diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000000..84373e1cc2b9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,128 @@ +# Build: +# docker build -t llama-cpp-fp8 . +# Run (needs the host to have NVIDIA drivers + nvidia-container-toolkit): +# docker run --rm -it --gpus all \ +# -v "$PWD":/workspace -w /workspace \ +# -v ~/.cache/huggingface:/root/.cache/huggingface \ +# llama-cpp-fp8 + +ARG CUDA_VERSION=12.4.0 +ARG UBUNTU_VERSION=22.04 +FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} + +ARG CUDA_DOCKER_ARCH=default +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake git curl ca-certificates pkg-config \ + libssl-dev libgomp1 libcurl4-openssl-dev \ + python3 python3-pip python3-dev python3-venv \ + && rm -rf /var/lib/apt/lists/* + +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 + +WORKDIR /app + +# Copy only what the C++ build needs. Editing scripts / assets / docs later +# will NOT invalidate this layer, so cmake stays cached across those edits. +COPY CMakeLists.txt CMakePresets.json ./ +COPY cmake ./cmake +COPY src ./src +COPY ggml ./ggml +COPY include ./include +COPY common ./common +COPY tools ./tools +COPY examples ./examples +COPY pocs ./pocs +COPY vendor ./vendor +COPY licenses ./licenses +COPY grammars ./grammars +COPY scripts ./scripts + +RUN if [ "${CUDA_DOCKER_ARCH}" != "default" ]; then \ + export CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CUDA_DOCKER_ARCH}"; \ + fi && \ + cmake -B build \ + -DGGML_NATIVE=OFF \ + -DGGML_CUDA=ON \ + -DGGML_BACKEND_DL=ON \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DLLAMA_BUILD_TESTS=OFF \ + ${CMAKE_ARGS} && \ + cmake --build build --config Release -j"$(nproc)" + +ENV PATH="/app/build/bin:${PATH}" +ENV LD_LIBRARY_PATH="/app/build/bin:${LD_LIBRARY_PATH}" + +# Copy only the requirements files. Editing scripts / assets later won't +# invalidate the pip install layer either. +COPY requirements.txt ./ +COPY requirements ./requirements + +# Install CUDA-capable torch first so that requirements.txt (which pulls the +# CPU wheel via --extra-index-url) will see torch already satisfied and skip it. +RUN pip3 install --break-system-packages --no-cache-dir --upgrade \ + pip setuptools wheel && \ + pip3 install --break-system-packages --no-cache-dir \ + --index-url https://download.pytorch.org/whl/cu124 \ + "torch~=2.6.0" && \ + pip3 install --break-system-packages --no-cache-dir \ + -r requirements.txt && \ + pip3 install --break-system-packages --no-cache-dir \ + "llmcompressor<0.12" "transformers<5" + +# No file dependency: safe to keep cached across script/asset changes. +# scipy is not directly used, but transformers 4.57's generation.candidate_generator +# imports sklearn at module load; sklearn then imports scipy. Missing scipy therefore +# breaks AutoTokenizer / AutoModel imports entirely. +# Pin scipy<1.14 because llama.cpp's requirements pin numpy~=1.26.4, and scipy>=1.14 +# requires numpy>=2.0 — otherwise sklearn refuses to load with a version-mismatch error. +RUN pip3 install --break-system-packages --no-cache-dir \ + "huggingface_hub[cli,hf_transfer]" \ + "scipy<1.14" + +# Mamba / Nemotron-H hybrid models require mamba_ssm + causal-conv1d for their +# custom kernels. These packages compile CUDA code against the installed torch, +# so --no-build-isolation is required (else pip's ephemeral build env has no +# torch/CUDA). Its own layer: compile is slow (~10 min) but rarely changes. +RUN pip3 install --break-system-packages --no-cache-dir --no-build-isolation \ + "causal-conv1d>=1.4.0" \ + mamba-ssm + +# vLLM for FP8 serving and for the test_conversion_fp8 --vllm cross-check. +# ~2 GB of wheels + deps, no file dependency, so editing scripts / assets +# never re-triggers this download. +# Pin vllm<0.24: from 0.24 onward vllm requires transformers>=5.5.3, which +# would upgrade transformers to v5 and break llmcompressor (pinned to +# transformers<=4.57.6). Re-assert transformers<5 on the same install so +# pip can't silently override the earlier constraint. +RUN pip3 install --break-system-packages --no-cache-dir \ + "vllm<0.24" "transformers<5" + +# Ollama binary (needed by test_conversion/test_main.py to compare the GGUF +# against the original HF model). test.py auto-starts `ollama serve` in the +# background when it isn't already reachable, provided this binary is on PATH. +# Placed as the last install layer. Ollama publishes as .tar.zst (zstd) since +# ~v0.31 — no more .tgz — hence the apt install of zstd and tar's --zstd flag. +# Rewrite `http://` to `https://` on Ubuntu mirrors before any apt call, so +# `apt-get update` works from networks that block plain-HTTP egress (common in +# managed environments). Also bundles `less` (interactive log paging) into the +# same apt hop as `zstd` so we don't need a second network round-trip. +RUN find /etc/apt -type f \( -name '*.sources' -o -name '*.list' \) -exec sed -i \ + -e 's|http://ports.ubuntu.com|https://ports.ubuntu.com|g' \ + -e 's|http://archive.ubuntu.com|https://archive.ubuntu.com|g' \ + -e 's|http://security.ubuntu.com|https://security.ubuntu.com|g' \ + {} + \ + && apt-get update && apt-get install -y --no-install-recommends zstd less \ + && rm -rf /var/lib/apt/lists/* \ + && arch=$(uname -m) \ + && case "$arch" in x86_64) o=amd64 ;; aarch64) o=arm64 ;; *) echo "unsupported arch: $arch" >&2; exit 1 ;; esac \ + && curl -fsSL "https://ollama.com/download/ollama-linux-${o}.tar.zst" | tar --zstd -xf - -C /usr + +# Everything else (Python scripts, assets, README, etc.). Placed last so that +# editing any of these only rebuilds this small layer, not cmake or pip. +COPY . . + +WORKDIR /workspace + +CMD ["/bin/bash"] diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 new file mode 100644 index 000000000000..8b8bca864328 --- /dev/null +++ b/Dockerfile.arm64 @@ -0,0 +1,135 @@ +# Variant of Dockerfile targeted at NVIDIA DGX Spark (GB10 Grace-Blackwell, +# arm64, compute capability sm_121). Requires a Blackwell-capable CUDA +# toolchain (13.0) and a PyTorch build with arm64 + cu128 wheels. + +ARG CUDA_VERSION=13.0.0 +ARG UBUNTU_VERSION=24.04 +FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} + +ARG CUDA_DOCKER_ARCH=121 +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake git curl ca-certificates pkg-config \ + libssl-dev libgomp1 libcurl4-openssl-dev \ + python3 python3-pip python3-dev python3-venv \ + && rm -rf /var/lib/apt/lists/* + +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 + +WORKDIR /app + +# Copy only what the C++ build needs. Editing scripts / assets / docs later +# will NOT invalidate this layer, so cmake stays cached across those edits. +COPY CMakeLists.txt CMakePresets.json ./ +COPY cmake ./cmake +COPY src ./src +COPY ggml ./ggml +COPY include ./include +COPY common ./common +COPY tools ./tools +COPY examples ./examples +COPY pocs ./pocs +COPY vendor ./vendor +COPY licenses ./licenses +COPY grammars ./grammars +COPY scripts ./scripts + +RUN cmake -B build \ + -DGGML_NATIVE=OFF \ + -DGGML_CUDA=ON \ + -DGGML_BACKEND_DL=ON \ + -DGGML_CPU_ALL_VARIANTS=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DCMAKE_CUDA_ARCHITECTURES=${CUDA_DOCKER_ARCH} && \ + cmake --build build --config Release -j"$(nproc)" + +ENV PATH="/app/build/bin:${PATH}" +ENV LD_LIBRARY_PATH="/app/build/bin:${LD_LIBRARY_PATH}" + +# Copy only the requirements files. Editing scripts / assets later won't +# invalidate the pip install layer either. +COPY requirements.txt ./ +COPY requirements ./requirements + +# Install CUDA-capable torch first from the cu128 index (arm64 + CUDA wheels +# are only published from cu126 onward; cu128 pairs with CUDA 13). Then strip +# the CPU-only torch line from requirements before installing the rest, so +# pip doesn't try to downgrade the arm64+CUDA torch we just put in. +RUN pip3 install --break-system-packages --no-cache-dir --upgrade \ + --ignore-installed \ + pip setuptools wheel && \ + pip3 install --break-system-packages --no-cache-dir \ + --index-url https://download.pytorch.org/whl/cu128 \ + "torch>=2.7,<2.9" && \ + sed -i -E '/^(--extra-index-url .*whl\/(cpu|nightly)|torch[~=>< ].*)$/d' \ + requirements/requirements-convert_hf_to_gguf.txt && \ + pip3 install --break-system-packages --no-cache-dir \ + -r requirements.txt && \ + pip3 install --break-system-packages --no-cache-dir \ + "llmcompressor<0.12" "transformers<5" + +# No file dependency: safe to keep cached across script/asset changes. +# scipy is not directly used, but transformers 4.57's generation.candidate_generator +# imports sklearn at module load; sklearn then imports scipy. Missing scipy therefore +# breaks AutoTokenizer / AutoModel imports entirely. +# Pin scipy<1.14 because llama.cpp's requirements pin numpy~=1.26.4, and scipy>=1.14 +# requires numpy>=2.0 — otherwise sklearn refuses to load with a version-mismatch error. +RUN pip3 install --break-system-packages --no-cache-dir \ + "huggingface_hub[cli,hf_transfer]" \ + "scipy<1.14" + +# Mamba / Nemotron-H hybrid models require mamba_ssm + causal-conv1d for their +# custom kernels. These packages compile CUDA code against the installed torch, +# so --no-build-isolation is required (else pip's ephemeral build env has no +# torch/CUDA). On arm64 there are no prebuilt wheels, so this compiles from +# source with nvcc — slow (~20-30 min on GB10) but rarely changes. +# mamba-ssm is installed from git main (not PyPI): the released wheel still +# ships a Triton kernel (_chunk_cumsum_fwd_kernel in ssd_chunk_state.py) that +# fails to specialize a dict argument under the Triton bundled with torch +# cu128 on Blackwell (sm_121). Symptom at first forward: +# TypeError: failed to specialize argument of type: dict +# git main has the fix. +RUN pip3 install --break-system-packages --no-cache-dir --no-build-isolation \ + "causal-conv1d>=1.4.0" \ + "git+https://github.com/state-spaces/mamba.git" + +# vLLM for FP8 serving and for the test_conversion_fp8 --vllm cross-check. +# On arm64 there are no prebuilt vLLM wheels, so this compiles from source +# against our cu128 torch; expect a long build. +# Pin vllm<0.24: from 0.24 onward vllm requires transformers>=5.5.3, which +# would upgrade transformers to v5 and break llmcompressor (pinned to +# transformers<=4.57.6). Re-assert transformers<5 on the same install so +# pip can't silently override the earlier constraint. +RUN pip3 install --break-system-packages --no-cache-dir \ + "vllm<0.24" "transformers<5" + +# Ollama binary (needed by test_conversion/test_main.py to compare the GGUF +# against the original HF model). test.py auto-starts `ollama serve` in the +# background when it isn't already reachable, provided this binary is on PATH. +# Placed as the last install layer. Ollama publishes as .tar.zst (zstd) since +# ~v0.31 — no more .tgz — hence the apt install of zstd and tar's --zstd flag. +# Rewrite `http://` to `https://` on Ubuntu mirrors before any apt call, so +# `apt-get update` works from networks that block plain-HTTP egress (common in +# managed environments; the base image ships DEB822 sources pointing at +# `http://ports.ubuntu.com/ubuntu-ports/` and would otherwise hang). Also +# bundles `less` (interactive log paging) into the same apt hop as `zstd` so +# we don't need a second network round-trip. +RUN find /etc/apt -type f \( -name '*.sources' -o -name '*.list' \) -exec sed -i \ + -e 's|http://ports.ubuntu.com|https://ports.ubuntu.com|g' \ + -e 's|http://archive.ubuntu.com|https://archive.ubuntu.com|g' \ + -e 's|http://security.ubuntu.com|https://security.ubuntu.com|g' \ + {} + \ + && apt-get update && apt-get install -y --no-install-recommends zstd less \ + && rm -rf /var/lib/apt/lists/* \ + && arch=$(uname -m) \ + && case "$arch" in x86_64) o=amd64 ;; aarch64) o=arm64 ;; *) echo "unsupported arch: $arch" >&2; exit 1 ;; esac \ + && curl -fsSL "https://ollama.com/download/ollama-linux-${o}.tar.zst" | tar --zstd -xf - -C /usr + +# Everything else (Python scripts, assets, README, etc.). Placed last so that +# editing any of these only rebuilds this small layer, not cmake or pip. +COPY . . + +WORKDIR /workspace + +CMD ["/bin/bash"] diff --git a/common/debug.cpp b/common/debug.cpp index 0df409a79dbb..4732f73c4432 100644 --- a/common/debug.cpp +++ b/common/debug.cpp @@ -3,6 +3,10 @@ #include "log.h" #include +#include +#include +#include +#include #include static std::string common_ggml_ne_string(const ggml_tensor * t) { @@ -155,6 +159,53 @@ template bool common_debug_cb_eval(struct ggml_tensor * t, b if (!ggml_is_quantized(t->type) && matches_filter) { uint8_t * data = is_host ? (uint8_t *) t->data : cb_data->data.data(); common_debug_print_tensor(data, t->type, t->ne, t->nb, 3); + + // Optional full-tensor binary dump for layer-by-layer comparison work. + // Activated by setting env var LLAMA_DUMP_TENSORS_FILE=/path/to/out.bin. + // Optionally narrow what gets dumped with LLAMA_DUMP_TENSORS_REGEX + // (a single regex; anchored implicitly with regex_search). If unset, + // every tensor that already matched cb_data's filter gets dumped. + // Per-tensor binary record (little-endian): + // u32 name_len, char name[name_len], + // u32 dtype (ggml_type), i64 ne[4], + // u64 n_bytes, u8 data[n_bytes] + const char * dump_path = std::getenv("LLAMA_DUMP_TENSORS_FILE"); + if (dump_path) { + static std::regex dump_regex; + static bool dump_regex_set = false; + static bool dump_regex_valid = false; + if (!dump_regex_set) { + dump_regex_set = true; + const char * pat = std::getenv("LLAMA_DUMP_TENSORS_REGEX"); + if (pat && *pat) { + try { dump_regex = std::regex(pat); dump_regex_valid = true; } + catch (const std::regex_error &) { dump_regex_valid = false; } + } + } + bool should_dump = !dump_regex_valid || std::regex_search(t->name, dump_regex); + if (should_dump) { + static FILE * dump_fout = nullptr; + static std::string opened_path; + if (!dump_fout || opened_path != dump_path) { + if (dump_fout) fclose(dump_fout); + dump_fout = std::fopen(dump_path, "wb"); + opened_path = dump_path; + } + if (dump_fout) { + uint32_t name_len = (uint32_t) std::strlen(t->name); + std::fwrite(&name_len, 4, 1, dump_fout); + std::fwrite(t->name, 1, name_len, dump_fout); + uint32_t dtype = (uint32_t) t->type; + std::fwrite(&dtype, 4, 1, dump_fout); + int64_t ne[4] = { t->ne[0], t->ne[1], t->ne[2], t->ne[3] }; + std::fwrite(ne, 8, 4, dump_fout); + uint64_t nbytes = (uint64_t) ggml_nbytes(t); + std::fwrite(&nbytes, 8, 1, dump_fout); + std::fwrite(data, 1, nbytes, dump_fout); + std::fflush(dump_fout); + } + } + } } return true; diff --git a/convert.sh b/convert.sh new file mode 100755 index 000000000000..64063fba6035 --- /dev/null +++ b/convert.sh @@ -0,0 +1,431 @@ +set -e + +SRCDIR=$(dirname "$0") + +usage() { + cat < [--name ] [--output ] + [--complete] [--test-vocab] [--transformers-fp8] + [--renderer NAME] [--parser NAME] + [--thinking | --no-thinking] + + input_folder HF-format model directory to convert + --name NAME basename used for output files (default: basename of input_folder) + --output DIR output directory (default: -GGUF) + --complete full build (all standard + dynamic quants), sets COMPLETE=1 + --test-vocab only build a vocab-only GGUF and exit, sets TEST_VOCAB=1 + --transformers-fp8 also run convert_hf_to_fp8.py; FP8 output goes to + -FP8, or /FP8 if --output is set + --renderer NAME Ollama Modelfile RENDERER directive (overrides auto-detect) + --parser NAME Ollama Modelfile PARSER directive (overrides auto-detect) + --thinking treat the model as thinking-capable (overrides auto-detect) + --no-thinking treat the model as non-thinking (overrides auto-detect) + +Auto-detected defaults for the Ollama directives (both overridable): + + regular Nemotron NemotronH* + non-thinking (none) RENDERER=qwen3-vl-instruct + PARSER=passthrough + thinking RENDERER=qwen3-vl-thinking (defensive: Ollama's Jinja engine may + PARSER=qwen3-vl-thinking choke on {%- generation %} blocks) + + Thinking auto-detected by looking for in chat_template.jinja or + tokenizer_config.json's chat_template field. +EOF + exit 1 +} + +INPUT_PATH="" +NAME="" +OUTPUT_PATH="" +OUTPUT_SPECIFIED=0 +RUN_FP8=0 +OLLAMA_RENDERER="" +OLLAMA_PARSER="" +OLLAMA_RENDERER_SET=0 +OLLAMA_PARSER_SET=0 +IS_THINKING="" +IS_THINKING_SET=0 + +while [ $# -gt 0 ]; do + case "$1" in + -h|--help) usage ;; + --name) NAME=$2; shift 2 ;; + --output) OUTPUT_PATH=$2; OUTPUT_SPECIFIED=1; shift 2 ;; + --complete) COMPLETE=1; shift ;; + --test-vocab) TEST_VOCAB=1; shift ;; + --transformers-fp8) RUN_FP8=1; shift ;; + --renderer) OLLAMA_RENDERER=$2; OLLAMA_RENDERER_SET=1; shift 2 ;; + --parser) OLLAMA_PARSER=$2; OLLAMA_PARSER_SET=1; shift 2 ;; + --thinking) IS_THINKING=1; IS_THINKING_SET=1; shift ;; + --no-thinking) IS_THINKING=0; IS_THINKING_SET=1; shift ;; + --) shift; break ;; + -*) echo "unknown option: $1" >&2; usage ;; + *) + if [ -z "$INPUT_PATH" ]; then + INPUT_PATH=$1 + shift + else + echo "unexpected positional argument: $1" >&2 + usage + fi + ;; + esac +done + +[ -z "$INPUT_PATH" ] && usage +[ -d "$INPUT_PATH" ] || { echo "input folder not found: $INPUT_PATH" >&2; exit 1; } + +INPUT_PATH=${INPUT_PATH%/} +: "${NAME:=$(basename "$INPUT_PATH")}" +: "${OUTPUT_PATH:=${INPUT_PATH}-GGUF}" + +# Flags ------------------------------------------------------------------ +# TEST_VOCAB=1 : only build a vocab-only GGUF and exit (sanity check) +# COMPLETE=0 : minimal build (F16 base + Q4_K_M) +# COMPLETE=1 : full build (BF16 + F16 base, all standard quants, +# plus unsloth-style "dynamic" _L / _XL variants) +# STRIP_CHAT_TEMPLATE=0 : drop tokenizer.chat_template from the converted +# GGUF. Experiment to see if Ollama's nemotron_h +# hardcoded-template path can be sidestepped (lets the +# Modelfile TEMPLATE take effect). WARNING: llama-server +# loses /apply-template and /v1/chat/completions support. +# CLI --test-vocab / --complete override these; the assignments here are just defaults. +: "${TEST_VOCAB:=0}" +: "${COMPLETE:=0}" +STRIP_CHAT_TEMPLATE=0 + +# Base type used as source for all quantizations. +# F16 is the de-facto base; BF16 is safer for models trained in bf16. +BASE_TYPE="bf16" + +# Detect the HF architecture from config.json, if any. +HF_ARCH="" +if [ -f "$INPUT_PATH/config.json" ]; then + HF_ARCH=$(python3 -c "import json,sys; d=json.load(open(sys.argv[1])); a=d.get('architectures') or ['']; print(a[0])" "$INPUT_PATH/config.json" 2>/dev/null || echo "") +fi + +# Detect whether the source has a thinking-capable chat template. +# The naive "contains " check misfires: non-thinking templates that +# support parsing legacy thinking-annotated history (Luciole's non-thinking +# variant does exactly this) contain '' inside a split() call in +# their assistant-parsing branch. Signals that are actually distinctive: +# 1. `enable_thinking` — the API toggle standard HF thinking templates +# expose; absent from non-thinking variants. +# 2. `\n\n` — the empty-thought fenced block emitted when +# thinking is disabled at render time; only thinking templates emit it. +# Either signal is enough. --thinking / --no-thinking overrides on the CLI. +if [ $IS_THINKING_SET -eq 0 ]; then + IS_THINKING=$(python3 - "$INPUT_PATH" <<'PY' 2>/dev/null || echo 0 +import json, os, sys +p = sys.argv[1] +tmpl = "" +jinja = os.path.join(p, "chat_template.jinja") +if os.path.isfile(jinja): + with open(jinja) as f: + tmpl = f.read() +else: + tc = os.path.join(p, "tokenizer_config.json") + if os.path.isfile(tc): + try: + with open(tc) as f: + tmpl = json.load(f).get("chat_template", "") or "" + except Exception: + pass +is_thinking = "enable_thinking" in tmpl or "\\n\\n" in tmpl +print(1 if is_thinking else 0) +PY +) +fi + +# Case matrix for the two Ollama Modelfile directives (only applied when the +# caller hasn't overridden them via --renderer/--parser): +# +# regular Nemotron NemotronH* +# non-thinking (none) qwen3-vl-instruct / passthrough +# thinking qwen3-vl-thinking qwen3-vl-thinking / qwen3-vl-thinking +# / qwen3-vl-thinking (both same) +# +# For the thinking case we also route away from Ollama's built-in Jinja engine +# even on regular nemotron, because Luciole's thinking template uses +# {%- generation %} / {%- endgeneration %} blocks that Ollama's Jinja support +# has historically been shaky on. +case "$HF_ARCH" in + NemotronHForCausalLM|NemotronHForConditionalGeneration|NemotronH*) + if [ "$IS_THINKING" = "1" ]; then + [ $OLLAMA_RENDERER_SET -eq 0 ] && OLLAMA_RENDERER="qwen3-vl-thinking" + [ $OLLAMA_PARSER_SET -eq 0 ] && OLLAMA_PARSER="qwen3-vl-thinking" + else + [ $OLLAMA_RENDERER_SET -eq 0 ] && OLLAMA_RENDERER="qwen3-vl-instruct" + [ $OLLAMA_PARSER_SET -eq 0 ] && OLLAMA_PARSER="passthrough" + fi + ;; + NemotronForCausalLM|Nemotron*) + if [ "$IS_THINKING" = "1" ]; then + [ $OLLAMA_RENDERER_SET -eq 0 ] && OLLAMA_RENDERER="qwen3-vl-thinking" + [ $OLLAMA_PARSER_SET -eq 0 ] && OLLAMA_PARSER="qwen3-vl-thinking" + fi + # non-thinking regular nemotron: no override needed — Ollama routes + # to llama-server's /v1/chat/completions and llama.cpp applies the + # GGUF's Jinja template cleanly. + ;; +esac +echo "[detect] HF_ARCH=$HF_ARCH IS_THINKING=$IS_THINKING RENDERER=${OLLAMA_RENDERER:-(none)} PARSER=${OLLAMA_PARSER:-(none)}" + +# Calibration text for importance matrix (imatrix). +# Required for IQ1_*, IQ2_*, IQ3_XXS and recommended for all other quants +# (matches unsloth's "Dynamic Quants 2.0" recipe). Set to "" to disable. +IMATRIX_DATA="$SRCDIR/calibration.txt" + +# Quants that REQUIRE an imatrix (will be skipped if none is available). +IMATRIX_REQUIRED_QUANTS=(IQ1_S IQ1_M IQ2_XXS IQ2_XS IQ2_S IQ2_M IQ3_XXS) + +needs_imatrix() { + local q=$1 + for r in "${IMATRIX_REQUIRED_QUANTS[@]}"; do + [ "$q" = "$r" ] && return 0 + done + return 1 +} + +# Quant types — matches unsloth/Llama-3.3-70B-Instruct-GGUF exactly. +# Other types supported by llama-quantize are listed below (commented out). +QUANTS_STD=( + Q4_K_M + Q5_K_M + Q8_0 + # Q6_K + # Q5_K_S + # Q4_K_S + # Q4_0 Q4_1 + # Q3_K_M Q3_K_S + # Q2_K + # IQ4_NL IQ4_XS + # IQ3_XXS + # IQ2_M IQ2_XXS + # IQ1_M IQ1_S + # Q3_K_L # extra K-quant tier (between Q3_K_M and Q4_K_S) + # Q2_K_S # smaller Q2_K variant + # Q5_0 Q5_1 # legacy 5-bit quants (superseded by Q5_K_*) + # IQ3_M IQ3_S IQ3_XS # extra IQ3 tiers + # IQ2_S IQ2_XS # extra IQ2 tiers + # TQ1_0 TQ2_0 # ternary quants (~1.7 / ~2.1 bpw, niche) + # MXFP4_MOE # MoE-only 4-bit +) + +# Unsloth-style "dynamic" variants: keep token_embd and/or output at higher +# precision than the body. Format: "::". +# - _L : token embeddings bumped to Q8_0 +# - _XL : token embeddings AND output bumped to Q8_0 (or BF16 for Q8_K_XL) +QUANTS_DYNAMIC=( + # "Q2_K_L:q8_0:" + # "Q2_K_XL:q8_0:q8_0" + # "Q3_K_XL:q8_0:q8_0" + # "Q4_K_XL:q8_0:q8_0" + # "Q5_K_XL:q8_0:q8_0" + # "Q6_K_XL:q8_0:q8_0" + # "Q8_K_XL:bf16:bf16" +) + +# ----------------------------------------------------------------------- + +mkdir -p "$OUTPUT_PATH" +cd "$SRCDIR" + +QUANTIZE=./build/bin/llama-quantize +IMATRIX_BIN=./build/bin/llama-imatrix + +# Per-model imatrix path (set after $NAME / $OUTPUT_PATH are known). +IMATRIX="" + +build_imatrix() { + # Generate the importance matrix from $BASE_TYPE GGUF + calibration text. + # No-op if disabled, already built, or calibration file missing. + if [ -z "$IMATRIX_DATA" ]; then + echo "[imatrix] disabled (IMATRIX_DATA empty)" + IMATRIX="" + return + fi + if [ ! -f "$IMATRIX_DATA" ]; then + echo "[imatrix] WARNING: calibration file $IMATRIX_DATA not found — skipping imatrix" + IMATRIX="" + return + fi + IMATRIX=$OUTPUT_PATH/$NAME-imatrix.gguf + if [ -f "$IMATRIX" ]; then + echo "[skip] $IMATRIX already exists" + return + fi + local base=$OUTPUT_PATH/$NAME-$(echo "$BASE_TYPE" | tr '[:lower:]' '[:upper:]').gguf + CMD="$IMATRIX_BIN -m $base -f $IMATRIX_DATA -o $IMATRIX --process-output" + echo "$CMD" + eval $CMD +} + +convert_hf() { + # outtype is the CLI arg for --outtype (lowercase: bf16/f16/f32); + # the filename uses the uppercase form (BF16/F16/F32) to match unsloth/bartowski. + local outtype=$1 + local suffix=$(echo "$outtype" | tr '[:lower:]' '[:upper:]') + local outfile=$OUTPUT_PATH/$NAME-$suffix.gguf + if [ -f "$outfile" ]; then + echo "[skip] $outfile already exists" + return + fi + CMD="STRIP_CHAT_TEMPLATE=$STRIP_CHAT_TEMPLATE python3 convert_hf_to_gguf.py $INPUT_PATH --outfile $outfile --outtype $outtype" + echo "$CMD" + eval $CMD +} + +quantize_std() { + local quant=$1 + local base=$OUTPUT_PATH/$NAME-$(echo "$BASE_TYPE" | tr '[:lower:]' '[:upper:]').gguf + local outfile=$OUTPUT_PATH/$NAME-$quant.gguf + if [ -f "$outfile" ]; then + echo "[skip] $outfile already exists" + return + fi + local imat="" + if [ -n "$IMATRIX" ] && [ -f "$IMATRIX" ]; then + imat="--imatrix $IMATRIX" + elif needs_imatrix "$quant"; then + echo "[skip] $quant requires imatrix but none available" + return + fi + CMD="$QUANTIZE $imat $base $outfile $quant" + echo "$CMD" + eval $CMD +} + +quantize_dynamic() { + # spec format: NAME:TOK_TYPE:OUT_TYPE (TOK_TYPE or OUT_TYPE may be empty) + local spec=$1 + local name=${spec%%:*} + local rest=${spec#*:} + local tok_type=${rest%%:*} + local out_type=${rest#*:} + + # Derive the underlying base quant from the variant name: + # Q2_K_L -> Q2_K, Q4_K_XL -> Q4_K, Q8_K_XL -> Q8_0 (special) + local base_quant=${name%_XL} + base_quant=${base_quant%_L} + [ "$name" = "Q8_K_XL" ] && base_quant=Q8_0 + + local base=$OUTPUT_PATH/$NAME-$(echo "$BASE_TYPE" | tr '[:lower:]' '[:upper:]').gguf + local outfile=$OUTPUT_PATH/$NAME-$name.gguf + if [ -f "$outfile" ]; then + echo "[skip] $outfile already exists" + return + fi + local flags="" + [ -n "$tok_type" ] && flags="$flags --token-embedding-type $tok_type" + [ -n "$out_type" ] && flags="$flags --output-tensor-type $out_type" + if [ -n "$IMATRIX" ] && [ -f "$IMATRIX" ]; then + flags="$flags --imatrix $IMATRIX" + elif needs_imatrix "$base_quant"; then + echo "[skip] $name (base $base_quant) requires imatrix but none available" + return + fi + CMD="$QUANTIZE $flags $base $outfile $base_quant" + echo "$CMD" + eval $CMD +} + +# ----------------------------------------------------------------------- + +if [ $TEST_VOCAB -eq 1 ]; then + CMD="python3 convert_hf_to_gguf.py $INPUT_PATH --outfile $OUTPUT_PATH/$NAME-vocab.gguf --vocab-only" + echo "$CMD" + eval $CMD + exit 0 +fi + +# Always build the base precision used for quantization. +convert_hf $BASE_TYPE + +# Build the importance matrix from the base GGUF (needed for IQ quants, +# recommended for all quants). Falls back gracefully if disabled / data missing. +build_imatrix + +if [ $COMPLETE -eq 1 ]; then + # Other precision bases — uncomment to publish them too (unsloth ships + # both BF16 and F16; F32 is rarely useful and ~4x the size of BF16/F16). + # [ "$BASE_TYPE" = "bf16" ] && convert_hf f16 + # [ "$BASE_TYPE" = "f16" ] && convert_hf bf16 + # convert_hf f32 + + # Standard quants + for q in "${QUANTS_STD[@]}"; do + quantize_std $q + done + + # Dynamic (unsloth-style) variants + for spec in "${QUANTS_DYNAMIC[@]}"; do + quantize_dynamic $spec + done +else + # Minimal build: one quant + quantize_std Q4_K_M +fi + +# Copy model-card assets into the output folder, substituting -> $NAME +# in text templates. Binary assets (logos, etc.) are copied verbatim. +# For the Modelfile, additionally inject RENDERER/PARSER lines after the FROM +# directive when OLLAMA_RENDERER / OLLAMA_PARSER are set (thinking-vs-not is +# encoded in those two directives, so a single Modelfile template covers both). +ASSETS_DIR="$SRCDIR/hf_assets_gguf" +if [ -d "$ASSETS_DIR" ]; then + for src in "$ASSETS_DIR"/*; do + [ -e "$src" ] || continue + dst="$OUTPUT_PATH/$(basename "$src")" + case "$(basename "$src")" in + Modelfile) + awk -v NAME="$NAME" \ + -v RENDERER="$OLLAMA_RENDERER" \ + -v PARSER="$OLLAMA_PARSER" ' + { gsub(//, NAME) } + /^FROM / && !injected { + print + if (RENDERER != "") print "RENDERER " RENDERER + if (PARSER != "") print "PARSER " PARSER + injected = 1 + next + } + { print } + ' "$src" > "$dst" + note="" + [ -n "$OLLAMA_RENDERER" ] && note="$note, RENDERER=$OLLAMA_RENDERER" + [ -n "$OLLAMA_PARSER" ] && note="$note, PARSER=$OLLAMA_PARSER" + echo "[assets] wrote $dst ( -> $NAME${note})" + ;; + README.md) + sed "s||$NAME|g" "$src" > "$dst" + echo "[assets] wrote $dst (with -> $NAME)" + ;; + *) + cp -f "$src" "$dst" + echo "[assets] copied $dst" + ;; + esac + done +fi + +if [ $RUN_FP8 -eq 1 ]; then + if [ $OUTPUT_SPECIFIED -eq 1 ]; then + FP8_PATH=$OUTPUT_PATH/FP8 + else + FP8_PATH=${INPUT_PATH}-FP8 + fi + mkdir -p "$FP8_PATH" + # FP8_DYNAMIC is data-free and weight-only — no CUDA needed. Hide the GPU + # so llmcompressor's data_free pipeline (which calls compressed_tensors' + # dispatch_model → get_device_memory → torch.accelerator.get_memory_info) + # doesn't trigger a CUDA context init that OOMs on unified-memory systems + # (DGX Spark) when other processes are already holding most of the pool. + CMD="CUDA_VISIBLE_DEVICES= python3 convert_hf_to_fp8.py $INPUT_PATH --outfile $FP8_PATH --split-max-size 5G --device cpu" + echo "$CMD" + eval $CMD +fi + + +# done diff --git a/convert_hf_to_fp8.py b/convert_hf_to_fp8.py new file mode 100644 index 000000000000..63216e37e3c2 --- /dev/null +++ b/convert_hf_to_fp8.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +""" +Quantize a HuggingFace transformers model to FP8 (compressed-tensors format). + +Mirrors the CLI surface of convert_hf_to_gguf.py so command lines are broadly +interchangeable. Arguments that have no meaning for an FP8 pass are accepted +and ignored (documented in each --help entry). + +Output is a standard HF transformers directory (config.json + safetensors + +tokenizer.*). It can be loaded with: + + from transformers import AutoModelForCausalLM + AutoModelForCausalLM.from_pretrained("") + +or served natively by vLLM / SGLang / TGI. + +Requires: + pip install "llmcompressor<0.12" "transformers<5" +""" + +from __future__ import annotations + +import argparse +import logging +import os +import re +import shutil +import sys +from pathlib import Path + +logger = logging.getLogger("convert-hf-to-fp8") + + +# Monkey-patch shutil.copy so every copied file gets owner-write. llmcompressor's +# copy_python_files_from_model_cache (helpers.py:74) uses shutil.copy on cached +# HF `.py` modules whose source mode is often 0464 or 0444 (read-only for owner). +# When llmcompressor then re-invokes the same copy path during save_pretrained, +# open('wb') on the read-only destination raises PermissionError. Patching here +# ensures the destination is always writable regardless of source mode. Must run +# before any llmcompressor import; llmcompressor calls `shutil.copy(...)` by +# module-attribute lookup, so replacing shutil.copy takes effect immediately. +_orig_shutil_copy = shutil.copy + + +def _copy_writable(src, dst, *args, **kwargs): + result = _orig_shutil_copy(src, dst, *args, **kwargs) + try: + st = os.stat(result) + os.chmod(result, st.st_mode | 0o200) + except OSError: + pass + return result + + +shutil.copy = _copy_writable + + +# vision encoders and multimodal projectors are activation-sensitive; keeping +# them in the original precision preserves quality with negligible size cost. +VISION_IGNORE_PATTERNS = [ + "re:.*vision_tower.*", + "re:.*vision_model.*", + "re:.*visual\\..*", + "re:.*multi_modal_projector.*", + "re:.*mm_projector.*", +] + + +# Model families whose Mamba/SSM projections we skip during FP8 quantization. +# The Mamba mixer's in_proj/out_proj feed into a stateful recurrent update +# whose numerics are markedly more sensitive to weight quantization than +# plain attention/MLP layers. Skipping them recovers most of the quality on +# hybrid models at negligible size cost (Mamba layers are ~5-10% of params). +MAMBA_HYBRID_MODEL_TYPES = ("nemotron_h", "nemotron-h", "bamba", "zamba", "zamba2", "jamba") + + +def _mamba_hybrid_ignore_patterns(config) -> list[str]: + """Build regex ignore patterns targeting Mamba layers of a hybrid model. + + Uses config.hybrid_override_pattern (a string like 'M-M-M*-M-...' where + each char = one layer: M=Mamba, -=MLP, *=Attention) to target only the + Mamba-position layers by index. Falls back to a broad name-based glob + when the pattern is missing.""" + pattern = getattr(config, "hybrid_override_pattern", None) or "" + mamba_indices = [i for i, c in enumerate(pattern) if c == "M"] + if not mamba_indices: + return ["re:.*(mamba|ssm)\\..*", "re:.*\\.mixer\\.(in_proj|out_proj)$"] + # Nemotron-H / similar layer path is backbone.layers..mixer.. + # Anchor on `..` so `.10.` does not also match `.1.` etc. + return [f"re:.*\\.layers\\.{i}\\.mixer\\..*" for i in mamba_indices] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Convert a HuggingFace transformers model to FP8 (compressed-tensors)" + ) + parser.add_argument( + "--vocab-only", action="store_true", + help="only export config + tokenizer (no weights)", + ) + parser.add_argument( + "--outfile", type=Path, + help="output directory; default: -FP8", + ) + parser.add_argument( + "--outtype", type=str, + choices=["fp8", "fp8_dynamic", "auto"], default="auto", + help=( + "FP8 scheme: 'fp8_dynamic' uses static per-channel weights + dynamic " + "per-token activations (no calibration needed); 'fp8' uses static " + "per-tensor activations (requires calibration; not implemented here). " + "'auto' selects fp8_dynamic." + ), + ) + parser.add_argument( + "--bigendian", action="store_true", + help="ignored (safetensors is endian-agnostic)", + ) + parser.add_argument( + "model", type=str, nargs="?", + help="directory containing model files or HuggingFace repository ID", + ) + parser.add_argument( + "--use-temp-file", action="store_true", + help="ignored (no bespoke temp-file path here)", + ) + parser.add_argument( + "--no-lazy", action="store_true", + help="ignored (weights are always fully materialized for FP8 quantization)", + ) + parser.add_argument( + "--model-name", type=str, default=None, + help="name used to derive the default output directory", + ) + parser.add_argument( + "--verbose", action="store_true", + help="increase output verbosity", + ) + parser.add_argument( + "--split-max-tensors", type=int, default=0, + help="ignored (safetensors sharding uses --split-max-size)", + ) + parser.add_argument( + "--split-max-size", type=str, default="0", + help="max shard size for saved safetensors, e.g. 5G. '0' = library default (5GB).", + ) + parser.add_argument( + "--dry-run", action="store_true", + help="print the plan without running the quantization", + ) + parser.add_argument( + "--no-tensor-first-split", action="store_true", + help="ignored", + ) + parser.add_argument( + "--metadata", type=Path, + help="ignored (GGUF-only)", + ) + parser.add_argument( + "--print-supported-models", action="store_true", + help="print the supported model families and exit", + ) + parser.add_argument( + "--remote", action="store_true", + help=( + "treat the model argument as a HuggingFace repo id. In practice this " + "flag is not required: local paths and repo ids are both accepted." + ), + ) + parser.add_argument( + "--mmproj", action="store_true", + help="ignored (vision components are auto-detected and kept in original precision)", + ) + parser.add_argument( + "--mistral-format", action="store_true", + help="not supported (this tool consumes HF transformers format only)", + ) + parser.add_argument( + "--disable-mistral-community-chat-template", action="store_true", + help="ignored", + ) + parser.add_argument( + "--sentence-transformers-dense-modules", action="store_true", + help="ignored", + ) + parser.add_argument( + "--fuse-gate-up-exps", action="store_true", + help="ignored (llm-compressor handles MoE gate/up layers automatically)", + ) + parser.add_argument( + "--offload-folder", type=Path, default=None, + help=( + "directory used by accelerate for disk offload when the model does " + "not fit in RAM+VRAM. Default: /.offload. Needs enough free " + "space to hold the parts of the model that don't fit in memory." + ), + ) + parser.add_argument( + "--device", type=str, choices=["auto", "cpu", "cuda"], default="auto", + help=( + "device used to run the quantization pass. 'cpu' avoids VRAM entirely " + "(FP8_DYNAMIC is a data-free / weight-only pass, so no forward runs) " + "and is the reliable fallback when GPU quantization OOMs. Requires " + "roughly 2 x (model size in bytes) of RAM. 'auto' = cuda if available." + ), + ) + parser.add_argument( + "--skip-mamba-fp8", action="store_true", + help=( + "for hybrid Mamba-Transformer models (Nemotron-H, Bamba, Jamba, Zamba…), " + "keep every Linear inside the Mamba mixer at the original precision. " + "Standard practice quantizes Mamba's in_proj/out_proj to FP8 just like " + "attention/MLP; this flag is stricter and useful if that hurts quality " + "on a specific model. llmcompressor's targets='Linear' already spares " + "the causal conv1d and the SSM state parameters regardless." + ), + ) + + args = parser.parse_args() + if not args.print_supported_models and args.model is None: + parser.error("the following arguments are required: model") + return args + + +def resolve_scheme(outtype: str) -> str: + if outtype in ("auto", "fp8_dynamic"): + return "FP8_DYNAMIC" + if outtype == "fp8": + return "FP8" + raise ValueError(f"unknown --outtype value: {outtype}") + + +def default_output_path(model_path: str, model_name: str | None) -> Path: + base = model_name or Path(model_path.rstrip("/")).name + return Path(f"{base}-FP8") + + +def translate_shard_size(spec: str) -> str | None: + """Translate the GGUF split-size grammar ('5G', '500M', '1024K', or a raw + byte count) into what transformers.save_pretrained expects ('5GB', '500MB', + ...). Returns None to mean 'use the library default'.""" + if not spec or spec == "0": + return None + m = re.fullmatch(r"(\d+)\s*([KMG]?)", spec.strip()) + if not m: + raise ValueError(f"invalid --split-max-size: {spec!r}") + n, unit = m.group(1), m.group(2) + if unit == "": + return n + return f"{n}{unit}B" + + +def looks_multimodal(config_dict: dict) -> bool: + return any( + k in config_dict + for k in ("vision_config", "vision_tower_config", "vision_model_config", + "audio_config", "speech_config") + ) + + +def print_supported_models() -> None: + print( + "llm-compressor's FP8_DYNAMIC pass works on any HuggingFace transformers\n" + "model whose weights live in torch.nn.Linear modules. In practice this\n" + "covers:\n" + " - Llama 1/2/3/3.1/3.2/4 and derivatives (Vicuna, WizardLM, ...)\n" + " - Mistral 7B/Small/Nemo/Large, Mixtral, Codestral\n" + " - Qwen 1/2/2.5/3 (dense + MoE), Qwen-VL\n" + " - Gemma 1/2/3\n" + " - Phi 2/3/3.5/4\n" + " - DeepSeek V2/V3, DeepSeek-Coder, DeepSeek-VL\n" + " - Falcon, MPT, StarCoder, InternLM, Yi, Baichuan, CommandR\n" + " - Vision-language variants of the above (vision tower is kept in the\n" + " original precision, only the LLM Linears are quantized)\n" + ) + + +def _load_model(model_ref: str, is_multimodal: bool, offload_folder: Path, device: str): + """Pick a suitable Auto* class. For image-text models the CausalLM head is + wrapped by a conditional-generation class; instantiating via + AutoModelForCausalLM would strip the vision tower. + + device 'cpu' places the whole model on CPU (no VRAM, no offload thrashing). + device 'cuda'/'auto' let accelerate spread across GPUs, spilling to + offload_folder when RAM+VRAM aren't enough.""" + from transformers import AutoModelForCausalLM + + kw = dict(torch_dtype="auto", trust_remote_code=True) + if device == "cpu": + kw["device_map"] = {"": "cpu"} + else: + offload_folder.mkdir(parents=True, exist_ok=True) + kw["device_map"] = "auto" + kw["offload_folder"] = str(offload_folder) + + if is_multimodal: + try: + from transformers import AutoModelForImageTextToText + return AutoModelForImageTextToText.from_pretrained(model_ref, **kw) + except (ImportError, ValueError): + pass + try: + from transformers import AutoModelForVision2Seq + return AutoModelForVision2Seq.from_pretrained(model_ref, **kw) + except (ImportError, ValueError): + pass + return AutoModelForCausalLM.from_pretrained(model_ref, **kw) + + +def _run_vocab_only(args: argparse.Namespace) -> int: + from transformers import AutoConfig, AutoTokenizer + + out_dir = args.outfile or default_output_path(args.model, args.model_name) + out_dir.mkdir(parents=True, exist_ok=True) + + AutoConfig.from_pretrained(args.model, trust_remote_code=True).save_pretrained(out_dir) + AutoTokenizer.from_pretrained(args.model, trust_remote_code=True).save_pretrained(out_dir) + logger.info("wrote config + tokenizer to %s", out_dir) + return 0 + + +def main() -> int: + args = parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + + if args.print_supported_models: + print_supported_models() + return 0 + + if args.mistral_format: + logger.error( + "--mistral-format is not supported: this tool consumes HuggingFace " + "transformers format only. Convert the model to HF format first, or " + "quantize the mistralai/... HF sibling repository." + ) + return 2 + + if args.vocab_only: + return _run_vocab_only(args) + + try: + from llmcompressor import oneshot + from llmcompressor.modifiers.quantization import QuantizationModifier + from transformers import AutoConfig, AutoProcessor, AutoTokenizer + except ImportError as e: + logger.error("Missing dependency: %s", e) + logger.error('Install with: pip install "llmcompressor<0.12" "transformers<5"') + return 3 + + # transformers v5 writes rope_scaling in a form that transformers v4 + # loaders reject; refuse to run so we can't silently produce a checkpoint + # that our downstream v4 environments would misload. + import transformers + tfx_major = int(transformers.__version__.split(".", 1)[0]) + if tfx_major >= 5: + logger.error( + "transformers %s detected. This tool targets transformers v4 output " + "for downstream compatibility. Reinstall with: " + 'pip install "llmcompressor<0.12" "transformers<5"', + transformers.__version__, + ) + return 4 + + scheme = resolve_scheme(args.outtype) + if scheme == "FP8" and not args.dry_run: + logger.error( + "--outtype fp8 (static activations) requires a calibration dataset " + "and is not implemented here. Use --outtype fp8_dynamic (default)." + ) + return 2 + + model_ref = args.model + out_dir = args.outfile or default_output_path(model_ref, args.model_name) + shard_size = translate_shard_size(args.split_max_size) + offload_folder = args.offload_folder or (out_dir / ".offload") + + config = AutoConfig.from_pretrained(model_ref, trust_remote_code=True) + is_multimodal = looks_multimodal(config.to_dict()) + model_type = (getattr(config, "model_type", "") or "").lower() + is_mamba_hybrid = model_type in MAMBA_HYBRID_MODEL_TYPES + + ignore = ["lm_head"] + if is_multimodal: + ignore.extend(VISION_IGNORE_PATTERNS) + if args.skip_mamba_fp8: + if not is_mamba_hybrid: + logger.warning( + "--skip-mamba-fp8 was passed but model_type=%r is not in the " + "known Mamba-hybrid list %s; extra Mamba ignores skipped.", + model_type, list(MAMBA_HYBRID_MODEL_TYPES), + ) + else: + mamba_patterns = _mamba_hybrid_ignore_patterns(config) + ignore.extend(mamba_patterns) + logger.info( + "--skip-mamba-fp8: keeping every Linear in %d Mamba layer(s) " + "of %s at original precision", + len(mamba_patterns), model_type, + ) + + device = args.device + if device == "auto": + try: + import torch + device = "cuda" if torch.cuda.is_available() else "cpu" + except ImportError: + device = "cpu" + + logger.info("model: %s", model_ref) + logger.info("output directory: %s", out_dir) + logger.info("scheme: %s", scheme) + logger.info("multimodal: %s", is_multimodal) + logger.info("ignored layers: %s", ignore) + logger.info("max shard size: %s", shard_size or "library default") + logger.info("device: %s", device) + if device != "cpu": + logger.info("offload folder: %s", offload_folder) + + if args.dry_run: + logger.info("--dry-run: nothing written.") + return 0 + + logger.info("loading model weights (this can take a while)...") + model = _load_model(model_ref, is_multimodal, offload_folder, device) + + recipe = QuantizationModifier( + targets="Linear", + scheme=scheme, + ignore=ignore, + ) + + logger.info("running one-shot FP8 quantization...") + oneshot(model=model, recipe=recipe) + + out_dir.mkdir(parents=True, exist_ok=True) + + # llmcompressor copies custom `.py` modules (trust_remote_code models) from + # the HF on-disk cache using shutil.copy, which preserves the source's 0444 + # mode. Then it re-copies them during save_pretrained, and open('wb') fails + # on the read-only destination. Pre-emptively make any pre-existing files + # writable so the second copy can overwrite them. + _make_tree_writable(out_dir) + + save_kwargs: dict = {"save_compressed": True} + if shard_size is not None: + save_kwargs["max_shard_size"] = shard_size + + logger.info("saving quantized model to %s", out_dir) + model.save_pretrained(out_dir, **save_kwargs) + + try: + AutoTokenizer.from_pretrained(model_ref, trust_remote_code=True).save_pretrained(out_dir) + except Exception as e: + logger.warning("could not save tokenizer: %s", e) + + if is_multimodal: + try: + AutoProcessor.from_pretrained(model_ref, trust_remote_code=True).save_pretrained(out_dir) + except Exception as e: + logger.warning("could not save processor: %s", e) + + # Restore normal mode on the .py files copied from cache so a subsequent + # rerun into the same directory doesn't hit the read-only wall again. + _make_tree_writable(out_dir) + + # accelerate created the offload dir but may have written nothing there + # (or already migrated everything out). Remove it if empty so it doesn't + # clutter the model card. + if offload_folder.exists() and offload_folder.is_dir() and not any(offload_folder.iterdir()): + try: + offload_folder.rmdir() + except OSError: + pass + + logger.info("done.") + return 0 + + +def _make_tree_writable(root: Path) -> None: + if not root.exists(): + return + for f in root.rglob("*"): + if f.is_file() or f.is_dir(): + try: + mode = f.stat().st_mode + f.chmod(mode | 0o200) + except OSError: + pass + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 46469c862000..2a16418e7234 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -1490,6 +1490,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "e4d54df1ebc1f2b91acd986c5b51aa50837d5faf7c7398e73c1f9e9ee5d19869": # ref: https://huggingface.co/kakaocorp/kanana-2-30b-a3b-instruct-2601 res = "kanana2" + if chkhsh == "5f9861fd826d8e124b222f41f41b928e78d8f6c8fbdf25625d06cc1e8736662c": + # ref: https://huggingface.co/OpenLLM-France/Luciole-1B-Base + res = "qwen2" if res is None: logger.warning("\n") @@ -1515,15 +1518,187 @@ def get_vocab_base_pre(self, tokenizer) -> str: def _set_vocab_none(self) -> None: self.gguf_writer.add_tokenizer_model("none") - def _set_vocab_gpt2(self) -> None: + @staticmethod + def _gpt2_bytes_to_unicode() -> dict[int, str]: + # Returns the GPT-2 byte-to-unicode mapping: each byte (0-255) maps to a + # printable unicode character. Printable ASCII and Latin-1 supplement bytes + # map to themselves; remaining bytes are shifted to 256+. + # This is the same as openai/gpt-2's bytes_to_unicode(). + bs = list(range(ord("!"), ord("~") + 1)) + list(range(0xA1, 0xAC + 1)) + list(range(0xAE, 0xFF + 1)) + cs = list(bs) + n = 0 + for b in range(256): + if b not in bs: + bs.append(b) + cs.append(256 + n) + n += 1 + return dict(zip(bs, (chr(c) for c in cs))) + + def _set_vocab_gpt2(self, convert_metaspace_to_gpt2=False) -> None: tokens, toktypes, tokpre = self.get_vocab_base() + + if convert_metaspace_to_gpt2: + # The tokenizer uses raw UTF-8 with Metaspace (▁ for spaces), but + # the "gpt2" tokenizer model in llama.cpp expects GPT-2 byte encoding + # (where each byte is mapped to a printable unicode char, e.g. space -> Ġ). + # Convert all tokens: replace ▁ back to space, then apply GPT-2 byte encoding. + byte_encoder = self._gpt2_bytes_to_unicode() + seen: set[str] = set() + for i, token in enumerate(tokens): + if toktypes[i] in (gguf.TokenType.NORMAL, gguf.TokenType.USER_DEFINED): + if token == " ": + # Useless token in Luciole + encoded = "".join(byte_encoder[b] for b in "\u2581".encode("utf-8")) + else: + encoded = "".join(byte_encoder[b] for b in token.replace("\u2581", " ").encode("utf-8")) + assert encoded not in seen, f"Unexpected collision in GPT-2 byte encoding: {encoded!r} for '{token}'" + seen.add(encoded) + tokens[i] = encoded + else: # gguf.TokenType.CONTROL + print("NOCOMMIT", i, token, toktypes[i]) + assert token not in seen, f"Unexpected collision in GPT-2 byte encoding: {token}" + seen.add(token) + self.gguf_writer.add_tokenizer_model("gpt2") self.gguf_writer.add_tokenizer_pre(tokpre) self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_types(toktypes) - special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) + if convert_metaspace_to_gpt2: + special_vocab.merges = [ + " ".join( + "".join(byte_encoder[b] for b in part.replace("\u2581", " ").encode("utf-8")) + for part in merge.split(" ") + ) + for merge in special_vocab.merges + ] special_vocab.add_to_gguf(self.gguf_writer) + return tokens + + def _set_vocab_bpe_as_spm(self) -> None: + """Convert a HuggingFace BPE tokenizer (with Metaspace ▁) to SPM format for llama.cpp. + + This reads the vocab from tokenizer.json, keeps tokens in their original + UTF-8 form (with ▁ preserved), assigns scores from merge ranks, and adds + byte fallback tokens <0x00>-<0xFF> required by the SPM tokenizer in C++. + """ + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(self.dir_model) + vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab)) + + reverse_vocab = {id_: tok for tok, id_ in tokenizer.vocab.items()} + added_vocab = tokenizer.get_added_vocab() + added_tokens_decoder = tokenizer.added_tokens_decoder + + # Build merge rank lookup: token_text -> rank (lower rank = merged earlier = higher priority) + merge_ranks: dict[str, int] = {} + merges_file = self.dir_model / "tokenizer.json" + if merges_file.is_file(): + import json as _json + with open(merges_file, "r", encoding="utf-8") as f: + tokenizer_json = _json.load(f) + merges = tokenizer_json.get("model", {}).get("merges", []) + for rank, merge in enumerate(merges): + # merge can be "token_a token_b" (str) or ["token_a", "token_b"] (list) + parts = merge.split(" ") if isinstance(merge, str) else merge + merged_token = "".join(parts) + if merged_token not in merge_ranks: + merge_ranks[merged_token] = rank + + # Prepare token arrays + tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)] + scores: list[float] = [-10000.0] * vocab_size + toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size + + # Track which byte values are covered (for byte fallback) + byte_token_ids: dict[int, int] = {} + + for token_id in range(vocab_size): + if token_id not in reverse_vocab: + continue + + token_text = reverse_vocab[token_id] + + if token_id in added_tokens_decoder: + info = added_tokens_decoder[token_id] + if info.special or self.does_token_look_special(token_text): + tokens[token_id] = token_text.encode("utf-8") + scores[token_id] = 0.0 + # USER_DEFINED instead of CONTROL: USER_DEFINED tokens are + # always pre-extracted atomically by llama.cpp's tokenizer + # (see llama-vocab.cpp:tokenizer_st_partition), whereas + # CONTROL tokens are only matched when the caller passes + # parse_special=true. Some runtimes (notably Ollama in + # /api/generate raw=true mode) leave parse_special=false, + # which would BPE-split tokens like <|im_start|> into ~12 + # pieces. USER_DEFINED avoids that and matches HF behavior. + toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED + continue + + # Check if this is a byte fallback token (<0xHH>) or a single-byte token + import re as _re + raw_bytes = token_text.encode("utf-8") + byte_match = _re.fullmatch(r"<0x([0-9A-Fa-f]{2})>", token_text) + if byte_match: + byte_val = int(byte_match.group(1), 16) + byte_token_ids[byte_val] = token_id + tokens[token_id] = token_text.encode("utf-8") + scores[token_id] = -10000.0 + toktypes[token_id] = SentencePieceTokenTypes.BYTE + continue + elif len(raw_bytes) == 1: + byte_token_ids[raw_bytes[0]] = token_id + + # Assign score based on merge rank or token_id + if token_text in merge_ranks: + # Merged tokens: earlier merges get higher (less negative) scores + # Use negative rank so that rank 0 (first merge) gets highest score + score = -float(merge_ranks[token_text]) + else: + # Base tokens (single chars) get high scores; unknown tokens get low scores + if len(raw_bytes) == 1: + score = 0.0 + else: + score = -10000.0 + float(token_id) + + tokens[token_id] = raw_bytes + scores[token_id] = score + toktypes[token_id] = SentencePieceTokenTypes.NORMAL + + # Add byte fallback tokens for any missing byte values + # SPM in llama.cpp requires <0x00> through <0xFF> with BYTE type + next_pad_idx = 0 + for byte_val in range(256): + if byte_val in byte_token_ids: + continue # already handled above + hex_str = f"<0x{byte_val:02X}>" + if byte_val in byte_token_ids: + tid = byte_token_ids[byte_val] + tokens[tid] = hex_str.encode("utf-8") + toktypes[tid] = SentencePieceTokenTypes.BYTE + scores[tid] = -10000.0 + else: + # Find an unused PAD slot + while next_pad_idx < len(tokens) and toktypes[next_pad_idx] != SentencePieceTokenTypes.UNUSED: + next_pad_idx += 1 + if next_pad_idx < vocab_size: + tokens[next_pad_idx] = hex_str.encode("utf-8") + toktypes[next_pad_idx] = SentencePieceTokenTypes.BYTE + scores[next_pad_idx] = -10000.0 + next_pad_idx += 1 + else: + logger.warning(f"No room to add byte fallback token {hex_str}") + + self.gguf_writer.add_tokenizer_model("llama") + self.gguf_writer.add_tokenizer_pre("default") + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_scores(scores) + self.gguf_writer.add_token_types(toktypes) + + special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens)) + special_vocab.add_to_gguf(self.gguf_writer) + return tokens def _set_vocab_qwen(self): dir_model = self.dir_model @@ -9607,14 +9782,61 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter yield from super().modify_tensors(data_torch, name, bid) +LUCIOLE_TO_BPE = False +def set_vocab_luciole(self): + # Luciole + # Promote every entry of added_tokens_decoder to an atomic token, even those + # flagged "special": false in tokenizer_config.json (e.g. , + # , , ). _set_vocab_bpe_as_spm + # then marks them USER_DEFINED, which means llama.cpp pre-extracts them + # atomically regardless of the runtime's parse_special flag — important + # because Ollama's /api/generate raw=true mode runs with parse_special=false + # and would otherwise BPE-split <|im_start|> into ~12 byte tokens. + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained(self.dir_model) + added_token_texts = {info.content for info in tokenizer.added_tokens_decoder.values()} + original_does_token_look_special = self.does_token_look_special + + def does_token_look_special_with_added(token): + token_text = token.decode("utf-8") if isinstance(token, (bytes, bytearray)) else token + if token_text in added_token_texts: + return True + return original_does_token_look_special(token) + + self.does_token_look_special = does_token_look_special_with_added + try: + if LUCIOLE_TO_BPE: + tokens = self._set_vocab_gpt2(convert_metaspace_to_gpt2=True) + self.gguf_writer.add_pad_token_id(tokens.index("")) + self.gguf_writer.add_unk_token_id(tokens.index("")) + # self.gguf_writer.add_tokenizer_pre("llama-bpe") # bloom, qwen2, llama-bpe ? + else: + tokens = self._set_vocab_bpe_as_spm() + self.gguf_writer.add_pad_token_id(tokens.index(b"")) + self.gguf_writer.add_unk_token_id(tokens.index(b"")) + finally: + self.does_token_look_special = original_does_token_look_special + # add_space_prefix=False because HF's metaspace prepend_scheme="first" + # only inserts `▁` at the very start of the input, while llama.cpp's flag + # is binary and would insert `▁` after EVERY special token (so + # <|im_start|>system → '<|im_start|>', '▁system' instead of the expected + # '<|im_start|>', 'system'). Since the model is only ever fed chat- + # templated inputs with many special-token boundaries, the per-boundary + # divergence is much more harmful than the raw-text leading-space miss. + self.gguf_writer.add_add_space_prefix(False) + + @ModelBase.register("NemotronForCausalLM") class NemotronModel(TextModel): model_arch = gguf.MODEL_ARCH.NEMOTRON def set_vocab(self): - self._set_vocab_sentencepiece() - self.gguf_writer.add_pad_token_id(0) - self.gguf_writer.add_unk_token_id(1) + if (self.dir_model / "tokenizer.model").is_file(): + self._set_vocab_sentencepiece() + self.gguf_writer.add_pad_token_id(0) + self.gguf_writer.add_unk_token_id(1) + else: + set_vocab_luciole(self) def set_gguf_parameters(self): super().set_gguf_parameters() @@ -9642,8 +9864,24 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter # model.layers.{l}.input_layernorm.weight # model.layers.{l}.post_attention_layernorm.weight # model.norm.weight + # NOTE: cast to fp32 BEFORE the +1 — source weights are bf16/fp16 and the + # add would otherwise happen at the source dtype, quantizing γ by ~3.9e-3 + # (bf16) / ~9.8e-4 (fp16) per element. GGUF stores these tensors as F32, + # so doing the arithmetic at full precision is free. if name.endswith("norm.weight"): - data_torch = data_torch + 1 + data_torch = data_torch.float() + 1 + + # for tied embeddings, duplicate token_embd as output.weight. + # NOTE: upstream llama.cpp's NEMOTRON loader treats output.weight as + # required (unlike NEMOTRON_H, which falls back to token_embd), so the + # duplicate must be present in the GGUF — it costs ~vocab*n_embd bytes + # but is necessary for the model to load on stock llama.cpp. + if self.hparams.get("tie_word_embeddings", False) and name == "model.embed_tokens.weight": + yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch) + + # skip lm_head.weight if tie_word_embeddings is True (already emitted from embed_tokens above) + if self.hparams.get("tie_word_embeddings", False) and name == "lm_head.weight": + return yield from super().modify_tensors(data_torch, name, bid) @@ -10091,6 +10329,8 @@ def __init__(self, *args, **kwargs): self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE self.is_moe = True + self.is_luciole = hparams.get("bos_token_id", -1) == 0 + super().__init__(*args, **kwargs) # Save the top-level head_dim for later @@ -10164,6 +10404,10 @@ def set_gguf_parameters(self): self.gguf_writer.add_moe_latent_size(latent_size) def set_vocab(self): + if self.is_luciole: + set_vocab_luciole(self) + return + super().set_vocab() # The tokenizer _does_ add a BOS token (via post_processor type diff --git a/examples/eval-callback/eval-callback.cpp b/examples/eval-callback/eval-callback.cpp index 17d162d95d37..3f5e382b1208 100644 --- a/examples/eval-callback/eval-callback.cpp +++ b/examples/eval-callback/eval-callback.cpp @@ -15,7 +15,14 @@ static bool run(llama_context * ctx, const common_params & params) { const bool add_bos = llama_vocab_get_add_bos(vocab); - std::vector tokens = common_tokenize(ctx, params.prompt, add_bos); + // Opt-in atomic tokenization of control strings: set + // LLAMA_TOKENIZE_PARSE_SPECIAL=1 to make chat-template tokens like + // <|im_start|> / <|im_end|> / tokenize as a single id instead + // of being byte-split. Default behaviour (env var unset) is unchanged. + const char * env_parse_special = std::getenv("LLAMA_TOKENIZE_PARSE_SPECIAL"); + const bool parse_special = env_parse_special != nullptr && + env_parse_special[0] != '\0' && env_parse_special[0] != '0'; + std::vector tokens = common_tokenize(ctx, params.prompt, add_bos, parse_special); if (tokens.empty()) { LOG_ERR("%s : there are not input tokens to process - (try to provide a prompt with '-p')\n", __func__); diff --git a/hf_assets_gguf/Modelfile b/hf_assets_gguf/Modelfile new file mode 100644 index 000000000000..021aa3b7c19a --- /dev/null +++ b/hf_assets_gguf/Modelfile @@ -0,0 +1,218 @@ +# Modelfile to be used with "ollama" +# adapt "./-Q4_K_M.gguf" with the path where you copy the GGUF file + +FROM ./-Q4_K_M.gguf +PARAMETER seed 1234 +PARAMETER num_ctx 32000 +PARAMETER temperature 0.2 +PARAMETER top_k 20 +PARAMETER top_p 0.95 +PARAMETER min_p 0.1 +SYSTEM "You are a helpful AI assistant named Luciole, trained by LINAGORA and OpenLLM France." +PARAMETER stop "<|im_end|>" +PARAMETER stop "<|im_start|>" +PARAMETER stop "" +PARAMETER stop "" +PARAMETER stop "" +LICENSE " + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + “License” shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + “Licensor” shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + “Legal Entity” shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + “control” means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + “You” (or “Your”) shall mean an individual or Legal Entity + exercising permissions granted by this License. + + “Source” form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + “Object” form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + “Work” shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + “Derivative Works” shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + “Contribution” shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, “submitted“ + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as “Not a Contribution.“ + + “Contributor” shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a “NOTICE” text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an “AS IS” BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets “[]“ + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same “printed page” as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the “License”); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an “AS IS” BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License." diff --git a/hf_assets_gguf/README.md b/hf_assets_gguf/README.md new file mode 100644 index 000000000000..0ccc1046f814 --- /dev/null +++ b/hf_assets_gguf/README.md @@ -0,0 +1,17 @@ +--- +license: apache-2.0 +language: +- fr +- en +pipeline_tag: text-generation +tags: +- openllm-france +base_model: +- OpenLLM-France/ +--- + +![luciole_logo.png](luciole_logo.png) + +-GGUF is a quantized version of [](https://huggingface.co/OpenLLM-France/) (see [llama.cpp](https://github.com/ggerganov/llama.cpp) for quantization details). + +see https://huggingface.co/OpenLLM-France/#test-with-ollama \ No newline at end of file diff --git a/hf_assets_gguf/luciole_logo.png b/hf_assets_gguf/luciole_logo.png new file mode 100644 index 000000000000..27682e877593 Binary files /dev/null and b/hf_assets_gguf/luciole_logo.png differ diff --git a/hf_upload_model.py b/hf_upload_model.py new file mode 100644 index 000000000000..c3b706690db8 --- /dev/null +++ b/hf_upload_model.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +""" +Upload a local folder (or single file) to a HuggingFace repository. + +Usage: + python hf_upload_model.py [--message MSG] + [--private | --public] + [--create-repo {auto,always,never}] + [--repo-type {model,dataset,space}] + +Examples: + python hf_upload_model.py ./Luciole-1B-Instruct-GGUF \\ + OpenLLM-France/Luciole-1B-Instruct-1.0-GGUF \\ + --message "add Q4_K_M + BF16 + assets" + +Authentication: uses your HF token from the env (HF_TOKEN or HUGGING_FACE_HUB_TOKEN) +or the cached token from `huggingface-cli login`. Falls back to an interactive +prompt only when neither is present. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +import huggingface_hub + + +def connect(repo_id: str, repo_type: str, create_mode: str, private: bool): + api = huggingface_hub.HfApi() + try: + api.whoami() + except Exception: + huggingface_hub.login() + api = huggingface_hub.HfApi() + + exists = True + try: + api.repo_info(repo_id, repo_type=repo_type) + except huggingface_hub.utils.RepositoryNotFoundError: + exists = False + + if create_mode == "never" and not exists: + raise SystemExit(f"repo {repo_id} does not exist and --create-repo=never") + if create_mode == "always" or (create_mode == "auto" and not exists): + print(f"creating https://huggingface.co/{repo_id}") + api.create_repo( + repo_id=repo_id, + repo_type=repo_type, + private=private, + exist_ok=True, + ) + return api + + +def upload( + input_path: str, + repo_id: str, + message: str | None = None, + create_repo: str = "auto", + private: bool = True, + repo_type: str = "model", +) -> None: + src = Path(input_path) + if not src.exists(): + raise SystemExit(f"{input_path} does not exist") + + api = connect(repo_id, repo_type, create_repo, private) + repo_url = f"https://huggingface.co/{repo_id}" + + if src.is_dir(): + print(f"uploading folder {src} -> {repo_url}") + api.upload_folder( + folder_path=str(src), + repo_id=repo_id, + repo_type=repo_type, + commit_message=message or f"upload {src.name}", + ignore_patterns=["__pycache__", ".offload", ".git", ".DS_Store"], + ) + else: + print(f"uploading file {src} -> {repo_url}") + api.upload_file( + path_or_fileobj=str(src), + path_in_repo=src.name, + repo_id=repo_id, + repo_type=repo_type, + commit_message=message or f"upload {src.name}", + ) + print(f"done: {repo_url}") + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Upload a folder or file to a HuggingFace repo") + p.add_argument("input", type=str, help="local folder or file to upload") + p.add_argument("repo_id", type=str, help="target repo id, e.g. OpenLLM-France/Luciole-1B-Instruct-1.0-GGUF") + p.add_argument("--message", type=str, default=None, help="commit message") + p.add_argument("--repo-type", choices=["model", "dataset", "space"], default="model") + p.add_argument("--create-repo", choices=["auto", "always", "never"], default="auto", + help="whether to create the repo if it does not exist (default: auto)") + visibility = p.add_mutually_exclusive_group() + visibility.add_argument("--private", dest="private", action="store_true", default=True, + help="create as private (default)") + visibility.add_argument("--public", dest="private", action="store_false", + help="create as public") + return p.parse_args() + + +def main() -> int: + args = parse_args() + upload( + input_path=args.input, + repo_id=args.repo_id, + message=args.message, + create_repo=args.create_repo, + private=args.private, + repo_type=args.repo_type, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test.py b/test.py new file mode 100755 index 000000000000..55dc13404223 --- /dev/null +++ b/test.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +""" +Run every applicable conversion test for a converted model. + +Usage: + python test.py + [--gguf ] [--fp8 ] + [--skip-tokenizer] [--skip-gguf] [--skip-fp8] + [--gguf-file ] [--vllm] [--verbose] + +Steps (each is skipped either on the corresponding --skip flag, or when its +inputs aren't available): + 1. Tokenizer round-trip test tests/test-tokenizer-random.py + 2. GGUF conversion test test_conversion/test_main.py + 3. FP8 conversion test test_conversion_fp8/test_fp8.py + +Defaults: + --gguf defaults to -GGUF (falls back to -gguf). + --fp8 defaults to -FP8 if that folder exists; otherwise the + FP8 test is silently skipped (no failure). +""" + +from __future__ import annotations + +import argparse +import shutil +import subprocess +import sys +import time +from pathlib import Path + +SRCDIR = Path(__file__).resolve().parent + + +def pick_gguf_file(gguf_dir: Path) -> Path | None: + if not gguf_dir.is_dir(): + return None + candidates = sorted( + p for p in gguf_dir.glob("*.gguf") + if "imatrix" not in p.name.lower() and "vocab" not in p.name.lower() + ) + return candidates[0] if candidates else None + + +def run(label: str, cmd: list[str]) -> bool: + print() + print("=" * 78) + print(f"[{label}]") + print("$ " + " ".join(str(x) for x in cmd)) + print("=" * 78) + rc = subprocess.run(cmd, cwd=str(SRCDIR)).returncode + ok = rc == 0 + print(f"[{label}] -> {'PASS' if ok else f'FAIL (exit {rc})'}") + return ok + + +def resolve_gguf_dir(input_folder: Path, override: Path | None) -> Path | None: + if override is not None: + return override + for suffix in ("-GGUF", "-gguf"): + candidate = input_folder.with_name(input_folder.name + suffix) + if candidate.is_dir(): + return candidate + return None + + +def resolve_fp8_dir(input_folder: Path, override: Path | None) -> Path | None: + if override is not None: + return override + candidate = input_folder.with_name(input_folder.name + "-FP8") + return candidate if candidate.is_dir() else None + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Run all applicable conversion tests for a Luciole-style model", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument("input_folder", type=Path, + help="original HF-format model directory (before conversion)") + p.add_argument("--gguf", type=Path, default=None, + help="GGUF conversion folder (default: -GGUF)") + p.add_argument("--fp8", type=Path, default=None, + help="FP8 conversion folder (default: -FP8 if present)") + p.add_argument("--gguf-file", type=Path, default=None, + help="specific .gguf file to feed the tokenizer test " + "(default: first non-imatrix/non-vocab .gguf in the GGUF dir)") + p.add_argument("--skip-tokenizer", action="store_true") + p.add_argument("--skip-gguf", action="store_true") + p.add_argument("--skip-fp8", action="store_true") + p.add_argument("--skip-vllm", action="store_true", + help="skip the vLLM cross-check inside the FP8 test (on by default)") + p.add_argument("--fp8-device", choices=["auto", "cpu", "cuda"], default="auto", + help="device for the FP8 test's HF inference (pass 'cpu' to work around " + "Triton/mamba-ssm crashes on Blackwell)") + p.add_argument("--disable-verbose", action="store_true", + help="do not pass --verbose to individual tests (verbose on by default)") + p.add_argument("--ollama-url", default="http://localhost:11434", + help="Ollama endpoint used by the GGUF conversion test (default: %(default)s)") + return p.parse_args() + + +def ollama_reachable(url: str, timeout: float = 2.0) -> bool: + try: + import urllib.error + import urllib.request + with urllib.request.urlopen(f"{url}/api/version", timeout=timeout) as r: + return r.status == 200 + except (OSError, urllib.error.URLError): + return False + + +def ensure_ollama(url: str, wait_seconds: int = 30) -> bool: + """Return True if Ollama is reachable at `url`, starting it in the + background if it isn't and the `ollama` binary is available on PATH.""" + if ollama_reachable(url): + return True + if not shutil.which("ollama"): + return False + print(f"[ollama] not reachable at {url}; starting `ollama serve` in the background...") + log_path = Path("/tmp/ollama-autostart.log") + log_file = open(log_path, "wb") + subprocess.Popen( + ["ollama", "serve"], + stdout=log_file, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + for _ in range(wait_seconds): + time.sleep(1) + if ollama_reachable(url): + print(f"[ollama] up (logs: {log_path})") + return True + print(f"[ollama] failed to come up within {wait_seconds}s (see {log_path})") + return False + + +def main() -> int: + args = parse_args() + + # .absolute() (not .resolve()) so the symlink itself is preserved: the + # user's naming convention (Luciole-1B-Instruct-1.1) is what -GGUF/-FP8 + # siblings sit next to, not the underlying dpo_luciole_...-step_N target. + input_folder = args.input_folder.absolute() + if not input_folder.is_dir(): + print(f"error: input folder not found: {input_folder}", file=sys.stderr) + return 2 + + gguf_dir = resolve_gguf_dir(input_folder, args.gguf) + fp8_dir = resolve_fp8_dir(input_folder, args.fp8) + + print(f"input: {input_folder}") + print(f"gguf dir: {gguf_dir or '(not found)'}") + print(f"fp8 dir: {fp8_dir or '(not found — FP8 test will be skipped)'}") + + results: dict[str, bool] = {} + + if not args.skip_tokenizer: + if gguf_dir is None: + print("[tokenizer] SKIP (no GGUF folder)") + else: + gguf_file = args.gguf_file or pick_gguf_file(gguf_dir) + if gguf_file is None: + print(f"[tokenizer] SKIP (no .gguf file in {gguf_dir})") + else: + cmd = [sys.executable, "tests/test-tokenizer-random.py", + str(gguf_file), str(input_folder)] + if not args.disable_verbose: + cmd.append("--verbose") + results["tokenizer"] = run("tokenizer", cmd) + + if not args.skip_gguf: + if gguf_dir is None: + print("[gguf-conversion] SKIP (no GGUF folder)") + elif not (gguf_dir / "Modelfile").is_file(): + print(f"[gguf-conversion] SKIP (no Modelfile in {gguf_dir})") + elif not ensure_ollama(args.ollama_url): + print(f"[gguf-conversion] SKIP (Ollama unreachable at {args.ollama_url};") + print( " `ollama` binary not on PATH — install it or pass --ollama-url)") + else: + cmd = [sys.executable, "test_conversion/test_main.py", + str(input_folder), str(gguf_dir), + "--ollama-url", args.ollama_url] + results["gguf-conversion"] = run("gguf-conversion", cmd) + + if not args.skip_fp8: + if fp8_dir is None: + pass + else: + cmd = [sys.executable, "test_conversion_fp8/test_fp8.py", + str(input_folder), str(fp8_dir), + "--device", args.fp8_device] + if not args.skip_vllm: + cmd.append("--vllm") + if not args.disable_verbose: + cmd.append("--verbose") + results["fp8-conversion"] = run("fp8-conversion", cmd) + + print() + print("=" * 78) + print("SUMMARY") + print("=" * 78) + if not results: + print("(no tests ran)") + return 2 + for name, ok in results.items(): + print(f" {name:20s} {'PASS' if ok else 'FAIL'}") + return 0 if all(results.values()) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test.sh b/test.sh new file mode 100755 index 000000000000..bef1831914fb --- /dev/null +++ b/test.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Thin wrapper — real logic lives in test.py. +set -e +cd "$(dirname "$0")" +exec python3 test.py "$@" diff --git a/test_conversion/README.md b/test_conversion/README.md new file mode 100644 index 000000000000..98ebf416ed6c --- /dev/null +++ b/test_conversion/README.md @@ -0,0 +1,283 @@ +# test_conversion + +Validate that an HF transformers model has been converted to GGUF +faithfully — both the tokenizer/chat-template AND the model weights — by +comparing the **transformers reference** against the **GGUF served by +Ollama**. Used to catch tokenizer drift, broken chat templates, +quantization regressions, conversion bugs in `convert_hf_to_gguf.py`, +etc., before publishing a release. + +The suite has two parts: + +1. **`test_main.py`** — main 5-step pipeline (tokenizer, chat + template, behaviour, logits). Run this for every release. +2. **`run_layer_diff.py` + `compare_layers.py`** — deeper layer-by-layer + activation comparison. Run when (1) flags a logit-level regression + and you need to localize which op causes it. + +## Prerequisites + +### Python + +```bash +pip install transformers torch requests numpy matplotlib safetensors +``` + +### llama.cpp built (with our patches) + +The layer-diff tool uses two env-gated patches in `llama.cpp`: + +- `common/debug.cpp` — binary tensor dump triggered by + `LLAMA_DUMP_TENSORS_FILE` and `LLAMA_DUMP_TENSORS_REGEX`. Pure no-op + when those vars are unset. +- `examples/eval-callback/eval-callback.cpp` — atomic tokenization of + control tokens (`<|im_start|>` etc.) when `LLAMA_TOKENIZE_PARSE_SPECIAL=1`. + Default behaviour unchanged. + +Build: + +```bash +cd /path/to/llama.cpp +cmake -B build +cmake --build build --target llama-eval-callback -j$(nproc) +``` + +### Ollama server running + +```bash +ollama serve # in another terminal +``` + +The main pipeline talks to it on `http://localhost:11434`. + +--- + +## Step 0 — Convert HF → GGUF + +Starting from a HuggingFace transformers checkpoint directory: + +```bash +HF_MODEL=/path/to/Luciole-1B-SFT-1.2 # contains config.json, model.safetensors, tokenizer.* +GGUF_DIR=/path/to/Luciole-1B-SFT-1.2-gguf # output directory + +mkdir -p "$GGUF_DIR" +python /path/to/llama.cpp/convert_hf_to_gguf.py \ + "$HF_MODEL" \ + --outfile "$GGUF_DIR/Luciole-1B-SFT-f16.gguf" \ + --outtype f16 +``` + +For a quantized variant (smaller, faster, with some precision loss): + +```bash +/path/to/llama.cpp/build/bin/llama-quantize \ + "$GGUF_DIR/Luciole-1B-SFT-f16.gguf" \ + "$GGUF_DIR/Luciole-1B-SFT-q4_k_m.gguf" \ + Q4_K_M +``` + +## Step 1 — Write the Ollama `Modelfile` + +In `$GGUF_DIR/Modelfile`: + +``` +FROM ./Luciole-1B-SFT-f16.gguf # or your quantized variant +PARAMETER seed 1234 +PARAMETER num_ctx 32000 +PARAMETER temperature 0.6 +SYSTEM "You are a helpful AI assistant named Luciole, trained by LINAGORA and OpenLLM France." +TEMPLATE """ +…your Go-template version of the jinja chat template, including {{- range .Tools }}{{ . }}{{- end }} for tool support… +""" +PARAMETER stop "<|im_end|>" +PARAMETER stop "<|im_start|>" +… +``` + +Two pitfalls Ollama 0.24 hits silently: + +- `FROM` must be **relative** to the Modelfile directory. Absolute paths + fail with `no Modelfile or safetensors files found`. +- Ollama detects tool-calling capability from the template body. For the + `nemotron` architecture only the literal `{{ . }}` form inside + `{{ range .Tools }}` is recognized — `{{ .Function }}` or + `{{ json . }}` will silently disable tool support (Ollama returns + `does not support tools` on any tool request). + +--- + +## Step 2 — Run the main pipeline + +```bash +cd /path/to/llama.cpp/test_conversion +python test_main.py "$HF_MODEL" "$GGUF_DIR" +``` + +This runs five steps; each writes a JSON into +`results/__vs__/` and is skipped on rerun if +its JSON already exists. Pass `--force` to recompute, or delete the +JSON manually for a partial rerun. Slow steps can be turned off with +`--no-behavior` and `--no-logits`. + +### What each step checks + +| step | script | output | what it tests | +|------|--------|--------|---------------| +| 1 | `run_transformers.py` | `transformers.json` | renders each test case with `tokenizer.apply_chat_template(...)` and tokenizes — the reference for everything else. | +| 2 | `run_ollama.py` | `ollama.json` | per case, asks Ollama for `prompt_eval_count` two ways: (a) `/api/chat` (Ollama applies its Modelfile template + GGUF tokenizer); (b) `/api/generate raw=true` fed the transformers-rendered prompt (GGUF tokenizer only). | +| 3 | `run_behavior.py` | `behavior.json` | for cases with `expected_behavior`, sends the prompt to the model via `/api/generate raw=true`, parses the generated text for `{…}`, verifies tool name + required args. Bypasses Ollama's `/api/chat` tool parser, which is unreliable on `nemotron`-arch models in 0.24. | +| 4 | `run_logits.py` | `logits.json` | for the same prompts, runs transformers forward pass and Ollama with `logprobs=true`, compares next-token top-K distributions. Catches quantization / conversion regressions invisible to a binary tool-call test. | +| 5 | `compare.py` | stdout + exit code | unified report. | + +### Reading the report + +`compare.py` prints three sections. + +**Token-count comparison** — per case, two columns: +- `tokenizer` = transformers `apply_chat_template(tokenize=True)` length vs + Ollama's `prompt_eval_count` on the same rendered prompt via raw mode. +- `chat template` = same but Ollama applies its own template. + +Some mismatches are flagged `[WARN]` (with note) instead of `[FAIL]`: +- **tokenizer +1 in tool cases**: known llama.cpp SentencePiece quirk + — a single space following a special/added token gets segmented as a + spurious `▁▁` (two-space) piece. See the *Known issues* section + below. +- **chat template −N in tool cases**: Ollama renders each tool via + Go's `json.Marshal` (compact JSON, no spaces); jinja's `tojson` uses + pretty JSON (with spaces). Same data, only whitespace differs. + +**Behavioural check** — for each `expected_behavior` case, did the +model emit a valid `` block with the right name + args? + +**Logit comparison** — for each case (skipping ones without a generation +prompt), how close are the next-token distributions? + +- `top1` ✓/✗: same most-likely next token (matched by vocab id) +- `|Δlp_top1|`: absolute logprob diff on the chosen token + (fp16-vs-fp16: typically < 0.1; Q4_K_M: < 0.5 normal) +- `mean|Δlp|_top3`: mean of `|Δlp|` over TF's top-3 tokens +- `top5_overlap` / `miss`: how many of TF's top-5 are even in Ollama's top-K + +The aggregate thresholds for FAIL/WARN are documented at the top of +`compare.py`. + +### Exit code + +- `0` — PASS, or PASS with known acceptable warnings. +- `1` — at least one `[FAIL]` somewhere. + +--- + +## Step 3 — Layer-by-layer diagnostic (optional) + +When the logit step flags an unexpected regression on a specific case, +this localizes which layer (and which op type within the layer) is +introducing the divergence. + +```bash +python run_layer_diff.py "$HF_MODEL" "$GGUF_DIR/Luciole-1B-SFT-f16.gguf" \ + --transformers-output results/__vs__/transformers.json \ + --case 02_system_user \ + --work-dir results/__vs__/layer_diff_02_system_user + +python compare_layers.py results/__vs__/layer_diff_02_system_user --top-tail-only +``` + +Outputs: +- `tf_layers.npz` — transformers per-layer hidden states + intermediate + hook outputs (input_layernorm, self_attn, post_attention_layernorm, + mlp, final_norm, logits). +- `gguf_layers.bin` — llama.cpp per-layer activations + (`attn_norm-i`, `ffn_inp-i`, `ffn_norm-i`, `l_out-i`, `result_norm`, + `result_output`). +- `layer_diff_report.txt` — per-pair max/mean abs diff, relative max, + cosine distance, l2 relative error. +- `layer_diff_overview.png` — log-scale divergence vs layer index, + one series per op type. +- `layer_diff_l_out.png` — focused view of per-layer block output drift. + +`--top-tail-only` restricts comparison to the **last token position** — +this is what matters for next-token prediction and avoids confusion at +the last layer where llama.cpp uses `inp_out_ids` to compute only the +last position. + +### How to read the layer-diff + +If `L00 attn_norm` matches to floating-point precision but `L00 ffn_inp` +diverges, the **attention block** is to blame. If `L00 attn_norm` already +diverges, the **input embedding or tokenization** is to blame (or the +input LN itself). And so on along the column of op types. + +--- + +## Known issues / current findings (Luciole 1B SFT 1.2) + +- **Tokenizer +1 in tool cases** — llama.cpp's SentencePiece-style + tokenizer emits a spurious `▁▁` (double-space) piece when a special + token is followed by exactly one literal space then text. Affects the + fixed instruction string `function name and arguments within + XML tags:` in the system prompt of every + tool-using conversation. Reported upstream; harmless (decoded string + unchanged), but the model sees one out-of-distribution token per + request. Flagged `[WARN]`. + +- **Chat template `−N` in tool cases** — Ollama renders each tool with + Go's `json.Marshal` (compact); jinja uses pretty JSON. ~21 tokens + saved per tool definition. Cosmetic; the model parses both + identically. Flagged `[WARN]`. + +- **Logit drift at layer 0, attention block** — even with f16 GGUF + matching f16 transformers, the attention output already diverges + significantly at L00 (cos_d ≈ 0.05 on case 02). Most likely + PyTorch SDPA vs llama.cpp attention kernel: different reduction + orders in fp16 give different accumulation. Drifts to ~0.3 cos_d by + the last layer. Top-1 token usually still matches. + +- **`convert_hf_to_gguf.py` precision pitfall** — for the Nemotron + LayerNorm1p hack, `data_torch + 1` must be done in fp32, otherwise + the bf16 source values round before storage. Use + `data_torch.float() + 1`. Other entries in the converter with the + same `+ 1` pattern (Gemma, Nemotron-H, line ~8887 MTP block, lines + 5731+ for some mamba variant) should be audited similarly. + +--- + +## File layout + +``` +test_conversion/ +├── README.md # this file +├── test_main.py # main orchestrator (steps 1–5) +├── test_cases.py # canonical test conversations +├── run_transformers.py # step 1 +├── run_ollama.py # step 2 +├── run_behavior.py # step 3 +├── run_logits.py # step 4 +├── compare.py # step 5 — unified report +├── run_layer_diff.py # layer-diff tool +├── compare_layers.py # layer-diff report + plots +├── test.sh # convenience wrapper (if present) +└── results/ # outputs land here, one subfolder per __vs__ + └── __vs__/ + ├── transformers.json + ├── ollama.json + ├── behavior.json + ├── logits.json + └── layer_diff_/ + ├── tf_layers.npz + ├── gguf_layers.bin + ├── meta.json + ├── layer_diff_report.txt + └── *.png +``` + +## Hard-coded paths to update + +`run_layer_diff.py` has the llama.cpp build path baked in at the top: + +```python +LLAMA_BIN_DIR = Path("/home/jlouradour/src.nowsl/llama.cpp/build/bin") +``` + +Change this if your build directory is elsewhere. diff --git a/test_conversion/compare.py b/test_conversion/compare.py new file mode 100644 index 000000000000..1cd182881db4 --- /dev/null +++ b/test_conversion/compare.py @@ -0,0 +1,348 @@ +""" +Compare transformers.json and ollama.json (token counts) and optionally +behavior.json (functional tool-call check), then print a per-test report. + +Token-count comparison (two checks per case): + + Tokenizer + Pass the transformers-rendered prompt through Ollama with raw=true + and compare prompt_eval_count to len(transformers token_ids). + Tests just the GGUF tokenizer, isolated from the chat template. + + Chat template + Pass the conversation through Ollama's /api/chat. + Tests Ollama's template + GGUF tokenizer together. + +Known acceptable divergences (reported as [WARN], not [FAIL]): + + [tokenizer +1 in tool cases] + SentencePiece single-space-after-special-token quirk in llama.cpp's + BPE tokenizer. The model sees one extra space-prefix token where the + HF tokenizer didn't. Harmless (same decoded string). See llama.cpp + issue tracker for the upstream bug. + + [chat template -N in tool cases] + Ollama renders each tool via Go's json.Marshal (compact JSON, no + spaces). The jinja template uses tojson (pretty JSON, with spaces). + Same data, same field order, just whitespace. The model parses both + identically; cosmetic. + +Behavioural section (only if --behavior path is provided): + + For each case with an `expected_behavior` field, run_behavior.py asked + Ollama to actually generate a turn and checked whether the assistant's + response satisfied the expectation (correct tool_call name + args). + +Logits section (only if --logits path is provided): + + Per-case next-token top-K log-probability comparison between + transformers (reference) and Ollama (GGUF). Catches subtle conversion + or quantization regressions that the binary behavioural test misses. + Aggregate metrics: top-1 agreement rate, mean KL divergence. + +Exit code is 0 iff no [FAIL] anywhere; [WARN]s are non-fatal. + + Heuristic thresholds for the logits section: + top-1 agreement < 50% -> [FAIL] (very likely conversion bug) + top-1 agreement < 80% -> [WARN] + aggregate |Δlp_top1| > 1.0 -> [FAIL] (model is very differently confident) + aggregate |Δlp_top1| > 0.3 -> [WARN] + aggregate mean|Δlp|_top3 > 3.0 -> [WARN] (not a FAIL — top-3 is + sensitive to tail noise; use + only as a soft signal) + + Notes: + - Top-1 lp diff is the primary numeric signal. For fp16-vs-fp16 it + is typically < 0.1. For Q4_K_M, < 0.5 is normal. + - mean|Δlp|_top3 is reported for completeness but is noisy by nature: + fp16 softmax precision degrades for low-probability tokens, so a + single tail outlier can drag the mean up. Use top-1 metrics for + pass/fail; treat mean_top3 as informational. + - Top-1 mismatches often differ only in vocab variant (e.g. `▁The` + vs `The` for the same word) — those are real model behaviour + differences worth noting but typically caused by SP/llama.cpp + tokenization quirks, not gross conversion errors. + +Usage: + python compare.py + [--behavior ] + [--logits ] +""" + +import argparse +import json +import sys +from pathlib import Path + + +def load(path): + return {r["name"]: r for r in json.loads(Path(path).read_text())} + + +def fmt(status): + return {"ok": "[ OK ]", "warn": "[WARN]", "fail": "[FAIL]"}[status] + + +def classify(case_has_tools, tf_count, actual_count, kind): + """Return (status, label, note) for one column.""" + if actual_count is None: + return "ok", "skipped", None # neutral; not a failure if intentionally skipped + diff = actual_count - tf_count + if diff == 0: + return "ok", f"{tf_count} vs {actual_count}", None + + if kind == "tokenizer" and diff == 1 and case_has_tools: + return ("warn", + f"{tf_count} vs {actual_count} (+1)", + "SPM single-space-after-special-token quirk (llama.cpp tokenizer bug; harmless)") + if kind == "chat" and diff < 0 and case_has_tools: + return ("warn", + f"{tf_count} vs {actual_count} ({diff:+d})", + "Ollama renders tools as compact JSON; jinja uses pretty JSON (whitespace only; cosmetic)") + return "fail", f"{tf_count} vs {actual_count} ({diff:+d})", None + + +def report_counts(tf_map, ol_map): + """Returns (has_any_fail, has_any_warn, collected_notes_by_name).""" + names = sorted(set(tf_map) | set(ol_map)) + notes_by_name = {} # name -> list of strings + any_fail = False + any_warn = False + + print(f"\n{'name':<40} {'tokenizer':<32} {'chat template':<32}") + print("-" * 110) + + for name in names: + tf_r = tf_map.get(name) + ol_r = ol_map.get(name) + + if tf_r is None or "error" in tf_r: + err = tf_r.get("error", "missing") if tf_r else "missing" + print(f"{name:<40} transformers ERROR: {err}") + any_fail = True + continue + if ol_r is None: + print(f"{name:<40} ollama ERROR: missing") + any_fail = True + continue + + tf_count = tf_r["token_count"] + has_tools = tf_r.get("tools") is not None + + # Tokenizer probe + raw_err = ol_r.get("raw_error") + if raw_err: + tok_status, tok_label = "fail", f"err: {raw_err[:24]}" + tok_note = None + else: + tok_status, tok_label, tok_note = classify( + has_tools, tf_count, ol_r.get("raw_prompt_eval_count"), "tokenizer") + + # Chat template probe + chat_err = ol_r.get("chat_error") + if chat_err: + chat_status, chat_label = "fail", f"err: {chat_err[:24]}" + chat_note = None + else: + chat_status, chat_label, chat_note = classify( + has_tools, tf_count, ol_r.get("chat_prompt_eval_count"), "chat") + + tok_cell = f"{fmt(tok_status)} {tok_label}" + chat_cell = f"{fmt(chat_status)} {chat_label}" + print(f"{name:<40} {tok_cell:<32} {chat_cell:<32}") + + notes = [] + if tok_note: notes.append(f"tokenizer: {tok_note}") + if chat_note: notes.append(f"chat: {chat_note}") + if notes: + notes_by_name[name] = notes + + any_fail = any_fail or (tok_status == "fail" or chat_status == "fail") + any_warn = any_warn or (tok_status == "warn" or chat_status == "warn") + + # Print accumulated WARN notes (each unique note once is more readable) + if notes_by_name: + print() + print("WARN notes:") + printed = set() + for name, notes in notes_by_name.items(): + for n in notes: + if n not in printed: + print(f" - {n}") + printed.add(n) + print(" (cases with these warnings: " + + ", ".join(sorted(notes_by_name)) + ")") + + return any_fail, any_warn + + +def report_logits(logits_list): + """Logits section. Returns (has_any_fail, has_any_warn).""" + print(f"\n{'name':<40} {'top1':<5} {'|Δlp_top1|':<11} {'mean|Δlp|_top3':<15} " + f"{'top5':<5} {'miss':<5} {'tf top-1':<22} {'ol top-1':<22}") + print("-" * 140) + + top1_total = 0 + top1_matches = 0 + top1_diffs = [] + top3_means = [] + skipped = [] + any_error = False + + for r in logits_list: + name = r["name"] + if "skipped" in r: + skipped.append((name, r["skipped"])) + continue + if "error" in r: + print(f"{name:<40} ERROR: {r['error']}") + any_error = True + continue + cmp = r.get("comparison") + if not cmp: + print(f"{name:<40} no comparison (empty logprobs?)") + any_error = True + continue + + top1_total += 1 + if cmp["top1_match"]: + top1_matches += 1 + if cmp.get("top1_lp_diff") is not None: + top1_diffs.append(cmp["top1_lp_diff"]) + if cmp.get("mean_lp_diff_top3") is not None: + top3_means.append(cmp["mean_lp_diff_top3"]) + + f = lambda v: (f"{v:.4f}" if v is not None else "n/a") + miss_s = f"{cmp['tf_top5_missing_in_ollama_topk']}/5" + tf1 = cmp["tf_top1"]; ol1 = cmp["ol_top1"] + tf_lbl = f"{tf1['tok']!r}@{tf1['lp']}" + ol_lbl = f"{ol1['tok']!r}@{ol1['lp']}" + marker = "✓" if cmp["top1_match"] else "✗" + print(f"{name:<40} {marker:<5} {f(cmp.get('top1_lp_diff')):<11} " + f"{f(cmp.get('mean_lp_diff_top3')):<15} {cmp['top5_overlap']}/5 " + f"{miss_s:<5} {tf_lbl[:21]:<22} {ol_lbl[:21]:<22}") + + if skipped: + print() + print(" Skipped (no add_generation_prompt — no canonical next token):") + for n, why in skipped: + print(f" - {n} ({why})") + + print() + if top1_total == 0: + print(" (no logit comparisons completed)") + return any_error, False + + top1_rate = top1_matches / top1_total + agg_top1 = (sum(top1_diffs) / len(top1_diffs)) if top1_diffs else None + agg_top3 = (sum(top3_means) / len(top3_means)) if top3_means else None + + print(f" aggregate over {top1_total} comparable case(s):") + print(f" top-1 agreement = {top1_matches}/{top1_total} = {top1_rate*100:.1f}%") + if agg_top1 is not None: + print(f" aggregate |Δlp_top1| = {agg_top1:.4f}") + if agg_top3 is not None: + print(f" aggregate mean|Δlp|_top3 = {agg_top3:.4f}") + + fail = False + warn = False + if top1_rate < 0.5: + print(f" [FAIL] top-1 agreement {top1_rate*100:.1f}% < 50% — likely a conversion bug") + fail = True + elif top1_rate < 0.8: + print(f" [WARN] top-1 agreement {top1_rate*100:.1f}% < 80% — investigate the mismatching cases") + warn = True + if agg_top1 is not None: + if agg_top1 > 1.0: + print(f" [FAIL] |Δlp_top1| {agg_top1:.4f} > 1.0 — confidence on top token diverges sharply") + fail = True + elif agg_top1 > 0.3: + print(f" [WARN] |Δlp_top1| {agg_top1:.4f} > 0.3 — model is less confident on chosen tokens; investigate") + warn = True + if agg_top3 is not None and agg_top3 > 3.0: + # WARN-only: top-3 mean is noisy by nature (fp16 softmax tail). + print(f" [WARN] mean|Δlp|_top3 {agg_top3:.4f} > 3.0 — distribution shifted in top-3 (soft signal)") + warn = True + return (fail or any_error), warn + + +def report_behavior(behavior_map): + """Behavioural section. Returns has_any_fail.""" + print(f"\n{'name':<40} {'behaviour':<60}") + print("-" * 110) + + any_fail = False + for name in sorted(behavior_map): + r = behavior_map[name] + ok = r.get("pass") is True + reason = r.get("fail_reason") or "" + status = "ok" if ok else "fail" + print(f"{name:<40} {fmt(status)} {reason[:55]}") + if not ok: + any_fail = True + # Print actual model output details for debugging + raw = r.get("raw_output") + if raw is not None: + snippet = raw if len(raw) <= 200 else raw[:200] + "...(truncated)" + print(f" raw model output: {snippet!r}") + parsed = r.get("parsed_tool_call") + if parsed: + print(f" parsed tool_call: name={parsed.get('name')!r} args={parsed.get('arguments')!r}") + return any_fail + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("transformers_json") + parser.add_argument("ollama_json") + parser.add_argument("--behavior", default=None, + help="Optional behaviour JSON from run_behavior.py") + parser.add_argument("--logits", default=None, + help="Optional logits JSON from run_logits.py") + args = parser.parse_args() + + tf_map = load(args.transformers_json) + ol_map = load(args.ollama_json) + + print("=" * 120) + print("TOKEN COUNT COMPARISON") + print("=" * 120) + count_fail, count_warn = report_counts(tf_map, ol_map) + + behavior_fail = False + if args.behavior and Path(args.behavior).exists(): + beh_map = load(args.behavior) + if beh_map: + print() + print("=" * 120) + print("BEHAVIOURAL CHECK (model actually called the right tool)") + print("=" * 120) + behavior_fail = report_behavior(beh_map) + + logits_fail = False + logits_warn = False + if args.logits and Path(args.logits).exists(): + logits_list = json.loads(Path(args.logits).read_text()) + if logits_list: + print() + print("=" * 120) + print("LOGIT COMPARISON (next-token top-K distribution; transformers vs Ollama)") + print("=" * 120) + logits_fail, logits_warn = report_logits(logits_list) + + print() + any_fail = count_fail or behavior_fail or logits_fail + any_warn = count_warn or logits_warn + if any_fail: + print(">>> RESULT: FAIL") + sys.exit(1) + elif any_warn: + print(">>> RESULT: PASS (with known acceptable warnings)") + sys.exit(0) + else: + print(">>> RESULT: PASS") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/test_conversion/compare_layers.py b/test_conversion/compare_layers.py new file mode 100644 index 000000000000..2522343068c4 --- /dev/null +++ b/test_conversion/compare_layers.py @@ -0,0 +1,283 @@ +""" +Compare per-layer activations from tf_layers.npz and gguf_layers.bin +and produce a divergence plot. + +For each pair (transformers tensor, GGUF tensor) referring to the same +position in the graph, we compute: + + max_abs_diff max |x_tf - x_gg| in fp32 + mean_abs_diff mean over all elements + rel_max max_abs_diff / max(|x_tf|, |x_gg|, 1e-8) + cosine 1 - cos(x_tf, x_gg) (smaller = closer) + l2_rel ||x_tf - x_gg|| / ||x_tf|| + +Mappings used (nemotron architecture): + GGUF transformers + l_out-i hidden-(i+1) per-layer output (post-residual) + attn_norm-i attn_norm-i input_layernorm output + ffn_inp-i (attn-residual sum) not directly hookable in transformers; + approximated as hidden-i + self_attn-i + ffn_norm-i post_norm-i post_attention_layernorm output + ffn_out-i (mlp + residual) approximated as ffn_inp + mlp output + result_norm final_norm output of model.model.norm + result_output logits final LM head + +The first three columns of the report use l_out / final_norm / logits, which +are the most reliable (no hook approximation). The fine-grained per-op +analysis uses the rest. + +Output: + /layer_diff_report.txt text report + /layer_diff_overview.png log-scale per-layer divergence plot + /layer_diff_by_op.png per-op-type breakdown + +Usage: + python compare_layers.py +""" + +import argparse +import json +import struct +import sys +from collections import defaultdict +from pathlib import Path + +import numpy as np + + +# GGML type ids → numpy dtype (for the ones we'll encounter on activation tensors). +GGML_TYPE = { + 0: ("f32", np.float32), + 1: ("f16", np.float16), + 24: ("bf16", None), # handled specially + 26: ("i32", np.int32), + 30: ("i64", np.int64), +} + + +def parse_gguf_dump(path: Path): + """Yield (name, np_array_fp32) from the binary dump.""" + with open(path, "rb") as f: + data = f.read() + i = 0 + n = 0 + while i < len(data): + if i + 4 > len(data): + break + name_len = struct.unpack_from(" 1024: + break + name = data[i:i+name_len].decode("utf-8", errors="replace"); i += name_len + dtype = struct.unpack_from(" 1) or (1,) + try: + arr = arr.reshape(shape[::-1]) # ggml stores ne in element-stride order + except ValueError: + # fallback: leave as flat + pass + yield name, arr + n += 1 + + +def compute_diff_metrics(x_tf, x_gg): + """Both inputs flattened to fp32 and same total element count.""" + if x_tf.size != x_gg.size: + return None + a = x_tf.astype(np.float32).reshape(-1) + b = x_gg.astype(np.float32).reshape(-1) + diff = a - b + abs_diff = np.abs(diff) + max_abs = float(abs_diff.max()) + mean_abs = float(abs_diff.mean()) + denom = float(max(np.abs(a).max(), np.abs(b).max(), 1e-8)) + rel_max = max_abs / denom + # cosine-distance + na = float(np.linalg.norm(a)) + nb = float(np.linalg.norm(b)) + cos = 1.0 - float(a @ b) / (na * nb + 1e-12) + l2_rel = float(np.linalg.norm(diff)) / (na + 1e-12) + return dict(max_abs=max_abs, mean_abs=mean_abs, rel_max=rel_max, cosine=cos, l2_rel=l2_rel) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("work_dir") + parser.add_argument("--top-tail-only", action="store_true", + help="For multi-token tensors, only compare the LAST token position " + "(matches what next-token prediction uses).") + args = parser.parse_args() + + work_dir = Path(args.work_dir).resolve() + tf_path = work_dir / "tf_layers.npz" + gg_path = work_dir / "gguf_layers.bin" + meta = json.loads((work_dir / "meta.json").read_text()) + + print(f"Comparing layer activations for case {meta['case']!r}") + print(f" HF dir: {meta['hf_model_dir']}") + print(f" GGUF: {meta['gguf_path']}") + print() + + tf = dict(np.load(tf_path)) + gg = dict(parse_gguf_dump(gg_path)) + print(f"transformers: {len(tf)} arrays") + print(f"gguf: {len(gg)} arrays") + + n_layers = max(int(k.split("-")[1]) for k in tf if k.startswith("hidden-")) + 1 - 1 + # hidden_states has N+1 entries (0..N); N = num_layers + print(f"layers: {n_layers}") + + T = int(tf["tokens"].shape[-1]) + print(f"tokens: {T}") + + # Mapping rules: each entry is (label, op_type, tf_array_or_callable, gg_key) + # tf entry can be a string (npz key) or a callable taking the tf dict and + # returning an array — to combine multiple hook outputs. + def add_resid(i): + """ffn_inp = hidden[i] + self_attn[i] (post-attention, pre-MLP, with residual)""" + return lambda tfd: tfd[f"hidden-{i}"] + tfd[f"self_attn-{i}"] + + mappings = [] + for i in range(n_layers): + mappings.append((f"L{i:02d} attn_norm", "norm", f"attn_norm-{i}", f"attn_norm-{i}")) + mappings.append((f"L{i:02d} ffn_inp", "post_attn", add_resid(i), f"ffn_inp-{i}")) + mappings.append((f"L{i:02d} ffn_norm", "norm", f"post_norm-{i}", f"ffn_norm-{i}")) + mappings.append((f"L{i:02d} l_out", "block_out", f"hidden-{i+1}", f"l_out-{i}")) + mappings.append(("final_norm", "norm", "final_norm", "result_norm")) + mappings.append(("logits", "head", "logits", "result_output")) + + rows = [] + for label, op, tk, gk in mappings: + # Resolve transformers tensor (string key or callable on the dict) + if callable(tk): + try: + x_tf = tk(tf) + except KeyError as e: + rows.append({"label": label, "op": op, "skip": f"missing tf key {e}"}) + continue + else: + if tk not in tf: + rows.append({"label": label, "op": op, "skip": f"missing tf={tk}"}) + continue + x_tf = tf[tk] + if gk not in gg: + rows.append({"label": label, "op": op, "skip": f"missing gg={gk}"}) + continue + x_gg = gg[gk] + if args.top_tail_only: + # Pick last token slice + x_tf = x_tf.reshape(-1, x_tf.shape[-1])[-1] if x_tf.ndim >= 2 else x_tf + x_gg = x_gg.reshape(-1, x_gg.shape[-1])[-1] if x_gg.ndim >= 2 else x_gg + m = compute_diff_metrics(x_tf, x_gg) + if m is None: + rows.append({"label": label, "op": op, "skip": f"shape mismatch tf={x_tf.shape} gg={x_gg.shape}"}) + continue + rows.append({"label": label, "op": op, **m, "shape": tuple(x_tf.shape)}) + + # ─── Text report ─── + out_txt = work_dir / "layer_diff_report.txt" + lines = [] + lines.append(f"Layer-by-layer activation comparison — case {meta['case']!r}") + lines.append(f"HF dtype: {meta['dtype']} on {meta['device']}") + lines.append("") + lines.append(f"{'label':<25} {'op':<10} {'shape':<22} {'max|Δ|':<10} {'mean|Δ|':<10} {'rel_max':<9} {'cos_d':<10} {'l2_rel':<10}") + lines.append("-" * 120) + for r in rows: + if "skip" in r: + lines.append(f"{r['label']:<25} {r['op']:<10} SKIP: {r['skip']}") + continue + shape = str(r["shape"]) + lines.append(f"{r['label']:<25} {r['op']:<10} {shape:<22} " + f"{r['max_abs']:<10.4g} {r['mean_abs']:<10.4g} " + f"{r['rel_max']:<9.3g} {r['cosine']:<10.4g} {r['l2_rel']:<10.4g}") + report = "\n".join(lines) + out_txt.write_text(report + "\n") + print() + print(report) + print() + print(f"Wrote {out_txt}") + + # ─── Plots ─── + try: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except ImportError: + print("matplotlib not installed; skipping plots (pip install matplotlib)") + return + + valid = [r for r in rows if "skip" not in r] + layer_rows = [r for r in valid if r["label"].startswith("L")] + + # Plot 1: overview, divergence vs layer index, separate lines per op + fig, axes = plt.subplots(1, 2, figsize=(14, 5)) + by_op = defaultdict(list) + for r in layer_rows: + # label is "L00 attn_norm" / "L00 ffn_norm" / "L00 l_out" + op_label = r["label"].split()[1] + layer_idx = int(r["label"][1:3]) + by_op[op_label].append((layer_idx, r)) + + for ax, metric in zip(axes, ["max_abs", "l2_rel"]): + for op_label, lst in by_op.items(): + lst.sort() + xs = [li for li, _ in lst] + ys = [r[metric] for _, r in lst] + ax.plot(xs, ys, marker="o", label=op_label) + # add final_norm and logits as scatter at x = N + for r in valid: + if r["label"] == "final_norm": + ax.scatter([n_layers], [r[metric]], marker="*", s=200, label="final_norm") + if r["label"] == "logits": + ax.scatter([n_layers + 0.3], [r[metric]], marker="X", s=150, label="logits") + ax.set_xlabel("layer index") + ax.set_ylabel(metric) + ax.set_yscale("log") + ax.set_title(f"divergence per layer ({metric})") + ax.grid(True, which="both", alpha=0.3) + ax.legend(fontsize=8) + fig.suptitle(f"transformers vs GGUF — case {meta['case']!r}") + fig.tight_layout() + p1 = work_dir / "layer_diff_overview.png" + fig.savefig(p1, dpi=110, bbox_inches="tight") + print(f"Wrote {p1}") + + # Plot 2: cumulative growth of l_out divergence to highlight where drift accumulates + fig, ax = plt.subplots(figsize=(10, 5)) + l_out_rows = sorted(by_op.get("l_out", [])) + if l_out_rows: + xs = [li for li, _ in l_out_rows] + for metric in ["max_abs", "l2_rel", "cosine"]: + ys = [r[metric] for _, r in l_out_rows] + ax.plot(xs, ys, marker="o", label=metric) + ax.set_xlabel("layer index") + ax.set_ylabel("metric") + ax.set_yscale("log") + ax.set_title(f"l_out divergence over depth — case {meta['case']!r}") + ax.grid(True, which="both", alpha=0.3) + ax.legend() + fig.tight_layout() + p2 = work_dir / "layer_diff_l_out.png" + fig.savefig(p2, dpi=110, bbox_inches="tight") + print(f"Wrote {p2}") + + +if __name__ == "__main__": + main() diff --git a/test_conversion/run_behavior.py b/test_conversion/run_behavior.py new file mode 100644 index 000000000000..8c1762d21949 --- /dev/null +++ b/test_conversion/run_behavior.py @@ -0,0 +1,224 @@ +""" +Behavioural test: for each test case that has an `expected_behavior` field, +actually run the model and verify the assistant response satisfies the +expectation (e.g. emits a tool_call with the right name + args). + +IMPORTANT design note — why /api/generate raw=true: + + Ollama's /api/chat tool-call parser (at least in 0.24) silently drops + model output when it detects a tool_call tag but fails to extract a + valid call mid-stream. The model can emit a perfectly well-formed + ... block and the chat API still returns + {"content": "", "tool_calls": null}. That hides whether the *model* + works. + + To test the model behind Ollama, we bypass the chat layer entirely: + 1. Take the transformers-rendered prompt (already computed in + transformers.json — same exact string the model would see at + training time). + 2. Feed it via /api/generate with raw=true (no template, no parsing). + 3. Read back the raw text the model emitted, parse blocks + ourselves with a regex, and check the expectation. + + This tests the GGUF model + tokenizer end-to-end without Ollama's chat + quirks getting in the way. + +Currently supports one kind of expectation: + + expected_behavior = { + "tool_call": { + "name": "get_weather", + "required_args": ["location"], + "args_must_contain": {"location": "paris"}, # case-insensitive substring + } + } + +Output JSON: one entry per case with expected_behavior, including +{name, expected, raw_output, parsed_tool_call, pass, fail_reason}. + +Usage: + python run_behavior.py + --transformers-output + [--model-name NAME] + [--ollama-url URL] + [--num-predict N] +""" + +import argparse +import json +import re +import sys +import traceback +from pathlib import Path + +try: + import requests +except ImportError: + sys.exit("ERROR: this script needs the 'requests' package (pip install requests).") + +sys.path.insert(0, str(Path(__file__).parent)) +from test_cases import TEST_CASES # noqa: E402 +from run_ollama import check_ollama_alive, ollama_create, ollama_delete, _post # noqa: E402 + + +# Matches " ... " with any whitespace inside. +TOOL_CALL_RE = re.compile(r"\s*(\{.*?\})\s*", re.DOTALL) + + +def ollama_generate_raw(url, model, prompt, num_predict): + payload = { + "model": model, + "prompt": prompt, + "raw": True, + "stream": False, + "options": { + "num_predict": num_predict, + "temperature": 0, + "seed": 0, + # Stop at the chat-message terminator so we don't waste tokens + # generating into the next turn. + "stop": ["<|im_end|>"], + }, + } + return _post(f"{url}/api/generate", payload) + + +def parse_tool_call_from_text(text): + """Find the first ... block and parse its JSON. + + Returns (call_dict_or_None, error_str_or_None) where call_dict is + {"name": str, "arguments": dict} on success. + """ + m = TOOL_CALL_RE.search(text) + if not m: + return None, "no ... block in output" + body = m.group(1) + try: + obj = json.loads(body) + except json.JSONDecodeError as e: + return None, f" body is not valid JSON: {e}; body={body!r}" + if "name" not in obj: + return None, f" body has no 'name' field; body={obj!r}" + args = obj.get("arguments", {}) + if isinstance(args, str): + # Some templates emit arguments as a JSON-encoded string. + try: + args = json.loads(args) + except json.JSONDecodeError: + pass + return {"name": obj["name"], "arguments": args}, None + + +def evaluate_tool_call(expected, parsed): + """Check the parsed tool call matches expectation. + + Returns (pass: bool, reason: str | None). + """ + if parsed["name"] != expected["name"]: + return False, f"wrong tool name: got {parsed['name']!r}, expected {expected['name']!r}" + + args = parsed["arguments"] + if not isinstance(args, dict): + return False, f"arguments not a dict: {args!r}" + + for key in expected.get("required_args", []): + if key not in args: + return False, f"missing required arg {key!r} (got args: {list(args)})" + + for key, needle in expected.get("args_must_contain", {}).items(): + val = args.get(key) + if val is None: + return False, f"missing arg {key!r}" + if needle.lower() not in str(val).lower(): + return False, f"arg {key!r}={val!r} does not contain {needle!r}" + + return True, None + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("modelfile_path") + parser.add_argument("output_json") + parser.add_argument("--transformers-output", required=True, + help="Path to transformers.json (provides the exact " + "rendered prompt for each case)") + parser.add_argument("--model-name", default="test-chat-template-tmp", + help="Temporary ollama model name (created/deleted by the script).") + parser.add_argument("--ollama-url", default="http://localhost:11434") + parser.add_argument("--num-predict", type=int, default=256, + help="Max tokens the model may generate per case.") + args = parser.parse_args() + + version = check_ollama_alive(args.ollama_url) + print(f"[behavior] Ollama reachable (version {version})") + + tf_path = Path(args.transformers_output) + if not tf_path.exists(): + sys.exit(f"ERROR: transformers output not found at {tf_path}. " + "Run run_transformers.py first.") + transformers_by_name = {r["name"]: r for r in json.loads(tf_path.read_text())} + + cases_with_behavior = [c for c in TEST_CASES if c.get("expected_behavior")] + if not cases_with_behavior: + print("[behavior] No test cases have an 'expected_behavior' field; nothing to do.") + Path(args.output_json).write_text("[]") + return + + print(f"[behavior] {len(cases_with_behavior)} behavioural case(s) to run") + + ollama_create(args.model_name, args.modelfile_path) + try: + results = [] + for case in cases_with_behavior: + print(f"[behavior] {case['name']}") + entry = { + "name": case["name"], + "expected_behavior": case["expected_behavior"], + } + try: + tf = transformers_by_name.get(case["name"]) + if tf is None or "rendered_prompt" not in tf: + raise RuntimeError( + f"no rendered_prompt for {case['name']} in transformers output" + ) + resp = ollama_generate_raw( + args.ollama_url, args.model_name, + tf["rendered_prompt"], num_predict=args.num_predict, + ) + raw_output = resp.get("response", "") + entry["raw_output"] = raw_output + entry["eval_count"] = resp.get("eval_count") + + eb = case["expected_behavior"] + if "tool_call" in eb: + parsed, parse_err = parse_tool_call_from_text(raw_output) + entry["parsed_tool_call"] = parsed + if parsed is None: + entry["pass"] = False + entry["fail_reason"] = parse_err + else: + ok, reason = evaluate_tool_call(eb["tool_call"], parsed) + entry["pass"] = ok + entry["fail_reason"] = reason + else: + entry["pass"] = False + entry["fail_reason"] = f"unknown expected_behavior keys: {list(eb)}" + except Exception as e: + traceback.print_exc() + entry["pass"] = False + entry["fail_reason"] = f"{type(e).__name__}: {e}" + + marker = "OK " if entry.get("pass") else "FAIL" + print(f" -> {marker} {entry.get('fail_reason') or ''}") + results.append(entry) + finally: + ollama_delete(args.model_name) + + out = Path(args.output_json) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(results, indent=2, ensure_ascii=False)) + print(f"[behavior] Wrote {len(results)} results to {out}") + + +if __name__ == "__main__": + main() diff --git a/test_conversion/run_layer_diff.py b/test_conversion/run_layer_diff.py new file mode 100644 index 000000000000..167eca64c19f --- /dev/null +++ b/test_conversion/run_layer_diff.py @@ -0,0 +1,180 @@ +""" +Layer-by-layer activation dump for both backends, on a single anchor prompt. + +Outputs two files in : + + tf_layers.npz — numpy archive with one array per intermediate tensor: + "tokens" : input token ids (1, T) + "hidden-i" for i in 0..N : per-layer output of the i-th block + (transformers .hidden_states) + "attn_norm-i" for i in 0..N-1 : output of input_layernorm + "self_attn-i" for i in 0..N-1 : output of the attention block (without residual) + "post_norm-i" for i in 0..N-1 : output of post_attention_layernorm + "mlp-i" for i in 0..N-1 : output of MLP (without residual) + "final_norm" : after model.model.norm + "logits" : final LM head output + + gguf_layers.bin — binary dump from llama-eval-callback (env-gated). + Records of: u32 name_len, name, u32 dtype, i64 ne[4], u64 nbytes, data. + +The companion compare_layers.py loads both and computes per-layer divergence. + +Usage: + python run_layer_diff.py + --transformers-output + --case + --work-dir + [--device cuda|cpu] + [--dtype fp16|fp32|bf16] +""" + +import argparse +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import numpy as np +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + + +LLAMA_BIN_DIR = Path("/home/jlouradour/src.nowsl/llama.cpp/build/bin") +EVAL_CALLBACK_BIN = LLAMA_BIN_DIR / "llama-eval-callback" + +# Tensor name regex passed to the patched llama.cpp dumper. +# Keep aligned with the names cb()'d by src/models/nemotron.cpp. +GGUF_DUMP_REGEX = r'^(attn_norm|ffn_inp|ffn_norm|ffn_out|l_out|result_norm|result_output)-?[0-9]*$' + + +def transformers_dump(model_dir: Path, prompt: str, device: str, dtype: torch.dtype, out_path: Path): + print(f"[tf] Loading model from {model_dir} ({device}, {dtype})") + tokenizer = AutoTokenizer.from_pretrained(str(model_dir), trust_remote_code=True) + model = AutoModelForCausalLM.from_pretrained( + str(model_dir), torch_dtype=dtype, trust_remote_code=True, + ).to(device) + model.eval() + + # Match the way our test renders prompts: tokenize the raw rendered prompt + # without adding extra special tokens — the prompt already contains them. + inputs = tokenizer(prompt, return_tensors="pt", add_special_tokens=False).to(device) + input_ids = inputs["input_ids"] + print(f"[tf] Input tokens: {input_ids.shape}, last-pos id = {int(input_ids[0,-1])}") + + captures = {} + + def hook_for(key): + def fn(_mod, _inp, out): + t = out[0] if isinstance(out, tuple) else out + captures[key] = t.detach().cpu().float().numpy() + return fn + + handles = [] + for i, layer in enumerate(model.model.layers): + handles.append(layer.input_layernorm.register_forward_hook(hook_for(f"attn_norm-{i}"))) + handles.append(layer.self_attn.register_forward_hook(hook_for(f"self_attn-{i}"))) + handles.append(layer.post_attention_layernorm.register_forward_hook(hook_for(f"post_norm-{i}"))) + handles.append(layer.mlp.register_forward_hook(hook_for(f"mlp-{i}"))) + handles.append(model.model.norm.register_forward_hook(hook_for("final_norm"))) + + with torch.no_grad(): + out = model(**inputs, output_hidden_states=True) + + for h in handles: + h.remove() + + # Per-layer hidden states (hidden[i] is the output of layer i; hidden[0] = embeddings) + for i, h in enumerate(out.hidden_states): + captures[f"hidden-{i}"] = h.detach().cpu().float().numpy() + captures["logits"] = out.logits.detach().cpu().float().numpy() + captures["tokens"] = input_ids.cpu().numpy() + + out_path.parent.mkdir(parents=True, exist_ok=True) + np.savez(str(out_path), **captures) + print(f"[tf] Saved {len(captures)} arrays to {out_path}") + + # Free memory: model is no longer needed + del model + if device == "cuda": + torch.cuda.empty_cache() + + +def gguf_dump(gguf_path: Path, prompt: str, out_path: Path): + out_path.parent.mkdir(parents=True, exist_ok=True) + if out_path.exists(): + out_path.unlink() + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = str(LLAMA_BIN_DIR) + ":" + env.get("LD_LIBRARY_PATH", "") + env["LLAMA_DUMP_TENSORS_FILE"] = str(out_path) + env["LLAMA_DUMP_TENSORS_REGEX"] = GGUF_DUMP_REGEX + # Atomic tokenization of <|im_start|> etc., so the token count matches + # what HF transformers produces on a fully-rendered chat-template prompt. + env["LLAMA_TOKENIZE_PARSE_SPECIAL"] = "1" + + cmd = [str(EVAL_CALLBACK_BIN), + "-m", str(gguf_path), + "-p", prompt, + "-n", "1"] + print(f"[gguf] Running {EVAL_CALLBACK_BIN.name} with regex={GGUF_DUMP_REGEX!r}") + res = subprocess.run(cmd, env=env, capture_output=True, text=True) + if res.returncode != 0: + print(res.stdout[-1000:]) + print(res.stderr[-1000:]) + sys.exit(f"[gguf] llama-eval-callback failed (exit {res.returncode})") + if not out_path.exists() or out_path.stat().st_size == 0: + sys.exit(f"[gguf] dump file empty: {out_path}") + print(f"[gguf] Dump size: {out_path.stat().st_size/1024:.1f} KB") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("hf_model_dir") + parser.add_argument("gguf_path") + parser.add_argument("--transformers-output", required=True, + help="Path to existing transformers.json (provides rendered_prompt per case)") + parser.add_argument("--case", required=True, + help="Test case name from test_cases.py to use as the anchor prompt") + parser.add_argument("--work-dir", required=True) + parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu", choices=["cuda", "cpu"]) + parser.add_argument("--dtype", default=None, choices=["fp16", "fp32", "bf16"], + help="Defaults to fp16 on cuda, fp32 on cpu") + args = parser.parse_args() + + hf_dir = Path(args.hf_model_dir).resolve() + gguf_path = Path(args.gguf_path).resolve() + work_dir = Path(args.work_dir).resolve() + work_dir.mkdir(parents=True, exist_ok=True) + + tf_json = json.loads(Path(args.transformers_output).read_text()) + case = next((c for c in tf_json if c["name"] == args.case), None) + if case is None: + sys.exit(f"case {args.case!r} not found in {args.transformers_output}") + prompt = case["rendered_prompt"] + + if args.dtype is None: + args.dtype = "fp16" if args.device == "cuda" else "fp32" + torch_dtype = {"fp16": torch.float16, "fp32": torch.float32, "bf16": torch.bfloat16}[args.dtype] + + tf_out = work_dir / "tf_layers.npz" + gg_out = work_dir / "gguf_layers.bin" + + transformers_dump(hf_dir, prompt, args.device, torch_dtype, tf_out) + gguf_dump(gguf_path, prompt, gg_out) + + # Also save the prompt + tokens so compare_layers can sanity check. + meta = { + "case": args.case, + "prompt": prompt, + "device": args.device, + "dtype": args.dtype, + "hf_model_dir": str(hf_dir), + "gguf_path": str(gguf_path), + } + (work_dir / "meta.json").write_text(json.dumps(meta, indent=2, ensure_ascii=False)) + print(f"Saved meta to {work_dir / 'meta.json'}") + + +if __name__ == "__main__": + main() diff --git a/test_conversion/run_logits.py b/test_conversion/run_logits.py new file mode 100644 index 000000000000..c1dd875f3e2a --- /dev/null +++ b/test_conversion/run_logits.py @@ -0,0 +1,308 @@ +""" +Logit-level comparison: for each test case, compute the next-token +top-K log-probability distribution from + (a) the transformers model (forward pass on the rendered prompt) + (b) Ollama serving the GGUF (logprobs API on the same prompt) +and report top-1 agreement, top-5 overlap, and KL divergence. + +This catches subtle numerical regressions in the GGUF (quantization, +conversion bugs, wrong activation, etc.) that the binary tool-call +behavioural test would not notice. + +Per-case metrics: + + top1_match bool — same most-likely next token (most important) + top1_lp_diff float — |TF top-1 logprob − Ollama top-1 logprob|. + Concrete confidence delta on the chosen token. + fp16-vs-fp16: typically < 0.1. + Q4_K_M: typically < 0.5. + top5_overlap int — how many of TF's top-5 are in Ollama's top-5 (0..5). + mean_lp_diff_top3 float — primary aggregate metric: mean |Δlp| over TF's + top-3 tokens (aligned by token ID). Top-3 covers + the bulk of the probability mass; excluding the + 4-5 tail tokens avoids the high noise that fp16 + softmax has on low-probability logits. + mean_lp_diff_top5 float — same but over top-5; reported for completeness. + Naturally noisier; use top-3 for judgement. + tf_top5_missing int — count of TF's top-5 tokens not in Ollama's + top-K. High counts mean Ollama wasn't even close + on those tokens (significant divergence). + kl_div_renorm float — secondary: KL on renormalized common-top-K. + Can be inflated; ignore unless other signals + also flag. + +Cases whose rendered prompt does NOT end at a generation point (i.e. +add_generation_prompt was False — last message was assistant text/tool_calls, +no `<|im_start|>assistant\\n` suffix) are SKIPPED: there is no canonical +"next token" to predict there. + +Usage: + python run_logits.py + --transformers-output + [--model-name NAME] + [--ollama-url URL] + [--top-k K] + [--device cuda|cpu] + [--dtype fp16|fp32] +""" + +import argparse +import json +import math +import sys +import traceback +from pathlib import Path + +try: + import requests # noqa: F401 (imported via run_ollama as well, but be explicit) +except ImportError: + sys.exit("ERROR: this script needs the 'requests' package (pip install requests).") + +try: + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer +except ImportError as e: + sys.exit(f"ERROR: this script needs torch + transformers ({e}).") + +sys.path.insert(0, str(Path(__file__).parent)) +from run_ollama import check_ollama_alive, ollama_create, ollama_delete, _post # noqa: E402 + + +def transformers_topk(model, tokenizer, prompt, top_k): + """Forward-pass the model and return top-K (token_id, token, logprob).""" + inputs = tokenizer(prompt, return_tensors="pt", add_special_tokens=False) + device = next(model.parameters()).device + inputs = {k: v.to(device) for k, v in inputs.items()} + with torch.no_grad(): + out = model(**inputs) + logits = out.logits[0, -1].float() + logprobs = torch.log_softmax(logits, dim=-1) + vals, idxs = torch.topk(logprobs, top_k) + return [ + { + "token_id": int(i), + "token": tokenizer.convert_ids_to_tokens(int(i)), + "logprob": float(lp), + } + for i, lp in zip(idxs.tolist(), vals.tolist()) + ] + + +def ollama_topk(url, model_name, prompt, top_k): + """Get Ollama's top-K next-token logprobs (raw=true so it doesn't apply the chat template).""" + payload = { + "model": model_name, + "prompt": prompt, + "raw": True, + "stream": False, + "options": {"num_predict": 1, "temperature": 0, "seed": 0}, + "logprobs": True, + "top_logprobs": top_k, + } + resp = _post(f"{url}/api/generate", payload) + lps = resp.get("logprobs") or [] + if not lps: + return None + first = lps[0] + return [ + {"token": t["token"], "logprob": t["logprob"], "bytes": t.get("bytes")} + for t in first.get("top_logprobs", []) + ] + + +_SPM_SPACE = "▁" # ▁ — SentencePiece's word-boundary marker + + +def ollama_token_to_id(tokenizer, ol_entry, vocab): + """Map an Ollama-reported token to the transformers vocab ID. + + Critical: we look the token up DIRECTLY in the vocab dict, not via + tokenizer.encode(). The encoder normalizes (e.g. always converts a + leading literal-space `' Bonjour'` to `▁Bonjour`), which would COLLIDE + distinct GGUF vocab entries (`' Bonjour'`/▁Bonjour id=34362 vs + `'Bonjour'` id=21327) and cause ol_by_id[id] to be set to the WRONG + logprob (whichever distinct token appears later in the top-K list). + """ + s = ol_entry["token"] + + # 1. Try the SentencePiece form: leading literal-space → ▁ prefix. + spm_form = (_SPM_SPACE + s[1:]) if s.startswith(" ") else s + if spm_form in vocab: + return vocab[spm_form] + + # 2. Try the raw string (for non-space-prefixed tokens like 'Bonjour'). + if s in vocab: + return vocab[s] + + # 3. Last resort: lossy re-tokenize. May collide; logged via caller. + ids = tokenizer.encode(s, add_special_tokens=False) + if len(ids) == 1: + return ids[0] + return None + + +def compare_topk(tf_top, ol_top, tokenizer): + """Compare next-token top-K distributions and return per-case metrics.""" + if not tf_top or not ol_top: + return None + + # Annotate Ollama entries with transformers vocab IDs (direct vocab lookup). + vocab = tokenizer.get_vocab() + ol_with_ids = [ + {**t, "token_id": ollama_token_to_id(tokenizer, t, vocab)} for t in ol_top + ] + + tf1 = tf_top[0] + ol1 = ol_with_ids[0] + top1_match = tf1["token_id"] == ol1["token_id"] + + tf_top5_ids = {t["token_id"] for t in tf_top[:5]} + ol_top5_ids = {t["token_id"] for t in ol_with_ids[:5] if t["token_id"] is not None} + top5_overlap = len(tf_top5_ids & ol_top5_ids) + + # PRIMARY: absolute logprob differences on TF's top-N tokens (aligned + # by token ID via Ollama's top-K). Reported on top-1 (concrete) and + # top-3 (aggregate). Top-5 also computed for completeness but is + # naturally noisy because fp16 softmax precision is lowest in the tail. + ol_by_id = {t["token_id"]: t["logprob"] for t in ol_with_ids if t["token_id"] is not None} + + def diffs_over(n): + d = [] + miss = 0 + for t in tf_top[:n]: + ol_lp = ol_by_id.get(t["token_id"]) + if ol_lp is None: + miss += 1 + else: + d.append(abs(t["logprob"] - ol_lp)) + return d, miss + + d3, _ = diffs_over(3) + d5, missing5 = diffs_over(5) + mean3 = (sum(d3) / len(d3)) if d3 else None + mean5 = (sum(d5) / len(d5)) if d5 else None + + # Top-1 logprob diff specifically (most interpretable; same token assumed). + top1_lp_diff = None + if top1_match: + ol_top1_lp = ol_by_id.get(tf1["token_id"]) + if ol_top1_lp is not None: + top1_lp_diff = abs(tf1["logprob"] - ol_top1_lp) + + # SECONDARY METRIC: KL on renormalized common-top-K (kept for reference; + # can be inflated when overlap is small). + tf_by_id = {t["token_id"]: t["logprob"] for t in tf_top} + common = set(tf_by_id) & set(ol_by_id) + kl = None + if common: + tf_p = {i: math.exp(tf_by_id[i]) for i in common} + ol_p = {i: math.exp(ol_by_id[i]) for i in common} + s_tf = sum(tf_p.values()) + s_ol = sum(ol_p.values()) + if s_tf > 0 and s_ol > 0: + kl = 0.0 + for i in common: + p = tf_p[i] / s_tf + q = ol_p[i] / s_ol + if p > 1e-12 and q > 1e-12: + kl += p * math.log(p / q) + + return { + "top1_match": top1_match, + "tf_top1": {"id": tf1["token_id"], "tok": tf1["token"], "lp": round(tf1["logprob"], 4)}, + "ol_top1": {"id": ol1["token_id"], "tok": ol1["token"], "lp": round(ol1["logprob"], 4)}, + "top1_lp_diff": top1_lp_diff, + "top5_overlap": top5_overlap, + "tf_top5_missing_in_ollama_topk": missing5, + "mean_lp_diff_top3": mean3, + "mean_lp_diff_top5": mean5, + "kl_div_renorm": kl, + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("hf_model_dir", help="Path to HuggingFace transformers model directory") + parser.add_argument("modelfile_path", help="Path to the GGUF Modelfile (for ollama create)") + parser.add_argument("output_json") + parser.add_argument("--transformers-output", required=True, + help="Path to transformers.json (provides the rendered prompts)") + parser.add_argument("--model-name", default="test-chat-template-tmp") + parser.add_argument("--ollama-url", default="http://localhost:11434") + parser.add_argument("--top-k", type=int, default=20) + parser.add_argument("--device", + default="cuda" if torch.cuda.is_available() else "cpu", + choices=["cuda", "cpu"]) + parser.add_argument("--dtype", default=None, choices=["fp16", "fp32", "bf16"], + help="Defaults to fp16 on cuda, fp32 on cpu") + args = parser.parse_args() + + version = check_ollama_alive(args.ollama_url) + print(f"[logits] Ollama reachable (version {version})") + + tf_path = Path(args.transformers_output) + if not tf_path.exists(): + sys.exit(f"ERROR: transformers output not found at {tf_path}") + transformers_data = json.loads(tf_path.read_text()) + + if args.dtype is None: + args.dtype = "fp16" if args.device == "cuda" else "fp32" + torch_dtype = {"fp16": torch.float16, "fp32": torch.float32, "bf16": torch.bfloat16}[args.dtype] + + print(f"[logits] Loading transformers model from {args.hf_model_dir} ({args.device}, {args.dtype})") + tokenizer = AutoTokenizer.from_pretrained(args.hf_model_dir, trust_remote_code=True) + model = AutoModelForCausalLM.from_pretrained( + args.hf_model_dir, torch_dtype=torch_dtype, trust_remote_code=True, + ).to(args.device) + model.eval() + + ollama_create(args.model_name, args.modelfile_path) + try: + results = [] + for entry in transformers_data: + name = entry["name"] + prompt = entry.get("rendered_prompt") + if not prompt: + continue + if entry.get("add_generation_prompt") is False: + # No canonical next-token prediction: the conversation ends + # on the assistant's own message (closed by <|im_end|>). + # Skip — there's nothing meaningful to compare. + print(f"[logits] {name} SKIP (no add_generation_prompt)") + results.append({"name": name, "skipped": "no add_generation_prompt"}) + continue + print(f"[logits] {name}") + try: + tf_top = transformers_topk(model, tokenizer, prompt, args.top_k) + ol_top = ollama_topk(args.ollama_url, args.model_name, prompt, args.top_k) + cmp = compare_topk(tf_top, ol_top, tokenizer) + results.append({ + "name": name, + "comparison": cmp, + "tf_top5": tf_top[:5], + "ol_top5": (ol_top or [])[:5], + }) + if cmp: + fmt = lambda v: (f"{v:.4f}" if v is not None else "n/a") + print(f" top1_match={cmp['top1_match']} " + f"|Δlp_top1|={fmt(cmp['top1_lp_diff'])} " + f"mean|Δlp|_top3={fmt(cmp['mean_lp_diff_top3'])} " + f"top5_overlap={cmp['top5_overlap']}/5 " + f"missing={cmp['tf_top5_missing_in_ollama_topk']}/5") + except Exception as e: + traceback.print_exc() + results.append({"name": name, "error": f"{type(e).__name__}: {e}"}) + finally: + ollama_delete(args.model_name) + del model + if args.device == "cuda": + torch.cuda.empty_cache() + + out = Path(args.output_json) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(results, indent=2, ensure_ascii=False)) + print(f"[logits] Wrote {len(results)} results to {out}") + + +if __name__ == "__main__": + main() diff --git a/test_conversion/run_ollama.py b/test_conversion/run_ollama.py new file mode 100644 index 000000000000..d7e836c01ea7 --- /dev/null +++ b/test_conversion/run_ollama.py @@ -0,0 +1,228 @@ +""" +For each test case, query Ollama and collect the input-token count. + +Two probes per case: + chat_prompt_eval_count : tokens fed to the model when the conversation + is passed through Ollama's /api/chat (i.e. + Ollama applies its Modelfile TEMPLATE, then + tokenizes with the GGUF tokenizer). + raw_prompt_eval_count : tokens fed when the *transformers-rendered* + prompt is passed through /api/generate with + raw=true (i.e. only the GGUF tokenizer runs; + no chat template applied). This isolates the + tokenizer from the template. + +The Ollama model is created from the Modelfile at startup and deleted at exit. + +Usage: + python run_ollama.py + [--model-name NAME] + [--transformers-output JSON] + [--ollama-url URL] +""" + +import argparse +import json +import subprocess +import sys +import traceback +from pathlib import Path + +try: + import requests +except ImportError: + sys.exit("ERROR: this script needs the 'requests' package (pip install requests).") + +sys.path.insert(0, str(Path(__file__).parent)) +from test_cases import TEST_CASES # noqa: E402 + + +def check_ollama_alive(url): + try: + r = requests.get(f"{url}/api/version", timeout=3) + r.raise_for_status() + return r.json().get("version", "?") + except Exception as e: + sys.exit( + f"ERROR: cannot reach Ollama at {url} ({e}).\n" + " Start it first with: ollama serve" + ) + + +def validate_from_target(modelfile_path): + """Check that the Modelfile's FROM target exists (relative to the Modelfile).""" + modelfile_path = Path(modelfile_path).resolve() + for line in modelfile_path.read_text().splitlines(): + line = line.strip() + if line.startswith("FROM "): + target = line[len("FROM "):].strip().strip('"') + # Ollama requires a relative path for local FROM files; absolute + # paths trigger a misleading "no Modelfile or safetensors files + # found" error. + if target.startswith("/"): + sys.exit( + f"ERROR: Modelfile {modelfile_path} has an absolute FROM path " + f"({target}). Ollama needs it relative to the Modelfile." + ) + resolved = (modelfile_path.parent / target).resolve() + if not resolved.exists(): + sys.exit( + f"ERROR: Modelfile {modelfile_path} references\n" + f" FROM {target}\n" + f"but {resolved} does not exist.\n" + f"Either rename the GGUF or edit the FROM line in the Modelfile.\n" + f"(NB: ollama reports this as 'invalid model name' — misleading.)" + ) + return + sys.exit(f"ERROR: no FROM line found in {modelfile_path}") + + +def ollama_create(name, modelfile_path): + """Create (or overwrite) an ollama model from the given Modelfile.""" + modelfile_path = Path(modelfile_path).resolve() + validate_from_target(modelfile_path) + print(f"[ollama] Creating model '{name}' from {modelfile_path}") + # Run from the modelfile's directory so its FROM ./X.gguf resolves. + result = subprocess.run( + ["ollama", "create", name, "-f", modelfile_path.name], + cwd=str(modelfile_path.parent), + capture_output=True, + text=True, + ) + if result.returncode != 0: + sys.exit( + "ERROR: 'ollama create' failed:\n" + f" STDOUT: {result.stdout}\n" + f" STDERR: {result.stderr}" + ) + + +def ollama_delete(name): + print(f"[ollama] Removing temporary model '{name}'") + subprocess.run(["ollama", "rm", name], capture_output=True, text=True) + + +def normalize_for_ollama(messages): + """Convert OpenAI-style messages to the variant Ollama's /api/chat accepts. + + Differences observed empirically: + - tool_calls[].function.arguments must be an OBJECT, not a JSON string. + - The OpenAI-style {"type": "function", "function": {...}} wrapper around + each tool_call is tolerated, but we strip it to be safe. + """ + out = [] + for msg in messages: + m = dict(msg) + tcs = m.get("tool_calls") + if tcs: + new_tcs = [] + for tc in tcs: + fn = tc.get("function", tc) + args = fn.get("arguments") + if isinstance(args, str): + try: + args = json.loads(args) + except json.JSONDecodeError: + pass # leave it; ollama will complain again + new_tcs.append({"function": {"name": fn["name"], "arguments": args}}) + m["tool_calls"] = new_tcs + out.append(m) + return out + + +def _post(url, payload): + r = requests.post(url, json=payload, timeout=300) + if r.status_code >= 400: + # Surface Ollama's actual complaint instead of a bare HTTPError. + raise RuntimeError(f"HTTP {r.status_code} from {url}: {r.text}") + return r.json() + + +def ollama_chat(url, model, messages, tools): + payload = { + "model": model, + "messages": normalize_for_ollama(messages), + "stream": False, + "options": {"num_predict": 1, "temperature": 0, "seed": 0}, + } + if tools is not None: + payload["tools"] = tools + return _post(f"{url}/api/chat", payload) + + +def ollama_generate_raw(url, model, prompt): + payload = { + "model": model, + "prompt": prompt, + "raw": True, + "stream": False, + "options": {"num_predict": 1, "temperature": 0, "seed": 0}, + } + return _post(f"{url}/api/generate", payload) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("modelfile_path", help="Path to the Modelfile") + parser.add_argument("output_json", help="Path to write the JSON results") + parser.add_argument("--model-name", default="test-chat-template-tmp", + help="Temporary ollama model name (will be created and removed). " + "Must match ollama's naming rules: lowercase letters, digits, " + "hyphens and periods only (no underscores).") + parser.add_argument("--transformers-output", default=None, + help="Path to the transformers.json (enables the raw tokenizer probe)") + parser.add_argument("--ollama-url", default="http://localhost:11434") + args = parser.parse_args() + + version = check_ollama_alive(args.ollama_url) + print(f"[ollama] Server reachable (version {version})") + + transformers_by_name = {} + if args.transformers_output and Path(args.transformers_output).exists(): + ref = json.loads(Path(args.transformers_output).read_text()) + transformers_by_name = {r["name"]: r for r in ref} + print(f"[ollama] Loaded {len(transformers_by_name)} transformers references " + f"(will probe tokenizer with raw=true)") + else: + print("[ollama] No transformers reference available; skipping raw-tokenizer probe") + + ollama_create(args.model_name, args.modelfile_path) + + results = [] + try: + for case in TEST_CASES: + print(f"[ollama] {case['name']}") + entry = {"name": case["name"]} + try: + chat_resp = ollama_chat( + args.ollama_url, args.model_name, + case["messages"], case.get("tools"), + ) + entry["chat_prompt_eval_count"] = chat_resp.get("prompt_eval_count") + except Exception as e: + traceback.print_exc() + entry["chat_error"] = f"{type(e).__name__}: {e}" + + ref = transformers_by_name.get(case["name"]) + if ref and "rendered_prompt" in ref: + try: + raw_resp = ollama_generate_raw( + args.ollama_url, args.model_name, ref["rendered_prompt"], + ) + entry["raw_prompt_eval_count"] = raw_resp.get("prompt_eval_count") + except Exception as e: + traceback.print_exc() + entry["raw_error"] = f"{type(e).__name__}: {e}" + + results.append(entry) + finally: + ollama_delete(args.model_name) + + out = Path(args.output_json) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(results, indent=2, ensure_ascii=False)) + print(f"[ollama] Wrote {len(results)} results to {out}") + + +if __name__ == "__main__": + main() diff --git a/test_conversion/run_transformers.py b/test_conversion/run_transformers.py new file mode 100644 index 000000000000..f98c29710884 --- /dev/null +++ b/test_conversion/run_transformers.py @@ -0,0 +1,81 @@ +""" +Render each test case with the transformers chat template + tokenize. + +Outputs a JSON file with, for each test case: + name, messages, tools, add_generation_prompt, + rendered_prompt, token_count, token_ids + +Usage: + python run_transformers.py +""" + +import argparse +import json +import sys +import traceback +from pathlib import Path + +from transformers import AutoTokenizer + +sys.path.insert(0, str(Path(__file__).parent)) +from test_cases import TEST_CASES # noqa: E402 + + +def render_case(tokenizer, case): + messages = case["messages"] + tools = case.get("tools") + + # If the conversation ends on an assistant turn, we are NOT prompting for + # another generation; otherwise we are (mirrors Ollama's behaviour). + add_generation_prompt = messages[-1]["role"] != "assistant" + + kwargs = {"add_generation_prompt": add_generation_prompt} + if tools is not None: + kwargs["tools"] = tools + + rendered = tokenizer.apply_chat_template(messages, tokenize=False, **kwargs) + token_ids = tokenizer.apply_chat_template(messages, tokenize=True, **kwargs) + + # apply_chat_template may return a tensor; normalize to list[int] + if hasattr(token_ids, "tolist"): + token_ids = token_ids.tolist() + if token_ids and isinstance(token_ids[0], list): + token_ids = token_ids[0] + + return { + "name": case["name"], + "messages": messages, + "tools": tools, + "add_generation_prompt": add_generation_prompt, + "rendered_prompt": rendered, + "token_count": len(token_ids), + "token_ids": token_ids, + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("hf_model_dir", help="Path to HuggingFace transformers model directory") + parser.add_argument("output_json", help="Path to write the JSON results") + args = parser.parse_args() + + print(f"[transformers] Loading tokenizer from {args.hf_model_dir}") + tokenizer = AutoTokenizer.from_pretrained(args.hf_model_dir, trust_remote_code=True) + + results = [] + for case in TEST_CASES: + print(f"[transformers] {case['name']}") + try: + results.append(render_case(tokenizer, case)) + except Exception as e: + traceback.print_exc() + results.append({"name": case["name"], "error": f"{type(e).__name__}: {e}"}) + + out = Path(args.output_json) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(results, indent=2, ensure_ascii=False)) + print(f"[transformers] Wrote {len(results)} results to {out}") + + +if __name__ == "__main__": + main() diff --git a/test_conversion/test.sh b/test_conversion/test.sh new file mode 100644 index 000000000000..0be9ffc3a306 --- /dev/null +++ b/test_conversion/test.sh @@ -0,0 +1,5 @@ +cd `dirname $0` + + +WHAT="/mnt/d/home/jlouradour/luciole/releases/SFT/Luciole-1B-Instruct-1.0" +python test_main.py $WHAT $WHAT-gguf diff --git a/test_conversion/test_cases.py b/test_conversion/test_cases.py new file mode 100644 index 000000000000..d75aed9779aa --- /dev/null +++ b/test_conversion/test_cases.py @@ -0,0 +1,283 @@ +""" +Test conversations used to compare the transformers chat template (jinja) +against the Ollama Modelfile template. + +Each test case is a dict with: + name : unique short identifier (used in filenames and reports) + messages : OpenAI-style list of message dicts + tools : list of OpenAI-style tool definitions, or None +""" + +WEATHER_TOOL = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City name, e.g. 'Paris'.", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "Temperature unit.", + }, + }, + "required": ["location"], + }, + }, +} + +CALCULATOR_TOOL = { + "type": "function", + "function": { + "name": "calculator", + "description": "Evaluate a math expression.", + "parameters": { + "type": "object", + "properties": { + "expression": { + "type": "string", + "description": "A math expression, e.g. '2 + 2 * 3'.", + }, + }, + "required": ["expression"], + }, + }, +} + + +TEST_CASES = [ + # --- No tools --- + { + "name": "01_user_only", + "messages": [ + {"role": "user", "content": "Bonjour, comment vas-tu ?"}, + ], + "tools": None, + }, + { + "name": "02_system_user", + "messages": [ + {"role": "system", "content": "Tu es un assistant qui répond en français."}, + {"role": "user", "content": "Quelle est la capitale de la France ?"}, + ], + "tools": None, + }, + { + "name": "03_multi_turn", + "messages": [ + {"role": "user", "content": "Hi!"}, + {"role": "assistant", "content": "Hello! How can I help you today?"}, + {"role": "user", "content": "What's 2 + 2?"}, + {"role": "assistant", "content": "2 + 2 equals 4."}, + {"role": "user", "content": "Thanks!"}, + ], + "tools": None, + }, + # --- Tools, no tool call yet --- + { + "name": "04_tools_available_no_call", + "messages": [ + {"role": "user", "content": "What's the weather in Paris?"}, + ], + "tools": [WEATHER_TOOL], + # Behavioural expectation: the model should emit a tool_call rather than text. + "expected_behavior": { + "tool_call": { + "name": "get_weather", + "required_args": ["location"], + "args_must_contain": {"location": "paris"}, # case-insensitive substring + }, + }, + }, + { + "name": "05_tools_with_system", + "messages": [ + {"role": "system", "content": "You are a weather assistant."}, + {"role": "user", "content": "Weather in Paris please."}, + ], + "tools": [WEATHER_TOOL, CALCULATOR_TOOL], + "expected_behavior": { + "tool_call": { + "name": "get_weather", + "required_args": ["location"], + "args_must_contain": {"location": "paris"}, + }, + }, + }, + # --- Tool call + tool response --- + { + "name": "06_single_tool_call_and_response", + "messages": [ + {"role": "user", "content": "What's the weather in Paris?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "Paris"}', + }, + }, + ], + }, + {"role": "tool", "content": '{"temperature": 18, "unit": "celsius"}'}, + {"role": "assistant", "content": "It's 18°C in Paris."}, + ], + "tools": [WEATHER_TOOL], + }, + # --- Multiple tool calls in one assistant turn --- + { + "name": "07_multiple_tool_calls", + "messages": [ + {"role": "user", "content": "Weather in Paris and London?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "Paris"}', + }, + }, + { + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "London"}', + }, + }, + ], + }, + ], + "tools": [WEATHER_TOOL], + }, + # --- Consecutive tool responses (must batch into one user turn in jinja) --- + { + "name": "08_consecutive_tool_responses", + "messages": [ + {"role": "user", "content": "Weather in Paris and London?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "Paris"}', + }, + }, + { + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "London"}', + }, + }, + ], + }, + {"role": "tool", "content": '{"location": "Paris", "temperature": 18}'}, + {"role": "tool", "content": '{"location": "London", "temperature": 15}'}, + ], + "tools": [WEATHER_TOOL], + }, + # --- Assistant with BOTH content AND tool_calls --- + { + "name": "09_assistant_content_and_tool_call", + "messages": [ + {"role": "user", "content": "What's the weather in Paris?"}, + { + "role": "assistant", + "content": "Let me check that for you.", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "Paris"}', + }, + }, + ], + }, + {"role": "tool", "content": '{"temperature": 18, "unit": "celsius"}'}, + ], + "tools": [WEATHER_TOOL], + }, + # --- Long-ish multi-turn with tools sprinkled in --- + { + "name": "10_full_tool_dialogue", + "messages": [ + {"role": "system", "content": "You are a helpful assistant with tools."}, + {"role": "user", "content": "Compute 12*34 then tell me the weather in Lyon."}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "calculator", + "arguments": '{"expression": "12*34"}', + }, + }, + ], + }, + {"role": "tool", "content": '{"result": 408}'}, + { + "role": "assistant", + "content": "12*34 = 408. Now checking the weather.", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "Lyon"}', + }, + }, + ], + }, + {"role": "tool", "content": '{"temperature": 21, "unit": "celsius"}'}, + {"role": "assistant", "content": "12*34 = 408 and it's 21°C in Lyon."}, + ], + "tools": [WEATHER_TOOL, CALCULATOR_TOOL], + }, + # --- Multi-turn (user/assistant/user/assistant/user), tokenization-focused --- + # Mixes French accents, an apostrophe word ("l'Allemagne"), a code fence + # and a Markdown inline code span — designed to surface BPE / special-token + # boundary divergences between the HF tokenizer and llama.cpp's SPM. + { + "name": "11_multi_turn_tokenization", + "messages": [ + {"role": "user", "content": "Quelle heure est-il à Paris ?"}, + {"role": "assistant", "content": "Il est 14h30 (heure de Paris)."}, + {"role": "user", "content": "Montre-moi `int x = 42;` dans un bloc Markdown."}, + {"role": "assistant", "content": "```cpp\nint x = 42;\n```"}, + {"role": "user", "content": "Merci !"}, + ], + "tools": None, + }, + # --- Multi-turn (user/assistant/user/assistant/user), logit-focused --- + # Short factual conversation ending on a question whose answer is a single + # well-known proper noun ("Madrid"), so the next-token top-K distribution + # should be sharply peaked and easy to compare across runtimes. + { + "name": "12_multi_turn_logit", + "messages": [ + {"role": "user", "content": "Quelle est la capitale de la France ?"}, + {"role": "assistant", "content": "La capitale de la France est Paris."}, + {"role": "user", "content": "Et celle de l'Allemagne ?"}, + {"role": "assistant", "content": "Berlin."}, + {"role": "user", "content": "Et celle de l'Espagne ?"}, + ], + "tools": None, + }, +] diff --git a/test_conversion/test_main.py b/test_conversion/test_main.py new file mode 100644 index 000000000000..75505c641b19 --- /dev/null +++ b/test_conversion/test_main.py @@ -0,0 +1,175 @@ +""" +Main orchestrator: compare the chat template + tokenizer of a transformers +model against an Ollama (GGUF + Modelfile) deployment. + +Pipeline: + 1. run_transformers.py -> /transformers.json + 2. run_ollama.py -> /ollama.json + 3. run_behavior.py -> /behavior.json (only cases with expected_behavior) + 4. run_logits.py -> /logits.json (per-case next-token logit comparison) + 5. compare.py -> prints per-test report, exit 1 on failure + +Each step is skipped if its output JSON already exists; delete the file (or +the whole ) to force recomputation. Or pass --force. Slow optional +steps can be turned off with --no-behavior and --no-logits. + +Requirements: + - transformers + torch (Python; transformers always; torch only for --logits) + - requests (Python) + - ollama (must be running: ollama serve) + +Usage: + python test_main.py + [--work-dir DIR] + [--ollama-model-name NAME] + [--ollama-url URL] + [--num-predict N] + [--logits-top-k K] + [--logits-device cuda|cpu] + [--no-behavior] + [--no-logits] + [--force] + +Where: + is a HuggingFace transformers model directory + (must contain tokenizer files + chat_template). + is a directory containing both: + - a 'Modelfile' file + - the .gguf file referenced by the Modelfile (FROM ./...) +""" + +import argparse +import subprocess +import sys +from pathlib import Path + + +def run_step(label, cmd, output_file, force): + if not force and output_file.exists(): + print(f"=== {label}: SKIP (using cached {output_file}) ===\n") + return + print(f"=== {label} ===") + print("$ " + " ".join(cmd)) + rc = subprocess.run(cmd).returncode + if rc != 0: + sys.exit(f"!!! {label} failed (exit {rc})") + print() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("hf_model_dir", help="Path to the HuggingFace transformers model directory") + parser.add_argument("gguf_dir", help="Directory containing the Modelfile and the .gguf file") + parser.add_argument("--work-dir", default=None, + help="Where to store intermediate JSON files " + "(default: ./results/__vs__/)") + parser.add_argument("--ollama-model-name", default="test-chat-template-tmp", + help="Temporary ollama model name (created and removed by the script). " + "No underscores: ollama rejects them.") + parser.add_argument("--ollama-url", default="http://localhost:11434") + parser.add_argument("--num-predict", type=int, default=256, + help="Max tokens generated per behavioural case (default 256)") + parser.add_argument("--logits-top-k", type=int, default=20, + help="K for next-token top-K logit comparison (default 20)") + parser.add_argument("--logits-device", default=None, choices=["cuda", "cpu"], + help="Device for the transformers forward pass (default: cuda if available, else cpu)") + parser.add_argument("--no-behavior", action="store_true", + help="Skip the behavioural step (slower; requires the model to actually generate)") + parser.add_argument("--no-logits", action="store_true", + help="Skip the logit-comparison step (loads the full transformers model; slow)") + parser.add_argument("--force", action="store_true", + help="Recompute all intermediate outputs, ignoring caches") + args = parser.parse_args() + + here = Path(__file__).resolve().parent + hf_dir = Path(args.hf_model_dir).resolve() + gguf_dir = Path(args.gguf_dir).resolve() + modelfile = gguf_dir / "Modelfile" + + if not hf_dir.is_dir(): + sys.exit(f"ERROR: HF model dir not found: {hf_dir}") + if not modelfile.is_file(): + sys.exit(f"ERROR: Modelfile not found at {modelfile}") + + work_dir = (Path(args.work_dir).resolve() + if args.work_dir + else here / "results" / f"{hf_dir.name}__vs__{gguf_dir.name}") + work_dir.mkdir(parents=True, exist_ok=True) + transformers_json = work_dir / "transformers.json" + ollama_json = work_dir / "ollama.json" + behavior_json = work_dir / "behavior.json" + logits_json = work_dir / "logits.json" + + print(f"HF model dir : {hf_dir}") + print(f"GGUF dir : {gguf_dir}") + print(f"Modelfile : {modelfile}") + print(f"Work dir : {work_dir}") + print(f"Ollama URL : {args.ollama_url}") + print() + + # Step 1: transformers + run_step( + "Step 1/5 — transformers (render + tokenize)", + [sys.executable, str(here / "run_transformers.py"), str(hf_dir), str(transformers_json)], + transformers_json, + args.force, + ) + + # Step 2: ollama (depends on Step 1's JSON for the raw-tokenizer probe) + run_step( + "Step 2/5 — ollama (chat + raw tokenizer probes)", + [sys.executable, str(here / "run_ollama.py"), str(modelfile), str(ollama_json), + "--model-name", args.ollama_model_name, + "--transformers-output", str(transformers_json), + "--ollama-url", args.ollama_url], + ollama_json, + args.force, + ) + + # Step 3: behavioural check (optional). The model actually generates here. + if args.no_behavior: + print("=== Step 3/5 — behavioural check: SKIPPED (--no-behavior) ===\n") + else: + run_step( + "Step 3/5 — behavioural check (model generates tool_calls)", + [sys.executable, str(here / "run_behavior.py"), str(modelfile), str(behavior_json), + "--transformers-output", str(transformers_json), + "--model-name", args.ollama_model_name, + "--ollama-url", args.ollama_url, + "--num-predict", str(args.num_predict)], + behavior_json, + args.force, + ) + + # Step 4: logit comparison (optional, slow — loads the full transformers model). + if args.no_logits: + print("=== Step 4/5 — logit comparison: SKIPPED (--no-logits) ===\n") + else: + logits_cmd = [sys.executable, str(here / "run_logits.py"), + str(hf_dir), str(modelfile), str(logits_json), + "--transformers-output", str(transformers_json), + "--model-name", args.ollama_model_name, + "--ollama-url", args.ollama_url, + "--top-k", str(args.logits_top_k)] + if args.logits_device: + logits_cmd += ["--device", args.logits_device] + run_step( + "Step 4/5 — logit comparison (transformers vs Ollama, next-token top-K)", + logits_cmd, + logits_json, + args.force, + ) + + # Step 5: compare (always runs) + print("=== Step 5/5 — compare ===") + compare_cmd = [sys.executable, str(here / "compare.py"), str(transformers_json), str(ollama_json)] + if not args.no_behavior and behavior_json.exists(): + compare_cmd += ["--behavior", str(behavior_json)] + if not args.no_logits and logits_json.exists(): + compare_cmd += ["--logits", str(logits_json)] + rc = subprocess.run(compare_cmd).returncode + sys.exit(rc) + + +if __name__ == "__main__": + main() diff --git a/test_conversion_fp8/test.sh b/test_conversion_fp8/test.sh new file mode 100755 index 000000000000..5554626ace42 --- /dev/null +++ b/test_conversion_fp8/test.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -e +cd "$(dirname "$0")" + +usage() { + cat < [--vllm] + [--max-new-tokens N] [--num-prompts N] + [--min-agreement F] [--seed N] + +Compare inference between an HF model and its FP8-quantized sibling +(produced by convert_hf_to_fp8.py). See test_fp8.py for details. +EOF + exit 1 +} + +[ $# -lt 2 ] && usage + +python3 test_fp8.py "$@" diff --git a/test_conversion_fp8/test_fp8.py b/test_conversion_fp8/test_fp8.py new file mode 100644 index 000000000000..e406f374cfc3 --- /dev/null +++ b/test_conversion_fp8/test_fp8.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python3 +""" +Check that an FP8 conversion (produced by convert_hf_to_fp8.py) is functionally +equivalent to its source model. + +Usage: + python test_fp8.py [--vllm] + [--max-new-tokens N] [--num-prompts N] + [--min-agreement 0.85] [--seed 42] + +Method: + - Load both models via transformers.AutoModelForCausalLM (transformers + auto-dequantizes the compressed-tensors FP8 checkpoint on load). + - For a fixed set of chat-templated prompts, generate greedy continuations. + - Compare first-N tokens between original and FP8 output. FP8_DYNAMIC on a + well-behaved model is expected to match the source for the first ~30 + greedy tokens on most prompts; small early divergences are treated as + warnings, and the overall pass criterion is per-prompt token-agreement + >= --min-agreement (default 0.85). + - With --vllm, additionally run vLLM on the FP8 model and check it matches + the transformers FP8 outputs. + +Exit codes: + 0 all checks passed + 1 agreement below threshold on at least one prompt + 2 missing dependency / setup error +""" + +from __future__ import annotations + +import argparse +import logging +import random +import sys +from dataclasses import dataclass +from pathlib import Path + +logger = logging.getLogger("test-fp8") + + +DEFAULT_PROMPTS = [ + "Qu'est-ce qu'une éclipse solaire ? Réponds brièvement.", + "List three uses of the number pi in engineering.", + "Écris une phrase courte pour expliquer la photosynthèse.", + "In one sentence: why is the sky blue?", + "Cite un philosophe des Lumières et son idée principale.", +] + + +@dataclass +class Generation: + prompt: str + text: str + token_ids: list[int] + + +def _make_mamba_cpu_safe() -> None: + """Route mamba_ssm's gated-RMSNorm through its own pure-torch reference on CPU. + + Nemotron-H's `torch_forward` (the CPU-friendly branch) still routes its + gated RMSNorm through mamba_ssm's Triton kernel, which raises + 'Pointer argument (at 0) cannot be accessed from Triton (cpu tensor?)' on + CPU tensors. Dispatch: CUDA inputs → original Triton kernel; CPU inputs → + `rms_norm_ref`, the reference implementation shipped in the same module. + + Called at the top of `load_hf_model`, before any modeling module imports + `rmsnorm_fn`, so the local binding picks up the wrapper. Idempotent.""" + import inspect + try: + from mamba_ssm.ops.triton import layernorm_gated + except ImportError: + return + if getattr(layernorm_gated, "_cpu_safe_patched", False): + return + + _orig_rmsnorm_fn = layernorm_gated.rmsnorm_fn + _rms_norm_ref = layernorm_gated.rms_norm_ref + # Older mamba_ssm rmsnorm_fn is 7 params (no is_rms_norm — hard-coded True + # since this is the RMS variant); newer takes 8. Detect once so we only + # forward args the installed version accepts. + _orig_supports_is_rms_norm = ( + "is_rms_norm" in inspect.signature(_orig_rmsnorm_fn).parameters + ) + + def rmsnorm_fn_dispatch(x, weight, bias=None, z=None, eps=1e-6, + group_size=None, norm_before_gate=True, + is_rms_norm=True): + if x.is_cuda: + if _orig_supports_is_rms_norm: + return _orig_rmsnorm_fn(x, weight, bias, z, eps, group_size, + norm_before_gate, is_rms_norm) + return _orig_rmsnorm_fn(x, weight, bias, z, eps, group_size, + norm_before_gate) + return _rms_norm_ref(x, weight, bias, z, eps, group_size, + norm_before_gate, is_rms_norm) + + layernorm_gated.rmsnorm_fn = rmsnorm_fn_dispatch + layernorm_gated._cpu_safe_patched = True + + +def _make_cuda_stream_cpu_safe() -> None: + """Make `torch.cuda.default_stream` accept CPU devices. + + Nemotron-H's custom modeling wraps its Mamba path in + `with torch.cuda.stream(torch.cuda.default_stream(hidden_states.device)):` + unconditionally. On CPU, `default_stream("cpu")` raises + `ValueError: Expected a cuda device`. `torch.cuda.stream(None)` is already + a no-op context, so we only need to make `default_stream` return None on + non-CUDA devices. Idempotent.""" + import torch + if getattr(torch.cuda, "_cpu_safe_patched", False): + return + _orig_default_stream = torch.cuda.default_stream + + def default_stream(device=None): + if device is not None: + try: + if torch.device(device).type != "cuda": + return None + except (TypeError, ValueError): + pass + return _orig_default_stream(device) + + torch.cuda.default_stream = default_stream + torch.cuda._cpu_safe_patched = True + + +def load_hf_model(model_dir: Path, device: str = "auto"): + from transformers import AutoModelForCausalLM, AutoTokenizer + _make_cuda_stream_cpu_safe() + _make_mamba_cpu_safe() + logger.info("loading %s (device=%s) ...", model_dir, device) + tok = AutoTokenizer.from_pretrained(str(model_dir), trust_remote_code=True) + kw = dict(dtype="auto", trust_remote_code=True) + kw["device_map"] = {"": "cpu"} if device == "cpu" else "auto" + model = AutoModelForCausalLM.from_pretrained(str(model_dir), **kw) + model.eval() + return model, tok + + +def apply_template(tok, user_msg: str) -> str: + if getattr(tok, "chat_template", None): + return tok.apply_chat_template( + [{"role": "user", "content": user_msg}], + tokenize=False, + add_generation_prompt=True, + ) + logger.warning("no chat_template found on tokenizer; using raw prompt") + return user_msg + + +def generate_hf(model, tok, prompt_text: str, max_new_tokens: int) -> Generation: + import torch + inputs = tok(prompt_text, return_tensors="pt").to(model.device) + input_len = inputs["input_ids"].shape[1] + with torch.no_grad(): + out = model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=False, + temperature=None, + top_p=None, + use_cache=True, + ) + new_ids = out[0, input_len:].tolist() + text = tok.decode(new_ids, skip_special_tokens=True) + return Generation(prompt=prompt_text, text=text, token_ids=new_ids) + + +def generate_vllm(model_dir: Path, prompts: list[str], max_new_tokens: int, + gpu_memory_utilization: float = 0.5, + max_model_len: int = 4096) -> list[Generation]: + from vllm import LLM, SamplingParams + logger.info("loading %s in vLLM (gpu_memory_utilization=%.2f, max_model_len=%d) ...", + model_dir, gpu_memory_utilization, max_model_len) + llm = LLM( + model=str(model_dir), + dtype="auto", + gpu_memory_utilization=gpu_memory_utilization, + max_model_len=max_model_len, + trust_remote_code=True, + ) + tok = llm.get_tokenizer() + outputs = llm.generate( + prompts, + SamplingParams(temperature=0.0, max_tokens=max_new_tokens), + ) + gens: list[Generation] = [] + for prompt, out in zip(prompts, outputs): + completion = out.outputs[0] + gens.append(Generation( + prompt=prompt, + text=completion.text, + token_ids=list(completion.token_ids), + )) + return gens + + +def token_agreement(a: list[int], b: list[int]) -> tuple[float, int]: + n = min(len(a), len(b)) + if n == 0: + return 0.0, 0 + matches = 0 + diverged_at = n + for i in range(n): + if a[i] == b[i]: + matches += 1 + else: + diverged_at = i + break + return matches / n, diverged_at + + +def compare(label_a: str, gens_a: list[Generation], label_b: str, gens_b: list[Generation], + min_agreement: float) -> bool: + print() + print(f"=== {label_a} vs {label_b} ===") + all_pass = True + for ga, gb in zip(gens_a, gens_b): + agree, first_diff = token_agreement(ga.token_ids, gb.token_ids) + status = "PASS" if agree >= min_agreement else "FAIL" + if agree < min_agreement: + all_pass = False + prompt_preview = ga.prompt.strip().replace("\n", " ")[:60] + print(f"[{status}] agreement={agree:.2%} first_diff@{first_diff} prompt={prompt_preview!r}") + if agree < 1.0: + print(f" {label_a}: {ga.text[:120]!r}") + print(f" {label_b}: {gb.text[:120]!r}") + return all_pass + + +def _fit_vllm_util(desired: float, safety: float = 0.9) -> float: + """Shrink gpu_memory_utilization to what is actually free on device 0. + + vLLM's `gpu_memory_utilization` is a fraction of *total* device memory, + not of free memory, and it refuses to start if it can't reserve that + much. On unified-memory boxes (DGX Spark) the "free" fraction of the + 120 GB pool can be well below the requested 0.5 when other processes + (or the OS page cache) are holding memory. Cap the fraction at + (free/total)*safety so vLLM always has a small headroom over what it + is guaranteed to obtain. Returns the desired value untouched when + CUDA is unavailable or the query fails.""" + try: + import torch + if not torch.cuda.is_available(): + return desired + free, total = torch.cuda.mem_get_info(0) + except Exception: + return desired + if total <= 0: + return desired + max_util = (free / total) * safety + if max_util >= desired: + return desired + logger.warning( + "shrinking vLLM gpu_memory_utilization %.2f -> %.2f " + "(free=%.1f GiB, total=%.1f GiB, safety=%.2f)", + desired, max_util, free / 2**30, total / 2**30, safety, + ) + return max_util + + +def _release_cuda_memory() -> None: + """Drop the caching allocator's block pool back to the driver. + + torch.cuda.empty_cache() alone is not enough — it releases *cached* blocks + but the allocator may still hold reserved chunks. gc.collect() first drops + any lingering Python-level references; then empty_cache + reset_peak + + ipc_collect actually returns memory to the driver so a subsequent + subprocess (like vLLM's engine core) sees it as free.""" + import gc + gc.collect() + try: + import torch + if torch.cuda.is_available(): + torch.cuda.synchronize() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + torch.cuda.reset_peak_memory_stats() + except ImportError: + pass + + +def _wait_cuda_memory_stable(max_wait_s: float = 5.0, + poll_interval_s: float = 0.2) -> None: + """Poll `mem_get_info` until free memory stops growing, or timeout. + + torch.cuda.empty_cache() releases blocks *asynchronously*; the CUDA driver + may take several hundred milliseconds to fully unmap. If vLLM spawns its + engine subprocess while the parent is still draining, vLLM's first memory + snapshot is lower than the memory available during its later profiling + call, and it fails with: + AssertionError: Error in memory profiling. + Initial free memory X, current free memory Y. + (Y > X → the parent kept releasing.) + Wait for two consecutive stable readings before letting vLLM start.""" + import time + try: + import torch + except ImportError: + return + if not torch.cuda.is_available(): + return + torch.cuda.synchronize() + prev = -1 + deadline = time.monotonic() + max_wait_s + while time.monotonic() < deadline: + free, _ = torch.cuda.mem_get_info(0) + if free == prev: + return + prev = free + time.sleep(poll_interval_s) + + +def set_seed(seed: int) -> None: + random.seed(seed) + try: + import torch + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + except ImportError: + pass + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Verify FP8-converted model against its source") + p.add_argument("original", type=Path, help="folder of the original (BF16/FP16) HF model") + p.add_argument("fp8", type=Path, help="folder of the FP8-converted HF model") + p.add_argument("--vllm", action="store_true", help="also run inference with vLLM on the FP8 model") + p.add_argument("--device", choices=["auto", "cpu", "cuda"], default="auto", + help="device used to load the two HF models. Use 'cpu' to bypass " + "Triton/mamba-ssm compatibility issues on Blackwell — much " + "slower but sidesteps the fast path entirely.") + p.add_argument("--vllm-gpu-memory-utilization", type=float, default=0.5, + help="fraction of GPU memory vLLM is allowed to reserve at startup. " + "Default 0.5 is conservative and safe when the transformers run " + "in the same process still holds cache. Raise to 0.9 on a dedicated GPU. " + "Automatically shrunk to fit actual free memory when needed.") + p.add_argument("--vllm-max-model-len", type=int, default=4096, + help="max context length passed to vLLM. Smaller = less KV cache reserved. " + "Default 4096 is plenty for the short-generation checks this test does; " + "raise it only if a specific prompt/generation is longer.") + p.add_argument("--max-new-tokens", type=int, default=30, help="tokens generated per prompt (default: 30)") + p.add_argument("--num-prompts", type=int, default=3, help="number of prompts sampled from the pool (default: 3)") + p.add_argument("--min-agreement", type=float, default=0.85, + help="minimum per-prompt token-agreement for PASS (default: 0.85)") + p.add_argument("--seed", type=int, default=42, help="random seed for prompt selection (default: 42)") + p.add_argument("--verbose", action="store_true") + return p.parse_args() + + +def main() -> int: + args = parse_args() + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + + for p in (args.original, args.fp8): + if not p.is_dir(): + logger.error("not a directory: %s", p) + return 2 + + set_seed(args.seed) + prompts = random.sample(DEFAULT_PROMPTS, k=min(args.num_prompts, len(DEFAULT_PROMPTS))) + logger.info("selected %d prompt(s)", len(prompts)) + for i, p in enumerate(prompts): + logger.info(" [%d] %s", i, p) + + try: + orig_model, orig_tok = load_hf_model(args.original, device=args.device) + except Exception as e: + logger.error("failed to load original model: %s", e) + return 2 + + orig_prompts = [apply_template(orig_tok, p) for p in prompts] + logger.info("generating with original model ...") + orig_gens = [generate_hf(orig_model, orig_tok, tp, args.max_new_tokens) for tp in orig_prompts] + + del orig_model + _release_cuda_memory() + + try: + fp8_model, fp8_tok = load_hf_model(args.fp8, device=args.device) + except Exception as e: + logger.error("failed to load FP8 model: %s", e) + return 2 + + fp8_prompts = [apply_template(fp8_tok, p) for p in prompts] + logger.info("generating with FP8 model (transformers) ...") + fp8_gens = [generate_hf(fp8_model, fp8_tok, tp, args.max_new_tokens) for tp in fp8_prompts] + + del fp8_model + _release_cuda_memory() + + ok = compare("HF-original", orig_gens, "HF-FP8", fp8_gens, args.min_agreement) + + if args.vllm: + _release_cuda_memory() + _wait_cuda_memory_stable() + util = _fit_vllm_util(args.vllm_gpu_memory_utilization) + # A vLLM engine that reserves less than ~5% of a large unified pool + # cannot hold even a small model + kv-cache, so bail out with a clear + # message rather than letting vLLM emit an obscure allocator failure. + if util < 0.05: + logger.error( + "only ~%.1f%% of GPU memory is currently free — vLLM cannot " + "start with usable headroom. Free memory (kill other CUDA " + "processes; drop OS page cache) and retry.", + util * 100, + ) + return 2 + try: + vllm_gens = generate_vllm( + args.fp8, fp8_prompts, args.max_new_tokens, + gpu_memory_utilization=util, + max_model_len=args.vllm_max_model_len, + ) + except ImportError: + logger.error("vLLM is not installed. `pip install vllm` and retry.") + return 2 + except Exception as e: + logger.error("vLLM run failed: %s", e) + return 2 + ok = compare("HF-FP8", fp8_gens, "vLLM-FP8", vllm_gens, args.min_agreement) and ok + + print() + print("=== SUMMARY ===") + print("PASS" if ok else "FAIL") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test-tokenizer-random.py b/tests/test-tokenizer-random.py index 93e697607e66..5348a9d8b54d 100644 --- a/tests/test-tokenizer-random.py +++ b/tests/test-tokenizer-random.py @@ -11,6 +11,7 @@ import time import logging import argparse +import itertools import subprocess import random import unicodedata @@ -26,11 +27,28 @@ logger = logging.getLogger("test-tokenizer-random") +# Characters that historically flood the mismatch log without teaching us +# anything about the tokenizer we're actually testing (they mostly exercise +# byte-fallback paths). We keep at most one dedicated test per representative +# character in the fixed test lists and filter these out of the random / +# brute-force generators so they don't dominate the report. +CONTROL_CHARS = frozenset( + chr(cp) for cp in ( + *range(0x00, 0x09), # NUL..BS + 0x0B, # VT + *range(0x0D, 0x20), # CR..US + 0x7F, # DEL + 0xFEFF, # BOM + ) +) +CURLY_QUOTES = frozenset("‘’“”") # U+2018..U+201D — represent all curly quotes + + class LibLlama: DEFAULT_PATH_LLAMA_H = "./include/llama.h" DEFAULT_PATH_INCLUDES = ["./ggml/include/", "./include/"] - DEFAULT_PATH_LIBLLAMA = "./build/src/libllama.so" # CMakeLists.txt: BUILD_SHARED_LIBS ON + DEFAULT_PATH_LIBLLAMA = "./build/bin/libllama.so" # CMakeLists.txt: BUILD_SHARED_LIBS ON def __init__(self, path_llama_h: str | None = None, path_includes: list[str] = [], path_libllama: str | None = None): path_llama_h = path_llama_h or self.DEFAULT_PATH_LLAMA_H @@ -79,6 +97,9 @@ def __init__(self, libllama: LibLlama, path_model: str, mparams={}, cparams={}): self.model = self.lib.llama_model_load_from_file(path_model.encode(), mparams) if not self.model: raise RuntimeError("error: failed to load model '%s'" % path_model) + self.vocab = self.lib.llama_model_get_vocab(self.model) + if not self.vocab: + raise RuntimeError("error: failed to get vocab for model '%s'" % path_model) if isinstance(cparams, dict): cparams = libllama.context_default_params(**cparams) self.ctx = self.lib.llama_new_context_with_model(self.model, cparams) @@ -99,10 +120,10 @@ def free(self): def tokenize(self, text: str, add_special: bool = False, parse_special: bool = False) -> list[int]: encoded_text: bytes = text.encode("utf-8") - num = self.lib.llama_tokenize(self.model, encoded_text, len(encoded_text), self.token_ids, len(self.token_ids), add_special, parse_special) + num = self.lib.llama_tokenize(self.vocab, encoded_text, len(encoded_text), self.token_ids, len(self.token_ids), add_special, parse_special) while num < 0 and len(self.token_ids) < (16 << 20): self.token_ids = self.ffi.new("llama_token[]", -2 * num) - num = self.lib.llama_tokenize(self.model, encoded_text, len(encoded_text), self.token_ids, len(self.token_ids), add_special, parse_special) + num = self.lib.llama_tokenize(self.vocab, encoded_text, len(encoded_text), self.token_ids, len(self.token_ids), add_special, parse_special) return list(self.token_ids[0:num]) def detokenize(self, ids: list[int], remove_special: bool = False, unparse_special: bool = False) -> str: @@ -110,10 +131,10 @@ def detokenize(self, ids: list[int], remove_special: bool = False, unparse_speci self.token_ids = self.ffi.new("llama_token[]", 2 * len(ids)) for i, id in enumerate(ids): self.token_ids[i] = id - num = self.lib.llama_detokenize(self.model, self.token_ids, len(ids), self.text_buff, len(self.text_buff), remove_special, unparse_special) + num = self.lib.llama_detokenize(self.vocab, self.token_ids, len(ids), self.text_buff, len(self.text_buff), remove_special, unparse_special) while num < 0 and len(self.text_buff) < (16 << 20): self.text_buff = self.ffi.new("uint8_t[]", -2 * num) - num = self.lib.llama_detokenize(self.model, self.token_ids, len(ids), self.text_buff, len(self.text_buff), remove_special, unparse_special) + num = self.lib.llama_detokenize(self.vocab, self.token_ids, len(ids), self.text_buff, len(self.text_buff), remove_special, unparse_special) return str(cast(Buffer, self.ffi.buffer(self.text_buff, num)), encoding="utf-8", errors="replace") # replace errors with '\uFFFD' @@ -147,12 +168,25 @@ def __init__(self, dir_tokenizer: str): self.bos_token = self.model.bos_token self.eos_token = self.model.eos_token - def encode(self, text: str) -> list[int]: - return self.model.encode(text, add_special_tokens=True) + def encode(self, text: str, add_special: bool = True) -> list[int]: + return self.model.encode(text, add_special_tokens=add_special) def decode(self, ids: list[int]) -> str: return self.model.decode(ids, skip_special_tokens=False) + def convert_ids_to_tokens(self, ids: list[int]) -> list[str]: + return self.model.convert_ids_to_tokens(ids) + + def has_chat_template(self) -> bool: + return bool(getattr(self.model, "chat_template", None)) + + def apply_chat_template(self, messages: list[dict], add_generation_prompt: bool = True) -> str: + return self.model.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=add_generation_prompt, + ) + class TokenizerLlamaCpp (Tokenizer): @@ -163,8 +197,8 @@ def __init__(self, vocab_file: str): self.libllama = LibLlama() self.model = LibLlamaModel(self.libllama, vocab_file, mparams=dict(vocab_only=True), cparams=dict(n_ctx=4096)) - def encode(self, text: str) -> list[int]: - return self.model.tokenize(text, add_special=True, parse_special=True) + def encode(self, text: str, add_special: bool = True) -> list[int]: + return self.model.tokenize(text, add_special=add_special, parse_special=True) def decode(self, ids: list[int]) -> str: return self.model.detokenize(ids, remove_special=False, unparse_special=True) @@ -204,6 +238,12 @@ def generator_custom_text() -> Iterator[str]: "\n =", "' era", "Hello, y'all! How are you 😁 ?我想在apple工作1314151天～", + ] + + +def generator_digit() -> Iterator[str]: + """Digits""" + yield from [ "3", "33", "333", @@ -213,13 +253,55 @@ def generator_custom_text() -> Iterator[str]: "3333333", "33333333", "333333333", + "333333333+333", + ] + + +def generator_contractions() -> Iterator[str]: + """Contractions and apostrophes. + + All French elisions use the straight ASCII apostrophe. Curly-quote coverage + is deliberately minimal (one U+201C double, one U+2019 single) — those two + stand in for every curly-quote form; the rest are filtered out of the random + generators to keep the mismatch log focused.""" + yield from [ + # English contractions + "I'll", + "We've they're", + "don't shouldn't wouldn't", + "I'm you're he'd she'll", + "y'all it's", + # French elisions (single) + "j'ai t'as l'homme d'un c'est s'il n'est m'a qu'il", + "s'il vous plaît, c'est l'heure d'y aller.", + # French elisions requiring the multi-letter prefix branch + "jusqu'à demain", + "lorsqu'il pleut", + "puisqu'après tout", + "quoiqu'il arrive", + "aujourd'hui", + "aujourd'hui, jusqu'à ce que lorsqu'ils viennent", + # Mixed English + French + "I'll dire qu'aujourd'hui c'est bien", + "she's saying qu'il ne l'a pas fait, isn't she?", + # Edge case (nonsense elision) already covered in the prior list + "j're", + # One curly-double + one curly-single, standing in for all curly quotes + "“Bonjour quoiqu'aujourd'hui”", + "puisqu’après", ] def generator_custom_text_edge_cases() -> Iterator[str]: - """Edge cases found while debugging""" + """Edge cases found while debugging. + + Control-character coverage is intentionally sparse: at most 3 entries here + touch a character matched by [\\x00-\\x08\\x0b\\x0d-\\x1f\\x7f\\ufeff], and + each specific control character appears at most once (currently: BOM, CR, + NUL). The random / brute-force generators additionally filter these + characters out at their source so byte-fallback noise doesn't dominate the + mismatch report.""" yield from [ - '\x1f-a', # unicode_ranges_control, {0x00001C, 0x00001F} '¼-a', # unicode_ranges_digit, 0x00BC '½-a', # unicode_ranges_digit, 0x00BD '¾-a', # unicode_ranges_digit, 0x00BE @@ -232,8 +314,8 @@ def generator_custom_text_edge_cases() -> Iterator[str]: 'a\na', # bert fail '"`', # falcon ' \u2e4e', # falcon - '\n\x0b ', # falcon - 'a\xa0\xa0\x00b', # jina-v2-es + 'a\r\nb', # CR/LF handling [ctrl 2/3] + 'a\xa0\xa0\x00b', # jina-v2-es (embedded NUL) [ctrl 3/3] 'one ', # jina-v2-es lstrip=true 'a b', # rstrip phi-3 'a b', # lstrip jina-v2 @@ -255,7 +337,7 @@ def generator_vocab_words(tokenizer: TokenizerGroundtruth) -> Iterator[str]: def generator_ascii_lr_strip() -> Iterator[str]: WHITESPACES = ["", " ", " "] - CHARACTERS = list(chr(i) for i in range(1, 0x80)) + [""] + CHARACTERS = [c for c in (chr(i) for i in range(1, 0x80)) if c not in CONTROL_CHARS] + [""] for char1 in CHARACTERS: for char2 in CHARACTERS: for lstrip in WHITESPACES: @@ -267,7 +349,7 @@ def generator_ascii_lr_strip() -> Iterator[str]: def generator_apostrophe() -> Iterator[str]: WHITESPACES = ["", " ", " "] - CHARACTERS = list(chr(i) for i in range(1, 0x80)) + [""] + CHARACTERS = [c for c in (chr(i) for i in range(1, 0x80)) if c not in CONTROL_CHARS] + [""] for char1 in CHARACTERS: for char2 in CHARACTERS: for lstrip in WHITESPACES: @@ -346,6 +428,12 @@ def _valid(cpt): # return False if unicodedata.category(chr(cpt)) in ("Cn", "Cs", "Co"): # undefined, surrogates, private return False + ch = chr(cpt) + # Filter out control chars and curly quotes at the source; they are + # covered once each in the fixed generators (contractions, + # custom_text_edge_cases) and would otherwise dominate the report. + if ch in CONTROL_CHARS or ch in CURLY_QUOTES: + return False return True characters = [chr(cpt) for cpt in range(0, MAX_CODEPOINTS) if _valid(cpt)] @@ -407,7 +495,59 @@ def generator_random_vocab_words(tokenizer: TokenizerGroundtruth, iterations=100 yield "".join(text) -def compare_tokenizers(tokenizer1: TokenizerGroundtruth, tokenizer2: TokenizerLlamaCpp, generator: Iterator[str]): +def generator_chat_wrap(generator: Iterator[str], tokenizer: TokenizerGroundtruth) -> Iterator[str]: + """Wrap each yielded text as a single user-turn chat template, with add_generation_prompt=True.""" + for text in generator: + messages = [{"role": "user", "content": text}] + try: + yield tokenizer.apply_chat_template(messages, add_generation_prompt=True) + except Exception as e: + logger.debug(f"chat template skipped for {repr(text)[:60]}: {e}") + continue + + +def _collect_texts(source: Iterator[str], limit: int = 2048) -> list[str]: + out: list[str] = [] + for text in source: + if not isinstance(text, str): + continue + out.append(text) + if len(out) >= limit: + break + return out + + +def generator_random_chat_multiturn( + tokenizer: TokenizerGroundtruth, + text_pool_source: Iterator[str], + iterations: int = 500, + max_turns: int = 11, +) -> Iterator[str]: + """Random multi-turn conversations [user, assistant, user, ..., user] with add_generation_prompt=True. + + The conversation always ends on a user message (odd number of turns). + """ + texts = _collect_texts(text_pool_source) + if not texts: + return + rand = random.Random() + for m in range(iterations): + rand.seed(m) + num_turns = rand.randint(1, max_turns) + if num_turns % 2 == 0: + num_turns += 1 # ensure odd → ends on user + messages = [] + for i in range(num_turns): + role = "user" if i % 2 == 0 else "assistant" + messages.append({"role": role, "content": rand.choice(texts)}) + try: + yield tokenizer.apply_chat_template(messages, add_generation_prompt=True) + except Exception as e: + logger.debug(f"multiturn chat template failed at iter {m} (turns={num_turns}): {e}") + continue + + +def compare_tokenizers(tokenizer1: TokenizerGroundtruth, tokenizer2: TokenizerLlamaCpp, generator: Iterator[str], add_special: bool = True): def find_first_mismatch(ids1: list[int] | str, ids2: list[int] | str): for i, (a, b) in enumerate(zip(ids1, ids2)): @@ -418,7 +558,7 @@ def find_first_mismatch(ids1: list[int] | str, ids2: list[int] | str): return min(len(ids1), len(ids2)) def check_detokenizer(text: str, text1: str, text2: str) -> bool: - if text1 == text2: # equal to TokenizerGroundtruth? + if text1 == text2 or text2 == text: # equal to TokenizerGroundtruth? return True # equal to source text? if tokenizer1.add_bos_token and tokenizer1.bos_token and isinstance(tokenizer1.bos_token, str): # remove BOS @@ -436,16 +576,16 @@ def check_detokenizer(text: str, text1: str, text2: str) -> bool: t_start = time.perf_counter() encode_errors = 0 decode_errors = 0 - MAX_ERRORS = 10 + MAX_ERRORS = 20 logger.info("%s: %s" % (generator.__qualname__, "ini")) for text in generator: # print(repr(text), text.encode()) # print(repr(text), hex(ord(text[0])), text.encode()) t0 = time.perf_counter() - ids1 = tokenizer1.encode(text) + ids1 = tokenizer1.encode(text, add_special=add_special) t1 = time.perf_counter() - ids2 = tokenizer2.encode(text) + ids2 = tokenizer2.encode(text, add_special=add_special) t2 = time.perf_counter() text1 = tokenizer1.decode(ids1) t3 = time.perf_counter() @@ -455,23 +595,30 @@ def check_detokenizer(text: str, text1: str, text2: str) -> bool: t_encode2 += t2 - t1 t_decode1 += t3 - t2 t_decode2 += t4 - t3 - if encode_errors < MAX_ERRORS and ids1 != ids2: + had_error = False + if (MAX_ERRORS is None or encode_errors < MAX_ERRORS) and ids1 != ids2: i = find_first_mismatch(ids1, ids2) - ids1 = list(ids1)[max(0, i - 2) : i + 5 + 1] - ids2 = list(ids2)[max(0, i - 2) : i + 5 + 1] - logger.error(" Expected: " + str(ids1)) - logger.error(" Result: " + str(ids2)) + ids1_ctx = list(ids1)[max(0, i - 2) : i + 5 + 1] + ids2_ctx = list(ids2)[max(0, i - 2) : i + 5 + 1] + logger.error(f" Input: {repr(text[:100])}") + logger.error(" Expected: " + str(ids1_ctx) + " " + str(tokenizer1.convert_ids_to_tokens(ids1_ctx))) + logger.error(" Result: " + str(ids2_ctx) + " " + str(tokenizer1.convert_ids_to_tokens(ids2_ctx))) encode_errors += 1 - logger.error(f" {encode_errors=}") - if decode_errors < MAX_ERRORS and not check_detokenizer(text, text1, text2): + # logger.error(f" {encode_errors=}") + had_error = True + if (MAX_ERRORS is None or decode_errors < MAX_ERRORS) and not check_detokenizer(text, text1, text2): i = find_first_mismatch(text1, text2) - text1 = list(text1[max(0, i - 2) : i + 5 + 1]) - text2 = list(text2[max(0, i - 2) : i + 5 + 1]) - logger.error(" Expected: " + " ".join(hex(ord(x)) for x in text1)) - logger.error(" Result: " + " ".join(hex(ord(x)) for x in text2)) + text1_ctx = text1[max(0, i - 2) : i + 5 + 1] + text2_ctx = text2[max(0, i - 2) : i + 5 + 1] + logger.error(f" Input: {repr(text[:100])}") + logger.error(" Expected: " + repr(text1_ctx)) + logger.error(" Result: " + repr(text2_ctx)) decode_errors += 1 - logger.error(f" {decode_errors=}") - if encode_errors >= MAX_ERRORS and decode_errors >= MAX_ERRORS: + # logger.error(f" {decode_errors=}") + had_error = True + if had_error: + logger.error("") + if MAX_ERRORS is not None and encode_errors >= MAX_ERRORS and decode_errors >= MAX_ERRORS: logger.error(f" EXIT: {encode_errors=} {decode_errors=}") # raise Exception() break @@ -485,6 +632,19 @@ def main(argv: list[str] | None = None): parser.add_argument("vocab_file", type=str, help="path to vocab 'gguf' file") parser.add_argument("dir_tokenizer", type=str, help="directory containing 'tokenizer.model' file") parser.add_argument("--verbose", action="store_true", help="increase output verbosity") + parser.add_argument( + "--chat_template", + type=str, + choices=["true", "false"], + default=None, + help=( + "Wrap each test input in the model's chat template before tokenizing. " + "'true' requires the tokenizer to define a chat template (fails otherwise) and " + "also runs multi-turn conversation tests. " + "'false' tests raw inputs (legacy behavior). " + "If omitted, defaults to 'true' when a chat template is present, else 'false'." + ), + ) args = parser.parse_args(argv) logging.basicConfig(level = logging.DEBUG if args.verbose else logging.INFO) @@ -493,74 +653,141 @@ def main(argv: list[str] | None = None): tokenizer1 = TokenizerGroundtruth(args.dir_tokenizer) tokenizer2 = TokenizerLlamaCpp(args.vocab_file) - # compare_tokenizers(tokenizer1, tokenizer2, generator_custom_text()) - # compare_tokenizers(tokenizer1, tokenizer2, generator_custom_text_edge_cases()) - compare_tokenizers(tokenizer1, tokenizer2, generator_ascii_lr_strip()) - compare_tokenizers(tokenizer1, tokenizer2, generator_apostrophe()) - compare_tokenizers(tokenizer1, tokenizer2, generator_unicodes()) - compare_tokenizers(tokenizer1, tokenizer2, generator_vocab_words(tokenizer1)) - compare_tokenizers(tokenizer1, tokenizer2, generator_added_lr_strip(tokenizer1)) - # compare_tokenizers(tokenizer1, tokenizer2, generator_random_added_tokens(tokenizer1, 10_000)) - # compare_tokenizers(tokenizer1, tokenizer2, generator_random_chars(10_000)) - # compare_tokenizers(tokenizer1, tokenizer2, generator_random_unicodes(10_000)) - # compare_tokenizers(tokenizer1, tokenizer2, generator_random_vocab_chars(tokenizer1, 10_000)) - # compare_tokenizers(tokenizer1, tokenizer2, generator_random_vocab_words(tokenizer1, 5_000)) + has_template = tokenizer1.has_chat_template() + if args.chat_template is None: + use_chat_template = has_template + elif args.chat_template == "true": + if not has_template: + raise ValueError( + "--chat_template true was requested but the tokenizer at " + f"'{args.dir_tokenizer}' has no chat_template set." + ) + use_chat_template = True + else: + use_chat_template = False + + logger.info( + f"chat_template: {'ENABLED' if use_chat_template else 'DISABLED'} " + f"(tokenizer {'has' if has_template else 'has NO'} template; " + f"--chat_template={args.chat_template})" + ) + + if not use_chat_template: + compare_tokenizers(tokenizer1, tokenizer2, generator_custom_text()) + compare_tokenizers(tokenizer1, tokenizer2, generator_digit()) + compare_tokenizers(tokenizer1, tokenizer2, generator_contractions()) + compare_tokenizers(tokenizer1, tokenizer2, generator_custom_text_edge_cases()) + compare_tokenizers(tokenizer1, tokenizer2, generator_ascii_lr_strip()) + compare_tokenizers(tokenizer1, tokenizer2, generator_apostrophe()) + compare_tokenizers(tokenizer1, tokenizer2, generator_unicodes()) + compare_tokenizers(tokenizer1, tokenizer2, generator_vocab_words(tokenizer1)) + compare_tokenizers(tokenizer1, tokenizer2, generator_added_lr_strip(tokenizer1)) + compare_tokenizers(tokenizer1, tokenizer2, generator_random_added_tokens(tokenizer1, 10_000)) + compare_tokenizers(tokenizer1, tokenizer2, generator_random_chars(10_000)) + compare_tokenizers(tokenizer1, tokenizer2, generator_random_unicodes(10_000)) + compare_tokenizers(tokenizer1, tokenizer2, generator_random_vocab_chars(tokenizer1, 10_000)) + compare_tokenizers(tokenizer1, tokenizer2, generator_random_vocab_words(tokenizer1, 5_000)) + else: + # Chat-templated single-turn runs. The chat template already injects BOS/special + # tokens as needed, so we disable add_special on both tokenizers to avoid + # double-BOS and to keep the inputs strictly identical. + # + # Each yielded item costs ~10–100x more than in raw mode (Jinja template render + + # tokenization of the full templated string), so we cap exhaustive generators and + # use smaller iteration counts for the random ones. The chat-template prefix is + # identical across items, so sampling gives equivalent coverage to enumeration. + CHAT_CAP = 2_000 + CHAT_RAND_ITER = 1_000 + + def _wrap(gen, cap=CHAT_CAP): + if cap is not None: + gen = itertools.islice(gen, cap) + return generator_chat_wrap(gen, tokenizer1) + + compare_tokenizers(tokenizer1, tokenizer2, _wrap(generator_custom_text()), add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, _wrap(generator_digit()), add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, _wrap(generator_contractions()), add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, _wrap(generator_custom_text_edge_cases()), add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, _wrap(generator_ascii_lr_strip()), add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, _wrap(generator_apostrophe()), add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, _wrap(generator_unicodes()), add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, _wrap(generator_vocab_words(tokenizer1)), add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, _wrap(generator_added_lr_strip(tokenizer1)), add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, _wrap(generator_random_added_tokens(tokenizer1, CHAT_RAND_ITER)), add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, _wrap(generator_random_chars(CHAT_RAND_ITER)), add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, _wrap(generator_random_unicodes(CHAT_RAND_ITER)), add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, _wrap(generator_random_vocab_chars(tokenizer1, CHAT_RAND_ITER)), add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, _wrap(generator_random_vocab_words(tokenizer1, CHAT_RAND_ITER)), add_special=False) + + # Multi-turn conversation tests (alternating user/assistant, ending on user). + compare_tokenizers(tokenizer1, tokenizer2, + generator_random_chat_multiturn(tokenizer1, generator_custom_text(), iterations=500), + add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, + generator_random_chat_multiturn(tokenizer1, generator_random_chars(500), iterations=500), + add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, + generator_random_chat_multiturn(tokenizer1, generator_random_unicodes(500), iterations=500), + add_special=False) + compare_tokenizers(tokenizer1, tokenizer2, + generator_random_chat_multiturn(tokenizer1, generator_random_vocab_words(tokenizer1, 500), iterations=500), + add_special=False) tokenizer2.model.free() if __name__ == "__main__": - # main() - - if True: - logging.basicConfig( - level = logging.DEBUG, - format = "%(asctime)s.%(msecs)03d %(name)s %(levelname)s %(message)s", - datefmt = "%Y-%m-%d %H:%M:%S", - filename = logger.name + ".log", - filemode = "a" - ) - logging.basicConfig( - level = logging.DEBUG, - format = "%(levelname)s %(message)s", - ) - - path_tokenizers = Path("./models/tokenizers/") - path_vocab_format = "./models/ggml-vocab-%s.gguf" - - tokenizers = [ - "llama-spm", # SPM - "phi-3", # SPM - "gemma", # SPM - "gemma-2", # SPM - "baichuan", # SPM - "bert-bge", # WPM - "jina-v2-en", # WPM - "llama-bpe", # BPE - "phi-2", # BPE - "deepseek-llm", # BPE - "deepseek-coder", # BPE - "falcon", # BPE - "mpt", # BPE - "starcoder", # BPE - "gpt-2", # BPE - "stablelm2", # BPE - "refact", # BPE - "qwen2", # BPE - "olmo", # BPE - "jina-v2-es", # BPE - "jina-v2-de", # BPE - "smaug-bpe", # BPE - "poro-chat", # BPE - "jina-v2-code", # BPE - "viking", # BPE - "jais", # BPE - ] - - logger.info("=" * 50) - for tokenizer in tokenizers: - logger.info("-" * 50) - logger.info(f"TOKENIZER: '{tokenizer}'") - vocab_file = Path(path_vocab_format % tokenizer) - dir_tokenizer = path_tokenizers / tokenizer - main([str(vocab_file), str(dir_tokenizer), "--verbose"]) + main() + + # if True: + # logging.basicConfig( + # level = logging.DEBUG, + # format = "%(asctime)s.%(msecs)03d %(name)s %(levelname)s %(message)s", + # datefmt = "%Y-%m-%d %H:%M:%S", + # filename = logger.name + ".log", + # filemode = "a" + # ) + # logging.basicConfig( + # level = logging.DEBUG, + # format = "%(levelname)s %(message)s", + # ) + + # path_tokenizers = Path("./models/tokenizers/") + # path_vocab_format = "./models/ggml-vocab-%s.gguf" + + # tokenizers = [ + # "llama-spm", # SPM + # "phi-3", # SPM + # "gemma", # SPM + # "gemma-2", # SPM + # "baichuan", # SPM + # "bert-bge", # WPM + # "jina-v2-en", # WPM + # "llama-bpe", # BPE + # "phi-2", # BPE + # "deepseek-llm", # BPE + # "deepseek-coder", # BPE + # "falcon", # BPE + # "mpt", # BPE + # "starcoder", # BPE + # "gpt-2", # BPE + # "stablelm2", # BPE + # "refact", # BPE + # "qwen2", # BPE + # "olmo", # BPE + # "jina-v2-es", # BPE + # "jina-v2-de", # BPE + # "smaug-bpe", # BPE + # "poro-chat", # BPE + # "jina-v2-code", # BPE + # "viking", # BPE + # "jais", # BPE + # ] + + # logger.info("=" * 50) + # for tokenizer in tokenizers: + # logger.info("-" * 50) + # logger.info(f"TOKENIZER: '{tokenizer}'") + # vocab_file = Path(path_vocab_format % tokenizer) + # dir_tokenizer = path_tokenizers / tokenizer + # main([str(vocab_file), str(dir_tokenizer), "--verbose"])