From b576c546acd42a22c4b39776478d5824a007191f Mon Sep 17 00:00:00 2001 From: abetlen Date: Sun, 5 Jul 2026 09:47:34 -0700 Subject: [PATCH 1/4] feat: update llama.cpp to 78d2f5246 --- CHANGELOG.md | 2 ++ llama_cpp/llama_cpp.py | 16 ++++++++++++++++ vendor/llama.cpp | 2 +- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b5e88578..5aa71dcdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: update llama.cpp to ggml-org/llama.cpp@78d2f5246 + ## [0.3.32] - feat(example): support chained NextN heads for server MTP drafting diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 176709d96..333e57df0 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1272,6 +1272,14 @@ def llama_flash_attn_type_name(flash_attn_type: int, /) -> Optional[bytes]: ... +# // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" +# LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype); +@ctypes_function("llama_ftype_name", [ctypes.c_int], ctypes.c_char_p) +def llama_ftype_name(ftype: int, /) -> Optional[bytes]: + """Get the model file type as a string.""" + ... + + # // Initialize the llama + ggml backend # // If numa is true, use NUMA optimizations # // Call once at the start of the program @@ -1910,6 +1918,14 @@ def llama_model_desc( ... +# // Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0 +# LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model); +@ctypes_function("llama_model_ftype", [llama_model_p_ctypes], ctypes.c_int) +def llama_model_ftype(model: llama_model_p, /) -> int: + """Get the model file type.""" + ... + + # // Returns the total size of all the tensors in the model in bytes # LLAMA_API uint64_t llama_model_size(const struct llama_model * model); @ctypes_function("llama_model_size", [llama_model_p_ctypes], ctypes.c_uint64) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index b3fed31b9..78d2f5246 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit b3fed31b99f9bd37725833674252bccb429bb183 +Subproject commit 78d2f524682d9fee790a6460c93d018dafeb5229 From 9290f80fdb72b6b48d56c3e279a19b43cbafc463 Mon Sep 17 00:00:00 2001 From: abetlen Date: Sun, 5 Jul 2026 09:50:08 -0700 Subject: [PATCH 2/4] fix: align llama.cpp binding comments --- llama_cpp/llama_cpp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 333e57df0..592bf3e14 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1276,7 +1276,7 @@ def llama_flash_attn_type_name(flash_attn_type: int, /) -> Optional[bytes]: # LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype); @ctypes_function("llama_ftype_name", [ctypes.c_int], ctypes.c_char_p) def llama_ftype_name(ftype: int, /) -> Optional[bytes]: - """Get the model file type as a string.""" + """Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium".""" ... @@ -1922,7 +1922,7 @@ def llama_model_desc( # LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model); @ctypes_function("llama_model_ftype", [llama_model_p_ctypes], ctypes.c_int) def llama_model_ftype(model: llama_model_p, /) -> int: - """Get the model file type.""" + """Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0.""" ... From d115822ec24c868580ceacbcc543c94840d34e05 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 5 Jul 2026 10:05:03 -0700 Subject: [PATCH 3/4] fix: align llama.cpp binding docstrings (#2328) * fix: align llama.cpp binding docstrings * fix: use None in Python binding docstrings --- llama_cpp/llama_cpp.py | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 592bf3e14..087e1130e 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1276,7 +1276,8 @@ def llama_flash_attn_type_name(flash_attn_type: int, /) -> Optional[bytes]: # LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype); @ctypes_function("llama_ftype_name", [ctypes.c_int], ctypes.c_char_p) def llama_ftype_name(ftype: int, /) -> Optional[bytes]: - """Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium".""" + """Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" + """ ... @@ -1785,7 +1786,8 @@ def llama_model_rope_freq_scale_train(model: llama_model_p, /) -> float: ... # LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model); @ctypes_function("llama_model_n_cls_out", [llama_model_p_ctypes], ctypes.c_uint32) def llama_model_n_cls_out(model: llama_model_p, /) -> int: - """Returns the number of classifier outputs (only valid for classifier models)""" + """Returns the number of classifier outputs (only valid for classifier models) + Undefined behavior for non-classifier models""" ... @@ -1851,7 +1853,7 @@ def llama_model_meta_count(model: llama_model_p, /) -> int: # LLAMA_API const char * llama_model_meta_key_str(enum llama_model_meta_key key); @ctypes_function("llama_model_meta_key_str", [ctypes.c_int], ctypes.c_char_p) def llama_model_meta_key_str(key: int, /) -> Optional[bytes]: - """Get sampling metadata key name. Returns None if the key is invalid.""" + """Get sampling metadata key name. Returns None if the key is invalid""" ... @@ -1922,7 +1924,7 @@ def llama_model_desc( # LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model); @ctypes_function("llama_model_ftype", [llama_model_p_ctypes], ctypes.c_int) def llama_model_ftype(model: llama_model_p, /) -> int: - """Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0.""" + """Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0""" ... @@ -2501,7 +2503,9 @@ def llama_memory_can_shift(mem: llama_memory_t, /) -> bool: # LLAMA_API size_t llama_state_get_size(struct llama_context * ctx); @ctypes_function("llama_state_get_size", [llama_context_p_ctypes], ctypes.c_size_t) def llama_state_get_size(ctx: llama_context_p, /) -> int: - """Returns the *actual* size in bytes of the state (logits, embedding and memory)""" + """Returns the *actual* size in bytes of the state + (logits, embedding and memory) + Only use when saving the state, not when restoring it, otherwise the size may be too small.""" ... @@ -3062,9 +3066,12 @@ def llama_batch_free(batch: llama_batch, /): # struct llama_batch batch); @ctypes_function("llama_encode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32) def llama_encode(ctx: llama_context_p, batch: llama_batch, /) -> int: - """Process a batch of tokens using the encoder. + """Process a batch of tokens. + In contrast to llama_decode() - this call does not use KV cache. + For encode-decoder contexts, processes the batch using the encoder. + Can store the encoder output internally for later use by the decoder's cross-attention layers. 0 - success - < 0 - error""" + < 0 - error. the memory state is restored to the state before this call""" ... @@ -3086,9 +3093,15 @@ def llama_encode(ctx: llama_context_p, batch: llama_batch, /) -> int: @ctypes_function("llama_decode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32) def llama_decode(ctx: llama_context_p, batch: llama_batch, /) -> int: """Process a batch of tokens. + Requires the context to have a memory. + For encode-decoder contexts, processes the batch using the decoder. + Positive return values does not mean a fatal error, but rather a warning. + Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context + To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max() + Upon other return values, the memory state is restored to the state before this call 0 - success 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) - 2 - aborted (processed ubatches will remain in the context's memory) + 2 - aborted (processed ubatches will remain in the context's memory) -1 - invalid input batch < -1 - fatal error (processed ubatches will remain in the context's memory)""" ... @@ -3124,7 +3137,7 @@ def llama_set_n_threads( # LLAMA_API int32_t llama_n_threads(struct llama_context * ctx); @ctypes_function("llama_n_threads", [llama_context_p_ctypes], ctypes.c_int32) def llama_n_threads(ctx: llama_context_p, /) -> int: - """Get the number of threads used for generation of a single token""" + """Get the number of threads used for generation of a single token.""" ... @@ -3132,7 +3145,7 @@ def llama_n_threads(ctx: llama_context_p, /) -> int: # LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx); @ctypes_function("llama_n_threads_batch", [llama_context_p_ctypes], ctypes.c_int32) def llama_n_threads_batch(ctx: llama_context_p, /) -> int: - """Get the number of threads used for prompt and batch processing (multiple token)""" + """Get the number of threads used for prompt and batch processing (multiple token).""" ... @@ -3141,7 +3154,8 @@ def llama_n_threads_batch(ctx: llama_context_p, /) -> int: # LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings); @ctypes_function("llama_set_embeddings", [llama_context_p_ctypes, ctypes.c_bool], None) def llama_set_embeddings(ctx: llama_context_p, embeddings: bool, /): - """Set whether the context outputs embeddings or not""" + """Set whether the context outputs embeddings or not + TODO: rename to avoid confusion with llama_get_embeddings()""" ... From 05c2b8873cc92ab84bb3cb298b565db75f2a0d65 Mon Sep 17 00:00:00 2001 From: abetlen Date: Sun, 5 Jul 2026 10:06:37 -0700 Subject: [PATCH 4/4] fix: format binding docstring --- llama_cpp/llama_cpp.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 087e1130e..d703a2fcd 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1276,8 +1276,7 @@ def llama_flash_attn_type_name(flash_attn_type: int, /) -> Optional[bytes]: # LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype); @ctypes_function("llama_ftype_name", [ctypes.c_int], ctypes.c_char_p) def llama_ftype_name(ftype: int, /) -> Optional[bytes]: - """Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" - """ + '''Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium"''' ...