Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 52 additions & 9 deletions llama_cpp/llama.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ def __init__(
yarn_beta_slow: float = 1.0,
yarn_orig_ctx: int = 0,
logits_all: bool = False,
embedding: bool = False,
embeddings: bool = False,
offload_kqv: bool = True,
flash_attn: bool = False,
op_offload: Optional[bool] = None,
Expand Down Expand Up @@ -173,7 +173,7 @@ def __init__(
yarn_beta_slow: YaRN high correction dim
yarn_orig_ctx: YaRN original context size
logits_all: Return logits for all tokens, not just the last token. Must be True for completion to return logprobs.
embedding: Embedding mode only.
embeddings: Embedding mode only. Must be set to True when using create_embedding() or embed() methods.
offload_kqv: Offload K, Q, V to GPU.
flash_attn: Use flash attention.
op_offload: offload host tensor operations to device
Expand All @@ -193,14 +193,35 @@ def __init__(
spm_infill: Use Suffix/Prefix/Middle pattern for infill (instead of Prefix/Suffix/Middle) as some models prefer this.

Raises:
ValueError: If the model path does not exist.
ValueError: If model_path is empty/None, or if n_ctx <= 0.
TypeError: If model_path is not a string.

Returns:
A Llama instance.
"""
# Parameter validation
if not model_path:
raise ValueError("model_path cannot be empty or None")

if not isinstance(model_path, str):
raise TypeError("model_path must be a string")

if n_ctx <= 0:
raise ValueError(f"n_ctx must be > 0, got {n_ctx}")

self.verbose = verbose
self._stack = contextlib.ExitStack()

# Handle deprecated 'embedding' kwarg (singular) as alias for 'embeddings' (plural)
if 'embedding' in kwargs:
warnings.warn(
"The 'embedding' parameter is deprecated. Use 'embeddings' instead. "
"Support for 'embedding' will be removed in a future version.",
DeprecationWarning,
stacklevel=2
)
embeddings = kwargs.pop('embedding')

self._stack = contextlib.ExitStack()
set_verbose(verbose)

if not Llama.__backend_initialized:
Expand Down Expand Up @@ -342,7 +363,7 @@ def __init__(
)
self.context_params.yarn_orig_ctx = yarn_orig_ctx if yarn_orig_ctx != 0 else 0
self._logits_all = logits_all if draft_model is None else True
self.context_params.embeddings = embedding # TODO: Rename to embeddings
self.context_params.embeddings = embeddings
self.context_params.offload_kqv = offload_kqv
self.context_params.flash_attn_type = (
llama_cpp.LLAMA_FLASH_ATTN_TYPE_ENABLED
Expand Down Expand Up @@ -397,7 +418,7 @@ def __init__(
self.context_params.n_batch = self.n_batch
self.context_params.n_ubatch = min(self.n_batch, n_ubatch)

if embedding:
if embeddings:
self.context_params.n_seq_max = min(
self.n_batch,
llama_cpp.llama_max_parallel_sequences(),
Expand Down Expand Up @@ -603,11 +624,16 @@ def tokenize(
special: Whether to tokenize special tokens.

Raises:
TypeError: If text is not str or bytes.
RuntimeError: If the tokenization failed.

Returns:
A list of tokens.
"""
# Type validation
if not isinstance(text, (str, bytes)):
raise TypeError(f"text must be str or bytes, got {type(text).__name__}")

return self.tokenizer_.tokenize(text, add_bos, special)

def detokenize(
Expand All @@ -626,6 +652,13 @@ def detokenize(
Returns:
The detokenized string.
"""
# Type validation
if not isinstance(tokens, list):
raise TypeError(f"tokens must be a list, got {type(tokens).__name__}")

if not all(isinstance(t, int) for t in tokens):
raise TypeError("all elements in tokens must be integers")

return self.tokenizer_.detokenize(
tokens, prev_tokens=prev_tokens, special=special
)
Expand Down Expand Up @@ -1088,7 +1121,7 @@ def embed(

if self.context_params.embeddings is False:
raise RuntimeError(
"Llama model must be created with embedding=True to call this method"
"Llama model must be created with embeddings=True to call this method"
)

if self.verbose:
Expand Down Expand Up @@ -1863,12 +1896,22 @@ def create_completion(
logit_bias: A logit bias to use.

Raises:
ValueError: If the requested tokens exceed the context window.
ValueError: If temperature < 0.0, top_p not in [0.0, 1.0], top_k < 0, or if the requested tokens exceed the context window.
RuntimeError: If the prompt fails to tokenize or the model fails to evaluate the prompt.

Returns:
Response object containing the generated text.
"""
# Parameter validation
if temperature < 0.0:
raise ValueError(f"temperature must be >= 0.0, got {temperature}")

if not 0.0 <= top_p <= 1.0:
raise ValueError(f"top_p must be in [0.0, 1.0], got {top_p}")

if top_k < 0:
raise ValueError(f"top_k must be >= 0, got {top_k}")

completion_or_chunks = self._create_completion(
prompt=prompt,
suffix=suffix,
Expand Down Expand Up @@ -2163,7 +2206,7 @@ def __getstate__(self):
yarn_beta_slow=self.context_params.yarn_beta_slow,
yarn_orig_ctx=self.context_params.yarn_orig_ctx,
logits_all=self._logits_all,
embedding=self.context_params.embeddings,
embeddings=self.context_params.embeddings,
offload_kqv=self.context_params.offload_kqv,
flash_attn=(
self.context_params.flash_attn_type
Expand Down
Loading