From 3faa07e99df9708605ea687e8cd90b1c40fa0276 Mon Sep 17 00:00:00 2001
From: Maximilian Kimmich <maximilian.kimmich@ims.uni-stuttgart.de>
Date: Tue, 1 Oct 2024 18:19:37 +0200
Subject: [PATCH] Offload all layers to GPU for llama_cpp
---
evoprompt/models.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/evoprompt/models.py b/evoprompt/models.py
index 2fa83f6..41b88a8 100644
--- a/evoprompt/models.py
+++ b/evoprompt/models.py
@@ -209,7 +209,7 @@ class Llama(LLMModel):
chat_handler: str | None = None,
verbose: int | bool = False,
llama_verbose: bool = False,
- n_gpu_layers: int = 60,
+ n_gpu_layers: int = -1,
n_threads: int = 8,
n_ctx: int = 8192,
**kwargs,
--
GitLab