From 6c0d57266dbb1d75a8b83bdee93402b69d7a4aa7 Mon Sep 17 00:00:00 2001 From: z3rOR0ne Date: Fri, 27 Sep 2024 16:26:11 -0700 Subject: [PATCH] :wrench: Added shell script for interacting with llama.cpp --- llama/llama.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 llama/llama.sh diff --git a/llama/llama.sh b/llama/llama.sh new file mode 100755 index 00000000..259c2ee2 --- /dev/null +++ b/llama/llama.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +#NOTE: You can adjust the behavior of the chat version by assigning it a role + +# Simple AI assistant +# ./main -ngl 32 -m ./models/TheBloke/Llama-2-7B-Chat-GGUF/llama-2-7b-chat.Q4_K_M.gguf --color -c 4096 --in-suffix "AI Assistant: " --temp 0.8 --repeat_penalty 1.1 --threads 6 -n -1 -i -ins + +# Good chatbot, but needs CUDA to run more efficiently +#--n-gpu-layers 15000 +# ./main -ngl 20 -m ./models/TheBloke/CodeLlama-13B-oasst-sft-v10-GGUF/codellama-13b-oasst-sft-v10.Q4_K_M.gguf --color -c 4096 --temp 0.1 --repeat_penalty 1.1 --threads 6 -n -1 -i -ins + +# Good for chat, but also has better storytelling capabilities +# ./main -ngl 20 -m ./models/TheBloke/MythoMax-L2-13B-GGUF/mythomax-l2-13b.Q4_K_M.gguf --color -c 4096 --temp 0.1 --repeat_penalty 1.1 --threads 6 -n -1 -i -ins + +# Silicon Maid (medium) +# ./main -ngl 30 -m ./models/TheBloke/Silicon-Maid-7B-GGUF/silicon-maid-7b.Q4_K_M.gguf --color -c 4096 --temp 0.1 --repeat_penalty 1.1 --threads 6 -n -1 -i -ins + +# Silicon Maid (large) +# ./main -ngl 25 -m ./models/TheBloke/Silicon-Maid-7B-GGUF/silicon-maid-7b.Q5_K_M.gguf --color -c 4096 --temp 0.1 --repeat_penalty 1.1 --threads 6 -n -1 -i -ins + +# Kunoichi-7B (medium) +./main -ngl 30 -m ./models/TheBloke/Kunoichi-7B-GGUF/kunoichi-7b.Q4_K_M.gguf --color -c 4096 --temp 0.1 --repeat_penalty 1.1 --threads 6 -n -1 -i -ins