File size: 2,491 Bytes
9a16162 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 |
#!/usr/bin/bash
# requires the gum package
# GUM: https://github.com/charmbracelet/gum#installation
# Uses conda & koboldcpp built from source
# Assumes you are using nvidia
# activates conda env named kobold
# if you get a conda error try this: `conda install conda-forge::libstdcxx-ng`
eval "$(conda shell.bash hook)"
conda activate kobold
# USER variables
# CHANGE
KOBOLD_DIR=~/Git/koboldcpp/
MODEL_FOLDER_DIR=~/Downloads/GGUF
API_PORT=8001
API_HOST=192.168.1.20
BLAS_THREADS=16
THREADS=8
gum style \
--foreground 212 --border-foreground 212 --border double \
--align center --width 100 --margin "1 2" --padding "2 4" --bold \
'Beep...Boop... Script created by Virt-io'
# refer to https://github.com/LostRuins/koboldcpp?tab=readme-ov-file#osx-and-linux-manual-compiling for dependencies
# CD to koboldcpp dir
cd $KOBOLD_DIR
# edit make flags to fit your needs
echo "Git pull & build?"
KOBOLD_BUILD=$(gum choose --selected.bold --selected.underline "NO" "YES")
if [ "$KOBOLD_BUILD" = "YES" ]; then
gum spin --spinner monkey --title "Pulling latest changes..." -- git pull && make clean
gum spin --show-output --spinner monkey --title "Re-building Koboldcpp..." -- make -j8 LLAMA_NO_LLAMAFILE=0 LLAMA_CUDA=1 LLAMA_CUBLAS=1 LLAMA_OPENBLAS=1 LLAMA_FAST=1 LLAMA_NO_CCACHE=1 LLAMA_CUDA_MMV_Y=2 LLAMA_CUDA_DMMV_X=64 LLAMA_CUDA_DMMV_F16=1 LLAMA_CUDA_F16=1 LLAMA_NATIVE=1 LLAMA_LTO=1 LLAMA_AVX=1 LLAMA_AVX2=1 LLAMA_FMA=1 LLAMA_F16C=1
echo "Rebuild completed"
elif [ "$KOBOLD_BUILD" = "NO" ]; then
echo "Skipping rebuild."
fi
# lists models
echo "Select Model"
MODEL=$(gum choose --height=30 --selected.bold --selected.underline $(ls $MODEL_FOLDER_DIR))
# uncomment if you use sharded models. It will take the first file in a dir and load it.
#SHARDED_MODEL=$(ls -p $MODEL_FOLDER_DIR/$MODEL | grep -v / | head -1)
#MODEL=$MODEL/$SHARDED_MODEL
echo "$MODEL has been selected"
echo "Layers to Offload"
LAYERS=$(gum input --placeholder "99")
echo "$LAYERS layers have been offloaded"
echo "Context Size"
CONTEXT=$(gum choose --height=10 --selected.bold --selected.underline "4096" "8192" "12288" "16384" "32768")
echo "Using a context size of $CONTEXT"
# combined user flags
USER_FLAGS="--host $API_HOST --port $API_PORT --flashattention --blasbatchsize 2048 --threads $THREADS --blasthreads $BLAS_THREADS --usecublas normal --contextsize $CONTEXT --gpulayers $LAYERS --model $MODEL_FOLDER_DIR/$MODEL"
# runs koboldcpp
python koboldcpp.py --skiplauncher $USER_FLAGS
|