diff --git a/ollama/docker-compose.yml b/ollama/docker-compose.yml index 83ab5e2..4b12c22 100644 --- a/ollama/docker-compose.yml +++ b/ollama/docker-compose.yml @@ -8,6 +8,8 @@ services: volumes: - "/srv/docker/ollama/data/models:/models:ro" - "./llama-swap.yaml:/etc/llama-swap/config.yaml:ro" + environment: + - HIP_VISIBLE_DEVICES=0 devices: - "/dev/kfd:/dev/kfd" - "/dev/dri:/dev/dri" diff --git a/ollama/llama-swap.yaml b/ollama/llama-swap.yaml index d9c919c..66a2b25 100644 --- a/ollama/llama-swap.yaml +++ b/ollama/llama-swap.yaml @@ -2,25 +2,10 @@ healthCheckTimeout: 180 logLevel: info models: - "qwen3.5:9b": - cmd: | - /app/llama-server - --host 0.0.0.0 --port ${PORT} - --model /models/qwen3.5-9b.gguf - --alias qwen3.5:9b - --n-gpu-layers 999 - --ctx-size 8192 - --temp 1 --top-k 20 --top-p 0.95 --presence-penalty 1.5 - - "qwen3.5:9bctxSmall": - cmd: | - /app/llama-server - --host 0.0.0.0 --port ${PORT} - --model /models/qwen3.5-9b.gguf - --alias qwen3.5:9bctxSmall - --n-gpu-layers 999 - --ctx-size 131072 - --temp 1 --top-k 20 --top-p 0.95 --presence-penalty 1.5 + # NOTE: qwen3.5:9b and qwen3.5:9bctxSmall are disabled — the GGUF extracted + # from Ollama uses the old mrope_sections format (3 elements) which this + # llama.cpp build rejects. Download a fresh quantization from HuggingFace + # (e.g. bartowski/Qwen3.5-9B-GGUF) and add them back. "hf.co/HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive:q4_k_m": cmd: |