1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
|
#!/usr/bin/env bash
# Download a GGUF model and serve it with llama-server (OpenAI-compatible API),
# using the OVHai llama.cpp fork (CPU only). Runs INSIDE the sandbox.
#
# Every step is idempotent: it is skipped when its result already exists.
#
# scripts/olala.sh setup # find llama-server (kit) or build the fork (skipped if found/built)
# scripts/olala.sh download # download the model (skipped if the file exists)
# scripts/olala.sh serve # start llama-server in the background (skipped if running)
# scripts/olala.sh up # setup + download + serve
# scripts/olala.sh stop | restart | status | logs
#
# Configuration (environment variables, defaults below):
# OLALA_MODEL_REPO Hugging Face repo of the GGUF file
# OLALA_MODEL_FILE GGUF file name in that repo
# OLALA_MODELS_DIR where models are stored; relative paths are relative to the repo root
# (default: ./models, kept in the workspace so dropping the sandbox keeps them)
# OLALA_PORT port llama-server listens on (0.0.0.0)
# OLALA_THREADS CPU threads (default: nproc - 2; using every CPU is ~30% slower)
# OLALA_CTX context size
# OLALA_PARALLEL parallel slots
# OLALA_ALIAS model name exposed by the API (default: file name without .gguf)
# OLALA_LOAD_MODE llama-server --load-mode (default: none = read the model into RAM;
# mmap over the workspace mount is ~20x slower at generation)
# OLALA_EXTRA_ARGS extra llama-server arguments
# OLALA_SERVER_BIN llama-server to use (default: the one on PATH, e.g. from the
# kits/llama-cpp-olala kit, else the one built in OLALA_SRC_DIR)
# OLALA_SRC_DIR where the fork is cloned and built when no llama-server is found
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
OLALA_MODEL_REPO="${OLALA_MODEL_REPO:-OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF}"
OLALA_MODEL_FILE="${OLALA_MODEL_FILE:-Olala-v1.0-mini-7A1B-q8_0-safe2.gguf}"
OLALA_MODELS_DIR="${OLALA_MODELS_DIR:-./models}"
OLALA_PORT="${OLALA_PORT:-8080}"
OLALA_THREADS="${OLALA_THREADS:-$(( $(nproc) > 2 ? $(nproc) - 2 : 1 ))}"
OLALA_CTX="${OLALA_CTX:-32768}"
OLALA_PARALLEL="${OLALA_PARALLEL:-4}"
OLALA_ALIAS="${OLALA_ALIAS:-${OLALA_MODEL_FILE%.gguf}}"
OLALA_LOAD_MODE="${OLALA_LOAD_MODE:-none}"
OLALA_EXTRA_ARGS="${OLALA_EXTRA_ARGS:-}"
OLALA_SRC_DIR="${OLALA_SRC_DIR:-$HOME/olala/llama.cpp-olala}"
OLALA_REPO_URL="${OLALA_REPO_URL:-https://github.com/OVHai-LLM/llama.cpp.git}"
case "$OLALA_MODELS_DIR" in
/*) ;;
*) OLALA_MODELS_DIR="$REPO_ROOT/${OLALA_MODELS_DIR#./}" ;;
esac
RUN_DIR="$HOME/.olala"
PID_FILE="$RUN_DIR/llama-server.pid"
LOG_FILE="$RUN_DIR/llama-server.log"
BUILT_BIN="$OLALA_SRC_DIR/build/bin/llama-server"
MODEL_PATH="$OLALA_MODELS_DIR/$OLALA_MODEL_FILE"
log() { printf '[olala] %s\n' "$*" >&2; }
# Prints the llama-server to use, or nothing.
server_bin() {
if [ -n "${OLALA_SERVER_BIN:-}" ]; then
echo "$OLALA_SERVER_BIN"
elif command -v llama-server >/dev/null; then
command -v llama-server
elif [ -x "$BUILT_BIN" ]; then
echo "$BUILT_BIN"
fi
}
setup() {
local bin
bin="$(server_bin)"
if [ -n "$bin" ]; then
log "llama-server found: $bin"
return
fi
log "no llama-server on PATH (kits/llama-cpp-olala not installed): building the fork"
if ! command -v cmake >/dev/null || ! command -v g++ >/dev/null; then
log "installing build tools (build-essential, cmake, libssl-dev)"
sudo apt-get update -qq
sudo apt-get install -y -qq build-essential cmake libssl-dev pkg-config git >/dev/null
fi
if [ ! -d "$OLALA_SRC_DIR/.git" ]; then
log "cloning $OLALA_REPO_URL"
mkdir -p "$(dirname "$OLALA_SRC_DIR")"
git clone --depth 1 "$OLALA_REPO_URL" "$OLALA_SRC_DIR"
fi
log "building (CPU only)"
# -DLLAMA_USE_PREBUILT_UI=OFF: the prebuilt web UI does not match the fork (per the model card).
cmake -S "$OLALA_SRC_DIR" -B "$OLALA_SRC_DIR/build" -DCMAKE_BUILD_TYPE=Release -DLLAMA_USE_PREBUILT_UI=OFF >/dev/null
cmake --build "$OLALA_SRC_DIR/build" -j "$(nproc)" --target llama-server llama-cli >/dev/null
log "built: $BUILT_BIN"
}
download() {
if [ -s "$MODEL_PATH" ]; then
log "model already present: $MODEL_PATH"
return
fi
if ! command -v hf >/dev/null; then
log "installing the Hugging Face CLI"
uv tool install "huggingface_hub[cli]" >/dev/null
export PATH="$HOME/.local/bin:$PATH"
fi
mkdir -p "$OLALA_MODELS_DIR"
log "downloading $OLALA_MODEL_REPO / $OLALA_MODEL_FILE -> $OLALA_MODELS_DIR"
hf download "$OLALA_MODEL_REPO" "$OLALA_MODEL_FILE" --local-dir "$OLALA_MODELS_DIR"
}
is_running() {
[ -f "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null
}
serve() {
if is_running; then
log "llama-server already running (pid $(cat "$PID_FILE")) on port $OLALA_PORT"
return
fi
local bin
bin="$(server_bin)"
[ -n "$bin" ] || { log "llama-server not found: run '$0 setup'"; exit 1; }
[ -s "$MODEL_PATH" ] || { log "model missing: run '$0 download'"; exit 1; }
mkdir -p "$RUN_DIR"
log "starting $bin: $MODEL_PATH on 0.0.0.0:$OLALA_PORT (threads=$OLALA_THREADS ctx=$OLALA_CTX load-mode=$OLALA_LOAD_MODE)"
# Flags from the model card's "CPU" server example; --host 0.0.0.0 so the port can be published.
# The inner shell writes its own pid then execs llama-server, so the pid file is exact
# (setsid may fork, which made $! point at a process that had already exited).
# shellcheck disable=SC2016,SC2086
nohup setsid bash -c 'echo $$ >"$0"; exec "$@"' "$PID_FILE" \
"$bin" \
-m "$MODEL_PATH" \
--load-mode "$OLALA_LOAD_MODE" \
-t "$OLALA_THREADS" -fa 1 -nr -ctk q8_0 -ctv q8_0 \
-c "$OLALA_CTX" -np "$OLALA_PARALLEL" \
--temp 0.8 --top-p 0.9 --top-k 40 \
--alias "$OLALA_ALIAS" \
--host 0.0.0.0 --port "$OLALA_PORT" \
$OLALA_EXTRA_ARGS \
>"$LOG_FILE" 2>&1 </dev/null &
log "waiting for /health (log: $LOG_FILE)"
for _ in $(seq 1 300); do
if curl -fsS "http://127.0.0.1:$OLALA_PORT/health" >/dev/null 2>&1; then
log "ready: http://127.0.0.1:$OLALA_PORT/v1"
return
fi
if [ -f "$PID_FILE" ] && ! is_running; then
log "llama-server exited, last lines:"
tail -20 "$LOG_FILE" >&2
exit 1
fi
sleep 1
done
log "not ready after 300 s, see $LOG_FILE"
exit 1
}
stop() {
if is_running; then
local pid
pid="$(cat "$PID_FILE")"
kill "$pid"
for _ in $(seq 1 30); do kill -0 "$pid" 2>/dev/null || break; sleep 0.5; done
kill -0 "$pid" 2>/dev/null && kill -9 "$pid"
rm -f "$PID_FILE"
log "stopped"
else
log "not running"
fi
}
status() {
if is_running; then
log "running (pid $(cat "$PID_FILE")), model $MODEL_PATH, port $OLALA_PORT"
curl -fsS "http://127.0.0.1:$OLALA_PORT/health" && echo
else
log "not running"
fi
}
case "${1:-up}" in
setup) setup ;;
download) download ;;
serve) serve ;;
up) setup; download; serve ;;
stop) stop ;;
restart) stop; serve ;;
status) status ;;
logs) tail -f "$LOG_FILE" ;;
*) echo "usage: $0 {up|setup|download|serve|stop|restart|status|logs}" >&2; exit 2 ;;
esac
|