bots-garden/houlapublic Fork
⑂ v0.0.1
Commits
⬇ Clone ▾
git clone https://git.rickub.com/bots-garden/houla.git
git clone ssh://git@rickub.com/bots-garden/houla.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Download ZIP
🛟 Updated. 1dfc968Unverified · on v0.0.1 · k33g · 4h ago
olala.sh · 190 lines · 7.0 KBBash Blame HistoryRaw
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
#!/usr/bin/env bash
# Download a GGUF model and serve it with llama-server (OpenAI-compatible API),
# using the OVHai llama.cpp fork (CPU only). Runs INSIDE the sandbox.
#
# Every step is idempotent: it is skipped when its result already exists.
#
#   scripts/olala.sh setup      # find llama-server (kit) or build the fork (skipped if found/built)
#   scripts/olala.sh download   # download the model (skipped if the file exists)
#   scripts/olala.sh serve      # start llama-server in the background (skipped if running)
#   scripts/olala.sh up         # setup + download + serve
#   scripts/olala.sh stop | restart | status | logs
#
# Configuration (environment variables, defaults below):
#   OLALA_MODEL_REPO   Hugging Face repo of the GGUF file
#   OLALA_MODEL_FILE   GGUF file name in that repo
#   OLALA_MODELS_DIR   where models are stored; relative paths are relative to the repo root
#                      (default: ./models, kept in the workspace so dropping the sandbox keeps them)
#   OLALA_PORT         port llama-server listens on (0.0.0.0)
#   OLALA_THREADS      CPU threads (default: nproc - 2; using every CPU is ~30% slower)
#   OLALA_CTX          context size
#   OLALA_PARALLEL     parallel slots
#   OLALA_ALIAS        model name exposed by the API (default: file name without .gguf)
#   OLALA_LOAD_MODE    llama-server --load-mode (default: none = read the model into RAM;
#                      mmap over the workspace mount is ~20x slower at generation)
#   OLALA_EXTRA_ARGS   extra llama-server arguments
#   OLALA_SERVER_BIN   llama-server to use (default: the one on PATH, e.g. from the
#                      kits/llama-cpp-olala kit, else the one built in OLALA_SRC_DIR)
#   OLALA_SRC_DIR      where the fork is cloned and built when no llama-server is found
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"

OLALA_MODEL_REPO="${OLALA_MODEL_REPO:-OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF}"
OLALA_MODEL_FILE="${OLALA_MODEL_FILE:-Olala-v1.0-mini-7A1B-q8_0-safe2.gguf}"
OLALA_MODELS_DIR="${OLALA_MODELS_DIR:-./models}"
OLALA_PORT="${OLALA_PORT:-8080}"
OLALA_THREADS="${OLALA_THREADS:-$(( $(nproc) > 2 ? $(nproc) - 2 : 1 ))}"
OLALA_CTX="${OLALA_CTX:-32768}"
OLALA_PARALLEL="${OLALA_PARALLEL:-4}"
OLALA_ALIAS="${OLALA_ALIAS:-${OLALA_MODEL_FILE%.gguf}}"
OLALA_LOAD_MODE="${OLALA_LOAD_MODE:-none}"
OLALA_EXTRA_ARGS="${OLALA_EXTRA_ARGS:-}"
OLALA_SRC_DIR="${OLALA_SRC_DIR:-$HOME/olala/llama.cpp-olala}"
OLALA_REPO_URL="${OLALA_REPO_URL:-https://github.com/OVHai-LLM/llama.cpp.git}"

case "$OLALA_MODELS_DIR" in
  /*) ;;
  *) OLALA_MODELS_DIR="$REPO_ROOT/${OLALA_MODELS_DIR#./}" ;;
esac

RUN_DIR="$HOME/.olala"
PID_FILE="$RUN_DIR/llama-server.pid"
LOG_FILE="$RUN_DIR/llama-server.log"
BUILT_BIN="$OLALA_SRC_DIR/build/bin/llama-server"
MODEL_PATH="$OLALA_MODELS_DIR/$OLALA_MODEL_FILE"

log() { printf '[olala] %s\n' "$*" >&2; }

# Prints the llama-server to use, or nothing.
server_bin() {
  if [ -n "${OLALA_SERVER_BIN:-}" ]; then
    echo "$OLALA_SERVER_BIN"
  elif command -v llama-server >/dev/null; then
    command -v llama-server
  elif [ -x "$BUILT_BIN" ]; then
    echo "$BUILT_BIN"
  fi
}

setup() {
  local bin
  bin="$(server_bin)"
  if [ -n "$bin" ]; then
    log "llama-server found: $bin"
    return
  fi
  log "no llama-server on PATH (kits/llama-cpp-olala not installed): building the fork"
  if ! command -v cmake >/dev/null || ! command -v g++ >/dev/null; then
    log "installing build tools (build-essential, cmake, libssl-dev)"
    sudo apt-get update -qq
    sudo apt-get install -y -qq build-essential cmake libssl-dev pkg-config git >/dev/null
  fi
  if [ ! -d "$OLALA_SRC_DIR/.git" ]; then
    log "cloning $OLALA_REPO_URL"
    mkdir -p "$(dirname "$OLALA_SRC_DIR")"
    git clone --depth 1 "$OLALA_REPO_URL" "$OLALA_SRC_DIR"
  fi
  log "building (CPU only)"
  # -DLLAMA_USE_PREBUILT_UI=OFF: the prebuilt web UI does not match the fork (per the model card).
  cmake -S "$OLALA_SRC_DIR" -B "$OLALA_SRC_DIR/build" -DCMAKE_BUILD_TYPE=Release -DLLAMA_USE_PREBUILT_UI=OFF >/dev/null
  cmake --build "$OLALA_SRC_DIR/build" -j "$(nproc)" --target llama-server llama-cli >/dev/null
  log "built: $BUILT_BIN"
}

download() {
  if [ -s "$MODEL_PATH" ]; then
    log "model already present: $MODEL_PATH"
    return
  fi
  if ! command -v hf >/dev/null; then
    log "installing the Hugging Face CLI"
    uv tool install "huggingface_hub[cli]" >/dev/null
    export PATH="$HOME/.local/bin:$PATH"
  fi
  mkdir -p "$OLALA_MODELS_DIR"
  log "downloading $OLALA_MODEL_REPO / $OLALA_MODEL_FILE -> $OLALA_MODELS_DIR"
  hf download "$OLALA_MODEL_REPO" "$OLALA_MODEL_FILE" --local-dir "$OLALA_MODELS_DIR"
}

is_running() {
  [ -f "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null
}

serve() {
  if is_running; then
    log "llama-server already running (pid $(cat "$PID_FILE")) on port $OLALA_PORT"
    return
  fi
  local bin
  bin="$(server_bin)"
  [ -n "$bin" ] || { log "llama-server not found: run '$0 setup'"; exit 1; }
  [ -s "$MODEL_PATH" ] || { log "model missing: run '$0 download'"; exit 1; }
  mkdir -p "$RUN_DIR"
  log "starting $bin: $MODEL_PATH on 0.0.0.0:$OLALA_PORT (threads=$OLALA_THREADS ctx=$OLALA_CTX load-mode=$OLALA_LOAD_MODE)"
  # Flags from the model card's "CPU" server example; --host 0.0.0.0 so the port can be published.
  # The inner shell writes its own pid then execs llama-server, so the pid file is exact
  # (setsid may fork, which made $! point at a process that had already exited).
  # shellcheck disable=SC2016,SC2086
  nohup setsid bash -c 'echo $$ >"$0"; exec "$@"' "$PID_FILE" \
    "$bin" \
    -m "$MODEL_PATH" \
    --load-mode "$OLALA_LOAD_MODE" \
    -t "$OLALA_THREADS" -fa 1 -nr -ctk q8_0 -ctv q8_0 \
    -c "$OLALA_CTX" -np "$OLALA_PARALLEL" \
    --temp 0.8 --top-p 0.9 --top-k 40 \
    --alias "$OLALA_ALIAS" \
    --host 0.0.0.0 --port "$OLALA_PORT" \
    $OLALA_EXTRA_ARGS \
    >"$LOG_FILE" 2>&1 </dev/null &
  log "waiting for /health (log: $LOG_FILE)"
  for _ in $(seq 1 300); do
    if curl -fsS "http://127.0.0.1:$OLALA_PORT/health" >/dev/null 2>&1; then
      log "ready: http://127.0.0.1:$OLALA_PORT/v1"
      return
    fi
    if [ -f "$PID_FILE" ] && ! is_running; then
      log "llama-server exited, last lines:"
      tail -20 "$LOG_FILE" >&2
      exit 1
    fi
    sleep 1
  done
  log "not ready after 300 s, see $LOG_FILE"
  exit 1
}

stop() {
  if is_running; then
    local pid
    pid="$(cat "$PID_FILE")"
    kill "$pid"
    for _ in $(seq 1 30); do kill -0 "$pid" 2>/dev/null || break; sleep 0.5; done
    kill -0 "$pid" 2>/dev/null && kill -9 "$pid"
    rm -f "$PID_FILE"
    log "stopped"
  else
    log "not running"
  fi
}

status() {
  if is_running; then
    log "running (pid $(cat "$PID_FILE")), model $MODEL_PATH, port $OLALA_PORT"
    curl -fsS "http://127.0.0.1:$OLALA_PORT/health" && echo
  else
    log "not running"
  fi
}

case "${1:-up}" in
  setup) setup ;;
  download) download ;;
  serve) serve ;;
  up) setup; download; serve ;;
  stop) stop ;;
  restart) stop; serve ;;
  status) status ;;
  logs) tail -f "$LOG_FILE" ;;
  *) echo "usage: $0 {up|setup|download|serve|stop|restart|status|logs}" >&2; exit 2 ;;
esac