🛟 Updated.Unverified
1dfc968 added
.gitignore +8 -0 | new file mode 100644 | ||
| @@ -0,0 +1,8 @@ | ||
| 1 | +set-hf-token.md | |
| 2 | + | |
| 3 | +# Downloaded GGUF models (scripts/olala.sh, default OLALA_MODELS_DIR) | |
| 4 | +/models/ | |
| 5 | +*.gguf | |
| 6 | +**/.cache/huggingface/ | |
| 7 | +.claude | |
| 8 | +.memory | |
| new file mode 100644 | |||
| @@ -0,0 +1,8 @@ | |||
| 1 | +set-hf-token.md | ||
| 2 | + | ||
| 3 | +# Downloaded GGUF models (scripts/olala.sh, default OLALA_MODELS_DIR) | ||
| 4 | +/models/ | ||
| 5 | +*.gguf | ||
| 6 | +**/.cache/huggingface/ | ||
| 7 | +.claude | ||
| 8 | +.memory | ||
added
.tickets/config.yaml +2 -0 | new file mode 100644 | ||
| @@ -0,0 +1,2 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +version: 1 | |
| new file mode 100644 | |||
| @@ -0,0 +1,2 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +version: 1 | ||
added
.tickets/epics.yaml +6 -0 | new file mode 100644 | ||
| @@ -0,0 +1,6 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +epics: | |
| 3 | + - title: Olala in sbx | |
| 4 | + state: open | |
| 5 | + color: "#5319e7" | |
| 6 | + description: Run OVHai's Olala-v1.0-mini-7A1B (GGUF, llama.cpp fork, CPU only) in a Docker sandbox and serve it to the host | |
| new file mode 100644 | |||
| @@ -0,0 +1,6 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +epics: | ||
| 3 | + - title: Olala in sbx | ||
| 4 | + state: open | ||
| 5 | + color: "#5319e7" | ||
| 6 | + description: Run OVHai's Olala-v1.0-mini-7A1B (GGUF, llama.cpp fork, CPU only) in a Docker sandbox and serve it to the host | ||
added
.tickets/issues/0001-initialize.yaml +36 -0 | new file mode 100644 | ||
| @@ -0,0 +1,36 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +id: 1 | |
| 3 | +title: initialize | |
| 4 | +state: open | |
| 5 | +author: | |
| 6 | + name: k33g | |
| 7 | + email: ph.charriere@gmail.com | |
| 8 | +createdAt: 2026-11-21T06:18:39.078Z | |
| 9 | +updatedAt: 2026-11-21T07:52:58.000Z | |
| 10 | +labels: [] | |
| 11 | +body: | | |
| 12 | + OVH released a new model: https://huggingface.co/OVHaiLLM/Olala-v1.0-mini-7A1B | |
| 13 | + | |
| 14 | + To run it in GGUF format we need a fork of llama.cpp, explained here: https://huggingface.co/OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF | |
| 15 | + (the links to the fork are on that page, along with how to build and run it) | |
| 16 | + | |
| 17 | + Everything happens in an sbx sandbox, so we use the build without GPU (CPU only). | |
| 18 | + | |
| 19 | + Download the model: Olala-v1.0-mini-7A1B-q8_0-safe2.gguf | |
| 20 | + | |
| 21 | + then serve it so it can be reached from outside. | |
| 22 | + | |
| 23 | + Create a sbxenv.yaml file so this is easy to repeat: it must handle the model being already downloaded or not, expose a port, and optionally allow downloading and serving another model. | |
| 24 | + | |
| 25 | + And give the curl commands to test it. | |
| 26 | +tasks: [] | |
| 27 | +comments: | |
| 28 | + - id: 1 | |
| 29 | + author: | |
| 30 | + name: k33g | |
| 31 | + email: ph.charriere@gmail.com | |
| 32 | + createdAt: 2026-11-21T06:37:13.000Z | |
| 33 | + body: | | |
| 34 | + Split into #2 to #8 (epic ❖ Olala in sbx). Done in sandbox `houla-houla`: fork built, model downloaded, llama-server running and answering on port 8080 (~14 tokens/s, aarch64 CPU). | |
| 35 | + Still to do on the host: publish the port (`sbx ports houla-houla --publish 8080:8080`) and try `sbx env create sbxenv.houla.yaml` (#6). Kit: #7. | |
| 36 | + See [docs/olala-sbx.en.md](docs/olala-sbx.en.md). | |
| new file mode 100644 | |||
| @@ -0,0 +1,36 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +id: 1 | ||
| 3 | +title: initialize | ||
| 4 | +state: open | ||
| 5 | +author: | ||
| 6 | + name: k33g | ||
| 7 | + email: ph.charriere@gmail.com | ||
| 8 | +createdAt: 2026-11-21T06:18:39.078Z | ||
| 9 | +updatedAt: 2026-11-21T07:52:58.000Z | ||
| 10 | +labels: [] | ||
| 11 | +body: | | ||
| 12 | + OVH released a new model: https://huggingface.co/OVHaiLLM/Olala-v1.0-mini-7A1B | ||
| 13 | + | ||
| 14 | + To run it in GGUF format we need a fork of llama.cpp, explained here: https://huggingface.co/OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF | ||
| 15 | + (the links to the fork are on that page, along with how to build and run it) | ||
| 16 | + | ||
| 17 | + Everything happens in an sbx sandbox, so we use the build without GPU (CPU only). | ||
| 18 | + | ||
| 19 | + Download the model: Olala-v1.0-mini-7A1B-q8_0-safe2.gguf | ||
| 20 | + | ||
| 21 | + then serve it so it can be reached from outside. | ||
| 22 | + | ||
| 23 | + Create a sbxenv.yaml file so this is easy to repeat: it must handle the model being already downloaded or not, expose a port, and optionally allow downloading and serving another model. | ||
| 24 | + | ||
| 25 | + And give the curl commands to test it. | ||
| 26 | +tasks: [] | ||
| 27 | +comments: | ||
| 28 | + - id: 1 | ||
| 29 | + author: | ||
| 30 | + name: k33g | ||
| 31 | + email: ph.charriere@gmail.com | ||
| 32 | + createdAt: 2026-11-21T06:37:13.000Z | ||
| 33 | + body: | | ||
| 34 | + Split into #2 to #8 (epic ❖ Olala in sbx). Done in sandbox `houla-houla`: fork built, model downloaded, llama-server running and answering on port 8080 (~14 tokens/s, aarch64 CPU). | ||
| 35 | + Still to do on the host: publish the port (`sbx ports houla-houla --publish 8080:8080`) and try `sbx env create sbxenv.houla.yaml` (#6). Kit: #7. | ||
| 36 | + See [docs/olala-sbx.en.md](docs/olala-sbx.en.md). | ||
added
.tickets/issues/0002-build-the-ovhai-llama-cpp-fork-cpu-only.yaml +60 -0 | new file mode 100644 | ||
| @@ -0,0 +1,60 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +id: 2 | |
| 3 | +title: Build the OVHai llama.cpp fork (CPU only) | |
| 4 | +state: closed | |
| 5 | +author: | |
| 6 | + name: k33g | |
| 7 | + email: ph.charriere@gmail.com | |
| 8 | +createdAt: 2026-11-21T06:35:00.000Z | |
| 9 | +updatedAt: 2026-11-21T06:37:13.000Z | |
| 10 | +labels: | |
| 11 | + - kind::chore | |
| 12 | + - priority::high | |
| 13 | +epic: Olala in sbx | |
| 14 | +related: | |
| 15 | + - 1 | |
| 16 | +body: | | |
| 17 | + Upstream llama.cpp does not know the `olala` architecture: the fork https://github.com/OVHai-LLM/llama.cpp is required | |
| 18 | + (see [the GGUF model card](https://huggingface.co/OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF)). | |
| 19 | + | |
| 20 | + CPU only build (the sandbox has no GPU; it is **aarch64**, 18 CPUs, 31 GB RAM — the card benchmarks x86 CPUs). | |
| 21 | + | |
| 22 | + ```bash | |
| 23 | + cmake -B build -DCMAKE_BUILD_TYPE=Release -DLLAMA_USE_PREBUILT_UI=OFF | |
| 24 | + cmake --build build -j --target llama-server llama-cli | |
| 25 | + ``` | |
| 26 | + | |
| 27 | + ## Acceptance criteria | |
| 28 | + - `llama-server` and `llama-cli` built in `~/olala/llama.cpp-olala/build/bin`. | |
| 29 | +tasks: | |
| 30 | + - id: 1 | |
| 31 | + title: Install build-essential, cmake, libssl-dev | |
| 32 | + state: closed | |
| 33 | + priority: high | |
| 34 | + author: | |
| 35 | + name: k33g | |
| 36 | + email: ph.charriere@gmail.com | |
| 37 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 38 | + updatedAt: 2026-11-21T06:35:00.000Z | |
| 39 | + closedAt: 2026-11-21T06:35:00.000Z | |
| 40 | + - id: 2 | |
| 41 | + title: Clone the fork (depth 1, commit 55c22dc) | |
| 42 | + state: closed | |
| 43 | + priority: high | |
| 44 | + author: | |
| 45 | + name: k33g | |
| 46 | + email: ph.charriere@gmail.com | |
| 47 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 48 | + updatedAt: 2026-11-21T06:35:00.000Z | |
| 49 | + closedAt: 2026-11-21T06:35:00.000Z | |
| 50 | + - id: 3 | |
| 51 | + title: Build llama-server and llama-cli | |
| 52 | + state: closed | |
| 53 | + priority: high | |
| 54 | + author: | |
| 55 | + name: k33g | |
| 56 | + email: ph.charriere@gmail.com | |
| 57 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 58 | + updatedAt: 2026-11-21T06:35:00.000Z | |
| 59 | + closedAt: 2026-11-21T06:35:00.000Z | |
| 60 | +comments: [] | |
| new file mode 100644 | |||
| @@ -0,0 +1,60 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +id: 2 | ||
| 3 | +title: Build the OVHai llama.cpp fork (CPU only) | ||
| 4 | +state: closed | ||
| 5 | +author: | ||
| 6 | + name: k33g | ||
| 7 | + email: ph.charriere@gmail.com | ||
| 8 | +createdAt: 2026-11-21T06:35:00.000Z | ||
| 9 | +updatedAt: 2026-11-21T06:37:13.000Z | ||
| 10 | +labels: | ||
| 11 | + - kind::chore | ||
| 12 | + - priority::high | ||
| 13 | +epic: Olala in sbx | ||
| 14 | +related: | ||
| 15 | + - 1 | ||
| 16 | +body: | | ||
| 17 | + Upstream llama.cpp does not know the `olala` architecture: the fork https://github.com/OVHai-LLM/llama.cpp is required | ||
| 18 | + (see [the GGUF model card](https://huggingface.co/OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF)). | ||
| 19 | + | ||
| 20 | + CPU only build (the sandbox has no GPU; it is **aarch64**, 18 CPUs, 31 GB RAM — the card benchmarks x86 CPUs). | ||
| 21 | + | ||
| 22 | + ```bash | ||
| 23 | + cmake -B build -DCMAKE_BUILD_TYPE=Release -DLLAMA_USE_PREBUILT_UI=OFF | ||
| 24 | + cmake --build build -j --target llama-server llama-cli | ||
| 25 | + ``` | ||
| 26 | + | ||
| 27 | + ## Acceptance criteria | ||
| 28 | + - `llama-server` and `llama-cli` built in `~/olala/llama.cpp-olala/build/bin`. | ||
| 29 | +tasks: | ||
| 30 | + - id: 1 | ||
| 31 | + title: Install build-essential, cmake, libssl-dev | ||
| 32 | + state: closed | ||
| 33 | + priority: high | ||
| 34 | + author: | ||
| 35 | + name: k33g | ||
| 36 | + email: ph.charriere@gmail.com | ||
| 37 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 38 | + updatedAt: 2026-11-21T06:35:00.000Z | ||
| 39 | + closedAt: 2026-11-21T06:35:00.000Z | ||
| 40 | + - id: 2 | ||
| 41 | + title: Clone the fork (depth 1, commit 55c22dc) | ||
| 42 | + state: closed | ||
| 43 | + priority: high | ||
| 44 | + author: | ||
| 45 | + name: k33g | ||
| 46 | + email: ph.charriere@gmail.com | ||
| 47 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 48 | + updatedAt: 2026-11-21T06:35:00.000Z | ||
| 49 | + closedAt: 2026-11-21T06:35:00.000Z | ||
| 50 | + - id: 3 | ||
| 51 | + title: Build llama-server and llama-cli | ||
| 52 | + state: closed | ||
| 53 | + priority: high | ||
| 54 | + author: | ||
| 55 | + name: k33g | ||
| 56 | + email: ph.charriere@gmail.com | ||
| 57 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 58 | + updatedAt: 2026-11-21T06:35:00.000Z | ||
| 59 | + closedAt: 2026-11-21T06:35:00.000Z | ||
| 60 | +comments: [] | ||
added
.tickets/issues/0003-download-olala-v1-0-mini-7a1b-q8-0-safe2-gguf.yaml +51 -0 | new file mode 100644 | ||
| @@ -0,0 +1,51 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +id: 3 | |
| 3 | +title: Download Olala-v1.0-mini-7A1B-q8_0-safe2.gguf | |
| 4 | +state: closed | |
| 5 | +author: | |
| 6 | + name: k33g | |
| 7 | + email: ph.charriere@gmail.com | |
| 8 | +createdAt: 2026-11-21T06:35:00.000Z | |
| 9 | +updatedAt: 2026-11-21T06:37:13.000Z | |
| 10 | +labels: | |
| 11 | + - kind::chore | |
| 12 | + - priority::high | |
| 13 | +epic: Olala in sbx | |
| 14 | +related: | |
| 15 | + - 1 | |
| 16 | +body: | | |
| 17 | + Download the 8-bit file (6.9 GB, recommended for CPUs) from `OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF` with `hf download`. | |
| 18 | + | |
| 19 | + The model is stored in `~/models` inside the sandbox (root overlay: 19 GB free). The workspace is on kDrive: storing 7 GB there would sync it to the cloud, so it is not the default. | |
| 20 | + | |
| 21 | + ## Acceptance criteria | |
| 22 | + - The file is in `~/models`; a second run does not download it again. | |
| 23 | +tasks: | |
| 24 | + - id: 1 | |
| 25 | + title: Download with hf download | |
| 26 | + state: closed | |
| 27 | + priority: high | |
| 28 | + author: | |
| 29 | + name: k33g | |
| 30 | + email: ph.charriere@gmail.com | |
| 31 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 32 | + updatedAt: 2026-11-21T06:37:13.000Z | |
| 33 | + closedAt: 2026-11-21T06:37:13.000Z | |
| 34 | + - id: 2 | |
| 35 | + title: Skip the download when the file already exists | |
| 36 | + state: closed | |
| 37 | + priority: medium | |
| 38 | + author: | |
| 39 | + name: k33g | |
| 40 | + email: ph.charriere@gmail.com | |
| 41 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 42 | + updatedAt: 2026-11-21T06:37:13.000Z | |
| 43 | + closedAt: 2026-11-21T06:37:13.000Z | |
| 44 | +comments: | |
| 45 | + - id: 1 | |
| 46 | + author: | |
| 47 | + name: k33g | |
| 48 | + email: ph.charriere@gmail.com | |
| 49 | + createdAt: 2026-11-21T06:37:13.000Z | |
| 50 | + body: | | |
| 51 | + Downloaded in about 2 minutes; second run of `scripts/olala.sh download` skips it (checked). | |
| new file mode 100644 | |||
| @@ -0,0 +1,51 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +id: 3 | ||
| 3 | +title: Download Olala-v1.0-mini-7A1B-q8_0-safe2.gguf | ||
| 4 | +state: closed | ||
| 5 | +author: | ||
| 6 | + name: k33g | ||
| 7 | + email: ph.charriere@gmail.com | ||
| 8 | +createdAt: 2026-11-21T06:35:00.000Z | ||
| 9 | +updatedAt: 2026-11-21T06:37:13.000Z | ||
| 10 | +labels: | ||
| 11 | + - kind::chore | ||
| 12 | + - priority::high | ||
| 13 | +epic: Olala in sbx | ||
| 14 | +related: | ||
| 15 | + - 1 | ||
| 16 | +body: | | ||
| 17 | + Download the 8-bit file (6.9 GB, recommended for CPUs) from `OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF` with `hf download`. | ||
| 18 | + | ||
| 19 | + The model is stored in `~/models` inside the sandbox (root overlay: 19 GB free). The workspace is on kDrive: storing 7 GB there would sync it to the cloud, so it is not the default. | ||
| 20 | + | ||
| 21 | + ## Acceptance criteria | ||
| 22 | + - The file is in `~/models`; a second run does not download it again. | ||
| 23 | +tasks: | ||
| 24 | + - id: 1 | ||
| 25 | + title: Download with hf download | ||
| 26 | + state: closed | ||
| 27 | + priority: high | ||
| 28 | + author: | ||
| 29 | + name: k33g | ||
| 30 | + email: ph.charriere@gmail.com | ||
| 31 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 32 | + updatedAt: 2026-11-21T06:37:13.000Z | ||
| 33 | + closedAt: 2026-11-21T06:37:13.000Z | ||
| 34 | + - id: 2 | ||
| 35 | + title: Skip the download when the file already exists | ||
| 36 | + state: closed | ||
| 37 | + priority: medium | ||
| 38 | + author: | ||
| 39 | + name: k33g | ||
| 40 | + email: ph.charriere@gmail.com | ||
| 41 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 42 | + updatedAt: 2026-11-21T06:37:13.000Z | ||
| 43 | + closedAt: 2026-11-21T06:37:13.000Z | ||
| 44 | +comments: | ||
| 45 | + - id: 1 | ||
| 46 | + author: | ||
| 47 | + name: k33g | ||
| 48 | + email: ph.charriere@gmail.com | ||
| 49 | + createdAt: 2026-11-21T06:37:13.000Z | ||
| 50 | + body: | | ||
| 51 | + Downloaded in about 2 minutes; second run of `scripts/olala.sh download` skips it (checked). | ||
added
.tickets/issues/0004-serve-the-model-with-llama-server-and-expose-the-port.yaml +67 -0 | new file mode 100644 | ||
| @@ -0,0 +1,67 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +id: 4 | |
| 3 | +title: Serve the model with llama-server and expose the port | |
| 4 | +state: open | |
| 5 | +author: | |
| 6 | + name: k33g | |
| 7 | + email: ph.charriere@gmail.com | |
| 8 | +createdAt: 2026-11-21T06:35:00.000Z | |
| 9 | +updatedAt: 2026-11-21T07:08:59.000Z | |
| 10 | +labels: | |
| 11 | + - kind::feature | |
| 12 | + - priority::high | |
| 13 | +epic: Olala in sbx | |
| 14 | +related: | |
| 15 | + - 1 | |
| 16 | +body: | | |
| 17 | + Run `llama-server` (OpenAI-compatible API) bound on `0.0.0.0:8080` so the port can be published to the host. | |
| 18 | + | |
| 19 | + CPU flags from the model card: `-t <threads> -fa 1 -nr -ctk q8_0 -ctv q8_0 -c 32768 -np 4`, sampling `--temp 0.8 --top-p 0.9 --top-k 40`. | |
| 20 | + | |
| 21 | + On the host, for the current sandbox: `sbx ports houla-houla --publish 8080:8080`. | |
| 22 | + | |
| 23 | + ## Acceptance criteria | |
| 24 | + - `curl http://localhost:8080/health` answers from the host. | |
| 25 | + - A chat completion answers. | |
| 26 | +tasks: | |
| 27 | + - id: 1 | |
| 28 | + title: Start llama-server in the background, wait for /health | |
| 29 | + state: closed | |
| 30 | + priority: high | |
| 31 | + author: | |
| 32 | + name: k33g | |
| 33 | + email: ph.charriere@gmail.com | |
| 34 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 35 | + updatedAt: 2026-11-21T06:37:13.000Z | |
| 36 | + closedAt: 2026-11-21T06:37:13.000Z | |
| 37 | + - id: 2 | |
| 38 | + title: Publish port 8080 on the host (user, host side) | |
| 39 | + state: open | |
| 40 | + priority: high | |
| 41 | + author: | |
| 42 | + name: k33g | |
| 43 | + email: ph.charriere@gmail.com | |
| 44 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 45 | + - id: 3 | |
| 46 | + title: Test /v1/chat/completions from the host | |
| 47 | + state: open | |
| 48 | + priority: high | |
| 49 | + author: | |
| 50 | + name: k33g | |
| 51 | + email: ph.charriere@gmail.com | |
| 52 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 53 | +comments: | |
| 54 | + - id: 1 | |
| 55 | + author: | |
| 56 | + name: k33g | |
| 57 | + email: ph.charriere@gmail.com | |
| 58 | + createdAt: 2026-11-21T06:37:13.000Z | |
| 59 | + body: | | |
| 60 | + Inside the sandbox: `/health`, `/v1/models` and `/v1/chat/completions` (normal and streaming) answer. Not yet checked from the host: needs `sbx ports houla-houla --publish 8080:8080` on the host. | |
| 61 | + - id: 2 | |
| 62 | + author: | |
| 63 | + name: k33g | |
| 64 | + email: ph.charriere@gmail.com | |
| 65 | + createdAt: 2026-11-21T07:08:59.000Z | |
| 66 | + body: | | |
| 67 | + Measured (q8_0-safe2, prompt ~20 tokens, 150 generated): with the model on the workspace mount, mmap gives ~1.7 tok/s; `--load-mode none` (model read into RAM) gives 35–44 tok/s with 16 threads. 18 threads (= every CPU): ~26 tok/s. Script defaults changed accordingly (`OLALA_LOAD_MODE=none`, threads = nproc − 2). Note: this fork has no `--no-mmap`; use `--load-mode`. | |
| new file mode 100644 | |||
| @@ -0,0 +1,67 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +id: 4 | ||
| 3 | +title: Serve the model with llama-server and expose the port | ||
| 4 | +state: open | ||
| 5 | +author: | ||
| 6 | + name: k33g | ||
| 7 | + email: ph.charriere@gmail.com | ||
| 8 | +createdAt: 2026-11-21T06:35:00.000Z | ||
| 9 | +updatedAt: 2026-11-21T07:08:59.000Z | ||
| 10 | +labels: | ||
| 11 | + - kind::feature | ||
| 12 | + - priority::high | ||
| 13 | +epic: Olala in sbx | ||
| 14 | +related: | ||
| 15 | + - 1 | ||
| 16 | +body: | | ||
| 17 | + Run `llama-server` (OpenAI-compatible API) bound on `0.0.0.0:8080` so the port can be published to the host. | ||
| 18 | + | ||
| 19 | + CPU flags from the model card: `-t <threads> -fa 1 -nr -ctk q8_0 -ctv q8_0 -c 32768 -np 4`, sampling `--temp 0.8 --top-p 0.9 --top-k 40`. | ||
| 20 | + | ||
| 21 | + On the host, for the current sandbox: `sbx ports houla-houla --publish 8080:8080`. | ||
| 22 | + | ||
| 23 | + ## Acceptance criteria | ||
| 24 | + - `curl http://localhost:8080/health` answers from the host. | ||
| 25 | + - A chat completion answers. | ||
| 26 | +tasks: | ||
| 27 | + - id: 1 | ||
| 28 | + title: Start llama-server in the background, wait for /health | ||
| 29 | + state: closed | ||
| 30 | + priority: high | ||
| 31 | + author: | ||
| 32 | + name: k33g | ||
| 33 | + email: ph.charriere@gmail.com | ||
| 34 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 35 | + updatedAt: 2026-11-21T06:37:13.000Z | ||
| 36 | + closedAt: 2026-11-21T06:37:13.000Z | ||
| 37 | + - id: 2 | ||
| 38 | + title: Publish port 8080 on the host (user, host side) | ||
| 39 | + state: open | ||
| 40 | + priority: high | ||
| 41 | + author: | ||
| 42 | + name: k33g | ||
| 43 | + email: ph.charriere@gmail.com | ||
| 44 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 45 | + - id: 3 | ||
| 46 | + title: Test /v1/chat/completions from the host | ||
| 47 | + state: open | ||
| 48 | + priority: high | ||
| 49 | + author: | ||
| 50 | + name: k33g | ||
| 51 | + email: ph.charriere@gmail.com | ||
| 52 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 53 | +comments: | ||
| 54 | + - id: 1 | ||
| 55 | + author: | ||
| 56 | + name: k33g | ||
| 57 | + email: ph.charriere@gmail.com | ||
| 58 | + createdAt: 2026-11-21T06:37:13.000Z | ||
| 59 | + body: | | ||
| 60 | + Inside the sandbox: `/health`, `/v1/models` and `/v1/chat/completions` (normal and streaming) answer. Not yet checked from the host: needs `sbx ports houla-houla --publish 8080:8080` on the host. | ||
| 61 | + - id: 2 | ||
| 62 | + author: | ||
| 63 | + name: k33g | ||
| 64 | + email: ph.charriere@gmail.com | ||
| 65 | + createdAt: 2026-11-21T07:08:59.000Z | ||
| 66 | + body: | | ||
| 67 | + Measured (q8_0-safe2, prompt ~20 tokens, 150 generated): with the model on the workspace mount, mmap gives ~1.7 tok/s; `--load-mode none` (model read into RAM) gives 35–44 tok/s with 16 threads. 18 threads (= every CPU): ~26 tok/s. Script defaults changed accordingly (`OLALA_LOAD_MODE=none`, threads = nproc − 2). Note: this fork has no `--no-mmap`; use `--load-mode`. | ||
added
.tickets/issues/0005-idempotent-script-scripts-olala-sh.yaml +66 -0 | new file mode 100644 | ||
| @@ -0,0 +1,66 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +id: 5 | |
| 3 | +title: Idempotent script scripts/olala.sh | |
| 4 | +state: open | |
| 5 | +author: | |
| 6 | + name: k33g | |
| 7 | + email: ph.charriere@gmail.com | |
| 8 | +createdAt: 2026-11-21T06:35:00.000Z | |
| 9 | +updatedAt: 2026-11-21T07:08:59.000Z | |
| 10 | +labels: | |
| 11 | + - kind::feature | |
| 12 | + - priority::medium | |
| 13 | +epic: Olala in sbx | |
| 14 | +related: | |
| 15 | + - 1 | |
| 16 | +body: | | |
| 17 | + One script, run inside the sandbox, with subcommands `setup | download | serve | up | stop | restart | status | logs`. | |
| 18 | + Every step is skipped when its result exists (binary built, model present, server running). | |
| 19 | + | |
| 20 | + Configured through env vars so another model can be downloaded and served: `OLALA_MODEL_REPO`, `OLALA_MODEL_FILE`, `OLALA_MODELS_DIR`, `OLALA_PORT`, `OLALA_THREADS`, `OLALA_CTX`, `OLALA_PARALLEL`, `OLALA_EXTRA_ARGS`. | |
| 21 | + | |
| 22 | + File: [scripts/olala.sh](scripts/olala.sh) | |
| 23 | +tasks: | |
| 24 | + - id: 1 | |
| 25 | + title: Write the script | |
| 26 | + state: closed | |
| 27 | + priority: medium | |
| 28 | + author: | |
| 29 | + name: k33g | |
| 30 | + email: ph.charriere@gmail.com | |
| 31 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 32 | + updatedAt: 2026-11-21T06:35:00.000Z | |
| 33 | + closedAt: 2026-11-21T06:35:00.000Z | |
| 34 | + - id: 2 | |
| 35 | + title: Run it end to end (second run = all skipped) | |
| 36 | + state: closed | |
| 37 | + priority: medium | |
| 38 | + author: | |
| 39 | + name: k33g | |
| 40 | + email: ph.charriere@gmail.com | |
| 41 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 42 | + updatedAt: 2026-11-21T06:37:13.000Z | |
| 43 | + closedAt: 2026-11-21T06:37:13.000Z | |
| 44 | + - id: 3 | |
| 45 | + title: Serve another model through env vars | |
| 46 | + state: open | |
| 47 | + priority: low | |
| 48 | + author: | |
| 49 | + name: k33g | |
| 50 | + email: ph.charriere@gmail.com | |
| 51 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 52 | +comments: | |
| 53 | + - id: 1 | |
| 54 | + author: | |
| 55 | + name: k33g | |
| 56 | + email: ph.charriere@gmail.com | |
| 57 | + createdAt: 2026-11-21T06:37:13.000Z | |
| 58 | + body: | | |
| 59 | + `up` run twice: the second run skips the three steps. Added `OLALA_ALIAS` (model name exposed by the API, default: file name without `.gguf`). | |
| 60 | + - id: 2 | |
| 61 | + author: | |
| 62 | + name: k33g | |
| 63 | + email: ph.charriere@gmail.com | |
| 64 | + createdAt: 2026-11-21T07:08:59.000Z | |
| 65 | + body: | | |
| 66 | + Fixes: pid file was wrong when `setsid` forks (now written by the process itself before `exec`); `stop` waits for the process to exit; uses `llama-server` from PATH (kit) before building; models dir default `./models` (relative to the repo root); `OLALA_LOAD_MODE`, `OLALA_SERVER_BIN`. Write resets the exec bit: keep `chmod +x`. | |
| new file mode 100644 | |||
| @@ -0,0 +1,66 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +id: 5 | ||
| 3 | +title: Idempotent script scripts/olala.sh | ||
| 4 | +state: open | ||
| 5 | +author: | ||
| 6 | + name: k33g | ||
| 7 | + email: ph.charriere@gmail.com | ||
| 8 | +createdAt: 2026-11-21T06:35:00.000Z | ||
| 9 | +updatedAt: 2026-11-21T07:08:59.000Z | ||
| 10 | +labels: | ||
| 11 | + - kind::feature | ||
| 12 | + - priority::medium | ||
| 13 | +epic: Olala in sbx | ||
| 14 | +related: | ||
| 15 | + - 1 | ||
| 16 | +body: | | ||
| 17 | + One script, run inside the sandbox, with subcommands `setup | download | serve | up | stop | restart | status | logs`. | ||
| 18 | + Every step is skipped when its result exists (binary built, model present, server running). | ||
| 19 | + | ||
| 20 | + Configured through env vars so another model can be downloaded and served: `OLALA_MODEL_REPO`, `OLALA_MODEL_FILE`, `OLALA_MODELS_DIR`, `OLALA_PORT`, `OLALA_THREADS`, `OLALA_CTX`, `OLALA_PARALLEL`, `OLALA_EXTRA_ARGS`. | ||
| 21 | + | ||
| 22 | + File: [scripts/olala.sh](scripts/olala.sh) | ||
| 23 | +tasks: | ||
| 24 | + - id: 1 | ||
| 25 | + title: Write the script | ||
| 26 | + state: closed | ||
| 27 | + priority: medium | ||
| 28 | + author: | ||
| 29 | + name: k33g | ||
| 30 | + email: ph.charriere@gmail.com | ||
| 31 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 32 | + updatedAt: 2026-11-21T06:35:00.000Z | ||
| 33 | + closedAt: 2026-11-21T06:35:00.000Z | ||
| 34 | + - id: 2 | ||
| 35 | + title: Run it end to end (second run = all skipped) | ||
| 36 | + state: closed | ||
| 37 | + priority: medium | ||
| 38 | + author: | ||
| 39 | + name: k33g | ||
| 40 | + email: ph.charriere@gmail.com | ||
| 41 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 42 | + updatedAt: 2026-11-21T06:37:13.000Z | ||
| 43 | + closedAt: 2026-11-21T06:37:13.000Z | ||
| 44 | + - id: 3 | ||
| 45 | + title: Serve another model through env vars | ||
| 46 | + state: open | ||
| 47 | + priority: low | ||
| 48 | + author: | ||
| 49 | + name: k33g | ||
| 50 | + email: ph.charriere@gmail.com | ||
| 51 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 52 | +comments: | ||
| 53 | + - id: 1 | ||
| 54 | + author: | ||
| 55 | + name: k33g | ||
| 56 | + email: ph.charriere@gmail.com | ||
| 57 | + createdAt: 2026-11-21T06:37:13.000Z | ||
| 58 | + body: | | ||
| 59 | + `up` run twice: the second run skips the three steps. Added `OLALA_ALIAS` (model name exposed by the API, default: file name without `.gguf`). | ||
| 60 | + - id: 2 | ||
| 61 | + author: | ||
| 62 | + name: k33g | ||
| 63 | + email: ph.charriere@gmail.com | ||
| 64 | + createdAt: 2026-11-21T07:08:59.000Z | ||
| 65 | + body: | | ||
| 66 | + Fixes: pid file was wrong when `setsid` forks (now written by the process itself before `exec`); `stop` waits for the process to exit; uses `llama-server` from PATH (kit) before building; models dir default `./models` (relative to the repo root); `OLALA_LOAD_MODE`, `OLALA_SERVER_BIN`. Write resets the exec bit: keep `chmod +x`. | ||
added
.tickets/issues/0006-environment-file-sbxenv-yaml.yaml +72 -0 | new file mode 100644 | ||
| @@ -0,0 +1,72 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +id: 6 | |
| 3 | +title: Environment file sbxenv.yaml | |
| 4 | +state: open | |
| 5 | +author: | |
| 6 | + name: k33g | |
| 7 | + email: ph.charriere@gmail.com | |
| 8 | +createdAt: 2026-11-21T06:35:00.000Z | |
| 9 | +updatedAt: 2026-11-21T07:52:58.000Z | |
| 10 | +labels: | |
| 11 | + - kind::feature | |
| 12 | + - priority::medium | |
| 13 | +epic: Olala in sbx | |
| 14 | +related: | |
| 15 | + - 1 | |
| 16 | +body: | | |
| 17 | + An sbx environment file to recreate the whole setup in one command: `sbx env run sbxenv.yaml`. | |
| 18 | + | |
| 19 | + - `args`: model repo, model file, host port, threads → `env` of the sandbox. | |
| 20 | + - `ports`: sandbox 8080 → host port. | |
| 21 | + - `lifecycle.postCreate` (runs on the host): `sbx env exec … -- scripts/olala.sh up`. | |
| 22 | + - Restart after a re-attach (postCreate does not run then): `sbx env exec sbxenv.yaml -- scripts/olala.sh serve`. | |
| 23 | + | |
| 24 | + Traps: env files are mounted read-only in the sandbox once used; lifecycle commands do not receive the file's `env` values; the `sbx env exec` syntax inside a lifecycle hook is not verified yet. | |
| 25 | +tasks: | |
| 26 | + - id: 1 | |
| 27 | + title: Write sbxenv.yaml | |
| 28 | + state: closed | |
| 29 | + priority: medium | |
| 30 | + author: | |
| 31 | + name: k33g | |
| 32 | + email: ph.charriere@gmail.com | |
| 33 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 34 | + updatedAt: 2026-11-21T07:52:58.000Z | |
| 35 | + closedAt: 2026-11-21T06:37:13.000Z | |
| 36 | + - id: 2 | |
| 37 | + title: Try it on the host (sbx env create) | |
| 38 | + state: open | |
| 39 | + priority: medium | |
| 40 | + author: | |
| 41 | + name: k33g | |
| 42 | + email: ph.charriere@gmail.com | |
| 43 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 44 | +comments: | |
| 45 | + - id: 1 | |
| 46 | + author: | |
| 47 | + name: k33g | |
| 48 | + email: ph.charriere@gmail.com | |
| 49 | + createdAt: 2026-11-21T06:37:13.000Z | |
| 50 | + body: | | |
| 51 | + Written: [sbxenv.houla.yaml](sbxenv.houla.yaml). Not tried with sbx (no sbx in the sandbox). Unverified: `sbx env exec` from a postCreate hook, `--env-arg` substitution in `ports.host`, server surviving the end of the `sbx env exec` session. | |
| 52 | + - id: 2 | |
| 53 | + author: | |
| 54 | + name: k33g | |
| 55 | + email: ph.charriere@gmail.com | |
| 56 | + createdAt: 2026-11-21T07:23:50.000Z | |
| 57 | + body: | | |
| 58 | + Second host run: the kit built, the sandbox was created, then postCreate failed: `no sbxenv.yaml found at …/houla/--name/sbxenv.yaml`. `sbx env exec` reads every word after the first path, up to `--`, as another path: flags must come first. Now `sbx env exec --name "$SBX_SANDBOX_NAME" --no-input "$SBX_ENV_FILE" -- …`. | |
| 59 | + - id: 3 | |
| 60 | + author: | |
| 61 | + name: k33g | |
| 62 | + email: ph.charriere@gmail.com | |
| 63 | + createdAt: 2026-11-21T07:40:26.000Z | |
| 64 | + body: | | |
| 65 | + Third host run: postCreate OK (`ready`), but curl got nothing: the next `sbx env exec` printed `sandbox houla-olala — started` and `olala.sh status` said `not running`. Per the sbx docs (usage), a sandbox created with `sbx create` stops when no session keeps it running; `sbx run -d --name <name>` makes it detached (permanent) so it keeps running. Added it as the first postCreate command. Not verified yet. | |
| 66 | + - id: 4 | |
| 67 | + author: | |
| 68 | + name: k33g | |
| 69 | + email: ph.charriere@gmail.com | |
| 70 | + createdAt: 2026-11-21T07:52:58.000Z | |
| 71 | + body: | | |
| 72 | + Renamed: this file is now [sbxenv.yaml](sbxenv.yaml) (was `sbxenv.houla.yaml`); the former `sbxenv.yaml` (dev sandbox `houla-houla`) is now [sbxenv.dev.yaml](sbxenv.dev.yaml). Older comments keep the old name. The last step, `mv sbxenv.houla.yaml sbxenv.yaml`, has to be run on the host (read-only mount inside the sandbox). | |
| new file mode 100644 | |||
| @@ -0,0 +1,72 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +id: 6 | ||
| 3 | +title: Environment file sbxenv.yaml | ||
| 4 | +state: open | ||
| 5 | +author: | ||
| 6 | + name: k33g | ||
| 7 | + email: ph.charriere@gmail.com | ||
| 8 | +createdAt: 2026-11-21T06:35:00.000Z | ||
| 9 | +updatedAt: 2026-11-21T07:52:58.000Z | ||
| 10 | +labels: | ||
| 11 | + - kind::feature | ||
| 12 | + - priority::medium | ||
| 13 | +epic: Olala in sbx | ||
| 14 | +related: | ||
| 15 | + - 1 | ||
| 16 | +body: | | ||
| 17 | + An sbx environment file to recreate the whole setup in one command: `sbx env run sbxenv.yaml`. | ||
| 18 | + | ||
| 19 | + - `args`: model repo, model file, host port, threads → `env` of the sandbox. | ||
| 20 | + - `ports`: sandbox 8080 → host port. | ||
| 21 | + - `lifecycle.postCreate` (runs on the host): `sbx env exec … -- scripts/olala.sh up`. | ||
| 22 | + - Restart after a re-attach (postCreate does not run then): `sbx env exec sbxenv.yaml -- scripts/olala.sh serve`. | ||
| 23 | + | ||
| 24 | + Traps: env files are mounted read-only in the sandbox once used; lifecycle commands do not receive the file's `env` values; the `sbx env exec` syntax inside a lifecycle hook is not verified yet. | ||
| 25 | +tasks: | ||
| 26 | + - id: 1 | ||
| 27 | + title: Write sbxenv.yaml | ||
| 28 | + state: closed | ||
| 29 | + priority: medium | ||
| 30 | + author: | ||
| 31 | + name: k33g | ||
| 32 | + email: ph.charriere@gmail.com | ||
| 33 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 34 | + updatedAt: 2026-11-21T07:52:58.000Z | ||
| 35 | + closedAt: 2026-11-21T06:37:13.000Z | ||
| 36 | + - id: 2 | ||
| 37 | + title: Try it on the host (sbx env create) | ||
| 38 | + state: open | ||
| 39 | + priority: medium | ||
| 40 | + author: | ||
| 41 | + name: k33g | ||
| 42 | + email: ph.charriere@gmail.com | ||
| 43 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 44 | +comments: | ||
| 45 | + - id: 1 | ||
| 46 | + author: | ||
| 47 | + name: k33g | ||
| 48 | + email: ph.charriere@gmail.com | ||
| 49 | + createdAt: 2026-11-21T06:37:13.000Z | ||
| 50 | + body: | | ||
| 51 | + Written: [sbxenv.houla.yaml](sbxenv.houla.yaml). Not tried with sbx (no sbx in the sandbox). Unverified: `sbx env exec` from a postCreate hook, `--env-arg` substitution in `ports.host`, server surviving the end of the `sbx env exec` session. | ||
| 52 | + - id: 2 | ||
| 53 | + author: | ||
| 54 | + name: k33g | ||
| 55 | + email: ph.charriere@gmail.com | ||
| 56 | + createdAt: 2026-11-21T07:23:50.000Z | ||
| 57 | + body: | | ||
| 58 | + Second host run: the kit built, the sandbox was created, then postCreate failed: `no sbxenv.yaml found at …/houla/--name/sbxenv.yaml`. `sbx env exec` reads every word after the first path, up to `--`, as another path: flags must come first. Now `sbx env exec --name "$SBX_SANDBOX_NAME" --no-input "$SBX_ENV_FILE" -- …`. | ||
| 59 | + - id: 3 | ||
| 60 | + author: | ||
| 61 | + name: k33g | ||
| 62 | + email: ph.charriere@gmail.com | ||
| 63 | + createdAt: 2026-11-21T07:40:26.000Z | ||
| 64 | + body: | | ||
| 65 | + Third host run: postCreate OK (`ready`), but curl got nothing: the next `sbx env exec` printed `sandbox houla-olala — started` and `olala.sh status` said `not running`. Per the sbx docs (usage), a sandbox created with `sbx create` stops when no session keeps it running; `sbx run -d --name <name>` makes it detached (permanent) so it keeps running. Added it as the first postCreate command. Not verified yet. | ||
| 66 | + - id: 4 | ||
| 67 | + author: | ||
| 68 | + name: k33g | ||
| 69 | + email: ph.charriere@gmail.com | ||
| 70 | + createdAt: 2026-11-21T07:52:58.000Z | ||
| 71 | + body: | | ||
| 72 | + Renamed: this file is now [sbxenv.yaml](sbxenv.yaml) (was `sbxenv.houla.yaml`); the former `sbxenv.yaml` (dev sandbox `houla-houla`) is now [sbxenv.dev.yaml](sbxenv.dev.yaml). Older comments keep the old name. The last step, `mv sbxenv.houla.yaml sbxenv.yaml`, has to be run on the host (read-only mount inside the sandbox). | ||
added
.tickets/issues/0007-package-the-llama-cpp-fork-as-an-sbx-kit.yaml +83 -0 | new file mode 100644 | ||
| @@ -0,0 +1,83 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +id: 7 | |
| 3 | +title: Package the llama.cpp fork as an sbx kit | |
| 4 | +state: open | |
| 5 | +author: | |
| 6 | + name: k33g | |
| 7 | + email: ph.charriere@gmail.com | |
| 8 | +createdAt: 2026-11-21T06:35:00.000Z | |
| 9 | +updatedAt: 2026-11-21T07:52:58.000Z | |
| 10 | +labels: | |
| 11 | + - kind::feature | |
| 12 | + - priority::medium | |
| 13 | +epic: Olala in sbx | |
| 14 | +related: | |
| 15 | + - 1 | |
| 16 | +body: | | |
| 17 | + Instead of building the fork at every sandbox creation (several minutes), ship it as a **kit v3 mixin** in this project (e.g. `kits/llama-cpp-olala/`), on the model of `go-toolchain` in mini-me: | |
| 18 | + | |
| 19 | + - `llama-cpp-olala.yaml`: `schemaVersion: "3"`, `kind: mixin`, arg `ref` (fork commit, `buildArg`), `provides`, `network-policy@1` allowing `huggingface.co`, `*.huggingface.co`, `*.hf.co`. | |
| 20 | + - `llama-cpp-olala.dockerfile`: build stage (debian trixie, cmake, CPU only, `-DLLAMA_USE_PREBUILT_UI=OFF`, multi-arch `TARGETARCH`), then `FROM scratch` overlay with `llama-server`, `llama-cli` and their `.so` (or a static build `-DBUILD_SHARED_LIBS=OFF`). | |
| 21 | + - Reference it from `sbxenv.yaml` `kits:` (local path first, then Docker Hub). | |
| 22 | + | |
| 23 | + ## Acceptance criteria | |
| 24 | + - A new sandbox has `llama-server` on the PATH without building anything. | |
| 25 | + - `scripts/olala.sh setup` detects it and skips the build. | |
| 26 | +tasks: | |
| 27 | + - id: 1 | |
| 28 | + title: Write the kit descriptor and Dockerfile | |
| 29 | + state: closed | |
| 30 | + priority: medium | |
| 31 | + author: | |
| 32 | + name: k33g | |
| 33 | + email: ph.charriere@gmail.com | |
| 34 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 35 | + updatedAt: 2026-11-21T07:08:59.000Z | |
| 36 | + closedAt: 2026-11-21T07:08:59.000Z | |
| 37 | + - id: 2 | |
| 38 | + title: Build statically (or copy the .so) and check llama-server runs | |
| 39 | + state: closed | |
| 40 | + priority: medium | |
| 41 | + author: | |
| 42 | + name: k33g | |
| 43 | + email: ph.charriere@gmail.com | |
| 44 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 45 | + updatedAt: 2026-11-21T07:08:59.000Z | |
| 46 | + closedAt: 2026-11-21T07:08:59.000Z | |
| 47 | + - id: 3 | |
| 48 | + title: Use the kit from sbxenv.yaml | |
| 49 | + state: closed | |
| 50 | + priority: medium | |
| 51 | + author: | |
| 52 | + name: k33g | |
| 53 | + email: ph.charriere@gmail.com | |
| 54 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 55 | + updatedAt: 2026-11-21T07:52:58.000Z | |
| 56 | + closedAt: 2026-11-21T07:08:59.000Z | |
| 57 | + - id: 4 | |
| 58 | + title: Publish it on Docker Hub (optional) | |
| 59 | + state: open | |
| 60 | + priority: low | |
| 61 | + author: | |
| 62 | + name: k33g | |
| 63 | + email: ph.charriere@gmail.com | |
| 64 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 65 | +comments: | |
| 66 | + - id: 1 | |
| 67 | + author: | |
| 68 | + name: k33g | |
| 69 | + email: ph.charriere@gmail.com | |
| 70 | + createdAt: 2026-11-21T07:08:59.000Z | |
| 71 | + body: | | |
| 72 | + Done: [kits/llama-cpp-olala/](kits/llama-cpp-olala/llama-cpp-olala.yaml) — v3 mixin, arg `ref` (full SHA of the fork), portable build (`GGML_NATIVE=OFF`, `GGML_BACKEND_DL=ON`, `GGML_CPU_ALL_VARIANTS=ON`, `LLAMA_OPENSSL=OFF`, RPATH `$ORIGIN`), installed in `/opt/llama-cpp-olala` + links in `/usr/local/bin`, `libgomp.so.1` shipped. Network policy: Hugging Face hosts; agent context file. | |
| 73 | + | |
| 74 | + Tested: `docker buildx build` of the Dockerfile (1m22, 26 MB overlay, arm64); `llama-server --version` in a bare `debian:trixie-slim`; installed in this sandbox and served by `scripts/olala.sh` (35–44 tok/s, 16 threads, `--load-mode none`). | |
| 75 | + Not tested: sbx building the kit itself (`sbx env create sbxenv.houla.yaml`), amd64. | |
| 76 | + - id: 2 | |
| 77 | + author: | |
| 78 | + name: k33g | |
| 79 | + email: ph.charriere@gmail.com | |
| 80 | + createdAt: 2026-11-21T07:13:05.000Z | |
| 81 | + body: | | |
| 82 | + First `sbx env create sbxenv.houla.yaml` on the host: the kit frontend rejected `provides: ["llama-cpp-olala@<sha>"]` (`invalid version`). A provides version must be `X.Y.Z`: now `llama-cpp-olala@0.5.0` (what the fork reports); the commit stays pinned by `ref`. | |
| 83 | + Checked with the same frontend: `docker buildx build -f kits/llama-cpp-olala/llama-cpp-olala.yaml .` passes (overlay exported). | |
| new file mode 100644 | |||
| @@ -0,0 +1,83 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +id: 7 | ||
| 3 | +title: Package the llama.cpp fork as an sbx kit | ||
| 4 | +state: open | ||
| 5 | +author: | ||
| 6 | + name: k33g | ||
| 7 | + email: ph.charriere@gmail.com | ||
| 8 | +createdAt: 2026-11-21T06:35:00.000Z | ||
| 9 | +updatedAt: 2026-11-21T07:52:58.000Z | ||
| 10 | +labels: | ||
| 11 | + - kind::feature | ||
| 12 | + - priority::medium | ||
| 13 | +epic: Olala in sbx | ||
| 14 | +related: | ||
| 15 | + - 1 | ||
| 16 | +body: | | ||
| 17 | + Instead of building the fork at every sandbox creation (several minutes), ship it as a **kit v3 mixin** in this project (e.g. `kits/llama-cpp-olala/`), on the model of `go-toolchain` in mini-me: | ||
| 18 | + | ||
| 19 | + - `llama-cpp-olala.yaml`: `schemaVersion: "3"`, `kind: mixin`, arg `ref` (fork commit, `buildArg`), `provides`, `network-policy@1` allowing `huggingface.co`, `*.huggingface.co`, `*.hf.co`. | ||
| 20 | + - `llama-cpp-olala.dockerfile`: build stage (debian trixie, cmake, CPU only, `-DLLAMA_USE_PREBUILT_UI=OFF`, multi-arch `TARGETARCH`), then `FROM scratch` overlay with `llama-server`, `llama-cli` and their `.so` (or a static build `-DBUILD_SHARED_LIBS=OFF`). | ||
| 21 | + - Reference it from `sbxenv.yaml` `kits:` (local path first, then Docker Hub). | ||
| 22 | + | ||
| 23 | + ## Acceptance criteria | ||
| 24 | + - A new sandbox has `llama-server` on the PATH without building anything. | ||
| 25 | + - `scripts/olala.sh setup` detects it and skips the build. | ||
| 26 | +tasks: | ||
| 27 | + - id: 1 | ||
| 28 | + title: Write the kit descriptor and Dockerfile | ||
| 29 | + state: closed | ||
| 30 | + priority: medium | ||
| 31 | + author: | ||
| 32 | + name: k33g | ||
| 33 | + email: ph.charriere@gmail.com | ||
| 34 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 35 | + updatedAt: 2026-11-21T07:08:59.000Z | ||
| 36 | + closedAt: 2026-11-21T07:08:59.000Z | ||
| 37 | + - id: 2 | ||
| 38 | + title: Build statically (or copy the .so) and check llama-server runs | ||
| 39 | + state: closed | ||
| 40 | + priority: medium | ||
| 41 | + author: | ||
| 42 | + name: k33g | ||
| 43 | + email: ph.charriere@gmail.com | ||
| 44 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 45 | + updatedAt: 2026-11-21T07:08:59.000Z | ||
| 46 | + closedAt: 2026-11-21T07:08:59.000Z | ||
| 47 | + - id: 3 | ||
| 48 | + title: Use the kit from sbxenv.yaml | ||
| 49 | + state: closed | ||
| 50 | + priority: medium | ||
| 51 | + author: | ||
| 52 | + name: k33g | ||
| 53 | + email: ph.charriere@gmail.com | ||
| 54 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 55 | + updatedAt: 2026-11-21T07:52:58.000Z | ||
| 56 | + closedAt: 2026-11-21T07:08:59.000Z | ||
| 57 | + - id: 4 | ||
| 58 | + title: Publish it on Docker Hub (optional) | ||
| 59 | + state: open | ||
| 60 | + priority: low | ||
| 61 | + author: | ||
| 62 | + name: k33g | ||
| 63 | + email: ph.charriere@gmail.com | ||
| 64 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 65 | +comments: | ||
| 66 | + - id: 1 | ||
| 67 | + author: | ||
| 68 | + name: k33g | ||
| 69 | + email: ph.charriere@gmail.com | ||
| 70 | + createdAt: 2026-11-21T07:08:59.000Z | ||
| 71 | + body: | | ||
| 72 | + Done: [kits/llama-cpp-olala/](kits/llama-cpp-olala/llama-cpp-olala.yaml) — v3 mixin, arg `ref` (full SHA of the fork), portable build (`GGML_NATIVE=OFF`, `GGML_BACKEND_DL=ON`, `GGML_CPU_ALL_VARIANTS=ON`, `LLAMA_OPENSSL=OFF`, RPATH `$ORIGIN`), installed in `/opt/llama-cpp-olala` + links in `/usr/local/bin`, `libgomp.so.1` shipped. Network policy: Hugging Face hosts; agent context file. | ||
| 73 | + | ||
| 74 | + Tested: `docker buildx build` of the Dockerfile (1m22, 26 MB overlay, arm64); `llama-server --version` in a bare `debian:trixie-slim`; installed in this sandbox and served by `scripts/olala.sh` (35–44 tok/s, 16 threads, `--load-mode none`). | ||
| 75 | + Not tested: sbx building the kit itself (`sbx env create sbxenv.houla.yaml`), amd64. | ||
| 76 | + - id: 2 | ||
| 77 | + author: | ||
| 78 | + name: k33g | ||
| 79 | + email: ph.charriere@gmail.com | ||
| 80 | + createdAt: 2026-11-21T07:13:05.000Z | ||
| 81 | + body: | | ||
| 82 | + First `sbx env create sbxenv.houla.yaml` on the host: the kit frontend rejected `provides: ["llama-cpp-olala@<sha>"]` (`invalid version`). A provides version must be `X.Y.Z`: now `llama-cpp-olala@0.5.0` (what the fork reports); the commit stays pinned by `ref`. | ||
| 83 | + Checked with the same frontend: `docker buildx build -f kits/llama-cpp-olala/llama-cpp-olala.yaml .` passes (overlay exported). | ||
added
.tickets/issues/0008-document-the-setup-and-the-curl-test-commands.yaml +61 -0 | new file mode 100644 | ||
| @@ -0,0 +1,61 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +id: 8 | |
| 3 | +title: Document the setup and the curl test commands | |
| 4 | +state: open | |
| 5 | +author: | |
| 6 | + name: k33g | |
| 7 | + email: ph.charriere@gmail.com | |
| 8 | +createdAt: 2026-11-21T06:35:00.000Z | |
| 9 | +updatedAt: 2026-11-21T07:08:59.000Z | |
| 10 | +labels: | |
| 11 | + - documentation | |
| 12 | + - priority::medium | |
| 13 | +epic: Olala in sbx | |
| 14 | +related: | |
| 15 | + - 1 | |
| 16 | +body: | | |
| 17 | + How-to (FR + EN) in `docs/`: create the environment, check the server, curl commands (`/health`, `/v1/models`, `/v1/chat/completions` with `reasoning_effort`, streaming), change model, restart after re-attach. | |
| 18 | +tasks: | |
| 19 | + - id: 1 | |
| 20 | + title: curl commands tested from the host | |
| 21 | + state: open | |
| 22 | + priority: medium | |
| 23 | + author: | |
| 24 | + name: k33g | |
| 25 | + email: ph.charriere@gmail.com | |
| 26 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 27 | + - id: 2 | |
| 28 | + title: How-to FR | |
| 29 | + state: closed | |
| 30 | + priority: medium | |
| 31 | + author: | |
| 32 | + name: k33g | |
| 33 | + email: ph.charriere@gmail.com | |
| 34 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 35 | + updatedAt: 2026-11-21T06:37:13.000Z | |
| 36 | + closedAt: 2026-11-21T06:37:13.000Z | |
| 37 | + - id: 3 | |
| 38 | + title: How-to EN | |
| 39 | + state: closed | |
| 40 | + priority: medium | |
| 41 | + author: | |
| 42 | + name: k33g | |
| 43 | + email: ph.charriere@gmail.com | |
| 44 | + createdAt: 2026-11-21T06:35:00.000Z | |
| 45 | + updatedAt: 2026-11-21T06:37:13.000Z | |
| 46 | + closedAt: 2026-11-21T06:37:13.000Z | |
| 47 | +comments: | |
| 48 | + - id: 1 | |
| 49 | + author: | |
| 50 | + name: k33g | |
| 51 | + email: ph.charriere@gmail.com | |
| 52 | + createdAt: 2026-11-21T06:37:13.000Z | |
| 53 | + body: | | |
| 54 | + Written: [docs/olala-sbx.md](docs/olala-sbx.md) (FR) and [docs/olala-sbx.en.md](docs/olala-sbx.en.md) (EN). curl commands tested inside the sandbox only. | |
| 55 | + - id: 2 | |
| 56 | + author: | |
| 57 | + name: k33g | |
| 58 | + email: ph.charriere@gmail.com | |
| 59 | + createdAt: 2026-11-21T07:08:59.000Z | |
| 60 | + body: | | |
| 61 | + Docs updated for the kit, `./models`, `models_dir`, load mode and threads. | |
| new file mode 100644 | |||
| @@ -0,0 +1,61 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +id: 8 | ||
| 3 | +title: Document the setup and the curl test commands | ||
| 4 | +state: open | ||
| 5 | +author: | ||
| 6 | + name: k33g | ||
| 7 | + email: ph.charriere@gmail.com | ||
| 8 | +createdAt: 2026-11-21T06:35:00.000Z | ||
| 9 | +updatedAt: 2026-11-21T07:08:59.000Z | ||
| 10 | +labels: | ||
| 11 | + - documentation | ||
| 12 | + - priority::medium | ||
| 13 | +epic: Olala in sbx | ||
| 14 | +related: | ||
| 15 | + - 1 | ||
| 16 | +body: | | ||
| 17 | + How-to (FR + EN) in `docs/`: create the environment, check the server, curl commands (`/health`, `/v1/models`, `/v1/chat/completions` with `reasoning_effort`, streaming), change model, restart after re-attach. | ||
| 18 | +tasks: | ||
| 19 | + - id: 1 | ||
| 20 | + title: curl commands tested from the host | ||
| 21 | + state: open | ||
| 22 | + priority: medium | ||
| 23 | + author: | ||
| 24 | + name: k33g | ||
| 25 | + email: ph.charriere@gmail.com | ||
| 26 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 27 | + - id: 2 | ||
| 28 | + title: How-to FR | ||
| 29 | + state: closed | ||
| 30 | + priority: medium | ||
| 31 | + author: | ||
| 32 | + name: k33g | ||
| 33 | + email: ph.charriere@gmail.com | ||
| 34 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 35 | + updatedAt: 2026-11-21T06:37:13.000Z | ||
| 36 | + closedAt: 2026-11-21T06:37:13.000Z | ||
| 37 | + - id: 3 | ||
| 38 | + title: How-to EN | ||
| 39 | + state: closed | ||
| 40 | + priority: medium | ||
| 41 | + author: | ||
| 42 | + name: k33g | ||
| 43 | + email: ph.charriere@gmail.com | ||
| 44 | + createdAt: 2026-11-21T06:35:00.000Z | ||
| 45 | + updatedAt: 2026-11-21T06:37:13.000Z | ||
| 46 | + closedAt: 2026-11-21T06:37:13.000Z | ||
| 47 | +comments: | ||
| 48 | + - id: 1 | ||
| 49 | + author: | ||
| 50 | + name: k33g | ||
| 51 | + email: ph.charriere@gmail.com | ||
| 52 | + createdAt: 2026-11-21T06:37:13.000Z | ||
| 53 | + body: | | ||
| 54 | + Written: [docs/olala-sbx.md](docs/olala-sbx.md) (FR) and [docs/olala-sbx.en.md](docs/olala-sbx.en.md) (EN). curl commands tested inside the sandbox only. | ||
| 55 | + - id: 2 | ||
| 56 | + author: | ||
| 57 | + name: k33g | ||
| 58 | + email: ph.charriere@gmail.com | ||
| 59 | + createdAt: 2026-11-21T07:08:59.000Z | ||
| 60 | + body: | | ||
| 61 | + Docs updated for the kit, `./models`, `models_dir`, load mode and threads. | ||
added
.tickets/issues/0009-store-models-in-a-configurable-repository-folder.yaml +65 -0 | new file mode 100644 | ||
| @@ -0,0 +1,65 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +id: 9 | |
| 3 | +title: Store models in a configurable repository folder (default ./models) | |
| 4 | +state: closed | |
| 5 | +author: | |
| 6 | + name: k33g | |
| 7 | + email: ph.charriere@gmail.com | |
| 8 | +createdAt: 2026-11-21T07:08:59.000Z | |
| 9 | +updatedAt: 2026-11-21T07:52:58.000Z | |
| 10 | +labels: | |
| 11 | + - kind::feature | |
| 12 | + - priority::high | |
| 13 | +epic: Olala in sbx | |
| 14 | +related: | |
| 15 | + - 3 | |
| 16 | + - 5 | |
| 17 | +body: | | |
| 18 | + Store downloaded models in a folder of the repository, so dropping the sandbox does not mean downloading them again. | |
| 19 | + | |
| 20 | + - Default: `./models` (relative to the repo root), configurable with `OLALA_MODELS_DIR` / the `models_dir` arg of [sbxenv.yaml](sbxenv.yaml); absolute paths accepted. | |
| 21 | + - `.gitignore`: `/models/`, `*.gguf`, `**/.cache/huggingface/` (the `hf download --local-dir` cache). | |
| 22 | + - The workspace is on kDrive: the models are synced by kDrive unless `models/` is excluded there. | |
| 23 | + - Serve with `--load-mode none`: mmap over the workspace mount is ~20x slower (see #4). | |
| 24 | +tasks: | |
| 25 | + - id: 1 | |
| 26 | + title: Default OLALA_MODELS_DIR to ./models, relative to the repo root | |
| 27 | + state: closed | |
| 28 | + priority: high | |
| 29 | + author: | |
| 30 | + name: k33g | |
| 31 | + email: ph.charriere@gmail.com | |
| 32 | + createdAt: 2026-11-21T07:08:59.000Z | |
| 33 | + updatedAt: 2026-11-21T07:08:59.000Z | |
| 34 | + closedAt: 2026-11-21T07:08:59.000Z | |
| 35 | + - id: 2 | |
| 36 | + title: models_dir arg in sbxenv.yaml | |
| 37 | + state: closed | |
| 38 | + priority: high | |
| 39 | + author: | |
| 40 | + name: k33g | |
| 41 | + email: ph.charriere@gmail.com | |
| 42 | + createdAt: 2026-11-21T07:08:59.000Z | |
| 43 | + updatedAt: 2026-11-21T07:52:58.000Z | |
| 44 | + closedAt: 2026-11-21T07:08:59.000Z | |
| 45 | + - id: 3 | |
| 46 | + title: Update .gitignore | |
| 47 | + state: closed | |
| 48 | + priority: high | |
| 49 | + author: | |
| 50 | + name: k33g | |
| 51 | + email: ph.charriere@gmail.com | |
| 52 | + createdAt: 2026-11-21T07:08:59.000Z | |
| 53 | + updatedAt: 2026-11-21T07:08:59.000Z | |
| 54 | + closedAt: 2026-11-21T07:08:59.000Z | |
| 55 | + - id: 4 | |
| 56 | + title: Move the downloaded model to ./models | |
| 57 | + state: closed | |
| 58 | + priority: high | |
| 59 | + author: | |
| 60 | + name: k33g | |
| 61 | + email: ph.charriere@gmail.com | |
| 62 | + createdAt: 2026-11-21T07:08:59.000Z | |
| 63 | + updatedAt: 2026-11-21T07:08:59.000Z | |
| 64 | + closedAt: 2026-11-21T07:08:59.000Z | |
| 65 | +comments: [] | |
| new file mode 100644 | |||
| @@ -0,0 +1,65 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +id: 9 | ||
| 3 | +title: Store models in a configurable repository folder (default ./models) | ||
| 4 | +state: closed | ||
| 5 | +author: | ||
| 6 | + name: k33g | ||
| 7 | + email: ph.charriere@gmail.com | ||
| 8 | +createdAt: 2026-11-21T07:08:59.000Z | ||
| 9 | +updatedAt: 2026-11-21T07:52:58.000Z | ||
| 10 | +labels: | ||
| 11 | + - kind::feature | ||
| 12 | + - priority::high | ||
| 13 | +epic: Olala in sbx | ||
| 14 | +related: | ||
| 15 | + - 3 | ||
| 16 | + - 5 | ||
| 17 | +body: | | ||
| 18 | + Store downloaded models in a folder of the repository, so dropping the sandbox does not mean downloading them again. | ||
| 19 | + | ||
| 20 | + - Default: `./models` (relative to the repo root), configurable with `OLALA_MODELS_DIR` / the `models_dir` arg of [sbxenv.yaml](sbxenv.yaml); absolute paths accepted. | ||
| 21 | + - `.gitignore`: `/models/`, `*.gguf`, `**/.cache/huggingface/` (the `hf download --local-dir` cache). | ||
| 22 | + - The workspace is on kDrive: the models are synced by kDrive unless `models/` is excluded there. | ||
| 23 | + - Serve with `--load-mode none`: mmap over the workspace mount is ~20x slower (see #4). | ||
| 24 | +tasks: | ||
| 25 | + - id: 1 | ||
| 26 | + title: Default OLALA_MODELS_DIR to ./models, relative to the repo root | ||
| 27 | + state: closed | ||
| 28 | + priority: high | ||
| 29 | + author: | ||
| 30 | + name: k33g | ||
| 31 | + email: ph.charriere@gmail.com | ||
| 32 | + createdAt: 2026-11-21T07:08:59.000Z | ||
| 33 | + updatedAt: 2026-11-21T07:08:59.000Z | ||
| 34 | + closedAt: 2026-11-21T07:08:59.000Z | ||
| 35 | + - id: 2 | ||
| 36 | + title: models_dir arg in sbxenv.yaml | ||
| 37 | + state: closed | ||
| 38 | + priority: high | ||
| 39 | + author: | ||
| 40 | + name: k33g | ||
| 41 | + email: ph.charriere@gmail.com | ||
| 42 | + createdAt: 2026-11-21T07:08:59.000Z | ||
| 43 | + updatedAt: 2026-11-21T07:52:58.000Z | ||
| 44 | + closedAt: 2026-11-21T07:08:59.000Z | ||
| 45 | + - id: 3 | ||
| 46 | + title: Update .gitignore | ||
| 47 | + state: closed | ||
| 48 | + priority: high | ||
| 49 | + author: | ||
| 50 | + name: k33g | ||
| 51 | + email: ph.charriere@gmail.com | ||
| 52 | + createdAt: 2026-11-21T07:08:59.000Z | ||
| 53 | + updatedAt: 2026-11-21T07:08:59.000Z | ||
| 54 | + closedAt: 2026-11-21T07:08:59.000Z | ||
| 55 | + - id: 4 | ||
| 56 | + title: Move the downloaded model to ./models | ||
| 57 | + state: closed | ||
| 58 | + priority: high | ||
| 59 | + author: | ||
| 60 | + name: k33g | ||
| 61 | + email: ph.charriere@gmail.com | ||
| 62 | + createdAt: 2026-11-21T07:08:59.000Z | ||
| 63 | + updatedAt: 2026-11-21T07:08:59.000Z | ||
| 64 | + closedAt: 2026-11-21T07:08:59.000Z | ||
| 65 | +comments: [] | ||
added
.tickets/labels.yaml +38 -0 | new file mode 100644 | ||
| @@ -0,0 +1,38 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +labels: | |
| 3 | + - name: bug | |
| 4 | + color: "#d73a4a" | |
| 5 | + description: Something is not working | |
| 6 | + - name: documentation | |
| 7 | + color: "#0075ca" | |
| 8 | + description: Improvements or additions to documentation | |
| 9 | + - name: enhancement | |
| 10 | + color: "#a2eeef" | |
| 11 | + description: New feature or request | |
| 12 | + - name: question | |
| 13 | + color: "#d876e3" | |
| 14 | + description: Further information is requested | |
| 15 | + - name: wontfix | |
| 16 | + color: "#ffffff" | |
| 17 | + description: This will not be worked on | |
| 18 | + - name: priority::low | |
| 19 | + color: "#c5def5" | |
| 20 | + description: Can wait | |
| 21 | + - name: priority::medium | |
| 22 | + color: "#fbca04" | |
| 23 | + description: Normal priority | |
| 24 | + - name: priority::high | |
| 25 | + color: "#d93f0b" | |
| 26 | + description: Should be picked up next | |
| 27 | + - name: priority::urgent | |
| 28 | + color: "#b60205" | |
| 29 | + description: Drop everything | |
| 30 | + - name: kind::bug | |
| 31 | + color: "#d73a4a" | |
| 32 | + description: Something is broken | |
| 33 | + - name: kind::feature | |
| 34 | + color: "#0e8a16" | |
| 35 | + description: Something new | |
| 36 | + - name: kind::chore | |
| 37 | + color: "#bfdadc" | |
| 38 | + description: Maintenance, no visible change | |
| new file mode 100644 | |||
| @@ -0,0 +1,38 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +labels: | ||
| 3 | + - name: bug | ||
| 4 | + color: "#d73a4a" | ||
| 5 | + description: Something is not working | ||
| 6 | + - name: documentation | ||
| 7 | + color: "#0075ca" | ||
| 8 | + description: Improvements or additions to documentation | ||
| 9 | + - name: enhancement | ||
| 10 | + color: "#a2eeef" | ||
| 11 | + description: New feature or request | ||
| 12 | + - name: question | ||
| 13 | + color: "#d876e3" | ||
| 14 | + description: Further information is requested | ||
| 15 | + - name: wontfix | ||
| 16 | + color: "#ffffff" | ||
| 17 | + description: This will not be worked on | ||
| 18 | + - name: priority::low | ||
| 19 | + color: "#c5def5" | ||
| 20 | + description: Can wait | ||
| 21 | + - name: priority::medium | ||
| 22 | + color: "#fbca04" | ||
| 23 | + description: Normal priority | ||
| 24 | + - name: priority::high | ||
| 25 | + color: "#d93f0b" | ||
| 26 | + description: Should be picked up next | ||
| 27 | + - name: priority::urgent | ||
| 28 | + color: "#b60205" | ||
| 29 | + description: Drop everything | ||
| 30 | + - name: kind::bug | ||
| 31 | + color: "#d73a4a" | ||
| 32 | + description: Something is broken | ||
| 33 | + - name: kind::feature | ||
| 34 | + color: "#0e8a16" | ||
| 35 | + description: Something new | ||
| 36 | + - name: kind::chore | ||
| 37 | + color: "#bfdadc" | ||
| 38 | + description: Maintenance, no visible change | ||
added
.tickets/milestones.yaml +2 -0 | new file mode 100644 | ||
| @@ -0,0 +1,2 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +milestones: [] | |
| new file mode 100644 | |||
| @@ -0,0 +1,2 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +milestones: [] | ||
added
.vscode/extensions.json +9 -0 | new file mode 100644 | ||
| @@ -0,0 +1,9 @@ | ||
| 1 | +{ | |
| 2 | + "recommendations": [ | |
| 3 | + "ms-azuretools.vscode-docker", | |
| 4 | + "pkief.material-icon-theme", | |
| 5 | + "pkief.material-product-icons", | |
| 6 | + "aaron-bond.better-comments", | |
| 7 | + "bierner.markdown-mermaid", | |
| 8 | + ] | |
| 9 | +} | |
| \ No newline at end of file | ||
| new file mode 100644 | |||
| @@ -0,0 +1,9 @@ | |||
| 1 | +{ | ||
| 2 | + "recommendations": [ | ||
| 3 | + "ms-azuretools.vscode-docker", | ||
| 4 | + "pkief.material-icon-theme", | ||
| 5 | + "pkief.material-product-icons", | ||
| 6 | + "aaron-bond.better-comments", | ||
| 7 | + "bierner.markdown-mermaid", | ||
| 8 | + ] | ||
| 9 | +} | ||
| \ No newline at end of file | \ No newline at end of file | ||
added
.vscode/settings.json +63 -0 | new file mode 100644 | ||
| @@ -0,0 +1,63 @@ | ||
| 1 | +{ | |
| 2 | + "workbench.iconTheme": "material-icon-theme", | |
| 3 | + "workbench.colorTheme": "Light Modern", | |
| 4 | + "editor.fontSize": 14, | |
| 5 | + "terminal.integrated.fontSize": 14, | |
| 6 | + "editor.insertSpaces": true, | |
| 7 | + "editor.tabSize": 4, | |
| 8 | + "editor.detectIndentation": true, | |
| 9 | + "files.autoSave": "afterDelay", | |
| 10 | + "files.autoSaveDelay": 1000, | |
| 11 | + // "editor.defaultFormatter": "esbenp.prettier-vscode", | |
| 12 | + "editor.formatOnSave": true, | |
| 13 | + "workbench.tree.indent": 20, | |
| 14 | + //"workbench.activityBar.location": "top", | |
| 15 | + "workbench.editor.showTabs": "multiple", | |
| 16 | + "window.zoomLevel": 0.0, | |
| 17 | + "[markdown]": { | |
| 18 | + "editor.unicodeHighlight.ambiguousCharacters": false, | |
| 19 | + "editor.unicodeHighlight.invisibleCharacters": false, | |
| 20 | + "diffEditor.ignoreTrimWhitespace": false, | |
| 21 | + "editor.fontWeight": "normal", | |
| 22 | + "editor.fontFamily": "'Droid Sans Mono', 'monospace', monospace", | |
| 23 | + "editor.fontSize": 14, | |
| 24 | + "editor.wordWrap": "on", | |
| 25 | + "editor.quickSuggestions": { | |
| 26 | + "comments": "off", | |
| 27 | + "strings": "off", | |
| 28 | + "other": "off" | |
| 29 | + } | |
| 30 | + }, | |
| 31 | + "markdown.preview.fontSize": 14, | |
| 32 | + // "workbench.editorAssociations": { | |
| 33 | + // "*.md": "vscode.markdown.preview.editor" | |
| 34 | + // }, | |
| 35 | + "markdown.marp.html": "all", | |
| 36 | + "[dockerfile]": { | |
| 37 | + "editor.fontSize": 14 | |
| 38 | + }, | |
| 39 | + "[dockercompose]": { | |
| 40 | + "editor.fontSize": 14 | |
| 41 | + }, | |
| 42 | + "[json]": { | |
| 43 | + "editor.fontSize": 14 | |
| 44 | + }, | |
| 45 | + "[yaml]": { | |
| 46 | + "editor.fontSize": 14 | |
| 47 | + }, | |
| 48 | + "[go]": { | |
| 49 | + "editor.fontSize": 14, | |
| 50 | + "editor.codeActionsOnSave": { | |
| 51 | + "source.organizeImports": "explicit" | |
| 52 | + } | |
| 53 | + }, | |
| 54 | + "go.lintTool": "golangci-lint", | |
| 55 | + "go.lintOnSave": "package", | |
| 56 | + "go.formatTool": "goimports", | |
| 57 | + "go.useLanguageServer": true, | |
| 58 | + "gopls": { | |
| 59 | + "ui.semanticTokens": true, | |
| 60 | + "ui.completion.usePlaceholders": true | |
| 61 | + }, | |
| 62 | + "issuespec.groupBy": "milestone" | |
| 63 | +} | |
| \ No newline at end of file | ||
| new file mode 100644 | |||
| @@ -0,0 +1,63 @@ | |||
| 1 | +{ | ||
| 2 | + "workbench.iconTheme": "material-icon-theme", | ||
| 3 | + "workbench.colorTheme": "Light Modern", | ||
| 4 | + "editor.fontSize": 14, | ||
| 5 | + "terminal.integrated.fontSize": 14, | ||
| 6 | + "editor.insertSpaces": true, | ||
| 7 | + "editor.tabSize": 4, | ||
| 8 | + "editor.detectIndentation": true, | ||
| 9 | + "files.autoSave": "afterDelay", | ||
| 10 | + "files.autoSaveDelay": 1000, | ||
| 11 | + // "editor.defaultFormatter": "esbenp.prettier-vscode", | ||
| 12 | + "editor.formatOnSave": true, | ||
| 13 | + "workbench.tree.indent": 20, | ||
| 14 | + //"workbench.activityBar.location": "top", | ||
| 15 | + "workbench.editor.showTabs": "multiple", | ||
| 16 | + "window.zoomLevel": 0.0, | ||
| 17 | + "[markdown]": { | ||
| 18 | + "editor.unicodeHighlight.ambiguousCharacters": false, | ||
| 19 | + "editor.unicodeHighlight.invisibleCharacters": false, | ||
| 20 | + "diffEditor.ignoreTrimWhitespace": false, | ||
| 21 | + "editor.fontWeight": "normal", | ||
| 22 | + "editor.fontFamily": "'Droid Sans Mono', 'monospace', monospace", | ||
| 23 | + "editor.fontSize": 14, | ||
| 24 | + "editor.wordWrap": "on", | ||
| 25 | + "editor.quickSuggestions": { | ||
| 26 | + "comments": "off", | ||
| 27 | + "strings": "off", | ||
| 28 | + "other": "off" | ||
| 29 | + } | ||
| 30 | + }, | ||
| 31 | + "markdown.preview.fontSize": 14, | ||
| 32 | + // "workbench.editorAssociations": { | ||
| 33 | + // "*.md": "vscode.markdown.preview.editor" | ||
| 34 | + // }, | ||
| 35 | + "markdown.marp.html": "all", | ||
| 36 | + "[dockerfile]": { | ||
| 37 | + "editor.fontSize": 14 | ||
| 38 | + }, | ||
| 39 | + "[dockercompose]": { | ||
| 40 | + "editor.fontSize": 14 | ||
| 41 | + }, | ||
| 42 | + "[json]": { | ||
| 43 | + "editor.fontSize": 14 | ||
| 44 | + }, | ||
| 45 | + "[yaml]": { | ||
| 46 | + "editor.fontSize": 14 | ||
| 47 | + }, | ||
| 48 | + "[go]": { | ||
| 49 | + "editor.fontSize": 14, | ||
| 50 | + "editor.codeActionsOnSave": { | ||
| 51 | + "source.organizeImports": "explicit" | ||
| 52 | + } | ||
| 53 | + }, | ||
| 54 | + "go.lintTool": "golangci-lint", | ||
| 55 | + "go.lintOnSave": "package", | ||
| 56 | + "go.formatTool": "goimports", | ||
| 57 | + "go.useLanguageServer": true, | ||
| 58 | + "gopls": { | ||
| 59 | + "ui.semanticTokens": true, | ||
| 60 | + "ui.completion.usePlaceholders": true | ||
| 61 | + }, | ||
| 62 | + "issuespec.groupBy": "milestone" | ||
| 63 | +} | ||
| \ No newline at end of file | \ No newline at end of file | ||
added
README.fr.md +110 -0 | new file mode 100644 | ||
| @@ -0,0 +1,110 @@ | ||
| 1 | +# Houla ... Houla | |
| 2 | + | |
| 3 | +🇬🇧 [English version](README.md) | |
| 4 | + | |
| 5 | +## 1. C'est quoi ce projet ? | |
| 6 | + | |
| 7 | +Houla fait tourner le modèle de langage **Olala-v1.0-mini-7A1B** (d'OVHai) sur ta machine, dans un sandbox Docker (`sbx`), et l'expose sous forme d'**API compatible OpenAI** sur `http://localhost:8080`. | |
| 8 | + | |
| 9 | +- Le modèle est servi par `llama-server`, issu du [fork llama.cpp d'OVHai](https://github.com/OVHai-LLM/llama.cpp) (CPU uniquement, pas besoin de GPU). | |
| 10 | +- Le fichier du modèle (GGUF, ~7 Go) est téléchargé une seule fois dans le dossier `./models`, puis réutilisé. | |
| 11 | +- Tout est décrit dans [`sbxenv.yaml`](sbxenv.yaml) : une seule commande crée le sandbox, télécharge le modèle et démarre le serveur. | |
| 12 | + | |
| 13 | +## 2. Créer le secret Hugging Face | |
| 14 | + | |
| 15 | +**Pourquoi ?** Le modèle est téléchargé depuis [Hugging Face](https://huggingface.co/OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF). Avec un token, les téléchargements sont authentifiés (moins de limites de débit, et accès aux modèles restreints ou privés). | |
| 16 | + | |
| 17 | +Le vrai token n'entre jamais dans le sandbox : le sandbox ne voit qu'une valeur factice dans `HF_TOKEN`, et le proxy sbx la remplace par le vrai token quand une requête part vers `huggingface.co`. | |
| 18 | + | |
| 19 | +1. Crée un token **Read** sur <https://huggingface.co/settings/tokens>. | |
| 20 | +2. Sur la machine hôte, lance une fois (remplace `hf_xxx` par ton token) : | |
| 21 | + | |
| 22 | +```bash | |
| 23 | +printf '%s' "hf_xxx" | sbx secret set-custom \ | |
| 24 | + --host huggingface.co \ | |
| 25 | + --host '*.huggingface.co' \ | |
| 26 | + --env HF_TOKEN \ | |
| 27 | + --value-stdin | |
| 28 | +``` | |
| 29 | + | |
| 30 | +Le secret est global (il s'applique à tous les sandboxes). Fais-le **avant** de créer le sandbox. Plus de détails : [docs/huggingface-token-sbx.fr.md](docs/huggingface-token-sbx.fr.md). | |
| 31 | + | |
| 32 | +> Ne mets jamais le token dans `sbxenv.yaml` : ce fichier est commité. | |
| 33 | + | |
| 34 | +## 3. Premier lancement | |
| 35 | + | |
| 36 | +Sur la machine hôte, à la racine du projet : | |
| 37 | + | |
| 38 | +```bash | |
| 39 | +sbx env create sbxenv.yaml | |
| 40 | +``` | |
| 41 | + | |
| 42 | +Cela crée le sandbox `houla-olala`, télécharge le modèle (la première fois seulement, quelques minutes) et démarre le serveur sur le port 8080. | |
| 43 | + | |
| 44 | +Pour tester depuis l'hôte : | |
| 45 | + | |
| 46 | +```bash | |
| 47 | +# Le serveur répond ? | |
| 48 | +curl -s http://localhost:8080/health | |
| 49 | + | |
| 50 | +# Quel modèle est servi ? | |
| 51 | +curl -s http://localhost:8080/v1/models | |
| 52 | + | |
| 53 | +# Poser une question | |
| 54 | +curl -s http://localhost:8080/v1/chat/completions \ | |
| 55 | + -H 'Content-Type: application/json' \ | |
| 56 | + -d '{ | |
| 57 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | |
| 58 | + "messages": [{"role": "user", "content": "Explique le fonctionnement d'\''une pompe à chaleur en deux phrases."}], | |
| 59 | + "reasoning_effort": "low" | |
| 60 | + }' | jq -r '.choices[0].message.content' | |
| 61 | + | |
| 62 | +# Réponse en streaming | |
| 63 | +curl -sN http://localhost:8080/v1/chat/completions \ | |
| 64 | + -H 'Content-Type: application/json' \ | |
| 65 | + -d '{ | |
| 66 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | |
| 67 | + "stream": true, | |
| 68 | + "reasoning_effort": "disabled", | |
| 69 | + "messages": [{"role": "user", "content": "Dis bonjour"}] | |
| 70 | + }' | |
| 71 | +``` | |
| 72 | + | |
| 73 | +`reasoning_effort` peut valoir `disabled`, `low`, `medium` (par défaut) ou `high`. | |
| 74 | + | |
| 75 | +## 4. Relancer après avoir supprimé le sandbox | |
| 76 | + | |
| 77 | +```bash | |
| 78 | +sbx env rm sbxenv.yaml # supprime le sandbox (le modèle reste dans ./models) | |
| 79 | +sbx env create sbxenv.yaml # le recrée : pas de nouveau téléchargement, le serveur démarre directement | |
| 80 | +``` | |
| 81 | + | |
| 82 | +Si le sandbox a seulement été **arrêté** (`sbx stop`, redémarrage de la machine), pas besoin de le recréer : | |
| 83 | + | |
| 84 | +```bash | |
| 85 | +sbx run -d --name houla-olala | |
| 86 | +sbx env exec --no-input sbxenv.yaml -- bash -lc "$PWD/scripts/olala.sh serve" | |
| 87 | +``` | |
| 88 | + | |
| 89 | +## 5. Paramètres : port et modèle | |
| 90 | + | |
| 91 | +Passe-les avec `--env-arg` à la création du sandbox : | |
| 92 | + | |
| 93 | +```bash | |
| 94 | +# Un autre port sur l'hôte | |
| 95 | +sbx env create sbxenv.yaml --env-arg host_port=9090 | |
| 96 | + | |
| 97 | +# Un autre fichier de modèle (du même dépôt Hugging Face) | |
| 98 | +sbx env create sbxenv.yaml --env-arg model_file=Olala-v1.0-mini-7A1B-q5_k_m.gguf | |
| 99 | +``` | |
| 100 | + | |
| 101 | +| Argument | Défaut | Rôle | | |
| 102 | +| --- | --- | --- | | |
| 103 | +| `host_port` | `8080` | Port sur l'hôte | | |
| 104 | +| `model_repo` | `OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF` | Dépôt Hugging Face | | |
| 105 | +| `model_file` | `Olala-v1.0-mini-7A1B-q8_0-safe2.gguf` | Fichier GGUF à télécharger et servir | | |
| 106 | + | |
| 107 | +> Utilise les **mêmes** valeurs `--env-arg` pour toutes les commandes `sbx env` de ce sandbox (`rm`, `exec`…). | |
| 108 | +> Avec un autre port, remplace `8080` dans les URL curl. Avec un autre modèle, le nom `"model"` dans les requêtes est le nom du fichier sans `.gguf`. | |
| 109 | + | |
| 110 | +Autres paramètres (`models_dir`, `threads`, `ctx`) et détails : [docs/olala-sbx.fr.md](docs/olala-sbx.fr.md). | |
| new file mode 100644 | |||
| @@ -0,0 +1,110 @@ | |||
| 1 | +# Houla ... Houla | ||
| 2 | + | ||
| 3 | +🇬🇧 [English version](README.md) | ||
| 4 | + | ||
| 5 | +## 1. C'est quoi ce projet ? | ||
| 6 | + | ||
| 7 | +Houla fait tourner le modèle de langage **Olala-v1.0-mini-7A1B** (d'OVHai) sur ta machine, dans un sandbox Docker (`sbx`), et l'expose sous forme d'**API compatible OpenAI** sur `http://localhost:8080`. | ||
| 8 | + | ||
| 9 | +- Le modèle est servi par `llama-server`, issu du [fork llama.cpp d'OVHai](https://github.com/OVHai-LLM/llama.cpp) (CPU uniquement, pas besoin de GPU). | ||
| 10 | +- Le fichier du modèle (GGUF, ~7 Go) est téléchargé une seule fois dans le dossier `./models`, puis réutilisé. | ||
| 11 | +- Tout est décrit dans [`sbxenv.yaml`](sbxenv.yaml) : une seule commande crée le sandbox, télécharge le modèle et démarre le serveur. | ||
| 12 | + | ||
| 13 | +## 2. Créer le secret Hugging Face | ||
| 14 | + | ||
| 15 | +**Pourquoi ?** Le modèle est téléchargé depuis [Hugging Face](https://huggingface.co/OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF). Avec un token, les téléchargements sont authentifiés (moins de limites de débit, et accès aux modèles restreints ou privés). | ||
| 16 | + | ||
| 17 | +Le vrai token n'entre jamais dans le sandbox : le sandbox ne voit qu'une valeur factice dans `HF_TOKEN`, et le proxy sbx la remplace par le vrai token quand une requête part vers `huggingface.co`. | ||
| 18 | + | ||
| 19 | +1. Crée un token **Read** sur <https://huggingface.co/settings/tokens>. | ||
| 20 | +2. Sur la machine hôte, lance une fois (remplace `hf_xxx` par ton token) : | ||
| 21 | + | ||
| 22 | +```bash | ||
| 23 | +printf '%s' "hf_xxx" | sbx secret set-custom \ | ||
| 24 | + --host huggingface.co \ | ||
| 25 | + --host '*.huggingface.co' \ | ||
| 26 | + --env HF_TOKEN \ | ||
| 27 | + --value-stdin | ||
| 28 | +``` | ||
| 29 | + | ||
| 30 | +Le secret est global (il s'applique à tous les sandboxes). Fais-le **avant** de créer le sandbox. Plus de détails : [docs/huggingface-token-sbx.fr.md](docs/huggingface-token-sbx.fr.md). | ||
| 31 | + | ||
| 32 | +> Ne mets jamais le token dans `sbxenv.yaml` : ce fichier est commité. | ||
| 33 | + | ||
| 34 | +## 3. Premier lancement | ||
| 35 | + | ||
| 36 | +Sur la machine hôte, à la racine du projet : | ||
| 37 | + | ||
| 38 | +```bash | ||
| 39 | +sbx env create sbxenv.yaml | ||
| 40 | +``` | ||
| 41 | + | ||
| 42 | +Cela crée le sandbox `houla-olala`, télécharge le modèle (la première fois seulement, quelques minutes) et démarre le serveur sur le port 8080. | ||
| 43 | + | ||
| 44 | +Pour tester depuis l'hôte : | ||
| 45 | + | ||
| 46 | +```bash | ||
| 47 | +# Le serveur répond ? | ||
| 48 | +curl -s http://localhost:8080/health | ||
| 49 | + | ||
| 50 | +# Quel modèle est servi ? | ||
| 51 | +curl -s http://localhost:8080/v1/models | ||
| 52 | + | ||
| 53 | +# Poser une question | ||
| 54 | +curl -s http://localhost:8080/v1/chat/completions \ | ||
| 55 | + -H 'Content-Type: application/json' \ | ||
| 56 | + -d '{ | ||
| 57 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | ||
| 58 | + "messages": [{"role": "user", "content": "Explique le fonctionnement d'\''une pompe à chaleur en deux phrases."}], | ||
| 59 | + "reasoning_effort": "low" | ||
| 60 | + }' | jq -r '.choices[0].message.content' | ||
| 61 | + | ||
| 62 | +# Réponse en streaming | ||
| 63 | +curl -sN http://localhost:8080/v1/chat/completions \ | ||
| 64 | + -H 'Content-Type: application/json' \ | ||
| 65 | + -d '{ | ||
| 66 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | ||
| 67 | + "stream": true, | ||
| 68 | + "reasoning_effort": "disabled", | ||
| 69 | + "messages": [{"role": "user", "content": "Dis bonjour"}] | ||
| 70 | + }' | ||
| 71 | +``` | ||
| 72 | + | ||
| 73 | +`reasoning_effort` peut valoir `disabled`, `low`, `medium` (par défaut) ou `high`. | ||
| 74 | + | ||
| 75 | +## 4. Relancer après avoir supprimé le sandbox | ||
| 76 | + | ||
| 77 | +```bash | ||
| 78 | +sbx env rm sbxenv.yaml # supprime le sandbox (le modèle reste dans ./models) | ||
| 79 | +sbx env create sbxenv.yaml # le recrée : pas de nouveau téléchargement, le serveur démarre directement | ||
| 80 | +``` | ||
| 81 | + | ||
| 82 | +Si le sandbox a seulement été **arrêté** (`sbx stop`, redémarrage de la machine), pas besoin de le recréer : | ||
| 83 | + | ||
| 84 | +```bash | ||
| 85 | +sbx run -d --name houla-olala | ||
| 86 | +sbx env exec --no-input sbxenv.yaml -- bash -lc "$PWD/scripts/olala.sh serve" | ||
| 87 | +``` | ||
| 88 | + | ||
| 89 | +## 5. Paramètres : port et modèle | ||
| 90 | + | ||
| 91 | +Passe-les avec `--env-arg` à la création du sandbox : | ||
| 92 | + | ||
| 93 | +```bash | ||
| 94 | +# Un autre port sur l'hôte | ||
| 95 | +sbx env create sbxenv.yaml --env-arg host_port=9090 | ||
| 96 | + | ||
| 97 | +# Un autre fichier de modèle (du même dépôt Hugging Face) | ||
| 98 | +sbx env create sbxenv.yaml --env-arg model_file=Olala-v1.0-mini-7A1B-q5_k_m.gguf | ||
| 99 | +``` | ||
| 100 | + | ||
| 101 | +| Argument | Défaut | Rôle | | ||
| 102 | +| --- | --- | --- | | ||
| 103 | +| `host_port` | `8080` | Port sur l'hôte | | ||
| 104 | +| `model_repo` | `OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF` | Dépôt Hugging Face | | ||
| 105 | +| `model_file` | `Olala-v1.0-mini-7A1B-q8_0-safe2.gguf` | Fichier GGUF à télécharger et servir | | ||
| 106 | + | ||
| 107 | +> Utilise les **mêmes** valeurs `--env-arg` pour toutes les commandes `sbx env` de ce sandbox (`rm`, `exec`…). | ||
| 108 | +> Avec un autre port, remplace `8080` dans les URL curl. Avec un autre modèle, le nom `"model"` dans les requêtes est le nom du fichier sans `.gguf`. | ||
| 109 | + | ||
| 110 | +Autres paramètres (`models_dir`, `threads`, `ctx`) et détails : [docs/olala-sbx.fr.md](docs/olala-sbx.fr.md). | ||
added
README.md +110 -0 | new file mode 100644 | ||
| @@ -0,0 +1,110 @@ | ||
| 1 | +# Houla ... Houla | |
| 2 | + | |
| 3 | +🇫🇷 [Version française](README.fr.md) | |
| 4 | + | |
| 5 | +## 1. What is this project? | |
| 6 | + | |
| 7 | +Houla runs the **Olala-v1.0-mini-7A1B** language model (by OVHai) on your machine, inside a Docker sandbox (`sbx`), and exposes it as an **OpenAI-compatible API** on `http://localhost:8080`. | |
| 8 | + | |
| 9 | +- The model is served by `llama-server`, from OVHai's [llama.cpp fork](https://github.com/OVHai-LLM/llama.cpp) (CPU only, no GPU needed). | |
| 10 | +- The model file (GGUF, ~7 GB) is downloaded once into the `./models` folder and reused afterwards. | |
| 11 | +- Everything is described in [`sbxenv.yaml`](sbxenv.yaml): one command creates the sandbox, downloads the model and starts the server. | |
| 12 | + | |
| 13 | +## 2. Create the Hugging Face secret | |
| 14 | + | |
| 15 | +**Why?** The model is downloaded from [Hugging Face](https://huggingface.co/OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF). With a token, downloads are authenticated (fewer rate limits, and access to gated or private models). | |
| 16 | + | |
| 17 | +The real token never enters the sandbox: the sandbox only sees a placeholder in `HF_TOKEN`, and the sbx proxy swaps in the real token when a request goes to `huggingface.co`. | |
| 18 | + | |
| 19 | +1. Create a **Read** token at <https://huggingface.co/settings/tokens>. | |
| 20 | +2. On the host, run once (replace `hf_xxx` with your token): | |
| 21 | + | |
| 22 | +```bash | |
| 23 | +printf '%s' "hf_xxx" | sbx secret set-custom \ | |
| 24 | + --host huggingface.co \ | |
| 25 | + --host '*.huggingface.co' \ | |
| 26 | + --env HF_TOKEN \ | |
| 27 | + --value-stdin | |
| 28 | +``` | |
| 29 | + | |
| 30 | +The secret is global (it applies to every sandbox). Do this **before** creating the sandbox. More details: [docs/huggingface-token-sbx.en.md](docs/huggingface-token-sbx.en.md). | |
| 31 | + | |
| 32 | +> Never put the token in `sbxenv.yaml`: that file is committed. | |
| 33 | + | |
| 34 | +## 3. First launch | |
| 35 | + | |
| 36 | +On the host, at the project root: | |
| 37 | + | |
| 38 | +```bash | |
| 39 | +sbx env create sbxenv.yaml | |
| 40 | +``` | |
| 41 | + | |
| 42 | +This creates the `houla-olala` sandbox, downloads the model (first time only, a few minutes) and starts the server on port 8080. | |
| 43 | + | |
| 44 | +Test it from the host: | |
| 45 | + | |
| 46 | +```bash | |
| 47 | +# Is the server up? | |
| 48 | +curl -s http://localhost:8080/health | |
| 49 | + | |
| 50 | +# Which model is served? | |
| 51 | +curl -s http://localhost:8080/v1/models | |
| 52 | + | |
| 53 | +# Ask a question | |
| 54 | +curl -s http://localhost:8080/v1/chat/completions \ | |
| 55 | + -H 'Content-Type: application/json' \ | |
| 56 | + -d '{ | |
| 57 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | |
| 58 | + "messages": [{"role": "user", "content": "Explain how a heat pump works in two sentences."}], | |
| 59 | + "reasoning_effort": "low" | |
| 60 | + }' | jq -r '.choices[0].message.content' | |
| 61 | + | |
| 62 | +# Streaming answer | |
| 63 | +curl -sN http://localhost:8080/v1/chat/completions \ | |
| 64 | + -H 'Content-Type: application/json' \ | |
| 65 | + -d '{ | |
| 66 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | |
| 67 | + "stream": true, | |
| 68 | + "reasoning_effort": "disabled", | |
| 69 | + "messages": [{"role": "user", "content": "Say hello"}] | |
| 70 | + }' | |
| 71 | +``` | |
| 72 | + | |
| 73 | +`reasoning_effort` can be `disabled`, `low`, `medium` (default) or `high`. | |
| 74 | + | |
| 75 | +## 4. Start again after deleting the sandbox | |
| 76 | + | |
| 77 | +```bash | |
| 78 | +sbx env rm sbxenv.yaml # delete the sandbox (the model stays in ./models) | |
| 79 | +sbx env create sbxenv.yaml # recreate it: no new download, the server starts directly | |
| 80 | +``` | |
| 81 | + | |
| 82 | +If the sandbox was only **stopped** (`sbx stop`, reboot), no need to recreate it: | |
| 83 | + | |
| 84 | +```bash | |
| 85 | +sbx run -d --name houla-olala | |
| 86 | +sbx env exec --no-input sbxenv.yaml -- bash -lc "$PWD/scripts/olala.sh serve" | |
| 87 | +``` | |
| 88 | + | |
| 89 | +## 5. Settings: port and model | |
| 90 | + | |
| 91 | +Pass them with `--env-arg` when creating the sandbox: | |
| 92 | + | |
| 93 | +```bash | |
| 94 | +# Another host port | |
| 95 | +sbx env create sbxenv.yaml --env-arg host_port=9090 | |
| 96 | + | |
| 97 | +# Another model file (from the same Hugging Face repository) | |
| 98 | +sbx env create sbxenv.yaml --env-arg model_file=Olala-v1.0-mini-7A1B-q5_k_m.gguf | |
| 99 | +``` | |
| 100 | + | |
| 101 | +| Argument | Default | Purpose | | |
| 102 | +| --- | --- | --- | | |
| 103 | +| `host_port` | `8080` | Port on the host | | |
| 104 | +| `model_repo` | `OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF` | Hugging Face repository | | |
| 105 | +| `model_file` | `Olala-v1.0-mini-7A1B-q8_0-safe2.gguf` | GGUF file to download and serve | | |
| 106 | + | |
| 107 | +> Use the **same** `--env-arg` values with every `sbx env` command of that sandbox (`rm`, `exec`…). | |
| 108 | +> With another port, replace `8080` in the curl URLs. With another model, the `"model"` name in the requests is the file name without `.gguf`. | |
| 109 | + | |
| 110 | +Other settings (`models_dir`, `threads`, `ctx`) and details: [docs/olala-sbx.en.md](docs/olala-sbx.en.md). | |
| new file mode 100644 | |||
| @@ -0,0 +1,110 @@ | |||
| 1 | +# Houla ... Houla | ||
| 2 | + | ||
| 3 | +🇫🇷 [Version française](README.fr.md) | ||
| 4 | + | ||
| 5 | +## 1. What is this project? | ||
| 6 | + | ||
| 7 | +Houla runs the **Olala-v1.0-mini-7A1B** language model (by OVHai) on your machine, inside a Docker sandbox (`sbx`), and exposes it as an **OpenAI-compatible API** on `http://localhost:8080`. | ||
| 8 | + | ||
| 9 | +- The model is served by `llama-server`, from OVHai's [llama.cpp fork](https://github.com/OVHai-LLM/llama.cpp) (CPU only, no GPU needed). | ||
| 10 | +- The model file (GGUF, ~7 GB) is downloaded once into the `./models` folder and reused afterwards. | ||
| 11 | +- Everything is described in [`sbxenv.yaml`](sbxenv.yaml): one command creates the sandbox, downloads the model and starts the server. | ||
| 12 | + | ||
| 13 | +## 2. Create the Hugging Face secret | ||
| 14 | + | ||
| 15 | +**Why?** The model is downloaded from [Hugging Face](https://huggingface.co/OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF). With a token, downloads are authenticated (fewer rate limits, and access to gated or private models). | ||
| 16 | + | ||
| 17 | +The real token never enters the sandbox: the sandbox only sees a placeholder in `HF_TOKEN`, and the sbx proxy swaps in the real token when a request goes to `huggingface.co`. | ||
| 18 | + | ||
| 19 | +1. Create a **Read** token at <https://huggingface.co/settings/tokens>. | ||
| 20 | +2. On the host, run once (replace `hf_xxx` with your token): | ||
| 21 | + | ||
| 22 | +```bash | ||
| 23 | +printf '%s' "hf_xxx" | sbx secret set-custom \ | ||
| 24 | + --host huggingface.co \ | ||
| 25 | + --host '*.huggingface.co' \ | ||
| 26 | + --env HF_TOKEN \ | ||
| 27 | + --value-stdin | ||
| 28 | +``` | ||
| 29 | + | ||
| 30 | +The secret is global (it applies to every sandbox). Do this **before** creating the sandbox. More details: [docs/huggingface-token-sbx.en.md](docs/huggingface-token-sbx.en.md). | ||
| 31 | + | ||
| 32 | +> Never put the token in `sbxenv.yaml`: that file is committed. | ||
| 33 | + | ||
| 34 | +## 3. First launch | ||
| 35 | + | ||
| 36 | +On the host, at the project root: | ||
| 37 | + | ||
| 38 | +```bash | ||
| 39 | +sbx env create sbxenv.yaml | ||
| 40 | +``` | ||
| 41 | + | ||
| 42 | +This creates the `houla-olala` sandbox, downloads the model (first time only, a few minutes) and starts the server on port 8080. | ||
| 43 | + | ||
| 44 | +Test it from the host: | ||
| 45 | + | ||
| 46 | +```bash | ||
| 47 | +# Is the server up? | ||
| 48 | +curl -s http://localhost:8080/health | ||
| 49 | + | ||
| 50 | +# Which model is served? | ||
| 51 | +curl -s http://localhost:8080/v1/models | ||
| 52 | + | ||
| 53 | +# Ask a question | ||
| 54 | +curl -s http://localhost:8080/v1/chat/completions \ | ||
| 55 | + -H 'Content-Type: application/json' \ | ||
| 56 | + -d '{ | ||
| 57 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | ||
| 58 | + "messages": [{"role": "user", "content": "Explain how a heat pump works in two sentences."}], | ||
| 59 | + "reasoning_effort": "low" | ||
| 60 | + }' | jq -r '.choices[0].message.content' | ||
| 61 | + | ||
| 62 | +# Streaming answer | ||
| 63 | +curl -sN http://localhost:8080/v1/chat/completions \ | ||
| 64 | + -H 'Content-Type: application/json' \ | ||
| 65 | + -d '{ | ||
| 66 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | ||
| 67 | + "stream": true, | ||
| 68 | + "reasoning_effort": "disabled", | ||
| 69 | + "messages": [{"role": "user", "content": "Say hello"}] | ||
| 70 | + }' | ||
| 71 | +``` | ||
| 72 | + | ||
| 73 | +`reasoning_effort` can be `disabled`, `low`, `medium` (default) or `high`. | ||
| 74 | + | ||
| 75 | +## 4. Start again after deleting the sandbox | ||
| 76 | + | ||
| 77 | +```bash | ||
| 78 | +sbx env rm sbxenv.yaml # delete the sandbox (the model stays in ./models) | ||
| 79 | +sbx env create sbxenv.yaml # recreate it: no new download, the server starts directly | ||
| 80 | +``` | ||
| 81 | + | ||
| 82 | +If the sandbox was only **stopped** (`sbx stop`, reboot), no need to recreate it: | ||
| 83 | + | ||
| 84 | +```bash | ||
| 85 | +sbx run -d --name houla-olala | ||
| 86 | +sbx env exec --no-input sbxenv.yaml -- bash -lc "$PWD/scripts/olala.sh serve" | ||
| 87 | +``` | ||
| 88 | + | ||
| 89 | +## 5. Settings: port and model | ||
| 90 | + | ||
| 91 | +Pass them with `--env-arg` when creating the sandbox: | ||
| 92 | + | ||
| 93 | +```bash | ||
| 94 | +# Another host port | ||
| 95 | +sbx env create sbxenv.yaml --env-arg host_port=9090 | ||
| 96 | + | ||
| 97 | +# Another model file (from the same Hugging Face repository) | ||
| 98 | +sbx env create sbxenv.yaml --env-arg model_file=Olala-v1.0-mini-7A1B-q5_k_m.gguf | ||
| 99 | +``` | ||
| 100 | + | ||
| 101 | +| Argument | Default | Purpose | | ||
| 102 | +| --- | --- | --- | | ||
| 103 | +| `host_port` | `8080` | Port on the host | | ||
| 104 | +| `model_repo` | `OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF` | Hugging Face repository | | ||
| 105 | +| `model_file` | `Olala-v1.0-mini-7A1B-q8_0-safe2.gguf` | GGUF file to download and serve | | ||
| 106 | + | ||
| 107 | +> Use the **same** `--env-arg` values with every `sbx env` command of that sandbox (`rm`, `exec`…). | ||
| 108 | +> With another port, replace `8080` in the curl URLs. With another model, the `"model"` name in the requests is the file name without `.gguf`. | ||
| 109 | + | ||
| 110 | +Other settings (`models_dir`, `threads`, `ctx`) and details: [docs/olala-sbx.en.md](docs/olala-sbx.en.md). | ||
added
docs/huggingface-token-sbx.en.md +74 -0 | new file mode 100644 | ||
| @@ -0,0 +1,74 @@ | ||
| 1 | +# Inject a Hugging Face token into an sbx sandbox | |
| 2 | + | |
| 3 | +This guide explains how to make a Hugging Face token usable in a Docker sandbox (`sbx`) without storing the real token in the container. | |
| 4 | + | |
| 5 | +> **Status:** `sbx secret set-custom` is **experimental**. Its behavior, options and placeholder format may change. | |
| 6 | + | |
| 7 | +## How it works | |
| 8 | + | |
| 9 | +Hugging Face is not one of the services built into `sbx` (like `github`). You therefore use a **custom secret**: | |
| 10 | + | |
| 11 | +- Inside the sandbox, the `HF_TOKEN` variable holds a **placeholder** (`sbx-cs-…`). | |
| 12 | +- The real token stays on the host machine, in the operating system's keychain. | |
| 13 | +- When a process in the sandbox sends an HTTP request to one of the declared domains, the proxy replaces the placeholder with the real token, **in HTTP headers only**, not in the request body. | |
| 14 | + | |
| 15 | +For Hugging Face, that is enough: the token is sent in the `Authorization: Bearer …` header. | |
| 16 | + | |
| 17 | +## Set up the secret | |
| 18 | + | |
| 19 | +All `sbx` commands run **on the host machine**, not in the sandbox. | |
| 20 | + | |
| 21 | +The secret is **global**: without `--sandbox`, it applies to every sandbox, so both the dev sandbox (`houla-houla`, `sbxenv.dev.yaml`) and the Olala sandbox (`houla-olala`, `sbxenv.yaml`). There is only one command to run, once. | |
| 22 | + | |
| 23 | +```bash | |
| 24 | +printf '%s' "hf_xxx" | sbx secret set-custom \ | |
| 25 | + --host huggingface.co \ | |
| 26 | + --host '*.huggingface.co' \ | |
| 27 | + --env HF_TOKEN \ | |
| 28 | + --value-stdin | |
| 29 | +``` | |
| 30 | + | |
| 31 | +| Option | Purpose | | |
| 32 | +| --- | --- | | |
| 33 | +| `--host` | Domain the token is sent to. Repeatable. `*` matches a single subdomain level (e.g. `router.huggingface.co`, `cdn-lfs.huggingface.co`), `**` matches several. | | |
| 34 | +| `--env` | Name of the environment variable set in the sandbox. | | |
| 35 | +| `--value-stdin` | Reads the token from standard input (recommended). Without this option, the command prompts for the value. | | |
| 36 | +| `--sandbox <name>` | Restricts the secret to a single sandbox. Not used here: we want the secret everywhere. | | |
| 37 | +| `--ref` | Reads the token from an external source when needed, e.g. `op://Vault/Item/credential` (1Password). | | |
| 38 | +| `--command` | Reads the token when needed by running a command on the host that prints it. | | |
| 39 | +| `--refresh` | How long a token read by `--ref` or `--command` is cached (`<duration>` or `on-demand`). | | |
| 40 | + | |
| 41 | +> **Warning:** avoid `--value hf_xxx`: the token would end up in your shell history and in the process list. | |
| 42 | + | |
| 43 | +To check that the secret is registered: | |
| 44 | + | |
| 45 | +```bash | |
| 46 | +sbx secret ls --host huggingface.co | |
| 47 | +``` | |
| 48 | + | |
| 49 | +## Check inside the sandbox | |
| 50 | + | |
| 51 | +```bash | |
| 52 | +echo "$HF_TOKEN" # prints the placeholder, not the real token | |
| 53 | +hf auth whoami # should print your Hugging Face account | |
| 54 | +``` | |
| 55 | + | |
| 56 | +## Possible problems | |
| 57 | + | |
| 58 | +- **`HF_TOKEN` is empty in an existing sandbox**: for built-in secrets (like `github`), a global secret only applies to sandboxes created afterwards. It is probably the same here (not verified): recreate the sandbox, e.g. `sbx env rm sbxenv.dev.yaml` then `sbx env create sbxenv.dev.yaml`. For a single existing sandbox, you can also re-run the command with `--sandbox <name>` (for `github`, a sandbox-scoped secret applies immediately). | |
| 59 | +- **A tool rejects the placeholder because it does not start with `hf_`**: a `--placeholder` option would let you choose its format (e.g. `--placeholder 'hf_{rand}'`). It does not appear on the detailed docs page: check it with `sbx secret set-custom --help`. | |
| 60 | +- **HTTP 403 error "no matching allow rule"**: the domain is blocked by the sandbox firewall. On the host: | |
| 61 | + | |
| 62 | + ```bash | |
| 63 | + sbx policy allow network huggingface.co,cdn-lfs.huggingface.co | |
| 64 | + ``` | |
| 65 | + | |
| 66 | +## Don't | |
| 67 | + | |
| 68 | +Don't put the token in the `env:` section of `sbxenv.yaml` or `sbxenv.dev.yaml`: these files are meant to be committed, and the token would end up in the repository. | |
| 69 | + | |
| 70 | +## References | |
| 71 | + | |
| 72 | +- [Manage credentials – Docker Sandboxes](https://docs.docker.com/ai/sandboxes/configuration/credentials/) | |
| 73 | +- [`sbx secret set-custom`](https://docs.docker.com/reference/cli/sbx/secret/set-custom/) | |
| 74 | +- [`sbx secret`](https://docs.docker.com/reference/cli/sbx/secret/) | |
| new file mode 100644 | |||
| @@ -0,0 +1,74 @@ | |||
| 1 | +# Inject a Hugging Face token into an sbx sandbox | ||
| 2 | + | ||
| 3 | +This guide explains how to make a Hugging Face token usable in a Docker sandbox (`sbx`) without storing the real token in the container. | ||
| 4 | + | ||
| 5 | +> **Status:** `sbx secret set-custom` is **experimental**. Its behavior, options and placeholder format may change. | ||
| 6 | + | ||
| 7 | +## How it works | ||
| 8 | + | ||
| 9 | +Hugging Face is not one of the services built into `sbx` (like `github`). You therefore use a **custom secret**: | ||
| 10 | + | ||
| 11 | +- Inside the sandbox, the `HF_TOKEN` variable holds a **placeholder** (`sbx-cs-…`). | ||
| 12 | +- The real token stays on the host machine, in the operating system's keychain. | ||
| 13 | +- When a process in the sandbox sends an HTTP request to one of the declared domains, the proxy replaces the placeholder with the real token, **in HTTP headers only**, not in the request body. | ||
| 14 | + | ||
| 15 | +For Hugging Face, that is enough: the token is sent in the `Authorization: Bearer …` header. | ||
| 16 | + | ||
| 17 | +## Set up the secret | ||
| 18 | + | ||
| 19 | +All `sbx` commands run **on the host machine**, not in the sandbox. | ||
| 20 | + | ||
| 21 | +The secret is **global**: without `--sandbox`, it applies to every sandbox, so both the dev sandbox (`houla-houla`, `sbxenv.dev.yaml`) and the Olala sandbox (`houla-olala`, `sbxenv.yaml`). There is only one command to run, once. | ||
| 22 | + | ||
| 23 | +```bash | ||
| 24 | +printf '%s' "hf_xxx" | sbx secret set-custom \ | ||
| 25 | + --host huggingface.co \ | ||
| 26 | + --host '*.huggingface.co' \ | ||
| 27 | + --env HF_TOKEN \ | ||
| 28 | + --value-stdin | ||
| 29 | +``` | ||
| 30 | + | ||
| 31 | +| Option | Purpose | | ||
| 32 | +| --- | --- | | ||
| 33 | +| `--host` | Domain the token is sent to. Repeatable. `*` matches a single subdomain level (e.g. `router.huggingface.co`, `cdn-lfs.huggingface.co`), `**` matches several. | | ||
| 34 | +| `--env` | Name of the environment variable set in the sandbox. | | ||
| 35 | +| `--value-stdin` | Reads the token from standard input (recommended). Without this option, the command prompts for the value. | | ||
| 36 | +| `--sandbox <name>` | Restricts the secret to a single sandbox. Not used here: we want the secret everywhere. | | ||
| 37 | +| `--ref` | Reads the token from an external source when needed, e.g. `op://Vault/Item/credential` (1Password). | | ||
| 38 | +| `--command` | Reads the token when needed by running a command on the host that prints it. | | ||
| 39 | +| `--refresh` | How long a token read by `--ref` or `--command` is cached (`<duration>` or `on-demand`). | | ||
| 40 | + | ||
| 41 | +> **Warning:** avoid `--value hf_xxx`: the token would end up in your shell history and in the process list. | ||
| 42 | + | ||
| 43 | +To check that the secret is registered: | ||
| 44 | + | ||
| 45 | +```bash | ||
| 46 | +sbx secret ls --host huggingface.co | ||
| 47 | +``` | ||
| 48 | + | ||
| 49 | +## Check inside the sandbox | ||
| 50 | + | ||
| 51 | +```bash | ||
| 52 | +echo "$HF_TOKEN" # prints the placeholder, not the real token | ||
| 53 | +hf auth whoami # should print your Hugging Face account | ||
| 54 | +``` | ||
| 55 | + | ||
| 56 | +## Possible problems | ||
| 57 | + | ||
| 58 | +- **`HF_TOKEN` is empty in an existing sandbox**: for built-in secrets (like `github`), a global secret only applies to sandboxes created afterwards. It is probably the same here (not verified): recreate the sandbox, e.g. `sbx env rm sbxenv.dev.yaml` then `sbx env create sbxenv.dev.yaml`. For a single existing sandbox, you can also re-run the command with `--sandbox <name>` (for `github`, a sandbox-scoped secret applies immediately). | ||
| 59 | +- **A tool rejects the placeholder because it does not start with `hf_`**: a `--placeholder` option would let you choose its format (e.g. `--placeholder 'hf_{rand}'`). It does not appear on the detailed docs page: check it with `sbx secret set-custom --help`. | ||
| 60 | +- **HTTP 403 error "no matching allow rule"**: the domain is blocked by the sandbox firewall. On the host: | ||
| 61 | + | ||
| 62 | + ```bash | ||
| 63 | + sbx policy allow network huggingface.co,cdn-lfs.huggingface.co | ||
| 64 | + ``` | ||
| 65 | + | ||
| 66 | +## Don't | ||
| 67 | + | ||
| 68 | +Don't put the token in the `env:` section of `sbxenv.yaml` or `sbxenv.dev.yaml`: these files are meant to be committed, and the token would end up in the repository. | ||
| 69 | + | ||
| 70 | +## References | ||
| 71 | + | ||
| 72 | +- [Manage credentials – Docker Sandboxes](https://docs.docker.com/ai/sandboxes/configuration/credentials/) | ||
| 73 | +- [`sbx secret set-custom`](https://docs.docker.com/reference/cli/sbx/secret/set-custom/) | ||
| 74 | +- [`sbx secret`](https://docs.docker.com/reference/cli/sbx/secret/) | ||
added
docs/huggingface-token-sbx.fr.md +74 -0 | new file mode 100644 | ||
| @@ -0,0 +1,74 @@ | ||
| 1 | +# Injecter un token Hugging Face dans un sandbox sbx | |
| 2 | + | |
| 3 | +Ce guide explique comment rendre un token Hugging Face utilisable dans un sandbox Docker (`sbx`) sans que le vrai token soit stocké dans le conteneur. | |
| 4 | + | |
| 5 | +> **Statut :** `sbx secret set-custom` est **expérimental**. Son comportement, ses options et le format de la valeur factice peuvent changer. | |
| 6 | + | |
| 7 | +## Principe | |
| 8 | + | |
| 9 | +Hugging Face ne fait pas partie des services intégrés à `sbx` (comme `github`). Il faut donc passer par un **secret personnalisé** : | |
| 10 | + | |
| 11 | +- Dans le sandbox, la variable `HF_TOKEN` contient une **valeur factice** (`sbx-cs-…`). | |
| 12 | +- Le vrai token reste sur la machine hôte, dans le trousseau du système d'exploitation. | |
| 13 | +- Quand un processus du sandbox envoie une requête HTTP vers un des domaines déclarés, le proxy remplace la valeur factice par le vrai token, **uniquement dans les en-têtes HTTP**, pas dans le corps de la requête. | |
| 14 | + | |
| 15 | +Pour Hugging Face, cela suffit : le token est envoyé dans l'en-tête `Authorization: Bearer …`. | |
| 16 | + | |
| 17 | +## Configurer le secret | |
| 18 | + | |
| 19 | +Toutes les commandes `sbx` se lancent **sur la machine hôte**, pas dans le sandbox. | |
| 20 | + | |
| 21 | +Le secret est **global** : sans `--sandbox`, il s'applique à tous les sandboxes, donc aussi bien au sandbox de dev (`houla-houla`, `sbxenv.dev.yaml`) qu'au sandbox Olala (`houla-olala`, `sbxenv.yaml`). Il n'y a qu'une commande à lancer, une seule fois. | |
| 22 | + | |
| 23 | +```bash | |
| 24 | +printf '%s' "hf_xxx" | sbx secret set-custom \ | |
| 25 | + --host huggingface.co \ | |
| 26 | + --host '*.huggingface.co' \ | |
| 27 | + --env HF_TOKEN \ | |
| 28 | + --value-stdin | |
| 29 | +``` | |
| 30 | + | |
| 31 | +| Option | Rôle | | |
| 32 | +| --- | --- | | |
| 33 | +| `--host` | Domaine auquel le token est envoyé. Répétable. `*` correspond à un seul niveau de sous-domaine (ex. `router.huggingface.co`, `cdn-lfs.huggingface.co`), `**` à plusieurs. | | |
| 34 | +| `--env` | Nom de la variable d'environnement définie dans le sandbox. | | |
| 35 | +| `--value-stdin` | Lit le token sur l'entrée standard (méthode recommandée). Sans cette option, la commande demande la valeur. | | |
| 36 | +| `--sandbox <nom>` | Limite le secret à un seul sandbox. Non utilisé ici : on veut le secret partout. | | |
| 37 | +| `--ref` | Lit le token au moment où il en a besoin dans une source externe, par ex. `op://Vault/Item/credential` (1Password). | | |
| 38 | +| `--command` | Lit le token au moment où il en a besoin en lançant une commande sur l'hôte qui l'affiche. | | |
| 39 | +| `--refresh` | Durée de mise en cache d'un token lu par `--ref` ou `--command` (`<durée>` ou `on-demand`). | | |
| 40 | + | |
| 41 | +> **Attention :** évite `--value hf_xxx` : le token se retrouverait dans l'historique du shell et dans la liste des processus. | |
| 42 | + | |
| 43 | +Pour vérifier que le secret est enregistré : | |
| 44 | + | |
| 45 | +```bash | |
| 46 | +sbx secret ls --host huggingface.co | |
| 47 | +``` | |
| 48 | + | |
| 49 | +## Vérifier dans le sandbox | |
| 50 | + | |
| 51 | +```bash | |
| 52 | +echo "$HF_TOKEN" # affiche la valeur factice, pas le vrai token | |
| 53 | +hf auth whoami # doit afficher ton compte Hugging Face | |
| 54 | +``` | |
| 55 | + | |
| 56 | +## Problèmes possibles | |
| 57 | + | |
| 58 | +- **`HF_TOKEN` est vide dans un sandbox déjà créé** : pour les secrets intégrés (comme `github`), un secret global ne s'applique qu'aux sandboxes créés ensuite. C'est sans doute pareil ici (pas vérifié) : recrée le sandbox, par ex. `sbx env rm sbxenv.dev.yaml` puis `sbx env create sbxenv.dev.yaml`. Pour un seul sandbox existant, tu peux aussi relancer la commande avec `--sandbox <nom>` (pour `github`, un secret limité à un sandbox s'applique tout de suite). | |
| 59 | +- **Un outil refuse la valeur factice parce qu'elle ne commence pas par `hf_`** : une option `--placeholder` permettrait de choisir son format (par ex. `--placeholder 'hf_{rand}'`). Elle n'apparaît pas dans la page de doc détaillée : vérifie-la avec `sbx secret set-custom --help`. | |
| 60 | +- **Erreur HTTP 403 « no matching allow rule »** : le domaine est bloqué par le pare-feu du sandbox. Sur l'hôte : | |
| 61 | + | |
| 62 | + ```bash | |
| 63 | + sbx policy allow network huggingface.co,cdn-lfs.huggingface.co | |
| 64 | + ``` | |
| 65 | + | |
| 66 | +## À ne pas faire | |
| 67 | + | |
| 68 | +Ne mets pas le token dans la section `env:` de `sbxenv.yaml` ou `sbxenv.dev.yaml` : ces fichiers sont faits pour être commités, et le token se retrouverait dans le dépôt. | |
| 69 | + | |
| 70 | +## Références | |
| 71 | + | |
| 72 | +- [Manage credentials – Docker Sandboxes](https://docs.docker.com/ai/sandboxes/configuration/credentials/) | |
| 73 | +- [`sbx secret set-custom`](https://docs.docker.com/reference/cli/sbx/secret/set-custom/) | |
| 74 | +- [`sbx secret`](https://docs.docker.com/reference/cli/sbx/secret/) | |
| new file mode 100644 | |||
| @@ -0,0 +1,74 @@ | |||
| 1 | +# Injecter un token Hugging Face dans un sandbox sbx | ||
| 2 | + | ||
| 3 | +Ce guide explique comment rendre un token Hugging Face utilisable dans un sandbox Docker (`sbx`) sans que le vrai token soit stocké dans le conteneur. | ||
| 4 | + | ||
| 5 | +> **Statut :** `sbx secret set-custom` est **expérimental**. Son comportement, ses options et le format de la valeur factice peuvent changer. | ||
| 6 | + | ||
| 7 | +## Principe | ||
| 8 | + | ||
| 9 | +Hugging Face ne fait pas partie des services intégrés à `sbx` (comme `github`). Il faut donc passer par un **secret personnalisé** : | ||
| 10 | + | ||
| 11 | +- Dans le sandbox, la variable `HF_TOKEN` contient une **valeur factice** (`sbx-cs-…`). | ||
| 12 | +- Le vrai token reste sur la machine hôte, dans le trousseau du système d'exploitation. | ||
| 13 | +- Quand un processus du sandbox envoie une requête HTTP vers un des domaines déclarés, le proxy remplace la valeur factice par le vrai token, **uniquement dans les en-têtes HTTP**, pas dans le corps de la requête. | ||
| 14 | + | ||
| 15 | +Pour Hugging Face, cela suffit : le token est envoyé dans l'en-tête `Authorization: Bearer …`. | ||
| 16 | + | ||
| 17 | +## Configurer le secret | ||
| 18 | + | ||
| 19 | +Toutes les commandes `sbx` se lancent **sur la machine hôte**, pas dans le sandbox. | ||
| 20 | + | ||
| 21 | +Le secret est **global** : sans `--sandbox`, il s'applique à tous les sandboxes, donc aussi bien au sandbox de dev (`houla-houla`, `sbxenv.dev.yaml`) qu'au sandbox Olala (`houla-olala`, `sbxenv.yaml`). Il n'y a qu'une commande à lancer, une seule fois. | ||
| 22 | + | ||
| 23 | +```bash | ||
| 24 | +printf '%s' "hf_xxx" | sbx secret set-custom \ | ||
| 25 | + --host huggingface.co \ | ||
| 26 | + --host '*.huggingface.co' \ | ||
| 27 | + --env HF_TOKEN \ | ||
| 28 | + --value-stdin | ||
| 29 | +``` | ||
| 30 | + | ||
| 31 | +| Option | Rôle | | ||
| 32 | +| --- | --- | | ||
| 33 | +| `--host` | Domaine auquel le token est envoyé. Répétable. `*` correspond à un seul niveau de sous-domaine (ex. `router.huggingface.co`, `cdn-lfs.huggingface.co`), `**` à plusieurs. | | ||
| 34 | +| `--env` | Nom de la variable d'environnement définie dans le sandbox. | | ||
| 35 | +| `--value-stdin` | Lit le token sur l'entrée standard (méthode recommandée). Sans cette option, la commande demande la valeur. | | ||
| 36 | +| `--sandbox <nom>` | Limite le secret à un seul sandbox. Non utilisé ici : on veut le secret partout. | | ||
| 37 | +| `--ref` | Lit le token au moment où il en a besoin dans une source externe, par ex. `op://Vault/Item/credential` (1Password). | | ||
| 38 | +| `--command` | Lit le token au moment où il en a besoin en lançant une commande sur l'hôte qui l'affiche. | | ||
| 39 | +| `--refresh` | Durée de mise en cache d'un token lu par `--ref` ou `--command` (`<durée>` ou `on-demand`). | | ||
| 40 | + | ||
| 41 | +> **Attention :** évite `--value hf_xxx` : le token se retrouverait dans l'historique du shell et dans la liste des processus. | ||
| 42 | + | ||
| 43 | +Pour vérifier que le secret est enregistré : | ||
| 44 | + | ||
| 45 | +```bash | ||
| 46 | +sbx secret ls --host huggingface.co | ||
| 47 | +``` | ||
| 48 | + | ||
| 49 | +## Vérifier dans le sandbox | ||
| 50 | + | ||
| 51 | +```bash | ||
| 52 | +echo "$HF_TOKEN" # affiche la valeur factice, pas le vrai token | ||
| 53 | +hf auth whoami # doit afficher ton compte Hugging Face | ||
| 54 | +``` | ||
| 55 | + | ||
| 56 | +## Problèmes possibles | ||
| 57 | + | ||
| 58 | +- **`HF_TOKEN` est vide dans un sandbox déjà créé** : pour les secrets intégrés (comme `github`), un secret global ne s'applique qu'aux sandboxes créés ensuite. C'est sans doute pareil ici (pas vérifié) : recrée le sandbox, par ex. `sbx env rm sbxenv.dev.yaml` puis `sbx env create sbxenv.dev.yaml`. Pour un seul sandbox existant, tu peux aussi relancer la commande avec `--sandbox <nom>` (pour `github`, un secret limité à un sandbox s'applique tout de suite). | ||
| 59 | +- **Un outil refuse la valeur factice parce qu'elle ne commence pas par `hf_`** : une option `--placeholder` permettrait de choisir son format (par ex. `--placeholder 'hf_{rand}'`). Elle n'apparaît pas dans la page de doc détaillée : vérifie-la avec `sbx secret set-custom --help`. | ||
| 60 | +- **Erreur HTTP 403 « no matching allow rule »** : le domaine est bloqué par le pare-feu du sandbox. Sur l'hôte : | ||
| 61 | + | ||
| 62 | + ```bash | ||
| 63 | + sbx policy allow network huggingface.co,cdn-lfs.huggingface.co | ||
| 64 | + ``` | ||
| 65 | + | ||
| 66 | +## À ne pas faire | ||
| 67 | + | ||
| 68 | +Ne mets pas le token dans la section `env:` de `sbxenv.yaml` ou `sbxenv.dev.yaml` : ces fichiers sont faits pour être commités, et le token se retrouverait dans le dépôt. | ||
| 69 | + | ||
| 70 | +## Références | ||
| 71 | + | ||
| 72 | +- [Manage credentials – Docker Sandboxes](https://docs.docker.com/ai/sandboxes/configuration/credentials/) | ||
| 73 | +- [`sbx secret set-custom`](https://docs.docker.com/reference/cli/sbx/secret/set-custom/) | ||
| 74 | +- [`sbx secret`](https://docs.docker.com/reference/cli/sbx/secret/) | ||
added
docs/olala-sbx.en.md +123 -0 | new file mode 100644 | ||
| @@ -0,0 +1,123 @@ | ||
| 1 | +# Serve Olala-v1.0-mini-7A1B in an sbx sandbox | |
| 2 | + | |
| 3 | +This guide creates a sandbox that serves an Olala GGUF model with `llama-server` (OpenAI-compatible API), reachable from the host. `llama-server` comes from OVHai's llama.cpp fork (CPU only), shipped by the [`kits/llama-cpp-olala`](../kits/llama-cpp-olala/) kit. | |
| 4 | + | |
| 5 | +> Upstream llama.cpp, Ollama and LM Studio do not support the `olala` architecture yet: the [OVHai-LLM/llama.cpp](https://github.com/OVHai-LLM/llama.cpp) fork is required (see the [GGUF model card](https://huggingface.co/OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF)). | |
| 6 | + | |
| 7 | +## Create the sandbox | |
| 8 | + | |
| 9 | +On the host, at the project root: | |
| 10 | + | |
| 11 | +```bash | |
| 12 | +sbx env create sbxenv.yaml | |
| 13 | +``` | |
| 14 | + | |
| 15 | +> The development sandbox (Claude Code + Go/Python/mini-me toolchains, `houla-houla`) is described separately in `sbxenv.dev.yaml`: `sbx env run sbxenv.dev.yaml`. | |
| 16 | + | |
| 17 | +The `postCreate` hook runs [`scripts/olala.sh up`](../scripts/olala.sh) in the sandbox: | |
| 18 | + | |
| 19 | +1. uses the kit's `llama-server` (without the kit, it builds the fork: a few minutes); | |
| 20 | +2. downloads `Olala-v1.0-mini-7A1B-q8_0-safe2.gguf` (6.9 GB) into `./models`, in the repository; | |
| 21 | +3. starts `llama-server` on `0.0.0.0:8080`, published on host port 8080. | |
| 22 | + | |
| 23 | +Each step is skipped when already done (`llama-server` found, model present, server running). Models stay in `./models` (ignored by git) when the sandbox is removed: a new sandbox does not download them again. | |
| 24 | + | |
| 25 | +> The repository folder is on kDrive, so the models are synced too. Exclude `models/` from kDrive sync if you do not want that. | |
| 26 | + | |
| 27 | +Options (pass them again to every `sbx env` command of this sandbox): | |
| 28 | + | |
| 29 | +| Argument | Default | Purpose | | |
| 30 | +| --- | --- | --- | | |
| 31 | +| `model_repo` | `OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF` | Hugging Face repository | | |
| 32 | +| `model_file` | `Olala-v1.0-mini-7A1B-q8_0-safe2.gguf` | GGUF file | | |
| 33 | +| `models_dir` | `./models` | Models folder (relative to the repository root, or absolute) | | |
| 34 | +| `host_port` | `8080` | Host port | | |
| 35 | +| `threads` | `16` | CPU threads (keep 2 CPUs free: using all of them is ~30% slower) | | |
| 36 | +| `ctx` | `32768` | Context size (shared by the 4 slots) | | |
| 37 | + | |
| 38 | +```bash | |
| 39 | +sbx env create sbxenv.yaml --env-arg model_file=Olala-v1.0-mini-7A1B-q5_k_m.gguf --env-arg host_port=9090 | |
| 40 | +``` | |
| 41 | + | |
| 42 | +## In an existing sandbox | |
| 43 | + | |
| 44 | +Inside the sandbox (or through `sbx env exec sbxenv.yaml -- bash -lc '$SBX_WS/scripts/olala.sh <command>'`): | |
| 45 | + | |
| 46 | +```bash | |
| 47 | +scripts/olala.sh up # build + download + serve | |
| 48 | +scripts/olala.sh status # state and /health | |
| 49 | +scripts/olala.sh restart # after a model change | |
| 50 | +scripts/olala.sh stop | |
| 51 | +scripts/olala.sh logs | |
| 52 | +``` | |
| 53 | + | |
| 54 | +Switch model: | |
| 55 | + | |
| 56 | +```bash | |
| 57 | +OLALA_MODEL_FILE=Olala-v1.0-mini-7A1B-q6_k.gguf scripts/olala.sh download | |
| 58 | +scripts/olala.sh stop | |
| 59 | +OLALA_MODEL_FILE=Olala-v1.0-mini-7A1B-q6_k.gguf scripts/olala.sh serve | |
| 60 | +``` | |
| 61 | + | |
| 62 | +The same settings exist as environment variables: `OLALA_MODELS_DIR`, `OLALA_THREADS`, `OLALA_CTX`, `OLALA_PORT`, `OLALA_LOAD_MODE`, `OLALA_EXTRA_ARGS`… (listed at the top of [`scripts/olala.sh`](../scripts/olala.sh)). | |
| 63 | + | |
| 64 | +> **Model loading:** the script runs `llama-server --load-mode none` (the model is read into RAM, ~7 GB). With `mmap` (llama.cpp's default) the file stays on the repository mount and generation drops to ~2 tokens/s. | |
| 65 | + | |
| 66 | +A local sandbox stops as soon as no session keeps it running, and `llama-server` with it. The `postCreate` hook therefore switches the sandbox to detached mode (`sbx run -d --name houla-olala`, permanent for this sandbox): it keeps running until `sbx stop`. After `sbx stop` or a reboot, start it again, then the server: | |
| 67 | + | |
| 68 | +```bash | |
| 69 | +sbx run -d --name houla-olala | |
| 70 | +sbx env exec --no-input sbxenv.yaml -- bash -lc "$PWD/scripts/olala.sh serve" | |
| 71 | +``` | |
| 72 | + | |
| 73 | +For a sandbox created without `sbxenv.yaml` (e.g. `houla-houla`, the dev sandbox of `sbxenv.dev.yaml`), publish the port from the host: | |
| 74 | + | |
| 75 | +```bash | |
| 76 | +sbx ports houla-houla --publish 8080:8080 | |
| 77 | +``` | |
| 78 | + | |
| 79 | +## Test with curl | |
| 80 | + | |
| 81 | +From the host (or the sandbox): | |
| 82 | + | |
| 83 | +```bash | |
| 84 | +# Health | |
| 85 | +curl -s http://localhost:8080/health | |
| 86 | + | |
| 87 | +# Served models (name = file name without .gguf) | |
| 88 | +curl -s http://localhost:8080/v1/models | |
| 89 | + | |
| 90 | +# Chat | |
| 91 | +curl -s http://localhost:8080/v1/chat/completions \ | |
| 92 | + -H 'Content-Type: application/json' \ | |
| 93 | + -d '{ | |
| 94 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | |
| 95 | + "messages": [{"role": "user", "content": "Explain how a heat pump works in two sentences."}], | |
| 96 | + "temperature": 0.8, "top_p": 0.9, "top_k": 40, | |
| 97 | + "reasoning_effort": "low" | |
| 98 | + }' | jq -r '.choices[0].message.content' | |
| 99 | + | |
| 100 | +# Streaming, no reasoning | |
| 101 | +curl -sN http://localhost:8080/v1/chat/completions \ | |
| 102 | + -H 'Content-Type: application/json' \ | |
| 103 | + -d '{ | |
| 104 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | |
| 105 | + "stream": true, | |
| 106 | + "reasoning_effort": "disabled", | |
| 107 | + "messages": [{"role": "user", "content": "Say hello"}] | |
| 108 | + }' | |
| 109 | +``` | |
| 110 | + | |
| 111 | +`reasoning_effort` is `disabled`, `low`, `medium` (default) or `high`. The reasoning comes back in `choices[0].message.reasoning_content`. | |
| 112 | + | |
| 113 | +Measured in an aarch64 sandbox (18 CPUs, 31 GB) with q8_0-safe2, the kit's `llama-server`, 16 threads: 35 to 44 tokens/s generation. | |
| 114 | + | |
| 115 | +## The `llama-cpp-olala` kit | |
| 116 | + | |
| 117 | +An sbx mixin (kits v3) that builds the fork once, when the kit is built, and puts `llama-server` and `llama-cli` in `/opt/llama-cpp-olala` (links in `/usr/local/bin`). Portable build (`GGML_NATIVE=OFF`, every CPU variant loaded at runtime), arm64 (tested) and amd64 (not tested yet). The `ref` argument pins the fork commit. | |
| 118 | + | |
| 119 | +To use it in another sandbox (syntax not yet tried with `sbx`): | |
| 120 | + | |
| 121 | +```bash | |
| 122 | +sbx run claude --kit ./kits/llama-cpp-olala | |
| 123 | +``` | |
| new file mode 100644 | |||
| @@ -0,0 +1,123 @@ | |||
| 1 | +# Serve Olala-v1.0-mini-7A1B in an sbx sandbox | ||
| 2 | + | ||
| 3 | +This guide creates a sandbox that serves an Olala GGUF model with `llama-server` (OpenAI-compatible API), reachable from the host. `llama-server` comes from OVHai's llama.cpp fork (CPU only), shipped by the [`kits/llama-cpp-olala`](../kits/llama-cpp-olala/) kit. | ||
| 4 | + | ||
| 5 | +> Upstream llama.cpp, Ollama and LM Studio do not support the `olala` architecture yet: the [OVHai-LLM/llama.cpp](https://github.com/OVHai-LLM/llama.cpp) fork is required (see the [GGUF model card](https://huggingface.co/OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF)). | ||
| 6 | + | ||
| 7 | +## Create the sandbox | ||
| 8 | + | ||
| 9 | +On the host, at the project root: | ||
| 10 | + | ||
| 11 | +```bash | ||
| 12 | +sbx env create sbxenv.yaml | ||
| 13 | +``` | ||
| 14 | + | ||
| 15 | +> The development sandbox (Claude Code + Go/Python/mini-me toolchains, `houla-houla`) is described separately in `sbxenv.dev.yaml`: `sbx env run sbxenv.dev.yaml`. | ||
| 16 | + | ||
| 17 | +The `postCreate` hook runs [`scripts/olala.sh up`](../scripts/olala.sh) in the sandbox: | ||
| 18 | + | ||
| 19 | +1. uses the kit's `llama-server` (without the kit, it builds the fork: a few minutes); | ||
| 20 | +2. downloads `Olala-v1.0-mini-7A1B-q8_0-safe2.gguf` (6.9 GB) into `./models`, in the repository; | ||
| 21 | +3. starts `llama-server` on `0.0.0.0:8080`, published on host port 8080. | ||
| 22 | + | ||
| 23 | +Each step is skipped when already done (`llama-server` found, model present, server running). Models stay in `./models` (ignored by git) when the sandbox is removed: a new sandbox does not download them again. | ||
| 24 | + | ||
| 25 | +> The repository folder is on kDrive, so the models are synced too. Exclude `models/` from kDrive sync if you do not want that. | ||
| 26 | + | ||
| 27 | +Options (pass them again to every `sbx env` command of this sandbox): | ||
| 28 | + | ||
| 29 | +| Argument | Default | Purpose | | ||
| 30 | +| --- | --- | --- | | ||
| 31 | +| `model_repo` | `OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF` | Hugging Face repository | | ||
| 32 | +| `model_file` | `Olala-v1.0-mini-7A1B-q8_0-safe2.gguf` | GGUF file | | ||
| 33 | +| `models_dir` | `./models` | Models folder (relative to the repository root, or absolute) | | ||
| 34 | +| `host_port` | `8080` | Host port | | ||
| 35 | +| `threads` | `16` | CPU threads (keep 2 CPUs free: using all of them is ~30% slower) | | ||
| 36 | +| `ctx` | `32768` | Context size (shared by the 4 slots) | | ||
| 37 | + | ||
| 38 | +```bash | ||
| 39 | +sbx env create sbxenv.yaml --env-arg model_file=Olala-v1.0-mini-7A1B-q5_k_m.gguf --env-arg host_port=9090 | ||
| 40 | +``` | ||
| 41 | + | ||
| 42 | +## In an existing sandbox | ||
| 43 | + | ||
| 44 | +Inside the sandbox (or through `sbx env exec sbxenv.yaml -- bash -lc '$SBX_WS/scripts/olala.sh <command>'`): | ||
| 45 | + | ||
| 46 | +```bash | ||
| 47 | +scripts/olala.sh up # build + download + serve | ||
| 48 | +scripts/olala.sh status # state and /health | ||
| 49 | +scripts/olala.sh restart # after a model change | ||
| 50 | +scripts/olala.sh stop | ||
| 51 | +scripts/olala.sh logs | ||
| 52 | +``` | ||
| 53 | + | ||
| 54 | +Switch model: | ||
| 55 | + | ||
| 56 | +```bash | ||
| 57 | +OLALA_MODEL_FILE=Olala-v1.0-mini-7A1B-q6_k.gguf scripts/olala.sh download | ||
| 58 | +scripts/olala.sh stop | ||
| 59 | +OLALA_MODEL_FILE=Olala-v1.0-mini-7A1B-q6_k.gguf scripts/olala.sh serve | ||
| 60 | +``` | ||
| 61 | + | ||
| 62 | +The same settings exist as environment variables: `OLALA_MODELS_DIR`, `OLALA_THREADS`, `OLALA_CTX`, `OLALA_PORT`, `OLALA_LOAD_MODE`, `OLALA_EXTRA_ARGS`… (listed at the top of [`scripts/olala.sh`](../scripts/olala.sh)). | ||
| 63 | + | ||
| 64 | +> **Model loading:** the script runs `llama-server --load-mode none` (the model is read into RAM, ~7 GB). With `mmap` (llama.cpp's default) the file stays on the repository mount and generation drops to ~2 tokens/s. | ||
| 65 | + | ||
| 66 | +A local sandbox stops as soon as no session keeps it running, and `llama-server` with it. The `postCreate` hook therefore switches the sandbox to detached mode (`sbx run -d --name houla-olala`, permanent for this sandbox): it keeps running until `sbx stop`. After `sbx stop` or a reboot, start it again, then the server: | ||
| 67 | + | ||
| 68 | +```bash | ||
| 69 | +sbx run -d --name houla-olala | ||
| 70 | +sbx env exec --no-input sbxenv.yaml -- bash -lc "$PWD/scripts/olala.sh serve" | ||
| 71 | +``` | ||
| 72 | + | ||
| 73 | +For a sandbox created without `sbxenv.yaml` (e.g. `houla-houla`, the dev sandbox of `sbxenv.dev.yaml`), publish the port from the host: | ||
| 74 | + | ||
| 75 | +```bash | ||
| 76 | +sbx ports houla-houla --publish 8080:8080 | ||
| 77 | +``` | ||
| 78 | + | ||
| 79 | +## Test with curl | ||
| 80 | + | ||
| 81 | +From the host (or the sandbox): | ||
| 82 | + | ||
| 83 | +```bash | ||
| 84 | +# Health | ||
| 85 | +curl -s http://localhost:8080/health | ||
| 86 | + | ||
| 87 | +# Served models (name = file name without .gguf) | ||
| 88 | +curl -s http://localhost:8080/v1/models | ||
| 89 | + | ||
| 90 | +# Chat | ||
| 91 | +curl -s http://localhost:8080/v1/chat/completions \ | ||
| 92 | + -H 'Content-Type: application/json' \ | ||
| 93 | + -d '{ | ||
| 94 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | ||
| 95 | + "messages": [{"role": "user", "content": "Explain how a heat pump works in two sentences."}], | ||
| 96 | + "temperature": 0.8, "top_p": 0.9, "top_k": 40, | ||
| 97 | + "reasoning_effort": "low" | ||
| 98 | + }' | jq -r '.choices[0].message.content' | ||
| 99 | + | ||
| 100 | +# Streaming, no reasoning | ||
| 101 | +curl -sN http://localhost:8080/v1/chat/completions \ | ||
| 102 | + -H 'Content-Type: application/json' \ | ||
| 103 | + -d '{ | ||
| 104 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | ||
| 105 | + "stream": true, | ||
| 106 | + "reasoning_effort": "disabled", | ||
| 107 | + "messages": [{"role": "user", "content": "Say hello"}] | ||
| 108 | + }' | ||
| 109 | +``` | ||
| 110 | + | ||
| 111 | +`reasoning_effort` is `disabled`, `low`, `medium` (default) or `high`. The reasoning comes back in `choices[0].message.reasoning_content`. | ||
| 112 | + | ||
| 113 | +Measured in an aarch64 sandbox (18 CPUs, 31 GB) with q8_0-safe2, the kit's `llama-server`, 16 threads: 35 to 44 tokens/s generation. | ||
| 114 | + | ||
| 115 | +## The `llama-cpp-olala` kit | ||
| 116 | + | ||
| 117 | +An sbx mixin (kits v3) that builds the fork once, when the kit is built, and puts `llama-server` and `llama-cli` in `/opt/llama-cpp-olala` (links in `/usr/local/bin`). Portable build (`GGML_NATIVE=OFF`, every CPU variant loaded at runtime), arm64 (tested) and amd64 (not tested yet). The `ref` argument pins the fork commit. | ||
| 118 | + | ||
| 119 | +To use it in another sandbox (syntax not yet tried with `sbx`): | ||
| 120 | + | ||
| 121 | +```bash | ||
| 122 | +sbx run claude --kit ./kits/llama-cpp-olala | ||
| 123 | +``` | ||
added
docs/olala-sbx.fr.md +123 -0 | new file mode 100644 | ||
| @@ -0,0 +1,123 @@ | ||
| 1 | +# Servir Olala-v1.0-mini-7A1B dans un sandbox sbx | |
| 2 | + | |
| 3 | +Ce guide crée un sandbox qui sert un modèle GGUF Olala avec `llama-server` (API compatible OpenAI), accessible depuis la machine hôte. `llama-server` vient du fork llama.cpp d'OVHai (CPU uniquement), fourni par le kit [`kits/llama-cpp-olala`](../kits/llama-cpp-olala/). | |
| 4 | + | |
| 5 | +> Upstream llama.cpp, Ollama et LM Studio ne connaissent pas encore l'architecture `olala` : il faut le fork [OVHai-LLM/llama.cpp](https://github.com/OVHai-LLM/llama.cpp) (voir la [fiche du modèle GGUF](https://huggingface.co/OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF)). | |
| 6 | + | |
| 7 | +## Créer le sandbox | |
| 8 | + | |
| 9 | +Sur la machine hôte, à la racine du projet : | |
| 10 | + | |
| 11 | +```bash | |
| 12 | +sbx env create sbxenv.yaml | |
| 13 | +``` | |
| 14 | + | |
| 15 | +> Le sandbox de développement (Claude Code + toolchains Go/Python/mini-me, `houla-houla`) est décrit à part dans `sbxenv.dev.yaml` : `sbx env run sbxenv.dev.yaml`. | |
| 16 | + | |
| 17 | +Le hook `postCreate` lance [`scripts/olala.sh up`](../scripts/olala.sh) dans le sandbox : | |
| 18 | + | |
| 19 | +1. utilise le `llama-server` du kit (sans le kit, il compile le fork : quelques minutes) ; | |
| 20 | +2. télécharge `Olala-v1.0-mini-7A1B-q8_0-safe2.gguf` (6,9 Go) dans `./models`, dans le dépôt ; | |
| 21 | +3. démarre `llama-server` sur `0.0.0.0:8080`, publié sur le port 8080 de l'hôte. | |
| 22 | + | |
| 23 | +Chaque étape est sautée si elle est déjà faite (`llama-server` trouvé, modèle présent, serveur lancé). Les modèles restent dans `./models` (ignoré par git) quand le sandbox est supprimé : un nouveau sandbox ne les retélécharge pas. | |
| 24 | + | |
| 25 | +> Le dossier du dépôt est sur kDrive : les modèles y sont donc aussi synchronisés. Exclus `models/` de la synchronisation kDrive si tu ne le veux pas. | |
| 26 | + | |
| 27 | +Options (à repasser à chaque commande `sbx env` de ce sandbox) : | |
| 28 | + | |
| 29 | +| Argument | Défaut | Rôle | | |
| 30 | +| --- | --- | --- | | |
| 31 | +| `model_repo` | `OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF` | Dépôt Hugging Face | | |
| 32 | +| `model_file` | `Olala-v1.0-mini-7A1B-q8_0-safe2.gguf` | Fichier GGUF | | |
| 33 | +| `models_dir` | `./models` | Dossier des modèles (relatif à la racine du dépôt, ou absolu) | | |
| 34 | +| `host_port` | `8080` | Port de l'hôte | | |
| 35 | +| `threads` | `16` | Threads CPU (garde 2 CPU libres : tous les utiliser est ~30 % plus lent) | | |
| 36 | +| `ctx` | `32768` | Taille du contexte (partagée par les 4 slots) | | |
| 37 | + | |
| 38 | +```bash | |
| 39 | +sbx env create sbxenv.yaml --env-arg model_file=Olala-v1.0-mini-7A1B-q5_k_m.gguf --env-arg host_port=9090 | |
| 40 | +``` | |
| 41 | + | |
| 42 | +## Dans un sandbox existant | |
| 43 | + | |
| 44 | +Dans le sandbox (ou via `sbx env exec sbxenv.yaml -- bash -lc '$SBX_WS/scripts/olala.sh <commande>'`) : | |
| 45 | + | |
| 46 | +```bash | |
| 47 | +scripts/olala.sh up # compile + télécharge + sert | |
| 48 | +scripts/olala.sh status # état et /health | |
| 49 | +scripts/olala.sh restart # après un changement de modèle | |
| 50 | +scripts/olala.sh stop | |
| 51 | +scripts/olala.sh logs | |
| 52 | +``` | |
| 53 | + | |
| 54 | +Changer de modèle : | |
| 55 | + | |
| 56 | +```bash | |
| 57 | +OLALA_MODEL_FILE=Olala-v1.0-mini-7A1B-q6_k.gguf scripts/olala.sh download | |
| 58 | +scripts/olala.sh stop | |
| 59 | +OLALA_MODEL_FILE=Olala-v1.0-mini-7A1B-q6_k.gguf scripts/olala.sh serve | |
| 60 | +``` | |
| 61 | + | |
| 62 | +Les mêmes réglages existent en variables d'environnement : `OLALA_MODELS_DIR`, `OLALA_THREADS`, `OLALA_CTX`, `OLALA_PORT`, `OLALA_LOAD_MODE`, `OLALA_EXTRA_ARGS`… (liste en tête de [`scripts/olala.sh`](../scripts/olala.sh)). | |
| 63 | + | |
| 64 | +> **Chargement du modèle :** le script lance `llama-server --load-mode none` (le modèle est lu en RAM, ~7 Go). En `mmap` (défaut de llama.cpp), le fichier reste sur le montage du dépôt et la génération tombe à ~2 tokens/s. | |
| 65 | + | |
| 66 | +Un sandbox local s'arrête dès qu'aucune session ne le garde en vie, et `llama-server` avec lui. Le hook `postCreate` passe donc le sandbox en mode détaché (`sbx run -d --name houla-olala`, permanent pour ce sandbox) : il tourne jusqu'à `sbx stop`. Après un `sbx stop` ou un redémarrage de la machine, relance-le puis relance le serveur : | |
| 67 | + | |
| 68 | +```bash | |
| 69 | +sbx run -d --name houla-olala | |
| 70 | +sbx env exec --no-input sbxenv.yaml -- bash -lc "$PWD/scripts/olala.sh serve" | |
| 71 | +``` | |
| 72 | + | |
| 73 | +Pour un sandbox créé sans `sbxenv.yaml` (par ex. `houla-houla`, le sandbox de dev de `sbxenv.dev.yaml`), publie le port depuis l'hôte : | |
| 74 | + | |
| 75 | +```bash | |
| 76 | +sbx ports houla-houla --publish 8080:8080 | |
| 77 | +``` | |
| 78 | + | |
| 79 | +## Tester avec curl | |
| 80 | + | |
| 81 | +Depuis l'hôte (ou le sandbox) : | |
| 82 | + | |
| 83 | +```bash | |
| 84 | +# Santé | |
| 85 | +curl -s http://localhost:8080/health | |
| 86 | + | |
| 87 | +# Modèles servis (le nom = fichier sans .gguf) | |
| 88 | +curl -s http://localhost:8080/v1/models | |
| 89 | + | |
| 90 | +# Chat | |
| 91 | +curl -s http://localhost:8080/v1/chat/completions \ | |
| 92 | + -H 'Content-Type: application/json' \ | |
| 93 | + -d '{ | |
| 94 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | |
| 95 | + "messages": [{"role": "user", "content": "Explique en deux phrases ce qu est une pompe à chaleur."}], | |
| 96 | + "temperature": 0.8, "top_p": 0.9, "top_k": 40, | |
| 97 | + "reasoning_effort": "low" | |
| 98 | + }' | jq -r '.choices[0].message.content' | |
| 99 | + | |
| 100 | +# Streaming, sans raisonnement | |
| 101 | +curl -sN http://localhost:8080/v1/chat/completions \ | |
| 102 | + -H 'Content-Type: application/json' \ | |
| 103 | + -d '{ | |
| 104 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | |
| 105 | + "stream": true, | |
| 106 | + "reasoning_effort": "disabled", | |
| 107 | + "messages": [{"role": "user", "content": "Dis bonjour"}] | |
| 108 | + }' | |
| 109 | +``` | |
| 110 | + | |
| 111 | +`reasoning_effort` vaut `disabled`, `low`, `medium` (défaut) ou `high`. Le raisonnement est renvoyé dans `choices[0].message.reasoning_content`. | |
| 112 | + | |
| 113 | +Mesuré dans un sandbox aarch64 (18 CPU, 31 Go) avec q8_0-safe2, `llama-server` du kit, 16 threads : 35 à 44 tokens/s en génération. | |
| 114 | + | |
| 115 | +## Le kit `llama-cpp-olala` | |
| 116 | + | |
| 117 | +Mixin sbx (kits v3) qui compile le fork une fois, au build du kit, et pose `llama-server` et `llama-cli` dans `/opt/llama-cpp-olala` (liens dans `/usr/local/bin`). Build portable (`GGML_NATIVE=OFF`, toutes les variantes CPU chargées à l'exécution), arm64 (testé) et amd64 (pas encore testé). L'argument `ref` fixe le commit du fork. | |
| 118 | + | |
| 119 | +Pour l'utiliser dans un autre sandbox (syntaxe pas encore testée avec `sbx`) : | |
| 120 | + | |
| 121 | +```bash | |
| 122 | +sbx run claude --kit ./kits/llama-cpp-olala | |
| 123 | +``` | |
| new file mode 100644 | |||
| @@ -0,0 +1,123 @@ | |||
| 1 | +# Servir Olala-v1.0-mini-7A1B dans un sandbox sbx | ||
| 2 | + | ||
| 3 | +Ce guide crée un sandbox qui sert un modèle GGUF Olala avec `llama-server` (API compatible OpenAI), accessible depuis la machine hôte. `llama-server` vient du fork llama.cpp d'OVHai (CPU uniquement), fourni par le kit [`kits/llama-cpp-olala`](../kits/llama-cpp-olala/). | ||
| 4 | + | ||
| 5 | +> Upstream llama.cpp, Ollama et LM Studio ne connaissent pas encore l'architecture `olala` : il faut le fork [OVHai-LLM/llama.cpp](https://github.com/OVHai-LLM/llama.cpp) (voir la [fiche du modèle GGUF](https://huggingface.co/OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF)). | ||
| 6 | + | ||
| 7 | +## Créer le sandbox | ||
| 8 | + | ||
| 9 | +Sur la machine hôte, à la racine du projet : | ||
| 10 | + | ||
| 11 | +```bash | ||
| 12 | +sbx env create sbxenv.yaml | ||
| 13 | +``` | ||
| 14 | + | ||
| 15 | +> Le sandbox de développement (Claude Code + toolchains Go/Python/mini-me, `houla-houla`) est décrit à part dans `sbxenv.dev.yaml` : `sbx env run sbxenv.dev.yaml`. | ||
| 16 | + | ||
| 17 | +Le hook `postCreate` lance [`scripts/olala.sh up`](../scripts/olala.sh) dans le sandbox : | ||
| 18 | + | ||
| 19 | +1. utilise le `llama-server` du kit (sans le kit, il compile le fork : quelques minutes) ; | ||
| 20 | +2. télécharge `Olala-v1.0-mini-7A1B-q8_0-safe2.gguf` (6,9 Go) dans `./models`, dans le dépôt ; | ||
| 21 | +3. démarre `llama-server` sur `0.0.0.0:8080`, publié sur le port 8080 de l'hôte. | ||
| 22 | + | ||
| 23 | +Chaque étape est sautée si elle est déjà faite (`llama-server` trouvé, modèle présent, serveur lancé). Les modèles restent dans `./models` (ignoré par git) quand le sandbox est supprimé : un nouveau sandbox ne les retélécharge pas. | ||
| 24 | + | ||
| 25 | +> Le dossier du dépôt est sur kDrive : les modèles y sont donc aussi synchronisés. Exclus `models/` de la synchronisation kDrive si tu ne le veux pas. | ||
| 26 | + | ||
| 27 | +Options (à repasser à chaque commande `sbx env` de ce sandbox) : | ||
| 28 | + | ||
| 29 | +| Argument | Défaut | Rôle | | ||
| 30 | +| --- | --- | --- | | ||
| 31 | +| `model_repo` | `OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF` | Dépôt Hugging Face | | ||
| 32 | +| `model_file` | `Olala-v1.0-mini-7A1B-q8_0-safe2.gguf` | Fichier GGUF | | ||
| 33 | +| `models_dir` | `./models` | Dossier des modèles (relatif à la racine du dépôt, ou absolu) | | ||
| 34 | +| `host_port` | `8080` | Port de l'hôte | | ||
| 35 | +| `threads` | `16` | Threads CPU (garde 2 CPU libres : tous les utiliser est ~30 % plus lent) | | ||
| 36 | +| `ctx` | `32768` | Taille du contexte (partagée par les 4 slots) | | ||
| 37 | + | ||
| 38 | +```bash | ||
| 39 | +sbx env create sbxenv.yaml --env-arg model_file=Olala-v1.0-mini-7A1B-q5_k_m.gguf --env-arg host_port=9090 | ||
| 40 | +``` | ||
| 41 | + | ||
| 42 | +## Dans un sandbox existant | ||
| 43 | + | ||
| 44 | +Dans le sandbox (ou via `sbx env exec sbxenv.yaml -- bash -lc '$SBX_WS/scripts/olala.sh <commande>'`) : | ||
| 45 | + | ||
| 46 | +```bash | ||
| 47 | +scripts/olala.sh up # compile + télécharge + sert | ||
| 48 | +scripts/olala.sh status # état et /health | ||
| 49 | +scripts/olala.sh restart # après un changement de modèle | ||
| 50 | +scripts/olala.sh stop | ||
| 51 | +scripts/olala.sh logs | ||
| 52 | +``` | ||
| 53 | + | ||
| 54 | +Changer de modèle : | ||
| 55 | + | ||
| 56 | +```bash | ||
| 57 | +OLALA_MODEL_FILE=Olala-v1.0-mini-7A1B-q6_k.gguf scripts/olala.sh download | ||
| 58 | +scripts/olala.sh stop | ||
| 59 | +OLALA_MODEL_FILE=Olala-v1.0-mini-7A1B-q6_k.gguf scripts/olala.sh serve | ||
| 60 | +``` | ||
| 61 | + | ||
| 62 | +Les mêmes réglages existent en variables d'environnement : `OLALA_MODELS_DIR`, `OLALA_THREADS`, `OLALA_CTX`, `OLALA_PORT`, `OLALA_LOAD_MODE`, `OLALA_EXTRA_ARGS`… (liste en tête de [`scripts/olala.sh`](../scripts/olala.sh)). | ||
| 63 | + | ||
| 64 | +> **Chargement du modèle :** le script lance `llama-server --load-mode none` (le modèle est lu en RAM, ~7 Go). En `mmap` (défaut de llama.cpp), le fichier reste sur le montage du dépôt et la génération tombe à ~2 tokens/s. | ||
| 65 | + | ||
| 66 | +Un sandbox local s'arrête dès qu'aucune session ne le garde en vie, et `llama-server` avec lui. Le hook `postCreate` passe donc le sandbox en mode détaché (`sbx run -d --name houla-olala`, permanent pour ce sandbox) : il tourne jusqu'à `sbx stop`. Après un `sbx stop` ou un redémarrage de la machine, relance-le puis relance le serveur : | ||
| 67 | + | ||
| 68 | +```bash | ||
| 69 | +sbx run -d --name houla-olala | ||
| 70 | +sbx env exec --no-input sbxenv.yaml -- bash -lc "$PWD/scripts/olala.sh serve" | ||
| 71 | +``` | ||
| 72 | + | ||
| 73 | +Pour un sandbox créé sans `sbxenv.yaml` (par ex. `houla-houla`, le sandbox de dev de `sbxenv.dev.yaml`), publie le port depuis l'hôte : | ||
| 74 | + | ||
| 75 | +```bash | ||
| 76 | +sbx ports houla-houla --publish 8080:8080 | ||
| 77 | +``` | ||
| 78 | + | ||
| 79 | +## Tester avec curl | ||
| 80 | + | ||
| 81 | +Depuis l'hôte (ou le sandbox) : | ||
| 82 | + | ||
| 83 | +```bash | ||
| 84 | +# Santé | ||
| 85 | +curl -s http://localhost:8080/health | ||
| 86 | + | ||
| 87 | +# Modèles servis (le nom = fichier sans .gguf) | ||
| 88 | +curl -s http://localhost:8080/v1/models | ||
| 89 | + | ||
| 90 | +# Chat | ||
| 91 | +curl -s http://localhost:8080/v1/chat/completions \ | ||
| 92 | + -H 'Content-Type: application/json' \ | ||
| 93 | + -d '{ | ||
| 94 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | ||
| 95 | + "messages": [{"role": "user", "content": "Explique en deux phrases ce qu est une pompe à chaleur."}], | ||
| 96 | + "temperature": 0.8, "top_p": 0.9, "top_k": 40, | ||
| 97 | + "reasoning_effort": "low" | ||
| 98 | + }' | jq -r '.choices[0].message.content' | ||
| 99 | + | ||
| 100 | +# Streaming, sans raisonnement | ||
| 101 | +curl -sN http://localhost:8080/v1/chat/completions \ | ||
| 102 | + -H 'Content-Type: application/json' \ | ||
| 103 | + -d '{ | ||
| 104 | + "model": "Olala-v1.0-mini-7A1B-q8_0-safe2", | ||
| 105 | + "stream": true, | ||
| 106 | + "reasoning_effort": "disabled", | ||
| 107 | + "messages": [{"role": "user", "content": "Dis bonjour"}] | ||
| 108 | + }' | ||
| 109 | +``` | ||
| 110 | + | ||
| 111 | +`reasoning_effort` vaut `disabled`, `low`, `medium` (défaut) ou `high`. Le raisonnement est renvoyé dans `choices[0].message.reasoning_content`. | ||
| 112 | + | ||
| 113 | +Mesuré dans un sandbox aarch64 (18 CPU, 31 Go) avec q8_0-safe2, `llama-server` du kit, 16 threads : 35 à 44 tokens/s en génération. | ||
| 114 | + | ||
| 115 | +## Le kit `llama-cpp-olala` | ||
| 116 | + | ||
| 117 | +Mixin sbx (kits v3) qui compile le fork une fois, au build du kit, et pose `llama-server` et `llama-cli` dans `/opt/llama-cpp-olala` (liens dans `/usr/local/bin`). Build portable (`GGML_NATIVE=OFF`, toutes les variantes CPU chargées à l'exécution), arm64 (testé) et amd64 (pas encore testé). L'argument `ref` fixe le commit du fork. | ||
| 118 | + | ||
| 119 | +Pour l'utiliser dans un autre sandbox (syntaxe pas encore testée avec `sbx`) : | ||
| 120 | + | ||
| 121 | +```bash | ||
| 122 | +sbx run claude --kit ./kits/llama-cpp-olala | ||
| 123 | +``` | ||
added
git.sh +136 -0 | new file mode 100755 | ||
| @@ -0,0 +1,136 @@ | ||
| 1 | +#!/bin/bash | |
| 2 | +message="" | |
| 3 | +case $1 in | |
| 4 | + | |
| 5 | + # 🎨: art | |
| 6 | + art) | |
| 7 | + message="Improve structure / format of the code" | |
| 8 | + emoji="🎨" | |
| 9 | + ;; | |
| 10 | + | |
| 11 | + # 🐛: bug | |
| 12 | + bug|fix) | |
| 13 | + message="Fix a bug" | |
| 14 | + emoji="🐛" | |
| 15 | + ;; | |
| 16 | + | |
| 17 | + # 🤓: geek | |
| 18 | + human|human-fixed) | |
| 19 | + message="Human Fixed" | |
| 20 | + emoji="🤓" | |
| 21 | + ;; | |
| 22 | + | |
| 23 | + # 🤖: robot | |
| 24 | + ai|ai-generated) | |
| 25 | + message="AI generated" | |
| 26 | + emoji="🤖" | |
| 27 | + ;; | |
| 28 | + | |
| 29 | + # ✨: sparkles | |
| 30 | + sparkles|feature) | |
| 31 | + message="Introduce new feature(s)" | |
| 32 | + emoji="✨" | |
| 33 | + ;; | |
| 34 | + | |
| 35 | + # 🧩: jigsaw | |
| 36 | + jigsaw|example|examples|demo|demos) | |
| 37 | + message="Introduce new example(s)" | |
| 38 | + emoji="🧩" | |
| 39 | + ;; | |
| 40 | + | |
| 41 | + | |
| 42 | + # 📝: memo | |
| 43 | + memo|doc|documentation) | |
| 44 | + message="Add or update documentation" | |
| 45 | + emoji="📝" | |
| 46 | + ;; | |
| 47 | + | |
| 48 | + # 🌸: cherry_blossom | |
| 49 | + gardening|garden|clean|cleaning) | |
| 50 | + message="Gardening" | |
| 51 | + emoji="🌸" | |
| 52 | + ;; | |
| 53 | + | |
| 54 | + # 🚀: rocket | |
| 55 | + rocket|deploy) | |
| 56 | + message="Deploy stuff" | |
| 57 | + emoji="🚀" | |
| 58 | + ;; | |
| 59 | + | |
| 60 | + # 🎉: tada | |
| 61 | + tada|first) | |
| 62 | + message="Begin a project" | |
| 63 | + emoji="🎉" | |
| 64 | + ;; | |
| 65 | + | |
| 66 | + # 🚧: construction | |
| 67 | + construction|wip) | |
| 68 | + message="Work in progress" | |
| 69 | + emoji="🚧" | |
| 70 | + ;; | |
| 71 | + | |
| 72 | + # 📦️: package | |
| 73 | + package|build) | |
| 74 | + message="Add or update compiled files or packages" | |
| 75 | + emoji="📦️" | |
| 76 | + ;; | |
| 77 | + | |
| 78 | + # 📦️: package | |
| 79 | + release) | |
| 80 | + message="Create a release" | |
| 81 | + emoji="📦️" | |
| 82 | + ;; | |
| 83 | + | |
| 84 | + # 👽️: alien | |
| 85 | + alien|api) | |
| 86 | + message="Update code due to external API changes" | |
| 87 | + emoji="👽️" | |
| 88 | + ;; | |
| 89 | + | |
| 90 | + # 🐳: whale | |
| 91 | + docker|container) | |
| 92 | + message="Docker" | |
| 93 | + emoji="🐳" | |
| 94 | + ;; | |
| 95 | + | |
| 96 | + # 🍊: tangerine | |
| 97 | + gitpod|gitpodify) | |
| 98 | + message="Gitpodify" | |
| 99 | + emoji="🍊" | |
| 100 | + ;; | |
| 101 | + | |
| 102 | + # 🧪: test tube | |
| 103 | + alembic|experiments|experiment|xp) | |
| 104 | + message="Perform experiments" | |
| 105 | + emoji="🧪" | |
| 106 | + ;; | |
| 107 | + | |
| 108 | + # ✅: check mark | |
| 109 | + test|tests|testing) | |
| 110 | + message="Add or update tests" | |
| 111 | + emoji="✅" | |
| 112 | + ;; | |
| 113 | + | |
| 114 | + # 💾: floppy-disk | |
| 115 | + save) | |
| 116 | + message="Saved" | |
| 117 | + emoji="💾" | |
| 118 | + ;; | |
| 119 | + | |
| 120 | + *) | |
| 121 | + message="Updated" | |
| 122 | + emoji="🛟" | |
| 123 | + ;; | |
| 124 | + | |
| 125 | +esac | |
| 126 | + | |
| 127 | +find . -name '.DS_Store' -type f -delete | |
| 128 | + | |
| 129 | +if [ -z "$2" ] | |
| 130 | +then | |
| 131 | + # empty | |
| 132 | + git add .; git commit -m "$emoji $message."; git push | |
| 133 | +else | |
| 134 | + # not empty | |
| 135 | + git add .; git commit -m "$emoji $message: $2"; git push | |
| 136 | +fi | |
| new file mode 100755 | |||
| @@ -0,0 +1,136 @@ | |||
| 1 | +#!/bin/bash | ||
| 2 | +message="" | ||
| 3 | +case $1 in | ||
| 4 | + | ||
| 5 | + # 🎨: art | ||
| 6 | + art) | ||
| 7 | + message="Improve structure / format of the code" | ||
| 8 | + emoji="🎨" | ||
| 9 | + ;; | ||
| 10 | + | ||
| 11 | + # 🐛: bug | ||
| 12 | + bug|fix) | ||
| 13 | + message="Fix a bug" | ||
| 14 | + emoji="🐛" | ||
| 15 | + ;; | ||
| 16 | + | ||
| 17 | + # 🤓: geek | ||
| 18 | + human|human-fixed) | ||
| 19 | + message="Human Fixed" | ||
| 20 | + emoji="🤓" | ||
| 21 | + ;; | ||
| 22 | + | ||
| 23 | + # 🤖: robot | ||
| 24 | + ai|ai-generated) | ||
| 25 | + message="AI generated" | ||
| 26 | + emoji="🤖" | ||
| 27 | + ;; | ||
| 28 | + | ||
| 29 | + # ✨: sparkles | ||
| 30 | + sparkles|feature) | ||
| 31 | + message="Introduce new feature(s)" | ||
| 32 | + emoji="✨" | ||
| 33 | + ;; | ||
| 34 | + | ||
| 35 | + # 🧩: jigsaw | ||
| 36 | + jigsaw|example|examples|demo|demos) | ||
| 37 | + message="Introduce new example(s)" | ||
| 38 | + emoji="🧩" | ||
| 39 | + ;; | ||
| 40 | + | ||
| 41 | + | ||
| 42 | + # 📝: memo | ||
| 43 | + memo|doc|documentation) | ||
| 44 | + message="Add or update documentation" | ||
| 45 | + emoji="📝" | ||
| 46 | + ;; | ||
| 47 | + | ||
| 48 | + # 🌸: cherry_blossom | ||
| 49 | + gardening|garden|clean|cleaning) | ||
| 50 | + message="Gardening" | ||
| 51 | + emoji="🌸" | ||
| 52 | + ;; | ||
| 53 | + | ||
| 54 | + # 🚀: rocket | ||
| 55 | + rocket|deploy) | ||
| 56 | + message="Deploy stuff" | ||
| 57 | + emoji="🚀" | ||
| 58 | + ;; | ||
| 59 | + | ||
| 60 | + # 🎉: tada | ||
| 61 | + tada|first) | ||
| 62 | + message="Begin a project" | ||
| 63 | + emoji="🎉" | ||
| 64 | + ;; | ||
| 65 | + | ||
| 66 | + # 🚧: construction | ||
| 67 | + construction|wip) | ||
| 68 | + message="Work in progress" | ||
| 69 | + emoji="🚧" | ||
| 70 | + ;; | ||
| 71 | + | ||
| 72 | + # 📦️: package | ||
| 73 | + package|build) | ||
| 74 | + message="Add or update compiled files or packages" | ||
| 75 | + emoji="📦️" | ||
| 76 | + ;; | ||
| 77 | + | ||
| 78 | + # 📦️: package | ||
| 79 | + release) | ||
| 80 | + message="Create a release" | ||
| 81 | + emoji="📦️" | ||
| 82 | + ;; | ||
| 83 | + | ||
| 84 | + # 👽️: alien | ||
| 85 | + alien|api) | ||
| 86 | + message="Update code due to external API changes" | ||
| 87 | + emoji="👽️" | ||
| 88 | + ;; | ||
| 89 | + | ||
| 90 | + # 🐳: whale | ||
| 91 | + docker|container) | ||
| 92 | + message="Docker" | ||
| 93 | + emoji="🐳" | ||
| 94 | + ;; | ||
| 95 | + | ||
| 96 | + # 🍊: tangerine | ||
| 97 | + gitpod|gitpodify) | ||
| 98 | + message="Gitpodify" | ||
| 99 | + emoji="🍊" | ||
| 100 | + ;; | ||
| 101 | + | ||
| 102 | + # 🧪: test tube | ||
| 103 | + alembic|experiments|experiment|xp) | ||
| 104 | + message="Perform experiments" | ||
| 105 | + emoji="🧪" | ||
| 106 | + ;; | ||
| 107 | + | ||
| 108 | + # ✅: check mark | ||
| 109 | + test|tests|testing) | ||
| 110 | + message="Add or update tests" | ||
| 111 | + emoji="✅" | ||
| 112 | + ;; | ||
| 113 | + | ||
| 114 | + # 💾: floppy-disk | ||
| 115 | + save) | ||
| 116 | + message="Saved" | ||
| 117 | + emoji="💾" | ||
| 118 | + ;; | ||
| 119 | + | ||
| 120 | + *) | ||
| 121 | + message="Updated" | ||
| 122 | + emoji="🛟" | ||
| 123 | + ;; | ||
| 124 | + | ||
| 125 | +esac | ||
| 126 | + | ||
| 127 | +find . -name '.DS_Store' -type f -delete | ||
| 128 | + | ||
| 129 | +if [ -z "$2" ] | ||
| 130 | +then | ||
| 131 | + # empty | ||
| 132 | + git add .; git commit -m "$emoji $message."; git push | ||
| 133 | +else | ||
| 134 | + # not empty | ||
| 135 | + git add .; git commit -m "$emoji $message: $2"; git push | ||
| 136 | +fi | ||
added
kits/llama-cpp-olala/llama-cpp-olala-context.md +9 -0 | new file mode 100644 | ||
| @@ -0,0 +1,9 @@ | ||
| 1 | +# llama.cpp (OVHai Olala fork) | |
| 2 | + | |
| 3 | +`llama-server` and `llama-cli` on the PATH come from the OVHai fork of llama.cpp | |
| 4 | +(`/opt/llama-cpp-olala`, commit in `/opt/llama-cpp-olala/REVISION`), CPU only. | |
| 5 | +They load `olala` GGUF models, which upstream llama.cpp cannot. | |
| 6 | + | |
| 7 | +- Serve: `llama-server -m <file.gguf> -t <threads> -fa 1 -nr -ctk q8_0 -ctv q8_0 -c 32768 -np 4 --host 0.0.0.0 --port 8080` | |
| 8 | +- In this project, prefer `scripts/olala.sh up|serve|stop|status` (downloads the model if needed). | |
| 9 | +- Recommended sampling: `temperature=0.8`, `top_p=0.9`, `top_k=40`; `reasoning_effort`: disabled, low, medium, high. | |
| new file mode 100644 | |||
| @@ -0,0 +1,9 @@ | |||
| 1 | +# llama.cpp (OVHai Olala fork) | ||
| 2 | + | ||
| 3 | +`llama-server` and `llama-cli` on the PATH come from the OVHai fork of llama.cpp | ||
| 4 | +(`/opt/llama-cpp-olala`, commit in `/opt/llama-cpp-olala/REVISION`), CPU only. | ||
| 5 | +They load `olala` GGUF models, which upstream llama.cpp cannot. | ||
| 6 | + | ||
| 7 | +- Serve: `llama-server -m <file.gguf> -t <threads> -fa 1 -nr -ctk q8_0 -ctv q8_0 -c 32768 -np 4 --host 0.0.0.0 --port 8080` | ||
| 8 | +- In this project, prefer `scripts/olala.sh up|serve|stop|status` (downloads the model if needed). | ||
| 9 | +- Recommended sampling: `temperature=0.8`, `top_p=0.9`, `top_k=40`; `reasoning_effort`: disabled, low, medium, high. | ||
added
kits/llama-cpp-olala/llama-cpp-olala.dockerfile +61 -0 | new file mode 100644 | ||
| @@ -0,0 +1,61 @@ | ||
| 1 | +# syntax=docker/dockerfile:1 | |
| 2 | + | |
| 3 | +# --- build stage: the OVHai llama.cpp fork, CPU only, portable ------------------ | |
| 4 | +# Same flags as the fork's .devops/cpu.Dockerfile: no -march=native, and every CPU | |
| 5 | +# variant built as a backend library loaded at runtime for the CPU it runs on. | |
| 6 | +FROM debian:trixie-slim AS build | |
| 7 | +ARG LLAMA_REF | |
| 8 | +ARG TARGETARCH | |
| 9 | + | |
| 10 | +# A heredoc RUN does not stop at the first failing line by itself: every | |
| 11 | +# script starts with `set -eu`. | |
| 12 | +RUN <<EOF | |
| 13 | +set -eu | |
| 14 | +apt-get update | |
| 15 | +apt-get install -y --no-install-recommends build-essential cmake git ca-certificates libgomp1 | |
| 16 | +rm -rf /var/lib/apt/lists/* | |
| 17 | +EOF | |
| 18 | + | |
| 19 | +RUN <<EOF | |
| 20 | +set -eu | |
| 21 | +git init -q /src | |
| 22 | +cd /src | |
| 23 | +git remote add origin https://github.com/OVHai-LLM/llama.cpp.git | |
| 24 | +git fetch -q --depth 1 origin "${LLAMA_REF}" | |
| 25 | +git checkout -q FETCH_HEAD | |
| 26 | +EOF | |
| 27 | + | |
| 28 | +# -DLLAMA_USE_PREBUILT_UI=OFF: the prebuilt web UI does not match the fork (model card). | |
| 29 | +# -DLLAMA_OPENSSL=OFF: llama-server serves plain HTTP; downloads are done with `hf`. | |
| 30 | +# RPATH=$ORIGIN: binaries find their libraries next to them, wherever they are installed. | |
| 31 | +RUN <<EOF | |
| 32 | +set -eu | |
| 33 | +case "${TARGETARCH}" in | |
| 34 | + amd64|arm64) ;; | |
| 35 | + *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; | |
| 36 | +esac | |
| 37 | +cd /src | |
| 38 | +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \ | |
| 39 | + -DGGML_NATIVE=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON \ | |
| 40 | + -DLLAMA_USE_PREBUILT_UI=OFF -DLLAMA_OPENSSL=OFF \ | |
| 41 | + -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF \ | |
| 42 | + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_RPATH='$ORIGIN' | |
| 43 | +cmake --build build -j "$(nproc)" --target llama-server llama-cli | |
| 44 | +EOF | |
| 45 | + | |
| 46 | +RUN <<EOF | |
| 47 | +set -eu | |
| 48 | +dest=/out/opt/llama-cpp-olala | |
| 49 | +mkdir -p "$dest" /out/usr/local/bin | |
| 50 | +cp -P /src/build/bin/llama-server /src/build/bin/llama-cli "$dest"/ | |
| 51 | +find /src/build/bin -name '*.so*' -exec cp -P {} "$dest"/ \; | |
| 52 | +# OpenMP runtime: not guaranteed on the sandbox image. | |
| 53 | +cp -L "$(gcc -print-file-name=libgomp.so.1)" "$dest"/libgomp.so.1 | |
| 54 | +git -C /src rev-parse HEAD > "$dest"/REVISION | |
| 55 | +ln -s /opt/llama-cpp-olala/llama-server /out/usr/local/bin/llama-server | |
| 56 | +ln -s /opt/llama-cpp-olala/llama-cli /out/usr/local/bin/llama-cli | |
| 57 | +EOF | |
| 58 | + | |
| 59 | +# --- overlay: only the binaries and their libraries land on the workload --------- | |
| 60 | +FROM scratch | |
| 61 | +COPY --from=build /out/ / | |
| new file mode 100644 | |||
| @@ -0,0 +1,61 @@ | |||
| 1 | +# syntax=docker/dockerfile:1 | ||
| 2 | + | ||
| 3 | +# --- build stage: the OVHai llama.cpp fork, CPU only, portable ------------------ | ||
| 4 | +# Same flags as the fork's .devops/cpu.Dockerfile: no -march=native, and every CPU | ||
| 5 | +# variant built as a backend library loaded at runtime for the CPU it runs on. | ||
| 6 | +FROM debian:trixie-slim AS build | ||
| 7 | +ARG LLAMA_REF | ||
| 8 | +ARG TARGETARCH | ||
| 9 | + | ||
| 10 | +# A heredoc RUN does not stop at the first failing line by itself: every | ||
| 11 | +# script starts with `set -eu`. | ||
| 12 | +RUN <<EOF | ||
| 13 | +set -eu | ||
| 14 | +apt-get update | ||
| 15 | +apt-get install -y --no-install-recommends build-essential cmake git ca-certificates libgomp1 | ||
| 16 | +rm -rf /var/lib/apt/lists/* | ||
| 17 | +EOF | ||
| 18 | + | ||
| 19 | +RUN <<EOF | ||
| 20 | +set -eu | ||
| 21 | +git init -q /src | ||
| 22 | +cd /src | ||
| 23 | +git remote add origin https://github.com/OVHai-LLM/llama.cpp.git | ||
| 24 | +git fetch -q --depth 1 origin "${LLAMA_REF}" | ||
| 25 | +git checkout -q FETCH_HEAD | ||
| 26 | +EOF | ||
| 27 | + | ||
| 28 | +# -DLLAMA_USE_PREBUILT_UI=OFF: the prebuilt web UI does not match the fork (model card). | ||
| 29 | +# -DLLAMA_OPENSSL=OFF: llama-server serves plain HTTP; downloads are done with `hf`. | ||
| 30 | +# RPATH=$ORIGIN: binaries find their libraries next to them, wherever they are installed. | ||
| 31 | +RUN <<EOF | ||
| 32 | +set -eu | ||
| 33 | +case "${TARGETARCH}" in | ||
| 34 | + amd64|arm64) ;; | ||
| 35 | + *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; | ||
| 36 | +esac | ||
| 37 | +cd /src | ||
| 38 | +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \ | ||
| 39 | + -DGGML_NATIVE=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON \ | ||
| 40 | + -DLLAMA_USE_PREBUILT_UI=OFF -DLLAMA_OPENSSL=OFF \ | ||
| 41 | + -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF \ | ||
| 42 | + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_RPATH='$ORIGIN' | ||
| 43 | +cmake --build build -j "$(nproc)" --target llama-server llama-cli | ||
| 44 | +EOF | ||
| 45 | + | ||
| 46 | +RUN <<EOF | ||
| 47 | +set -eu | ||
| 48 | +dest=/out/opt/llama-cpp-olala | ||
| 49 | +mkdir -p "$dest" /out/usr/local/bin | ||
| 50 | +cp -P /src/build/bin/llama-server /src/build/bin/llama-cli "$dest"/ | ||
| 51 | +find /src/build/bin -name '*.so*' -exec cp -P {} "$dest"/ \; | ||
| 52 | +# OpenMP runtime: not guaranteed on the sandbox image. | ||
| 53 | +cp -L "$(gcc -print-file-name=libgomp.so.1)" "$dest"/libgomp.so.1 | ||
| 54 | +git -C /src rev-parse HEAD > "$dest"/REVISION | ||
| 55 | +ln -s /opt/llama-cpp-olala/llama-server /out/usr/local/bin/llama-server | ||
| 56 | +ln -s /opt/llama-cpp-olala/llama-cli /out/usr/local/bin/llama-cli | ||
| 57 | +EOF | ||
| 58 | + | ||
| 59 | +# --- overlay: only the binaries and their libraries land on the workload --------- | ||
| 60 | +FROM scratch | ||
| 61 | +COPY --from=build /out/ / | ||
added
kits/llama-cpp-olala/llama-cpp-olala.yaml +29 -0 | new file mode 100644 | ||
| @@ -0,0 +1,29 @@ | ||
| 1 | +# syntax=docker/sandbox-kit:3 | |
| 2 | +schemaVersion: "3" | |
| 3 | +kind: mixin | |
| 4 | +displayName: llama.cpp (OVHai Olala fork, CPU) | |
| 5 | +description: llama-server and llama-cli from the OVHai llama.cpp fork, which supports the `olala` architecture (CPU only) | |
| 6 | + | |
| 7 | +args: | |
| 8 | + ref: | |
| 9 | + # Commit of https://github.com/OVHai-LLM/llama.cpp to build. Pin a full SHA for reproducible builds. | |
| 10 | + default: "55c22dca0c987f247c21b427c3b3a09e1a28187f" | |
| 11 | + pattern: '^[0-9a-f]{40}$' | |
| 12 | + buildArg: LLAMA_REF | |
| 13 | + | |
| 14 | +# provides needs an X.Y.Z version (a commit SHA is rejected by the kit frontend): | |
| 15 | +# 0.5.0 = what the fork reports (`llama-server --version`: 0.5.0-dev). The commit is pinned by `ref`. | |
| 16 | +provides: ["llama-cpp-olala@0.5.0"] | |
| 17 | + | |
| 18 | +capabilities: | |
| 19 | + # Runtime access to download GGUF models from Hugging Face (hf download, llama-server -hf). | |
| 20 | + - type: com.docker.sandbox/network-policy@1 | |
| 21 | + config: | |
| 22 | + runtime: | |
| 23 | + allow: | |
| 24 | + - huggingface.co | |
| 25 | + - "*.huggingface.co" | |
| 26 | + - "*.hf.co" | |
| 27 | + - type: com.docker.sandbox/agent-context@1 | |
| 28 | + config: | |
| 29 | + contentFile: ./llama-cpp-olala-context.md | |
| new file mode 100644 | |||
| @@ -0,0 +1,29 @@ | |||
| 1 | +# syntax=docker/sandbox-kit:3 | ||
| 2 | +schemaVersion: "3" | ||
| 3 | +kind: mixin | ||
| 4 | +displayName: llama.cpp (OVHai Olala fork, CPU) | ||
| 5 | +description: llama-server and llama-cli from the OVHai llama.cpp fork, which supports the `olala` architecture (CPU only) | ||
| 6 | + | ||
| 7 | +args: | ||
| 8 | + ref: | ||
| 9 | + # Commit of https://github.com/OVHai-LLM/llama.cpp to build. Pin a full SHA for reproducible builds. | ||
| 10 | + default: "55c22dca0c987f247c21b427c3b3a09e1a28187f" | ||
| 11 | + pattern: '^[0-9a-f]{40}$' | ||
| 12 | + buildArg: LLAMA_REF | ||
| 13 | + | ||
| 14 | +# provides needs an X.Y.Z version (a commit SHA is rejected by the kit frontend): | ||
| 15 | +# 0.5.0 = what the fork reports (`llama-server --version`: 0.5.0-dev). The commit is pinned by `ref`. | ||
| 16 | +provides: ["llama-cpp-olala@0.5.0"] | ||
| 17 | + | ||
| 18 | +capabilities: | ||
| 19 | + # Runtime access to download GGUF models from Hugging Face (hf download, llama-server -hf). | ||
| 20 | + - type: com.docker.sandbox/network-policy@1 | ||
| 21 | + config: | ||
| 22 | + runtime: | ||
| 23 | + allow: | ||
| 24 | + - huggingface.co | ||
| 25 | + - "*.huggingface.co" | ||
| 26 | + - "*.hf.co" | ||
| 27 | + - type: com.docker.sandbox/agent-context@1 | ||
| 28 | + config: | ||
| 29 | + contentFile: ./llama-cpp-olala-context.md | ||
added
sbxenv.dev.yaml +39 -0 | new file mode 100644 | ||
| @@ -0,0 +1,39 @@ | ||
| 1 | +schemaVersion: "1" | |
| 2 | + | |
| 3 | +# Development sandbox (Claude Code + Go/mini-me toolchains). | |
| 4 | +# Python and uv come from the claude agent kit itself: do not add python-toolchain-kit | |
| 5 | +# (two kits cannot provide the same capability). | |
| 6 | +# The Olala model server is described by sbxenv.yaml (sandbox houla-olala). | |
| 7 | + | |
| 8 | +name: houla-houla | |
| 9 | + | |
| 10 | +agent: claude | |
| 11 | + | |
| 12 | +# Primary workspace: this folder (relative paths resolve against this file). | |
| 13 | +workspace: . | |
| 14 | + | |
| 15 | +additionalWorkspaces: | |
| 16 | + - path: /Users/k33g/Documents | |
| 17 | + - path: /Users/k33g/kDrive/Rickub/bots-garden/mini-me | |
| 18 | + | |
| 19 | +kits: | |
| 20 | + - docker.io/k33g/go-toolchain-kit:0.2.0 | |
| 21 | + - docker.io/k33g/mini-me-agent-kit:0.2.0 | |
| 22 | + | |
| 23 | +env: | |
| 24 | + # Claude Code default model | |
| 25 | + ANTHROPIC_MODEL: claude-opus-5-5 | |
| 26 | + | |
| 27 | +# Lifecycle commands run on the HOST, so we reach into the sandbox with `sbx exec`. | |
| 28 | +# postCreate runs once, right after a new sandbox is created (not on re-attach). | |
| 29 | +lifecycle: | |
| 30 | + postCreate: | |
| 31 | + - name: Install the Hugging Face CLI (hf) | |
| 32 | + command: sbx exec "$SBX_SANDBOX_NAME" bash -lc 'uv tool install "huggingface_hub[cli]" && hf version' | |
| 33 | + timeout: 5m | |
| 34 | + | |
| 35 | + | |
| 36 | + | |
| 37 | +# sbx env create sbxenv.dev.yaml # shows the plan and asks for confirmation (--force to skip) | |
| 38 | +# sbx env run sbxenv.dev.yaml # re-attach (creates the sandbox first if it does not exist) | |
| 39 | +# sbx env rm sbxenv.dev.yaml # delete the sandbox | |
| new file mode 100644 | |||
| @@ -0,0 +1,39 @@ | |||
| 1 | +schemaVersion: "1" | ||
| 2 | + | ||
| 3 | +# Development sandbox (Claude Code + Go/mini-me toolchains). | ||
| 4 | +# Python and uv come from the claude agent kit itself: do not add python-toolchain-kit | ||
| 5 | +# (two kits cannot provide the same capability). | ||
| 6 | +# The Olala model server is described by sbxenv.yaml (sandbox houla-olala). | ||
| 7 | + | ||
| 8 | +name: houla-houla | ||
| 9 | + | ||
| 10 | +agent: claude | ||
| 11 | + | ||
| 12 | +# Primary workspace: this folder (relative paths resolve against this file). | ||
| 13 | +workspace: . | ||
| 14 | + | ||
| 15 | +additionalWorkspaces: | ||
| 16 | + - path: /Users/k33g/Documents | ||
| 17 | + - path: /Users/k33g/kDrive/Rickub/bots-garden/mini-me | ||
| 18 | + | ||
| 19 | +kits: | ||
| 20 | + - docker.io/k33g/go-toolchain-kit:0.2.0 | ||
| 21 | + - docker.io/k33g/mini-me-agent-kit:0.2.0 | ||
| 22 | + | ||
| 23 | +env: | ||
| 24 | + # Claude Code default model | ||
| 25 | + ANTHROPIC_MODEL: claude-opus-5-5 | ||
| 26 | + | ||
| 27 | +# Lifecycle commands run on the HOST, so we reach into the sandbox with `sbx exec`. | ||
| 28 | +# postCreate runs once, right after a new sandbox is created (not on re-attach). | ||
| 29 | +lifecycle: | ||
| 30 | + postCreate: | ||
| 31 | + - name: Install the Hugging Face CLI (hf) | ||
| 32 | + command: sbx exec "$SBX_SANDBOX_NAME" bash -lc 'uv tool install "huggingface_hub[cli]" && hf version' | ||
| 33 | + timeout: 5m | ||
| 34 | + | ||
| 35 | + | ||
| 36 | + | ||
| 37 | +# sbx env create sbxenv.dev.yaml # shows the plan and asks for confirmation (--force to skip) | ||
| 38 | +# sbx env run sbxenv.dev.yaml # re-attach (creates the sandbox first if it does not exist) | ||
| 39 | +# sbx env rm sbxenv.dev.yaml # delete the sandbox | ||
added
sbxenv.yaml +76 -0 | new file mode 100644 | ||
| @@ -0,0 +1,76 @@ | ||
| 1 | +schemaVersion: "1" | |
| 2 | + | |
| 3 | +# Sandbox that serves an Olala GGUF model with llama-server (OpenAI-compatible API) | |
| 4 | +# on the host, using the OVHai llama.cpp fork (CPU only) from kits/llama-cpp-olala. | |
| 5 | +# | |
| 6 | +# sbx env create sbxenv.yaml # create + download + serve | |
| 7 | +# sbx env create sbxenv.yaml --env-arg host_port=9090 # another host port | |
| 8 | +# sbx env create sbxenv.yaml --env-arg models_dir=./my-models # another models folder | |
| 9 | +# sbx env run sbxenv.yaml # attach Claude Code | |
| 10 | +# sbx env exec --no-input sbxenv.yaml -- bash -lc '$SBX_WS/scripts/olala.sh serve' # restart after `sbx stop` | |
| 11 | +# sbx env rm sbxenv.yaml # models stay in models_dir | |
| 12 | +# | |
| 13 | +# Every step of scripts/olala.sh is skipped when already done (llama-server found, | |
| 14 | +# model present, server running), so re-running it is cheap. | |
| 15 | +# Use the same --env-arg values with every sbx env command of a given sandbox. | |
| 16 | + | |
| 17 | +name: houla-olala | |
| 18 | + | |
| 19 | +agent: claude | |
| 20 | + | |
| 21 | +workspace: . | |
| 22 | + | |
| 23 | +kits: | |
| 24 | + # llama-server / llama-cli from the OVHai fork (built by sbx from the local kit). | |
| 25 | + - ./kits/llama-cpp-olala | |
| 26 | + | |
| 27 | +args: | |
| 28 | + model_repo: | |
| 29 | + default: OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF | |
| 30 | + description: Hugging Face repository of the GGUF file | |
| 31 | + model_file: | |
| 32 | + default: Olala-v1.0-mini-7A1B-q8_0-safe2.gguf | |
| 33 | + description: GGUF file to download and serve (q8_0-safe2 = recommended for CPUs) | |
| 34 | + models_dir: | |
| 35 | + default: ./models | |
| 36 | + description: Where models are stored; relative to the repository root, kept when the sandbox is removed | |
| 37 | + host_port: | |
| 38 | + default: "8080" | |
| 39 | + pattern: "[1-9][0-9]*" | |
| 40 | + description: Host port that publishes llama-server | |
| 41 | + threads: | |
| 42 | + default: "16" | |
| 43 | + pattern: "[1-9][0-9]*" | |
| 44 | + description: CPU threads used by llama-server (leave a couple of CPUs free) | |
| 45 | + ctx: | |
| 46 | + default: "32768" | |
| 47 | + pattern: "[1-9][0-9]*" | |
| 48 | + description: Context size (shared by the parallel slots) | |
| 49 | + | |
| 50 | +env: | |
| 51 | + ANTHROPIC_MODEL: claude-opus-5-5 | |
| 52 | + OLALA_MODEL_REPO: "${{ env.args.model_repo }}" | |
| 53 | + OLALA_MODEL_FILE: "${{ env.args.model_file }}" | |
| 54 | + OLALA_MODELS_DIR: "${{ env.args.models_dir }}" | |
| 55 | + OLALA_THREADS: "${{ env.args.threads }}" | |
| 56 | + OLALA_CTX: "${{ env.args.ctx }}" | |
| 57 | + OLALA_PORT: "8080" | |
| 58 | + SBX_WS: "${{ env.fileDir }}" | |
| 59 | + | |
| 60 | +ports: | |
| 61 | + - sandbox: 8080 | |
| 62 | + host: ${{ env.args.host_port }} | |
| 63 | + | |
| 64 | +# Lifecycle commands run on the HOST; they reach into the sandbox with `sbx env exec`. | |
| 65 | +# postCreate runs once, after creation (not on re-attach). | |
| 66 | +lifecycle: | |
| 67 | + postCreate: | |
| 68 | + # A local sandbox stops as soon as no session keeps it running (and llama-server with it). | |
| 69 | + # Detached mode keeps it running until `sbx stop`; it is permanent for this sandbox. | |
| 70 | + - name: Keep the sandbox running in the background (detached) | |
| 71 | + command: sbx run -d --name "$SBX_SANDBOX_NAME" | |
| 72 | + timeout: 2m | |
| 73 | + - name: Download the model (if missing) and start llama-server | |
| 74 | + # Flags go BEFORE the env file path: after it, sbx reads every word up to `--` as another path. | |
| 75 | + command: sbx env exec --name "$SBX_SANDBOX_NAME" --no-input "$SBX_ENV_FILE" -- bash -lc "$SBX_WORKSPACE/scripts/olala.sh up" | |
| 76 | + timeout: 30m | |
| new file mode 100644 | |||
| @@ -0,0 +1,76 @@ | |||
| 1 | +schemaVersion: "1" | ||
| 2 | + | ||
| 3 | +# Sandbox that serves an Olala GGUF model with llama-server (OpenAI-compatible API) | ||
| 4 | +# on the host, using the OVHai llama.cpp fork (CPU only) from kits/llama-cpp-olala. | ||
| 5 | +# | ||
| 6 | +# sbx env create sbxenv.yaml # create + download + serve | ||
| 7 | +# sbx env create sbxenv.yaml --env-arg host_port=9090 # another host port | ||
| 8 | +# sbx env create sbxenv.yaml --env-arg models_dir=./my-models # another models folder | ||
| 9 | +# sbx env run sbxenv.yaml # attach Claude Code | ||
| 10 | +# sbx env exec --no-input sbxenv.yaml -- bash -lc '$SBX_WS/scripts/olala.sh serve' # restart after `sbx stop` | ||
| 11 | +# sbx env rm sbxenv.yaml # models stay in models_dir | ||
| 12 | +# | ||
| 13 | +# Every step of scripts/olala.sh is skipped when already done (llama-server found, | ||
| 14 | +# model present, server running), so re-running it is cheap. | ||
| 15 | +# Use the same --env-arg values with every sbx env command of a given sandbox. | ||
| 16 | + | ||
| 17 | +name: houla-olala | ||
| 18 | + | ||
| 19 | +agent: claude | ||
| 20 | + | ||
| 21 | +workspace: . | ||
| 22 | + | ||
| 23 | +kits: | ||
| 24 | + # llama-server / llama-cli from the OVHai fork (built by sbx from the local kit). | ||
| 25 | + - ./kits/llama-cpp-olala | ||
| 26 | + | ||
| 27 | +args: | ||
| 28 | + model_repo: | ||
| 29 | + default: OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF | ||
| 30 | + description: Hugging Face repository of the GGUF file | ||
| 31 | + model_file: | ||
| 32 | + default: Olala-v1.0-mini-7A1B-q8_0-safe2.gguf | ||
| 33 | + description: GGUF file to download and serve (q8_0-safe2 = recommended for CPUs) | ||
| 34 | + models_dir: | ||
| 35 | + default: ./models | ||
| 36 | + description: Where models are stored; relative to the repository root, kept when the sandbox is removed | ||
| 37 | + host_port: | ||
| 38 | + default: "8080" | ||
| 39 | + pattern: "[1-9][0-9]*" | ||
| 40 | + description: Host port that publishes llama-server | ||
| 41 | + threads: | ||
| 42 | + default: "16" | ||
| 43 | + pattern: "[1-9][0-9]*" | ||
| 44 | + description: CPU threads used by llama-server (leave a couple of CPUs free) | ||
| 45 | + ctx: | ||
| 46 | + default: "32768" | ||
| 47 | + pattern: "[1-9][0-9]*" | ||
| 48 | + description: Context size (shared by the parallel slots) | ||
| 49 | + | ||
| 50 | +env: | ||
| 51 | + ANTHROPIC_MODEL: claude-opus-5-5 | ||
| 52 | + OLALA_MODEL_REPO: "${{ env.args.model_repo }}" | ||
| 53 | + OLALA_MODEL_FILE: "${{ env.args.model_file }}" | ||
| 54 | + OLALA_MODELS_DIR: "${{ env.args.models_dir }}" | ||
| 55 | + OLALA_THREADS: "${{ env.args.threads }}" | ||
| 56 | + OLALA_CTX: "${{ env.args.ctx }}" | ||
| 57 | + OLALA_PORT: "8080" | ||
| 58 | + SBX_WS: "${{ env.fileDir }}" | ||
| 59 | + | ||
| 60 | +ports: | ||
| 61 | + - sandbox: 8080 | ||
| 62 | + host: ${{ env.args.host_port }} | ||
| 63 | + | ||
| 64 | +# Lifecycle commands run on the HOST; they reach into the sandbox with `sbx env exec`. | ||
| 65 | +# postCreate runs once, after creation (not on re-attach). | ||
| 66 | +lifecycle: | ||
| 67 | + postCreate: | ||
| 68 | + # A local sandbox stops as soon as no session keeps it running (and llama-server with it). | ||
| 69 | + # Detached mode keeps it running until `sbx stop`; it is permanent for this sandbox. | ||
| 70 | + - name: Keep the sandbox running in the background (detached) | ||
| 71 | + command: sbx run -d --name "$SBX_SANDBOX_NAME" | ||
| 72 | + timeout: 2m | ||
| 73 | + - name: Download the model (if missing) and start llama-server | ||
| 74 | + # Flags go BEFORE the env file path: after it, sbx reads every word up to `--` as another path. | ||
| 75 | + command: sbx env exec --name "$SBX_SANDBOX_NAME" --no-input "$SBX_ENV_FILE" -- bash -lc "$SBX_WORKSPACE/scripts/olala.sh up" | ||
| 76 | + timeout: 30m | ||
added
scripts/olala.sh +190 -0 | new file mode 100755 | ||
| @@ -0,0 +1,190 @@ | ||
| 1 | +#!/usr/bin/env bash | |
| 2 | +# Download a GGUF model and serve it with llama-server (OpenAI-compatible API), | |
| 3 | +# using the OVHai llama.cpp fork (CPU only). Runs INSIDE the sandbox. | |
| 4 | +# | |
| 5 | +# Every step is idempotent: it is skipped when its result already exists. | |
| 6 | +# | |
| 7 | +# scripts/olala.sh setup # find llama-server (kit) or build the fork (skipped if found/built) | |
| 8 | +# scripts/olala.sh download # download the model (skipped if the file exists) | |
| 9 | +# scripts/olala.sh serve # start llama-server in the background (skipped if running) | |
| 10 | +# scripts/olala.sh up # setup + download + serve | |
| 11 | +# scripts/olala.sh stop | restart | status | logs | |
| 12 | +# | |
| 13 | +# Configuration (environment variables, defaults below): | |
| 14 | +# OLALA_MODEL_REPO Hugging Face repo of the GGUF file | |
| 15 | +# OLALA_MODEL_FILE GGUF file name in that repo | |
| 16 | +# OLALA_MODELS_DIR where models are stored; relative paths are relative to the repo root | |
| 17 | +# (default: ./models, kept in the workspace so dropping the sandbox keeps them) | |
| 18 | +# OLALA_PORT port llama-server listens on (0.0.0.0) | |
| 19 | +# OLALA_THREADS CPU threads (default: nproc - 2; using every CPU is ~30% slower) | |
| 20 | +# OLALA_CTX context size | |
| 21 | +# OLALA_PARALLEL parallel slots | |
| 22 | +# OLALA_ALIAS model name exposed by the API (default: file name without .gguf) | |
| 23 | +# OLALA_LOAD_MODE llama-server --load-mode (default: none = read the model into RAM; | |
| 24 | +# mmap over the workspace mount is ~20x slower at generation) | |
| 25 | +# OLALA_EXTRA_ARGS extra llama-server arguments | |
| 26 | +# OLALA_SERVER_BIN llama-server to use (default: the one on PATH, e.g. from the | |
| 27 | +# kits/llama-cpp-olala kit, else the one built in OLALA_SRC_DIR) | |
| 28 | +# OLALA_SRC_DIR where the fork is cloned and built when no llama-server is found | |
| 29 | +set -euo pipefail | |
| 30 | + | |
| 31 | +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" | |
| 32 | + | |
| 33 | +OLALA_MODEL_REPO="${OLALA_MODEL_REPO:-OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF}" | |
| 34 | +OLALA_MODEL_FILE="${OLALA_MODEL_FILE:-Olala-v1.0-mini-7A1B-q8_0-safe2.gguf}" | |
| 35 | +OLALA_MODELS_DIR="${OLALA_MODELS_DIR:-./models}" | |
| 36 | +OLALA_PORT="${OLALA_PORT:-8080}" | |
| 37 | +OLALA_THREADS="${OLALA_THREADS:-$(( $(nproc) > 2 ? $(nproc) - 2 : 1 ))}" | |
| 38 | +OLALA_CTX="${OLALA_CTX:-32768}" | |
| 39 | +OLALA_PARALLEL="${OLALA_PARALLEL:-4}" | |
| 40 | +OLALA_ALIAS="${OLALA_ALIAS:-${OLALA_MODEL_FILE%.gguf}}" | |
| 41 | +OLALA_LOAD_MODE="${OLALA_LOAD_MODE:-none}" | |
| 42 | +OLALA_EXTRA_ARGS="${OLALA_EXTRA_ARGS:-}" | |
| 43 | +OLALA_SRC_DIR="${OLALA_SRC_DIR:-$HOME/olala/llama.cpp-olala}" | |
| 44 | +OLALA_REPO_URL="${OLALA_REPO_URL:-https://github.com/OVHai-LLM/llama.cpp.git}" | |
| 45 | + | |
| 46 | +case "$OLALA_MODELS_DIR" in | |
| 47 | + /*) ;; | |
| 48 | + *) OLALA_MODELS_DIR="$REPO_ROOT/${OLALA_MODELS_DIR#./}" ;; | |
| 49 | +esac | |
| 50 | + | |
| 51 | +RUN_DIR="$HOME/.olala" | |
| 52 | +PID_FILE="$RUN_DIR/llama-server.pid" | |
| 53 | +LOG_FILE="$RUN_DIR/llama-server.log" | |
| 54 | +BUILT_BIN="$OLALA_SRC_DIR/build/bin/llama-server" | |
| 55 | +MODEL_PATH="$OLALA_MODELS_DIR/$OLALA_MODEL_FILE" | |
| 56 | + | |
| 57 | +log() { printf '[olala] %s\n' "$*" >&2; } | |
| 58 | + | |
| 59 | +# Prints the llama-server to use, or nothing. | |
| 60 | +server_bin() { | |
| 61 | + if [ -n "${OLALA_SERVER_BIN:-}" ]; then | |
| 62 | + echo "$OLALA_SERVER_BIN" | |
| 63 | + elif command -v llama-server >/dev/null; then | |
| 64 | + command -v llama-server | |
| 65 | + elif [ -x "$BUILT_BIN" ]; then | |
| 66 | + echo "$BUILT_BIN" | |
| 67 | + fi | |
| 68 | +} | |
| 69 | + | |
| 70 | +setup() { | |
| 71 | + local bin | |
| 72 | + bin="$(server_bin)" | |
| 73 | + if [ -n "$bin" ]; then | |
| 74 | + log "llama-server found: $bin" | |
| 75 | + return | |
| 76 | + fi | |
| 77 | + log "no llama-server on PATH (kits/llama-cpp-olala not installed): building the fork" | |
| 78 | + if ! command -v cmake >/dev/null || ! command -v g++ >/dev/null; then | |
| 79 | + log "installing build tools (build-essential, cmake, libssl-dev)" | |
| 80 | + sudo apt-get update -qq | |
| 81 | + sudo apt-get install -y -qq build-essential cmake libssl-dev pkg-config git >/dev/null | |
| 82 | + fi | |
| 83 | + if [ ! -d "$OLALA_SRC_DIR/.git" ]; then | |
| 84 | + log "cloning $OLALA_REPO_URL" | |
| 85 | + mkdir -p "$(dirname "$OLALA_SRC_DIR")" | |
| 86 | + git clone --depth 1 "$OLALA_REPO_URL" "$OLALA_SRC_DIR" | |
| 87 | + fi | |
| 88 | + log "building (CPU only)" | |
| 89 | + # -DLLAMA_USE_PREBUILT_UI=OFF: the prebuilt web UI does not match the fork (per the model card). | |
| 90 | + cmake -S "$OLALA_SRC_DIR" -B "$OLALA_SRC_DIR/build" -DCMAKE_BUILD_TYPE=Release -DLLAMA_USE_PREBUILT_UI=OFF >/dev/null | |
| 91 | + cmake --build "$OLALA_SRC_DIR/build" -j "$(nproc)" --target llama-server llama-cli >/dev/null | |
| 92 | + log "built: $BUILT_BIN" | |
| 93 | +} | |
| 94 | + | |
| 95 | +download() { | |
| 96 | + if [ -s "$MODEL_PATH" ]; then | |
| 97 | + log "model already present: $MODEL_PATH" | |
| 98 | + return | |
| 99 | + fi | |
| 100 | + if ! command -v hf >/dev/null; then | |
| 101 | + log "installing the Hugging Face CLI" | |
| 102 | + uv tool install "huggingface_hub[cli]" >/dev/null | |
| 103 | + export PATH="$HOME/.local/bin:$PATH" | |
| 104 | + fi | |
| 105 | + mkdir -p "$OLALA_MODELS_DIR" | |
| 106 | + log "downloading $OLALA_MODEL_REPO / $OLALA_MODEL_FILE -> $OLALA_MODELS_DIR" | |
| 107 | + hf download "$OLALA_MODEL_REPO" "$OLALA_MODEL_FILE" --local-dir "$OLALA_MODELS_DIR" | |
| 108 | +} | |
| 109 | + | |
| 110 | +is_running() { | |
| 111 | + [ -f "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null | |
| 112 | +} | |
| 113 | + | |
| 114 | +serve() { | |
| 115 | + if is_running; then | |
| 116 | + log "llama-server already running (pid $(cat "$PID_FILE")) on port $OLALA_PORT" | |
| 117 | + return | |
| 118 | + fi | |
| 119 | + local bin | |
| 120 | + bin="$(server_bin)" | |
| 121 | + [ -n "$bin" ] || { log "llama-server not found: run '$0 setup'"; exit 1; } | |
| 122 | + [ -s "$MODEL_PATH" ] || { log "model missing: run '$0 download'"; exit 1; } | |
| 123 | + mkdir -p "$RUN_DIR" | |
| 124 | + log "starting $bin: $MODEL_PATH on 0.0.0.0:$OLALA_PORT (threads=$OLALA_THREADS ctx=$OLALA_CTX load-mode=$OLALA_LOAD_MODE)" | |
| 125 | + # Flags from the model card's "CPU" server example; --host 0.0.0.0 so the port can be published. | |
| 126 | + # The inner shell writes its own pid then execs llama-server, so the pid file is exact | |
| 127 | + # (setsid may fork, which made $! point at a process that had already exited). | |
| 128 | + # shellcheck disable=SC2016,SC2086 | |
| 129 | + nohup setsid bash -c 'echo $$ >"$0"; exec "$@"' "$PID_FILE" \ | |
| 130 | + "$bin" \ | |
| 131 | + -m "$MODEL_PATH" \ | |
| 132 | + --load-mode "$OLALA_LOAD_MODE" \ | |
| 133 | + -t "$OLALA_THREADS" -fa 1 -nr -ctk q8_0 -ctv q8_0 \ | |
| 134 | + -c "$OLALA_CTX" -np "$OLALA_PARALLEL" \ | |
| 135 | + --temp 0.8 --top-p 0.9 --top-k 40 \ | |
| 136 | + --alias "$OLALA_ALIAS" \ | |
| 137 | + --host 0.0.0.0 --port "$OLALA_PORT" \ | |
| 138 | + $OLALA_EXTRA_ARGS \ | |
| 139 | + >"$LOG_FILE" 2>&1 </dev/null & | |
| 140 | + log "waiting for /health (log: $LOG_FILE)" | |
| 141 | + for _ in $(seq 1 300); do | |
| 142 | + if curl -fsS "http://127.0.0.1:$OLALA_PORT/health" >/dev/null 2>&1; then | |
| 143 | + log "ready: http://127.0.0.1:$OLALA_PORT/v1" | |
| 144 | + return | |
| 145 | + fi | |
| 146 | + if [ -f "$PID_FILE" ] && ! is_running; then | |
| 147 | + log "llama-server exited, last lines:" | |
| 148 | + tail -20 "$LOG_FILE" >&2 | |
| 149 | + exit 1 | |
| 150 | + fi | |
| 151 | + sleep 1 | |
| 152 | + done | |
| 153 | + log "not ready after 300 s, see $LOG_FILE" | |
| 154 | + exit 1 | |
| 155 | +} | |
| 156 | + | |
| 157 | +stop() { | |
| 158 | + if is_running; then | |
| 159 | + local pid | |
| 160 | + pid="$(cat "$PID_FILE")" | |
| 161 | + kill "$pid" | |
| 162 | + for _ in $(seq 1 30); do kill -0 "$pid" 2>/dev/null || break; sleep 0.5; done | |
| 163 | + kill -0 "$pid" 2>/dev/null && kill -9 "$pid" | |
| 164 | + rm -f "$PID_FILE" | |
| 165 | + log "stopped" | |
| 166 | + else | |
| 167 | + log "not running" | |
| 168 | + fi | |
| 169 | +} | |
| 170 | + | |
| 171 | +status() { | |
| 172 | + if is_running; then | |
| 173 | + log "running (pid $(cat "$PID_FILE")), model $MODEL_PATH, port $OLALA_PORT" | |
| 174 | + curl -fsS "http://127.0.0.1:$OLALA_PORT/health" && echo | |
| 175 | + else | |
| 176 | + log "not running" | |
| 177 | + fi | |
| 178 | +} | |
| 179 | + | |
| 180 | +case "${1:-up}" in | |
| 181 | + setup) setup ;; | |
| 182 | + download) download ;; | |
| 183 | + serve) serve ;; | |
| 184 | + up) setup; download; serve ;; | |
| 185 | + stop) stop ;; | |
| 186 | + restart) stop; serve ;; | |
| 187 | + status) status ;; | |
| 188 | + logs) tail -f "$LOG_FILE" ;; | |
| 189 | + *) echo "usage: $0 {up|setup|download|serve|stop|restart|status|logs}" >&2; exit 2 ;; | |
| 190 | +esac | |
| new file mode 100755 | |||
| @@ -0,0 +1,190 @@ | |||
| 1 | +#!/usr/bin/env bash | ||
| 2 | +# Download a GGUF model and serve it with llama-server (OpenAI-compatible API), | ||
| 3 | +# using the OVHai llama.cpp fork (CPU only). Runs INSIDE the sandbox. | ||
| 4 | +# | ||
| 5 | +# Every step is idempotent: it is skipped when its result already exists. | ||
| 6 | +# | ||
| 7 | +# scripts/olala.sh setup # find llama-server (kit) or build the fork (skipped if found/built) | ||
| 8 | +# scripts/olala.sh download # download the model (skipped if the file exists) | ||
| 9 | +# scripts/olala.sh serve # start llama-server in the background (skipped if running) | ||
| 10 | +# scripts/olala.sh up # setup + download + serve | ||
| 11 | +# scripts/olala.sh stop | restart | status | logs | ||
| 12 | +# | ||
| 13 | +# Configuration (environment variables, defaults below): | ||
| 14 | +# OLALA_MODEL_REPO Hugging Face repo of the GGUF file | ||
| 15 | +# OLALA_MODEL_FILE GGUF file name in that repo | ||
| 16 | +# OLALA_MODELS_DIR where models are stored; relative paths are relative to the repo root | ||
| 17 | +# (default: ./models, kept in the workspace so dropping the sandbox keeps them) | ||
| 18 | +# OLALA_PORT port llama-server listens on (0.0.0.0) | ||
| 19 | +# OLALA_THREADS CPU threads (default: nproc - 2; using every CPU is ~30% slower) | ||
| 20 | +# OLALA_CTX context size | ||
| 21 | +# OLALA_PARALLEL parallel slots | ||
| 22 | +# OLALA_ALIAS model name exposed by the API (default: file name without .gguf) | ||
| 23 | +# OLALA_LOAD_MODE llama-server --load-mode (default: none = read the model into RAM; | ||
| 24 | +# mmap over the workspace mount is ~20x slower at generation) | ||
| 25 | +# OLALA_EXTRA_ARGS extra llama-server arguments | ||
| 26 | +# OLALA_SERVER_BIN llama-server to use (default: the one on PATH, e.g. from the | ||
| 27 | +# kits/llama-cpp-olala kit, else the one built in OLALA_SRC_DIR) | ||
| 28 | +# OLALA_SRC_DIR where the fork is cloned and built when no llama-server is found | ||
| 29 | +set -euo pipefail | ||
| 30 | + | ||
| 31 | +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" | ||
| 32 | + | ||
| 33 | +OLALA_MODEL_REPO="${OLALA_MODEL_REPO:-OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF}" | ||
| 34 | +OLALA_MODEL_FILE="${OLALA_MODEL_FILE:-Olala-v1.0-mini-7A1B-q8_0-safe2.gguf}" | ||
| 35 | +OLALA_MODELS_DIR="${OLALA_MODELS_DIR:-./models}" | ||
| 36 | +OLALA_PORT="${OLALA_PORT:-8080}" | ||
| 37 | +OLALA_THREADS="${OLALA_THREADS:-$(( $(nproc) > 2 ? $(nproc) - 2 : 1 ))}" | ||
| 38 | +OLALA_CTX="${OLALA_CTX:-32768}" | ||
| 39 | +OLALA_PARALLEL="${OLALA_PARALLEL:-4}" | ||
| 40 | +OLALA_ALIAS="${OLALA_ALIAS:-${OLALA_MODEL_FILE%.gguf}}" | ||
| 41 | +OLALA_LOAD_MODE="${OLALA_LOAD_MODE:-none}" | ||
| 42 | +OLALA_EXTRA_ARGS="${OLALA_EXTRA_ARGS:-}" | ||
| 43 | +OLALA_SRC_DIR="${OLALA_SRC_DIR:-$HOME/olala/llama.cpp-olala}" | ||
| 44 | +OLALA_REPO_URL="${OLALA_REPO_URL:-https://github.com/OVHai-LLM/llama.cpp.git}" | ||
| 45 | + | ||
| 46 | +case "$OLALA_MODELS_DIR" in | ||
| 47 | + /*) ;; | ||
| 48 | + *) OLALA_MODELS_DIR="$REPO_ROOT/${OLALA_MODELS_DIR#./}" ;; | ||
| 49 | +esac | ||
| 50 | + | ||
| 51 | +RUN_DIR="$HOME/.olala" | ||
| 52 | +PID_FILE="$RUN_DIR/llama-server.pid" | ||
| 53 | +LOG_FILE="$RUN_DIR/llama-server.log" | ||
| 54 | +BUILT_BIN="$OLALA_SRC_DIR/build/bin/llama-server" | ||
| 55 | +MODEL_PATH="$OLALA_MODELS_DIR/$OLALA_MODEL_FILE" | ||
| 56 | + | ||
| 57 | +log() { printf '[olala] %s\n' "$*" >&2; } | ||
| 58 | + | ||
| 59 | +# Prints the llama-server to use, or nothing. | ||
| 60 | +server_bin() { | ||
| 61 | + if [ -n "${OLALA_SERVER_BIN:-}" ]; then | ||
| 62 | + echo "$OLALA_SERVER_BIN" | ||
| 63 | + elif command -v llama-server >/dev/null; then | ||
| 64 | + command -v llama-server | ||
| 65 | + elif [ -x "$BUILT_BIN" ]; then | ||
| 66 | + echo "$BUILT_BIN" | ||
| 67 | + fi | ||
| 68 | +} | ||
| 69 | + | ||
| 70 | +setup() { | ||
| 71 | + local bin | ||
| 72 | + bin="$(server_bin)" | ||
| 73 | + if [ -n "$bin" ]; then | ||
| 74 | + log "llama-server found: $bin" | ||
| 75 | + return | ||
| 76 | + fi | ||
| 77 | + log "no llama-server on PATH (kits/llama-cpp-olala not installed): building the fork" | ||
| 78 | + if ! command -v cmake >/dev/null || ! command -v g++ >/dev/null; then | ||
| 79 | + log "installing build tools (build-essential, cmake, libssl-dev)" | ||
| 80 | + sudo apt-get update -qq | ||
| 81 | + sudo apt-get install -y -qq build-essential cmake libssl-dev pkg-config git >/dev/null | ||
| 82 | + fi | ||
| 83 | + if [ ! -d "$OLALA_SRC_DIR/.git" ]; then | ||
| 84 | + log "cloning $OLALA_REPO_URL" | ||
| 85 | + mkdir -p "$(dirname "$OLALA_SRC_DIR")" | ||
| 86 | + git clone --depth 1 "$OLALA_REPO_URL" "$OLALA_SRC_DIR" | ||
| 87 | + fi | ||
| 88 | + log "building (CPU only)" | ||
| 89 | + # -DLLAMA_USE_PREBUILT_UI=OFF: the prebuilt web UI does not match the fork (per the model card). | ||
| 90 | + cmake -S "$OLALA_SRC_DIR" -B "$OLALA_SRC_DIR/build" -DCMAKE_BUILD_TYPE=Release -DLLAMA_USE_PREBUILT_UI=OFF >/dev/null | ||
| 91 | + cmake --build "$OLALA_SRC_DIR/build" -j "$(nproc)" --target llama-server llama-cli >/dev/null | ||
| 92 | + log "built: $BUILT_BIN" | ||
| 93 | +} | ||
| 94 | + | ||
| 95 | +download() { | ||
| 96 | + if [ -s "$MODEL_PATH" ]; then | ||
| 97 | + log "model already present: $MODEL_PATH" | ||
| 98 | + return | ||
| 99 | + fi | ||
| 100 | + if ! command -v hf >/dev/null; then | ||
| 101 | + log "installing the Hugging Face CLI" | ||
| 102 | + uv tool install "huggingface_hub[cli]" >/dev/null | ||
| 103 | + export PATH="$HOME/.local/bin:$PATH" | ||
| 104 | + fi | ||
| 105 | + mkdir -p "$OLALA_MODELS_DIR" | ||
| 106 | + log "downloading $OLALA_MODEL_REPO / $OLALA_MODEL_FILE -> $OLALA_MODELS_DIR" | ||
| 107 | + hf download "$OLALA_MODEL_REPO" "$OLALA_MODEL_FILE" --local-dir "$OLALA_MODELS_DIR" | ||
| 108 | +} | ||
| 109 | + | ||
| 110 | +is_running() { | ||
| 111 | + [ -f "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null | ||
| 112 | +} | ||
| 113 | + | ||
| 114 | +serve() { | ||
| 115 | + if is_running; then | ||
| 116 | + log "llama-server already running (pid $(cat "$PID_FILE")) on port $OLALA_PORT" | ||
| 117 | + return | ||
| 118 | + fi | ||
| 119 | + local bin | ||
| 120 | + bin="$(server_bin)" | ||
| 121 | + [ -n "$bin" ] || { log "llama-server not found: run '$0 setup'"; exit 1; } | ||
| 122 | + [ -s "$MODEL_PATH" ] || { log "model missing: run '$0 download'"; exit 1; } | ||
| 123 | + mkdir -p "$RUN_DIR" | ||
| 124 | + log "starting $bin: $MODEL_PATH on 0.0.0.0:$OLALA_PORT (threads=$OLALA_THREADS ctx=$OLALA_CTX load-mode=$OLALA_LOAD_MODE)" | ||
| 125 | + # Flags from the model card's "CPU" server example; --host 0.0.0.0 so the port can be published. | ||
| 126 | + # The inner shell writes its own pid then execs llama-server, so the pid file is exact | ||
| 127 | + # (setsid may fork, which made $! point at a process that had already exited). | ||
| 128 | + # shellcheck disable=SC2016,SC2086 | ||
| 129 | + nohup setsid bash -c 'echo $$ >"$0"; exec "$@"' "$PID_FILE" \ | ||
| 130 | + "$bin" \ | ||
| 131 | + -m "$MODEL_PATH" \ | ||
| 132 | + --load-mode "$OLALA_LOAD_MODE" \ | ||
| 133 | + -t "$OLALA_THREADS" -fa 1 -nr -ctk q8_0 -ctv q8_0 \ | ||
| 134 | + -c "$OLALA_CTX" -np "$OLALA_PARALLEL" \ | ||
| 135 | + --temp 0.8 --top-p 0.9 --top-k 40 \ | ||
| 136 | + --alias "$OLALA_ALIAS" \ | ||
| 137 | + --host 0.0.0.0 --port "$OLALA_PORT" \ | ||
| 138 | + $OLALA_EXTRA_ARGS \ | ||
| 139 | + >"$LOG_FILE" 2>&1 </dev/null & | ||
| 140 | + log "waiting for /health (log: $LOG_FILE)" | ||
| 141 | + for _ in $(seq 1 300); do | ||
| 142 | + if curl -fsS "http://127.0.0.1:$OLALA_PORT/health" >/dev/null 2>&1; then | ||
| 143 | + log "ready: http://127.0.0.1:$OLALA_PORT/v1" | ||
| 144 | + return | ||
| 145 | + fi | ||
| 146 | + if [ -f "$PID_FILE" ] && ! is_running; then | ||
| 147 | + log "llama-server exited, last lines:" | ||
| 148 | + tail -20 "$LOG_FILE" >&2 | ||
| 149 | + exit 1 | ||
| 150 | + fi | ||
| 151 | + sleep 1 | ||
| 152 | + done | ||
| 153 | + log "not ready after 300 s, see $LOG_FILE" | ||
| 154 | + exit 1 | ||
| 155 | +} | ||
| 156 | + | ||
| 157 | +stop() { | ||
| 158 | + if is_running; then | ||
| 159 | + local pid | ||
| 160 | + pid="$(cat "$PID_FILE")" | ||
| 161 | + kill "$pid" | ||
| 162 | + for _ in $(seq 1 30); do kill -0 "$pid" 2>/dev/null || break; sleep 0.5; done | ||
| 163 | + kill -0 "$pid" 2>/dev/null && kill -9 "$pid" | ||
| 164 | + rm -f "$PID_FILE" | ||
| 165 | + log "stopped" | ||
| 166 | + else | ||
| 167 | + log "not running" | ||
| 168 | + fi | ||
| 169 | +} | ||
| 170 | + | ||
| 171 | +status() { | ||
| 172 | + if is_running; then | ||
| 173 | + log "running (pid $(cat "$PID_FILE")), model $MODEL_PATH, port $OLALA_PORT" | ||
| 174 | + curl -fsS "http://127.0.0.1:$OLALA_PORT/health" && echo | ||
| 175 | + else | ||
| 176 | + log "not running" | ||
| 177 | + fi | ||
| 178 | +} | ||
| 179 | + | ||
| 180 | +case "${1:-up}" in | ||
| 181 | + setup) setup ;; | ||
| 182 | + download) download ;; | ||
| 183 | + serve) serve ;; | ||
| 184 | + up) setup; download; serve ;; | ||
| 185 | + stop) stop ;; | ||
| 186 | + restart) stop; serve ;; | ||
| 187 | + status) status ;; | ||
| 188 | + logs) tail -f "$LOG_FILE" ;; | ||
| 189 | + *) echo "usage: $0 {up|setup|download|serve|stop|restart|status|logs}" >&2; exit 2 ;; | ||
| 190 | +esac | ||