modified
.tickets/issues/0092-how-to-unload-a-model.yaml +2 -2 | @@ -1,12 +1,12 @@ | ||
| 1 | 1 | # Managed by IssueSpec. Hand edits are welcome; keep the schema valid. |
| 2 | 2 | id: 92 |
| 3 | 3 | title: How to unload a model |
| 4 | -state: open | |
| 4 | +state: closed | |
| 5 | 5 | author: |
| 6 | 6 | name: k33g |
| 7 | 7 | email: ph.charriere@gmail.com |
| 8 | 8 | createdAt: 2026-10-17T14:24:18.694Z |
| 9 | -updatedAt: 2026-10-18T16:00:00.000Z | |
| 9 | +updatedAt: 2026-10-18T19:30:22.673Z | |
| 10 | 10 | labels: |
| 11 | 11 | - priority::urgent |
| 12 | 12 | milestone: v0.7.0 |
| @@ -1,12 +1,12 @@ | |||
| 1 | # Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | 1 | # Managed by IssueSpec. Hand edits are welcome; keep the schema valid. |
| 2 | id: 92 | 2 | id: 92 |
| 3 | title: How to unload a model | 3 | title: How to unload a model |
| 4 | -state: open | 4 | +state: closed |
| 5 | author: | 5 | author: |
| 6 | name: k33g | 6 | name: k33g |
| 7 | email: ph.charriere@gmail.com | 7 | email: ph.charriere@gmail.com |
| 8 | createdAt: 2026-10-17T14:24:18.694Z | 8 | createdAt: 2026-10-17T14:24:18.694Z |
| 9 | -updatedAt: 2026-10-18T16:00:00.000Z | 9 | +updatedAt: 2026-10-18T19:30:22.673Z |
| 10 | labels: | 10 | labels: |
| 11 | - priority::urgent | 11 | - priority::urgent |
| 12 | milestone: v0.7.0 | 12 | milestone: v0.7.0 |
deleted
.tickets/issues/0101-unload-a-model-command-and-tool.yaml +0 -116 | deleted file mode 100644 | ||
| @@ -1,116 +0,0 @@ | ||
| 1 | -# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | -id: 101 | |
| 3 | -title: Unload a model from memory — /unload command and unload_model tool | |
| 4 | -state: open | |
| 5 | -author: | |
| 6 | - name: k33g | |
| 7 | - email: ph.charriere@gmail.com | |
| 8 | -createdAt: 2026-10-18T16:30:00.000Z | |
| 9 | -updatedAt: 2026-10-18T16:30:00.000Z | |
| 10 | -labels: | |
| 11 | - - kind::feature | |
| 12 | -milestone: v0.7.0 | |
| 13 | -body: | | |
| 14 | - Follows ticket 92 (research: how each server unloads a model). Give mm the | |
| 15 | - capacity to unload a model from the server's memory, for the llmman, ollama | |
| 16 | - and llamacpp providers, in three forms: | |
| 17 | - | |
| 18 | - 1. a slash command `/unload [model]`, in the line mode, the window (-tui) | |
| 19 | - AND over ACP (an editor's slash-command menu); | |
| 20 | - 2. a built-in tool `unload_model`, like `ask_agent`, that the model calls | |
| 21 | - itself when its instructions say so — e.g. "unload your model before you | |
| 22 | - hand the task to another agent", or before handing control back; | |
| 23 | - 3. a demo on llmman in `demos/unload-models/`. | |
| 24 | - | |
| 25 | - The capacity can be turned off in the configuration file. | |
| 26 | - | |
| 27 | - Proposal (to confirm before coding): | |
| 28 | - - Key `unloadModels` (bool, default true). `false`: no `/unload` command | |
| 29 | - (not listed, not advertised over ACP) and no `unload_model` tool. | |
| 30 | - - `/unload` alone unloads THIS agent's model; `/unload <model>` another one | |
| 31 | - served by the same server. The next question reloads the model, as usual. | |
| 32 | - - Tool `unload_model {model?: string}`: empty = this agent's model. The | |
| 33 | - output says what happened, as text (unloaded / was not loaded / not | |
| 34 | - supported by this server, and why), never a Go error. | |
| 35 | - - Per provider (ticket 92 comment 1): | |
| 36 | - - llmman, ollama: POST <root>/api/generate {"model": m, "keep_alive": 0} | |
| 37 | - (root = baseUrl without /v1); success = done_reason "unload". | |
| 38 | - - llamacpp: POST <root>/models/unload {"model": m} — router mode only; a | |
| 39 | - single-model llama-server has no such route: say so, and point to | |
| 40 | - --sleep-idle-seconds. | |
| 41 | - - dmr: out of scope for now — "not supported by this provider". | |
| 42 | - - Permission: `unload_model` changes nothing on disk; it is gated like | |
| 43 | - `ask_agent` (asked when `yolo: false`)? To decide. | |
| 44 | - - Hooks: the tool goes through preToolCall/postToolCall like every tool. | |
| 45 | - | |
| 46 | - Open questions: | |
| 47 | - - Default on or off? (proposal: on, since unloading is harmless — the next | |
| 48 | - request reloads.) | |
| 49 | - - Should the tool be allowed to unload a model other than its own, or only | |
| 50 | - its own (safer for a small model that might unload the wrong one)? | |
| 51 | - - Permission with `yolo: false`: ask, or never ask? | |
| 52 | -tasks: | |
| 53 | - - id: 1 | |
| 54 | - title: "Agree on the key name and default, the tool's scope (own model only or any), and its permission" | |
| 55 | - state: open | |
| 56 | - priority: none | |
| 57 | - author: | |
| 58 | - name: k33g | |
| 59 | - email: ph.charriere@gmail.com | |
| 60 | - createdAt: 2026-10-18T16:30:00.000Z | |
| 61 | - updatedAt: 2026-10-18T16:30:00.000Z | |
| 62 | - - id: 2 | |
| 63 | - title: "engine: Unload(ctx, model) per provider — llmman/ollama /api/generate keep_alive 0, llamacpp /models/unload, others not supported — with tests on a fake server" | |
| 64 | - state: open | |
| 65 | - priority: none | |
| 66 | - author: | |
| 67 | - name: k33g | |
| 68 | - email: ph.charriere@gmail.com | |
| 69 | - createdAt: 2026-10-18T16:30:00.000Z | |
| 70 | - updatedAt: 2026-10-18T16:30:00.000Z | |
| 71 | - - id: 3 | |
| 72 | - title: "config key (unloadModels), validation and banner" | |
| 73 | - state: open | |
| 74 | - priority: none | |
| 75 | - author: | |
| 76 | - name: k33g | |
| 77 | - email: ph.charriere@gmail.com | |
| 78 | - createdAt: 2026-10-18T16:30:00.000Z | |
| 79 | - updatedAt: 2026-10-18T16:30:00.000Z | |
| 80 | - - id: 4 | |
| 81 | - title: "/unload [model] in the command table for Terminal | Window | Editor, handled by the three front ends" | |
| 82 | - state: open | |
| 83 | - priority: none | |
| 84 | - author: | |
| 85 | - name: k33g | |
| 86 | - email: ph.charriere@gmail.com | |
| 87 | - createdAt: 2026-10-18T16:30:00.000Z | |
| 88 | - updatedAt: 2026-10-18T16:30:00.000Z | |
| 89 | - - id: 5 | |
| 90 | - title: "built-in tool unload_model (reserved name, declared unless unloadModels is false)" | |
| 91 | - state: open | |
| 92 | - priority: none | |
| 93 | - author: | |
| 94 | - name: k33g | |
| 95 | - email: ph.charriere@gmail.com | |
| 96 | - createdAt: 2026-10-18T16:30:00.000Z | |
| 97 | - updatedAt: 2026-10-18T16:30:00.000Z | |
| 98 | - - id: 6 | |
| 99 | - title: "demos/unload-models/ on llmman: a coordinator that unloads its model before handing a task to another agent (different model), READMEs EN+FR, measured" | |
| 100 | - state: open | |
| 101 | - priority: none | |
| 102 | - author: | |
| 103 | - name: k33g | |
| 104 | - email: ph.charriere@gmail.com | |
| 105 | - createdAt: 2026-10-18T16:30:00.000Z | |
| 106 | - updatedAt: 2026-10-18T16:30:00.000Z | |
| 107 | - - id: 7 | |
| 108 | - title: "Docs EN+FR (reference: configuration, tools, slash-commands, acp; how-to), quality gate, .memory" | |
| 109 | - state: open | |
| 110 | - priority: none | |
| 111 | - author: | |
| 112 | - name: k33g | |
| 113 | - email: ph.charriere@gmail.com | |
| 114 | - createdAt: 2026-10-18T16:30:00.000Z | |
| 115 | - updatedAt: 2026-10-18T16:30:00.000Z | |
| 116 | -comments: [] | |
| deleted file mode 100644 | |||
| @@ -1,116 +0,0 @@ | |||
| 1 | -# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | -id: 101 | ||
| 3 | -title: Unload a model from memory — /unload command and unload_model tool | ||
| 4 | -state: open | ||
| 5 | -author: | ||
| 6 | - name: k33g | ||
| 7 | - email: ph.charriere@gmail.com | ||
| 8 | -createdAt: 2026-10-18T16:30:00.000Z | ||
| 9 | -updatedAt: 2026-10-18T16:30:00.000Z | ||
| 10 | -labels: | ||
| 11 | - - kind::feature | ||
| 12 | -milestone: v0.7.0 | ||
| 13 | -body: | | ||
| 14 | - Follows ticket 92 (research: how each server unloads a model). Give mm the | ||
| 15 | - capacity to unload a model from the server's memory, for the llmman, ollama | ||
| 16 | - and llamacpp providers, in three forms: | ||
| 17 | - | ||
| 18 | - 1. a slash command `/unload [model]`, in the line mode, the window (-tui) | ||
| 19 | - AND over ACP (an editor's slash-command menu); | ||
| 20 | - 2. a built-in tool `unload_model`, like `ask_agent`, that the model calls | ||
| 21 | - itself when its instructions say so — e.g. "unload your model before you | ||
| 22 | - hand the task to another agent", or before handing control back; | ||
| 23 | - 3. a demo on llmman in `demos/unload-models/`. | ||
| 24 | - | ||
| 25 | - The capacity can be turned off in the configuration file. | ||
| 26 | - | ||
| 27 | - Proposal (to confirm before coding): | ||
| 28 | - - Key `unloadModels` (bool, default true). `false`: no `/unload` command | ||
| 29 | - (not listed, not advertised over ACP) and no `unload_model` tool. | ||
| 30 | - - `/unload` alone unloads THIS agent's model; `/unload <model>` another one | ||
| 31 | - served by the same server. The next question reloads the model, as usual. | ||
| 32 | - - Tool `unload_model {model?: string}`: empty = this agent's model. The | ||
| 33 | - output says what happened, as text (unloaded / was not loaded / not | ||
| 34 | - supported by this server, and why), never a Go error. | ||
| 35 | - - Per provider (ticket 92 comment 1): | ||
| 36 | - - llmman, ollama: POST <root>/api/generate {"model": m, "keep_alive": 0} | ||
| 37 | - (root = baseUrl without /v1); success = done_reason "unload". | ||
| 38 | - - llamacpp: POST <root>/models/unload {"model": m} — router mode only; a | ||
| 39 | - single-model llama-server has no such route: say so, and point to | ||
| 40 | - --sleep-idle-seconds. | ||
| 41 | - - dmr: out of scope for now — "not supported by this provider". | ||
| 42 | - - Permission: `unload_model` changes nothing on disk; it is gated like | ||
| 43 | - `ask_agent` (asked when `yolo: false`)? To decide. | ||
| 44 | - - Hooks: the tool goes through preToolCall/postToolCall like every tool. | ||
| 45 | - | ||
| 46 | - Open questions: | ||
| 47 | - - Default on or off? (proposal: on, since unloading is harmless — the next | ||
| 48 | - request reloads.) | ||
| 49 | - - Should the tool be allowed to unload a model other than its own, or only | ||
| 50 | - its own (safer for a small model that might unload the wrong one)? | ||
| 51 | - - Permission with `yolo: false`: ask, or never ask? | ||
| 52 | -tasks: | ||
| 53 | - - id: 1 | ||
| 54 | - title: "Agree on the key name and default, the tool's scope (own model only or any), and its permission" | ||
| 55 | - state: open | ||
| 56 | - priority: none | ||
| 57 | - author: | ||
| 58 | - name: k33g | ||
| 59 | - email: ph.charriere@gmail.com | ||
| 60 | - createdAt: 2026-10-18T16:30:00.000Z | ||
| 61 | - updatedAt: 2026-10-18T16:30:00.000Z | ||
| 62 | - - id: 2 | ||
| 63 | - title: "engine: Unload(ctx, model) per provider — llmman/ollama /api/generate keep_alive 0, llamacpp /models/unload, others not supported — with tests on a fake server" | ||
| 64 | - state: open | ||
| 65 | - priority: none | ||
| 66 | - author: | ||
| 67 | - name: k33g | ||
| 68 | - email: ph.charriere@gmail.com | ||
| 69 | - createdAt: 2026-10-18T16:30:00.000Z | ||
| 70 | - updatedAt: 2026-10-18T16:30:00.000Z | ||
| 71 | - - id: 3 | ||
| 72 | - title: "config key (unloadModels), validation and banner" | ||
| 73 | - state: open | ||
| 74 | - priority: none | ||
| 75 | - author: | ||
| 76 | - name: k33g | ||
| 77 | - email: ph.charriere@gmail.com | ||
| 78 | - createdAt: 2026-10-18T16:30:00.000Z | ||
| 79 | - updatedAt: 2026-10-18T16:30:00.000Z | ||
| 80 | - - id: 4 | ||
| 81 | - title: "/unload [model] in the command table for Terminal | Window | Editor, handled by the three front ends" | ||
| 82 | - state: open | ||
| 83 | - priority: none | ||
| 84 | - author: | ||
| 85 | - name: k33g | ||
| 86 | - email: ph.charriere@gmail.com | ||
| 87 | - createdAt: 2026-10-18T16:30:00.000Z | ||
| 88 | - updatedAt: 2026-10-18T16:30:00.000Z | ||
| 89 | - - id: 5 | ||
| 90 | - title: "built-in tool unload_model (reserved name, declared unless unloadModels is false)" | ||
| 91 | - state: open | ||
| 92 | - priority: none | ||
| 93 | - author: | ||
| 94 | - name: k33g | ||
| 95 | - email: ph.charriere@gmail.com | ||
| 96 | - createdAt: 2026-10-18T16:30:00.000Z | ||
| 97 | - updatedAt: 2026-10-18T16:30:00.000Z | ||
| 98 | - - id: 6 | ||
| 99 | - title: "demos/unload-models/ on llmman: a coordinator that unloads its model before handing a task to another agent (different model), READMEs EN+FR, measured" | ||
| 100 | - state: open | ||
| 101 | - priority: none | ||
| 102 | - author: | ||
| 103 | - name: k33g | ||
| 104 | - email: ph.charriere@gmail.com | ||
| 105 | - createdAt: 2026-10-18T16:30:00.000Z | ||
| 106 | - updatedAt: 2026-10-18T16:30:00.000Z | ||
| 107 | - - id: 7 | ||
| 108 | - title: "Docs EN+FR (reference: configuration, tools, slash-commands, acp; how-to), quality gate, .memory" | ||
| 109 | - state: open | ||
| 110 | - priority: none | ||
| 111 | - author: | ||
| 112 | - name: k33g | ||
| 113 | - email: ph.charriere@gmail.com | ||
| 114 | - createdAt: 2026-10-18T16:30:00.000Z | ||
| 115 | - updatedAt: 2026-10-18T16:30:00.000Z | ||
| 116 | -comments: [] | ||
added
.tickets/issues/0101-unload-a-model-from-memory-unload-command-and-un.yaml +197 -0 | new file mode 100644 | ||
| @@ -0,0 +1,197 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +id: 101 | |
| 3 | +title: Unload a model from memory — /unload command and unload_model tool | |
| 4 | +state: closed | |
| 5 | +author: | |
| 6 | + name: k33g | |
| 7 | + email: ph.charriere@gmail.com | |
| 8 | +createdAt: 2026-10-18T16:30:00.000Z | |
| 9 | +updatedAt: 2026-10-18T19:30:11.700Z | |
| 10 | +labels: | |
| 11 | + - kind::feature | |
| 12 | +milestone: v0.7.0 | |
| 13 | +related: | |
| 14 | + - 92 | |
| 15 | +body: | | |
| 16 | + Follows ticket 92 (research: how each server unloads a model). Give mm the | |
| 17 | + capacity to unload a model from the server's memory, for the llmman, ollama | |
| 18 | + and llamacpp providers, in three forms: | |
| 19 | + | |
| 20 | + 1. a slash command `/unload [model]`, in the line mode, the window (-tui) | |
| 21 | + AND over ACP (an editor's slash-command menu); | |
| 22 | + 2. a built-in tool `unload_model`, like `ask_agent`, that the model calls | |
| 23 | + itself when its instructions say so — e.g. "unload your model before you | |
| 24 | + hand the task to another agent", or before handing control back; | |
| 25 | + 3. a demo on llmman in `demos/unload-models/`. | |
| 26 | + | |
| 27 | + The capacity can be turned off in the configuration file. | |
| 28 | + | |
| 29 | + Proposal (to confirm before coding): | |
| 30 | + - Key `unloadModels` (bool, default true). `false`: no `/unload` command | |
| 31 | + (not listed, not advertised over ACP) and no `unload_model` tool. | |
| 32 | + - `/unload` alone unloads THIS agent's model; `/unload <model>` another one | |
| 33 | + served by the same server. The next question reloads the model, as usual. | |
| 34 | + - Tool `unload_model {model?: string}`: empty = this agent's model. The | |
| 35 | + output says what happened, as text (unloaded / was not loaded / not | |
| 36 | + supported by this server, and why), never a Go error. | |
| 37 | + - Per provider (ticket 92 comment 1): | |
| 38 | + - llmman, ollama: POST <root>/api/generate {"model": m, "keep_alive": 0} | |
| 39 | + (root = baseUrl without /v1); success = done_reason "unload". | |
| 40 | + - llamacpp: POST <root>/models/unload {"model": m} — router mode only; a | |
| 41 | + single-model llama-server has no such route: say so, and point to | |
| 42 | + --sleep-idle-seconds. | |
| 43 | + - dmr: out of scope for now — "not supported by this provider". | |
| 44 | + - Permission: `unload_model` changes nothing on disk; it is gated like | |
| 45 | + `ask_agent` (asked when `yolo: false`)? To decide. | |
| 46 | + - Hooks: the tool goes through preToolCall/postToolCall like every tool. | |
| 47 | + | |
| 48 | + Open questions: | |
| 49 | + - Default on or off? (proposal: on, since unloading is harmless — the next | |
| 50 | + request reloads.) | |
| 51 | + - Should the tool be allowed to unload a model other than its own, or only | |
| 52 | + its own (safer for a small model that might unload the wrong one)? | |
| 53 | + - Permission with `yolo: false`: ask, or never ask? | |
| 54 | +tasks: | |
| 55 | + - id: 1 | |
| 56 | + title: Agree on the key name and default, the tool's scope (own model only or any), and its permission | |
| 57 | + state: closed | |
| 58 | + priority: none | |
| 59 | + author: | |
| 60 | + name: k33g | |
| 61 | + email: ph.charriere@gmail.com | |
| 62 | + createdAt: 2026-10-18T16:30:00.000Z | |
| 63 | + updatedAt: 2026-10-18T16:30:00.000Z | |
| 64 | + - id: 2 | |
| 65 | + title: "engine: Unload(ctx, model) per provider — llmman/ollama /api/generate keep_alive 0, llamacpp /models/unload, others not supported — with tests on a fake server" | |
| 66 | + state: closed | |
| 67 | + priority: none | |
| 68 | + author: | |
| 69 | + name: k33g | |
| 70 | + email: ph.charriere@gmail.com | |
| 71 | + createdAt: 2026-10-18T16:30:00.000Z | |
| 72 | + updatedAt: 2026-10-18T16:30:00.000Z | |
| 73 | + - id: 3 | |
| 74 | + title: config key (unloadModels), validation and banner | |
| 75 | + state: closed | |
| 76 | + priority: none | |
| 77 | + author: | |
| 78 | + name: k33g | |
| 79 | + email: ph.charriere@gmail.com | |
| 80 | + createdAt: 2026-10-18T16:30:00.000Z | |
| 81 | + updatedAt: 2026-10-18T16:30:00.000Z | |
| 82 | + - id: 4 | |
| 83 | + title: /unload [model] in the command table for Terminal | Window | Editor, handled by the three front ends | |
| 84 | + state: closed | |
| 85 | + priority: none | |
| 86 | + author: | |
| 87 | + name: k33g | |
| 88 | + email: ph.charriere@gmail.com | |
| 89 | + createdAt: 2026-10-18T16:30:00.000Z | |
| 90 | + updatedAt: 2026-10-18T16:30:00.000Z | |
| 91 | + - id: 5 | |
| 92 | + title: built-in tool unload_model (reserved name, declared unless unloadModels is false) | |
| 93 | + state: closed | |
| 94 | + priority: none | |
| 95 | + author: | |
| 96 | + name: k33g | |
| 97 | + email: ph.charriere@gmail.com | |
| 98 | + createdAt: 2026-10-18T16:30:00.000Z | |
| 99 | + updatedAt: 2026-10-18T16:30:00.000Z | |
| 100 | + - id: 6 | |
| 101 | + title: "demos/unload-models/ on llmman: a coordinator that unloads its model before handing a task to another agent (different model), READMEs EN+FR, measured" | |
| 102 | + state: closed | |
| 103 | + priority: none | |
| 104 | + author: | |
| 105 | + name: k33g | |
| 106 | + email: ph.charriere@gmail.com | |
| 107 | + createdAt: 2026-10-18T16:30:00.000Z | |
| 108 | + updatedAt: 2026-10-18T16:30:00.000Z | |
| 109 | + - id: 7 | |
| 110 | + title: "Docs EN+FR (reference: configuration, tools, slash-commands, acp; how-to), quality gate, .memory" | |
| 111 | + state: open | |
| 112 | + priority: none | |
| 113 | + author: | |
| 114 | + name: k33g | |
| 115 | + email: ph.charriere@gmail.com | |
| 116 | + createdAt: 2026-10-18T16:30:00.000Z | |
| 117 | + updatedAt: 2026-10-18T16:30:00.000Z | |
| 118 | +comments: | |
| 119 | + - id: 1 | |
| 120 | + author: | |
| 121 | + name: k33g | |
| 122 | + email: ph.charriere@gmail.com | |
| 123 | + createdAt: 2026-10-18T16:45:00.000Z | |
| 124 | + body: | | |
| 125 | + Decided by the user: `unloadModels` ON by default; the tool may unload ANY | |
| 126 | + model (`model` empty = this agent's); never asks for permission, even with | |
| 127 | + `yolo: false` (like read_file: nothing changes on disk, the next request | |
| 128 | + reloads). Branch created by the user. | |
| 129 | + - id: 2 | |
| 130 | + author: | |
| 131 | + name: k33g | |
| 132 | + email: ph.charriere@gmail.com | |
| 133 | + createdAt: 2026-10-18T18:00:00.000Z | |
| 134 | + body: | | |
| 135 | + Done on feature/unload-model (tasks 2-6; task 7 open: quality gate not run, | |
| 136 | + qlty absent from this sandbox — gofmt/go vet clean, go test ./... green). | |
| 137 | + - engine/unload.go: Engine.Unload(ctx, model) → one line; per provider: | |
| 138 | + llmman/ollama = /api/ps lookup first (a keep_alive 0 generate on a model | |
| 139 | + that is NOT loaded would load it), then POST /api/generate keep_alive 0, | |
| 140 | + done_reason "unload"; llamacpp = POST /models/unload (router mode; a 404 | |
| 141 | + explains single-model mode and --sleep-idle-seconds); dmr = not supported. | |
| 142 | + - config unloadModels (default true, user's choice, confirmed after being | |
| 143 | + told every existing agent gets one more tool); reserved name unload_model. | |
| 144 | + - /unload [model]: command.SetUnload, Terminal|Window|Editor; REPL, window | |
| 145 | + (off the event loop) and ACP handlers. Tool unload_model (any model, empty | |
| 146 | + = own), never gated, not beside a structuredOutput. | |
| 147 | + - Finding: unloading the agent's OWN model through the tool lasts one step | |
| 148 | + (it needs its model to read the result) — said in the tool description; | |
| 149 | + the demo unloads the asked agent's model instead. | |
| 150 | + - Live: /unload over ACP on llmman (unloaded → "nothing to unload" → | |
| 151 | + unknown model); demos/guard, demos/guards and structured-output-pizza | |
| 152 | + still work with the default. demos/unload-models: MiniCPM5 2B coordinator | |
| 153 | + + Qwen3.5 9B expert, unload_model called 4/5 (after a prompt fix; first | |
| 154 | + prompt skipped it), expert gone from /api/ps each time it was called. | |
| 155 | + - Docs EN+FR: reference configuration / slash-commands / tools, how-to | |
| 156 | + unload-a-model, indexes, README, demo READMEs. | |
| 157 | + - id: 3 | |
| 158 | + author: | |
| 159 | + name: k33g | |
| 160 | + email: ph.charriere@gmail.com | |
| 161 | + createdAt: 2026-10-18T19:30:00.000Z | |
| 162 | + body: | | |
| 163 | + Follow-ups (user's requests): | |
| 164 | + - "unload the main model before the other agent starts": the model cannot | |
| 165 | + (it is needed until the ask_agent call and again to read the answer). | |
| 166 | + User chose a preAsk hook in the demo over an mmConfigs option: | |
| 167 | + demos/unload-models/hooks/unload-self.js (keep_alive 0 on the asking | |
| 168 | + agent's model, read from the hook input "model"). Measured with /api/ps | |
| 169 | + every 0.5 s on 3 questions: coordinator alone → nothing → expert ALONE → | |
| 170 | + both (coordinator reloads to read) → coordinator; hook 3/3, tool 3/3. | |
| 171 | + - "/unload not visible in -tui": the "/" picker drew only the first 8 | |
| 172 | + choices (pickerRows) and never scrolled; /unload is the 10th command. | |
| 173 | + Pre-existing bug (it also hid /ask-<name> commands). Fixed: the window | |
| 174 | + follows the selection (tui/draw.go pickerFirst, + test). | |
| 175 | + - "unload_model calls not visible over ACP": over the real protocol the | |
| 176 | + editor receives the tool_call ("unload_model: <model>", pending → | |
| 177 | + in_progress → completed) — checked with a scripted client on the demo. | |
| 178 | + Most likely the editor runs an older installed mm; not reproduced here. | |
| 179 | + - id: 4 | |
| 180 | + author: | |
| 181 | + name: k33g | |
| 182 | + email: ph.charriere@gmail.com | |
| 183 | + createdAt: 2026-10-18T21:30:00.000Z | |
| 184 | + body: | | |
| 185 | + Demo follow-ups (user's requests): | |
| 186 | + - Models: coordinator Qwen3.5 9B, expert Qwen3.5 4B (was Ornith 1.5 9B / | |
| 187 | + Qwen3.5 9B). With preAsk + unload_model: 7/7 clean, but both models | |
| 188 | + loaded together ~18 s per question (the coordinator reloads to read the | |
| 189 | + answer before it can call unload_model). | |
| 190 | + - "never two models loaded at once": hooks/unload-self.js became | |
| 191 | + hooks/unload.js self|asked; a postAsk hook (unload.js asked, model read | |
| 192 | + from ask.agentModel) unloads the expert before the coordinator reloads. | |
| 193 | + unload_model step removed from the prompt (kept for "free the expert's | |
| 194 | + memory" requests; expert model name in the prompt). Measured 9/9: | |
| 195 | + 9B → nothing → 4B → nothing → 9B; "unload the expert" 4/4. Prompt | |
| 196 | + needed "ask_agent is ALWAYS your first action" (short prompt let the | |
| 197 | + coordinator answer yield itself, 1/7). | |
| new file mode 100644 | |||
| @@ -0,0 +1,197 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +id: 101 | ||
| 3 | +title: Unload a model from memory — /unload command and unload_model tool | ||
| 4 | +state: closed | ||
| 5 | +author: | ||
| 6 | + name: k33g | ||
| 7 | + email: ph.charriere@gmail.com | ||
| 8 | +createdAt: 2026-10-18T16:30:00.000Z | ||
| 9 | +updatedAt: 2026-10-18T19:30:11.700Z | ||
| 10 | +labels: | ||
| 11 | + - kind::feature | ||
| 12 | +milestone: v0.7.0 | ||
| 13 | +related: | ||
| 14 | + - 92 | ||
| 15 | +body: | | ||
| 16 | + Follows ticket 92 (research: how each server unloads a model). Give mm the | ||
| 17 | + capacity to unload a model from the server's memory, for the llmman, ollama | ||
| 18 | + and llamacpp providers, in three forms: | ||
| 19 | + | ||
| 20 | + 1. a slash command `/unload [model]`, in the line mode, the window (-tui) | ||
| 21 | + AND over ACP (an editor's slash-command menu); | ||
| 22 | + 2. a built-in tool `unload_model`, like `ask_agent`, that the model calls | ||
| 23 | + itself when its instructions say so — e.g. "unload your model before you | ||
| 24 | + hand the task to another agent", or before handing control back; | ||
| 25 | + 3. a demo on llmman in `demos/unload-models/`. | ||
| 26 | + | ||
| 27 | + The capacity can be turned off in the configuration file. | ||
| 28 | + | ||
| 29 | + Proposal (to confirm before coding): | ||
| 30 | + - Key `unloadModels` (bool, default true). `false`: no `/unload` command | ||
| 31 | + (not listed, not advertised over ACP) and no `unload_model` tool. | ||
| 32 | + - `/unload` alone unloads THIS agent's model; `/unload <model>` another one | ||
| 33 | + served by the same server. The next question reloads the model, as usual. | ||
| 34 | + - Tool `unload_model {model?: string}`: empty = this agent's model. The | ||
| 35 | + output says what happened, as text (unloaded / was not loaded / not | ||
| 36 | + supported by this server, and why), never a Go error. | ||
| 37 | + - Per provider (ticket 92 comment 1): | ||
| 38 | + - llmman, ollama: POST <root>/api/generate {"model": m, "keep_alive": 0} | ||
| 39 | + (root = baseUrl without /v1); success = done_reason "unload". | ||
| 40 | + - llamacpp: POST <root>/models/unload {"model": m} — router mode only; a | ||
| 41 | + single-model llama-server has no such route: say so, and point to | ||
| 42 | + --sleep-idle-seconds. | ||
| 43 | + - dmr: out of scope for now — "not supported by this provider". | ||
| 44 | + - Permission: `unload_model` changes nothing on disk; it is gated like | ||
| 45 | + `ask_agent` (asked when `yolo: false`)? To decide. | ||
| 46 | + - Hooks: the tool goes through preToolCall/postToolCall like every tool. | ||
| 47 | + | ||
| 48 | + Open questions: | ||
| 49 | + - Default on or off? (proposal: on, since unloading is harmless — the next | ||
| 50 | + request reloads.) | ||
| 51 | + - Should the tool be allowed to unload a model other than its own, or only | ||
| 52 | + its own (safer for a small model that might unload the wrong one)? | ||
| 53 | + - Permission with `yolo: false`: ask, or never ask? | ||
| 54 | +tasks: | ||
| 55 | + - id: 1 | ||
| 56 | + title: Agree on the key name and default, the tool's scope (own model only or any), and its permission | ||
| 57 | + state: closed | ||
| 58 | + priority: none | ||
| 59 | + author: | ||
| 60 | + name: k33g | ||
| 61 | + email: ph.charriere@gmail.com | ||
| 62 | + createdAt: 2026-10-18T16:30:00.000Z | ||
| 63 | + updatedAt: 2026-10-18T16:30:00.000Z | ||
| 64 | + - id: 2 | ||
| 65 | + title: "engine: Unload(ctx, model) per provider — llmman/ollama /api/generate keep_alive 0, llamacpp /models/unload, others not supported — with tests on a fake server" | ||
| 66 | + state: closed | ||
| 67 | + priority: none | ||
| 68 | + author: | ||
| 69 | + name: k33g | ||
| 70 | + email: ph.charriere@gmail.com | ||
| 71 | + createdAt: 2026-10-18T16:30:00.000Z | ||
| 72 | + updatedAt: 2026-10-18T16:30:00.000Z | ||
| 73 | + - id: 3 | ||
| 74 | + title: config key (unloadModels), validation and banner | ||
| 75 | + state: closed | ||
| 76 | + priority: none | ||
| 77 | + author: | ||
| 78 | + name: k33g | ||
| 79 | + email: ph.charriere@gmail.com | ||
| 80 | + createdAt: 2026-10-18T16:30:00.000Z | ||
| 81 | + updatedAt: 2026-10-18T16:30:00.000Z | ||
| 82 | + - id: 4 | ||
| 83 | + title: /unload [model] in the command table for Terminal | Window | Editor, handled by the three front ends | ||
| 84 | + state: closed | ||
| 85 | + priority: none | ||
| 86 | + author: | ||
| 87 | + name: k33g | ||
| 88 | + email: ph.charriere@gmail.com | ||
| 89 | + createdAt: 2026-10-18T16:30:00.000Z | ||
| 90 | + updatedAt: 2026-10-18T16:30:00.000Z | ||
| 91 | + - id: 5 | ||
| 92 | + title: built-in tool unload_model (reserved name, declared unless unloadModels is false) | ||
| 93 | + state: closed | ||
| 94 | + priority: none | ||
| 95 | + author: | ||
| 96 | + name: k33g | ||
| 97 | + email: ph.charriere@gmail.com | ||
| 98 | + createdAt: 2026-10-18T16:30:00.000Z | ||
| 99 | + updatedAt: 2026-10-18T16:30:00.000Z | ||
| 100 | + - id: 6 | ||
| 101 | + title: "demos/unload-models/ on llmman: a coordinator that unloads its model before handing a task to another agent (different model), READMEs EN+FR, measured" | ||
| 102 | + state: closed | ||
| 103 | + priority: none | ||
| 104 | + author: | ||
| 105 | + name: k33g | ||
| 106 | + email: ph.charriere@gmail.com | ||
| 107 | + createdAt: 2026-10-18T16:30:00.000Z | ||
| 108 | + updatedAt: 2026-10-18T16:30:00.000Z | ||
| 109 | + - id: 7 | ||
| 110 | + title: "Docs EN+FR (reference: configuration, tools, slash-commands, acp; how-to), quality gate, .memory" | ||
| 111 | + state: open | ||
| 112 | + priority: none | ||
| 113 | + author: | ||
| 114 | + name: k33g | ||
| 115 | + email: ph.charriere@gmail.com | ||
| 116 | + createdAt: 2026-10-18T16:30:00.000Z | ||
| 117 | + updatedAt: 2026-10-18T16:30:00.000Z | ||
| 118 | +comments: | ||
| 119 | + - id: 1 | ||
| 120 | + author: | ||
| 121 | + name: k33g | ||
| 122 | + email: ph.charriere@gmail.com | ||
| 123 | + createdAt: 2026-10-18T16:45:00.000Z | ||
| 124 | + body: | | ||
| 125 | + Decided by the user: `unloadModels` ON by default; the tool may unload ANY | ||
| 126 | + model (`model` empty = this agent's); never asks for permission, even with | ||
| 127 | + `yolo: false` (like read_file: nothing changes on disk, the next request | ||
| 128 | + reloads). Branch created by the user. | ||
| 129 | + - id: 2 | ||
| 130 | + author: | ||
| 131 | + name: k33g | ||
| 132 | + email: ph.charriere@gmail.com | ||
| 133 | + createdAt: 2026-10-18T18:00:00.000Z | ||
| 134 | + body: | | ||
| 135 | + Done on feature/unload-model (tasks 2-6; task 7 open: quality gate not run, | ||
| 136 | + qlty absent from this sandbox — gofmt/go vet clean, go test ./... green). | ||
| 137 | + - engine/unload.go: Engine.Unload(ctx, model) → one line; per provider: | ||
| 138 | + llmman/ollama = /api/ps lookup first (a keep_alive 0 generate on a model | ||
| 139 | + that is NOT loaded would load it), then POST /api/generate keep_alive 0, | ||
| 140 | + done_reason "unload"; llamacpp = POST /models/unload (router mode; a 404 | ||
| 141 | + explains single-model mode and --sleep-idle-seconds); dmr = not supported. | ||
| 142 | + - config unloadModels (default true, user's choice, confirmed after being | ||
| 143 | + told every existing agent gets one more tool); reserved name unload_model. | ||
| 144 | + - /unload [model]: command.SetUnload, Terminal|Window|Editor; REPL, window | ||
| 145 | + (off the event loop) and ACP handlers. Tool unload_model (any model, empty | ||
| 146 | + = own), never gated, not beside a structuredOutput. | ||
| 147 | + - Finding: unloading the agent's OWN model through the tool lasts one step | ||
| 148 | + (it needs its model to read the result) — said in the tool description; | ||
| 149 | + the demo unloads the asked agent's model instead. | ||
| 150 | + - Live: /unload over ACP on llmman (unloaded → "nothing to unload" → | ||
| 151 | + unknown model); demos/guard, demos/guards and structured-output-pizza | ||
| 152 | + still work with the default. demos/unload-models: MiniCPM5 2B coordinator | ||
| 153 | + + Qwen3.5 9B expert, unload_model called 4/5 (after a prompt fix; first | ||
| 154 | + prompt skipped it), expert gone from /api/ps each time it was called. | ||
| 155 | + - Docs EN+FR: reference configuration / slash-commands / tools, how-to | ||
| 156 | + unload-a-model, indexes, README, demo READMEs. | ||
| 157 | + - id: 3 | ||
| 158 | + author: | ||
| 159 | + name: k33g | ||
| 160 | + email: ph.charriere@gmail.com | ||
| 161 | + createdAt: 2026-10-18T19:30:00.000Z | ||
| 162 | + body: | | ||
| 163 | + Follow-ups (user's requests): | ||
| 164 | + - "unload the main model before the other agent starts": the model cannot | ||
| 165 | + (it is needed until the ask_agent call and again to read the answer). | ||
| 166 | + User chose a preAsk hook in the demo over an mmConfigs option: | ||
| 167 | + demos/unload-models/hooks/unload-self.js (keep_alive 0 on the asking | ||
| 168 | + agent's model, read from the hook input "model"). Measured with /api/ps | ||
| 169 | + every 0.5 s on 3 questions: coordinator alone → nothing → expert ALONE → | ||
| 170 | + both (coordinator reloads to read) → coordinator; hook 3/3, tool 3/3. | ||
| 171 | + - "/unload not visible in -tui": the "/" picker drew only the first 8 | ||
| 172 | + choices (pickerRows) and never scrolled; /unload is the 10th command. | ||
| 173 | + Pre-existing bug (it also hid /ask-<name> commands). Fixed: the window | ||
| 174 | + follows the selection (tui/draw.go pickerFirst, + test). | ||
| 175 | + - "unload_model calls not visible over ACP": over the real protocol the | ||
| 176 | + editor receives the tool_call ("unload_model: <model>", pending → | ||
| 177 | + in_progress → completed) — checked with a scripted client on the demo. | ||
| 178 | + Most likely the editor runs an older installed mm; not reproduced here. | ||
| 179 | + - id: 4 | ||
| 180 | + author: | ||
| 181 | + name: k33g | ||
| 182 | + email: ph.charriere@gmail.com | ||
| 183 | + createdAt: 2026-10-18T21:30:00.000Z | ||
| 184 | + body: | | ||
| 185 | + Demo follow-ups (user's requests): | ||
| 186 | + - Models: coordinator Qwen3.5 9B, expert Qwen3.5 4B (was Ornith 1.5 9B / | ||
| 187 | + Qwen3.5 9B). With preAsk + unload_model: 7/7 clean, but both models | ||
| 188 | + loaded together ~18 s per question (the coordinator reloads to read the | ||
| 189 | + answer before it can call unload_model). | ||
| 190 | + - "never two models loaded at once": hooks/unload-self.js became | ||
| 191 | + hooks/unload.js self|asked; a postAsk hook (unload.js asked, model read | ||
| 192 | + from ask.agentModel) unloads the expert before the coordinator reloads. | ||
| 193 | + unload_model step removed from the prompt (kept for "free the expert's | ||
| 194 | + memory" requests; expert model name in the prompt). Measured 9/9: | ||
| 195 | + 9B → nothing → 4B → nothing → 9B; "unload the expert" 4/4. Prompt | ||
| 196 | + needed "ask_agent is ALWAYS your first action" (short prompt let the | ||
| 197 | + coordinator answer yield itself, 1/7). | ||
modified
README.md +2 -0 | @@ -120,6 +120,7 @@ Anything that speaks ACP can host `mm -acp` the same way: **JetBrains IDEs** (In | ||
| 120 | 120 | - **Guard models**: have every question checked by guard models (IBM's Granite Guardian), one agent per risk, before your assistant answers — a `preGeneration` hook asks them without the conversation history, and the assistant refuses or answers according to their verdicts. See [Screen questions with guard models](docs/en/how-to/screen-questions-with-guard-models.md), `demos/guard/` and `demos/guards/`. |
| 121 | 121 | - **Plans**: the built-in `plan` tool lets the agent turn a list of tasks into a tracked plan and show its progress after each answer. See [Manage plans and tasks](docs/en/how-to/manage-plans-and-tasks.md). |
| 122 | 122 | - **Save an answer**: `/save` writes the last answer to a Markdown file; with `saveAnswerTool: true`, the model does it itself when you ask in your own words (`save_answer`). Never overwrites a file. See [Save an answer to a file](docs/en/how-to/save-an-answer.md). |
| 123 | +- **Unload a model**: `/unload [model]` frees a model from the server's memory — in the terminal, the window and an editor —, and the `unload_model` tool lets the agent do it itself, e.g. on another agent's model once it has answered; llmman, Ollama and llama-server in router mode. `unloadModels: false` turns both off. See [Free a model's memory](docs/en/how-to/unload-a-model.md) and `demos/unload-models/`. | |
| 123 | 124 | - **Docker Sandboxes**: the sandbox kit lives in its own repository, [mini-me-kit](https://rickub.com/bots-garden/mini-me-kit); `sbx/` explains how to use it (`sbx run …`), with two mixins (`go-toolchain`, `cli-tools`) and ready-made configurations — see `sbx/README.md` and [Run mini-me in a Docker Sandbox](docs/en/how-to/run-in-a-docker-sandbox.md). |
| 124 | 125 | |
| 125 | 126 | ## Demos |
| @@ -152,6 +153,7 @@ The `demos/` directory holds ready-made set-ups (most with a README explaining h | ||
| 152 | 153 | | `demos/programming-answers-2/` | **Tuto**: an agent that answers with mini tutorials in the conversation and saves one to Markdown only when asked (`fetch_content` + `save_answer`), Mellum2.1 12B Thinking on llmman, measured results | |
| 153 | 154 | | `demos/guard/` | one guard model (Granite Guardian 3.2 3B-A800M on llmman) answering `Yes`/`No` to "is this question harmful?"; why its `system:` line is a risk name, not a prompt; custom risks | |
| 154 | 155 | | `demos/guards/` | an assistant (Gemma 4 E4B) protected by seven guard agents, one per risk: a Node.js `preGeneration` hook asks them in parallel without the history, the assistant refuses or answers according to their verdicts; `displayHooks: true`; measured results | |
| 156 | +| `demos/unload-models/` | a coordinator (Qwen3.5 9B) hands programming questions to an expert (Qwen3.5 4B) with `ask_agent`, a `preAsk` hook unloads the coordinator's own model before the expert starts and a `postAsk` hook unloads the expert's once it has answered — the two models are never loaded together; `/unload` and `unload_model` on request; llmman, measured results | | |
| 155 | 157 | | `demos/declared-tools/` | an ops desk with no `bash` and no skills, acting only through four tools declared under `tools:` — release, version bump, service logs, slow requests — covering every parameter type (string, enum, boolean, integer, number), a safe default (`dryRun: true`) and an optional argument; Mellum2 12B on DMR, measured results | |
| 156 | 158 | |
| 157 | 159 | ## Learn more |
| @@ -120,6 +120,7 @@ Anything that speaks ACP can host `mm -acp` the same way: **JetBrains IDEs** (In | |||
| 120 | - **Guard models**: have every question checked by guard models (IBM's Granite Guardian), one agent per risk, before your assistant answers — a `preGeneration` hook asks them without the conversation history, and the assistant refuses or answers according to their verdicts. See [Screen questions with guard models](docs/en/how-to/screen-questions-with-guard-models.md), `demos/guard/` and `demos/guards/`. | 120 | - **Guard models**: have every question checked by guard models (IBM's Granite Guardian), one agent per risk, before your assistant answers — a `preGeneration` hook asks them without the conversation history, and the assistant refuses or answers according to their verdicts. See [Screen questions with guard models](docs/en/how-to/screen-questions-with-guard-models.md), `demos/guard/` and `demos/guards/`. |
| 121 | - **Plans**: the built-in `plan` tool lets the agent turn a list of tasks into a tracked plan and show its progress after each answer. See [Manage plans and tasks](docs/en/how-to/manage-plans-and-tasks.md). | 121 | - **Plans**: the built-in `plan` tool lets the agent turn a list of tasks into a tracked plan and show its progress after each answer. See [Manage plans and tasks](docs/en/how-to/manage-plans-and-tasks.md). |
| 122 | - **Save an answer**: `/save` writes the last answer to a Markdown file; with `saveAnswerTool: true`, the model does it itself when you ask in your own words (`save_answer`). Never overwrites a file. See [Save an answer to a file](docs/en/how-to/save-an-answer.md). | 122 | - **Save an answer**: `/save` writes the last answer to a Markdown file; with `saveAnswerTool: true`, the model does it itself when you ask in your own words (`save_answer`). Never overwrites a file. See [Save an answer to a file](docs/en/how-to/save-an-answer.md). |
| 123 | +- **Unload a model**: `/unload [model]` frees a model from the server's memory — in the terminal, the window and an editor —, and the `unload_model` tool lets the agent do it itself, e.g. on another agent's model once it has answered; llmman, Ollama and llama-server in router mode. `unloadModels: false` turns both off. See [Free a model's memory](docs/en/how-to/unload-a-model.md) and `demos/unload-models/`. | ||
| 123 | - **Docker Sandboxes**: the sandbox kit lives in its own repository, [mini-me-kit](https://rickub.com/bots-garden/mini-me-kit); `sbx/` explains how to use it (`sbx run …`), with two mixins (`go-toolchain`, `cli-tools`) and ready-made configurations — see `sbx/README.md` and [Run mini-me in a Docker Sandbox](docs/en/how-to/run-in-a-docker-sandbox.md). | 124 | - **Docker Sandboxes**: the sandbox kit lives in its own repository, [mini-me-kit](https://rickub.com/bots-garden/mini-me-kit); `sbx/` explains how to use it (`sbx run …`), with two mixins (`go-toolchain`, `cli-tools`) and ready-made configurations — see `sbx/README.md` and [Run mini-me in a Docker Sandbox](docs/en/how-to/run-in-a-docker-sandbox.md). |
| 124 | 125 | ||
| 125 | ## Demos | 126 | ## Demos |
| @@ -152,6 +153,7 @@ The `demos/` directory holds ready-made set-ups (most with a README explaining h | |||
| 152 | | `demos/programming-answers-2/` | **Tuto**: an agent that answers with mini tutorials in the conversation and saves one to Markdown only when asked (`fetch_content` + `save_answer`), Mellum2.1 12B Thinking on llmman, measured results | | 153 | | `demos/programming-answers-2/` | **Tuto**: an agent that answers with mini tutorials in the conversation and saves one to Markdown only when asked (`fetch_content` + `save_answer`), Mellum2.1 12B Thinking on llmman, measured results | |
| 153 | | `demos/guard/` | one guard model (Granite Guardian 3.2 3B-A800M on llmman) answering `Yes`/`No` to "is this question harmful?"; why its `system:` line is a risk name, not a prompt; custom risks | | 154 | | `demos/guard/` | one guard model (Granite Guardian 3.2 3B-A800M on llmman) answering `Yes`/`No` to "is this question harmful?"; why its `system:` line is a risk name, not a prompt; custom risks | |
| 154 | | `demos/guards/` | an assistant (Gemma 4 E4B) protected by seven guard agents, one per risk: a Node.js `preGeneration` hook asks them in parallel without the history, the assistant refuses or answers according to their verdicts; `displayHooks: true`; measured results | | 155 | | `demos/guards/` | an assistant (Gemma 4 E4B) protected by seven guard agents, one per risk: a Node.js `preGeneration` hook asks them in parallel without the history, the assistant refuses or answers according to their verdicts; `displayHooks: true`; measured results | |
| 156 | +| `demos/unload-models/` | a coordinator (Qwen3.5 9B) hands programming questions to an expert (Qwen3.5 4B) with `ask_agent`, a `preAsk` hook unloads the coordinator's own model before the expert starts and a `postAsk` hook unloads the expert's once it has answered — the two models are never loaded together; `/unload` and `unload_model` on request; llmman, measured results | | ||
| 155 | | `demos/declared-tools/` | an ops desk with no `bash` and no skills, acting only through four tools declared under `tools:` — release, version bump, service logs, slow requests — covering every parameter type (string, enum, boolean, integer, number), a safe default (`dryRun: true`) and an optional argument; Mellum2 12B on DMR, measured results | | 157 | | `demos/declared-tools/` | an ops desk with no `bash` and no skills, acting only through four tools declared under `tools:` — release, version bump, service logs, slow requests — covering every parameter type (string, enum, boolean, integer, number), a safe default (`dryRun: true`) and an optional argument; Mellum2 12B on DMR, measured results | |
| 156 | 158 | ||
| 157 | ## Learn more | 159 | ## Learn more |
modified
demos/guards/README.en.md +4 -0 | @@ -30,6 +30,10 @@ The hook runs the guards with the same `mm` as the assistant: mini-me exports it | ||
| 30 | 30 | |
| 31 | 31 | ## How it works |
| 32 | 32 | |
| 33 | + | |
| 34 | + | |
| 35 | +Editable source: `guards-flow.drawio` (draw.io); `guards-flow.svg` is the same picture. | |
| 36 | + | |
| 33 | 37 | ```text |
| 34 | 38 | user's question |
| 35 | 39 | │ |
| @@ -30,6 +30,10 @@ The hook runs the guards with the same `mm` as the assistant: mini-me exports it | |||
| 30 | 30 | ||
| 31 | ## How it works | 31 | ## How it works |
| 32 | 32 | ||
| 33 | + | ||
| 34 | + | ||
| 35 | +Editable source: `guards-flow.drawio` (draw.io); `guards-flow.svg` is the same picture. | ||
| 36 | + | ||
| 33 | ```text | 37 | ```text |
| 34 | user's question | 38 | user's question |
| 35 | │ | 39 | │ |
modified
demos/guards/README.fr.md +4 -0 | @@ -30,6 +30,10 @@ Le hook lance les gardes avec le même `mm` que l'assistant : mini-me exporte so | ||
| 30 | 30 | |
| 31 | 31 | ## Comment ça marche |
| 32 | 32 | |
| 33 | + | |
| 34 | + | |
| 35 | +Source modifiable : `guards-flow.drawio` (draw.io) ; `guards-flow.svg` est la même image. | |
| 36 | + | |
| 33 | 37 | ```text |
| 34 | 38 | question de l'utilisateur |
| 35 | 39 | │ |
| @@ -30,6 +30,10 @@ Le hook lance les gardes avec le même `mm` que l'assistant : mini-me exporte so | |||
| 30 | 30 | ||
| 31 | ## Comment ça marche | 31 | ## Comment ça marche |
| 32 | 32 | ||
| 33 | + | ||
| 34 | + | ||
| 35 | +Source modifiable : `guards-flow.drawio` (draw.io) ; `guards-flow.svg` est la même image. | ||
| 36 | + | ||
| 33 | ```text | 37 | ```text |
| 34 | question de l'utilisateur | 38 | question de l'utilisateur |
| 35 | │ | 39 | │ |
added
demos/guards/guards-flow.drawio +30 -0 | new file mode 100644 | ||
| @@ -0,0 +1,30 @@ | ||
| 1 | +<mxfile host="app.diagrams.net" agent="drawio-fancy-diagram-skill" version="24.7.17" type="device"> | |
| 2 | + <diagram name="demos/guards" id="guards-flow"> | |
| 3 | + <mxGraphModel dx="1400" dy="800" grid="0" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1200" pageHeight="680" math="0" shadow="0"> | |
| 4 | + <root> | |
| 5 | + <mxCell id="0"/> | |
| 6 | + <mxCell id="1" parent="0"/> | |
| 7 | + <mxCell id="bg" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#FFFFFF;gradientColor=#EEF2FA;strokeColor=none;" vertex="1" parent="1"><mxGeometry x="0" y="0" width="1200" height="680" as="geometry"/></mxCell> | |
| 8 | + <mxCell id="title" value="demos/guards — every question screened first" style="text;html=1;align=center;verticalAlign=middle;fontSize=24;fontStyle=1;fontColor=#1A237E;" vertex="1" parent="1"><mxGeometry x="200" y="18" width="800" height="30" as="geometry"/></mxCell> | |
| 9 | + <mxCell id="subtitle" value="a preGeneration hook asks seven guard agents, the question alone, before the assistant answers" style="text;html=1;align=center;verticalAlign=middle;fontSize=13;fontStyle=2;fontColor=#546E7A;" vertex="1" parent="1"><mxGeometry x="150" y="48" width="900" height="20" as="geometry"/></mxCell> | |
| 10 | + <mxCell id="user" value="🧑<br><b>User</b><br><font style="font-size:11px">asks a question<br>terminal · -tui · editor</font>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#E3F2FD;gradientColor=#BBDEFB;strokeColor=#1565C0;strokeWidth=2;shadow=1;fontSize=15;fontColor=#0D47A1;" vertex="1" parent="1"><mxGeometry x="40" y="140" width="190" height="120" as="geometry"/></mxCell> | |
| 11 | + <mxCell id="hook" value="🔍<br><b>preGeneration hook</b><br><font style="font-size:11px">hooks/guards.js<br>no verdict → stop (fail closed)</font>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#E1F5FE;gradientColor=#B3E5FC;strokeColor=#0277BD;strokeWidth=2;shadow=1;fontSize=15;fontColor=#01579B;" vertex="1" parent="1"><mxGeometry x="330" y="140" width="250" height="120" as="geometry"/></mxCell> | |
| 12 | + <mxCell id="assistant" value="🤖<br><b>Assistant</b><br><font style="font-size:11px">Gemma 4 E4B · no tools<br>all No → answers · a Yes → refuses</font>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#F3E5F5;gradientColor=#E1BEE7;strokeColor=#6A1B9A;strokeWidth=3;shadow=1;fontSize=15;fontColor=#4A148C;" vertex="1" parent="1"><mxGeometry x="900" y="140" width="260" height="120" as="geometry"/></mxCell> | |
| 13 | + <mxCell id="badge" value="★ decides" style="rounded=1;arcSize=50;whiteSpace=wrap;html=1;fillColor=#6A1B9A;strokeColor=none;fontSize=11;fontStyle=1;fontColor=#FFFFFF;" vertex="1" parent="1"><mxGeometry x="1060" y="128" width="88" height="24" as="geometry"/></mxCell> | |
| 14 | + <mxCell id="guards" value="7 guard agents · Granite Guardian 3.2 3B · in parallel, 1.2 s in total" style="rounded=1;arcSize=4;whiteSpace=wrap;html=1;fillColor=#FAFAFA;strokeColor=#90A4AE;strokeWidth=2;dashed=1;verticalAlign=top;align=center;fontSize=13;fontStyle=3;fontColor=#607D8B;spacingTop=8;" vertex="1" parent="1"><mxGeometry x="200" y="330" width="800" height="300" as="geometry"/></mxCell> | |
| 15 | + <mxCell id="g0" value="<b>harm</b>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#ECEFF1;gradientColor=#CFD8DC;strokeColor=#455A64;strokeWidth=2;shadow=1;fontSize=14;fontColor=#263238;" vertex="1" parent="1"><mxGeometry x="236" y="380" width="170" height="64" as="geometry"/></mxCell> | |
| 16 | + <mxCell id="g1" value="<b>jailbreak</b>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#ECEFF1;gradientColor=#CFD8DC;strokeColor=#455A64;strokeWidth=2;shadow=1;fontSize=14;fontColor=#263238;" vertex="1" parent="1"><mxGeometry x="422" y="380" width="170" height="64" as="geometry"/></mxCell> | |
| 17 | + <mxCell id="g2" value="<b>violence</b>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#ECEFF1;gradientColor=#CFD8DC;strokeColor=#455A64;strokeWidth=2;shadow=1;fontSize=14;fontColor=#263238;" vertex="1" parent="1"><mxGeometry x="608" y="380" width="170" height="64" as="geometry"/></mxCell> | |
| 18 | + <mxCell id="g3" value="<b>unethical_behavior</b>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#ECEFF1;gradientColor=#CFD8DC;strokeColor=#455A64;strokeWidth=2;shadow=1;fontSize=14;fontColor=#263238;" vertex="1" parent="1"><mxGeometry x="794" y="380" width="170" height="64" as="geometry"/></mxCell> | |
| 19 | + <mxCell id="g4" value="<b>social_bias</b>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#ECEFF1;gradientColor=#CFD8DC;strokeColor=#455A64;strokeWidth=2;shadow=1;fontSize=14;fontColor=#263238;" vertex="1" parent="1"><mxGeometry x="329" y="460" width="170" height="64" as="geometry"/></mxCell> | |
| 20 | + <mxCell id="g5" value="<b>profanity</b>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#ECEFF1;gradientColor=#CFD8DC;strokeColor=#455A64;strokeWidth=2;shadow=1;fontSize=14;fontColor=#263238;" vertex="1" parent="1"><mxGeometry x="515" y="460" width="170" height="64" as="geometry"/></mxCell> | |
| 21 | + <mxCell id="g6" value="<b>sexual_content</b>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#ECEFF1;gradientColor=#CFD8DC;strokeColor=#455A64;strokeWidth=2;shadow=1;fontSize=14;fontColor=#263238;" vertex="1" parent="1"><mxGeometry x="701" y="460" width="170" height="64" as="geometry"/></mxCell> | |
| 22 | + <mxCell id="notes" value="each guard: mm -config guards/&lt;risk&gt;.yaml -p="&lt;question&gt;"<br>system: &lt;risk name&gt; · a fresh conversation, never the history<br><i><font color="#78909C" style="font-size:11px">answers "Yes" or "No" + &lt;confidence&gt;</font></i>" style="text;html=1;align=center;verticalAlign=middle;fontSize=12;fontColor=#455A64;" vertex="1" parent="1"><mxGeometry x="250" y="550" width="700" height="70" as="geometry"/></mxCell> | |
| 23 | + <mxCell id="e1" value="question" style="edgeStyle=orthogonalEdgeStyle;rounded=1;html=1;strokeColor=#37474F;strokeWidth=2;endArrow=block;endFill=1;fontSize=12;fontStyle=1;fontColor=#263238;labelBackgroundColor=#FFFFFF;" edge="1" parent="1" source="user" target="hook"><mxGeometry relative="1" as="geometry"></mxGeometry></mxCell> | |
| 24 | + <mxCell id="e2" value="question + safety report<br><font style="font-size:10px;font-weight:normal" color="#607D8B">[context added by a hook]</font>" style="edgeStyle=orthogonalEdgeStyle;rounded=1;html=1;strokeColor=#37474F;strokeWidth=2;endArrow=block;endFill=1;fontSize=12;fontStyle=1;fontColor=#263238;labelBackgroundColor=#FFFFFF;" edge="1" parent="1" source="hook" target="assistant"><mxGeometry relative="1" as="geometry"></mxGeometry></mxCell> | |
| 25 | + <mxCell id="e3" value="question alone<br><font style="font-size:10px;font-weight:normal" color="#607D8B">← Yes / No per risk</font>" style="edgeStyle=orthogonalEdgeStyle;rounded=1;html=1;strokeColor=#37474F;strokeWidth=2;endArrow=block;endFill=1;fontSize=12;fontStyle=1;fontColor=#263238;labelBackgroundColor=#FFFFFF;startArrow=block;startFill=1;align=left;spacingLeft=14;" edge="1" parent="1" source="hook" target="guards"><mxGeometry relative="1" as="geometry"></mxGeometry></mxCell> | |
| 26 | + <mxCell id="e4" value="answer · or refusal" style="edgeStyle=orthogonalEdgeStyle;rounded=1;html=1;strokeColor=#37474F;strokeWidth=2;endArrow=block;endFill=1;fontSize=12;fontStyle=1;fontColor=#263238;labelBackgroundColor=#FFFFFF;exitX=0.5;exitY=0;entryX=0.5;entryY=0;" edge="1" parent="1" source="assistant" target="user"><mxGeometry relative="1" as="geometry"><Array as="points"><mxPoint x="1030" y="100"/><mxPoint x="135" y="100"/></Array></mxGeometry></mxCell> | |
| 27 | + </root> | |
| 28 | + </mxGraphModel> | |
| 29 | + </diagram> | |
| 30 | +</mxfile> | |
| new file mode 100644 | |||
| @@ -0,0 +1,30 @@ | |||
| 1 | +<mxfile host="app.diagrams.net" agent="drawio-fancy-diagram-skill" version="24.7.17" type="device"> | ||
| 2 | + <diagram name="demos/guards" id="guards-flow"> | ||
| 3 | + <mxGraphModel dx="1400" dy="800" grid="0" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1200" pageHeight="680" math="0" shadow="0"> | ||
| 4 | + <root> | ||
| 5 | + <mxCell id="0"/> | ||
| 6 | + <mxCell id="1" parent="0"/> | ||
| 7 | + <mxCell id="bg" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#FFFFFF;gradientColor=#EEF2FA;strokeColor=none;" vertex="1" parent="1"><mxGeometry x="0" y="0" width="1200" height="680" as="geometry"/></mxCell> | ||
| 8 | + <mxCell id="title" value="demos/guards — every question screened first" style="text;html=1;align=center;verticalAlign=middle;fontSize=24;fontStyle=1;fontColor=#1A237E;" vertex="1" parent="1"><mxGeometry x="200" y="18" width="800" height="30" as="geometry"/></mxCell> | ||
| 9 | + <mxCell id="subtitle" value="a preGeneration hook asks seven guard agents, the question alone, before the assistant answers" style="text;html=1;align=center;verticalAlign=middle;fontSize=13;fontStyle=2;fontColor=#546E7A;" vertex="1" parent="1"><mxGeometry x="150" y="48" width="900" height="20" as="geometry"/></mxCell> | ||
| 10 | + <mxCell id="user" value="🧑<br><b>User</b><br><font style="font-size:11px">asks a question<br>terminal · -tui · editor</font>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#E3F2FD;gradientColor=#BBDEFB;strokeColor=#1565C0;strokeWidth=2;shadow=1;fontSize=15;fontColor=#0D47A1;" vertex="1" parent="1"><mxGeometry x="40" y="140" width="190" height="120" as="geometry"/></mxCell> | ||
| 11 | + <mxCell id="hook" value="🔍<br><b>preGeneration hook</b><br><font style="font-size:11px">hooks/guards.js<br>no verdict → stop (fail closed)</font>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#E1F5FE;gradientColor=#B3E5FC;strokeColor=#0277BD;strokeWidth=2;shadow=1;fontSize=15;fontColor=#01579B;" vertex="1" parent="1"><mxGeometry x="330" y="140" width="250" height="120" as="geometry"/></mxCell> | ||
| 12 | + <mxCell id="assistant" value="🤖<br><b>Assistant</b><br><font style="font-size:11px">Gemma 4 E4B · no tools<br>all No → answers · a Yes → refuses</font>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#F3E5F5;gradientColor=#E1BEE7;strokeColor=#6A1B9A;strokeWidth=3;shadow=1;fontSize=15;fontColor=#4A148C;" vertex="1" parent="1"><mxGeometry x="900" y="140" width="260" height="120" as="geometry"/></mxCell> | ||
| 13 | + <mxCell id="badge" value="★ decides" style="rounded=1;arcSize=50;whiteSpace=wrap;html=1;fillColor=#6A1B9A;strokeColor=none;fontSize=11;fontStyle=1;fontColor=#FFFFFF;" vertex="1" parent="1"><mxGeometry x="1060" y="128" width="88" height="24" as="geometry"/></mxCell> | ||
| 14 | + <mxCell id="guards" value="7 guard agents · Granite Guardian 3.2 3B · in parallel, 1.2 s in total" style="rounded=1;arcSize=4;whiteSpace=wrap;html=1;fillColor=#FAFAFA;strokeColor=#90A4AE;strokeWidth=2;dashed=1;verticalAlign=top;align=center;fontSize=13;fontStyle=3;fontColor=#607D8B;spacingTop=8;" vertex="1" parent="1"><mxGeometry x="200" y="330" width="800" height="300" as="geometry"/></mxCell> | ||
| 15 | + <mxCell id="g0" value="<b>harm</b>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#ECEFF1;gradientColor=#CFD8DC;strokeColor=#455A64;strokeWidth=2;shadow=1;fontSize=14;fontColor=#263238;" vertex="1" parent="1"><mxGeometry x="236" y="380" width="170" height="64" as="geometry"/></mxCell> | ||
| 16 | + <mxCell id="g1" value="<b>jailbreak</b>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#ECEFF1;gradientColor=#CFD8DC;strokeColor=#455A64;strokeWidth=2;shadow=1;fontSize=14;fontColor=#263238;" vertex="1" parent="1"><mxGeometry x="422" y="380" width="170" height="64" as="geometry"/></mxCell> | ||
| 17 | + <mxCell id="g2" value="<b>violence</b>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#ECEFF1;gradientColor=#CFD8DC;strokeColor=#455A64;strokeWidth=2;shadow=1;fontSize=14;fontColor=#263238;" vertex="1" parent="1"><mxGeometry x="608" y="380" width="170" height="64" as="geometry"/></mxCell> | ||
| 18 | + <mxCell id="g3" value="<b>unethical_behavior</b>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#ECEFF1;gradientColor=#CFD8DC;strokeColor=#455A64;strokeWidth=2;shadow=1;fontSize=14;fontColor=#263238;" vertex="1" parent="1"><mxGeometry x="794" y="380" width="170" height="64" as="geometry"/></mxCell> | ||
| 19 | + <mxCell id="g4" value="<b>social_bias</b>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#ECEFF1;gradientColor=#CFD8DC;strokeColor=#455A64;strokeWidth=2;shadow=1;fontSize=14;fontColor=#263238;" vertex="1" parent="1"><mxGeometry x="329" y="460" width="170" height="64" as="geometry"/></mxCell> | ||
| 20 | + <mxCell id="g5" value="<b>profanity</b>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#ECEFF1;gradientColor=#CFD8DC;strokeColor=#455A64;strokeWidth=2;shadow=1;fontSize=14;fontColor=#263238;" vertex="1" parent="1"><mxGeometry x="515" y="460" width="170" height="64" as="geometry"/></mxCell> | ||
| 21 | + <mxCell id="g6" value="<b>sexual_content</b>" style="rounded=1;arcSize=14;whiteSpace=wrap;html=1;fillColor=#ECEFF1;gradientColor=#CFD8DC;strokeColor=#455A64;strokeWidth=2;shadow=1;fontSize=14;fontColor=#263238;" vertex="1" parent="1"><mxGeometry x="701" y="460" width="170" height="64" as="geometry"/></mxCell> | ||
| 22 | + <mxCell id="notes" value="each guard: mm -config guards/&lt;risk&gt;.yaml -p="&lt;question&gt;"<br>system: &lt;risk name&gt; · a fresh conversation, never the history<br><i><font color="#78909C" style="font-size:11px">answers "Yes" or "No" + &lt;confidence&gt;</font></i>" style="text;html=1;align=center;verticalAlign=middle;fontSize=12;fontColor=#455A64;" vertex="1" parent="1"><mxGeometry x="250" y="550" width="700" height="70" as="geometry"/></mxCell> | ||
| 23 | + <mxCell id="e1" value="question" style="edgeStyle=orthogonalEdgeStyle;rounded=1;html=1;strokeColor=#37474F;strokeWidth=2;endArrow=block;endFill=1;fontSize=12;fontStyle=1;fontColor=#263238;labelBackgroundColor=#FFFFFF;" edge="1" parent="1" source="user" target="hook"><mxGeometry relative="1" as="geometry"></mxGeometry></mxCell> | ||
| 24 | + <mxCell id="e2" value="question + safety report<br><font style="font-size:10px;font-weight:normal" color="#607D8B">[context added by a hook]</font>" style="edgeStyle=orthogonalEdgeStyle;rounded=1;html=1;strokeColor=#37474F;strokeWidth=2;endArrow=block;endFill=1;fontSize=12;fontStyle=1;fontColor=#263238;labelBackgroundColor=#FFFFFF;" edge="1" parent="1" source="hook" target="assistant"><mxGeometry relative="1" as="geometry"></mxGeometry></mxCell> | ||
| 25 | + <mxCell id="e3" value="question alone<br><font style="font-size:10px;font-weight:normal" color="#607D8B">← Yes / No per risk</font>" style="edgeStyle=orthogonalEdgeStyle;rounded=1;html=1;strokeColor=#37474F;strokeWidth=2;endArrow=block;endFill=1;fontSize=12;fontStyle=1;fontColor=#263238;labelBackgroundColor=#FFFFFF;startArrow=block;startFill=1;align=left;spacingLeft=14;" edge="1" parent="1" source="hook" target="guards"><mxGeometry relative="1" as="geometry"></mxGeometry></mxCell> | ||
| 26 | + <mxCell id="e4" value="answer · or refusal" style="edgeStyle=orthogonalEdgeStyle;rounded=1;html=1;strokeColor=#37474F;strokeWidth=2;endArrow=block;endFill=1;fontSize=12;fontStyle=1;fontColor=#263238;labelBackgroundColor=#FFFFFF;exitX=0.5;exitY=0;entryX=0.5;entryY=0;" edge="1" parent="1" source="assistant" target="user"><mxGeometry relative="1" as="geometry"><Array as="points"><mxPoint x="1030" y="100"/><mxPoint x="135" y="100"/></Array></mxGeometry></mxCell> | ||
| 27 | + </root> | ||
| 28 | + </mxGraphModel> | ||
| 29 | + </diagram> | ||
| 30 | +</mxfile> | ||
added
demos/guards/guards-flow.png +0 -0 | new file mode 100644 | ||
| Binary files /dev/null and b/demos/guards/guards-flow.png differ | ||
| new file mode 100644 | |||
| Binary files /dev/null and b/demos/guards/guards-flow.png differ | Binary files /dev/null and b/demos/guards/guards-flow.png differ | ||
added
demos/guards/guards-flow.svg +117 -0 | new file mode 100644 | ||
| @@ -0,0 +1,117 @@ | ||
| 1 | +<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="680" viewBox="0 0 1200 680" font-family="DejaVu Sans, Verdana, sans-serif"> | |
| 2 | + <defs> | |
| 3 | + <linearGradient id="gBg" x1="0" y1="0" x2="0" y2="1"> | |
| 4 | + <stop offset="0%" stop-color="#FFFFFF"/> | |
| 5 | + <stop offset="100%" stop-color="#EEF2FA"/> | |
| 6 | + </linearGradient> | |
| 7 | + <linearGradient id="gBlue" x1="0" y1="0" x2="0" y2="1"> | |
| 8 | + <stop offset="0%" stop-color="#E3F2FD"/> | |
| 9 | + <stop offset="100%" stop-color="#BBDEFB"/> | |
| 10 | + </linearGradient> | |
| 11 | + <linearGradient id="gBlue2" x1="0" y1="0" x2="0" y2="1"> | |
| 12 | + <stop offset="0%" stop-color="#E1F5FE"/> | |
| 13 | + <stop offset="100%" stop-color="#B3E5FC"/> | |
| 14 | + </linearGradient> | |
| 15 | + <linearGradient id="gPurple" x1="0" y1="0" x2="0" y2="1"> | |
| 16 | + <stop offset="0%" stop-color="#F3E5F5"/> | |
| 17 | + <stop offset="100%" stop-color="#E1BEE7"/> | |
| 18 | + </linearGradient> | |
| 19 | + <linearGradient id="gGrey" x1="0" y1="0" x2="0" y2="1"> | |
| 20 | + <stop offset="0%" stop-color="#ECEFF1"/> | |
| 21 | + <stop offset="100%" stop-color="#CFD8DC"/> | |
| 22 | + </linearGradient> | |
| 23 | + <filter id="shadow" x="-30%" y="-30%" width="160%" height="160%"> | |
| 24 | + <feDropShadow dx="2" dy="3" stdDeviation="4" flood-color="#000000" flood-opacity="0.25"/> | |
| 25 | + </filter> | |
| 26 | + <marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto-start-reverse"> | |
| 27 | + <path d="M0,0 L10,5 L0,10 z" fill="#37474F"/> | |
| 28 | + </marker> | |
| 29 | + </defs> | |
| 30 | + | |
| 31 | + <rect x="0" y="0" width="1200" height="680" fill="url(#gBg)"/> | |
| 32 | + | |
| 33 | + <text x="600" y="38" text-anchor="middle" font-size="25" font-weight="700" fill="#1A237E">demos/guards — every question screened first</text> | |
| 34 | + <text x="600" y="61" text-anchor="middle" font-size="14" font-style="italic" fill="#546E7A">a preGeneration hook asks seven guard agents, the question alone, before the assistant answers</text> | |
| 35 | + | |
| 36 | + <!-- Answer: Assistant -> User (above the row) --> | |
| 37 | + <polyline points="1030,140 1030,100 135,100 135,138" fill="none" stroke="#37474F" stroke-width="2" marker-end="url(#arrow)"/> | |
| 38 | + <rect x="505" y="88" width="190" height="24" rx="4" fill="#FFFFFF"/> | |
| 39 | + <text x="600" y="105" text-anchor="middle" font-size="12.5" font-weight="700" fill="#263238">answer · or refusal</text> | |
| 40 | + | |
| 41 | + <!-- User --> | |
| 42 | + <g filter="url(#shadow)"> | |
| 43 | + <rect x="40" y="140" width="190" height="120" rx="16" fill="url(#gBlue)" stroke="#1565C0" stroke-width="2"/> | |
| 44 | + </g> | |
| 45 | + <text x="135" y="182" text-anchor="middle" font-size="24">🧑</text> | |
| 46 | + <text x="135" y="210" text-anchor="middle" font-size="16" font-weight="700" fill="#0D47A1">User</text> | |
| 47 | + <text x="135" y="230" text-anchor="middle" font-size="11.5" fill="#0D47A1">asks a question</text> | |
| 48 | + <text x="135" y="246" text-anchor="middle" font-size="11.5" fill="#0D47A1">terminal · -tui · editor</text> | |
| 49 | + | |
| 50 | + <!-- User -> Hook --> | |
| 51 | + <line x1="230" y1="200" x2="328" y2="200" stroke="#37474F" stroke-width="2" marker-end="url(#arrow)"/> | |
| 52 | + <rect x="243" y="190" width="72" height="20" rx="4" fill="#FFFFFF"/> | |
| 53 | + <text x="279" y="204" text-anchor="middle" font-size="12" font-weight="700" fill="#263238">question</text> | |
| 54 | + | |
| 55 | + <!-- Hook --> | |
| 56 | + <g filter="url(#shadow)"> | |
| 57 | + <rect x="330" y="140" width="250" height="120" rx="16" fill="url(#gBlue2)" stroke="#0277BD" stroke-width="2"/> | |
| 58 | + </g> | |
| 59 | + <text x="455" y="178" text-anchor="middle" font-size="22">🔍</text> | |
| 60 | + <text x="455" y="204" text-anchor="middle" font-size="16" font-weight="700" fill="#01579B">preGeneration hook</text> | |
| 61 | + <text x="455" y="223" text-anchor="middle" font-size="11.5" fill="#01579B">hooks/guards.js</text> | |
| 62 | + <text x="455" y="240" text-anchor="middle" font-size="11.5" fill="#01579B">no verdict → stop (fail closed)</text> | |
| 63 | + | |
| 64 | + <!-- Hook -> Assistant --> | |
| 65 | + <line x1="580" y1="200" x2="898" y2="200" stroke="#37474F" stroke-width="2" marker-end="url(#arrow)"/> | |
| 66 | + <rect x="632" y="180" width="216" height="40" rx="4" fill="#FFFFFF"/> | |
| 67 | + <text x="740" y="196" text-anchor="middle" font-size="12.5" font-weight="700" fill="#263238">question + safety report</text> | |
| 68 | + <text x="740" y="212" text-anchor="middle" font-size="10.5" fill="#607D8B">[context added by a hook]</text> | |
| 69 | + | |
| 70 | + <!-- Assistant (highlighted) --> | |
| 71 | + <g filter="url(#shadow)"> | |
| 72 | + <rect x="900" y="140" width="260" height="120" rx="16" fill="url(#gPurple)" stroke="#6A1B9A" stroke-width="3"/> | |
| 73 | + </g> | |
| 74 | + <rect x="1060" y="128" width="88" height="24" rx="12" fill="#6A1B9A"/> | |
| 75 | + <text x="1104" y="144.5" text-anchor="middle" font-size="11" font-weight="700" fill="#FFFFFF">★ decides</text> | |
| 76 | + <text x="1030" y="180" text-anchor="middle" font-size="22">🤖</text> | |
| 77 | + <text x="1030" y="206" text-anchor="middle" font-size="16" font-weight="700" fill="#4A148C">Assistant</text> | |
| 78 | + <text x="1030" y="225" text-anchor="middle" font-size="11.5" fill="#4A148C">Gemma 4 E4B · no tools</text> | |
| 79 | + <text x="1030" y="242" text-anchor="middle" font-size="11.5" fill="#4A148C">all No → answers · a Yes → refuses</text> | |
| 80 | + | |
| 81 | + <!-- Hook <-> guards --> | |
| 82 | + <line x1="455" y1="262" x2="455" y2="328" stroke="#37474F" stroke-width="2" marker-start="url(#arrow)" marker-end="url(#arrow)"/> | |
| 83 | + <rect x="468" y="274" width="170" height="40" rx="4" fill="#FFFFFF"/> | |
| 84 | + <text x="474" y="290" font-size="12.5" font-weight="700" fill="#263238">question alone</text> | |
| 85 | + <text x="474" y="306" font-size="10.5" fill="#607D8B">← Yes / No per risk</text> | |
| 86 | + | |
| 87 | + <!-- Guards container --> | |
| 88 | + <rect x="200" y="330" width="800" height="300" rx="10" fill="#FAFAFA" stroke="#90A4AE" stroke-width="2" stroke-dasharray="7,5"/> | |
| 89 | + <text x="600" y="355" text-anchor="middle" font-size="13" font-style="italic" font-weight="700" fill="#607D8B">7 guard agents · Granite Guardian 3.2 3B · in parallel, 1.2 s in total</text> | |
| 90 | + | |
| 91 | + <!-- Row 1 --> | |
| 92 | + <g filter="url(#shadow)"> | |
| 93 | + <rect x="236" y="380" width="170" height="64" rx="14" fill="url(#gGrey)" stroke="#455A64" stroke-width="2"/> | |
| 94 | + <rect x="422" y="380" width="170" height="64" rx="14" fill="url(#gGrey)" stroke="#455A64" stroke-width="2"/> | |
| 95 | + <rect x="608" y="380" width="170" height="64" rx="14" fill="url(#gGrey)" stroke="#455A64" stroke-width="2"/> | |
| 96 | + <rect x="794" y="380" width="170" height="64" rx="14" fill="url(#gGrey)" stroke="#455A64" stroke-width="2"/> | |
| 97 | + </g> | |
| 98 | + <text x="321" y="417" text-anchor="middle" font-size="14" font-weight="700" fill="#263238">harm</text> | |
| 99 | + <text x="507" y="417" text-anchor="middle" font-size="14" font-weight="700" fill="#263238">jailbreak</text> | |
| 100 | + <text x="693" y="417" text-anchor="middle" font-size="14" font-weight="700" fill="#263238">violence</text> | |
| 101 | + <text x="879" y="417" text-anchor="middle" font-size="14" font-weight="700" fill="#263238">unethical_behavior</text> | |
| 102 | + | |
| 103 | + <!-- Row 2 --> | |
| 104 | + <g filter="url(#shadow)"> | |
| 105 | + <rect x="329" y="460" width="170" height="64" rx="14" fill="url(#gGrey)" stroke="#455A64" stroke-width="2"/> | |
| 106 | + <rect x="515" y="460" width="170" height="64" rx="14" fill="url(#gGrey)" stroke="#455A64" stroke-width="2"/> | |
| 107 | + <rect x="701" y="460" width="170" height="64" rx="14" fill="url(#gGrey)" stroke="#455A64" stroke-width="2"/> | |
| 108 | + </g> | |
| 109 | + <text x="414" y="497" text-anchor="middle" font-size="14" font-weight="700" fill="#263238">social_bias</text> | |
| 110 | + <text x="600" y="497" text-anchor="middle" font-size="14" font-weight="700" fill="#263238">profanity</text> | |
| 111 | + <text x="786" y="497" text-anchor="middle" font-size="14" font-weight="700" fill="#263238">sexual_content</text> | |
| 112 | + | |
| 113 | + <!-- Container notes --> | |
| 114 | + <text x="600" y="566" text-anchor="middle" font-size="12" fill="#455A64">each guard: mm -config guards/<risk>.yaml -p="<question>"</text> | |
| 115 | + <text x="600" y="586" text-anchor="middle" font-size="12" fill="#455A64">system: <risk name> · a fresh conversation, never the history</text> | |
| 116 | + <text x="600" y="610" text-anchor="middle" font-size="11" font-style="italic" fill="#78909C">answers "Yes" or "No" + <confidence></text> | |
| 117 | +</svg> | |
| new file mode 100644 | |||
| @@ -0,0 +1,117 @@ | |||
| 1 | +<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="680" viewBox="0 0 1200 680" font-family="DejaVu Sans, Verdana, sans-serif"> | ||
| 2 | + <defs> | ||
| 3 | + <linearGradient id="gBg" x1="0" y1="0" x2="0" y2="1"> | ||
| 4 | + <stop offset="0%" stop-color="#FFFFFF"/> | ||
| 5 | + <stop offset="100%" stop-color="#EEF2FA"/> | ||
| 6 | + </linearGradient> | ||
| 7 | + <linearGradient id="gBlue" x1="0" y1="0" x2="0" y2="1"> | ||
| 8 | + <stop offset="0%" stop-color="#E3F2FD"/> | ||
| 9 | + <stop offset="100%" stop-color="#BBDEFB"/> | ||
| 10 | + </linearGradient> | ||
| 11 | + <linearGradient id="gBlue2" x1="0" y1="0" x2="0" y2="1"> | ||
| 12 | + <stop offset="0%" stop-color="#E1F5FE"/> | ||
| 13 | + <stop offset="100%" stop-color="#B3E5FC"/> | ||
| 14 | + </linearGradient> | ||
| 15 | + <linearGradient id="gPurple" x1="0" y1="0" x2="0" y2="1"> | ||
| 16 | + <stop offset="0%" stop-color="#F3E5F5"/> | ||
| 17 | + <stop offset="100%" stop-color="#E1BEE7"/> | ||
| 18 | + </linearGradient> | ||
| 19 | + <linearGradient id="gGrey" x1="0" y1="0" x2="0" y2="1"> | ||
| 20 | + <stop offset="0%" stop-color="#ECEFF1"/> | ||
| 21 | + <stop offset="100%" stop-color="#CFD8DC"/> | ||
| 22 | + </linearGradient> | ||
| 23 | + <filter id="shadow" x="-30%" y="-30%" width="160%" height="160%"> | ||
| 24 | + <feDropShadow dx="2" dy="3" stdDeviation="4" flood-color="#000000" flood-opacity="0.25"/> | ||
| 25 | + </filter> | ||
| 26 | + <marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto-start-reverse"> | ||
| 27 | + <path d="M0,0 L10,5 L0,10 z" fill="#37474F"/> | ||
| 28 | + </marker> | ||
| 29 | + </defs> | ||
| 30 | + | ||
| 31 | + <rect x="0" y="0" width="1200" height="680" fill="url(#gBg)"/> | ||
| 32 | + | ||
| 33 | + <text x="600" y="38" text-anchor="middle" font-size="25" font-weight="700" fill="#1A237E">demos/guards — every question screened first</text> | ||
| 34 | + <text x="600" y="61" text-anchor="middle" font-size="14" font-style="italic" fill="#546E7A">a preGeneration hook asks seven guard agents, the question alone, before the assistant answers</text> | ||
| 35 | + | ||
| 36 | + <!-- Answer: Assistant -> User (above the row) --> | ||
| 37 | + <polyline points="1030,140 1030,100 135,100 135,138" fill="none" stroke="#37474F" stroke-width="2" marker-end="url(#arrow)"/> | ||
| 38 | + <rect x="505" y="88" width="190" height="24" rx="4" fill="#FFFFFF"/> | ||
| 39 | + <text x="600" y="105" text-anchor="middle" font-size="12.5" font-weight="700" fill="#263238">answer · or refusal</text> | ||
| 40 | + | ||
| 41 | + <!-- User --> | ||
| 42 | + <g filter="url(#shadow)"> | ||
| 43 | + <rect x="40" y="140" width="190" height="120" rx="16" fill="url(#gBlue)" stroke="#1565C0" stroke-width="2"/> | ||
| 44 | + </g> | ||
| 45 | + <text x="135" y="182" text-anchor="middle" font-size="24">🧑</text> | ||
| 46 | + <text x="135" y="210" text-anchor="middle" font-size="16" font-weight="700" fill="#0D47A1">User</text> | ||
| 47 | + <text x="135" y="230" text-anchor="middle" font-size="11.5" fill="#0D47A1">asks a question</text> | ||
| 48 | + <text x="135" y="246" text-anchor="middle" font-size="11.5" fill="#0D47A1">terminal · -tui · editor</text> | ||
| 49 | + | ||
| 50 | + <!-- User -> Hook --> | ||
| 51 | + <line x1="230" y1="200" x2="328" y2="200" stroke="#37474F" stroke-width="2" marker-end="url(#arrow)"/> | ||
| 52 | + <rect x="243" y="190" width="72" height="20" rx="4" fill="#FFFFFF"/> | ||
| 53 | + <text x="279" y="204" text-anchor="middle" font-size="12" font-weight="700" fill="#263238">question</text> | ||
| 54 | + | ||
| 55 | + <!-- Hook --> | ||
| 56 | + <g filter="url(#shadow)"> | ||
| 57 | + <rect x="330" y="140" width="250" height="120" rx="16" fill="url(#gBlue2)" stroke="#0277BD" stroke-width="2"/> | ||
| 58 | + </g> | ||
| 59 | + <text x="455" y="178" text-anchor="middle" font-size="22">🔍</text> | ||
| 60 | + <text x="455" y="204" text-anchor="middle" font-size="16" font-weight="700" fill="#01579B">preGeneration hook</text> | ||
| 61 | + <text x="455" y="223" text-anchor="middle" font-size="11.5" fill="#01579B">hooks/guards.js</text> | ||
| 62 | + <text x="455" y="240" text-anchor="middle" font-size="11.5" fill="#01579B">no verdict → stop (fail closed)</text> | ||
| 63 | + | ||
| 64 | + <!-- Hook -> Assistant --> | ||
| 65 | + <line x1="580" y1="200" x2="898" y2="200" stroke="#37474F" stroke-width="2" marker-end="url(#arrow)"/> | ||
| 66 | + <rect x="632" y="180" width="216" height="40" rx="4" fill="#FFFFFF"/> | ||
| 67 | + <text x="740" y="196" text-anchor="middle" font-size="12.5" font-weight="700" fill="#263238">question + safety report</text> | ||
| 68 | + <text x="740" y="212" text-anchor="middle" font-size="10.5" fill="#607D8B">[context added by a hook]</text> | ||
| 69 | + | ||
| 70 | + <!-- Assistant (highlighted) --> | ||
| 71 | + <g filter="url(#shadow)"> | ||
| 72 | + <rect x="900" y="140" width="260" height="120" rx="16" fill="url(#gPurple)" stroke="#6A1B9A" stroke-width="3"/> | ||
| 73 | + </g> | ||
| 74 | + <rect x="1060" y="128" width="88" height="24" rx="12" fill="#6A1B9A"/> | ||
| 75 | + <text x="1104" y="144.5" text-anchor="middle" font-size="11" font-weight="700" fill="#FFFFFF">★ decides</text> | ||
| 76 | + <text x="1030" y="180" text-anchor="middle" font-size="22">🤖</text> | ||
| 77 | + <text x="1030" y="206" text-anchor="middle" font-size="16" font-weight="700" fill="#4A148C">Assistant</text> | ||
| 78 | + <text x="1030" y="225" text-anchor="middle" font-size="11.5" fill="#4A148C">Gemma 4 E4B · no tools</text> | ||
| 79 | + <text x="1030" y="242" text-anchor="middle" font-size="11.5" fill="#4A148C">all No → answers · a Yes → refuses</text> | ||
| 80 | + | ||
| 81 | + <!-- Hook <-> guards --> | ||
| 82 | + <line x1="455" y1="262" x2="455" y2="328" stroke="#37474F" stroke-width="2" marker-start="url(#arrow)" marker-end="url(#arrow)"/> | ||
| 83 | + <rect x="468" y="274" width="170" height="40" rx="4" fill="#FFFFFF"/> | ||
| 84 | + <text x="474" y="290" font-size="12.5" font-weight="700" fill="#263238">question alone</text> | ||
| 85 | + <text x="474" y="306" font-size="10.5" fill="#607D8B">← Yes / No per risk</text> | ||
| 86 | + | ||
| 87 | + <!-- Guards container --> | ||
| 88 | + <rect x="200" y="330" width="800" height="300" rx="10" fill="#FAFAFA" stroke="#90A4AE" stroke-width="2" stroke-dasharray="7,5"/> | ||
| 89 | + <text x="600" y="355" text-anchor="middle" font-size="13" font-style="italic" font-weight="700" fill="#607D8B">7 guard agents · Granite Guardian 3.2 3B · in parallel, 1.2 s in total</text> | ||
| 90 | + | ||
| 91 | + <!-- Row 1 --> | ||
| 92 | + <g filter="url(#shadow)"> | ||
| 93 | + <rect x="236" y="380" width="170" height="64" rx="14" fill="url(#gGrey)" stroke="#455A64" stroke-width="2"/> | ||
| 94 | + <rect x="422" y="380" width="170" height="64" rx="14" fill="url(#gGrey)" stroke="#455A64" stroke-width="2"/> | ||
| 95 | + <rect x="608" y="380" width="170" height="64" rx="14" fill="url(#gGrey)" stroke="#455A64" stroke-width="2"/> | ||
| 96 | + <rect x="794" y="380" width="170" height="64" rx="14" fill="url(#gGrey)" stroke="#455A64" stroke-width="2"/> | ||
| 97 | + </g> | ||
| 98 | + <text x="321" y="417" text-anchor="middle" font-size="14" font-weight="700" fill="#263238">harm</text> | ||
| 99 | + <text x="507" y="417" text-anchor="middle" font-size="14" font-weight="700" fill="#263238">jailbreak</text> | ||
| 100 | + <text x="693" y="417" text-anchor="middle" font-size="14" font-weight="700" fill="#263238">violence</text> | ||
| 101 | + <text x="879" y="417" text-anchor="middle" font-size="14" font-weight="700" fill="#263238">unethical_behavior</text> | ||
| 102 | + | ||
| 103 | + <!-- Row 2 --> | ||
| 104 | + <g filter="url(#shadow)"> | ||
| 105 | + <rect x="329" y="460" width="170" height="64" rx="14" fill="url(#gGrey)" stroke="#455A64" stroke-width="2"/> | ||
| 106 | + <rect x="515" y="460" width="170" height="64" rx="14" fill="url(#gGrey)" stroke="#455A64" stroke-width="2"/> | ||
| 107 | + <rect x="701" y="460" width="170" height="64" rx="14" fill="url(#gGrey)" stroke="#455A64" stroke-width="2"/> | ||
| 108 | + </g> | ||
| 109 | + <text x="414" y="497" text-anchor="middle" font-size="14" font-weight="700" fill="#263238">social_bias</text> | ||
| 110 | + <text x="600" y="497" text-anchor="middle" font-size="14" font-weight="700" fill="#263238">profanity</text> | ||
| 111 | + <text x="786" y="497" text-anchor="middle" font-size="14" font-weight="700" fill="#263238">sexual_content</text> | ||
| 112 | + | ||
| 113 | + <!-- Container notes --> | ||
| 114 | + <text x="600" y="566" text-anchor="middle" font-size="12" fill="#455A64">each guard: mm -config guards/<risk>.yaml -p="<question>"</text> | ||
| 115 | + <text x="600" y="586" text-anchor="middle" font-size="12" fill="#455A64">system: <risk name> · a fresh conversation, never the history</text> | ||
| 116 | + <text x="600" y="610" text-anchor="middle" font-size="11" font-style="italic" fill="#78909C">answers "Yes" or "No" + <confidence></text> | ||
| 117 | +</svg> | ||
added
demos/unload-models/README.en.md +86 -0 | new file mode 100644 | ||
| @@ -0,0 +1,86 @@ | ||
| 1 | +# mini-me demo: unloading models | |
| 2 | + | |
| 3 | +*[Version française](README.fr.md)* | |
| 4 | + | |
| 5 | +A coordinator hands each programming question to an expert, and the two models take turns in the server's memory without ever being loaded together: a `preAsk` hook unloads the coordinator's model right before the expert starts, and a `postAsk` hook unloads the expert's model once it has answered, before the coordinator is reloaded to read the answer. The models are served by [llmman](https://github.com/llmmanorg/llmman). | |
| 6 | + | |
| 7 | +## What is here | |
| 8 | + | |
| 9 | +| File | Role | | |
| 10 | +|---|---| | |
| 11 | +| `agent.yaml` | The **coordinator** (Qwen3.5 9B): `ask_agent` to the expert, then the answer. The two hooks. `/unload` and the `unload_model` tool work here too. `displayHooks: true` shows the hook runs. | | |
| 12 | +| `hooks/unload.js` | The hook (Node.js). `unload.js self` (`preAsk`): unloads the coordinator's own model right before mm starts the expert. `unload.js asked` (`postAsk`): unloads the expert's model once it has answered. | | |
| 13 | +| `agent.expert.yaml` | The **expert** (Qwen3.5 4B): answers programming questions, no tools at all (`unloadModels: false`). | | |
| 14 | + | |
| 15 | +## Run the demo | |
| 16 | + | |
| 17 | +```bash | |
| 18 | +llmman pull hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | |
| 19 | +llmman pull hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M | |
| 20 | +cd demos/unload-models | |
| 21 | +mm -tui agent.yaml | |
| 22 | +``` | |
| 23 | + | |
| 24 | +Ask a programming question, then look at what the server holds: | |
| 25 | + | |
| 26 | +```bash | |
| 27 | +llmman ps # or: curl -s http://127.0.0.1:17434/api/ps | |
| 28 | +``` | |
| 29 | + | |
| 30 | +After the answer, only the coordinator's Qwen3.5 9B is loaded. In the terminal the hand-off shows as: | |
| 31 | + | |
| 32 | +```text | |
| 33 | +🗣 ask_agent: expert (llmman/hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M) — Explain what a closure is in Ja… | |
| 34 | + 🪝 1. preAsk expert · node "$MM_CONFIG_DIR/hooks/unload.js" self → continue | |
| 35 | + 🪝 2. postAsk expert · node "$MM_CONFIG_DIR/hooks/unload.js" asked → continue | |
| 36 | +``` | |
| 37 | + | |
| 38 | +By hand: `/unload` unloads the coordinator's own model, `/unload <model>` any other one of the same server — in the terminal, the window and an editor. Asking the coordinator to free the expert's memory ("Unload the expert's model.", "Libère la mémoire de l'expert.") makes it call `unload_model` on the expert's model (4/4 measured): its name is written in the prompt, as the model cannot guess how the server names a model. | |
| 39 | + | |
| 40 | +## How it works | |
| 41 | + | |
| 42 | +What the server holds during one question (measured with `/api/ps` twice a second): | |
| 43 | + | |
| 44 | +| Moment | Loaded | | |
| 45 | +|---|---| | |
| 46 | +| the coordinator reads the question and calls `ask_agent` | Qwen3.5 9B | | |
| 47 | +| the `preAsk` hook unloads the coordinator | — | | |
| 48 | +| the expert works | Qwen3.5 4B alone | | |
| 49 | +| the `postAsk` hook unloads the expert | — | | |
| 50 | +| the coordinator reads the answer (the server reloads it) | Qwen3.5 9B alone | | |
| 51 | + | |
| 52 | +**Why hooks, and not the model.** Both unloads must happen at a moment when the coordinator's model is not running: | |
| 53 | + | |
| 54 | +- *Before the expert*: the coordinator's model has just decided to call `ask_agent`, and it is needed again only to read the answer. The model cannot unload itself — `unload_model` on its own model lasts until its next step. | |
| 55 | +- *After the expert*: if the coordinator called `unload_model` on the expert's model, it would already be running again — reloaded beside the 4B — when it does so. The two models would be loaded together for the whole step. (That is what the previous version of this demo did; see below.) | |
| 56 | + | |
| 57 | +mm runs the `preAsk` hooks right before it starts the asked agent and the `postAsk` hooks right after its answer, before the asking agent reads it. `hooks/unload.js` reads the model to unload from its input — `"model": "llmman/<model>"` for `self`, `"ask": {"agentModel": "llmman/<model>"}` for `asked` — and sends `keep_alive: 0` to the server. `matcher: ^expert$` keeps both hooks to this hand-off. `onError: warn`: if an unload fails, the hand-off goes on. | |
| 58 | + | |
| 59 | +The hooks are also certain: a model can skip an instruction, a hook always runs. | |
| 60 | + | |
| 61 | +## Measured results (2026-10-18, `mm -p`, llmman) | |
| 62 | + | |
| 63 | +Coordinator Qwen3.5 9B, expert Qwen3.5 4B, both hooks on. "Never together" means `/api/ps` (sampled twice a second) never showed both models at once during the question: 9B → nothing → 4B → nothing → 9B. | |
| 64 | + | |
| 65 | +| Question | `ask_agent` | `preAsk` unloads the 9B | `postAsk` unloads the 4B | never together | Time | | |
| 66 | +|---|---|---|---|---|---| | |
| 67 | +| How do I reverse a string in Go? | ✓ | ✓ | ✓ | ✓ | 62 s | | |
| 68 | +| What is a closure in JavaScript? | ✓ | ✓ | ✓ | ✓ | 65 s | | |
| 69 | +| How do I read a file line by line in Python? | ✓ | ✓ | ✓ | ✓ | 49 s | | |
| 70 | +| Explain Rust ownership briefly. | ✓ | ✓ | ✓ | ✓ | 82 s | | |
| 71 | +| Comment trier un tableau en Java ? | ✓ | ✓ | ✓ | ✓ | 119 s | | |
| 72 | +| How do I make an HTTP GET request in Go? | ✓ | ✓ | ✓ | ✓ | 106 s | | |
| 73 | +| What does the yield keyword do in Python? (×3) | ✓ ✓ ✓ | ✓ ✓ ✓ | ✓ ✓ ✓ | ✓ ✓ ✓ | 57 s, 60 s, 85 s | | |
| 74 | + | |
| 75 | +9 out of 9. The French question is answered in French. A greeting ("Hello, who are you?") is answered in 2 s without any tool. The times include the loads: each question loads the 9B twice and the 4B once, and vary a lot from one run to the next. | |
| 76 | + | |
| 77 | +**The prompt.** Without the `unload_model` step, a short prompt ("1. Call ask_agent… 2. Then give the expert's answer") let the coordinator answer the `yield` question by itself, without the expert (1 time out of 7). Saying that `ask_agent` is ALWAYS the first action, "even for a question you know the answer to", gave the table above. | |
| 78 | + | |
| 79 | +**Earlier versions.** With the coordinator freeing the expert through `unload_model` (no `postAsk` hook), the 9B and the 4B were loaded together for ~18 s per question while the coordinator read the answer — 7/7 questions otherwise clean. Before that, with a MiniCPM5 2B coordinator and a Qwen3.5 9B expert, the coordinator skipped `unload_model` once in five. | |
| 80 | + | |
| 81 | +## Good to know | |
| 82 | + | |
| 83 | +- **`postAsk` runs after every hand-off to the expert**, answered, failed or cancelled: the expert's model is unloaded even when its turn failed. | |
| 84 | +- **The next question reloads the 4B**: a hand-off costs one load of each model, plus a reload of the coordinator. Keeping both loaded is faster, when the memory allows it. | |
| 85 | +- **llmman also unloads by itself** a model idle for five minutes (`keep_alive`); the hooks free the memory at once, which matters when the next model needs it right away. | |
| 86 | +- **Turn off `/unload` and `unload_model`** with `unloadModels: false`, as the expert does: an agent that only answers does not need one more tool. The hooks do not depend on that key. | |
| new file mode 100644 | |||
| @@ -0,0 +1,86 @@ | |||
| 1 | +# mini-me demo: unloading models | ||
| 2 | + | ||
| 3 | +*[Version française](README.fr.md)* | ||
| 4 | + | ||
| 5 | +A coordinator hands each programming question to an expert, and the two models take turns in the server's memory without ever being loaded together: a `preAsk` hook unloads the coordinator's model right before the expert starts, and a `postAsk` hook unloads the expert's model once it has answered, before the coordinator is reloaded to read the answer. The models are served by [llmman](https://github.com/llmmanorg/llmman). | ||
| 6 | + | ||
| 7 | +## What is here | ||
| 8 | + | ||
| 9 | +| File | Role | | ||
| 10 | +|---|---| | ||
| 11 | +| `agent.yaml` | The **coordinator** (Qwen3.5 9B): `ask_agent` to the expert, then the answer. The two hooks. `/unload` and the `unload_model` tool work here too. `displayHooks: true` shows the hook runs. | | ||
| 12 | +| `hooks/unload.js` | The hook (Node.js). `unload.js self` (`preAsk`): unloads the coordinator's own model right before mm starts the expert. `unload.js asked` (`postAsk`): unloads the expert's model once it has answered. | | ||
| 13 | +| `agent.expert.yaml` | The **expert** (Qwen3.5 4B): answers programming questions, no tools at all (`unloadModels: false`). | | ||
| 14 | + | ||
| 15 | +## Run the demo | ||
| 16 | + | ||
| 17 | +```bash | ||
| 18 | +llmman pull hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | ||
| 19 | +llmman pull hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M | ||
| 20 | +cd demos/unload-models | ||
| 21 | +mm -tui agent.yaml | ||
| 22 | +``` | ||
| 23 | + | ||
| 24 | +Ask a programming question, then look at what the server holds: | ||
| 25 | + | ||
| 26 | +```bash | ||
| 27 | +llmman ps # or: curl -s http://127.0.0.1:17434/api/ps | ||
| 28 | +``` | ||
| 29 | + | ||
| 30 | +After the answer, only the coordinator's Qwen3.5 9B is loaded. In the terminal the hand-off shows as: | ||
| 31 | + | ||
| 32 | +```text | ||
| 33 | +🗣 ask_agent: expert (llmman/hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M) — Explain what a closure is in Ja… | ||
| 34 | + 🪝 1. preAsk expert · node "$MM_CONFIG_DIR/hooks/unload.js" self → continue | ||
| 35 | + 🪝 2. postAsk expert · node "$MM_CONFIG_DIR/hooks/unload.js" asked → continue | ||
| 36 | +``` | ||
| 37 | + | ||
| 38 | +By hand: `/unload` unloads the coordinator's own model, `/unload <model>` any other one of the same server — in the terminal, the window and an editor. Asking the coordinator to free the expert's memory ("Unload the expert's model.", "Libère la mémoire de l'expert.") makes it call `unload_model` on the expert's model (4/4 measured): its name is written in the prompt, as the model cannot guess how the server names a model. | ||
| 39 | + | ||
| 40 | +## How it works | ||
| 41 | + | ||
| 42 | +What the server holds during one question (measured with `/api/ps` twice a second): | ||
| 43 | + | ||
| 44 | +| Moment | Loaded | | ||
| 45 | +|---|---| | ||
| 46 | +| the coordinator reads the question and calls `ask_agent` | Qwen3.5 9B | | ||
| 47 | +| the `preAsk` hook unloads the coordinator | — | | ||
| 48 | +| the expert works | Qwen3.5 4B alone | | ||
| 49 | +| the `postAsk` hook unloads the expert | — | | ||
| 50 | +| the coordinator reads the answer (the server reloads it) | Qwen3.5 9B alone | | ||
| 51 | + | ||
| 52 | +**Why hooks, and not the model.** Both unloads must happen at a moment when the coordinator's model is not running: | ||
| 53 | + | ||
| 54 | +- *Before the expert*: the coordinator's model has just decided to call `ask_agent`, and it is needed again only to read the answer. The model cannot unload itself — `unload_model` on its own model lasts until its next step. | ||
| 55 | +- *After the expert*: if the coordinator called `unload_model` on the expert's model, it would already be running again — reloaded beside the 4B — when it does so. The two models would be loaded together for the whole step. (That is what the previous version of this demo did; see below.) | ||
| 56 | + | ||
| 57 | +mm runs the `preAsk` hooks right before it starts the asked agent and the `postAsk` hooks right after its answer, before the asking agent reads it. `hooks/unload.js` reads the model to unload from its input — `"model": "llmman/<model>"` for `self`, `"ask": {"agentModel": "llmman/<model>"}` for `asked` — and sends `keep_alive: 0` to the server. `matcher: ^expert$` keeps both hooks to this hand-off. `onError: warn`: if an unload fails, the hand-off goes on. | ||
| 58 | + | ||
| 59 | +The hooks are also certain: a model can skip an instruction, a hook always runs. | ||
| 60 | + | ||
| 61 | +## Measured results (2026-10-18, `mm -p`, llmman) | ||
| 62 | + | ||
| 63 | +Coordinator Qwen3.5 9B, expert Qwen3.5 4B, both hooks on. "Never together" means `/api/ps` (sampled twice a second) never showed both models at once during the question: 9B → nothing → 4B → nothing → 9B. | ||
| 64 | + | ||
| 65 | +| Question | `ask_agent` | `preAsk` unloads the 9B | `postAsk` unloads the 4B | never together | Time | | ||
| 66 | +|---|---|---|---|---|---| | ||
| 67 | +| How do I reverse a string in Go? | ✓ | ✓ | ✓ | ✓ | 62 s | | ||
| 68 | +| What is a closure in JavaScript? | ✓ | ✓ | ✓ | ✓ | 65 s | | ||
| 69 | +| How do I read a file line by line in Python? | ✓ | ✓ | ✓ | ✓ | 49 s | | ||
| 70 | +| Explain Rust ownership briefly. | ✓ | ✓ | ✓ | ✓ | 82 s | | ||
| 71 | +| Comment trier un tableau en Java ? | ✓ | ✓ | ✓ | ✓ | 119 s | | ||
| 72 | +| How do I make an HTTP GET request in Go? | ✓ | ✓ | ✓ | ✓ | 106 s | | ||
| 73 | +| What does the yield keyword do in Python? (×3) | ✓ ✓ ✓ | ✓ ✓ ✓ | ✓ ✓ ✓ | ✓ ✓ ✓ | 57 s, 60 s, 85 s | | ||
| 74 | + | ||
| 75 | +9 out of 9. The French question is answered in French. A greeting ("Hello, who are you?") is answered in 2 s without any tool. The times include the loads: each question loads the 9B twice and the 4B once, and vary a lot from one run to the next. | ||
| 76 | + | ||
| 77 | +**The prompt.** Without the `unload_model` step, a short prompt ("1. Call ask_agent… 2. Then give the expert's answer") let the coordinator answer the `yield` question by itself, without the expert (1 time out of 7). Saying that `ask_agent` is ALWAYS the first action, "even for a question you know the answer to", gave the table above. | ||
| 78 | + | ||
| 79 | +**Earlier versions.** With the coordinator freeing the expert through `unload_model` (no `postAsk` hook), the 9B and the 4B were loaded together for ~18 s per question while the coordinator read the answer — 7/7 questions otherwise clean. Before that, with a MiniCPM5 2B coordinator and a Qwen3.5 9B expert, the coordinator skipped `unload_model` once in five. | ||
| 80 | + | ||
| 81 | +## Good to know | ||
| 82 | + | ||
| 83 | +- **`postAsk` runs after every hand-off to the expert**, answered, failed or cancelled: the expert's model is unloaded even when its turn failed. | ||
| 84 | +- **The next question reloads the 4B**: a hand-off costs one load of each model, plus a reload of the coordinator. Keeping both loaded is faster, when the memory allows it. | ||
| 85 | +- **llmman also unloads by itself** a model idle for five minutes (`keep_alive`); the hooks free the memory at once, which matters when the next model needs it right away. | ||
| 86 | +- **Turn off `/unload` and `unload_model`** with `unloadModels: false`, as the expert does: an agent that only answers does not need one more tool. The hooks do not depend on that key. | ||
added
demos/unload-models/README.fr.md +86 -0 | new file mode 100644 | ||
| @@ -0,0 +1,86 @@ | ||
| 1 | +# Démo mini-me : décharger des modèles | |
| 2 | + | |
| 3 | +*[English version](README.en.md)* | |
| 4 | + | |
| 5 | +Un coordinateur confie chaque question de programmation à un expert, et les deux modèles se relaient dans la mémoire du serveur sans jamais y être chargés ensemble : un hook `preAsk` décharge le modèle du coordinateur juste avant le lancement de l'expert, et un hook `postAsk` décharge le modèle de l'expert une fois sa réponse donnée, avant que le coordinateur soit rechargé pour la lire. Les modèles sont servis par [llmman](https://github.com/llmmanorg/llmman). | |
| 6 | + | |
| 7 | +## Ce qu'il y a ici | |
| 8 | + | |
| 9 | +| Fichier | Rôle | | |
| 10 | +|---|---| | |
| 11 | +| `agent.yaml` | Le **coordinateur** (Qwen3.5 9B) : `ask_agent` vers l'expert, puis la réponse. Les deux hooks. `/unload` et l'outil `unload_model` fonctionnent aussi ici. `displayHooks: true` affiche l'exécution des hooks. | | |
| 12 | +| `hooks/unload.js` | Le hook (Node.js). `unload.js self` (`preAsk`) : décharge le modèle du coordinateur juste avant que mm lance l'expert. `unload.js asked` (`postAsk`) : décharge le modèle de l'expert une fois sa réponse donnée. | | |
| 13 | +| `agent.expert.yaml` | L'**expert** (Qwen3.5 4B) : répond aux questions de programmation, sans aucun outil (`unloadModels: false`). | | |
| 14 | + | |
| 15 | +## Lancer la démo | |
| 16 | + | |
| 17 | +```bash | |
| 18 | +llmman pull hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | |
| 19 | +llmman pull hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M | |
| 20 | +cd demos/unload-models | |
| 21 | +mm -tui agent.yaml | |
| 22 | +``` | |
| 23 | + | |
| 24 | +Pose une question de programmation, puis regarde ce que le serveur garde en mémoire : | |
| 25 | + | |
| 26 | +```bash | |
| 27 | +llmman ps # ou : curl -s http://127.0.0.1:17434/api/ps | |
| 28 | +``` | |
| 29 | + | |
| 30 | +Après la réponse, seul le Qwen3.5 9B du coordinateur est chargé. Dans le terminal, le relais s'affiche ainsi : | |
| 31 | + | |
| 32 | +```text | |
| 33 | +🗣 ask_agent: expert (llmman/hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M) — Explain what a closure is in Ja… | |
| 34 | + 🪝 1. preAsk expert · node "$MM_CONFIG_DIR/hooks/unload.js" self → continue | |
| 35 | + 🪝 2. postAsk expert · node "$MM_CONFIG_DIR/hooks/unload.js" asked → continue | |
| 36 | +``` | |
| 37 | + | |
| 38 | +À la main : `/unload` décharge le modèle du coordinateur, `/unload <modèle>` n'importe quel autre modèle du même serveur — dans le terminal, la fenêtre et un éditeur. Demander au coordinateur de libérer la mémoire de l'expert (« Unload the expert's model. », « Libère la mémoire de l'expert. ») lui fait appeler `unload_model` sur le modèle de l'expert (4 sur 4 mesurés) : son nom est écrit dans le prompt, car le modèle ne peut pas deviner comment le serveur nomme un modèle. | |
| 39 | + | |
| 40 | +## Comment ça marche | |
| 41 | + | |
| 42 | +Ce que le serveur garde en mémoire pendant une question (mesuré avec `/api/ps` deux fois par seconde) : | |
| 43 | + | |
| 44 | +| Moment | Chargé | | |
| 45 | +|---|---| | |
| 46 | +| le coordinateur lit la question et appelle `ask_agent` | Qwen3.5 9B | | |
| 47 | +| le hook `preAsk` décharge le coordinateur | — | | |
| 48 | +| l'expert travaille | Qwen3.5 4B seul | | |
| 49 | +| le hook `postAsk` décharge l'expert | — | | |
| 50 | +| le coordinateur lit la réponse (le serveur le recharge) | Qwen3.5 9B seul | | |
| 51 | + | |
| 52 | +**Pourquoi des hooks, et pas le modèle.** Les deux déchargements doivent avoir lieu à un moment où le modèle du coordinateur ne tourne pas : | |
| 53 | + | |
| 54 | +- *Avant l'expert* : le modèle du coordinateur vient de décider d'appeler `ask_agent`, et il n'en a de nouveau besoin que pour lire la réponse. Le modèle ne peut pas se décharger lui-même — `unload_model` sur son propre modèle ne dure que jusqu'à son étape suivante. | |
| 55 | +- *Après l'expert* : si le coordinateur appelait `unload_model` sur le modèle de l'expert, il tournerait déjà de nouveau — rechargé à côté du 4B — au moment de le faire. Les deux modèles seraient chargés ensemble pendant toute cette étape. (C'est ce que faisait la version précédente de cette démo ; voir plus bas.) | |
| 56 | + | |
| 57 | +mm exécute les hooks `preAsk` juste avant de lancer l'agent interrogé et les hooks `postAsk` juste après sa réponse, avant que l'agent qui l'a interrogé la lise. `hooks/unload.js` lit dans son entrée le modèle à décharger — `"model": "llmman/<modèle>"` pour `self`, `"ask": {"agentModel": "llmman/<modèle>"}` pour `asked` — et envoie `keep_alive: 0` au serveur. `matcher: ^expert$` limite les deux hooks à ce relais. `onError: warn` : si un déchargement échoue, le relais continue. | |
| 58 | + | |
| 59 | +Les hooks sont en plus certains : un modèle peut sauter une consigne, un hook s'exécute toujours. | |
| 60 | + | |
| 61 | +## Résultats mesurés (2026-10-18, `mm -p`, llmman) | |
| 62 | + | |
| 63 | +Coordinateur Qwen3.5 9B, expert Qwen3.5 4B, les deux hooks actifs. « Jamais ensemble » : `/api/ps` (interrogé deux fois par seconde) n'a jamais montré les deux modèles à la fois pendant la question : 9B → rien → 4B → rien → 9B. | |
| 64 | + | |
| 65 | +| Question | `ask_agent` | `preAsk` décharge le 9B | `postAsk` décharge le 4B | jamais ensemble | Durée | | |
| 66 | +|---|---|---|---|---|---| | |
| 67 | +| How do I reverse a string in Go? | ✓ | ✓ | ✓ | ✓ | 62 s | | |
| 68 | +| What is a closure in JavaScript? | ✓ | ✓ | ✓ | ✓ | 65 s | | |
| 69 | +| How do I read a file line by line in Python? | ✓ | ✓ | ✓ | ✓ | 49 s | | |
| 70 | +| Explain Rust ownership briefly. | ✓ | ✓ | ✓ | ✓ | 82 s | | |
| 71 | +| Comment trier un tableau en Java ? | ✓ | ✓ | ✓ | ✓ | 119 s | | |
| 72 | +| How do I make an HTTP GET request in Go? | ✓ | ✓ | ✓ | ✓ | 106 s | | |
| 73 | +| What does the yield keyword do in Python? (×3) | ✓ ✓ ✓ | ✓ ✓ ✓ | ✓ ✓ ✓ | ✓ ✓ ✓ | 57 s, 60 s, 85 s | | |
| 74 | + | |
| 75 | +9 sur 9. La question en français reçoit une réponse en français. Une salutation (« Hello, who are you? ») reçoit une réponse en 2 s sans aucun outil. Les durées comprennent les chargements : chaque question charge le 9B deux fois et le 4B une fois, et elles varient beaucoup d'une exécution à l'autre. | |
| 76 | + | |
| 77 | +**Le prompt.** Sans l'étape `unload_model`, un prompt court (« 1. Call ask_agent… 2. Then give the expert's answer ») a laissé le coordinateur répondre seul à la question sur `yield`, sans l'expert (1 fois sur 7). Préciser que `ask_agent` est TOUJOURS la première action, « even for a question you know the answer to », a donné le tableau ci-dessus. | |
| 78 | + | |
| 79 | +**Versions précédentes.** Quand le coordinateur libérait l'expert avec `unload_model` (sans hook `postAsk`), le 9B et le 4B étaient chargés ensemble pendant ~18 s par question, le temps que le coordinateur lise la réponse — 7 questions sur 7 sans autre défaut. Avant cela, avec un coordinateur MiniCPM5 2B et un expert Qwen3.5 9B, le coordinateur a sauté `unload_model` une fois sur cinq. | |
| 80 | + | |
| 81 | +## Ce qu'il faut savoir | |
| 82 | + | |
| 83 | +- **`postAsk` s'exécute après chaque relais vers l'expert**, qu'il ait répondu, échoué ou été annulé : le modèle de l'expert est déchargé même quand son tour a échoué. | |
| 84 | +- **La question suivante recharge le 4B** : un relais coûte un chargement de chaque modèle, plus un rechargement du coordinateur. Garder les deux chargés est plus rapide, quand la mémoire le permet. | |
| 85 | +- **llmman décharge aussi tout seul** un modèle inactif depuis cinq minutes (`keep_alive`) ; les hooks libèrent la mémoire tout de suite, ce qui compte quand le modèle suivant en a besoin aussitôt. | |
| 86 | +- **Désactive `/unload` et `unload_model`** avec `unloadModels: false`, comme l'expert : un agent qui ne fait que répondre n'a pas besoin d'un outil de plus. Les hooks ne dépendent pas de cette clé. | |
| new file mode 100644 | |||
| @@ -0,0 +1,86 @@ | |||
| 1 | +# Démo mini-me : décharger des modèles | ||
| 2 | + | ||
| 3 | +*[English version](README.en.md)* | ||
| 4 | + | ||
| 5 | +Un coordinateur confie chaque question de programmation à un expert, et les deux modèles se relaient dans la mémoire du serveur sans jamais y être chargés ensemble : un hook `preAsk` décharge le modèle du coordinateur juste avant le lancement de l'expert, et un hook `postAsk` décharge le modèle de l'expert une fois sa réponse donnée, avant que le coordinateur soit rechargé pour la lire. Les modèles sont servis par [llmman](https://github.com/llmmanorg/llmman). | ||
| 6 | + | ||
| 7 | +## Ce qu'il y a ici | ||
| 8 | + | ||
| 9 | +| Fichier | Rôle | | ||
| 10 | +|---|---| | ||
| 11 | +| `agent.yaml` | Le **coordinateur** (Qwen3.5 9B) : `ask_agent` vers l'expert, puis la réponse. Les deux hooks. `/unload` et l'outil `unload_model` fonctionnent aussi ici. `displayHooks: true` affiche l'exécution des hooks. | | ||
| 12 | +| `hooks/unload.js` | Le hook (Node.js). `unload.js self` (`preAsk`) : décharge le modèle du coordinateur juste avant que mm lance l'expert. `unload.js asked` (`postAsk`) : décharge le modèle de l'expert une fois sa réponse donnée. | | ||
| 13 | +| `agent.expert.yaml` | L'**expert** (Qwen3.5 4B) : répond aux questions de programmation, sans aucun outil (`unloadModels: false`). | | ||
| 14 | + | ||
| 15 | +## Lancer la démo | ||
| 16 | + | ||
| 17 | +```bash | ||
| 18 | +llmman pull hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | ||
| 19 | +llmman pull hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M | ||
| 20 | +cd demos/unload-models | ||
| 21 | +mm -tui agent.yaml | ||
| 22 | +``` | ||
| 23 | + | ||
| 24 | +Pose une question de programmation, puis regarde ce que le serveur garde en mémoire : | ||
| 25 | + | ||
| 26 | +```bash | ||
| 27 | +llmman ps # ou : curl -s http://127.0.0.1:17434/api/ps | ||
| 28 | +``` | ||
| 29 | + | ||
| 30 | +Après la réponse, seul le Qwen3.5 9B du coordinateur est chargé. Dans le terminal, le relais s'affiche ainsi : | ||
| 31 | + | ||
| 32 | +```text | ||
| 33 | +🗣 ask_agent: expert (llmman/hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M) — Explain what a closure is in Ja… | ||
| 34 | + 🪝 1. preAsk expert · node "$MM_CONFIG_DIR/hooks/unload.js" self → continue | ||
| 35 | + 🪝 2. postAsk expert · node "$MM_CONFIG_DIR/hooks/unload.js" asked → continue | ||
| 36 | +``` | ||
| 37 | + | ||
| 38 | +À la main : `/unload` décharge le modèle du coordinateur, `/unload <modèle>` n'importe quel autre modèle du même serveur — dans le terminal, la fenêtre et un éditeur. Demander au coordinateur de libérer la mémoire de l'expert (« Unload the expert's model. », « Libère la mémoire de l'expert. ») lui fait appeler `unload_model` sur le modèle de l'expert (4 sur 4 mesurés) : son nom est écrit dans le prompt, car le modèle ne peut pas deviner comment le serveur nomme un modèle. | ||
| 39 | + | ||
| 40 | +## Comment ça marche | ||
| 41 | + | ||
| 42 | +Ce que le serveur garde en mémoire pendant une question (mesuré avec `/api/ps` deux fois par seconde) : | ||
| 43 | + | ||
| 44 | +| Moment | Chargé | | ||
| 45 | +|---|---| | ||
| 46 | +| le coordinateur lit la question et appelle `ask_agent` | Qwen3.5 9B | | ||
| 47 | +| le hook `preAsk` décharge le coordinateur | — | | ||
| 48 | +| l'expert travaille | Qwen3.5 4B seul | | ||
| 49 | +| le hook `postAsk` décharge l'expert | — | | ||
| 50 | +| le coordinateur lit la réponse (le serveur le recharge) | Qwen3.5 9B seul | | ||
| 51 | + | ||
| 52 | +**Pourquoi des hooks, et pas le modèle.** Les deux déchargements doivent avoir lieu à un moment où le modèle du coordinateur ne tourne pas : | ||
| 53 | + | ||
| 54 | +- *Avant l'expert* : le modèle du coordinateur vient de décider d'appeler `ask_agent`, et il n'en a de nouveau besoin que pour lire la réponse. Le modèle ne peut pas se décharger lui-même — `unload_model` sur son propre modèle ne dure que jusqu'à son étape suivante. | ||
| 55 | +- *Après l'expert* : si le coordinateur appelait `unload_model` sur le modèle de l'expert, il tournerait déjà de nouveau — rechargé à côté du 4B — au moment de le faire. Les deux modèles seraient chargés ensemble pendant toute cette étape. (C'est ce que faisait la version précédente de cette démo ; voir plus bas.) | ||
| 56 | + | ||
| 57 | +mm exécute les hooks `preAsk` juste avant de lancer l'agent interrogé et les hooks `postAsk` juste après sa réponse, avant que l'agent qui l'a interrogé la lise. `hooks/unload.js` lit dans son entrée le modèle à décharger — `"model": "llmman/<modèle>"` pour `self`, `"ask": {"agentModel": "llmman/<modèle>"}` pour `asked` — et envoie `keep_alive: 0` au serveur. `matcher: ^expert$` limite les deux hooks à ce relais. `onError: warn` : si un déchargement échoue, le relais continue. | ||
| 58 | + | ||
| 59 | +Les hooks sont en plus certains : un modèle peut sauter une consigne, un hook s'exécute toujours. | ||
| 60 | + | ||
| 61 | +## Résultats mesurés (2026-10-18, `mm -p`, llmman) | ||
| 62 | + | ||
| 63 | +Coordinateur Qwen3.5 9B, expert Qwen3.5 4B, les deux hooks actifs. « Jamais ensemble » : `/api/ps` (interrogé deux fois par seconde) n'a jamais montré les deux modèles à la fois pendant la question : 9B → rien → 4B → rien → 9B. | ||
| 64 | + | ||
| 65 | +| Question | `ask_agent` | `preAsk` décharge le 9B | `postAsk` décharge le 4B | jamais ensemble | Durée | | ||
| 66 | +|---|---|---|---|---|---| | ||
| 67 | +| How do I reverse a string in Go? | ✓ | ✓ | ✓ | ✓ | 62 s | | ||
| 68 | +| What is a closure in JavaScript? | ✓ | ✓ | ✓ | ✓ | 65 s | | ||
| 69 | +| How do I read a file line by line in Python? | ✓ | ✓ | ✓ | ✓ | 49 s | | ||
| 70 | +| Explain Rust ownership briefly. | ✓ | ✓ | ✓ | ✓ | 82 s | | ||
| 71 | +| Comment trier un tableau en Java ? | ✓ | ✓ | ✓ | ✓ | 119 s | | ||
| 72 | +| How do I make an HTTP GET request in Go? | ✓ | ✓ | ✓ | ✓ | 106 s | | ||
| 73 | +| What does the yield keyword do in Python? (×3) | ✓ ✓ ✓ | ✓ ✓ ✓ | ✓ ✓ ✓ | ✓ ✓ ✓ | 57 s, 60 s, 85 s | | ||
| 74 | + | ||
| 75 | +9 sur 9. La question en français reçoit une réponse en français. Une salutation (« Hello, who are you? ») reçoit une réponse en 2 s sans aucun outil. Les durées comprennent les chargements : chaque question charge le 9B deux fois et le 4B une fois, et elles varient beaucoup d'une exécution à l'autre. | ||
| 76 | + | ||
| 77 | +**Le prompt.** Sans l'étape `unload_model`, un prompt court (« 1. Call ask_agent… 2. Then give the expert's answer ») a laissé le coordinateur répondre seul à la question sur `yield`, sans l'expert (1 fois sur 7). Préciser que `ask_agent` est TOUJOURS la première action, « even for a question you know the answer to », a donné le tableau ci-dessus. | ||
| 78 | + | ||
| 79 | +**Versions précédentes.** Quand le coordinateur libérait l'expert avec `unload_model` (sans hook `postAsk`), le 9B et le 4B étaient chargés ensemble pendant ~18 s par question, le temps que le coordinateur lise la réponse — 7 questions sur 7 sans autre défaut. Avant cela, avec un coordinateur MiniCPM5 2B et un expert Qwen3.5 9B, le coordinateur a sauté `unload_model` une fois sur cinq. | ||
| 80 | + | ||
| 81 | +## Ce qu'il faut savoir | ||
| 82 | + | ||
| 83 | +- **`postAsk` s'exécute après chaque relais vers l'expert**, qu'il ait répondu, échoué ou été annulé : le modèle de l'expert est déchargé même quand son tour a échoué. | ||
| 84 | +- **La question suivante recharge le 4B** : un relais coûte un chargement de chaque modèle, plus un rechargement du coordinateur. Garder les deux chargés est plus rapide, quand la mémoire le permet. | ||
| 85 | +- **llmman décharge aussi tout seul** un modèle inactif depuis cinq minutes (`keep_alive`) ; les hooks libèrent la mémoire tout de suite, ce qui compte quand le modèle suivant en a besoin aussitôt. | ||
| 86 | +- **Désactive `/unload` et `unload_model`** avec `unloadModels: false`, comme l'expert : un agent qui ne fait que répondre n'a pas besoin d'un outil de plus. Les hooks ne dépendent pas de cette clé. | ||
added
demos/unload-models/agent.expert.yaml +37 -0 | new file mode 100644 | ||
| @@ -0,0 +1,37 @@ | ||
| 1 | +# The expert of demos/unload-models: answers programming questions. Asked by | |
| 2 | +# agent.yaml through ask_agent; its model is unloaded after each answer. | |
| 3 | + | |
| 4 | +provider: llmman | |
| 5 | +model: hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M | |
| 6 | +baseUrl: http://127.0.0.1:17434/v1 | |
| 7 | +fallback: http://host.docker.internal:17434/v1 | |
| 8 | + | |
| 9 | +sampling: | |
| 10 | + temperature: 0.2 | |
| 11 | + max_tokens: 2048 | |
| 12 | + chat_template_kwargs: | |
| 13 | + enable_thinking: false | |
| 14 | + | |
| 15 | +watchdogTimeout: 5m | |
| 16 | +firstTokenTimeout: 3m | |
| 17 | +maxTurns: 2 | |
| 18 | + | |
| 19 | +# The expert only answers: no tools at all, unload_model included. | |
| 20 | +unloadModels: false | |
| 21 | +bashTool: false | |
| 22 | +editTools: false | |
| 23 | +planTool: false | |
| 24 | +fetchContentTool: false | |
| 25 | +askUserTool: false | |
| 26 | +skillsDir: no-skills | |
| 27 | + | |
| 28 | +agentsMd: | |
| 29 | + enabled: false | |
| 30 | + | |
| 31 | +acp: | |
| 32 | + name: expert | |
| 33 | + title: Programming expert | |
| 34 | + | |
| 35 | +system: | | |
| 36 | + You are a programming expert. Answer the question clearly and briefly: | |
| 37 | + a short explanation, then one short code example when it helps. | |
| new file mode 100644 | |||
| @@ -0,0 +1,37 @@ | |||
| 1 | +# The expert of demos/unload-models: answers programming questions. Asked by | ||
| 2 | +# agent.yaml through ask_agent; its model is unloaded after each answer. | ||
| 3 | + | ||
| 4 | +provider: llmman | ||
| 5 | +model: hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M | ||
| 6 | +baseUrl: http://127.0.0.1:17434/v1 | ||
| 7 | +fallback: http://host.docker.internal:17434/v1 | ||
| 8 | + | ||
| 9 | +sampling: | ||
| 10 | + temperature: 0.2 | ||
| 11 | + max_tokens: 2048 | ||
| 12 | + chat_template_kwargs: | ||
| 13 | + enable_thinking: false | ||
| 14 | + | ||
| 15 | +watchdogTimeout: 5m | ||
| 16 | +firstTokenTimeout: 3m | ||
| 17 | +maxTurns: 2 | ||
| 18 | + | ||
| 19 | +# The expert only answers: no tools at all, unload_model included. | ||
| 20 | +unloadModels: false | ||
| 21 | +bashTool: false | ||
| 22 | +editTools: false | ||
| 23 | +planTool: false | ||
| 24 | +fetchContentTool: false | ||
| 25 | +askUserTool: false | ||
| 26 | +skillsDir: no-skills | ||
| 27 | + | ||
| 28 | +agentsMd: | ||
| 29 | + enabled: false | ||
| 30 | + | ||
| 31 | +acp: | ||
| 32 | + name: expert | ||
| 33 | + title: Programming expert | ||
| 34 | + | ||
| 35 | +system: | | ||
| 36 | + You are a programming expert. Answer the question clearly and briefly: | ||
| 37 | + a short explanation, then one short code example when it helps. | ||
added
demos/unload-models/agent.yaml +108 -0 | new file mode 100644 | ||
| @@ -0,0 +1,108 @@ | ||
| 1 | +# Unload models — a coordinator hands each programming question to an expert, | |
| 2 | +# and the two models take turns in memory, never loaded together: | |
| 3 | +# | |
| 4 | +# hooks/unload.js self preAsk unloads the COORDINATOR's model right | |
| 5 | +# before the expert starts (only mm can do | |
| 6 | +# it at that moment) | |
| 7 | +# hooks/unload.js asked postAsk unloads the EXPERT's model once it has | |
| 8 | +# answered, before the coordinator is | |
| 9 | +# reloaded to read the answer | |
| 10 | +# | |
| 11 | +# coordinator Qwen3.5 9B hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M (this file) | |
| 12 | +# expert Qwen3.5 4B hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M (agent.expert.yaml) | |
| 13 | +# | |
| 14 | +# llmman pull hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | |
| 15 | +# llmman pull hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M | |
| 16 | +# llmman serve | |
| 17 | +# | |
| 18 | +# cd demos/unload-models | |
| 19 | +# mm -tui agent.yaml or mm -config agent.yaml -p "…" | |
| 20 | +# | |
| 21 | +# Watch the server between questions: llmman ps (or curl :17434/api/ps). | |
| 22 | +# `/unload` unloads the coordinator's own model by hand, `/unload <model>` | |
| 23 | +# any other one — in the terminal, the window and an editor. | |
| 24 | + | |
| 25 | +provider: llmman | |
| 26 | +model: hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | |
| 27 | +baseUrl: http://127.0.0.1:17434/v1 | |
| 28 | +fallback: http://host.docker.internal:17434/v1 | |
| 29 | + | |
| 30 | +mmConfigs: | |
| 31 | + expert: | |
| 32 | + path: agent.expert.yaml | |
| 33 | + description: >- | |
| 34 | + Programming expert. Answers any question about code, languages, | |
| 35 | + tools and how to do something in a program. | |
| 36 | + yolo: true | |
| 37 | + | |
| 38 | +# /unload and unload_model are on by default (unloadModels: true); written | |
| 39 | +# here because this demo is about them. The hooks do the hand-off's unloads: | |
| 40 | +# a step of the model would come too late (the coordinator is already | |
| 41 | +# reloaded beside the expert) and a small model may skip it. unload_model | |
| 42 | +# stays for when the user asks to free a model. | |
| 43 | +unloadModels: true | |
| 44 | + | |
| 45 | +sampling: | |
| 46 | + # Low temperature: routing wants a steady hand. | |
| 47 | + temperature: 0.4 | |
| 48 | + top_p: 0.8 | |
| 49 | + top_k: 20 | |
| 50 | + max_tokens: 2048 | |
| 51 | + parallel_tool_calls: false | |
| 52 | + chat_template_kwargs: | |
| 53 | + enable_thinking: false | |
| 54 | + | |
| 55 | +# During ask_agent every event of the expert counts as a sign of life, but | |
| 56 | +# loading a 9B model is silent for a while. | |
| 57 | +watchdogTimeout: 5m | |
| 58 | +firstTokenTimeout: 3m | |
| 59 | + | |
| 60 | +# ask the expert, answer: two steps, one spare. | |
| 61 | +maxTurns: 3 | |
| 62 | +maxOutput: 20000 | |
| 63 | +displayCommands: true | |
| 64 | +displayHooks: true | |
| 65 | + | |
| 66 | +hooks: | |
| 67 | + # onError: warn — an unload is a saving, not a check: the hand-off goes on. | |
| 68 | + preAsk: | |
| 69 | + - matcher: ^expert$ | |
| 70 | + command: node "$MM_CONFIG_DIR/hooks/unload.js" self | |
| 71 | + timeout: 30s | |
| 72 | + onError: warn | |
| 73 | + postAsk: | |
| 74 | + - matcher: ^expert$ | |
| 75 | + command: node "$MM_CONFIG_DIR/hooks/unload.js" asked | |
| 76 | + timeout: 30s | |
| 77 | + onError: warn | |
| 78 | + | |
| 79 | +yolo: true | |
| 80 | +bashTool: false | |
| 81 | +editTools: false | |
| 82 | +planTool: false | |
| 83 | +fetchContentTool: false | |
| 84 | +askUserTool: false | |
| 85 | +skillsDir: no-skills | |
| 86 | + | |
| 87 | +agentsMd: | |
| 88 | + enabled: false | |
| 89 | + | |
| 90 | +acp: | |
| 91 | + name: unloader | |
| 92 | + title: 🧹 Unload models | |
| 93 | + | |
| 94 | +system: | | |
| 95 | + You are a coordinator. You do not answer programming questions yourself. | |
| 96 | + | |
| 97 | + For every question about programming, even one you know the answer to, | |
| 98 | + do these two steps, in order: | |
| 99 | + 1. Call ask_agent with agent "expert" and the user's question. It is | |
| 100 | + ALWAYS your first action: never answer a programming question before | |
| 101 | + the expert has answered it. | |
| 102 | + 2. Then give the expert's answer to the user, in the user's language. | |
| 103 | + | |
| 104 | + If the user asks you to free or unload a model, call unload_model. The | |
| 105 | + expert's model is "hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M". | |
| 106 | + | |
| 107 | + For anything else (a greeting, a question about you), answer in one short | |
| 108 | + sentence, without any tool. | |
| new file mode 100644 | |||
| @@ -0,0 +1,108 @@ | |||
| 1 | +# Unload models — a coordinator hands each programming question to an expert, | ||
| 2 | +# and the two models take turns in memory, never loaded together: | ||
| 3 | +# | ||
| 4 | +# hooks/unload.js self preAsk unloads the COORDINATOR's model right | ||
| 5 | +# before the expert starts (only mm can do | ||
| 6 | +# it at that moment) | ||
| 7 | +# hooks/unload.js asked postAsk unloads the EXPERT's model once it has | ||
| 8 | +# answered, before the coordinator is | ||
| 9 | +# reloaded to read the answer | ||
| 10 | +# | ||
| 11 | +# coordinator Qwen3.5 9B hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M (this file) | ||
| 12 | +# expert Qwen3.5 4B hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M (agent.expert.yaml) | ||
| 13 | +# | ||
| 14 | +# llmman pull hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | ||
| 15 | +# llmman pull hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M | ||
| 16 | +# llmman serve | ||
| 17 | +# | ||
| 18 | +# cd demos/unload-models | ||
| 19 | +# mm -tui agent.yaml or mm -config agent.yaml -p "…" | ||
| 20 | +# | ||
| 21 | +# Watch the server between questions: llmman ps (or curl :17434/api/ps). | ||
| 22 | +# `/unload` unloads the coordinator's own model by hand, `/unload <model>` | ||
| 23 | +# any other one — in the terminal, the window and an editor. | ||
| 24 | + | ||
| 25 | +provider: llmman | ||
| 26 | +model: hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | ||
| 27 | +baseUrl: http://127.0.0.1:17434/v1 | ||
| 28 | +fallback: http://host.docker.internal:17434/v1 | ||
| 29 | + | ||
| 30 | +mmConfigs: | ||
| 31 | + expert: | ||
| 32 | + path: agent.expert.yaml | ||
| 33 | + description: >- | ||
| 34 | + Programming expert. Answers any question about code, languages, | ||
| 35 | + tools and how to do something in a program. | ||
| 36 | + yolo: true | ||
| 37 | + | ||
| 38 | +# /unload and unload_model are on by default (unloadModels: true); written | ||
| 39 | +# here because this demo is about them. The hooks do the hand-off's unloads: | ||
| 40 | +# a step of the model would come too late (the coordinator is already | ||
| 41 | +# reloaded beside the expert) and a small model may skip it. unload_model | ||
| 42 | +# stays for when the user asks to free a model. | ||
| 43 | +unloadModels: true | ||
| 44 | + | ||
| 45 | +sampling: | ||
| 46 | + # Low temperature: routing wants a steady hand. | ||
| 47 | + temperature: 0.4 | ||
| 48 | + top_p: 0.8 | ||
| 49 | + top_k: 20 | ||
| 50 | + max_tokens: 2048 | ||
| 51 | + parallel_tool_calls: false | ||
| 52 | + chat_template_kwargs: | ||
| 53 | + enable_thinking: false | ||
| 54 | + | ||
| 55 | +# During ask_agent every event of the expert counts as a sign of life, but | ||
| 56 | +# loading a 9B model is silent for a while. | ||
| 57 | +watchdogTimeout: 5m | ||
| 58 | +firstTokenTimeout: 3m | ||
| 59 | + | ||
| 60 | +# ask the expert, answer: two steps, one spare. | ||
| 61 | +maxTurns: 3 | ||
| 62 | +maxOutput: 20000 | ||
| 63 | +displayCommands: true | ||
| 64 | +displayHooks: true | ||
| 65 | + | ||
| 66 | +hooks: | ||
| 67 | + # onError: warn — an unload is a saving, not a check: the hand-off goes on. | ||
| 68 | + preAsk: | ||
| 69 | + - matcher: ^expert$ | ||
| 70 | + command: node "$MM_CONFIG_DIR/hooks/unload.js" self | ||
| 71 | + timeout: 30s | ||
| 72 | + onError: warn | ||
| 73 | + postAsk: | ||
| 74 | + - matcher: ^expert$ | ||
| 75 | + command: node "$MM_CONFIG_DIR/hooks/unload.js" asked | ||
| 76 | + timeout: 30s | ||
| 77 | + onError: warn | ||
| 78 | + | ||
| 79 | +yolo: true | ||
| 80 | +bashTool: false | ||
| 81 | +editTools: false | ||
| 82 | +planTool: false | ||
| 83 | +fetchContentTool: false | ||
| 84 | +askUserTool: false | ||
| 85 | +skillsDir: no-skills | ||
| 86 | + | ||
| 87 | +agentsMd: | ||
| 88 | + enabled: false | ||
| 89 | + | ||
| 90 | +acp: | ||
| 91 | + name: unloader | ||
| 92 | + title: 🧹 Unload models | ||
| 93 | + | ||
| 94 | +system: | | ||
| 95 | + You are a coordinator. You do not answer programming questions yourself. | ||
| 96 | + | ||
| 97 | + For every question about programming, even one you know the answer to, | ||
| 98 | + do these two steps, in order: | ||
| 99 | + 1. Call ask_agent with agent "expert" and the user's question. It is | ||
| 100 | + ALWAYS your first action: never answer a programming question before | ||
| 101 | + the expert has answered it. | ||
| 102 | + 2. Then give the expert's answer to the user, in the user's language. | ||
| 103 | + | ||
| 104 | + If the user asks you to free or unload a model, call unload_model. The | ||
| 105 | + expert's model is "hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M". | ||
| 106 | + | ||
| 107 | + For anything else (a greeting, a question about you), answer in one short | ||
| 108 | + sentence, without any tool. | ||
added
demos/unload-models/hooks/unload.js +58 -0 | new file mode 100644 | ||
| @@ -0,0 +1,58 @@ | ||
| 1 | +#!/usr/bin/env node | |
| 2 | +// preAsk / postAsk hook: unloads one model so that two models are never in | |
| 3 | +// memory together during a hand-off. | |
| 4 | +// | |
| 5 | +// node unload.js self preAsk: the ASKING agent's own model, right before | |
| 6 | +// mm starts the asked agent. Only mm can do it at | |
| 7 | +// that moment: the model has just decided to call | |
| 8 | +// ask_agent, and it is needed again only to read the | |
| 9 | +// answer — the server reloads it then by itself. The | |
| 10 | +// model cannot do it (unload_model on its own model | |
| 11 | +// lasts until its next step). | |
| 12 | +// node unload.js asked postAsk: the ASKED agent's model, once it has | |
| 13 | +// answered and before the asking agent reads the | |
| 14 | +// answer — so the asker's model reloads alone. | |
| 15 | +// | |
| 16 | +// Input: {"model": "llmman/<model>", "ask": {"agentModel": "llmman/<model>"}, | |
| 17 | +// ...}. Server: $LLMMAN_URL, else llmman's default address, else the one a | |
| 18 | +// Docker sandbox reaches the host by. Prints nothing: the hand-off goes on | |
| 19 | +// whatever happens (onError: warn in agent.yaml), an unload is a saving, not | |
| 20 | +// a check. | |
| 21 | + | |
| 22 | +const fs = require("fs"); | |
| 23 | + | |
| 24 | +const input = JSON.parse(fs.readFileSync(0, "utf8")); | |
| 25 | +const which = process.argv[2]; | |
| 26 | +const ref = which === "asked" ? input.ask && input.ask.agentModel : input.model; | |
| 27 | +if (!ref) { | |
| 28 | + console.error(`unload.js ${which}: no model in the input`); | |
| 29 | + process.exit(1); | |
| 30 | +} | |
| 31 | +const model = ref.replace(/^[^/]+\//, ""); // drop the "llmman/" provider prefix | |
| 32 | + | |
| 33 | +const servers = process.env.LLMMAN_URL | |
| 34 | + ? [process.env.LLMMAN_URL] | |
| 35 | + : ["http://127.0.0.1:17434", "http://host.docker.internal:17434"]; | |
| 36 | + | |
| 37 | +async function unload() { | |
| 38 | + for (const server of servers) { | |
| 39 | + try { | |
| 40 | + const resp = await fetch(server + "/api/generate", { | |
| 41 | + method: "POST", | |
| 42 | + headers: { "Content-Type": "application/json" }, | |
| 43 | + body: JSON.stringify({ model, keep_alive: 0 }), | |
| 44 | + signal: AbortSignal.timeout(20_000), | |
| 45 | + }); | |
| 46 | + const reply = await resp.json(); | |
| 47 | + if (reply.done_reason === "unload") return; | |
| 48 | + throw new Error(`${server}: not confirmed: ${JSON.stringify(reply)}`); | |
| 49 | + } catch (err) { | |
| 50 | + if (server === servers[servers.length - 1]) throw err; | |
| 51 | + } | |
| 52 | + } | |
| 53 | +} | |
| 54 | + | |
| 55 | +unload().catch((err) => { | |
| 56 | + console.error(`could not unload ${model}: ${err.message}`); | |
| 57 | + process.exit(1); | |
| 58 | +}); | |
| new file mode 100644 | |||
| @@ -0,0 +1,58 @@ | |||
| 1 | +#!/usr/bin/env node | ||
| 2 | +// preAsk / postAsk hook: unloads one model so that two models are never in | ||
| 3 | +// memory together during a hand-off. | ||
| 4 | +// | ||
| 5 | +// node unload.js self preAsk: the ASKING agent's own model, right before | ||
| 6 | +// mm starts the asked agent. Only mm can do it at | ||
| 7 | +// that moment: the model has just decided to call | ||
| 8 | +// ask_agent, and it is needed again only to read the | ||
| 9 | +// answer — the server reloads it then by itself. The | ||
| 10 | +// model cannot do it (unload_model on its own model | ||
| 11 | +// lasts until its next step). | ||
| 12 | +// node unload.js asked postAsk: the ASKED agent's model, once it has | ||
| 13 | +// answered and before the asking agent reads the | ||
| 14 | +// answer — so the asker's model reloads alone. | ||
| 15 | +// | ||
| 16 | +// Input: {"model": "llmman/<model>", "ask": {"agentModel": "llmman/<model>"}, | ||
| 17 | +// ...}. Server: $LLMMAN_URL, else llmman's default address, else the one a | ||
| 18 | +// Docker sandbox reaches the host by. Prints nothing: the hand-off goes on | ||
| 19 | +// whatever happens (onError: warn in agent.yaml), an unload is a saving, not | ||
| 20 | +// a check. | ||
| 21 | + | ||
| 22 | +const fs = require("fs"); | ||
| 23 | + | ||
| 24 | +const input = JSON.parse(fs.readFileSync(0, "utf8")); | ||
| 25 | +const which = process.argv[2]; | ||
| 26 | +const ref = which === "asked" ? input.ask && input.ask.agentModel : input.model; | ||
| 27 | +if (!ref) { | ||
| 28 | + console.error(`unload.js ${which}: no model in the input`); | ||
| 29 | + process.exit(1); | ||
| 30 | +} | ||
| 31 | +const model = ref.replace(/^[^/]+\//, ""); // drop the "llmman/" provider prefix | ||
| 32 | + | ||
| 33 | +const servers = process.env.LLMMAN_URL | ||
| 34 | + ? [process.env.LLMMAN_URL] | ||
| 35 | + : ["http://127.0.0.1:17434", "http://host.docker.internal:17434"]; | ||
| 36 | + | ||
| 37 | +async function unload() { | ||
| 38 | + for (const server of servers) { | ||
| 39 | + try { | ||
| 40 | + const resp = await fetch(server + "/api/generate", { | ||
| 41 | + method: "POST", | ||
| 42 | + headers: { "Content-Type": "application/json" }, | ||
| 43 | + body: JSON.stringify({ model, keep_alive: 0 }), | ||
| 44 | + signal: AbortSignal.timeout(20_000), | ||
| 45 | + }); | ||
| 46 | + const reply = await resp.json(); | ||
| 47 | + if (reply.done_reason === "unload") return; | ||
| 48 | + throw new Error(`${server}: not confirmed: ${JSON.stringify(reply)}`); | ||
| 49 | + } catch (err) { | ||
| 50 | + if (server === servers[servers.length - 1]) throw err; | ||
| 51 | + } | ||
| 52 | + } | ||
| 53 | +} | ||
| 54 | + | ||
| 55 | +unload().catch((err) => { | ||
| 56 | + console.error(`could not unload ${model}: ${err.message}`); | ||
| 57 | + process.exit(1); | ||
| 58 | +}); | ||
modified
docs/en/README.md +2 -1 | @@ -27,6 +27,7 @@ This documentation is organised along the four [Diátaxis](https://diataxis.fr) | ||
| 27 | 27 | - [Add a skill](how-to/add-a-skill.md) |
| 28 | 28 | - [Declare a custom tool](how-to/declare-a-custom-tool.md) |
| 29 | 29 | - [Save an answer to a file](how-to/save-an-answer.md) |
| 30 | +- [Free a model's memory](how-to/unload-a-model.md) — `/unload [model]` in every front end, the `unload_model` tool after another agent's answer, `unloadModels` | |
| 30 | 31 | - [Measure whether the model picks the right skill](how-to/measure-skill-routing.md) — the routing eval: run it, read its table, use it as a before-and-after |
| 31 | 32 | - [Keep a long session inside the context window](how-to/manage-the-context-window.md) |
| 32 | 33 | - [Start a new session without restarting](how-to/start-a-new-session.md) |
| @@ -51,7 +52,7 @@ This documentation is organised along the four [Diátaxis](https://diataxis.fr) | ||
| 51 | 52 | - [Command line](reference/cli.md) — flags, environment variables, start-up banner |
| 52 | 53 | - [Slash commands](reference/slash-commands.md) — `/quit`, `/abort`, `/compact`, `/new`, `/sessions`, `/resume`, `/help`, keyboard shortcuts, window keys |
| 53 | 54 | - [Configuration file](reference/configuration.md) — every YAML key, its default and its validation rule |
| 54 | -- [Built-in tools](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`, `lsp`, `fetch_content`, `ask_agent`, `ask_user`, `save_answer`, declared tools | |
| 55 | +- [Built-in tools](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`, `lsp`, `fetch_content`, `ask_agent`, `ask_user`, `save_answer`, `unload_model`, declared tools | |
| 55 | 56 | - [Skill file format](reference/skill-format.md) |
| 56 | 57 | - [Hooks](reference/hooks.md) — the six events, the JSON a hook reads and writes, `matcher`, `timeout`, `onError`, validation errors |
| 57 | 58 | - [Agent Client Protocol surface](reference/acp.md) |
| @@ -27,6 +27,7 @@ This documentation is organised along the four [Diátaxis](https://diataxis.fr) | |||
| 27 | - [Add a skill](how-to/add-a-skill.md) | 27 | - [Add a skill](how-to/add-a-skill.md) |
| 28 | - [Declare a custom tool](how-to/declare-a-custom-tool.md) | 28 | - [Declare a custom tool](how-to/declare-a-custom-tool.md) |
| 29 | - [Save an answer to a file](how-to/save-an-answer.md) | 29 | - [Save an answer to a file](how-to/save-an-answer.md) |
| 30 | +- [Free a model's memory](how-to/unload-a-model.md) — `/unload [model]` in every front end, the `unload_model` tool after another agent's answer, `unloadModels` | ||
| 30 | - [Measure whether the model picks the right skill](how-to/measure-skill-routing.md) — the routing eval: run it, read its table, use it as a before-and-after | 31 | - [Measure whether the model picks the right skill](how-to/measure-skill-routing.md) — the routing eval: run it, read its table, use it as a before-and-after |
| 31 | - [Keep a long session inside the context window](how-to/manage-the-context-window.md) | 32 | - [Keep a long session inside the context window](how-to/manage-the-context-window.md) |
| 32 | - [Start a new session without restarting](how-to/start-a-new-session.md) | 33 | - [Start a new session without restarting](how-to/start-a-new-session.md) |
| @@ -51,7 +52,7 @@ This documentation is organised along the four [Diátaxis](https://diataxis.fr) | |||
| 51 | - [Command line](reference/cli.md) — flags, environment variables, start-up banner | 52 | - [Command line](reference/cli.md) — flags, environment variables, start-up banner |
| 52 | - [Slash commands](reference/slash-commands.md) — `/quit`, `/abort`, `/compact`, `/new`, `/sessions`, `/resume`, `/help`, keyboard shortcuts, window keys | 53 | - [Slash commands](reference/slash-commands.md) — `/quit`, `/abort`, `/compact`, `/new`, `/sessions`, `/resume`, `/help`, keyboard shortcuts, window keys |
| 53 | - [Configuration file](reference/configuration.md) — every YAML key, its default and its validation rule | 54 | - [Configuration file](reference/configuration.md) — every YAML key, its default and its validation rule |
| 54 | -- [Built-in tools](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`, `lsp`, `fetch_content`, `ask_agent`, `ask_user`, `save_answer`, declared tools | 55 | +- [Built-in tools](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`, `lsp`, `fetch_content`, `ask_agent`, `ask_user`, `save_answer`, `unload_model`, declared tools |
| 55 | - [Skill file format](reference/skill-format.md) | 56 | - [Skill file format](reference/skill-format.md) |
| 56 | - [Hooks](reference/hooks.md) — the six events, the JSON a hook reads and writes, `matcher`, `timeout`, `onError`, validation errors | 57 | - [Hooks](reference/hooks.md) — the six events, the JSON a hook reads and writes, `matcher`, `timeout`, `onError`, validation errors |
| 57 | - [Agent Client Protocol surface](reference/acp.md) | 58 | - [Agent Client Protocol surface](reference/acp.md) |
added
docs/en/how-to/unload-a-model.md +58 -0 | new file mode 100644 | ||
| @@ -0,0 +1,58 @@ | ||
| 1 | +# How to free a model's memory | |
| 2 | + | |
| 3 | +This guide shows how to unload a model from the model server's memory without stopping the server: by hand with `/unload`, or by the agent itself with its `unload_model` tool — typically to free the memory of another agent's model once its answer is in. It assumes mm runs on llmman, Ollama, or llama-server in router mode. The model reloads by itself the next time something uses it. | |
| 4 | + | |
| 5 | +## Unload by hand | |
| 6 | + | |
| 7 | +Type, in the terminal, the window or an editor: | |
| 8 | + | |
| 9 | +```text | |
| 10 | +/unload | |
| 11 | +``` | |
| 12 | + | |
| 13 | +The agent's own model is unloaded: | |
| 14 | + | |
| 15 | +```text | |
| 16 | +🧹 unloaded hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | |
| 17 | +``` | |
| 18 | + | |
| 19 | +Name another model served by the same server to unload that one instead: | |
| 20 | + | |
| 21 | +```text | |
| 22 | +/unload hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | |
| 23 | +``` | |
| 24 | + | |
| 25 | +A model that is not loaded gives `🧹 <model> was not loaded: nothing to unload`; nothing is sent to the server, so nothing is loaded either. Check what is loaded with `llmman ps`, `ollama ps`, or `curl <server>/api/ps`. | |
| 26 | + | |
| 27 | +## Let the agent unload a model | |
| 28 | + | |
| 29 | +The `unload_model` tool is declared by default. Tell the model when to use it, in its `system` prompt. The case it serves best is another agent's model, once that agent has answered: | |
| 30 | + | |
| 31 | +```yaml | |
| 32 | +mmConfigs: | |
| 33 | + expert: | |
| 34 | + path: agent.expert.yaml | |
| 35 | + description: Programming expert. | |
| 36 | + | |
| 37 | +system: | | |
| 38 | + For every question about programming, do these three steps, in order: | |
| 39 | + 1. Call ask_agent with agent "expert" and the user's question. | |
| 40 | + 2. Call unload_model with model "hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M". | |
| 41 | + It is ALWAYS your next action after ask_agent: never answer the user | |
| 42 | + before you have called unload_model. | |
| 43 | + 3. Then give the expert's answer to the user. | |
| 44 | +``` | |
| 45 | + | |
| 46 | +Write the model's name exactly as the server lists it: the model cannot guess it. `demos/unload-models/` does the expert's unload with a `postAsk` hook instead (see the variants below); its coordinator still calls `unload_model` when the user asks it to free the expert's memory. | |
| 47 | + | |
| 48 | +## Variants | |
| 49 | + | |
| 50 | +- **The agent's own model.** `unload_model` with an empty `model` unloads it, but only until the agent's next step: the agent needs its model to read the tool's result, and the server reloads it. Use `/unload` between two questions instead — or, to free it while another agent works, a `preAsk` hook that unloads the model named in its input (`"model": "<provider>/<model>"`) right before the asked agent starts: `demos/unload-models/hooks/unload.js self`. | |
| 51 | +- **A guarantee instead of an instruction.** A small model may skip the step (measured: 4 times out of 5 with MiniCPM5 2B in the first version of `demos/unload-models`), and the agent's own model is already reloaded beside the other one when it calls the tool. For a certain unload, with the two models never loaded together, have a `postAsk` hook call the server itself — `curl -s <server>/api/generate -d '{"model":"…","keep_alive":0}'` — once the asked agent has answered and before the asking agent reads the answer: `demos/unload-models/hooks/unload.js asked` reads the asked agent's model from `ask.agentModel`. See [Control the hand-offs with hooks](control-agent-calls-with-hooks.md). | |
| 52 | +- **llama-server.** Only router mode (started without `-m`) can unload a model. A single-model server answers with the alternatives: `--sleep-idle-seconds N` frees the memory after N idle seconds, or stop the server. | |
| 53 | +- **Turn it off.** `unloadModels: false` removes both `/unload` and the tool — for an agent that must not have one more tool, such as a guard or a narrow specialist. | |
| 54 | + | |
| 55 | +## See also | |
| 56 | + | |
| 57 | +- [`unload_model` reference](../reference/tools.md#unload_model) and [`/unload`](../reference/slash-commands.md#commands) | |
| 58 | +- [`unloadModels`](../reference/configuration.md) | |
| new file mode 100644 | |||
| @@ -0,0 +1,58 @@ | |||
| 1 | +# How to free a model's memory | ||
| 2 | + | ||
| 3 | +This guide shows how to unload a model from the model server's memory without stopping the server: by hand with `/unload`, or by the agent itself with its `unload_model` tool — typically to free the memory of another agent's model once its answer is in. It assumes mm runs on llmman, Ollama, or llama-server in router mode. The model reloads by itself the next time something uses it. | ||
| 4 | + | ||
| 5 | +## Unload by hand | ||
| 6 | + | ||
| 7 | +Type, in the terminal, the window or an editor: | ||
| 8 | + | ||
| 9 | +```text | ||
| 10 | +/unload | ||
| 11 | +``` | ||
| 12 | + | ||
| 13 | +The agent's own model is unloaded: | ||
| 14 | + | ||
| 15 | +```text | ||
| 16 | +🧹 unloaded hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | ||
| 17 | +``` | ||
| 18 | + | ||
| 19 | +Name another model served by the same server to unload that one instead: | ||
| 20 | + | ||
| 21 | +```text | ||
| 22 | +/unload hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | ||
| 23 | +``` | ||
| 24 | + | ||
| 25 | +A model that is not loaded gives `🧹 <model> was not loaded: nothing to unload`; nothing is sent to the server, so nothing is loaded either. Check what is loaded with `llmman ps`, `ollama ps`, or `curl <server>/api/ps`. | ||
| 26 | + | ||
| 27 | +## Let the agent unload a model | ||
| 28 | + | ||
| 29 | +The `unload_model` tool is declared by default. Tell the model when to use it, in its `system` prompt. The case it serves best is another agent's model, once that agent has answered: | ||
| 30 | + | ||
| 31 | +```yaml | ||
| 32 | +mmConfigs: | ||
| 33 | + expert: | ||
| 34 | + path: agent.expert.yaml | ||
| 35 | + description: Programming expert. | ||
| 36 | + | ||
| 37 | +system: | | ||
| 38 | + For every question about programming, do these three steps, in order: | ||
| 39 | + 1. Call ask_agent with agent "expert" and the user's question. | ||
| 40 | + 2. Call unload_model with model "hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M". | ||
| 41 | + It is ALWAYS your next action after ask_agent: never answer the user | ||
| 42 | + before you have called unload_model. | ||
| 43 | + 3. Then give the expert's answer to the user. | ||
| 44 | +``` | ||
| 45 | + | ||
| 46 | +Write the model's name exactly as the server lists it: the model cannot guess it. `demos/unload-models/` does the expert's unload with a `postAsk` hook instead (see the variants below); its coordinator still calls `unload_model` when the user asks it to free the expert's memory. | ||
| 47 | + | ||
| 48 | +## Variants | ||
| 49 | + | ||
| 50 | +- **The agent's own model.** `unload_model` with an empty `model` unloads it, but only until the agent's next step: the agent needs its model to read the tool's result, and the server reloads it. Use `/unload` between two questions instead — or, to free it while another agent works, a `preAsk` hook that unloads the model named in its input (`"model": "<provider>/<model>"`) right before the asked agent starts: `demos/unload-models/hooks/unload.js self`. | ||
| 51 | +- **A guarantee instead of an instruction.** A small model may skip the step (measured: 4 times out of 5 with MiniCPM5 2B in the first version of `demos/unload-models`), and the agent's own model is already reloaded beside the other one when it calls the tool. For a certain unload, with the two models never loaded together, have a `postAsk` hook call the server itself — `curl -s <server>/api/generate -d '{"model":"…","keep_alive":0}'` — once the asked agent has answered and before the asking agent reads the answer: `demos/unload-models/hooks/unload.js asked` reads the asked agent's model from `ask.agentModel`. See [Control the hand-offs with hooks](control-agent-calls-with-hooks.md). | ||
| 52 | +- **llama-server.** Only router mode (started without `-m`) can unload a model. A single-model server answers with the alternatives: `--sleep-idle-seconds N` frees the memory after N idle seconds, or stop the server. | ||
| 53 | +- **Turn it off.** `unloadModels: false` removes both `/unload` and the tool — for an agent that must not have one more tool, such as a guard or a narrow specialist. | ||
| 54 | + | ||
| 55 | +## See also | ||
| 56 | + | ||
| 57 | +- [`unload_model` reference](../reference/tools.md#unload_model) and [`/unload`](../reference/slash-commands.md#commands) | ||
| 58 | +- [`unloadModels`](../reference/configuration.md) | ||
modified
docs/en/reference/configuration.md +2 -1 | @@ -26,6 +26,7 @@ | ||
| 26 | 26 | | `fetchContentTool` | bool | `false` | Declare `fetch_content`, which fetches web pages as Markdown. Off by default: it is the only built-in tool that reaches the network. | |
| 27 | 27 | | `askUserTool` | bool | `true` | Declare `ask_user`, through which the model asks the user questions (pick one, pick several, type a text, yes or no) and waits for the answers. `false` removes it: the model cannot ask. Never declared beside a `structuredOutput`. Overridden by `AGENT_ASK_USER`; for an agent asked by another one, also by the asker's `mmConfigs.<name>.askUser`. See [tools](tools.md#ask_user). | |
| 28 | 28 | | `saveAnswerTool` | bool | `false` | Declare `save_answer`, which writes an answer the model already gave to a new file: the model sends a file name, not the text again. Off by default. The `/save` command exists either way. See [tools](tools.md#save_answer). | |
| 29 | +| `unloadModels` | bool | `true` | The `/unload [model]` command (terminal, window, editor) and the `unload_model` tool, which free a model from the server's memory: llmman and Ollama, llama-server in router mode. `false` removes both. The tool is never declared beside a `structuredOutput`. See [tools](tools.md#unload_model) and [How to free a model's memory](../how-to/unload-a-model.md). | | |
| 29 | 30 | | `sampling` | map | `temperature: 0.0`, `top_p: 0.9`, `max_tokens: 4096` | Generation settings, OpenAI API key names, passed to the server. A key the OpenAI API does not declare is sent as it is, at the top level of the request — for example `chat_template_kwargs: {enable_thinking: false}`, which tells a llama.cpp server (`dmr`, `llamacpp`, `llmman`) not to reason; `reasoning_effort: none` does the same on Ollama. `messages`, `tools`, `tool_choice` (and the legacy `functions`, `function_call`) are ignored: Genkit builds them from the conversation. The shipped `agent.yaml` adds `parallel_tool_calls: false`. `max_tokens` also bounds one tool call: a `write_file` of a long document (or a thinking model's reasoning plus the call) can hit it, and the call is then cut off. mm hands it back to the model twice, asking for smaller pieces, then stops with `the model hit max_tokens (N) while writing a tool call`; raise the value if that happens. With streaming, mm also asks the server for token counts (`stream_options.include_usage`). | |
| 30 | 31 | | `watchdogTimeout` | duration | `20s` | Silence tolerated between two streamed tokens before the connection is declared hung. Doubled for summary requests. | |
| 31 | 32 | | `firstTokenTimeout` | duration | `3m` | Silence tolerated before a model call's first token, while the server reads the prompt (every round trip of a turn has one). Never shorter than `watchdogTimeout`; must be >= 0. With a llama.cpp server (`dmr`, `llamacpp`, `llmman` — whose local GGUF models run in llama-server) mm asks for `return_progress`: each progress report restarts this clock, and the spinner shows `Reading the prompt 12.0k/25.3k tokens`. Ollama sends no progress: this one wait covers the whole prompt (and the model's loading). On a timeout the watchdog says `no first token from the server after …` (before the first token) or `connection hang detected after …` (after). | |
| @@ -128,7 +129,7 @@ The shipped `agent.lsp.yaml` is `agent.dmr.yaml` with this block on (gopls activ | ||
| 128 | 129 | | Key | Type | Default | Description | |
| 129 | 130 | |-----|------|---------|-------------| |
| 130 | 131 | | `tools` | list | `[]` | One entry per declared tool. | |
| 131 | -| `tools[].name` | string | — | What the model calls. Required. Starts with a letter; letters, digits, `_`, `-`; at most 64 characters. Unique. Not the name of a built-in tool (`ask_agent`, `ask_user`, `bash`, `edit_file`, `fetch_content`, `lsp`, `plan`, `read_file`, `read_skill`, `save_answer`, `write_file`), even one this agent turns off. | | |
| 132 | +| `tools[].name` | string | — | What the model calls. Required. Starts with a letter; letters, digits, `_`, `-`; at most 64 characters. Unique. Not the name of a built-in tool (`ask_agent`, `ask_user`, `bash`, `edit_file`, `fetch_content`, `lsp`, `plan`, `read_file`, `read_skill`, `save_answer`, `unload_model`, `write_file`), even one this agent turns off. | | |
| 132 | 133 | | `tools[].description` | string | — | What the tool does and when to use it, shown to the model. Required. A line `Required parameters: …` is appended. | |
| 133 | 134 | | `tools[].parameters` | map name → parameter | `{}` | The arguments, in the order written. Names follow the `name` rules. | |
| 134 | 135 | | `tools[].parameters.<p>.type` | string | `string` | `string`, `number`, `integer` or `boolean`. | |
| @@ -26,6 +26,7 @@ | |||
| 26 | | `fetchContentTool` | bool | `false` | Declare `fetch_content`, which fetches web pages as Markdown. Off by default: it is the only built-in tool that reaches the network. | | 26 | | `fetchContentTool` | bool | `false` | Declare `fetch_content`, which fetches web pages as Markdown. Off by default: it is the only built-in tool that reaches the network. | |
| 27 | | `askUserTool` | bool | `true` | Declare `ask_user`, through which the model asks the user questions (pick one, pick several, type a text, yes or no) and waits for the answers. `false` removes it: the model cannot ask. Never declared beside a `structuredOutput`. Overridden by `AGENT_ASK_USER`; for an agent asked by another one, also by the asker's `mmConfigs.<name>.askUser`. See [tools](tools.md#ask_user). | | 27 | | `askUserTool` | bool | `true` | Declare `ask_user`, through which the model asks the user questions (pick one, pick several, type a text, yes or no) and waits for the answers. `false` removes it: the model cannot ask. Never declared beside a `structuredOutput`. Overridden by `AGENT_ASK_USER`; for an agent asked by another one, also by the asker's `mmConfigs.<name>.askUser`. See [tools](tools.md#ask_user). | |
| 28 | | `saveAnswerTool` | bool | `false` | Declare `save_answer`, which writes an answer the model already gave to a new file: the model sends a file name, not the text again. Off by default. The `/save` command exists either way. See [tools](tools.md#save_answer). | | 28 | | `saveAnswerTool` | bool | `false` | Declare `save_answer`, which writes an answer the model already gave to a new file: the model sends a file name, not the text again. Off by default. The `/save` command exists either way. See [tools](tools.md#save_answer). | |
| 29 | +| `unloadModels` | bool | `true` | The `/unload [model]` command (terminal, window, editor) and the `unload_model` tool, which free a model from the server's memory: llmman and Ollama, llama-server in router mode. `false` removes both. The tool is never declared beside a `structuredOutput`. See [tools](tools.md#unload_model) and [How to free a model's memory](../how-to/unload-a-model.md). | | ||
| 29 | | `sampling` | map | `temperature: 0.0`, `top_p: 0.9`, `max_tokens: 4096` | Generation settings, OpenAI API key names, passed to the server. A key the OpenAI API does not declare is sent as it is, at the top level of the request — for example `chat_template_kwargs: {enable_thinking: false}`, which tells a llama.cpp server (`dmr`, `llamacpp`, `llmman`) not to reason; `reasoning_effort: none` does the same on Ollama. `messages`, `tools`, `tool_choice` (and the legacy `functions`, `function_call`) are ignored: Genkit builds them from the conversation. The shipped `agent.yaml` adds `parallel_tool_calls: false`. `max_tokens` also bounds one tool call: a `write_file` of a long document (or a thinking model's reasoning plus the call) can hit it, and the call is then cut off. mm hands it back to the model twice, asking for smaller pieces, then stops with `the model hit max_tokens (N) while writing a tool call`; raise the value if that happens. With streaming, mm also asks the server for token counts (`stream_options.include_usage`). | | 30 | | `sampling` | map | `temperature: 0.0`, `top_p: 0.9`, `max_tokens: 4096` | Generation settings, OpenAI API key names, passed to the server. A key the OpenAI API does not declare is sent as it is, at the top level of the request — for example `chat_template_kwargs: {enable_thinking: false}`, which tells a llama.cpp server (`dmr`, `llamacpp`, `llmman`) not to reason; `reasoning_effort: none` does the same on Ollama. `messages`, `tools`, `tool_choice` (and the legacy `functions`, `function_call`) are ignored: Genkit builds them from the conversation. The shipped `agent.yaml` adds `parallel_tool_calls: false`. `max_tokens` also bounds one tool call: a `write_file` of a long document (or a thinking model's reasoning plus the call) can hit it, and the call is then cut off. mm hands it back to the model twice, asking for smaller pieces, then stops with `the model hit max_tokens (N) while writing a tool call`; raise the value if that happens. With streaming, mm also asks the server for token counts (`stream_options.include_usage`). | |
| 30 | | `watchdogTimeout` | duration | `20s` | Silence tolerated between two streamed tokens before the connection is declared hung. Doubled for summary requests. | | 31 | | `watchdogTimeout` | duration | `20s` | Silence tolerated between two streamed tokens before the connection is declared hung. Doubled for summary requests. | |
| 31 | | `firstTokenTimeout` | duration | `3m` | Silence tolerated before a model call's first token, while the server reads the prompt (every round trip of a turn has one). Never shorter than `watchdogTimeout`; must be >= 0. With a llama.cpp server (`dmr`, `llamacpp`, `llmman` — whose local GGUF models run in llama-server) mm asks for `return_progress`: each progress report restarts this clock, and the spinner shows `Reading the prompt 12.0k/25.3k tokens`. Ollama sends no progress: this one wait covers the whole prompt (and the model's loading). On a timeout the watchdog says `no first token from the server after …` (before the first token) or `connection hang detected after …` (after). | | 32 | | `firstTokenTimeout` | duration | `3m` | Silence tolerated before a model call's first token, while the server reads the prompt (every round trip of a turn has one). Never shorter than `watchdogTimeout`; must be >= 0. With a llama.cpp server (`dmr`, `llamacpp`, `llmman` — whose local GGUF models run in llama-server) mm asks for `return_progress`: each progress report restarts this clock, and the spinner shows `Reading the prompt 12.0k/25.3k tokens`. Ollama sends no progress: this one wait covers the whole prompt (and the model's loading). On a timeout the watchdog says `no first token from the server after …` (before the first token) or `connection hang detected after …` (after). | |
| @@ -128,7 +129,7 @@ The shipped `agent.lsp.yaml` is `agent.dmr.yaml` with this block on (gopls activ | |||
| 128 | | Key | Type | Default | Description | | 129 | | Key | Type | Default | Description | |
| 129 | |-----|------|---------|-------------| | 130 | |-----|------|---------|-------------| |
| 130 | | `tools` | list | `[]` | One entry per declared tool. | | 131 | | `tools` | list | `[]` | One entry per declared tool. | |
| 131 | -| `tools[].name` | string | — | What the model calls. Required. Starts with a letter; letters, digits, `_`, `-`; at most 64 characters. Unique. Not the name of a built-in tool (`ask_agent`, `ask_user`, `bash`, `edit_file`, `fetch_content`, `lsp`, `plan`, `read_file`, `read_skill`, `save_answer`, `write_file`), even one this agent turns off. | | 132 | +| `tools[].name` | string | — | What the model calls. Required. Starts with a letter; letters, digits, `_`, `-`; at most 64 characters. Unique. Not the name of a built-in tool (`ask_agent`, `ask_user`, `bash`, `edit_file`, `fetch_content`, `lsp`, `plan`, `read_file`, `read_skill`, `save_answer`, `unload_model`, `write_file`), even one this agent turns off. | |
| 132 | | `tools[].description` | string | — | What the tool does and when to use it, shown to the model. Required. A line `Required parameters: …` is appended. | | 133 | | `tools[].description` | string | — | What the tool does and when to use it, shown to the model. Required. A line `Required parameters: …` is appended. | |
| 133 | | `tools[].parameters` | map name → parameter | `{}` | The arguments, in the order written. Names follow the `name` rules. | | 134 | | `tools[].parameters` | map name → parameter | `{}` | The arguments, in the order written. Names follow the `name` rules. | |
| 134 | | `tools[].parameters.<p>.type` | string | `string` | `string`, `number`, `integer` or `boolean`. | | 135 | | `tools[].parameters.<p>.type` | string | `string` | `string`, `number`, `integer` or `boolean`. | |
modified
docs/en/reference/slash-commands.md +3 -1 | @@ -15,6 +15,7 @@ | ||
| 15 | 15 | | `/help` | Prints the commands of this front end, one per line with their description, aligned. | Sent to the model as a new message. | |
| 16 | 16 | | `/copy [n\|all]` | Copies the last answer (`n`: the n-th from the end) to the clipboard through `pbcopy`/`wl-copy`/`xclip`/`xsel` or, failing that, OSC 52 when stdout is a terminal, and prints `📋 copied N chars (<road>)` or `⚠️ …`. `/copy all` is window-mode only: here it says so. | Sent to the model as a new message. | |
| 17 | 17 | | `/save <file> [n]` | Writes the last answer (`n`: the n-th from the end) as it is to a new file, creating the missing directories, and prints `💾 saved the last answer to <file> (N bytes)`. Never overwrites: an existing file gives `⚠️ <file> already exists — nothing was written; choose another name`. A file name with spaces is not supported. The model is not called. | Sent to the model as a new message. | |
| 18 | +| `/unload [model]` | With `unloadModels` (default on): unloads a model from the server's memory — this agent's own, or the one named — and prints one line: `🧹 unloaded <model>`, `🧹 <model> was not loaded: nothing to unload`, or `🧹 cannot unload <model>: <reason>` (a provider or a server that has no way to). The history is not touched; the next question reloads the agent's model. | Sent to the model as a new message. | | |
| 18 | 19 | |
| 19 | 20 | With `sessions.enabled`, `/new` leaves the current session's file as it is; the next question opens a new file. An empty line at the prompt is ignored. |
| 20 | 21 | |
| @@ -169,12 +170,13 @@ The notation works in the line mode and over ACP too, typed by hand: a `$skill` | ||
| 169 | 170 | |
| 170 | 171 | ## ACP mode |
| 171 | 172 | |
| 172 | -Two commands exist in ACP mode: `/new` and `/compact`. They are advertised to the editor through an `available_commands_update` notification and intercepted by `session/prompt` when the prompt text, trimmed, is exactly the command. Each answers with one `agent_message_chunk` carrying the line the terminal would print, then `stopReason: end_turn`; the model is not consulted. | |
| 173 | +Two commands exist in ACP mode, three with `unloadModels` (the default): `/new`, `/compact` and `/unload`. They are advertised to the editor through an `available_commands_update` notification and intercepted by `session/prompt` when the prompt text, trimmed, is exactly the command (`/unload` may be followed by a model name). Each answers with one `agent_message_chunk` carrying the line the terminal would print, then `stopReason: end_turn`; the model is not consulted. | |
| 173 | 174 | |
| 174 | 175 | | Command | Over ACP | |
| 175 | 176 | |---------|----------| |
| 176 | 177 | | `/new` | Same as in the terminal: history back to the system prompt, token count forgotten; the session id, `cwd` and allow-always grants are kept. | |
| 177 | 178 | | `/compact` | Same as in the terminal, always forced: `🗜️ compressed …`, `🗜️ nothing to compact: …` or `[compact: failed, history kept: …]`. Waits for a running turn to finish first. | |
| 179 | +| `/unload [model]` | With `unloadModels` (default on), advertised with the input hint `[model]`: same as in the terminal, the `🧹` line as the agent's text; the history is not touched. | | |
| 178 | 180 | |
| 179 | 181 | With `mmConfigs`, `/ask-<name>` and `/ask-with-context-<name>` are advertised too, each with the input hint `<prompt>`; the generic `/ask <name>` is not, since an editor's menu completes a command name but not its argument. See [Asking another agent](#asking-another-agent). |
| 180 | 182 | |
| @@ -15,6 +15,7 @@ | |||
| 15 | | `/help` | Prints the commands of this front end, one per line with their description, aligned. | Sent to the model as a new message. | | 15 | | `/help` | Prints the commands of this front end, one per line with their description, aligned. | Sent to the model as a new message. | |
| 16 | | `/copy [n\|all]` | Copies the last answer (`n`: the n-th from the end) to the clipboard through `pbcopy`/`wl-copy`/`xclip`/`xsel` or, failing that, OSC 52 when stdout is a terminal, and prints `📋 copied N chars (<road>)` or `⚠️ …`. `/copy all` is window-mode only: here it says so. | Sent to the model as a new message. | | 16 | | `/copy [n\|all]` | Copies the last answer (`n`: the n-th from the end) to the clipboard through `pbcopy`/`wl-copy`/`xclip`/`xsel` or, failing that, OSC 52 when stdout is a terminal, and prints `📋 copied N chars (<road>)` or `⚠️ …`. `/copy all` is window-mode only: here it says so. | Sent to the model as a new message. | |
| 17 | | `/save <file> [n]` | Writes the last answer (`n`: the n-th from the end) as it is to a new file, creating the missing directories, and prints `💾 saved the last answer to <file> (N bytes)`. Never overwrites: an existing file gives `⚠️ <file> already exists — nothing was written; choose another name`. A file name with spaces is not supported. The model is not called. | Sent to the model as a new message. | | 17 | | `/save <file> [n]` | Writes the last answer (`n`: the n-th from the end) as it is to a new file, creating the missing directories, and prints `💾 saved the last answer to <file> (N bytes)`. Never overwrites: an existing file gives `⚠️ <file> already exists — nothing was written; choose another name`. A file name with spaces is not supported. The model is not called. | Sent to the model as a new message. | |
| 18 | +| `/unload [model]` | With `unloadModels` (default on): unloads a model from the server's memory — this agent's own, or the one named — and prints one line: `🧹 unloaded <model>`, `🧹 <model> was not loaded: nothing to unload`, or `🧹 cannot unload <model>: <reason>` (a provider or a server that has no way to). The history is not touched; the next question reloads the agent's model. | Sent to the model as a new message. | | ||
| 18 | 19 | ||
| 19 | With `sessions.enabled`, `/new` leaves the current session's file as it is; the next question opens a new file. An empty line at the prompt is ignored. | 20 | With `sessions.enabled`, `/new` leaves the current session's file as it is; the next question opens a new file. An empty line at the prompt is ignored. |
| 20 | 21 | ||
| @@ -169,12 +170,13 @@ The notation works in the line mode and over ACP too, typed by hand: a `$skill` | |||
| 169 | 170 | ||
| 170 | ## ACP mode | 171 | ## ACP mode |
| 171 | 172 | ||
| 172 | -Two commands exist in ACP mode: `/new` and `/compact`. They are advertised to the editor through an `available_commands_update` notification and intercepted by `session/prompt` when the prompt text, trimmed, is exactly the command. Each answers with one `agent_message_chunk` carrying the line the terminal would print, then `stopReason: end_turn`; the model is not consulted. | 173 | +Two commands exist in ACP mode, three with `unloadModels` (the default): `/new`, `/compact` and `/unload`. They are advertised to the editor through an `available_commands_update` notification and intercepted by `session/prompt` when the prompt text, trimmed, is exactly the command (`/unload` may be followed by a model name). Each answers with one `agent_message_chunk` carrying the line the terminal would print, then `stopReason: end_turn`; the model is not consulted. |
| 173 | 174 | ||
| 174 | | Command | Over ACP | | 175 | | Command | Over ACP | |
| 175 | |---------|----------| | 176 | |---------|----------| |
| 176 | | `/new` | Same as in the terminal: history back to the system prompt, token count forgotten; the session id, `cwd` and allow-always grants are kept. | | 177 | | `/new` | Same as in the terminal: history back to the system prompt, token count forgotten; the session id, `cwd` and allow-always grants are kept. | |
| 177 | | `/compact` | Same as in the terminal, always forced: `🗜️ compressed …`, `🗜️ nothing to compact: …` or `[compact: failed, history kept: …]`. Waits for a running turn to finish first. | | 178 | | `/compact` | Same as in the terminal, always forced: `🗜️ compressed …`, `🗜️ nothing to compact: …` or `[compact: failed, history kept: …]`. Waits for a running turn to finish first. | |
| 179 | +| `/unload [model]` | With `unloadModels` (default on), advertised with the input hint `[model]`: same as in the terminal, the `🧹` line as the agent's text; the history is not touched. | | ||
| 178 | 180 | ||
| 179 | With `mmConfigs`, `/ask-<name>` and `/ask-with-context-<name>` are advertised too, each with the input hint `<prompt>`; the generic `/ask <name>` is not, since an editor's menu completes a command name but not its argument. See [Asking another agent](#asking-another-agent). | 181 | With `mmConfigs`, `/ask-<name>` and `/ask-with-context-<name>` are advertised too, each with the input hint `<prompt>`; the generic `/ask <name>` is not, since an editor's menu completes a command name but not its argument. See [Asking another agent](#asking-another-agent). |
| 180 | 182 | ||
modified
docs/en/reference/tools.md +20 -0 | @@ -268,6 +268,26 @@ Declared when `saveAnswerTool: true` (default `false`). | ||
| 268 | 268 | | Terminal display | `💾 save_answer: <path> (this answer\|the previous answer)`, then the result. | |
| 269 | 269 | | ACP and window | Tool call of kind `edit`, with the file's absolute path. | |
| 270 | 270 | |
| 271 | +## `unload_model` | |
| 272 | + | |
| 273 | +Declared when `unloadModels` is true (the default), except beside a `structuredOutput`. | |
| 274 | + | |
| 275 | +| Field | Type | Description | | |
| 276 | +|-------|------|-------------| | |
| 277 | +| `model` | string | The model to unload, as the server names it. Empty: the agent's own model. | | |
| 278 | + | |
| 279 | +| Aspect | Behaviour | | |
| 280 | +|--------|-----------| | |
| 281 | +| llmman, `ollama` | Looks the model up in `GET /api/ps` first; when it is loaded, `POST /api/generate {"model": …, "keep_alive": 0}` and expects `done_reason: "unload"`. A model that is not loaded is never asked for: the request would load it first. | | |
| 282 | +| `llamacpp` | `POST /models/unload {"model": …}`, which exists in router mode only (llama-server started without a model). A single-model server's 404 gives the reason and the alternatives: router mode, `--sleep-idle-seconds N`, stopping the server. | | |
| 283 | +| `dmr` | `cannot unload <model>: the dmr provider cannot unload a model`. | | |
| 284 | +| Result | One line: `unloaded <model>`, `<model> was not loaded: nothing to unload`, or `cannot unload <model>: <reason>` (unreachable server, the server's own error, no confirmation). | | |
| 285 | +| Own model | Unloading the agent's own model lasts until its next step: the model is needed to read the result, and the server reloads it. | | |
| 286 | +| Permission | Never asked, even with `yolo: false`: nothing changes on disk, and the model reloads by itself. | | |
| 287 | +| Terminal display | `🧹 unload_model: <model>` (`(own model)` when empty), then the result. | | |
| 288 | +| ACP and window | Tool call of kind `other`; a `cannot unload` result is a failed call. | | |
| 289 | +| Timeout | 30 s per request to the server. | | |
| 290 | + | |
| 271 | 291 | ## Declared tools |
| 272 | 292 | |
| 273 | 293 | Declared from the `tools:` block of the agent file, one tool per entry (see [configuration](configuration.md#tools-keys)). Each has the entry's `name`; its fields are the entry's `parameters`. |
| @@ -268,6 +268,26 @@ Declared when `saveAnswerTool: true` (default `false`). | |||
| 268 | | Terminal display | `💾 save_answer: <path> (this answer\|the previous answer)`, then the result. | | 268 | | Terminal display | `💾 save_answer: <path> (this answer\|the previous answer)`, then the result. | |
| 269 | | ACP and window | Tool call of kind `edit`, with the file's absolute path. | | 269 | | ACP and window | Tool call of kind `edit`, with the file's absolute path. | |
| 270 | 270 | ||
| 271 | +## `unload_model` | ||
| 272 | + | ||
| 273 | +Declared when `unloadModels` is true (the default), except beside a `structuredOutput`. | ||
| 274 | + | ||
| 275 | +| Field | Type | Description | | ||
| 276 | +|-------|------|-------------| | ||
| 277 | +| `model` | string | The model to unload, as the server names it. Empty: the agent's own model. | | ||
| 278 | + | ||
| 279 | +| Aspect | Behaviour | | ||
| 280 | +|--------|-----------| | ||
| 281 | +| llmman, `ollama` | Looks the model up in `GET /api/ps` first; when it is loaded, `POST /api/generate {"model": …, "keep_alive": 0}` and expects `done_reason: "unload"`. A model that is not loaded is never asked for: the request would load it first. | | ||
| 282 | +| `llamacpp` | `POST /models/unload {"model": …}`, which exists in router mode only (llama-server started without a model). A single-model server's 404 gives the reason and the alternatives: router mode, `--sleep-idle-seconds N`, stopping the server. | | ||
| 283 | +| `dmr` | `cannot unload <model>: the dmr provider cannot unload a model`. | | ||
| 284 | +| Result | One line: `unloaded <model>`, `<model> was not loaded: nothing to unload`, or `cannot unload <model>: <reason>` (unreachable server, the server's own error, no confirmation). | | ||
| 285 | +| Own model | Unloading the agent's own model lasts until its next step: the model is needed to read the result, and the server reloads it. | | ||
| 286 | +| Permission | Never asked, even with `yolo: false`: nothing changes on disk, and the model reloads by itself. | | ||
| 287 | +| Terminal display | `🧹 unload_model: <model>` (`(own model)` when empty), then the result. | | ||
| 288 | +| ACP and window | Tool call of kind `other`; a `cannot unload` result is a failed call. | | ||
| 289 | +| Timeout | 30 s per request to the server. | | ||
| 290 | + | ||
| 271 | ## Declared tools | 291 | ## Declared tools |
| 272 | 292 | ||
| 273 | Declared from the `tools:` block of the agent file, one tool per entry (see [configuration](configuration.md#tools-keys)). Each has the entry's `name`; its fields are the entry's `parameters`. | 293 | Declared from the `tools:` block of the agent file, one tool per entry (see [configuration](configuration.md#tools-keys)). Each has the entry's `name`; its fields are the entry's `parameters`. |
modified
docs/fr/README.md +2 -1 | @@ -27,6 +27,7 @@ Cette documentation est organisée selon les quatre quadrants [Diátaxis](https: | ||
| 27 | 27 | - [Ajouter un skill](how-to/add-a-skill.md) |
| 28 | 28 | - [Déclarer un outil personnalisé](how-to/declare-a-custom-tool.md) |
| 29 | 29 | - [Sauvegarder une réponse dans un fichier](how-to/save-an-answer.md) |
| 30 | +- [Libérer la mémoire d'un modèle](how-to/unload-a-model.md) — `/unload [modèle]` dans toutes les façades, l'outil `unload_model` après la réponse d'un autre agent, `unloadModels` | |
| 30 | 31 | - [Mesurer si le modèle choisit le bon skill](how-to/measure-skill-routing.md) — l'eval de routage : le lancer, lire son tableau, s'en servir en avant/après |
| 31 | 32 | - [Garder une longue session dans la fenêtre de contexte](how-to/manage-the-context-window.md) |
| 32 | 33 | - [Démarrer une nouvelle session sans relancer](how-to/start-a-new-session.md) |
| @@ -51,7 +52,7 @@ Cette documentation est organisée selon les quatre quadrants [Diátaxis](https: | ||
| 51 | 52 | - [Ligne de commande](reference/cli.md) — options, variables d'environnement, bannière de démarrage |
| 52 | 53 | - [Commandes slash](reference/slash-commands.md) — `/quit`, `/abort`, `/compact`, `/new`, `/sessions`, `/resume`, `/help`, raccourcis clavier, touches de la fenêtre |
| 53 | 54 | - [Fichier de configuration](reference/configuration.md) — chaque clé YAML, sa valeur par défaut et sa règle de validation |
| 54 | -- [Outils intégrés](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`, `lsp`, `fetch_content`, `ask_agent`, `ask_user`, `save_answer`, outils déclarés | |
| 55 | +- [Outils intégrés](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`, `lsp`, `fetch_content`, `ask_agent`, `ask_user`, `save_answer`, `unload_model`, outils déclarés | |
| 55 | 56 | - [Format d'un fichier de skill](reference/skill-format.md) |
| 56 | 57 | - [Les hooks](reference/hooks.md) — les six événements, le JSON qu'un hook lit et écrit, `matcher`, `timeout`, `onError`, erreurs de validation |
| 57 | 58 | - [Surface Agent Client Protocol](reference/acp.md) |
| @@ -27,6 +27,7 @@ Cette documentation est organisée selon les quatre quadrants [Diátaxis](https: | |||
| 27 | - [Ajouter un skill](how-to/add-a-skill.md) | 27 | - [Ajouter un skill](how-to/add-a-skill.md) |
| 28 | - [Déclarer un outil personnalisé](how-to/declare-a-custom-tool.md) | 28 | - [Déclarer un outil personnalisé](how-to/declare-a-custom-tool.md) |
| 29 | - [Sauvegarder une réponse dans un fichier](how-to/save-an-answer.md) | 29 | - [Sauvegarder une réponse dans un fichier](how-to/save-an-answer.md) |
| 30 | +- [Libérer la mémoire d'un modèle](how-to/unload-a-model.md) — `/unload [modèle]` dans toutes les façades, l'outil `unload_model` après la réponse d'un autre agent, `unloadModels` | ||
| 30 | - [Mesurer si le modèle choisit le bon skill](how-to/measure-skill-routing.md) — l'eval de routage : le lancer, lire son tableau, s'en servir en avant/après | 31 | - [Mesurer si le modèle choisit le bon skill](how-to/measure-skill-routing.md) — l'eval de routage : le lancer, lire son tableau, s'en servir en avant/après |
| 31 | - [Garder une longue session dans la fenêtre de contexte](how-to/manage-the-context-window.md) | 32 | - [Garder une longue session dans la fenêtre de contexte](how-to/manage-the-context-window.md) |
| 32 | - [Démarrer une nouvelle session sans relancer](how-to/start-a-new-session.md) | 33 | - [Démarrer une nouvelle session sans relancer](how-to/start-a-new-session.md) |
| @@ -51,7 +52,7 @@ Cette documentation est organisée selon les quatre quadrants [Diátaxis](https: | |||
| 51 | - [Ligne de commande](reference/cli.md) — options, variables d'environnement, bannière de démarrage | 52 | - [Ligne de commande](reference/cli.md) — options, variables d'environnement, bannière de démarrage |
| 52 | - [Commandes slash](reference/slash-commands.md) — `/quit`, `/abort`, `/compact`, `/new`, `/sessions`, `/resume`, `/help`, raccourcis clavier, touches de la fenêtre | 53 | - [Commandes slash](reference/slash-commands.md) — `/quit`, `/abort`, `/compact`, `/new`, `/sessions`, `/resume`, `/help`, raccourcis clavier, touches de la fenêtre |
| 53 | - [Fichier de configuration](reference/configuration.md) — chaque clé YAML, sa valeur par défaut et sa règle de validation | 54 | - [Fichier de configuration](reference/configuration.md) — chaque clé YAML, sa valeur par défaut et sa règle de validation |
| 54 | -- [Outils intégrés](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`, `lsp`, `fetch_content`, `ask_agent`, `ask_user`, `save_answer`, outils déclarés | 55 | +- [Outils intégrés](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`, `lsp`, `fetch_content`, `ask_agent`, `ask_user`, `save_answer`, `unload_model`, outils déclarés |
| 55 | - [Format d'un fichier de skill](reference/skill-format.md) | 56 | - [Format d'un fichier de skill](reference/skill-format.md) |
| 56 | - [Les hooks](reference/hooks.md) — les six événements, le JSON qu'un hook lit et écrit, `matcher`, `timeout`, `onError`, erreurs de validation | 57 | - [Les hooks](reference/hooks.md) — les six événements, le JSON qu'un hook lit et écrit, `matcher`, `timeout`, `onError`, erreurs de validation |
| 57 | - [Surface Agent Client Protocol](reference/acp.md) | 58 | - [Surface Agent Client Protocol](reference/acp.md) |
added
docs/fr/how-to/unload-a-model.md +58 -0 | new file mode 100644 | ||
| @@ -0,0 +1,58 @@ | ||
| 1 | +# Libérer la mémoire d'un modèle | |
| 2 | + | |
| 3 | +Ce guide montre comment décharger un modèle de la mémoire du serveur sans arrêter le serveur : à la main avec `/unload`, ou par l'agent lui-même avec son outil `unload_model` — typiquement pour libérer la mémoire du modèle d'un autre agent une fois sa réponse obtenue. Il suppose que mm tourne sur llmman, Ollama, ou llama-server en mode router. Le modèle se recharge tout seul la prochaine fois qu'il sert. | |
| 4 | + | |
| 5 | +## Décharger à la main | |
| 6 | + | |
| 7 | +Tapez, dans le terminal, la fenêtre ou un éditeur : | |
| 8 | + | |
| 9 | +```text | |
| 10 | +/unload | |
| 11 | +``` | |
| 12 | + | |
| 13 | +Le modèle de l'agent est déchargé : | |
| 14 | + | |
| 15 | +```text | |
| 16 | +🧹 unloaded hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | |
| 17 | +``` | |
| 18 | + | |
| 19 | +Nommez un autre modèle servi par le même serveur pour décharger celui-là : | |
| 20 | + | |
| 21 | +```text | |
| 22 | +/unload hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | |
| 23 | +``` | |
| 24 | + | |
| 25 | +Un modèle qui n'est pas chargé donne `🧹 <modèle> was not loaded: nothing to unload` ; rien n'est envoyé au serveur, donc rien n'est chargé non plus. Vérifiez ce qui est chargé avec `llmman ps`, `ollama ps`, ou `curl <serveur>/api/ps`. | |
| 26 | + | |
| 27 | +## Laisser l'agent décharger un modèle | |
| 28 | + | |
| 29 | +L'outil `unload_model` est déclaré par défaut. Dites au modèle quand s'en servir, dans son prompt `system`. Le cas où il sert le mieux est le modèle d'un autre agent, une fois que cet agent a répondu : | |
| 30 | + | |
| 31 | +```yaml | |
| 32 | +mmConfigs: | |
| 33 | + expert: | |
| 34 | + path: agent.expert.yaml | |
| 35 | + description: Programming expert. | |
| 36 | + | |
| 37 | +system: | | |
| 38 | + For every question about programming, do these three steps, in order: | |
| 39 | + 1. Call ask_agent with agent "expert" and the user's question. | |
| 40 | + 2. Call unload_model with model "hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M". | |
| 41 | + It is ALWAYS your next action after ask_agent: never answer the user | |
| 42 | + before you have called unload_model. | |
| 43 | + 3. Then give the expert's answer to the user. | |
| 44 | +``` | |
| 45 | + | |
| 46 | +Écrivez le nom du modèle exactement comme le serveur le liste : le modèle ne peut pas le deviner. `demos/unload-models/` décharge plutôt l'expert avec un hook `postAsk` (voir les variantes ci-dessous) ; son coordinateur appelle toujours `unload_model` quand l'utilisateur lui demande de libérer la mémoire de l'expert. | |
| 47 | + | |
| 48 | +## Variantes | |
| 49 | + | |
| 50 | +- **Le modèle de l'agent lui-même.** `unload_model` avec un `model` vide le décharge, mais seulement jusqu'à l'étape suivante de l'agent : il a besoin de son modèle pour lire le résultat de l'outil, et le serveur le recharge. Utilisez plutôt `/unload` entre deux questions — ou, pour le libérer pendant qu'un autre agent travaille, un hook `preAsk` qui décharge le modèle nommé dans son entrée (`"model": "<fournisseur>/<modèle>"`) juste avant le lancement de l'agent interrogé : `demos/unload-models/hooks/unload.js self`. | |
| 51 | +- **Une garantie plutôt qu'une consigne.** Un petit modèle peut sauter l'étape (mesuré : 4 fois sur 5 avec MiniCPM5 2B dans la première version de `demos/unload-models`), et le modèle de l'agent est déjà rechargé à côté de l'autre quand il appelle l'outil. Pour un déchargement certain, sans jamais les deux modèles chargés ensemble, faites appeler le serveur par un hook `postAsk` — `curl -s <serveur>/api/generate -d '{"model":"…","keep_alive":0}'` — une fois que l'agent interrogé a répondu et avant que l'agent qui l'a interrogé lise la réponse : `demos/unload-models/hooks/unload.js asked` lit le modèle de l'agent interrogé dans `ask.agentModel`. Voir [Contrôler les relais avec des hooks](control-agent-calls-with-hooks.md). | |
| 52 | +- **llama-server.** Seul le mode router (lancé sans `-m`) peut décharger un modèle. Un serveur à modèle unique répond avec les alternatives : `--sleep-idle-seconds N` libère la mémoire après N secondes d'inactivité, ou arrêtez le serveur. | |
| 53 | +- **Le désactiver.** `unloadModels: false` retire à la fois `/unload` et l'outil — pour un agent qui ne doit pas avoir un outil de plus, comme une garde ou un spécialiste étroit. | |
| 54 | + | |
| 55 | +## Voir aussi | |
| 56 | + | |
| 57 | +- [Référence de `unload_model`](../reference/tools.md#unload_model) et de [`/unload`](../reference/slash-commands.md#commandes) | |
| 58 | +- [`unloadModels`](../reference/configuration.md) | |
| new file mode 100644 | |||
| @@ -0,0 +1,58 @@ | |||
| 1 | +# Libérer la mémoire d'un modèle | ||
| 2 | + | ||
| 3 | +Ce guide montre comment décharger un modèle de la mémoire du serveur sans arrêter le serveur : à la main avec `/unload`, ou par l'agent lui-même avec son outil `unload_model` — typiquement pour libérer la mémoire du modèle d'un autre agent une fois sa réponse obtenue. Il suppose que mm tourne sur llmman, Ollama, ou llama-server en mode router. Le modèle se recharge tout seul la prochaine fois qu'il sert. | ||
| 4 | + | ||
| 5 | +## Décharger à la main | ||
| 6 | + | ||
| 7 | +Tapez, dans le terminal, la fenêtre ou un éditeur : | ||
| 8 | + | ||
| 9 | +```text | ||
| 10 | +/unload | ||
| 11 | +``` | ||
| 12 | + | ||
| 13 | +Le modèle de l'agent est déchargé : | ||
| 14 | + | ||
| 15 | +```text | ||
| 16 | +🧹 unloaded hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | ||
| 17 | +``` | ||
| 18 | + | ||
| 19 | +Nommez un autre modèle servi par le même serveur pour décharger celui-là : | ||
| 20 | + | ||
| 21 | +```text | ||
| 22 | +/unload hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M | ||
| 23 | +``` | ||
| 24 | + | ||
| 25 | +Un modèle qui n'est pas chargé donne `🧹 <modèle> was not loaded: nothing to unload` ; rien n'est envoyé au serveur, donc rien n'est chargé non plus. Vérifiez ce qui est chargé avec `llmman ps`, `ollama ps`, ou `curl <serveur>/api/ps`. | ||
| 26 | + | ||
| 27 | +## Laisser l'agent décharger un modèle | ||
| 28 | + | ||
| 29 | +L'outil `unload_model` est déclaré par défaut. Dites au modèle quand s'en servir, dans son prompt `system`. Le cas où il sert le mieux est le modèle d'un autre agent, une fois que cet agent a répondu : | ||
| 30 | + | ||
| 31 | +```yaml | ||
| 32 | +mmConfigs: | ||
| 33 | + expert: | ||
| 34 | + path: agent.expert.yaml | ||
| 35 | + description: Programming expert. | ||
| 36 | + | ||
| 37 | +system: | | ||
| 38 | + For every question about programming, do these three steps, in order: | ||
| 39 | + 1. Call ask_agent with agent "expert" and the user's question. | ||
| 40 | + 2. Call unload_model with model "hf.co/unsloth/Qwen3.5-9B-GGUF:Q4_K_M". | ||
| 41 | + It is ALWAYS your next action after ask_agent: never answer the user | ||
| 42 | + before you have called unload_model. | ||
| 43 | + 3. Then give the expert's answer to the user. | ||
| 44 | +``` | ||
| 45 | + | ||
| 46 | +Écrivez le nom du modèle exactement comme le serveur le liste : le modèle ne peut pas le deviner. `demos/unload-models/` décharge plutôt l'expert avec un hook `postAsk` (voir les variantes ci-dessous) ; son coordinateur appelle toujours `unload_model` quand l'utilisateur lui demande de libérer la mémoire de l'expert. | ||
| 47 | + | ||
| 48 | +## Variantes | ||
| 49 | + | ||
| 50 | +- **Le modèle de l'agent lui-même.** `unload_model` avec un `model` vide le décharge, mais seulement jusqu'à l'étape suivante de l'agent : il a besoin de son modèle pour lire le résultat de l'outil, et le serveur le recharge. Utilisez plutôt `/unload` entre deux questions — ou, pour le libérer pendant qu'un autre agent travaille, un hook `preAsk` qui décharge le modèle nommé dans son entrée (`"model": "<fournisseur>/<modèle>"`) juste avant le lancement de l'agent interrogé : `demos/unload-models/hooks/unload.js self`. | ||
| 51 | +- **Une garantie plutôt qu'une consigne.** Un petit modèle peut sauter l'étape (mesuré : 4 fois sur 5 avec MiniCPM5 2B dans la première version de `demos/unload-models`), et le modèle de l'agent est déjà rechargé à côté de l'autre quand il appelle l'outil. Pour un déchargement certain, sans jamais les deux modèles chargés ensemble, faites appeler le serveur par un hook `postAsk` — `curl -s <serveur>/api/generate -d '{"model":"…","keep_alive":0}'` — une fois que l'agent interrogé a répondu et avant que l'agent qui l'a interrogé lise la réponse : `demos/unload-models/hooks/unload.js asked` lit le modèle de l'agent interrogé dans `ask.agentModel`. Voir [Contrôler les relais avec des hooks](control-agent-calls-with-hooks.md). | ||
| 52 | +- **llama-server.** Seul le mode router (lancé sans `-m`) peut décharger un modèle. Un serveur à modèle unique répond avec les alternatives : `--sleep-idle-seconds N` libère la mémoire après N secondes d'inactivité, ou arrêtez le serveur. | ||
| 53 | +- **Le désactiver.** `unloadModels: false` retire à la fois `/unload` et l'outil — pour un agent qui ne doit pas avoir un outil de plus, comme une garde ou un spécialiste étroit. | ||
| 54 | + | ||
| 55 | +## Voir aussi | ||
| 56 | + | ||
| 57 | +- [Référence de `unload_model`](../reference/tools.md#unload_model) et de [`/unload`](../reference/slash-commands.md#commandes) | ||
| 58 | +- [`unloadModels`](../reference/configuration.md) | ||
modified
docs/fr/reference/configuration.md +2 -1 | @@ -26,6 +26,7 @@ | ||
| 26 | 26 | | `fetchContentTool` | booléen | `false` | Déclare `fetch_content`, qui récupère des pages web en Markdown. Désactivé par défaut : c'est le seul outil intégré qui accède au réseau. | |
| 27 | 27 | | `askUserTool` | booléen | `true` | Déclare `ask_user`, par lequel le modèle pose des questions à l'utilisateur (choisir une option, en choisir plusieurs, taper un texte, oui ou non) et attend les réponses. `false` le retire : le modèle ne peut pas demander. Jamais déclaré à côté d'un `structuredOutput`. Surchargé par `AGENT_ASK_USER` ; pour un agent interrogé par un autre, aussi par le `mmConfigs.<nom>.askUser` de celui qui l'interroge. Voir [outils](tools.md#ask_user). | |
| 28 | 28 | | `saveAnswerTool` | bool | `false` | Déclare `save_answer`, qui écrit dans un nouveau fichier une réponse que le modèle a déjà donnée : le modèle envoie un nom de fichier, pas le texte une seconde fois. Désactivé par défaut. La commande `/save` existe dans tous les cas. Voir les [outils](tools.md#save_answer). | |
| 29 | +| `unloadModels` | bool | `true` | La commande `/unload [modèle]` (terminal, fenêtre, éditeur) et l'outil `unload_model`, qui libèrent un modèle de la mémoire du serveur : llmman et Ollama, llama-server en mode router. `false` retire les deux. L'outil n'est jamais déclaré à côté d'une `structuredOutput`. Voir les [outils](tools.md#unload_model) et [Libérer la mémoire d'un modèle](../how-to/unload-a-model.md). | | |
| 29 | 30 | | `sampling` | dictionnaire | `temperature: 0.0`, `top_p: 0.9`, `max_tokens: 4096` | Paramètres de génération, noms de l'API OpenAI, transmis au serveur. Une clé que l'API OpenAI ne déclare pas est envoyée telle quelle, au premier niveau de la requête — par exemple `chat_template_kwargs: {enable_thinking: false}`, qui demande à un serveur llama.cpp (`dmr`, `llamacpp`, `llmman`) de ne pas raisonner ; `reasoning_effort: none` fait de même avec Ollama. `messages`, `tools`, `tool_choice` (et les anciens `functions`, `function_call`) sont ignorées : Genkit les construit à partir de la conversation. L'`agent.yaml` livré ajoute `parallel_tool_calls: false`. `max_tokens` borne aussi un appel d'outil : un `write_file` de long document (ou le raisonnement d'un modèle « thinking » plus l'appel) peut l'atteindre, et l'appel est alors coupé. mm le rend deux fois au modèle en lui demandant de procéder par morceaux, puis s'arrête avec `the model hit max_tokens (N) while writing a tool call` ; augmentez la valeur si cela arrive. En streaming, mm demande aussi au serveur le compte des tokens (`stream_options.include_usage`). | |
| 30 | 31 | | `watchdogTimeout` | durée | `20s` | Silence toléré entre deux tokens diffusés avant de déclarer la connexion bloquée. Doublé pour les requêtes de résumé. | |
| 31 | 32 | | `firstTokenTimeout` | durée | `3m` | Silence toléré avant le premier token d'un appel au modèle, pendant que le serveur lit le prompt (chaque aller-retour d'un tour en a un). Jamais plus court que `watchdogTimeout` ; doit être >= 0. Avec un serveur llama.cpp (`dmr`, `llamacpp`, `llmman` — dont les modèles GGUF locaux tournent dans llama-server), mm demande `return_progress` : chaque rapport d'avancement relance ce délai, et le spinner affiche `Reading the prompt 12.0k/25.3k tokens`. Ollama n'envoie pas d'avancement : cette seule attente couvre tout le prompt (et le chargement du modèle). À l'expiration, le watchdog dit `no first token from the server after …` (avant le premier token) ou `connection hang detected after …` (après). | |
| @@ -128,7 +129,7 @@ Le fichier livré `agent.lsp.yaml` est `agent.dmr.yaml` avec ce bloc actif (gopl | ||
| 128 | 129 | | Clé | Type | Défaut | Description | |
| 129 | 130 | |-----|------|--------|-------------| |
| 130 | 131 | | `tools` | liste | `[]` | Une entrée par outil déclaré. | |
| 131 | -| `tools[].name` | chaîne | — | Le nom que le modèle appelle. Obligatoire. Commence par une lettre ; lettres, chiffres, `_`, `-` ; 64 caractères au plus. Unique. Pas le nom d'un outil intégré (`ask_agent`, `ask_user`, `bash`, `edit_file`, `fetch_content`, `lsp`, `plan`, `read_file`, `read_skill`, `save_answer`, `write_file`), même désactivé dans cet agent. | | |
| 132 | +| `tools[].name` | chaîne | — | Le nom que le modèle appelle. Obligatoire. Commence par une lettre ; lettres, chiffres, `_`, `-` ; 64 caractères au plus. Unique. Pas le nom d'un outil intégré (`ask_agent`, `ask_user`, `bash`, `edit_file`, `fetch_content`, `lsp`, `plan`, `read_file`, `read_skill`, `save_answer`, `unload_model`, `write_file`), même désactivé dans cet agent. | | |
| 132 | 133 | | `tools[].description` | chaîne | — | Ce que fait l'outil et quand s'en servir, montré au modèle. Obligatoire. Une ligne `Required parameters: …` y est ajoutée. | |
| 133 | 134 | | `tools[].parameters` | map nom → paramètre | `{}` | Les arguments, dans l'ordre écrit. Les noms suivent les règles de `name`. | |
| 134 | 135 | | `tools[].parameters.<p>.type` | chaîne | `string` | `string`, `number`, `integer` ou `boolean`. | |
| @@ -26,6 +26,7 @@ | |||
| 26 | | `fetchContentTool` | booléen | `false` | Déclare `fetch_content`, qui récupère des pages web en Markdown. Désactivé par défaut : c'est le seul outil intégré qui accède au réseau. | | 26 | | `fetchContentTool` | booléen | `false` | Déclare `fetch_content`, qui récupère des pages web en Markdown. Désactivé par défaut : c'est le seul outil intégré qui accède au réseau. | |
| 27 | | `askUserTool` | booléen | `true` | Déclare `ask_user`, par lequel le modèle pose des questions à l'utilisateur (choisir une option, en choisir plusieurs, taper un texte, oui ou non) et attend les réponses. `false` le retire : le modèle ne peut pas demander. Jamais déclaré à côté d'un `structuredOutput`. Surchargé par `AGENT_ASK_USER` ; pour un agent interrogé par un autre, aussi par le `mmConfigs.<nom>.askUser` de celui qui l'interroge. Voir [outils](tools.md#ask_user). | | 27 | | `askUserTool` | booléen | `true` | Déclare `ask_user`, par lequel le modèle pose des questions à l'utilisateur (choisir une option, en choisir plusieurs, taper un texte, oui ou non) et attend les réponses. `false` le retire : le modèle ne peut pas demander. Jamais déclaré à côté d'un `structuredOutput`. Surchargé par `AGENT_ASK_USER` ; pour un agent interrogé par un autre, aussi par le `mmConfigs.<nom>.askUser` de celui qui l'interroge. Voir [outils](tools.md#ask_user). | |
| 28 | | `saveAnswerTool` | bool | `false` | Déclare `save_answer`, qui écrit dans un nouveau fichier une réponse que le modèle a déjà donnée : le modèle envoie un nom de fichier, pas le texte une seconde fois. Désactivé par défaut. La commande `/save` existe dans tous les cas. Voir les [outils](tools.md#save_answer). | | 28 | | `saveAnswerTool` | bool | `false` | Déclare `save_answer`, qui écrit dans un nouveau fichier une réponse que le modèle a déjà donnée : le modèle envoie un nom de fichier, pas le texte une seconde fois. Désactivé par défaut. La commande `/save` existe dans tous les cas. Voir les [outils](tools.md#save_answer). | |
| 29 | +| `unloadModels` | bool | `true` | La commande `/unload [modèle]` (terminal, fenêtre, éditeur) et l'outil `unload_model`, qui libèrent un modèle de la mémoire du serveur : llmman et Ollama, llama-server en mode router. `false` retire les deux. L'outil n'est jamais déclaré à côté d'une `structuredOutput`. Voir les [outils](tools.md#unload_model) et [Libérer la mémoire d'un modèle](../how-to/unload-a-model.md). | | ||
| 29 | | `sampling` | dictionnaire | `temperature: 0.0`, `top_p: 0.9`, `max_tokens: 4096` | Paramètres de génération, noms de l'API OpenAI, transmis au serveur. Une clé que l'API OpenAI ne déclare pas est envoyée telle quelle, au premier niveau de la requête — par exemple `chat_template_kwargs: {enable_thinking: false}`, qui demande à un serveur llama.cpp (`dmr`, `llamacpp`, `llmman`) de ne pas raisonner ; `reasoning_effort: none` fait de même avec Ollama. `messages`, `tools`, `tool_choice` (et les anciens `functions`, `function_call`) sont ignorées : Genkit les construit à partir de la conversation. L'`agent.yaml` livré ajoute `parallel_tool_calls: false`. `max_tokens` borne aussi un appel d'outil : un `write_file` de long document (ou le raisonnement d'un modèle « thinking » plus l'appel) peut l'atteindre, et l'appel est alors coupé. mm le rend deux fois au modèle en lui demandant de procéder par morceaux, puis s'arrête avec `the model hit max_tokens (N) while writing a tool call` ; augmentez la valeur si cela arrive. En streaming, mm demande aussi au serveur le compte des tokens (`stream_options.include_usage`). | | 30 | | `sampling` | dictionnaire | `temperature: 0.0`, `top_p: 0.9`, `max_tokens: 4096` | Paramètres de génération, noms de l'API OpenAI, transmis au serveur. Une clé que l'API OpenAI ne déclare pas est envoyée telle quelle, au premier niveau de la requête — par exemple `chat_template_kwargs: {enable_thinking: false}`, qui demande à un serveur llama.cpp (`dmr`, `llamacpp`, `llmman`) de ne pas raisonner ; `reasoning_effort: none` fait de même avec Ollama. `messages`, `tools`, `tool_choice` (et les anciens `functions`, `function_call`) sont ignorées : Genkit les construit à partir de la conversation. L'`agent.yaml` livré ajoute `parallel_tool_calls: false`. `max_tokens` borne aussi un appel d'outil : un `write_file` de long document (ou le raisonnement d'un modèle « thinking » plus l'appel) peut l'atteindre, et l'appel est alors coupé. mm le rend deux fois au modèle en lui demandant de procéder par morceaux, puis s'arrête avec `the model hit max_tokens (N) while writing a tool call` ; augmentez la valeur si cela arrive. En streaming, mm demande aussi au serveur le compte des tokens (`stream_options.include_usage`). | |
| 30 | | `watchdogTimeout` | durée | `20s` | Silence toléré entre deux tokens diffusés avant de déclarer la connexion bloquée. Doublé pour les requêtes de résumé. | | 31 | | `watchdogTimeout` | durée | `20s` | Silence toléré entre deux tokens diffusés avant de déclarer la connexion bloquée. Doublé pour les requêtes de résumé. | |
| 31 | | `firstTokenTimeout` | durée | `3m` | Silence toléré avant le premier token d'un appel au modèle, pendant que le serveur lit le prompt (chaque aller-retour d'un tour en a un). Jamais plus court que `watchdogTimeout` ; doit être >= 0. Avec un serveur llama.cpp (`dmr`, `llamacpp`, `llmman` — dont les modèles GGUF locaux tournent dans llama-server), mm demande `return_progress` : chaque rapport d'avancement relance ce délai, et le spinner affiche `Reading the prompt 12.0k/25.3k tokens`. Ollama n'envoie pas d'avancement : cette seule attente couvre tout le prompt (et le chargement du modèle). À l'expiration, le watchdog dit `no first token from the server after …` (avant le premier token) ou `connection hang detected after …` (après). | | 32 | | `firstTokenTimeout` | durée | `3m` | Silence toléré avant le premier token d'un appel au modèle, pendant que le serveur lit le prompt (chaque aller-retour d'un tour en a un). Jamais plus court que `watchdogTimeout` ; doit être >= 0. Avec un serveur llama.cpp (`dmr`, `llamacpp`, `llmman` — dont les modèles GGUF locaux tournent dans llama-server), mm demande `return_progress` : chaque rapport d'avancement relance ce délai, et le spinner affiche `Reading the prompt 12.0k/25.3k tokens`. Ollama n'envoie pas d'avancement : cette seule attente couvre tout le prompt (et le chargement du modèle). À l'expiration, le watchdog dit `no first token from the server after …` (avant le premier token) ou `connection hang detected after …` (après). | |
| @@ -128,7 +129,7 @@ Le fichier livré `agent.lsp.yaml` est `agent.dmr.yaml` avec ce bloc actif (gopl | |||
| 128 | | Clé | Type | Défaut | Description | | 129 | | Clé | Type | Défaut | Description | |
| 129 | |-----|------|--------|-------------| | 130 | |-----|------|--------|-------------| |
| 130 | | `tools` | liste | `[]` | Une entrée par outil déclaré. | | 131 | | `tools` | liste | `[]` | Une entrée par outil déclaré. | |
| 131 | -| `tools[].name` | chaîne | — | Le nom que le modèle appelle. Obligatoire. Commence par une lettre ; lettres, chiffres, `_`, `-` ; 64 caractères au plus. Unique. Pas le nom d'un outil intégré (`ask_agent`, `ask_user`, `bash`, `edit_file`, `fetch_content`, `lsp`, `plan`, `read_file`, `read_skill`, `save_answer`, `write_file`), même désactivé dans cet agent. | | 132 | +| `tools[].name` | chaîne | — | Le nom que le modèle appelle. Obligatoire. Commence par une lettre ; lettres, chiffres, `_`, `-` ; 64 caractères au plus. Unique. Pas le nom d'un outil intégré (`ask_agent`, `ask_user`, `bash`, `edit_file`, `fetch_content`, `lsp`, `plan`, `read_file`, `read_skill`, `save_answer`, `unload_model`, `write_file`), même désactivé dans cet agent. | |
| 132 | | `tools[].description` | chaîne | — | Ce que fait l'outil et quand s'en servir, montré au modèle. Obligatoire. Une ligne `Required parameters: …` y est ajoutée. | | 133 | | `tools[].description` | chaîne | — | Ce que fait l'outil et quand s'en servir, montré au modèle. Obligatoire. Une ligne `Required parameters: …` y est ajoutée. | |
| 133 | | `tools[].parameters` | map nom → paramètre | `{}` | Les arguments, dans l'ordre écrit. Les noms suivent les règles de `name`. | | 134 | | `tools[].parameters` | map nom → paramètre | `{}` | Les arguments, dans l'ordre écrit. Les noms suivent les règles de `name`. | |
| 134 | | `tools[].parameters.<p>.type` | chaîne | `string` | `string`, `number`, `integer` ou `boolean`. | | 135 | | `tools[].parameters.<p>.type` | chaîne | `string` | `string`, `number`, `integer` ou `boolean`. | |
modified
docs/fr/reference/slash-commands.md +3 -1 | @@ -14,6 +14,7 @@ | ||
| 14 | 14 | | `/resume [id]` | Reprend une session enregistrée : la plus récemment mise à jour autre que la courante, en ignorant les vides, ou l'identifiant donné. Affiche `↩️ resumed <id> — N message(s), started with "<première question>"` (suivi de ` · ⚠️ N error(s)` quand la session a enregistré des erreurs) ; réinitialise le détecteur de boucles et le compte de tokens du serveur ; les tours suivants sont enregistrés dans cette session. Refusée avec une raison `⚠️` quand les sessions sont désactivées, qu'il n'y a pas de session antérieure, que l'identifiant est inconnu ou désigne la session courante. | Envoyée au modèle comme nouveau message. | |
| 15 | 15 | | `/help` | Affiche les commandes de cette façade, une par ligne avec leur description, alignées. | Envoyée au modèle comme nouveau message. | |
| 16 | 16 | | `/save <fichier> [n]` | Écrit la dernière réponse (`n` : la n-ième depuis la fin) telle quelle dans un nouveau fichier, en créant les répertoires manquants, et affiche `💾 saved the last answer to <fichier> (N bytes)`. N'écrase jamais : un fichier existant donne `⚠️ <fichier> already exists — nothing was written; choose another name`. Un nom de fichier avec des espaces n'est pas pris en charge. Le modèle n'est pas appelé. | Envoyée au modèle comme nouveau message. | |
| 17 | +| `/unload [modèle]` | Avec `unloadModels` (activé par défaut) : décharge un modèle de la mémoire du serveur — celui de l'agent, ou celui nommé — et affiche une ligne : `🧹 unloaded <modèle>`, `🧹 <modèle> was not loaded: nothing to unload`, ou `🧹 cannot unload <modèle>: <raison>` (un fournisseur ou un serveur qui n'a aucun moyen de le faire). L'historique n'est pas touché ; la question suivante recharge le modèle de l'agent. | Envoyée au modèle comme nouveau message. | | |
| 17 | 18 | | `/copy [n\|all]` | Copie la dernière réponse (`n` : la n-ième depuis la fin) dans le presse-papiers via `pbcopy`/`wl-copy`/`xclip`/`xsel` ou, à défaut, OSC 52 quand stdout est un terminal, et affiche `📋 copied N chars (<chemin>)` ou `⚠️ …`. `/copy all` est réservé au mode fenêtre : ici la commande le dit. | Envoyée au modèle comme nouveau message. | |
| 18 | 19 | |
| 19 | 20 | Avec `sessions.enabled`, `/new` laisse le fichier de la session courante tel quel ; la question suivante ouvre un nouveau fichier. Une ligne vide à l'invite est ignorée. |
| @@ -169,12 +170,13 @@ La notation marche aussi en mode ligne et en ACP, tapée à la main : un `$skill | ||
| 169 | 170 | |
| 170 | 171 | ## Mode ACP |
| 171 | 172 | |
| 172 | -Deux commandes existent en mode ACP : `/new` et `/compact`. Elles sont annoncées à l'éditeur par une notification `available_commands_update` et interceptées par `session/prompt` quand le texte du prompt, débarrassé de ses espaces, est exactement la commande. Chacune répond par un `agent_message_chunk` portant la ligne que le terminal afficherait, puis `stopReason: end_turn` ; le modèle n'est pas consulté. | |
| 173 | +Deux commandes existent en mode ACP, trois avec `unloadModels` (le défaut) : `/new`, `/compact` et `/unload`. Elles sont annoncées à l'éditeur par une notification `available_commands_update` et interceptées par `session/prompt` quand le texte du prompt, débarrassé de ses espaces, est exactement la commande (`/unload` peut être suivie d'un nom de modèle). Chacune répond par un `agent_message_chunk` portant la ligne que le terminal afficherait, puis `stopReason: end_turn` ; le modèle n'est pas consulté. | |
| 173 | 174 | |
| 174 | 175 | | Commande | En ACP | |
| 175 | 176 | |----------|--------| |
| 176 | 177 | | `/new` | Comme dans le terminal : historique ramené au prompt système, compte de tokens oublié ; l'identifiant de session, le `cwd` et les autorisations « toujours autoriser » sont conservés. | |
| 177 | 178 | | `/compact` | Comme dans le terminal, toujours forcée : `🗜️ compressed …`, `🗜️ nothing to compact: …` ou `[compact: failed, history kept: …]`. Attend d'abord la fin d'un tour en cours. | |
| 179 | +| `/unload [modèle]` | Avec `unloadModels` (activé par défaut), annoncée avec l'indication de saisie `[model]` : comme dans le terminal, la ligne `🧹` comme texte de l'agent ; l'historique n'est pas touché. | | |
| 178 | 180 | |
| 179 | 181 | Chaque **skill** y est proposé comme une commande à part entière : ACP n'a pas de sélecteur pour les skills, donc `/greetings Bob` dans le menu de l'éditeur est l'équivalent de `$greetings Bob` dans la fenêtre, et il est réécrit en exactement ça — voir la [référence ACP](acp.md#les-skills-comme-commandes). |
| 180 | 182 | |
| @@ -14,6 +14,7 @@ | |||
| 14 | | `/resume [id]` | Reprend une session enregistrée : la plus récemment mise à jour autre que la courante, en ignorant les vides, ou l'identifiant donné. Affiche `↩️ resumed <id> — N message(s), started with "<première question>"` (suivi de ` · ⚠️ N error(s)` quand la session a enregistré des erreurs) ; réinitialise le détecteur de boucles et le compte de tokens du serveur ; les tours suivants sont enregistrés dans cette session. Refusée avec une raison `⚠️` quand les sessions sont désactivées, qu'il n'y a pas de session antérieure, que l'identifiant est inconnu ou désigne la session courante. | Envoyée au modèle comme nouveau message. | | 14 | | `/resume [id]` | Reprend une session enregistrée : la plus récemment mise à jour autre que la courante, en ignorant les vides, ou l'identifiant donné. Affiche `↩️ resumed <id> — N message(s), started with "<première question>"` (suivi de ` · ⚠️ N error(s)` quand la session a enregistré des erreurs) ; réinitialise le détecteur de boucles et le compte de tokens du serveur ; les tours suivants sont enregistrés dans cette session. Refusée avec une raison `⚠️` quand les sessions sont désactivées, qu'il n'y a pas de session antérieure, que l'identifiant est inconnu ou désigne la session courante. | Envoyée au modèle comme nouveau message. | |
| 15 | | `/help` | Affiche les commandes de cette façade, une par ligne avec leur description, alignées. | Envoyée au modèle comme nouveau message. | | 15 | | `/help` | Affiche les commandes de cette façade, une par ligne avec leur description, alignées. | Envoyée au modèle comme nouveau message. | |
| 16 | | `/save <fichier> [n]` | Écrit la dernière réponse (`n` : la n-ième depuis la fin) telle quelle dans un nouveau fichier, en créant les répertoires manquants, et affiche `💾 saved the last answer to <fichier> (N bytes)`. N'écrase jamais : un fichier existant donne `⚠️ <fichier> already exists — nothing was written; choose another name`. Un nom de fichier avec des espaces n'est pas pris en charge. Le modèle n'est pas appelé. | Envoyée au modèle comme nouveau message. | | 16 | | `/save <fichier> [n]` | Écrit la dernière réponse (`n` : la n-ième depuis la fin) telle quelle dans un nouveau fichier, en créant les répertoires manquants, et affiche `💾 saved the last answer to <fichier> (N bytes)`. N'écrase jamais : un fichier existant donne `⚠️ <fichier> already exists — nothing was written; choose another name`. Un nom de fichier avec des espaces n'est pas pris en charge. Le modèle n'est pas appelé. | Envoyée au modèle comme nouveau message. | |
| 17 | +| `/unload [modèle]` | Avec `unloadModels` (activé par défaut) : décharge un modèle de la mémoire du serveur — celui de l'agent, ou celui nommé — et affiche une ligne : `🧹 unloaded <modèle>`, `🧹 <modèle> was not loaded: nothing to unload`, ou `🧹 cannot unload <modèle>: <raison>` (un fournisseur ou un serveur qui n'a aucun moyen de le faire). L'historique n'est pas touché ; la question suivante recharge le modèle de l'agent. | Envoyée au modèle comme nouveau message. | | ||
| 17 | | `/copy [n\|all]` | Copie la dernière réponse (`n` : la n-ième depuis la fin) dans le presse-papiers via `pbcopy`/`wl-copy`/`xclip`/`xsel` ou, à défaut, OSC 52 quand stdout est un terminal, et affiche `📋 copied N chars (<chemin>)` ou `⚠️ …`. `/copy all` est réservé au mode fenêtre : ici la commande le dit. | Envoyée au modèle comme nouveau message. | | 18 | | `/copy [n\|all]` | Copie la dernière réponse (`n` : la n-ième depuis la fin) dans le presse-papiers via `pbcopy`/`wl-copy`/`xclip`/`xsel` ou, à défaut, OSC 52 quand stdout est un terminal, et affiche `📋 copied N chars (<chemin>)` ou `⚠️ …`. `/copy all` est réservé au mode fenêtre : ici la commande le dit. | Envoyée au modèle comme nouveau message. | |
| 18 | 19 | ||
| 19 | Avec `sessions.enabled`, `/new` laisse le fichier de la session courante tel quel ; la question suivante ouvre un nouveau fichier. Une ligne vide à l'invite est ignorée. | 20 | Avec `sessions.enabled`, `/new` laisse le fichier de la session courante tel quel ; la question suivante ouvre un nouveau fichier. Une ligne vide à l'invite est ignorée. |
| @@ -169,12 +170,13 @@ La notation marche aussi en mode ligne et en ACP, tapée à la main : un `$skill | |||
| 169 | 170 | ||
| 170 | ## Mode ACP | 171 | ## Mode ACP |
| 171 | 172 | ||
| 172 | -Deux commandes existent en mode ACP : `/new` et `/compact`. Elles sont annoncées à l'éditeur par une notification `available_commands_update` et interceptées par `session/prompt` quand le texte du prompt, débarrassé de ses espaces, est exactement la commande. Chacune répond par un `agent_message_chunk` portant la ligne que le terminal afficherait, puis `stopReason: end_turn` ; le modèle n'est pas consulté. | 173 | +Deux commandes existent en mode ACP, trois avec `unloadModels` (le défaut) : `/new`, `/compact` et `/unload`. Elles sont annoncées à l'éditeur par une notification `available_commands_update` et interceptées par `session/prompt` quand le texte du prompt, débarrassé de ses espaces, est exactement la commande (`/unload` peut être suivie d'un nom de modèle). Chacune répond par un `agent_message_chunk` portant la ligne que le terminal afficherait, puis `stopReason: end_turn` ; le modèle n'est pas consulté. |
| 173 | 174 | ||
| 174 | | Commande | En ACP | | 175 | | Commande | En ACP | |
| 175 | |----------|--------| | 176 | |----------|--------| |
| 176 | | `/new` | Comme dans le terminal : historique ramené au prompt système, compte de tokens oublié ; l'identifiant de session, le `cwd` et les autorisations « toujours autoriser » sont conservés. | | 177 | | `/new` | Comme dans le terminal : historique ramené au prompt système, compte de tokens oublié ; l'identifiant de session, le `cwd` et les autorisations « toujours autoriser » sont conservés. | |
| 177 | | `/compact` | Comme dans le terminal, toujours forcée : `🗜️ compressed …`, `🗜️ nothing to compact: …` ou `[compact: failed, history kept: …]`. Attend d'abord la fin d'un tour en cours. | | 178 | | `/compact` | Comme dans le terminal, toujours forcée : `🗜️ compressed …`, `🗜️ nothing to compact: …` ou `[compact: failed, history kept: …]`. Attend d'abord la fin d'un tour en cours. | |
| 179 | +| `/unload [modèle]` | Avec `unloadModels` (activé par défaut), annoncée avec l'indication de saisie `[model]` : comme dans le terminal, la ligne `🧹` comme texte de l'agent ; l'historique n'est pas touché. | | ||
| 178 | 180 | ||
| 179 | Chaque **skill** y est proposé comme une commande à part entière : ACP n'a pas de sélecteur pour les skills, donc `/greetings Bob` dans le menu de l'éditeur est l'équivalent de `$greetings Bob` dans la fenêtre, et il est réécrit en exactement ça — voir la [référence ACP](acp.md#les-skills-comme-commandes). | 181 | Chaque **skill** y est proposé comme une commande à part entière : ACP n'a pas de sélecteur pour les skills, donc `/greetings Bob` dans le menu de l'éditeur est l'équivalent de `$greetings Bob` dans la fenêtre, et il est réécrit en exactement ça — voir la [référence ACP](acp.md#les-skills-comme-commandes). |
| 180 | 182 | ||
modified
docs/fr/reference/tools.md +20 -0 | @@ -268,6 +268,26 @@ Déclaré quand `saveAnswerTool: true` (défaut `false`). | ||
| 268 | 268 | | Affichage terminal | `💾 save_answer: <path> (this answer\|the previous answer)`, puis le résultat. | |
| 269 | 269 | | ACP et fenêtre | Appel d'outil de type `edit`, avec le chemin absolu du fichier. | |
| 270 | 270 | |
| 271 | +## `unload_model` | |
| 272 | + | |
| 273 | +Déclaré quand `unloadModels` vaut true (le défaut), sauf à côté d'une `structuredOutput`. | |
| 274 | + | |
| 275 | +| Champ | Type | Description | | |
| 276 | +|-------|------|-------------| | |
| 277 | +| `model` | chaîne | Le modèle à décharger, tel que le serveur le nomme. Vide : le modèle de l'agent. | | |
| 278 | + | |
| 279 | +| Aspect | Comportement | | |
| 280 | +|--------|--------------| | |
| 281 | +| llmman, `ollama` | Cherche d'abord le modèle dans `GET /api/ps` ; quand il est chargé, `POST /api/generate {"model": …, "keep_alive": 0}` et attend `done_reason: "unload"`. Un modèle non chargé n'est jamais demandé : la requête le chargerait d'abord. | | |
| 282 | +| `llamacpp` | `POST /models/unload {"model": …}`, qui n'existe qu'en mode router (llama-server lancé sans modèle). Le 404 d'un serveur à modèle unique donne la raison et les alternatives : mode router, `--sleep-idle-seconds N`, arrêt du serveur. | | |
| 283 | +| `dmr` | `cannot unload <modèle>: the dmr provider cannot unload a model`. | | |
| 284 | +| Résultat | Une ligne : `unloaded <modèle>`, `<modèle> was not loaded: nothing to unload`, ou `cannot unload <modèle>: <raison>` (serveur injoignable, erreur du serveur lui-même, absence de confirmation). | | |
| 285 | +| Son propre modèle | Décharger le modèle de l'agent ne dure que jusqu'à son étape suivante : le modèle est nécessaire pour lire le résultat, et le serveur le recharge. | | |
| 286 | +| Permission | Jamais demandée, même avec `yolo: false` : rien ne change sur disque, et le modèle se recharge tout seul. | | |
| 287 | +| Affichage terminal | `🧹 unload_model: <modèle>` (`(own model)` quand il est vide), puis le résultat. | | |
| 288 | +| ACP et fenêtre | Appel d'outil de type `other` ; un résultat `cannot unload` est un appel en échec. | | |
| 289 | +| Délai | 30 s par requête au serveur. | | |
| 290 | + | |
| 271 | 291 | ## Outils déclarés |
| 272 | 292 | |
| 273 | 293 | Déclarés à partir du bloc `tools:` du fichier d'agent, un outil par entrée (voir la [configuration](configuration.md#clés-de-tools)). Chacun porte le `name` de l'entrée ; ses champs sont les `parameters` de l'entrée. |
| @@ -268,6 +268,26 @@ Déclaré quand `saveAnswerTool: true` (défaut `false`). | |||
| 268 | | Affichage terminal | `💾 save_answer: <path> (this answer\|the previous answer)`, puis le résultat. | | 268 | | Affichage terminal | `💾 save_answer: <path> (this answer\|the previous answer)`, puis le résultat. | |
| 269 | | ACP et fenêtre | Appel d'outil de type `edit`, avec le chemin absolu du fichier. | | 269 | | ACP et fenêtre | Appel d'outil de type `edit`, avec le chemin absolu du fichier. | |
| 270 | 270 | ||
| 271 | +## `unload_model` | ||
| 272 | + | ||
| 273 | +Déclaré quand `unloadModels` vaut true (le défaut), sauf à côté d'une `structuredOutput`. | ||
| 274 | + | ||
| 275 | +| Champ | Type | Description | | ||
| 276 | +|-------|------|-------------| | ||
| 277 | +| `model` | chaîne | Le modèle à décharger, tel que le serveur le nomme. Vide : le modèle de l'agent. | | ||
| 278 | + | ||
| 279 | +| Aspect | Comportement | | ||
| 280 | +|--------|--------------| | ||
| 281 | +| llmman, `ollama` | Cherche d'abord le modèle dans `GET /api/ps` ; quand il est chargé, `POST /api/generate {"model": …, "keep_alive": 0}` et attend `done_reason: "unload"`. Un modèle non chargé n'est jamais demandé : la requête le chargerait d'abord. | | ||
| 282 | +| `llamacpp` | `POST /models/unload {"model": …}`, qui n'existe qu'en mode router (llama-server lancé sans modèle). Le 404 d'un serveur à modèle unique donne la raison et les alternatives : mode router, `--sleep-idle-seconds N`, arrêt du serveur. | | ||
| 283 | +| `dmr` | `cannot unload <modèle>: the dmr provider cannot unload a model`. | | ||
| 284 | +| Résultat | Une ligne : `unloaded <modèle>`, `<modèle> was not loaded: nothing to unload`, ou `cannot unload <modèle>: <raison>` (serveur injoignable, erreur du serveur lui-même, absence de confirmation). | | ||
| 285 | +| Son propre modèle | Décharger le modèle de l'agent ne dure que jusqu'à son étape suivante : le modèle est nécessaire pour lire le résultat, et le serveur le recharge. | | ||
| 286 | +| Permission | Jamais demandée, même avec `yolo: false` : rien ne change sur disque, et le modèle se recharge tout seul. | | ||
| 287 | +| Affichage terminal | `🧹 unload_model: <modèle>` (`(own model)` quand il est vide), puis le résultat. | | ||
| 288 | +| ACP et fenêtre | Appel d'outil de type `other` ; un résultat `cannot unload` est un appel en échec. | | ||
| 289 | +| Délai | 30 s par requête au serveur. | | ||
| 290 | + | ||
| 271 | ## Outils déclarés | 291 | ## Outils déclarés |
| 272 | 292 | ||
| 273 | Déclarés à partir du bloc `tools:` du fichier d'agent, un outil par entrée (voir la [configuration](configuration.md#clés-de-tools)). Chacun porte le `name` de l'entrée ; ses champs sont les `parameters` de l'entrée. | 293 | Déclarés à partir du bloc `tools:` du fichier d'agent, un outil par entrée (voir la [configuration](configuration.md#clés-de-tools)). Chacun porte le `name` de l'entrée ; ses champs sont les `parameters` de l'entrée. |
modified
internal/acp/acp.go +12 -0 | @@ -492,9 +492,21 @@ func (f *front) slashCommand(ctx context.Context, sid sdk.SessionId, s *session, | ||
| 492 | 492 | case command.Is(input, command.Compact): |
| 493 | 493 | return f.compactSession(ctx, sid, s), true |
| 494 | 494 | } |
| 495 | + if cmd, arg, ok := command.Parse(input); ok && cmd.Name == command.Unload { | |
| 496 | + return f.unloadModel(ctx, sid, s, arg), true | |
| 497 | + } | |
| 495 | 498 | return sdk.PromptResponse{}, false |
| 496 | 499 | } |
| 497 | 500 | |
| 501 | +// unloadModel is /unload [model] over ACP: the engine's one line, sent as the | |
| 502 | +// agent's answer; the history is not touched. | |
| 503 | +func (f *front) unloadModel(ctx context.Context, sid sdk.SessionId, s *session, model string) sdk.PromptResponse { | |
| 504 | + line := "🧹 " + f.e.Unload(ctx, model) | |
| 505 | + fmt.Fprintf(os.Stderr, "[acp] session %s: /unload — %s\n", sid, line) | |
| 506 | + _ = f.conn.SessionUpdate(ctx, sdk.SessionNotification{SessionId: sid, Update: sdk.UpdateAgentMessageText(line)}) | |
| 507 | + return sdk.PromptResponse{StopReason: sdk.StopReasonEndTurn, Usage: s.usage.totals()} | |
| 508 | +} | |
| 509 | + | |
| 498 | 510 | // skillCommand turns a slash command that names a skill into the `$skill` |
| 499 | 511 | // notation, so one code path — skills.ExpandRefs, shared by all three front |
| 500 | 512 | // ends — decides what a named skill means. |
| @@ -492,9 +492,21 @@ func (f *front) slashCommand(ctx context.Context, sid sdk.SessionId, s *session, | |||
| 492 | case command.Is(input, command.Compact): | 492 | case command.Is(input, command.Compact): |
| 493 | return f.compactSession(ctx, sid, s), true | 493 | return f.compactSession(ctx, sid, s), true |
| 494 | } | 494 | } |
| 495 | + if cmd, arg, ok := command.Parse(input); ok && cmd.Name == command.Unload { | ||
| 496 | + return f.unloadModel(ctx, sid, s, arg), true | ||
| 497 | + } | ||
| 495 | return sdk.PromptResponse{}, false | 498 | return sdk.PromptResponse{}, false |
| 496 | } | 499 | } |
| 497 | 500 | ||
| 501 | +// unloadModel is /unload [model] over ACP: the engine's one line, sent as the | ||
| 502 | +// agent's answer; the history is not touched. | ||
| 503 | +func (f *front) unloadModel(ctx context.Context, sid sdk.SessionId, s *session, model string) sdk.PromptResponse { | ||
| 504 | + line := "🧹 " + f.e.Unload(ctx, model) | ||
| 505 | + fmt.Fprintf(os.Stderr, "[acp] session %s: /unload — %s\n", sid, line) | ||
| 506 | + _ = f.conn.SessionUpdate(ctx, sdk.SessionNotification{SessionId: sid, Update: sdk.UpdateAgentMessageText(line)}) | ||
| 507 | + return sdk.PromptResponse{StopReason: sdk.StopReasonEndTurn, Usage: s.usage.totals()} | ||
| 508 | +} | ||
| 509 | + | ||
| 498 | // skillCommand turns a slash command that names a skill into the `$skill` | 510 | // skillCommand turns a slash command that names a skill into the `$skill` |
| 499 | // notation, so one code path — skills.ExpandRefs, shared by all three front | 511 | // notation, so one code path — skills.ExpandRefs, shared by all three front |
| 500 | // ends — decides what a named skill means. | 512 | // ends — decides what a named skill means. |
added
internal/acp/unload_test.go +55 -0 | new file mode 100644 | ||
| @@ -0,0 +1,55 @@ | ||
| 1 | +package acp | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "strings" | |
| 5 | + "testing" | |
| 6 | + | |
| 7 | + "mm/internal/command" | |
| 8 | + "mm/internal/engine" | |
| 9 | + | |
| 10 | + sdk "github.com/coder/acp-go-sdk" | |
| 11 | +) | |
| 12 | + | |
| 13 | +// With unloadModels on, /unload is advertised to the editor with an input | |
| 14 | +// hint, and session/prompt handles it: the engine's line comes back as the | |
| 15 | +// agent's text, the turn ends, and the history is untouched. | |
| 16 | +func TestUnloadOverACP(t *testing.T) { | |
| 17 | + command.SetUnload(true) | |
| 18 | + t.Cleanup(func() { command.SetUnload(false) }) | |
| 19 | + | |
| 20 | + advertised := false | |
| 21 | + for _, c := range availableCommands() { | |
| 22 | + if c.Name == command.Unload { | |
| 23 | + advertised = c.Input != nil | |
| 24 | + } | |
| 25 | + } | |
| 26 | + if !advertised { | |
| 27 | + t.Error("/unload is not advertised with an input hint") | |
| 28 | + } | |
| 29 | + | |
| 30 | + w := newWire(t, "sys") | |
| 31 | + f := frontOf(t, w) | |
| 32 | + dmr, _ := engine.Lookup("dmr") | |
| 33 | + f.e = &engine.Engine{Provider: dmr, Backend: engine.Backend{Provider: "dmr", Model: "m"}} | |
| 34 | + w.send(1, "session/new", `{"cwd":"`+t.TempDir()+`","mcpServers":[]}`) | |
| 35 | + _, resp := w.until(1) | |
| 36 | + sid := resp["result"].(map[string]any)["sessionId"].(string) | |
| 37 | + | |
| 38 | + w.send(2, "session/prompt", `{"sessionId":"`+sid+`","prompt":[{"type":"text","text":"/unload"}]}`) | |
| 39 | + notes, resp := w.until(2) | |
| 40 | + if stop := resp["result"].(map[string]any)["stopReason"]; stop != "end_turn" { | |
| 41 | + t.Errorf("stopReason = %v", stop) | |
| 42 | + } | |
| 43 | + var text strings.Builder | |
| 44 | + for _, n := range notes { | |
| 45 | + if n["sessionUpdate"] == "agent_message_chunk" { | |
| 46 | + text.WriteString(n["content"].(map[string]any)["text"].(string)) | |
| 47 | + } | |
| 48 | + } | |
| 49 | + if got := text.String(); got != "🧹 cannot unload m: the dmr provider cannot unload a model" { | |
| 50 | + t.Errorf("agent text = %q", got) | |
| 51 | + } | |
| 52 | + if msgs := f.sessions[sdk.SessionId(sid)].messages; len(msgs) != 1 { | |
| 53 | + t.Errorf("history: %d messages, want the system prompt alone", len(msgs)) | |
| 54 | + } | |
| 55 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,55 @@ | |||
| 1 | +package acp | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "strings" | ||
| 5 | + "testing" | ||
| 6 | + | ||
| 7 | + "mm/internal/command" | ||
| 8 | + "mm/internal/engine" | ||
| 9 | + | ||
| 10 | + sdk "github.com/coder/acp-go-sdk" | ||
| 11 | +) | ||
| 12 | + | ||
| 13 | +// With unloadModels on, /unload is advertised to the editor with an input | ||
| 14 | +// hint, and session/prompt handles it: the engine's line comes back as the | ||
| 15 | +// agent's text, the turn ends, and the history is untouched. | ||
| 16 | +func TestUnloadOverACP(t *testing.T) { | ||
| 17 | + command.SetUnload(true) | ||
| 18 | + t.Cleanup(func() { command.SetUnload(false) }) | ||
| 19 | + | ||
| 20 | + advertised := false | ||
| 21 | + for _, c := range availableCommands() { | ||
| 22 | + if c.Name == command.Unload { | ||
| 23 | + advertised = c.Input != nil | ||
| 24 | + } | ||
| 25 | + } | ||
| 26 | + if !advertised { | ||
| 27 | + t.Error("/unload is not advertised with an input hint") | ||
| 28 | + } | ||
| 29 | + | ||
| 30 | + w := newWire(t, "sys") | ||
| 31 | + f := frontOf(t, w) | ||
| 32 | + dmr, _ := engine.Lookup("dmr") | ||
| 33 | + f.e = &engine.Engine{Provider: dmr, Backend: engine.Backend{Provider: "dmr", Model: "m"}} | ||
| 34 | + w.send(1, "session/new", `{"cwd":"`+t.TempDir()+`","mcpServers":[]}`) | ||
| 35 | + _, resp := w.until(1) | ||
| 36 | + sid := resp["result"].(map[string]any)["sessionId"].(string) | ||
| 37 | + | ||
| 38 | + w.send(2, "session/prompt", `{"sessionId":"`+sid+`","prompt":[{"type":"text","text":"/unload"}]}`) | ||
| 39 | + notes, resp := w.until(2) | ||
| 40 | + if stop := resp["result"].(map[string]any)["stopReason"]; stop != "end_turn" { | ||
| 41 | + t.Errorf("stopReason = %v", stop) | ||
| 42 | + } | ||
| 43 | + var text strings.Builder | ||
| 44 | + for _, n := range notes { | ||
| 45 | + if n["sessionUpdate"] == "agent_message_chunk" { | ||
| 46 | + text.WriteString(n["content"].(map[string]any)["text"].(string)) | ||
| 47 | + } | ||
| 48 | + } | ||
| 49 | + if got := text.String(); got != "🧹 cannot unload m: the dmr provider cannot unload a model" { | ||
| 50 | + t.Errorf("agent text = %q", got) | ||
| 51 | + } | ||
| 52 | + if msgs := f.sessions[sdk.SessionId(sid)].messages; len(msgs) != 1 { | ||
| 53 | + t.Errorf("history: %d messages, want the system prompt alone", len(msgs)) | ||
| 54 | + } | ||
| 55 | +} | ||
modified
internal/agent/agent.go +2 -0 | @@ -562,6 +562,8 @@ func (l *loop) historyCommand(input string, messages []*ai.Message) ([]*ai.Messa | ||
| 562 | 562 | fmt.Println(copyAnswer(messages, arg, clipboard.New(terminalOSC52()))) |
| 563 | 563 | case command.Save: |
| 564 | 564 | fmt.Println(saved.Command(modelAnswers(messages), arg)) |
| 565 | + case command.Unload: | |
| 566 | + fmt.Println("🧹 " + l.e.Unload(l.ctx, arg)) | |
| 565 | 567 | case command.Resume: |
| 566 | 568 | resumed, line, err := l.keep.Resume(arg, agentsmd.System(l.system, "")) |
| 567 | 569 | if err != nil { |
| @@ -562,6 +562,8 @@ func (l *loop) historyCommand(input string, messages []*ai.Message) ([]*ai.Messa | |||
| 562 | fmt.Println(copyAnswer(messages, arg, clipboard.New(terminalOSC52()))) | 562 | fmt.Println(copyAnswer(messages, arg, clipboard.New(terminalOSC52()))) |
| 563 | case command.Save: | 563 | case command.Save: |
| 564 | fmt.Println(saved.Command(modelAnswers(messages), arg)) | 564 | fmt.Println(saved.Command(modelAnswers(messages), arg)) |
| 565 | + case command.Unload: | ||
| 566 | + fmt.Println("🧹 " + l.e.Unload(l.ctx, arg)) | ||
| 565 | case command.Resume: | 567 | case command.Resume: |
| 566 | resumed, line, err := l.keep.Resume(arg, agentsmd.System(l.system, "")) | 568 | resumed, line, err := l.keep.Resume(arg, agentsmd.System(l.system, "")) |
| 567 | if err != nil { | 569 | if err != nil { |
modified
internal/command/command.go +7 -3 | @@ -41,6 +41,7 @@ const ( | ||
| 41 | 41 | Help = "help" |
| 42 | 42 | Copy = "copy" |
| 43 | 43 | Save = "save" |
| 44 | + Unload = "unload" | |
| 44 | 45 | ) |
| 45 | 46 | |
| 46 | 47 | // Command is one slash command: what it is called, what it does, who offers |
| @@ -76,9 +77,12 @@ var table = []Command{ | ||
| 76 | 77 | {Name: Save, Description: "Save the last answer to a file, never overwriting one (/save notes.md 2: the answer before)", Fronts: Terminal | Window, TakesArg: true, Hint: "<file> [n]"}, |
| 77 | 78 | } |
| 78 | 79 | |
| 79 | -// All returns the whole table, in display order: the fixed commands, then | |
| 80 | -// the /ask family generated from the configured agents (ask.go). | |
| 81 | -func All() []Command { return append(append([]Command(nil), table...), askCommands...) } | |
| 80 | +// All returns the whole table, in display order: the fixed commands, /unload | |
| 81 | +// when the configuration allows it (unload.go), then the /ask family generated | |
| 82 | +// from the configured agents (ask.go). | |
| 83 | +func All() []Command { | |
| 84 | + return append(append(append([]Command(nil), table...), unloadCommands...), askCommands...) | |
| 85 | +} | |
| 82 | 86 | |
| 83 | 87 | // For returns the commands front f offers, in display order. |
| 84 | 88 | func For(f Front) []Command { |
| @@ -41,6 +41,7 @@ const ( | |||
| 41 | Help = "help" | 41 | Help = "help" |
| 42 | Copy = "copy" | 42 | Copy = "copy" |
| 43 | Save = "save" | 43 | Save = "save" |
| 44 | + Unload = "unload" | ||
| 44 | ) | 45 | ) |
| 45 | 46 | ||
| 46 | // Command is one slash command: what it is called, what it does, who offers | 47 | // Command is one slash command: what it is called, what it does, who offers |
| @@ -76,9 +77,12 @@ var table = []Command{ | |||
| 76 | {Name: Save, Description: "Save the last answer to a file, never overwriting one (/save notes.md 2: the answer before)", Fronts: Terminal | Window, TakesArg: true, Hint: "<file> [n]"}, | 77 | {Name: Save, Description: "Save the last answer to a file, never overwriting one (/save notes.md 2: the answer before)", Fronts: Terminal | Window, TakesArg: true, Hint: "<file> [n]"}, |
| 77 | } | 78 | } |
| 78 | 79 | ||
| 79 | -// All returns the whole table, in display order: the fixed commands, then | 80 | +// All returns the whole table, in display order: the fixed commands, /unload |
| 80 | -// the /ask family generated from the configured agents (ask.go). | 81 | +// when the configuration allows it (unload.go), then the /ask family generated |
| 81 | -func All() []Command { return append(append([]Command(nil), table...), askCommands...) } | 82 | +// from the configured agents (ask.go). |
| 83 | +func All() []Command { | ||
| 84 | + return append(append(append([]Command(nil), table...), unloadCommands...), askCommands...) | ||
| 85 | +} | ||
| 82 | 86 | ||
| 83 | // For returns the commands front f offers, in display order. | 87 | // For returns the commands front f offers, in display order. |
| 84 | func For(f Front) []Command { | 88 | func For(f Front) []Command { |
added
internal/command/unload.go +23 -0 | new file mode 100644 | ||
| @@ -0,0 +1,23 @@ | ||
| 1 | +package command | |
| 2 | + | |
| 3 | +// unloadCommands holds /unload when config `unloadModels` allows it; empty | |
| 4 | +// otherwise, so /help and the menus do not show a command that is off. | |
| 5 | +var unloadCommands []Command | |
| 6 | + | |
| 7 | +// SetUnload adds /unload [model] to every front end, the editor included, or | |
| 8 | +// removes it. Called once by main after the config is loaded. | |
| 9 | +// | |
| 10 | +// command.SetUnload(config.Cfg.UnloadModels) | |
| 11 | +// cmd, arg, _ := command.Parse("/unload qwen3.5:2b") // Unload, "qwen3.5:2b" | |
| 12 | +func SetUnload(enabled bool) { | |
| 13 | + unloadCommands = nil | |
| 14 | + if enabled { | |
| 15 | + unloadCommands = []Command{{ | |
| 16 | + Name: Unload, | |
| 17 | + Description: "Unload a model from the server's memory: this agent's, or the one named (the next question reloads it)", | |
| 18 | + Fronts: Terminal | Editor | Window, | |
| 19 | + TakesArg: true, | |
| 20 | + Hint: "[model]", | |
| 21 | + }} | |
| 22 | + } | |
| 23 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,23 @@ | |||
| 1 | +package command | ||
| 2 | + | ||
| 3 | +// unloadCommands holds /unload when config `unloadModels` allows it; empty | ||
| 4 | +// otherwise, so /help and the menus do not show a command that is off. | ||
| 5 | +var unloadCommands []Command | ||
| 6 | + | ||
| 7 | +// SetUnload adds /unload [model] to every front end, the editor included, or | ||
| 8 | +// removes it. Called once by main after the config is loaded. | ||
| 9 | +// | ||
| 10 | +// command.SetUnload(config.Cfg.UnloadModels) | ||
| 11 | +// cmd, arg, _ := command.Parse("/unload qwen3.5:2b") // Unload, "qwen3.5:2b" | ||
| 12 | +func SetUnload(enabled bool) { | ||
| 13 | + unloadCommands = nil | ||
| 14 | + if enabled { | ||
| 15 | + unloadCommands = []Command{{ | ||
| 16 | + Name: Unload, | ||
| 17 | + Description: "Unload a model from the server's memory: this agent's, or the one named (the next question reloads it)", | ||
| 18 | + Fronts: Terminal | Editor | Window, | ||
| 19 | + TakesArg: true, | ||
| 20 | + Hint: "[model]", | ||
| 21 | + }} | ||
| 22 | + } | ||
| 23 | +} | ||
added
internal/command/unload_test.go +30 -0 | new file mode 100644 | ||
| @@ -0,0 +1,30 @@ | ||
| 1 | +package command | |
| 2 | + | |
| 3 | +import "testing" | |
| 4 | + | |
| 5 | +func TestSetUnloadAddsAndRemovesTheCommand(t *testing.T) { | |
| 6 | + t.Cleanup(func() { SetUnload(false) }) | |
| 7 | + | |
| 8 | + SetUnload(true) | |
| 9 | + cmd, arg, ok := Parse("/unload qwen3.5:2b") | |
| 10 | + if !ok || cmd.Name != Unload || arg != "qwen3.5:2b" { | |
| 11 | + t.Fatalf("Parse = %+v %q %v", cmd, arg, ok) | |
| 12 | + } | |
| 13 | + if !Is("/unload", Unload) { | |
| 14 | + t.Error("/unload alone is not the command") | |
| 15 | + } | |
| 16 | + for _, f := range []Front{Terminal, Window, Editor} { | |
| 17 | + found := false | |
| 18 | + for _, c := range For(f) { | |
| 19 | + found = found || c.Name == Unload | |
| 20 | + } | |
| 21 | + if !found { | |
| 22 | + t.Errorf("front %d does not offer /unload", f) | |
| 23 | + } | |
| 24 | + } | |
| 25 | + | |
| 26 | + SetUnload(false) | |
| 27 | + if _, _, ok := Parse("/unload"); ok { | |
| 28 | + t.Error("/unload still parsed with unloadModels off") | |
| 29 | + } | |
| 30 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,30 @@ | |||
| 1 | +package command | ||
| 2 | + | ||
| 3 | +import "testing" | ||
| 4 | + | ||
| 5 | +func TestSetUnloadAddsAndRemovesTheCommand(t *testing.T) { | ||
| 6 | + t.Cleanup(func() { SetUnload(false) }) | ||
| 7 | + | ||
| 8 | + SetUnload(true) | ||
| 9 | + cmd, arg, ok := Parse("/unload qwen3.5:2b") | ||
| 10 | + if !ok || cmd.Name != Unload || arg != "qwen3.5:2b" { | ||
| 11 | + t.Fatalf("Parse = %+v %q %v", cmd, arg, ok) | ||
| 12 | + } | ||
| 13 | + if !Is("/unload", Unload) { | ||
| 14 | + t.Error("/unload alone is not the command") | ||
| 15 | + } | ||
| 16 | + for _, f := range []Front{Terminal, Window, Editor} { | ||
| 17 | + found := false | ||
| 18 | + for _, c := range For(f) { | ||
| 19 | + found = found || c.Name == Unload | ||
| 20 | + } | ||
| 21 | + if !found { | ||
| 22 | + t.Errorf("front %d does not offer /unload", f) | ||
| 23 | + } | ||
| 24 | + } | ||
| 25 | + | ||
| 26 | + SetUnload(false) | ||
| 27 | + if _, _, ok := Parse("/unload"); ok { | ||
| 28 | + t.Error("/unload still parsed with unloadModels off") | ||
| 29 | + } | ||
| 30 | +} | ||
modified
internal/config/config.go +8 -0 | @@ -139,6 +139,13 @@ type Config struct { | ||
| 139 | 139 | // way. |
| 140 | 140 | SaveAnswerTool bool `yaml:"saveAnswerTool"` |
| 141 | 141 | |
| 142 | + // UnloadModels: the /unload command and the `unload_model` tool, which | |
| 143 | + // free a model from the server's memory (ticket 101; llmman, ollama, | |
| 144 | + // llamacpp in router mode). On by default: unloading is harmless, the | |
| 145 | + // next request reloads. The tool is never declared beside a | |
| 146 | + // structuredOutput, like ask_user. | |
| 147 | + UnloadModels bool `yaml:"unloadModels"` | |
| 148 | + | |
| 142 | 149 | // Sampling: generation settings (OpenAI API keys; the plugin converts them.) |
| 143 | 150 | Sampling map[string]any `yaml:"sampling"` |
| 144 | 151 | |
| @@ -534,6 +541,7 @@ var Cfg = Config{ | ||
| 534 | 541 | EditTools: true, |
| 535 | 542 | PlanTool: true, |
| 536 | 543 | AskUserTool: true, |
| 544 | + UnloadModels: true, | |
| 537 | 545 | // FetchContentTool stays false (the zero value): see its field comment. |
| 538 | 546 | System: `You are a coding agent working in a terminal. |
| 539 | 547 | You have a "bash" tool to run shell commands. |
| @@ -139,6 +139,13 @@ type Config struct { | |||
| 139 | // way. | 139 | // way. |
| 140 | SaveAnswerTool bool `yaml:"saveAnswerTool"` | 140 | SaveAnswerTool bool `yaml:"saveAnswerTool"` |
| 141 | 141 | ||
| 142 | + // UnloadModels: the /unload command and the `unload_model` tool, which | ||
| 143 | + // free a model from the server's memory (ticket 101; llmman, ollama, | ||
| 144 | + // llamacpp in router mode). On by default: unloading is harmless, the | ||
| 145 | + // next request reloads. The tool is never declared beside a | ||
| 146 | + // structuredOutput, like ask_user. | ||
| 147 | + UnloadModels bool `yaml:"unloadModels"` | ||
| 148 | + | ||
| 142 | // Sampling: generation settings (OpenAI API keys; the plugin converts them.) | 149 | // Sampling: generation settings (OpenAI API keys; the plugin converts them.) |
| 143 | Sampling map[string]any `yaml:"sampling"` | 150 | Sampling map[string]any `yaml:"sampling"` |
| 144 | 151 | ||
| @@ -534,6 +541,7 @@ var Cfg = Config{ | |||
| 534 | EditTools: true, | 541 | EditTools: true, |
| 535 | PlanTool: true, | 542 | PlanTool: true, |
| 536 | AskUserTool: true, | 543 | AskUserTool: true, |
| 544 | + UnloadModels: true, | ||
| 537 | // FetchContentTool stays false (the zero value): see its field comment. | 545 | // FetchContentTool stays false (the zero value): see its field comment. |
| 538 | System: `You are a coding agent working in a terminal. | 546 | System: `You are a coding agent working in a terminal. |
| 539 | You have a "bash" tool to run shell commands. | 547 | You have a "bash" tool to run shell commands. |
modified
internal/config/config_test.go +20 -0 | @@ -238,3 +238,23 @@ func TestDisplayHooksConfig(t *testing.T) { | ||
| 238 | 238 | t.Error("displayHooks: true was not read") |
| 239 | 239 | } |
| 240 | 240 | } |
| 241 | + | |
| 242 | +// /unload and unload_model are on unless the YAML turns them off. | |
| 243 | +func TestUnloadModelsConfig(t *testing.T) { | |
| 244 | + defaults := Cfg | |
| 245 | + t.Cleanup(func() { Cfg = defaults }) | |
| 246 | + | |
| 247 | + if !Cfg.UnloadModels { | |
| 248 | + t.Fatal("unloadModels is off by default, want on") | |
| 249 | + } | |
| 250 | + off := filepath.Join(t.TempDir(), "off.yaml") | |
| 251 | + if err := os.WriteFile(off, []byte("unloadModels: false\n"), 0o644); err != nil { | |
| 252 | + t.Fatal(err) | |
| 253 | + } | |
| 254 | + if _, err := Load(off); err != nil { | |
| 255 | + t.Fatal(err) | |
| 256 | + } | |
| 257 | + if Cfg.UnloadModels { | |
| 258 | + t.Error("unloadModels: false was not read") | |
| 259 | + } | |
| 260 | +} | |
| @@ -238,3 +238,23 @@ func TestDisplayHooksConfig(t *testing.T) { | |||
| 238 | t.Error("displayHooks: true was not read") | 238 | t.Error("displayHooks: true was not read") |
| 239 | } | 239 | } |
| 240 | } | 240 | } |
| 241 | + | ||
| 242 | +// /unload and unload_model are on unless the YAML turns them off. | ||
| 243 | +func TestUnloadModelsConfig(t *testing.T) { | ||
| 244 | + defaults := Cfg | ||
| 245 | + t.Cleanup(func() { Cfg = defaults }) | ||
| 246 | + | ||
| 247 | + if !Cfg.UnloadModels { | ||
| 248 | + t.Fatal("unloadModels is off by default, want on") | ||
| 249 | + } | ||
| 250 | + off := filepath.Join(t.TempDir(), "off.yaml") | ||
| 251 | + if err := os.WriteFile(off, []byte("unloadModels: false\n"), 0o644); err != nil { | ||
| 252 | + t.Fatal(err) | ||
| 253 | + } | ||
| 254 | + if _, err := Load(off); err != nil { | ||
| 255 | + t.Fatal(err) | ||
| 256 | + } | ||
| 257 | + if Cfg.UnloadModels { | ||
| 258 | + t.Error("unloadModels: false was not read") | ||
| 259 | + } | ||
| 260 | +} | ||
modified
internal/config/tools.go +1 -0 | @@ -44,6 +44,7 @@ var BuiltinToolNames = []string{ | ||
| 44 | 44 | "read_file", |
| 45 | 45 | "read_skill", |
| 46 | 46 | "save_answer", |
| 47 | + "unload_model", | |
| 47 | 48 | "write_file", |
| 48 | 49 | } |
| 49 | 50 | |
| @@ -44,6 +44,7 @@ var BuiltinToolNames = []string{ | |||
| 44 | "read_file", | 44 | "read_file", |
| 45 | "read_skill", | 45 | "read_skill", |
| 46 | "save_answer", | 46 | "save_answer", |
| 47 | + "unload_model", | ||
| 47 | "write_file", | 48 | "write_file", |
| 48 | } | 49 | } |
| 49 | 50 | ||
modified
internal/engine/openai_compat.go +6 -0 | @@ -59,6 +59,9 @@ type openaiCompat struct { | ||
| 59 | 59 | // thinkingOff: how this server is told not to reason, for the summary of |
| 60 | 60 | // a compaction (usage.go). Zero value = no known way. |
| 61 | 61 | thinkingOff thinkingOff |
| 62 | + | |
| 63 | + // unload: how this server unloads a model (unload.go); nil = it cannot. | |
| 64 | + unload unloadFunc | |
| 62 | 65 | } |
| 63 | 66 | |
| 64 | 67 | func init() { |
| @@ -86,6 +89,7 @@ func init() { | ||
| 86 | 89 | probe: probeLlamaCpp, |
| 87 | 90 | progress: true, |
| 88 | 91 | thinkingOff: thinkingOffTemplate, |
| 92 | + unload: unloadRouter, | |
| 89 | 93 | }) |
| 90 | 94 | register(&openaiCompat{ |
| 91 | 95 | name: "llmman", |
| @@ -104,6 +108,7 @@ func init() { | ||
| 104 | 108 | // enable_thinking:false answers without reasoning). |
| 105 | 109 | progress: true, |
| 106 | 110 | thinkingOff: thinkingOffTemplate, |
| 111 | + unload: unloadKeepAlive, | |
| 107 | 112 | }) |
| 108 | 113 | register(&openaiCompat{ |
| 109 | 114 | name: "ollama", |
| @@ -120,6 +125,7 @@ func init() { | ||
| 120 | 125 | // it), chat_template_kwargs ignored but reasoning_effort "none" obeyed. |
| 121 | 126 | progress: false, |
| 122 | 127 | thinkingOff: thinkingOffEffort, |
| 128 | + unload: unloadKeepAlive, | |
| 123 | 129 | }) |
| 124 | 130 | } |
| 125 | 131 | |
| @@ -59,6 +59,9 @@ type openaiCompat struct { | |||
| 59 | // thinkingOff: how this server is told not to reason, for the summary of | 59 | // thinkingOff: how this server is told not to reason, for the summary of |
| 60 | // a compaction (usage.go). Zero value = no known way. | 60 | // a compaction (usage.go). Zero value = no known way. |
| 61 | thinkingOff thinkingOff | 61 | thinkingOff thinkingOff |
| 62 | + | ||
| 63 | + // unload: how this server unloads a model (unload.go); nil = it cannot. | ||
| 64 | + unload unloadFunc | ||
| 62 | } | 65 | } |
| 63 | 66 | ||
| 64 | func init() { | 67 | func init() { |
| @@ -86,6 +89,7 @@ func init() { | |||
| 86 | probe: probeLlamaCpp, | 89 | probe: probeLlamaCpp, |
| 87 | progress: true, | 90 | progress: true, |
| 88 | thinkingOff: thinkingOffTemplate, | 91 | thinkingOff: thinkingOffTemplate, |
| 92 | + unload: unloadRouter, | ||
| 89 | }) | 93 | }) |
| 90 | register(&openaiCompat{ | 94 | register(&openaiCompat{ |
| 91 | name: "llmman", | 95 | name: "llmman", |
| @@ -104,6 +108,7 @@ func init() { | |||
| 104 | // enable_thinking:false answers without reasoning). | 108 | // enable_thinking:false answers without reasoning). |
| 105 | progress: true, | 109 | progress: true, |
| 106 | thinkingOff: thinkingOffTemplate, | 110 | thinkingOff: thinkingOffTemplate, |
| 111 | + unload: unloadKeepAlive, | ||
| 107 | }) | 112 | }) |
| 108 | register(&openaiCompat{ | 113 | register(&openaiCompat{ |
| 109 | name: "ollama", | 114 | name: "ollama", |
| @@ -120,6 +125,7 @@ func init() { | |||
| 120 | // it), chat_template_kwargs ignored but reasoning_effort "none" obeyed. | 125 | // it), chat_template_kwargs ignored but reasoning_effort "none" obeyed. |
| 121 | progress: false, | 126 | progress: false, |
| 122 | thinkingOff: thinkingOffEffort, | 127 | thinkingOff: thinkingOffEffort, |
| 128 | + unload: unloadKeepAlive, | ||
| 123 | }) | 129 | }) |
| 124 | } | 130 | } |
| 125 | 131 | ||
added
internal/engine/unload.go +196 -0 | new file mode 100644 | ||
| @@ -0,0 +1,196 @@ | ||
| 1 | +package engine | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "bytes" | |
| 5 | + "context" | |
| 6 | + "encoding/json" | |
| 7 | + "errors" | |
| 8 | + "fmt" | |
| 9 | + "io" | |
| 10 | + "net/http" | |
| 11 | + "slices" | |
| 12 | + "strings" | |
| 13 | + "time" | |
| 14 | +) | |
| 15 | + | |
| 16 | +// Unloading a model from the server's memory (ticket 101): /unload and the | |
| 17 | +// unload_model tool. The model comes back on the next request that names it, | |
| 18 | +// so unloading is harmless — it frees the RAM a model held, e.g. before | |
| 19 | +// another agent's model loads. Each server does it its own way (ticket 92): | |
| 20 | +// | |
| 21 | +// - llmman, Ollama: POST /api/generate {"model": m, "keep_alive": 0}; | |
| 22 | +// - llama-server in router mode: POST /models/unload {"model": m}; | |
| 23 | +// - a single-model llama-server, Docker Model Runner: no route. | |
| 24 | + | |
| 25 | +// unloadFunc unloads model from the server behind b and returns one line | |
| 26 | +// saying what happened. An error is a failure the caller words itself. | |
| 27 | +type unloadFunc func(ctx context.Context, b Backend, model string) (string, error) | |
| 28 | + | |
| 29 | +// Unloader is the optional side of a Provider that can unload a model. | |
| 30 | +type Unloader interface { | |
| 31 | + Unload(ctx context.Context, b Backend, model string) (string, error) | |
| 32 | +} | |
| 33 | + | |
| 34 | +// unloadClient bounds one unload: the server answers once the backend process | |
| 35 | +// is stopped, which takes a few seconds, never minutes. | |
| 36 | +var unloadClient = &http.Client{Timeout: 30 * time.Second} | |
| 37 | + | |
| 38 | +// Unload unloads model (this engine's own model when empty) and returns ONE | |
| 39 | +// line for the user or the model: what was unloaded, that there was nothing | |
| 40 | +// to unload, or why it could not be done. Never an error: the line is the | |
| 41 | +// answer, the same way a failed command is text for the model. | |
| 42 | +// | |
| 43 | +// fmt.Println(e.Unload(ctx, "")) // "unloaded hf.co/…:Q4_K_M" | |
| 44 | +// fmt.Println(e.Unload(ctx, "qwen3.5:2b")) // another model of the same server | |
| 45 | +func (e *Engine) Unload(ctx context.Context, model string) string { | |
| 46 | + model = strings.TrimSpace(model) | |
| 47 | + if model == "" { | |
| 48 | + model = e.Backend.Model | |
| 49 | + } | |
| 50 | + u, ok := e.Provider.(Unloader) | |
| 51 | + if !ok { | |
| 52 | + return fmt.Sprintf("cannot unload %s: the %s provider cannot unload a model", model, e.Backend.Provider) | |
| 53 | + } | |
| 54 | + line, err := u.Unload(ctx, e.Backend, model) | |
| 55 | + if err != nil { | |
| 56 | + return fmt.Sprintf("cannot unload %s: %v", model, err) | |
| 57 | + } | |
| 58 | + return line | |
| 59 | +} | |
| 60 | + | |
| 61 | +// Unload implements Unloader with the server's own way, when it has one. | |
| 62 | +func (p *openaiCompat) Unload(ctx context.Context, b Backend, model string) (string, error) { | |
| 63 | + if p.unload == nil { | |
| 64 | + return "", fmt.Errorf("the %s provider cannot unload a model", p.name) | |
| 65 | + } | |
| 66 | + return p.unload(ctx, b, model) | |
| 67 | +} | |
| 68 | + | |
| 69 | +// rootURL is the server's address without the OpenAI prefix: where the | |
| 70 | +// Ollama routes (/api/…) and llama-server's router routes (/models/…) live. | |
| 71 | +func rootURL(baseURL string) string { | |
| 72 | + return strings.TrimSuffix(strings.TrimSuffix(baseURL, "/"), "/v1") | |
| 73 | +} | |
| 74 | + | |
| 75 | +// unloadKeepAlive is the llmman and Ollama way: a generate request with | |
| 76 | +// keep_alive 0. The model is first looked up in /api/ps: a generate for a | |
| 77 | +// model that is NOT loaded would load it only to unload it — minutes for a | |
| 78 | +// large one, and memory taken from whatever is running. | |
| 79 | +func unloadKeepAlive(ctx context.Context, b Backend, model string) (string, error) { | |
| 80 | + root := rootURL(b.BaseURL) | |
| 81 | + loaded, err := loadedModels(ctx, b, root) | |
| 82 | + if err != nil { | |
| 83 | + return "", err | |
| 84 | + } | |
| 85 | + if !slices.Contains(loaded, model) { | |
| 86 | + return fmt.Sprintf("%s was not loaded: nothing to unload", model), nil | |
| 87 | + } | |
| 88 | + var reply struct { | |
| 89 | + DoneReason string `json:"done_reason"` | |
| 90 | + Error string `json:"error"` | |
| 91 | + } | |
| 92 | + if err := postJSON(ctx, b, root+"/api/generate", map[string]any{"model": model, "keep_alive": 0}, &reply); err != nil { | |
| 93 | + return "", err | |
| 94 | + } | |
| 95 | + if reply.Error != "" { | |
| 96 | + return "", fmt.Errorf("the server says: %s", reply.Error) | |
| 97 | + } | |
| 98 | + if reply.DoneReason != "unload" { | |
| 99 | + return "", fmt.Errorf("the server did not confirm (done_reason %q)", reply.DoneReason) | |
| 100 | + } | |
| 101 | + return "unloaded " + model, nil | |
| 102 | +} | |
| 103 | + | |
| 104 | +// loadedModels lists the names /api/ps reports as loaded. | |
| 105 | +func loadedModels(ctx context.Context, b Backend, root string) ([]string, error) { | |
| 106 | + var ps struct { | |
| 107 | + Models []struct { | |
| 108 | + Name string `json:"name"` | |
| 109 | + } `json:"models"` | |
| 110 | + } | |
| 111 | + if err := getWithKey(ctx, b, root+"/api/ps", &ps); err != nil { | |
| 112 | + return nil, err | |
| 113 | + } | |
| 114 | + names := make([]string, 0, len(ps.Models)) | |
| 115 | + for _, m := range ps.Models { | |
| 116 | + names = append(names, m.Name) | |
| 117 | + } | |
| 118 | + return names, nil | |
| 119 | +} | |
| 120 | + | |
| 121 | +// unloadRouter is the llama-server way, in router mode only. A single-model | |
| 122 | +// llama-server has no /models/unload: its 404 is turned into the two ways it | |
| 123 | +// does free memory. | |
| 124 | +func unloadRouter(ctx context.Context, b Backend, model string) (string, error) { | |
| 125 | + var reply struct { | |
| 126 | + Success bool `json:"success"` | |
| 127 | + Error any `json:"error"` | |
| 128 | + } | |
| 129 | + err := postJSON(ctx, b, rootURL(b.BaseURL)+"/models/unload", map[string]any{"model": model}, &reply) | |
| 130 | + var status *httpStatusError | |
| 131 | + switch { | |
| 132 | + case errors.As(err, &status) && status.code == http.StatusNotFound: | |
| 133 | + return "", fmt.Errorf("this llama-server serves a single model and has no unload route — start it in router mode (no -m), or with --sleep-idle-seconds N to free memory when idle, or stop it") | |
| 134 | + case err != nil: | |
| 135 | + return "", err | |
| 136 | + case !reply.Success: | |
| 137 | + return "", fmt.Errorf("the server did not confirm: %v", reply.Error) | |
| 138 | + } | |
| 139 | + return "unloaded " + model, nil | |
| 140 | +} | |
| 141 | + | |
| 142 | +// httpStatusError is a non-2xx answer, with the start of its body. | |
| 143 | +type httpStatusError struct { | |
| 144 | + code int | |
| 145 | + body string | |
| 146 | +} | |
| 147 | + | |
| 148 | +func (e *httpStatusError) Error() string { | |
| 149 | + if e.body == "" { | |
| 150 | + return fmt.Sprintf("the server answered %d", e.code) | |
| 151 | + } | |
| 152 | + return fmt.Sprintf("the server answered %d: %s", e.code, e.body) | |
| 153 | +} | |
| 154 | + | |
| 155 | +// postJSON posts body as JSON and decodes a 2xx answer into out. | |
| 156 | +func postJSON(ctx context.Context, b Backend, url string, body, out any) error { | |
| 157 | + data, err := json.Marshal(body) | |
| 158 | + if err != nil { | |
| 159 | + return err | |
| 160 | + } | |
| 161 | + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data)) | |
| 162 | + if err != nil { | |
| 163 | + return err | |
| 164 | + } | |
| 165 | + req.Header.Set("Content-Type", "application/json") | |
| 166 | + return send(req, b, out) | |
| 167 | +} | |
| 168 | + | |
| 169 | +// getWithKey is GET url, decoded into out, with the backend's key. | |
| 170 | +func getWithKey(ctx context.Context, b Backend, url string, out any) error { | |
| 171 | + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) | |
| 172 | + if err != nil { | |
| 173 | + return err | |
| 174 | + } | |
| 175 | + return send(req, b, out) | |
| 176 | +} | |
| 177 | + | |
| 178 | +// send runs req with the backend's key and decodes a 2xx JSON answer. | |
| 179 | +func send(req *http.Request, b Backend, out any) error { | |
| 180 | + if b.APIKey != "" { | |
| 181 | + req.Header.Set("Authorization", "Bearer "+b.APIKey) | |
| 182 | + } | |
| 183 | + resp, err := unloadClient.Do(req) | |
| 184 | + if err != nil { | |
| 185 | + return fmt.Errorf("the server cannot be reached: %w", err) | |
| 186 | + } | |
| 187 | + defer func() { _ = resp.Body.Close() }() | |
| 188 | + if resp.StatusCode/100 != 2 { | |
| 189 | + body, _ := io.ReadAll(io.LimitReader(resp.Body, 300)) | |
| 190 | + return &httpStatusError{code: resp.StatusCode, body: strings.TrimSpace(string(body))} | |
| 191 | + } | |
| 192 | + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { | |
| 193 | + return fmt.Errorf("unreadable answer: %w", err) | |
| 194 | + } | |
| 195 | + return nil | |
| 196 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,196 @@ | |||
| 1 | +package engine | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "bytes" | ||
| 5 | + "context" | ||
| 6 | + "encoding/json" | ||
| 7 | + "errors" | ||
| 8 | + "fmt" | ||
| 9 | + "io" | ||
| 10 | + "net/http" | ||
| 11 | + "slices" | ||
| 12 | + "strings" | ||
| 13 | + "time" | ||
| 14 | +) | ||
| 15 | + | ||
| 16 | +// Unloading a model from the server's memory (ticket 101): /unload and the | ||
| 17 | +// unload_model tool. The model comes back on the next request that names it, | ||
| 18 | +// so unloading is harmless — it frees the RAM a model held, e.g. before | ||
| 19 | +// another agent's model loads. Each server does it its own way (ticket 92): | ||
| 20 | +// | ||
| 21 | +// - llmman, Ollama: POST /api/generate {"model": m, "keep_alive": 0}; | ||
| 22 | +// - llama-server in router mode: POST /models/unload {"model": m}; | ||
| 23 | +// - a single-model llama-server, Docker Model Runner: no route. | ||
| 24 | + | ||
| 25 | +// unloadFunc unloads model from the server behind b and returns one line | ||
| 26 | +// saying what happened. An error is a failure the caller words itself. | ||
| 27 | +type unloadFunc func(ctx context.Context, b Backend, model string) (string, error) | ||
| 28 | + | ||
| 29 | +// Unloader is the optional side of a Provider that can unload a model. | ||
| 30 | +type Unloader interface { | ||
| 31 | + Unload(ctx context.Context, b Backend, model string) (string, error) | ||
| 32 | +} | ||
| 33 | + | ||
| 34 | +// unloadClient bounds one unload: the server answers once the backend process | ||
| 35 | +// is stopped, which takes a few seconds, never minutes. | ||
| 36 | +var unloadClient = &http.Client{Timeout: 30 * time.Second} | ||
| 37 | + | ||
| 38 | +// Unload unloads model (this engine's own model when empty) and returns ONE | ||
| 39 | +// line for the user or the model: what was unloaded, that there was nothing | ||
| 40 | +// to unload, or why it could not be done. Never an error: the line is the | ||
| 41 | +// answer, the same way a failed command is text for the model. | ||
| 42 | +// | ||
| 43 | +// fmt.Println(e.Unload(ctx, "")) // "unloaded hf.co/…:Q4_K_M" | ||
| 44 | +// fmt.Println(e.Unload(ctx, "qwen3.5:2b")) // another model of the same server | ||
| 45 | +func (e *Engine) Unload(ctx context.Context, model string) string { | ||
| 46 | + model = strings.TrimSpace(model) | ||
| 47 | + if model == "" { | ||
| 48 | + model = e.Backend.Model | ||
| 49 | + } | ||
| 50 | + u, ok := e.Provider.(Unloader) | ||
| 51 | + if !ok { | ||
| 52 | + return fmt.Sprintf("cannot unload %s: the %s provider cannot unload a model", model, e.Backend.Provider) | ||
| 53 | + } | ||
| 54 | + line, err := u.Unload(ctx, e.Backend, model) | ||
| 55 | + if err != nil { | ||
| 56 | + return fmt.Sprintf("cannot unload %s: %v", model, err) | ||
| 57 | + } | ||
| 58 | + return line | ||
| 59 | +} | ||
| 60 | + | ||
| 61 | +// Unload implements Unloader with the server's own way, when it has one. | ||
| 62 | +func (p *openaiCompat) Unload(ctx context.Context, b Backend, model string) (string, error) { | ||
| 63 | + if p.unload == nil { | ||
| 64 | + return "", fmt.Errorf("the %s provider cannot unload a model", p.name) | ||
| 65 | + } | ||
| 66 | + return p.unload(ctx, b, model) | ||
| 67 | +} | ||
| 68 | + | ||
| 69 | +// rootURL is the server's address without the OpenAI prefix: where the | ||
| 70 | +// Ollama routes (/api/…) and llama-server's router routes (/models/…) live. | ||
| 71 | +func rootURL(baseURL string) string { | ||
| 72 | + return strings.TrimSuffix(strings.TrimSuffix(baseURL, "/"), "/v1") | ||
| 73 | +} | ||
| 74 | + | ||
| 75 | +// unloadKeepAlive is the llmman and Ollama way: a generate request with | ||
| 76 | +// keep_alive 0. The model is first looked up in /api/ps: a generate for a | ||
| 77 | +// model that is NOT loaded would load it only to unload it — minutes for a | ||
| 78 | +// large one, and memory taken from whatever is running. | ||
| 79 | +func unloadKeepAlive(ctx context.Context, b Backend, model string) (string, error) { | ||
| 80 | + root := rootURL(b.BaseURL) | ||
| 81 | + loaded, err := loadedModels(ctx, b, root) | ||
| 82 | + if err != nil { | ||
| 83 | + return "", err | ||
| 84 | + } | ||
| 85 | + if !slices.Contains(loaded, model) { | ||
| 86 | + return fmt.Sprintf("%s was not loaded: nothing to unload", model), nil | ||
| 87 | + } | ||
| 88 | + var reply struct { | ||
| 89 | + DoneReason string `json:"done_reason"` | ||
| 90 | + Error string `json:"error"` | ||
| 91 | + } | ||
| 92 | + if err := postJSON(ctx, b, root+"/api/generate", map[string]any{"model": model, "keep_alive": 0}, &reply); err != nil { | ||
| 93 | + return "", err | ||
| 94 | + } | ||
| 95 | + if reply.Error != "" { | ||
| 96 | + return "", fmt.Errorf("the server says: %s", reply.Error) | ||
| 97 | + } | ||
| 98 | + if reply.DoneReason != "unload" { | ||
| 99 | + return "", fmt.Errorf("the server did not confirm (done_reason %q)", reply.DoneReason) | ||
| 100 | + } | ||
| 101 | + return "unloaded " + model, nil | ||
| 102 | +} | ||
| 103 | + | ||
| 104 | +// loadedModels lists the names /api/ps reports as loaded. | ||
| 105 | +func loadedModels(ctx context.Context, b Backend, root string) ([]string, error) { | ||
| 106 | + var ps struct { | ||
| 107 | + Models []struct { | ||
| 108 | + Name string `json:"name"` | ||
| 109 | + } `json:"models"` | ||
| 110 | + } | ||
| 111 | + if err := getWithKey(ctx, b, root+"/api/ps", &ps); err != nil { | ||
| 112 | + return nil, err | ||
| 113 | + } | ||
| 114 | + names := make([]string, 0, len(ps.Models)) | ||
| 115 | + for _, m := range ps.Models { | ||
| 116 | + names = append(names, m.Name) | ||
| 117 | + } | ||
| 118 | + return names, nil | ||
| 119 | +} | ||
| 120 | + | ||
| 121 | +// unloadRouter is the llama-server way, in router mode only. A single-model | ||
| 122 | +// llama-server has no /models/unload: its 404 is turned into the two ways it | ||
| 123 | +// does free memory. | ||
| 124 | +func unloadRouter(ctx context.Context, b Backend, model string) (string, error) { | ||
| 125 | + var reply struct { | ||
| 126 | + Success bool `json:"success"` | ||
| 127 | + Error any `json:"error"` | ||
| 128 | + } | ||
| 129 | + err := postJSON(ctx, b, rootURL(b.BaseURL)+"/models/unload", map[string]any{"model": model}, &reply) | ||
| 130 | + var status *httpStatusError | ||
| 131 | + switch { | ||
| 132 | + case errors.As(err, &status) && status.code == http.StatusNotFound: | ||
| 133 | + return "", fmt.Errorf("this llama-server serves a single model and has no unload route — start it in router mode (no -m), or with --sleep-idle-seconds N to free memory when idle, or stop it") | ||
| 134 | + case err != nil: | ||
| 135 | + return "", err | ||
| 136 | + case !reply.Success: | ||
| 137 | + return "", fmt.Errorf("the server did not confirm: %v", reply.Error) | ||
| 138 | + } | ||
| 139 | + return "unloaded " + model, nil | ||
| 140 | +} | ||
| 141 | + | ||
| 142 | +// httpStatusError is a non-2xx answer, with the start of its body. | ||
| 143 | +type httpStatusError struct { | ||
| 144 | + code int | ||
| 145 | + body string | ||
| 146 | +} | ||
| 147 | + | ||
| 148 | +func (e *httpStatusError) Error() string { | ||
| 149 | + if e.body == "" { | ||
| 150 | + return fmt.Sprintf("the server answered %d", e.code) | ||
| 151 | + } | ||
| 152 | + return fmt.Sprintf("the server answered %d: %s", e.code, e.body) | ||
| 153 | +} | ||
| 154 | + | ||
| 155 | +// postJSON posts body as JSON and decodes a 2xx answer into out. | ||
| 156 | +func postJSON(ctx context.Context, b Backend, url string, body, out any) error { | ||
| 157 | + data, err := json.Marshal(body) | ||
| 158 | + if err != nil { | ||
| 159 | + return err | ||
| 160 | + } | ||
| 161 | + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data)) | ||
| 162 | + if err != nil { | ||
| 163 | + return err | ||
| 164 | + } | ||
| 165 | + req.Header.Set("Content-Type", "application/json") | ||
| 166 | + return send(req, b, out) | ||
| 167 | +} | ||
| 168 | + | ||
| 169 | +// getWithKey is GET url, decoded into out, with the backend's key. | ||
| 170 | +func getWithKey(ctx context.Context, b Backend, url string, out any) error { | ||
| 171 | + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) | ||
| 172 | + if err != nil { | ||
| 173 | + return err | ||
| 174 | + } | ||
| 175 | + return send(req, b, out) | ||
| 176 | +} | ||
| 177 | + | ||
| 178 | +// send runs req with the backend's key and decodes a 2xx JSON answer. | ||
| 179 | +func send(req *http.Request, b Backend, out any) error { | ||
| 180 | + if b.APIKey != "" { | ||
| 181 | + req.Header.Set("Authorization", "Bearer "+b.APIKey) | ||
| 182 | + } | ||
| 183 | + resp, err := unloadClient.Do(req) | ||
| 184 | + if err != nil { | ||
| 185 | + return fmt.Errorf("the server cannot be reached: %w", err) | ||
| 186 | + } | ||
| 187 | + defer func() { _ = resp.Body.Close() }() | ||
| 188 | + if resp.StatusCode/100 != 2 { | ||
| 189 | + body, _ := io.ReadAll(io.LimitReader(resp.Body, 300)) | ||
| 190 | + return &httpStatusError{code: resp.StatusCode, body: strings.TrimSpace(string(body))} | ||
| 191 | + } | ||
| 192 | + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { | ||
| 193 | + return fmt.Errorf("unreadable answer: %w", err) | ||
| 194 | + } | ||
| 195 | + return nil | ||
| 196 | +} | ||
added
internal/engine/unload_test.go +139 -0 | new file mode 100644 | ||
| @@ -0,0 +1,139 @@ | ||
| 1 | +package engine | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "context" | |
| 5 | + "encoding/json" | |
| 6 | + "net/http" | |
| 7 | + "net/http/httptest" | |
| 8 | + "strings" | |
| 9 | + "sync" | |
| 10 | + "testing" | |
| 11 | +) | |
| 12 | + | |
| 13 | +// fakeOllama is an Ollama-style server: /api/ps lists loaded, and a | |
| 14 | +// keep_alive 0 generate unloads the model it names. | |
| 15 | +type fakeOllama struct { | |
| 16 | + mu sync.Mutex | |
| 17 | + loaded []string | |
| 18 | + generate []map[string]any | |
| 19 | + reply string // the generate answer; "" = the real one | |
| 20 | +} | |
| 21 | + | |
| 22 | +func (f *fakeOllama) server(t *testing.T) *httptest.Server { | |
| 23 | + t.Helper() | |
| 24 | + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| 25 | + f.mu.Lock() | |
| 26 | + defer f.mu.Unlock() | |
| 27 | + switch r.URL.Path { | |
| 28 | + case "/api/ps": | |
| 29 | + models := []map[string]any{} | |
| 30 | + for _, m := range f.loaded { | |
| 31 | + models = append(models, map[string]any{"name": m}) | |
| 32 | + } | |
| 33 | + _ = json.NewEncoder(w).Encode(map[string]any{"models": models}) | |
| 34 | + case "/api/generate": | |
| 35 | + var body map[string]any | |
| 36 | + _ = json.NewDecoder(r.Body).Decode(&body) | |
| 37 | + f.generate = append(f.generate, body) | |
| 38 | + if f.reply != "" { | |
| 39 | + _, _ = w.Write([]byte(f.reply)) | |
| 40 | + return | |
| 41 | + } | |
| 42 | + f.loaded = nil | |
| 43 | + _, _ = w.Write([]byte(`{"model":"` + body["model"].(string) + `","done":true,"done_reason":"unload"}`)) | |
| 44 | + default: | |
| 45 | + http.NotFound(w, r) | |
| 46 | + } | |
| 47 | + })) | |
| 48 | + t.Cleanup(srv.Close) | |
| 49 | + return srv | |
| 50 | +} | |
| 51 | + | |
| 52 | +func TestUnloadKeepAliveUnloadsALoadedModel(t *testing.T) { | |
| 53 | + f := &fakeOllama{loaded: []string{"qwen3.5:2b"}} | |
| 54 | + srv := f.server(t) | |
| 55 | + line, err := unloadKeepAlive(context.Background(), Backend{BaseURL: srv.URL + "/v1"}, "qwen3.5:2b") | |
| 56 | + if err != nil || line != "unloaded qwen3.5:2b" { | |
| 57 | + t.Fatalf("unload = %q, %v", line, err) | |
| 58 | + } | |
| 59 | + if len(f.generate) != 1 || f.generate[0]["keep_alive"] != float64(0) || f.generate[0]["model"] != "qwen3.5:2b" { | |
| 60 | + t.Errorf("generate requests = %v, want one keep_alive 0 for the model", f.generate) | |
| 61 | + } | |
| 62 | +} | |
| 63 | + | |
| 64 | +// A model that is not loaded is not asked for: a generate would load it | |
| 65 | +// first, only to unload it. | |
| 66 | +func TestUnloadKeepAliveSkipsAModelThatIsNotLoaded(t *testing.T) { | |
| 67 | + f := &fakeOllama{loaded: []string{"other"}} | |
| 68 | + srv := f.server(t) | |
| 69 | + line, err := unloadKeepAlive(context.Background(), Backend{BaseURL: srv.URL + "/v1"}, "qwen3.5:2b") | |
| 70 | + if err != nil || line != "qwen3.5:2b was not loaded: nothing to unload" { | |
| 71 | + t.Fatalf("unload = %q, %v", line, err) | |
| 72 | + } | |
| 73 | + if len(f.generate) != 0 { | |
| 74 | + t.Errorf("a generate was sent for a model that was not loaded: %v", f.generate) | |
| 75 | + } | |
| 76 | +} | |
| 77 | + | |
| 78 | +func TestUnloadKeepAliveReportsAServerThatDoesNotConfirm(t *testing.T) { | |
| 79 | + f := &fakeOllama{loaded: []string{"m"}, reply: `{"error":"model is busy"}`} | |
| 80 | + srv := f.server(t) | |
| 81 | + if _, err := unloadKeepAlive(context.Background(), Backend{BaseURL: srv.URL + "/v1"}, "m"); err == nil || !strings.Contains(err.Error(), "model is busy") { | |
| 82 | + t.Errorf("err = %v, want the server's words", err) | |
| 83 | + } | |
| 84 | + f.reply = `{"done":true,"done_reason":"stop"}` | |
| 85 | + if _, err := unloadKeepAlive(context.Background(), Backend{BaseURL: srv.URL + "/v1"}, "m"); err == nil || !strings.Contains(err.Error(), "did not confirm") { | |
| 86 | + t.Errorf("err = %v, want an unconfirmed unload", err) | |
| 87 | + } | |
| 88 | +} | |
| 89 | + | |
| 90 | +func TestUnloadRouter(t *testing.T) { | |
| 91 | + var got map[string]any | |
| 92 | + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| 93 | + if r.URL.Path != "/models/unload" { | |
| 94 | + http.NotFound(w, r) | |
| 95 | + return | |
| 96 | + } | |
| 97 | + _ = json.NewDecoder(r.Body).Decode(&got) | |
| 98 | + _, _ = w.Write([]byte(`{"success":true}`)) | |
| 99 | + })) | |
| 100 | + t.Cleanup(srv.Close) | |
| 101 | + line, err := unloadRouter(context.Background(), Backend{BaseURL: srv.URL + "/v1"}, "gemma") | |
| 102 | + if err != nil || line != "unloaded gemma" || got["model"] != "gemma" { | |
| 103 | + t.Fatalf("unload = %q, %v; request %v", line, err, got) | |
| 104 | + } | |
| 105 | +} | |
| 106 | + | |
| 107 | +// A single-model llama-server answers 404: the error says what it can do | |
| 108 | +// instead. | |
| 109 | +func TestUnloadRouterSingleModelServer(t *testing.T) { | |
| 110 | + srv := httptest.NewServer(http.NotFoundHandler()) | |
| 111 | + t.Cleanup(srv.Close) | |
| 112 | + _, err := unloadRouter(context.Background(), Backend{BaseURL: srv.URL + "/v1"}, "gemma") | |
| 113 | + if err == nil || !strings.Contains(err.Error(), "--sleep-idle-seconds") { | |
| 114 | + t.Errorf("err = %v, want the single-model explanation", err) | |
| 115 | + } | |
| 116 | +} | |
| 117 | + | |
| 118 | +// Engine.Unload: the engine's own model when none is named, one line in | |
| 119 | +// every case, and a provider without a way says so. | |
| 120 | +func TestEngineUnloadLines(t *testing.T) { | |
| 121 | + f := &fakeOllama{loaded: []string{"own"}} | |
| 122 | + srv := f.server(t) | |
| 123 | + llmman, _ := Lookup("llmman") | |
| 124 | + e := &Engine{Provider: llmman, Backend: Backend{Provider: "llmman", BaseURL: srv.URL + "/v1", Model: "own"}} | |
| 125 | + if got := e.Unload(context.Background(), " "); got != "unloaded own" { | |
| 126 | + t.Errorf("Unload(\"\") = %q, want the engine's own model unloaded", got) | |
| 127 | + } | |
| 128 | + | |
| 129 | + dmr, _ := Lookup("dmr") | |
| 130 | + e = &Engine{Provider: dmr, Backend: Backend{Provider: "dmr", Model: "m"}} | |
| 131 | + if got := e.Unload(context.Background(), ""); !strings.HasPrefix(got, "cannot unload m:") { | |
| 132 | + t.Errorf("dmr Unload = %q, want a refusal", got) | |
| 133 | + } | |
| 134 | + | |
| 135 | + e = &Engine{Provider: llmman, Backend: Backend{Provider: "llmman", BaseURL: "http://127.0.0.1:1/v1", Model: "m"}} | |
| 136 | + if got := e.Unload(context.Background(), ""); !strings.Contains(got, "cannot be reached") { | |
| 137 | + t.Errorf("unreachable Unload = %q", got) | |
| 138 | + } | |
| 139 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,139 @@ | |||
| 1 | +package engine | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "context" | ||
| 5 | + "encoding/json" | ||
| 6 | + "net/http" | ||
| 7 | + "net/http/httptest" | ||
| 8 | + "strings" | ||
| 9 | + "sync" | ||
| 10 | + "testing" | ||
| 11 | +) | ||
| 12 | + | ||
| 13 | +// fakeOllama is an Ollama-style server: /api/ps lists loaded, and a | ||
| 14 | +// keep_alive 0 generate unloads the model it names. | ||
| 15 | +type fakeOllama struct { | ||
| 16 | + mu sync.Mutex | ||
| 17 | + loaded []string | ||
| 18 | + generate []map[string]any | ||
| 19 | + reply string // the generate answer; "" = the real one | ||
| 20 | +} | ||
| 21 | + | ||
| 22 | +func (f *fakeOllama) server(t *testing.T) *httptest.Server { | ||
| 23 | + t.Helper() | ||
| 24 | + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| 25 | + f.mu.Lock() | ||
| 26 | + defer f.mu.Unlock() | ||
| 27 | + switch r.URL.Path { | ||
| 28 | + case "/api/ps": | ||
| 29 | + models := []map[string]any{} | ||
| 30 | + for _, m := range f.loaded { | ||
| 31 | + models = append(models, map[string]any{"name": m}) | ||
| 32 | + } | ||
| 33 | + _ = json.NewEncoder(w).Encode(map[string]any{"models": models}) | ||
| 34 | + case "/api/generate": | ||
| 35 | + var body map[string]any | ||
| 36 | + _ = json.NewDecoder(r.Body).Decode(&body) | ||
| 37 | + f.generate = append(f.generate, body) | ||
| 38 | + if f.reply != "" { | ||
| 39 | + _, _ = w.Write([]byte(f.reply)) | ||
| 40 | + return | ||
| 41 | + } | ||
| 42 | + f.loaded = nil | ||
| 43 | + _, _ = w.Write([]byte(`{"model":"` + body["model"].(string) + `","done":true,"done_reason":"unload"}`)) | ||
| 44 | + default: | ||
| 45 | + http.NotFound(w, r) | ||
| 46 | + } | ||
| 47 | + })) | ||
| 48 | + t.Cleanup(srv.Close) | ||
| 49 | + return srv | ||
| 50 | +} | ||
| 51 | + | ||
| 52 | +func TestUnloadKeepAliveUnloadsALoadedModel(t *testing.T) { | ||
| 53 | + f := &fakeOllama{loaded: []string{"qwen3.5:2b"}} | ||
| 54 | + srv := f.server(t) | ||
| 55 | + line, err := unloadKeepAlive(context.Background(), Backend{BaseURL: srv.URL + "/v1"}, "qwen3.5:2b") | ||
| 56 | + if err != nil || line != "unloaded qwen3.5:2b" { | ||
| 57 | + t.Fatalf("unload = %q, %v", line, err) | ||
| 58 | + } | ||
| 59 | + if len(f.generate) != 1 || f.generate[0]["keep_alive"] != float64(0) || f.generate[0]["model"] != "qwen3.5:2b" { | ||
| 60 | + t.Errorf("generate requests = %v, want one keep_alive 0 for the model", f.generate) | ||
| 61 | + } | ||
| 62 | +} | ||
| 63 | + | ||
| 64 | +// A model that is not loaded is not asked for: a generate would load it | ||
| 65 | +// first, only to unload it. | ||
| 66 | +func TestUnloadKeepAliveSkipsAModelThatIsNotLoaded(t *testing.T) { | ||
| 67 | + f := &fakeOllama{loaded: []string{"other"}} | ||
| 68 | + srv := f.server(t) | ||
| 69 | + line, err := unloadKeepAlive(context.Background(), Backend{BaseURL: srv.URL + "/v1"}, "qwen3.5:2b") | ||
| 70 | + if err != nil || line != "qwen3.5:2b was not loaded: nothing to unload" { | ||
| 71 | + t.Fatalf("unload = %q, %v", line, err) | ||
| 72 | + } | ||
| 73 | + if len(f.generate) != 0 { | ||
| 74 | + t.Errorf("a generate was sent for a model that was not loaded: %v", f.generate) | ||
| 75 | + } | ||
| 76 | +} | ||
| 77 | + | ||
| 78 | +func TestUnloadKeepAliveReportsAServerThatDoesNotConfirm(t *testing.T) { | ||
| 79 | + f := &fakeOllama{loaded: []string{"m"}, reply: `{"error":"model is busy"}`} | ||
| 80 | + srv := f.server(t) | ||
| 81 | + if _, err := unloadKeepAlive(context.Background(), Backend{BaseURL: srv.URL + "/v1"}, "m"); err == nil || !strings.Contains(err.Error(), "model is busy") { | ||
| 82 | + t.Errorf("err = %v, want the server's words", err) | ||
| 83 | + } | ||
| 84 | + f.reply = `{"done":true,"done_reason":"stop"}` | ||
| 85 | + if _, err := unloadKeepAlive(context.Background(), Backend{BaseURL: srv.URL + "/v1"}, "m"); err == nil || !strings.Contains(err.Error(), "did not confirm") { | ||
| 86 | + t.Errorf("err = %v, want an unconfirmed unload", err) | ||
| 87 | + } | ||
| 88 | +} | ||
| 89 | + | ||
| 90 | +func TestUnloadRouter(t *testing.T) { | ||
| 91 | + var got map[string]any | ||
| 92 | + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| 93 | + if r.URL.Path != "/models/unload" { | ||
| 94 | + http.NotFound(w, r) | ||
| 95 | + return | ||
| 96 | + } | ||
| 97 | + _ = json.NewDecoder(r.Body).Decode(&got) | ||
| 98 | + _, _ = w.Write([]byte(`{"success":true}`)) | ||
| 99 | + })) | ||
| 100 | + t.Cleanup(srv.Close) | ||
| 101 | + line, err := unloadRouter(context.Background(), Backend{BaseURL: srv.URL + "/v1"}, "gemma") | ||
| 102 | + if err != nil || line != "unloaded gemma" || got["model"] != "gemma" { | ||
| 103 | + t.Fatalf("unload = %q, %v; request %v", line, err, got) | ||
| 104 | + } | ||
| 105 | +} | ||
| 106 | + | ||
| 107 | +// A single-model llama-server answers 404: the error says what it can do | ||
| 108 | +// instead. | ||
| 109 | +func TestUnloadRouterSingleModelServer(t *testing.T) { | ||
| 110 | + srv := httptest.NewServer(http.NotFoundHandler()) | ||
| 111 | + t.Cleanup(srv.Close) | ||
| 112 | + _, err := unloadRouter(context.Background(), Backend{BaseURL: srv.URL + "/v1"}, "gemma") | ||
| 113 | + if err == nil || !strings.Contains(err.Error(), "--sleep-idle-seconds") { | ||
| 114 | + t.Errorf("err = %v, want the single-model explanation", err) | ||
| 115 | + } | ||
| 116 | +} | ||
| 117 | + | ||
| 118 | +// Engine.Unload: the engine's own model when none is named, one line in | ||
| 119 | +// every case, and a provider without a way says so. | ||
| 120 | +func TestEngineUnloadLines(t *testing.T) { | ||
| 121 | + f := &fakeOllama{loaded: []string{"own"}} | ||
| 122 | + srv := f.server(t) | ||
| 123 | + llmman, _ := Lookup("llmman") | ||
| 124 | + e := &Engine{Provider: llmman, Backend: Backend{Provider: "llmman", BaseURL: srv.URL + "/v1", Model: "own"}} | ||
| 125 | + if got := e.Unload(context.Background(), " "); got != "unloaded own" { | ||
| 126 | + t.Errorf("Unload(\"\") = %q, want the engine's own model unloaded", got) | ||
| 127 | + } | ||
| 128 | + | ||
| 129 | + dmr, _ := Lookup("dmr") | ||
| 130 | + e = &Engine{Provider: dmr, Backend: Backend{Provider: "dmr", Model: "m"}} | ||
| 131 | + if got := e.Unload(context.Background(), ""); !strings.HasPrefix(got, "cannot unload m:") { | ||
| 132 | + t.Errorf("dmr Unload = %q, want a refusal", got) | ||
| 133 | + } | ||
| 134 | + | ||
| 135 | + e = &Engine{Provider: llmman, Backend: Backend{Provider: "llmman", BaseURL: "http://127.0.0.1:1/v1", Model: "m"}} | ||
| 136 | + if got := e.Unload(context.Background(), ""); !strings.Contains(got, "cannot be reached") { | ||
| 137 | + t.Errorf("unreachable Unload = %q", got) | ||
| 138 | + } | ||
| 139 | +} | ||
added
internal/tools/unload.go +69 -0 | new file mode 100644 | ||
| @@ -0,0 +1,69 @@ | ||
| 1 | +package tools | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "context" | |
| 5 | + "strings" | |
| 6 | + | |
| 7 | + "mm/internal/detector" | |
| 8 | + "mm/internal/spinner" | |
| 9 | + "mm/internal/ui" | |
| 10 | + | |
| 11 | + "github.com/firebase/genkit/go/ai" | |
| 12 | + "github.com/firebase/genkit/go/genkit" | |
| 13 | +) | |
| 14 | + | |
| 15 | +// --- unload_model (ticket 101) ----------------------------------------------- | |
| 16 | +// | |
| 17 | +// Frees a model from the server's memory. The model comes back by itself on | |
| 18 | +// the next request that names it, so the tool is never gated, even with | |
| 19 | +// yolo off: nothing changes on disk or in the project. | |
| 20 | +// | |
| 21 | +// Unloading the model that RUNS this agent lasts one step: the agent needs it | |
| 22 | +// to read the tool's result. The description says so, and points at the case | |
| 23 | +// that holds: the model of another agent, once its answer is in. | |
| 24 | + | |
| 25 | +type unloadModelInput struct { | |
| 26 | + Model string `json:"model,omitempty" jsonschema_description:"The model to unload, exactly as the server names it. Empty: your own model."` | |
| 27 | +} | |
| 28 | + | |
| 29 | +// UnloadFunc unloads a model and says what happened in one line — the | |
| 30 | +// engine's Unload. | |
| 31 | +type UnloadFunc func(ctx context.Context, model string) string | |
| 32 | + | |
| 33 | +// UnloadModel declares the `unload_model` tool (config unloadModels). | |
| 34 | +// | |
| 35 | +// set.tools = append(set.tools, tools.UnloadModel(e.G, d, e.Unload)) | |
| 36 | +func UnloadModel(g *genkit.Genkit, d *detector.LoopDetector, unload UnloadFunc) ai.ToolRef { | |
| 37 | + description := `Unload a model from the model server's memory, to free RAM for another one. The model reloads by itself the next time something uses it. | |
| 38 | +Use it when your instructions say so, typically on the model of another agent once you have its answer and no longer need it. | |
| 39 | +Unloading your own model (model left empty) only lasts until your next step: you need it to read this result. | |
| 40 | +Returns one line: unloaded, not loaded (nothing to do), or why it could not be done.` | |
| 41 | + | |
| 42 | + return genkit.DefineTool(g, "unload_model", description, | |
| 43 | + func(tctx *ai.ToolContext, in unloadModelInput) (string, error) { | |
| 44 | + title := "unload_model: " + target(in.Model) | |
| 45 | + if s := ui.ActiveSink(); s != nil { | |
| 46 | + ev := ui.ToolEvent{ID: ui.NextCallID(), Tool: "unload_model", Title: title, Kind: "other", Input: map[string]any{"model": in.Model}} | |
| 47 | + s.ToolStart(ev) | |
| 48 | + s.ToolRunning(ev.ID) | |
| 49 | + out := unload(tctx, in.Model) | |
| 50 | + s.ToolEnd(ev.ID, ui.ToolResult{Output: out, Failed: unloadFailed(out)}) | |
| 51 | + return record(d, "unload_model", in.Model, out) | |
| 52 | + } | |
| 53 | + announce("🧹 " + title) | |
| 54 | + out := unload(tctx, in.Model) | |
| 55 | + spinner.Suspend(func() { echoOutput(out) }) | |
| 56 | + return record(d, "unload_model", in.Model, out) | |
| 57 | + }) | |
| 58 | +} | |
| 59 | + | |
| 60 | +// target names the model in a title: "(own model)" when none is given. | |
| 61 | +func target(model string) string { | |
| 62 | + if strings.TrimSpace(model) == "" { | |
| 63 | + return "(own model)" | |
| 64 | + } | |
| 65 | + return model | |
| 66 | +} | |
| 67 | + | |
| 68 | +// unloadFailed reports whether the engine's line is a refusal. | |
| 69 | +func unloadFailed(line string) bool { return strings.HasPrefix(line, "cannot unload") } | |
| new file mode 100644 | |||
| @@ -0,0 +1,69 @@ | |||
| 1 | +package tools | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "context" | ||
| 5 | + "strings" | ||
| 6 | + | ||
| 7 | + "mm/internal/detector" | ||
| 8 | + "mm/internal/spinner" | ||
| 9 | + "mm/internal/ui" | ||
| 10 | + | ||
| 11 | + "github.com/firebase/genkit/go/ai" | ||
| 12 | + "github.com/firebase/genkit/go/genkit" | ||
| 13 | +) | ||
| 14 | + | ||
| 15 | +// --- unload_model (ticket 101) ----------------------------------------------- | ||
| 16 | +// | ||
| 17 | +// Frees a model from the server's memory. The model comes back by itself on | ||
| 18 | +// the next request that names it, so the tool is never gated, even with | ||
| 19 | +// yolo off: nothing changes on disk or in the project. | ||
| 20 | +// | ||
| 21 | +// Unloading the model that RUNS this agent lasts one step: the agent needs it | ||
| 22 | +// to read the tool's result. The description says so, and points at the case | ||
| 23 | +// that holds: the model of another agent, once its answer is in. | ||
| 24 | + | ||
| 25 | +type unloadModelInput struct { | ||
| 26 | + Model string `json:"model,omitempty" jsonschema_description:"The model to unload, exactly as the server names it. Empty: your own model."` | ||
| 27 | +} | ||
| 28 | + | ||
| 29 | +// UnloadFunc unloads a model and says what happened in one line — the | ||
| 30 | +// engine's Unload. | ||
| 31 | +type UnloadFunc func(ctx context.Context, model string) string | ||
| 32 | + | ||
| 33 | +// UnloadModel declares the `unload_model` tool (config unloadModels). | ||
| 34 | +// | ||
| 35 | +// set.tools = append(set.tools, tools.UnloadModel(e.G, d, e.Unload)) | ||
| 36 | +func UnloadModel(g *genkit.Genkit, d *detector.LoopDetector, unload UnloadFunc) ai.ToolRef { | ||
| 37 | + description := `Unload a model from the model server's memory, to free RAM for another one. The model reloads by itself the next time something uses it. | ||
| 38 | +Use it when your instructions say so, typically on the model of another agent once you have its answer and no longer need it. | ||
| 39 | +Unloading your own model (model left empty) only lasts until your next step: you need it to read this result. | ||
| 40 | +Returns one line: unloaded, not loaded (nothing to do), or why it could not be done.` | ||
| 41 | + | ||
| 42 | + return genkit.DefineTool(g, "unload_model", description, | ||
| 43 | + func(tctx *ai.ToolContext, in unloadModelInput) (string, error) { | ||
| 44 | + title := "unload_model: " + target(in.Model) | ||
| 45 | + if s := ui.ActiveSink(); s != nil { | ||
| 46 | + ev := ui.ToolEvent{ID: ui.NextCallID(), Tool: "unload_model", Title: title, Kind: "other", Input: map[string]any{"model": in.Model}} | ||
| 47 | + s.ToolStart(ev) | ||
| 48 | + s.ToolRunning(ev.ID) | ||
| 49 | + out := unload(tctx, in.Model) | ||
| 50 | + s.ToolEnd(ev.ID, ui.ToolResult{Output: out, Failed: unloadFailed(out)}) | ||
| 51 | + return record(d, "unload_model", in.Model, out) | ||
| 52 | + } | ||
| 53 | + announce("🧹 " + title) | ||
| 54 | + out := unload(tctx, in.Model) | ||
| 55 | + spinner.Suspend(func() { echoOutput(out) }) | ||
| 56 | + return record(d, "unload_model", in.Model, out) | ||
| 57 | + }) | ||
| 58 | +} | ||
| 59 | + | ||
| 60 | +// target names the model in a title: "(own model)" when none is given. | ||
| 61 | +func target(model string) string { | ||
| 62 | + if strings.TrimSpace(model) == "" { | ||
| 63 | + return "(own model)" | ||
| 64 | + } | ||
| 65 | + return model | ||
| 66 | +} | ||
| 67 | + | ||
| 68 | +// unloadFailed reports whether the engine's line is a refusal. | ||
| 69 | +func unloadFailed(line string) bool { return strings.HasPrefix(line, "cannot unload") } | ||
added
internal/tools/unload_test.go +62 -0 | new file mode 100644 | ||
| @@ -0,0 +1,62 @@ | ||
| 1 | +package tools | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "context" | |
| 5 | + "testing" | |
| 6 | + | |
| 7 | + "mm/internal/detector" | |
| 8 | + "mm/internal/ui" | |
| 9 | + | |
| 10 | + "github.com/firebase/genkit/go/genkit" | |
| 11 | +) | |
| 12 | + | |
| 13 | +// unload_model hands the model to the engine as given — empty for its own —, | |
| 14 | +// shows a finished call, and never asks, even with yolo off: nothing changes | |
| 15 | +// on disk, and the model reloads by itself. | |
| 16 | +func TestUnloadModelNeverAsksAndPassesTheModelOn(t *testing.T) { | |
| 17 | + yoloOff(t) | |
| 18 | + g := genkit.Init(context.Background()) | |
| 19 | + var asked []string | |
| 20 | + UnloadModel(g, detector.NewLoopDetector(10, 3), func(_ context.Context, model string) string { | |
| 21 | + asked = append(asked, model) | |
| 22 | + return "unloaded " + model | |
| 23 | + }) | |
| 24 | + tool := genkit.LookupTool(g, "unload_model") | |
| 25 | + sink := &recordingSink{allow: false} | |
| 26 | + ui.SetActive(sink) | |
| 27 | + t.Cleanup(func() { ui.SetActive(nil) }) | |
| 28 | + | |
| 29 | + out, err := tool.RunRaw(context.Background(), map[string]any{"model": "qwen3.5:2b"}) | |
| 30 | + if err != nil || out != "unloaded qwen3.5:2b" { | |
| 31 | + t.Fatalf("out = %v, %v", out, err) | |
| 32 | + } | |
| 33 | + if sink.asked != 0 { | |
| 34 | + t.Errorf("asked %d time(s) for permission, want never", sink.asked) | |
| 35 | + } | |
| 36 | + if sink.started.Title != "unload_model: qwen3.5:2b" || sink.ended.Failed { | |
| 37 | + t.Errorf("shown as %+v → %+v", sink.started, sink.ended) | |
| 38 | + } | |
| 39 | + | |
| 40 | + if _, err := tool.RunRaw(context.Background(), map[string]any{}); err != nil { | |
| 41 | + t.Fatal(err) | |
| 42 | + } | |
| 43 | + if len(asked) != 2 || asked[1] != "" || sink.started.Title != "unload_model: (own model)" { | |
| 44 | + t.Errorf("own model: engine asked %q, title %q", asked, sink.started.Title) | |
| 45 | + } | |
| 46 | +} | |
| 47 | + | |
| 48 | +func TestUnloadModelRefusalIsAFailedCall(t *testing.T) { | |
| 49 | + g := genkit.Init(context.Background()) | |
| 50 | + UnloadModel(g, detector.NewLoopDetector(10, 3), func(context.Context, string) string { | |
| 51 | + return "cannot unload m: the dmr provider cannot unload a model" | |
| 52 | + }) | |
| 53 | + sink := &recordingSink{} | |
| 54 | + ui.SetActive(sink) | |
| 55 | + t.Cleanup(func() { ui.SetActive(nil) }) | |
| 56 | + if _, err := genkit.LookupTool(g, "unload_model").RunRaw(context.Background(), map[string]any{"model": "m"}); err != nil { | |
| 57 | + t.Fatal(err) | |
| 58 | + } | |
| 59 | + if !sink.ended.Failed { | |
| 60 | + t.Error("a refusal is not shown as a failed call") | |
| 61 | + } | |
| 62 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,62 @@ | |||
| 1 | +package tools | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "context" | ||
| 5 | + "testing" | ||
| 6 | + | ||
| 7 | + "mm/internal/detector" | ||
| 8 | + "mm/internal/ui" | ||
| 9 | + | ||
| 10 | + "github.com/firebase/genkit/go/genkit" | ||
| 11 | +) | ||
| 12 | + | ||
| 13 | +// unload_model hands the model to the engine as given — empty for its own —, | ||
| 14 | +// shows a finished call, and never asks, even with yolo off: nothing changes | ||
| 15 | +// on disk, and the model reloads by itself. | ||
| 16 | +func TestUnloadModelNeverAsksAndPassesTheModelOn(t *testing.T) { | ||
| 17 | + yoloOff(t) | ||
| 18 | + g := genkit.Init(context.Background()) | ||
| 19 | + var asked []string | ||
| 20 | + UnloadModel(g, detector.NewLoopDetector(10, 3), func(_ context.Context, model string) string { | ||
| 21 | + asked = append(asked, model) | ||
| 22 | + return "unloaded " + model | ||
| 23 | + }) | ||
| 24 | + tool := genkit.LookupTool(g, "unload_model") | ||
| 25 | + sink := &recordingSink{allow: false} | ||
| 26 | + ui.SetActive(sink) | ||
| 27 | + t.Cleanup(func() { ui.SetActive(nil) }) | ||
| 28 | + | ||
| 29 | + out, err := tool.RunRaw(context.Background(), map[string]any{"model": "qwen3.5:2b"}) | ||
| 30 | + if err != nil || out != "unloaded qwen3.5:2b" { | ||
| 31 | + t.Fatalf("out = %v, %v", out, err) | ||
| 32 | + } | ||
| 33 | + if sink.asked != 0 { | ||
| 34 | + t.Errorf("asked %d time(s) for permission, want never", sink.asked) | ||
| 35 | + } | ||
| 36 | + if sink.started.Title != "unload_model: qwen3.5:2b" || sink.ended.Failed { | ||
| 37 | + t.Errorf("shown as %+v → %+v", sink.started, sink.ended) | ||
| 38 | + } | ||
| 39 | + | ||
| 40 | + if _, err := tool.RunRaw(context.Background(), map[string]any{}); err != nil { | ||
| 41 | + t.Fatal(err) | ||
| 42 | + } | ||
| 43 | + if len(asked) != 2 || asked[1] != "" || sink.started.Title != "unload_model: (own model)" { | ||
| 44 | + t.Errorf("own model: engine asked %q, title %q", asked, sink.started.Title) | ||
| 45 | + } | ||
| 46 | +} | ||
| 47 | + | ||
| 48 | +func TestUnloadModelRefusalIsAFailedCall(t *testing.T) { | ||
| 49 | + g := genkit.Init(context.Background()) | ||
| 50 | + UnloadModel(g, detector.NewLoopDetector(10, 3), func(context.Context, string) string { | ||
| 51 | + return "cannot unload m: the dmr provider cannot unload a model" | ||
| 52 | + }) | ||
| 53 | + sink := &recordingSink{} | ||
| 54 | + ui.SetActive(sink) | ||
| 55 | + t.Cleanup(func() { ui.SetActive(nil) }) | ||
| 56 | + if _, err := genkit.LookupTool(g, "unload_model").RunRaw(context.Background(), map[string]any{"model": "m"}); err != nil { | ||
| 57 | + t.Fatal(err) | ||
| 58 | + } | ||
| 59 | + if !sink.ended.Failed { | ||
| 60 | + t.Error("a refusal is not shown as a failed call") | ||
| 61 | + } | ||
| 62 | +} | ||
modified
internal/tui/commands.go +12 -0 | @@ -5,6 +5,7 @@ | ||
| 5 | 5 | package tui |
| 6 | 6 | |
| 7 | 7 | import ( |
| 8 | + "context" | |
| 8 | 9 | "fmt" |
| 9 | 10 | |
| 10 | 11 | "mm/internal/agentsmd" |
| @@ -101,9 +102,20 @@ func (r *runner) dispatch(p Prompt) { | ||
| 101 | 102 | r.copy(arg) |
| 102 | 103 | case command.Save: |
| 103 | 104 | r.saveAnswer(arg) |
| 105 | + case command.Unload: | |
| 106 | + r.unload(arg) | |
| 104 | 107 | } |
| 105 | 108 | } |
| 106 | 109 | |
| 110 | +// unload runs /unload off the event loop — the server takes a few seconds to | |
| 111 | +// stop a model — and shows its one line as a notice. | |
| 112 | +func (r *runner) unload(model string) { | |
| 113 | + go func() { | |
| 114 | + r.t.Notice("🧹 " + r.opts.Model.Unload(context.Background(), model)) | |
| 115 | + r.wake() | |
| 116 | + }() | |
| 117 | +} | |
| 118 | + | |
| 107 | 119 | // help lists the window's commands and keys. |
| 108 | 120 | func (r *runner) help() { |
| 109 | 121 | r.t.Notice(command.Usage(command.Window) + "\n\nEnter sends · Ctrl-J new line · Esc cancels the turn · Ctrl-C cancels then quits (twice to leave a turn that will not stop) · Ctrl-L clears · Ctrl-Y copies the last answer · F2 mouse on/off (off: select text with the mouse as usual) · PgUp/PgDn scroll · Tab moves the focus · @path attaches a file") |
| @@ -5,6 +5,7 @@ | |||
| 5 | package tui | 5 | package tui |
| 6 | 6 | ||
| 7 | import ( | 7 | import ( |
| 8 | + "context" | ||
| 8 | "fmt" | 9 | "fmt" |
| 9 | 10 | ||
| 10 | "mm/internal/agentsmd" | 11 | "mm/internal/agentsmd" |
| @@ -101,9 +102,20 @@ func (r *runner) dispatch(p Prompt) { | |||
| 101 | r.copy(arg) | 102 | r.copy(arg) |
| 102 | case command.Save: | 103 | case command.Save: |
| 103 | r.saveAnswer(arg) | 104 | r.saveAnswer(arg) |
| 105 | + case command.Unload: | ||
| 106 | + r.unload(arg) | ||
| 104 | } | 107 | } |
| 105 | } | 108 | } |
| 106 | 109 | ||
| 110 | +// unload runs /unload off the event loop — the server takes a few seconds to | ||
| 111 | +// stop a model — and shows its one line as a notice. | ||
| 112 | +func (r *runner) unload(model string) { | ||
| 113 | + go func() { | ||
| 114 | + r.t.Notice("🧹 " + r.opts.Model.Unload(context.Background(), model)) | ||
| 115 | + r.wake() | ||
| 116 | + }() | ||
| 117 | +} | ||
| 118 | + | ||
| 107 | // help lists the window's commands and keys. | 119 | // help lists the window's commands and keys. |
| 108 | func (r *runner) help() { | 120 | func (r *runner) help() { |
| 109 | r.t.Notice(command.Usage(command.Window) + "\n\nEnter sends · Ctrl-J new line · Esc cancels the turn · Ctrl-C cancels then quits (twice to leave a turn that will not stop) · Ctrl-L clears · Ctrl-Y copies the last answer · F2 mouse on/off (off: select text with the mouse as usual) · PgUp/PgDn scroll · Tab moves the focus · @path attaches a file") | 121 | r.t.Notice(command.Usage(command.Window) + "\n\nEnter sends · Ctrl-J new line · Esc cancels the turn · Ctrl-C cancels then quits (twice to leave a turn that will not stop) · Ctrl-L clears · Ctrl-Y copies the last answer · F2 mouse on/off (off: select text with the mouse as usual) · PgUp/PgDn scroll · Tab moves the focus · @path attaches a file") |
modified
internal/tui/draw.go +13 -3 | @@ -202,7 +202,7 @@ func (v *View) rightBar(leftWidth, width int) string { | ||
| 202 | 202 | } |
| 203 | 203 | |
| 204 | 204 | // drawPicker paints the popup over the bottom of the conversation: one row |
| 205 | -// per choice, the highlighted one in reverse. | |
| 205 | +// per choice, the highlighted one in reverse, scrolled so that it is shown. | |
| 206 | 206 | func (v *View) drawPicker(p painter) { |
| 207 | 207 | cs := v.choices() |
| 208 | 208 | if len(cs) == 0 { |
| @@ -222,10 +222,11 @@ func (v *View) drawPicker(p painter) { | ||
| 222 | 222 | width = max(width, labels+2+len([]rune(c.detail))) |
| 223 | 223 | } |
| 224 | 224 | width = min(width+4, p.width()) |
| 225 | + first := pickerFirst(len(cs), rows, v.pick.selected) | |
| 225 | 226 | for i := 0; i < rows; i++ { |
| 226 | - c := cs[i] | |
| 227 | + c := cs[first+i] | |
| 227 | 228 | style := stBody |
| 228 | - if i == v.pick.selected { | |
| 229 | + if first+i == v.pick.selected { | |
| 229 | 230 | style = stPick |
| 230 | 231 | } |
| 231 | 232 | text := " " + c.label + strings.Repeat(" ", max(labels-len([]rune(c.label))+2, 1)) + c.detail |
| @@ -233,3 +234,12 @@ func (v *View) drawPicker(p painter) { | ||
| 233 | 234 | p.textLimited(0, top+i, width, text, style) |
| 234 | 235 | } |
| 235 | 236 | } |
| 237 | + | |
| 238 | +// pickerFirst is the first choice shown when only rows of n fit: 0 until the | |
| 239 | +// selection goes past the last row, then the window follows it — without | |
| 240 | +// it, the arrows selected choices nobody could see (/unload, the tenth | |
| 241 | +// command, was never shown). | |
| 242 | +func pickerFirst(n, rows, selected int) int { | |
| 243 | + selected = min(selected, n-1) | |
| 244 | + return max(0, selected-rows+1) | |
| 245 | +} | |
| @@ -202,7 +202,7 @@ func (v *View) rightBar(leftWidth, width int) string { | |||
| 202 | } | 202 | } |
| 203 | 203 | ||
| 204 | // drawPicker paints the popup over the bottom of the conversation: one row | 204 | // drawPicker paints the popup over the bottom of the conversation: one row |
| 205 | -// per choice, the highlighted one in reverse. | 205 | +// per choice, the highlighted one in reverse, scrolled so that it is shown. |
| 206 | func (v *View) drawPicker(p painter) { | 206 | func (v *View) drawPicker(p painter) { |
| 207 | cs := v.choices() | 207 | cs := v.choices() |
| 208 | if len(cs) == 0 { | 208 | if len(cs) == 0 { |
| @@ -222,10 +222,11 @@ func (v *View) drawPicker(p painter) { | |||
| 222 | width = max(width, labels+2+len([]rune(c.detail))) | 222 | width = max(width, labels+2+len([]rune(c.detail))) |
| 223 | } | 223 | } |
| 224 | width = min(width+4, p.width()) | 224 | width = min(width+4, p.width()) |
| 225 | + first := pickerFirst(len(cs), rows, v.pick.selected) | ||
| 225 | for i := 0; i < rows; i++ { | 226 | for i := 0; i < rows; i++ { |
| 226 | - c := cs[i] | 227 | + c := cs[first+i] |
| 227 | style := stBody | 228 | style := stBody |
| 228 | - if i == v.pick.selected { | 229 | + if first+i == v.pick.selected { |
| 229 | style = stPick | 230 | style = stPick |
| 230 | } | 231 | } |
| 231 | text := " " + c.label + strings.Repeat(" ", max(labels-len([]rune(c.label))+2, 1)) + c.detail | 232 | text := " " + c.label + strings.Repeat(" ", max(labels-len([]rune(c.label))+2, 1)) + c.detail |
| @@ -233,3 +234,12 @@ func (v *View) drawPicker(p painter) { | |||
| 233 | p.textLimited(0, top+i, width, text, style) | 234 | p.textLimited(0, top+i, width, text, style) |
| 234 | } | 235 | } |
| 235 | } | 236 | } |
| 237 | + | ||
| 238 | +// pickerFirst is the first choice shown when only rows of n fit: 0 until the | ||
| 239 | +// selection goes past the last row, then the window follows it — without | ||
| 240 | +// it, the arrows selected choices nobody could see (/unload, the tenth | ||
| 241 | +// command, was never shown). | ||
| 242 | +func pickerFirst(n, rows, selected int) int { | ||
| 243 | + selected = min(selected, n-1) | ||
| 244 | + return max(0, selected-rows+1) | ||
| 245 | +} | ||
modified
internal/tui/pickers_test.go +18 -0 | @@ -195,3 +195,21 @@ func TestSlashIsACommandOnlyAtTheStart(t *testing.T) { | ||
| 195 | 195 | t.Errorf("the command popup opened mid-sentence: %v", labels(hs.v.choices())) |
| 196 | 196 | } |
| 197 | 197 | } |
| 198 | + | |
| 199 | +// The popup shows pickerRows choices; once the selection goes past the last | |
| 200 | +// one, the window follows it. | |
| 201 | +func TestPickerFirstFollowsTheSelection(t *testing.T) { | |
| 202 | + cases := []struct{ n, rows, selected, want int }{ | |
| 203 | + {3, 8, 2, 0}, // everything fits | |
| 204 | + {10, 8, 0, 0}, // top | |
| 205 | + {10, 8, 7, 0}, // last visible row | |
| 206 | + {10, 8, 8, 1}, // one past: scroll by one | |
| 207 | + {10, 8, 9, 2}, // the last choice, at the bottom | |
| 208 | + {10, 8, 42, 2}, // a stale selection is clamped | |
| 209 | + } | |
| 210 | + for _, c := range cases { | |
| 211 | + if got := pickerFirst(c.n, c.rows, c.selected); got != c.want { | |
| 212 | + t.Errorf("pickerFirst(%d, %d, %d) = %d, want %d", c.n, c.rows, c.selected, got, c.want) | |
| 213 | + } | |
| 214 | + } | |
| 215 | +} | |
| @@ -195,3 +195,21 @@ func TestSlashIsACommandOnlyAtTheStart(t *testing.T) { | |||
| 195 | t.Errorf("the command popup opened mid-sentence: %v", labels(hs.v.choices())) | 195 | t.Errorf("the command popup opened mid-sentence: %v", labels(hs.v.choices())) |
| 196 | } | 196 | } |
| 197 | } | 197 | } |
| 198 | + | ||
| 199 | +// The popup shows pickerRows choices; once the selection goes past the last | ||
| 200 | +// one, the window follows it. | ||
| 201 | +func TestPickerFirstFollowsTheSelection(t *testing.T) { | ||
| 202 | + cases := []struct{ n, rows, selected, want int }{ | ||
| 203 | + {3, 8, 2, 0}, // everything fits | ||
| 204 | + {10, 8, 0, 0}, // top | ||
| 205 | + {10, 8, 7, 0}, // last visible row | ||
| 206 | + {10, 8, 8, 1}, // one past: scroll by one | ||
| 207 | + {10, 8, 9, 2}, // the last choice, at the bottom | ||
| 208 | + {10, 8, 42, 2}, // a stale selection is clamped | ||
| 209 | + } | ||
| 210 | + for _, c := range cases { | ||
| 211 | + if got := pickerFirst(c.n, c.rows, c.selected); got != c.want { | ||
| 212 | + t.Errorf("pickerFirst(%d, %d, %d) = %d, want %d", c.n, c.rows, c.selected, got, c.want) | ||
| 213 | + } | ||
| 214 | + } | ||
| 215 | +} | ||
modified
internal/tui/turn.go +3 -0 | @@ -32,6 +32,9 @@ type Model interface { | ||
| 32 | 32 | Explain(err error) string |
| 33 | 33 | LastInputTokens() int |
| 34 | 34 | ForgetInputTokens() |
| 35 | + // Unload frees a model from the server's memory and says what happened | |
| 36 | + // in one line (engine.Engine.Unload). | |
| 37 | + Unload(ctx context.Context, model string) string | |
| 35 | 38 | } |
| 36 | 39 | |
| 37 | 40 | // Options is what Run needs from main: the same wiring as the two other |
| @@ -32,6 +32,9 @@ type Model interface { | |||
| 32 | Explain(err error) string | 32 | Explain(err error) string |
| 33 | LastInputTokens() int | 33 | LastInputTokens() int |
| 34 | ForgetInputTokens() | 34 | ForgetInputTokens() |
| 35 | + // Unload frees a model from the server's memory and says what happened | ||
| 36 | + // in one line (engine.Engine.Unload). | ||
| 37 | + Unload(ctx context.Context, model string) string | ||
| 35 | } | 38 | } |
| 36 | 39 | ||
| 37 | // Options is what Run needs from main: the same wiring as the two other | 40 | // Options is what Run needs from main: the same wiring as the two other |
modified
internal/tui/turn_test.go +25 -0 | @@ -10,6 +10,7 @@ import ( | ||
| 10 | 10 | "testing" |
| 11 | 11 | "time" |
| 12 | 12 | |
| 13 | + "mm/internal/command" | |
| 13 | 14 | "mm/internal/config" |
| 14 | 15 | "mm/internal/detector" |
| 15 | 16 | mmhooks "mm/internal/hooks" |
| @@ -70,6 +71,12 @@ func (m model) Summarize(context.Context, []*ai.Message, int) (string, error) { | ||
| 70 | 71 | func (m model) Explain(err error) string { return "explained: " + err.Error() } |
| 71 | 72 | func (m model) LastInputTokens() int { return m.tokens } |
| 72 | 73 | func (m model) ForgetInputTokens() { m.tokens = 0 } |
| 74 | +func (m model) Unload(_ context.Context, name string) string { | |
| 75 | + if name == "" { | |
| 76 | + name = "own" | |
| 77 | + } | |
| 78 | + return "unloaded " + name | |
| 79 | +} | |
| 73 | 80 | |
| 74 | 81 | func newTestRunner(t *testing.T, m *fakeModel) (*runner, *Transcript) { |
| 75 | 82 | t.Helper() |
| @@ -585,3 +592,21 @@ func TestRunnerHidesHooksWhenOff(t *testing.T) { | ||
| 585 | 592 | t.Errorf("hooks shown with displayHooks off:\n%s", notices(tr)) |
| 586 | 593 | } |
| 587 | 594 | } |
| 595 | + | |
| 596 | +// /unload runs off the event loop and shows the engine's line as a notice. | |
| 597 | +func TestRunnerUnload(t *testing.T) { | |
| 598 | + command.SetUnload(true) | |
| 599 | + t.Cleanup(func() { command.SetUnload(false) }) | |
| 600 | + r, tr := newTestRunner(t, &fakeModel{}) | |
| 601 | + r.submit(promptOf("/unload qwen3.5:2b")) | |
| 602 | + deadline := time.Now().Add(5 * time.Second) | |
| 603 | + for !strings.Contains(notices(tr), "🧹 unloaded qwen3.5:2b") { | |
| 604 | + if time.Now().After(deadline) { | |
| 605 | + t.Fatalf("notices:\n%s", notices(tr)) | |
| 606 | + } | |
| 607 | + time.Sleep(5 * time.Millisecond) | |
| 608 | + } | |
| 609 | + if len(r.messages) != 1 { | |
| 610 | + t.Errorf("the history changed: %d messages", len(r.messages)) | |
| 611 | + } | |
| 612 | +} | |
| @@ -10,6 +10,7 @@ import ( | |||
| 10 | "testing" | 10 | "testing" |
| 11 | "time" | 11 | "time" |
| 12 | 12 | ||
| 13 | + "mm/internal/command" | ||
| 13 | "mm/internal/config" | 14 | "mm/internal/config" |
| 14 | "mm/internal/detector" | 15 | "mm/internal/detector" |
| 15 | mmhooks "mm/internal/hooks" | 16 | mmhooks "mm/internal/hooks" |
| @@ -70,6 +71,12 @@ func (m model) Summarize(context.Context, []*ai.Message, int) (string, error) { | |||
| 70 | func (m model) Explain(err error) string { return "explained: " + err.Error() } | 71 | func (m model) Explain(err error) string { return "explained: " + err.Error() } |
| 71 | func (m model) LastInputTokens() int { return m.tokens } | 72 | func (m model) LastInputTokens() int { return m.tokens } |
| 72 | func (m model) ForgetInputTokens() { m.tokens = 0 } | 73 | func (m model) ForgetInputTokens() { m.tokens = 0 } |
| 74 | +func (m model) Unload(_ context.Context, name string) string { | ||
| 75 | + if name == "" { | ||
| 76 | + name = "own" | ||
| 77 | + } | ||
| 78 | + return "unloaded " + name | ||
| 79 | +} | ||
| 73 | 80 | ||
| 74 | func newTestRunner(t *testing.T, m *fakeModel) (*runner, *Transcript) { | 81 | func newTestRunner(t *testing.T, m *fakeModel) (*runner, *Transcript) { |
| 75 | t.Helper() | 82 | t.Helper() |
| @@ -585,3 +592,21 @@ func TestRunnerHidesHooksWhenOff(t *testing.T) { | |||
| 585 | t.Errorf("hooks shown with displayHooks off:\n%s", notices(tr)) | 592 | t.Errorf("hooks shown with displayHooks off:\n%s", notices(tr)) |
| 586 | } | 593 | } |
| 587 | } | 594 | } |
| 595 | + | ||
| 596 | +// /unload runs off the event loop and shows the engine's line as a notice. | ||
| 597 | +func TestRunnerUnload(t *testing.T) { | ||
| 598 | + command.SetUnload(true) | ||
| 599 | + t.Cleanup(func() { command.SetUnload(false) }) | ||
| 600 | + r, tr := newTestRunner(t, &fakeModel{}) | ||
| 601 | + r.submit(promptOf("/unload qwen3.5:2b")) | ||
| 602 | + deadline := time.Now().Add(5 * time.Second) | ||
| 603 | + for !strings.Contains(notices(tr), "🧹 unloaded qwen3.5:2b") { | ||
| 604 | + if time.Now().After(deadline) { | ||
| 605 | + t.Fatalf("notices:\n%s", notices(tr)) | ||
| 606 | + } | ||
| 607 | + time.Sleep(5 * time.Millisecond) | ||
| 608 | + } | ||
| 609 | + if len(r.messages) != 1 { | ||
| 610 | + t.Errorf("the history changed: %d messages", len(r.messages)) | ||
| 611 | + } | ||
| 612 | +} | ||
modified
main.go +5 -0 | @@ -211,6 +211,7 @@ func main() { | ||
| 211 | 211 | // commands join the shared table before any front end reads it, and their |
| 212 | 212 | // processes start on the first ask and stop when this one exits. |
| 213 | 213 | command.SetAgents(config.AgentNames()) |
| 214 | + command.SetUnload(config.Cfg.UnloadModels) | |
| 214 | 215 | asker := ask.NewPool() |
| 215 | 216 | defer asker.Close() |
| 216 | 217 | |
| @@ -393,6 +394,10 @@ func assembleTools(e *engine.Engine, d *detector.LoopDetector, asker tools.Agent | ||
| 393 | 394 | set.tools = append(set.tools, tools.SaveAnswer(e.G, d)) |
| 394 | 395 | set.names = append(set.names, "save_answer") |
| 395 | 396 | } |
| 397 | + if config.Cfg.UnloadModels && config.Cfg.OutputSchema == nil { | |
| 398 | + set.tools = append(set.tools, tools.UnloadModel(e.G, d, e.Unload)) | |
| 399 | + set.names = append(set.names, "unload_model") | |
| 400 | + } | |
| 396 | 401 | if config.Cfg.FetchContentTool { |
| 397 | 402 | // The product token "mm" is what the sites' robots.txt are read for. |
| 398 | 403 | fetcher := webmd.NewFetcher("mm/" + version.Short()) |
| @@ -211,6 +211,7 @@ func main() { | |||
| 211 | // commands join the shared table before any front end reads it, and their | 211 | // commands join the shared table before any front end reads it, and their |
| 212 | // processes start on the first ask and stop when this one exits. | 212 | // processes start on the first ask and stop when this one exits. |
| 213 | command.SetAgents(config.AgentNames()) | 213 | command.SetAgents(config.AgentNames()) |
| 214 | + command.SetUnload(config.Cfg.UnloadModels) | ||
| 214 | asker := ask.NewPool() | 215 | asker := ask.NewPool() |
| 215 | defer asker.Close() | 216 | defer asker.Close() |
| 216 | 217 | ||
| @@ -393,6 +394,10 @@ func assembleTools(e *engine.Engine, d *detector.LoopDetector, asker tools.Agent | |||
| 393 | set.tools = append(set.tools, tools.SaveAnswer(e.G, d)) | 394 | set.tools = append(set.tools, tools.SaveAnswer(e.G, d)) |
| 394 | set.names = append(set.names, "save_answer") | 395 | set.names = append(set.names, "save_answer") |
| 395 | } | 396 | } |
| 397 | + if config.Cfg.UnloadModels && config.Cfg.OutputSchema == nil { | ||
| 398 | + set.tools = append(set.tools, tools.UnloadModel(e.G, d, e.Unload)) | ||
| 399 | + set.names = append(set.names, "unload_model") | ||
| 400 | + } | ||
| 396 | if config.Cfg.FetchContentTool { | 401 | if config.Cfg.FetchContentTool { |
| 397 | // The product token "mm" is what the sites' robots.txt are read for. | 402 | // The product token "mm" is what the sites' robots.txt are read for. |
| 398 | fetcher := webmd.NewFetcher("mm/" + version.Short()) | 403 | fetcher := webmd.NewFetcher("mm/" + version.Short()) |