modified README.md +50 -0
| @@ -1,2 +1,52 @@ |
| 1 | 1 | # [Docker Agent] Work with `llama.cpp` |
| 2 | 2 | |
| 3 | + |
| 4 | +## Prerequisites |
| 5 | + |
| 6 | +### Install `llama.cpp` |
| 7 | +> [https://llama.app/](https://llama.app/) |
| 8 | + |
| 9 | +```bash |
| 10 | +curl -LsSf https://llama.app/install.sh | sh |
| 11 | +``` |
| 12 | + |
| 13 | +### Start `llama.cpp` and download a model |
| 14 | + |
| 15 | +Run this: |
| 16 | +```bash |
| 17 | +llama serve -hf JetBrains/Mellum2-12B-A2.5B-Instruct-GGUF-Q4_K_M:Q4_K_M |
| 18 | +``` |
| 19 | + |
| 20 | + |
| 21 | + |
| 22 | + |
| 23 | + |
| 24 | + |
| 25 | +### Check the `llama.cpp server` |
| 26 | + |
| 27 | +```bash |
| 28 | +# Server health |
| 29 | +curl http://localhost:8080/health |
| 30 | +# {"status":"ok"} |
| 31 | + |
| 32 | +# Exact name of the exposed model |
| 33 | +curl http://localhost:8080/v1/models |
| 34 | + |
| 35 | +# First chat completion |
| 36 | +curl http://localhost:8080/v1/chat/completions \ |
| 37 | + -H "Content-Type: application/json" \ |
| 38 | + -d '{ |
| 39 | + "model": "JetBrains/Mellum2-12B-A2.5B-Instruct-GGUF-Q4_K_M:Q4_K_M", |
| 40 | + "messages": [{"role": "user", "content": "Say hello in one short sentence."}], |
| 41 | + "max_tokens": 800 |
| 42 | + }' |
| 43 | + |
| 44 | +# {"choices":[{"finish_reason":"stop","index":0,"message":{"role":"assistant","content":"Hello!"}}], ... |
| 45 | +``` |
| 46 | + |
| 47 | +## Create a Docker Agent configuration |
| 48 | + |
| 49 | +**`agent.yaml`**: |
| 50 | +```yaml |
| 51 | + |
| 52 | +``` |
| | \ No newline at end of file |
| @@ -1,2 +1,52 @@ |
| 1 | # [Docker Agent] Work with `llama.cpp` | 1 | # [Docker Agent] Work with `llama.cpp` |
| 2 | | 2 | |
| | 3 | + |
| | 4 | +## Prerequisites |
| | 5 | + |
| | 6 | +### Install `llama.cpp` |
| | 7 | +> [https://llama.app/](https://llama.app/) |
| | 8 | + |
| | 9 | +```bash |
| | 10 | +curl -LsSf https://llama.app/install.sh | sh |
| | 11 | +``` |
| | 12 | + |
| | 13 | +### Start `llama.cpp` and download a model |
| | 14 | + |
| | 15 | +Run this: |
| | 16 | +```bash |
| | 17 | +llama serve -hf JetBrains/Mellum2-12B-A2.5B-Instruct-GGUF-Q4_K_M:Q4_K_M |
| | 18 | +``` |
| | 19 | + |
| | 20 | + |
| | 21 | + |
| | 22 | + |
| | 23 | + |
| | 24 | + |
| | 25 | +### Check the `llama.cpp server` |
| | 26 | + |
| | 27 | +```bash |
| | 28 | +# Server health |
| | 29 | +curl http://localhost:8080/health |
| | 30 | +# {"status":"ok"} |
| | 31 | + |
| | 32 | +# Exact name of the exposed model |
| | 33 | +curl http://localhost:8080/v1/models |
| | 34 | + |
| | 35 | +# First chat completion |
| | 36 | +curl http://localhost:8080/v1/chat/completions \ |
| | 37 | + -H "Content-Type: application/json" \ |
| | 38 | + -d '{ |
| | 39 | + "model": "JetBrains/Mellum2-12B-A2.5B-Instruct-GGUF-Q4_K_M:Q4_K_M", |
| | 40 | + "messages": [{"role": "user", "content": "Say hello in one short sentence."}], |
| | 41 | + "max_tokens": 800 |
| | 42 | + }' |
| | 43 | + |
| | 44 | +# {"choices":[{"finish_reason":"stop","index":0,"message":{"role":"assistant","content":"Hello!"}}], ... |
| | 45 | +``` |
| | 46 | + |
| | 47 | +## Create a Docker Agent configuration |
| | 48 | + |
| | 49 | +**`agent.yaml`**: |
| | 50 | +```yaml |
| | 51 | + |
| | 52 | +``` |
| \ No newline at end of file | | \ No newline at end of file |
added agent.yaml +29 -0
| new file mode 100644 |
| @@ -0,0 +1,29 @@ |
| 1 | +providers: |
| 2 | + llamacpp: |
| 3 | + api_type: openai_chatcompletions |
| 4 | + base_url: http://localhost:8080/v1 |
| 5 | + |
| 6 | +models: |
| 7 | + mellum2: |
| 8 | + provider: llamacpp |
| 9 | + model: JetBrains/Mellum2-12B-A2.5B-Instruct-GGUF-Q4_K_M:Q4_K_M |
| 10 | + #max_tokens: 8192 |
| 11 | + temperature: 0.7 |
| 12 | + provider_opts: |
| 13 | + context_size: 262144 |
| 14 | + |
| 15 | +agents: |
| 16 | + root: |
| 17 | + model: mellum2 |
| 18 | + description: A helpful AI assistant running on a local llama.cpp server |
| 19 | + instruction: | |
| 20 | + You name is Bob 🤓, you are a knowledgeable code assistant that helps users with various tasks. |
| 21 | + Be helpful, accurate, and concise in your responses. |
| 22 | + You have access to the local filesystem and shell: use these tools |
| 23 | + welcome_message: | |
| 24 | + |
| 25 | + 🤖 Local Assistant propulsed by **llama.cpp** 🦙 |
| 26 | + |
| 27 | + toolsets: |
| 28 | + - type: filesystem |
| 29 | + - type: shell |
| new file mode 100644 |
| @@ -0,0 +1,29 @@ |
| | 1 | +providers: |
| | 2 | + llamacpp: |
| | 3 | + api_type: openai_chatcompletions |
| | 4 | + base_url: http://localhost:8080/v1 |
| | 5 | + |
| | 6 | +models: |
| | 7 | + mellum2: |
| | 8 | + provider: llamacpp |
| | 9 | + model: JetBrains/Mellum2-12B-A2.5B-Instruct-GGUF-Q4_K_M:Q4_K_M |
| | 10 | + #max_tokens: 8192 |
| | 11 | + temperature: 0.7 |
| | 12 | + provider_opts: |
| | 13 | + context_size: 262144 |
| | 14 | + |
| | 15 | +agents: |
| | 16 | + root: |
| | 17 | + model: mellum2 |
| | 18 | + description: A helpful AI assistant running on a local llama.cpp server |
| | 19 | + instruction: | |
| | 20 | + You name is Bob 🤓, you are a knowledgeable code assistant that helps users with various tasks. |
| | 21 | + Be helpful, accurate, and concise in your responses. |
| | 22 | + You have access to the local filesystem and shell: use these tools |
| | 23 | + welcome_message: | |
| | 24 | + |
| | 25 | + 🤖 Local Assistant propulsed by **llama.cpp** 🦙 |
| | 26 | + |
| | 27 | + toolsets: |
| | 28 | + - type: filesystem |
| | 29 | + - type: shell |