🛟 Updated.
0d48ec0 parent: a96cf99 added
.tickets/issues/0049-recreate-the-readme.yaml +13 -0 | new file mode 100644 | ||
| @@ -0,0 +1,13 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +id: 49 | |
| 3 | +title: Recreate the README | |
| 4 | +state: open | |
| 5 | +author: | |
| 6 | + name: k33g | |
| 7 | + email: ph.charriere@gmail.com | |
| 8 | +createdAt: 2026-09-28T18:44:44.821Z | |
| 9 | +updatedAt: 2026-09-28T18:44:44.821Z | |
| 10 | +labels: [] | |
| 11 | +body: "" | |
| 12 | +tasks: [] | |
| 13 | +comments: [] | |
| new file mode 100644 | |||
| @@ -0,0 +1,13 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +id: 49 | ||
| 3 | +title: Recreate the README | ||
| 4 | +state: open | ||
| 5 | +author: | ||
| 6 | + name: k33g | ||
| 7 | + email: ph.charriere@gmail.com | ||
| 8 | +createdAt: 2026-09-28T18:44:44.821Z | ||
| 9 | +updatedAt: 2026-09-28T18:44:44.821Z | ||
| 10 | +labels: [] | ||
| 11 | +body: "" | ||
| 12 | +tasks: [] | ||
| 13 | +comments: [] | ||
added
.tickets/issues/0050-create-a-web-site.yaml +13 -0 | new file mode 100644 | ||
| @@ -0,0 +1,13 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +id: 50 | |
| 3 | +title: Create a web site | |
| 4 | +state: open | |
| 5 | +author: | |
| 6 | + name: k33g | |
| 7 | + email: ph.charriere@gmail.com | |
| 8 | +createdAt: 2026-09-28T18:44:59.825Z | |
| 9 | +updatedAt: 2026-09-28T18:44:59.825Z | |
| 10 | +labels: [] | |
| 11 | +body: "" | |
| 12 | +tasks: [] | |
| 13 | +comments: [] | |
| new file mode 100644 | |||
| @@ -0,0 +1,13 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +id: 50 | ||
| 3 | +title: Create a web site | ||
| 4 | +state: open | ||
| 5 | +author: | ||
| 6 | + name: k33g | ||
| 7 | + email: ph.charriere@gmail.com | ||
| 8 | +createdAt: 2026-09-28T18:44:59.825Z | ||
| 9 | +updatedAt: 2026-09-28T18:44:59.825Z | ||
| 10 | +labels: [] | ||
| 11 | +body: "" | ||
| 12 | +tasks: [] | ||
| 13 | +comments: [] | ||
added
.tickets/issues/0051-query-other-models-from-the-prompt.yaml +19 -0 | new file mode 100644 | ||
| @@ -0,0 +1,19 @@ | ||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | |
| 2 | +id: 51 | |
| 3 | +title: Query other models from the prompt | |
| 4 | +state: open | |
| 5 | +author: | |
| 6 | + name: k33g | |
| 7 | + email: ph.charriere@gmail.com | |
| 8 | +createdAt: 2026-09-29T04:49:19.839Z | |
| 9 | +updatedAt: 2026-09-29T04:51:59.493Z | |
| 10 | +labels: | |
| 11 | + - priority::high | |
| 12 | + - enhancement | |
| 13 | + - builtin tool | |
| 14 | +body: | | |
| 15 | + - Add a list of model (and provider) to the agent config file (with an alias / nickname) | |
| 16 | + - Add a builtin tool "query_model" | |
| 17 | + - Be able to force a json output | |
| 18 | +tasks: [] | |
| 19 | +comments: [] | |
| new file mode 100644 | |||
| @@ -0,0 +1,19 @@ | |||
| 1 | +# Managed by IssueSpec. Hand edits are welcome; keep the schema valid. | ||
| 2 | +id: 51 | ||
| 3 | +title: Query other models from the prompt | ||
| 4 | +state: open | ||
| 5 | +author: | ||
| 6 | + name: k33g | ||
| 7 | + email: ph.charriere@gmail.com | ||
| 8 | +createdAt: 2026-09-29T04:49:19.839Z | ||
| 9 | +updatedAt: 2026-09-29T04:51:59.493Z | ||
| 10 | +labels: | ||
| 11 | + - priority::high | ||
| 12 | + - enhancement | ||
| 13 | + - builtin tool | ||
| 14 | +body: | | ||
| 15 | + - Add a list of model (and provider) to the agent config file (with an alias / nickname) | ||
| 16 | + - Add a builtin tool "query_model" | ||
| 17 | + - Be able to force a json output | ||
| 18 | +tasks: [] | ||
| 19 | +comments: [] | ||
modified
README.md +1 -0 | @@ -100,3 +100,4 @@ Anything that speaks ACP can host `mm -acp` the same way: **JetBrains IDEs** (In | ||
| 100 | 100 | - [Documentation](docs/README.md) — tutorials, how-to guides, reference and explanations, in English and French. |
| 101 | 101 | - The `skills/` directory: Markdown procedures the agent can load with its `read_skill` tool. |
| 102 | 102 | - `agent.lsp.yaml`: the same agent with a language server behind an `lsp` tool (definition, references, hover, symbols, diagnostics, rename) — see [Navigate and refactor code with a language server](docs/en/how-to/navigate-code-with-lsp.md). |
| 103 | +- `fetchContentTool: true` in `agent.dmr.yaml`: the built-in `fetch_content` tool, which fetches a web page as Markdown (honouring its `robots.txt`) and can save it to a file without showing it — see [Fetch a web page as Markdown](docs/en/how-to/fetch-a-web-page.md). | |
| @@ -100,3 +100,4 @@ Anything that speaks ACP can host `mm -acp` the same way: **JetBrains IDEs** (In | |||
| 100 | - [Documentation](docs/README.md) — tutorials, how-to guides, reference and explanations, in English and French. | 100 | - [Documentation](docs/README.md) — tutorials, how-to guides, reference and explanations, in English and French. |
| 101 | - The `skills/` directory: Markdown procedures the agent can load with its `read_skill` tool. | 101 | - The `skills/` directory: Markdown procedures the agent can load with its `read_skill` tool. |
| 102 | - `agent.lsp.yaml`: the same agent with a language server behind an `lsp` tool (definition, references, hover, symbols, diagnostics, rename) — see [Navigate and refactor code with a language server](docs/en/how-to/navigate-code-with-lsp.md). | 102 | - `agent.lsp.yaml`: the same agent with a language server behind an `lsp` tool (definition, references, hover, symbols, diagnostics, rename) — see [Navigate and refactor code with a language server](docs/en/how-to/navigate-code-with-lsp.md). |
| 103 | +- `fetchContentTool: true` in `agent.dmr.yaml`: the built-in `fetch_content` tool, which fetches a web page as Markdown (honouring its `robots.txt`) and can save it to a file without showing it — see [Fetch a web page as Markdown](docs/en/how-to/fetch-a-web-page.md). | ||
modified
agent.dmr.yaml +5 -0 | @@ -31,6 +31,11 @@ editTools: true | ||
| 31 | 31 | # Set to false to disable plan creation and tracking via the `plan` tool. |
| 32 | 32 | planTool: true |
| 33 | 33 | |
| 34 | +# The built-in fetch_content tool: fetch a web page and get it as Markdown | |
| 35 | +# ("récupère le contenu de https://…", optionally "et sauve le dans page.md"). | |
| 36 | +# Honours the site's robots.txt. Off by default — it reaches the network. | |
| 37 | +fetchContentTool: true | |
| 38 | + | |
| 34 | 39 | # Language servers for the `lsp` tool (definition, references, hover, symbols, diagnostics, rename). |
| 35 | 40 | # The file extension picks the server. A server missing from the PATH is skipped with a warning. |
| 36 | 41 | # No server listed = no `lsp` tool. Examples (uncomment what you have installed): |
| @@ -31,6 +31,11 @@ editTools: true | |||
| 31 | # Set to false to disable plan creation and tracking via the `plan` tool. | 31 | # Set to false to disable plan creation and tracking via the `plan` tool. |
| 32 | planTool: true | 32 | planTool: true |
| 33 | 33 | ||
| 34 | +# The built-in fetch_content tool: fetch a web page and get it as Markdown | ||
| 35 | +# ("récupère le contenu de https://…", optionally "et sauve le dans page.md"). | ||
| 36 | +# Honours the site's robots.txt. Off by default — it reaches the network. | ||
| 37 | +fetchContentTool: true | ||
| 38 | + | ||
| 34 | # Language servers for the `lsp` tool (definition, references, hover, symbols, diagnostics, rename). | 39 | # Language servers for the `lsp` tool (definition, references, hover, symbols, diagnostics, rename). |
| 35 | # The file extension picks the server. A server missing from the PATH is skipped with a warning. | 40 | # The file extension picks the server. A server missing from the PATH is skipped with a warning. |
| 36 | # No server listed = no `lsp` tool. Examples (uncomment what you have installed): | 41 | # No server listed = no `lsp` tool. Examples (uncomment what you have installed): |
modified
demo/agent.dmr.2.yaml +3 -0 | @@ -29,6 +29,9 @@ bashTool: true | ||
| 29 | 29 | # The built-in file tools: read_file, write_file, edit_file. |
| 30 | 30 | editTools: true |
| 31 | 31 | |
| 32 | +# fetch content on the web | |
| 33 | +fetchContentTool: true | |
| 34 | + | |
| 32 | 35 | # The built-in plan tool for managing multi-step plans. |
| 33 | 36 | # Set to false to disable plan creation and tracking via the `plan` tool. |
| 34 | 37 | planTool: true |
| @@ -29,6 +29,9 @@ bashTool: true | |||
| 29 | # The built-in file tools: read_file, write_file, edit_file. | 29 | # The built-in file tools: read_file, write_file, edit_file. |
| 30 | editTools: true | 30 | editTools: true |
| 31 | 31 | ||
| 32 | +# fetch content on the web | ||
| 33 | +fetchContentTool: true | ||
| 34 | + | ||
| 32 | # The built-in plan tool for managing multi-step plans. | 35 | # The built-in plan tool for managing multi-step plans. |
| 33 | # Set to false to disable plan creation and tracking via the `plan` tool. | 36 | # Set to false to disable plan creation and tracking via the `plan` tool. |
| 34 | planTool: true | 37 | planTool: true |
modified
docs/diagrams/packages.drawio +2 -2 | @@ -1,3 +1,3 @@ | ||
| 1 | -<mxfile host="app.diagrams.net" modified="2026-09-15T00:00:00.000Z" agent="mm docs" version="24.0.0" type="device"><diagram id="packages" name="packages"><mxGraphModel dx="1400" dy="900" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1400" pageHeight="900"><root><mxCell id="0"/><mxCell id="1" parent="0"/><mxCell id="main" value="<b>main (mm)</b><br/><font style="font-size:10px">wiring: config, engine, tools, then one front end</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="1"><mxGeometry x="420" y="40" width="200" height="60" as="geometry"/></mxCell><mxCell id="agent" value="<b>internal/agent</b><br/><font style="font-size:10px">terminal REPL: /quit /abort /compact /new</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1"><mxGeometry x="160" y="180" width="200" height="60" as="geometry"/></mxCell><mxCell id="acp" value="<b>internal/acp</b><br/><font style="font-size:10px">editor front end: Agent Client Protocol on stdio</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1"><mxGeometry x="700" y="180" width="200" height="60" as="geometry"/></mxCell><mxCell id="session" value="<b>internal/session</b><br/><font style="font-size:10px">what a conversation starts from; the /new reset</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="320" width="200" height="60" as="geometry"/></mxCell><mxCell id="compact" value="<b>internal/compact</b><br/><font style="font-size:10px">history compression into a model-written summary</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="40" y="320" width="200" height="60" as="geometry"/></mxCell><mxCell id="engine" value="<b>internal/engine</b><br/><font style="font-size:10px">LLM connection + generation, one Provider per server</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="700" y="320" width="200" height="60" as="geometry"/></mxCell><mxCell id="tools" value="<b>internal/tools</b><br/><font style="font-size:10px">built-in tools: bash, read_skill, file tools, lsp</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="460" width="200" height="60" as="geometry"/></mxCell><mxCell id="detector" value="<b>internal/detector</b><br/><font style="font-size:10px">repeated-action (loop) detection</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="40" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="config" value="<b>internal/config</b><br/><font style="font-size:10px">settings with YAML overrides (agent.yaml)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="260" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="spinner" value="<b>internal/spinner</b><br/><font style="font-size:10px">terminal spinner and styling</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="480" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="ui" value="<b>internal/ui</b><br/><font style="font-size:10px">human output destination + shared event sink</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="700" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="fileedit" value="<b>internal/fileedit</b><br/><font style="font-size:10px">exact-replacement file editing</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="920" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="skills" value="<b>internal/skills</b><br/><font style="font-size:10px">markdown skills catalogue (./skills)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1140" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="genkit" value="<b>github.com/firebase/genkit</b><br/><font style="font-size:10px">ai.Message, tools, model calls</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="160" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="acpsdk" value="<b>github.com/coder/acp-go-sdk</b><br/><font style="font-size:10px">JSON-RPC transport for ACP</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="700" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="openai" value="<b>github.com/openai/openai-go</b><br/><font style="font-size:10px">OpenAI-compatible HTTP client</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="920" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="yaml" value="<b>gopkg.in/yaml.v3</b><br/><font style="font-size:10px">YAML parsing</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="400" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="e0" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="agent"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="version" value="<b>internal/version</b><br/><font style="font-size:10px">which mm this is: build stamp, else the commit</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1380" y="600" width="200" height="60" as="geometry"/></mxCell> | |
| 1 | +<mxfile host="app.diagrams.net" modified="2026-09-15T00:00:00.000Z" agent="mm docs" version="24.0.0" type="device"><diagram id="packages" name="packages"><mxGraphModel dx="1400" dy="900" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1400" pageHeight="900"><root><mxCell id="0"/><mxCell id="1" parent="0"/><mxCell id="main" value="<b>main (mm)</b><br/><font style="font-size:10px">wiring: config, engine, tools, then one front end</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="1"><mxGeometry x="420" y="40" width="200" height="60" as="geometry"/></mxCell><mxCell id="agent" value="<b>internal/agent</b><br/><font style="font-size:10px">terminal REPL: /quit /abort /compact /new</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1"><mxGeometry x="160" y="180" width="200" height="60" as="geometry"/></mxCell><mxCell id="acp" value="<b>internal/acp</b><br/><font style="font-size:10px">editor front end: Agent Client Protocol on stdio</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1"><mxGeometry x="700" y="180" width="200" height="60" as="geometry"/></mxCell><mxCell id="session" value="<b>internal/session</b><br/><font style="font-size:10px">what a conversation starts from; the /new reset</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="320" width="200" height="60" as="geometry"/></mxCell><mxCell id="compact" value="<b>internal/compact</b><br/><font style="font-size:10px">history compression into a model-written summary</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="40" y="320" width="200" height="60" as="geometry"/></mxCell><mxCell id="engine" value="<b>internal/engine</b><br/><font style="font-size:10px">LLM connection + generation, one Provider per server</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="700" y="320" width="200" height="60" as="geometry"/></mxCell><mxCell id="tools" value="<b>internal/tools</b><br/><font style="font-size:10px">built-in tools: bash, read_skill, file tools, lsp, fetch_content</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="460" width="200" height="60" as="geometry"/></mxCell><mxCell id="detector" value="<b>internal/detector</b><br/><font style="font-size:10px">repeated-action (loop) detection</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="40" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="config" value="<b>internal/config</b><br/><font style="font-size:10px">settings with YAML overrides (agent.yaml)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="260" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="spinner" value="<b>internal/spinner</b><br/><font style="font-size:10px">terminal spinner and styling</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="480" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="ui" value="<b>internal/ui</b><br/><font style="font-size:10px">human output destination + shared event sink</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="700" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="fileedit" value="<b>internal/fileedit</b><br/><font style="font-size:10px">exact-replacement file editing</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="920" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="skills" value="<b>internal/skills</b><br/><font style="font-size:10px">markdown skills catalogue (./skills)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1140" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="genkit" value="<b>github.com/firebase/genkit</b><br/><font style="font-size:10px">ai.Message, tools, model calls</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="160" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="acpsdk" value="<b>github.com/coder/acp-go-sdk</b><br/><font style="font-size:10px">JSON-RPC transport for ACP</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="700" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="openai" value="<b>github.com/openai/openai-go</b><br/><font style="font-size:10px">OpenAI-compatible HTTP client</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="920" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="yaml" value="<b>gopkg.in/yaml.v3</b><br/><font style="font-size:10px">YAML parsing</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="400" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="e0" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="agent"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="version" value="<b>internal/version</b><br/><font style="font-size:10px">which mm this is: build stamp, else the commit</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1380" y="600" width="200" height="60" as="geometry"/></mxCell> | |
| 2 | 2 | <mxCell id="e61" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="version"><mxGeometry relative="1" as="geometry"/></mxCell> |
| 3 | - <mxCell id="e1" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="acp"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e2" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e3" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e4" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e5" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="skills"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e6" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e7" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="tools"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e8" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e9" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e10" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e11" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e12" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="acp" target="acpsdk"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e13" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="acp" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e14" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="compact"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e15" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e16" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e17" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e18" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e19" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e20" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="agent" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e21" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="compact" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e22" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="compact" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e23" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="config" target="yaml"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e24" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="engine" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e25" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="engine" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e26" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="engine" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e27" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="engine" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e28" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="engine" target="openai"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e29" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e30" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e31" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="fileedit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e32" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="skills"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e33" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e34" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e35" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="tools" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e36" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="session" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="mention" value="<b>internal/mention</b><br/><font style="font-size:10px">the @path notation of a question</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="400" width="200" height="60" as="geometry"/></mxCell><mxCell id="e37" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="mention"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e38" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="mention"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="store" value="<b>internal/store</b><br/><font style="font-size:10px">per-project session files (.mm/sessions/<id>.json)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="480" width="200" height="60" as="geometry"/></mxCell><mxCell id="e39" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="store"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e40" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="store"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="tui" value="<b>internal/tui</b><br/><font style="font-size:10px">window front end: one screen on tcell (mm -tui)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1"><mxGeometry x="430" y="160" width="200" height="60" as="geometry"/></mxCell><mxCell id="command" value="<b>internal/command</b><br/><font style="font-size:10px">the one slash-command table, per front end</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="560" width="200" height="60" as="geometry"/></mxCell><mxCell id="tcell" value="<b>github.com/gdamore/tcell/v2</b><br/><font style="font-size:10px">terminal screen, keys, mouse</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="920" y="840" width="200" height="60" as="geometry"/></mxCell><mxCell id="e41" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="tui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e42" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e43" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e44" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="compact"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e45" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="store"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e46" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="mention"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e47" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e48" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e49" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e50" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="command"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e51" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="tcell"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e52" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="command"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e53" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="command"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e54" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="command" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="clipboard" value="<b>internal/clipboard</b><br/><font style="font-size:10px">system clipboard: native tool, then OSC 52</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1140" y="680" width="200" height="60" as="geometry"/></mxCell><mxCell id="e55" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="clipboard"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e56" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="clipboard"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="lsp" value="<b>internal/lsp</b><br/><font style="font-size:10px">language-server client: definition, references, hover, symbols, diagnostics, rename</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1160" y="600" width="220" height="60" as="geometry"/></mxCell><mxCell id="lspdev" value="<b>go.lsp.dev/protocol + jsonrpc2</b><br/><font style="font-size:10px">LSP types and JSON-RPC 2.0 over stdio</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#ffffff;strokeColor=#999999;" vertex="1" parent="1"><mxGeometry x="1160" y="700" width="220" height="60" as="geometry"/></mxCell><mxCell id="e57" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="lsp"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e58" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="lsp"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e59" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="lsp" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e60" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="lsp" target="lspdev"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="legend" value="<b>mm — package dependencies</b><br/>Arrow: "imports". Yellow: entry point. Blue: front ends. Green: shared logic. Grey: leaf packages. Dashed purple: third-party modules.<br/>Regenerate the edge list from: go list -f '{{.ImportPath}}: {{join .Imports " "}}' ./..." style="text;html=1;whiteSpace=wrap;align=left;verticalAlign=top;fillColor=#ffffff;strokeColor=#999999;" vertex="1" parent="1"><mxGeometry x="900" y="40" width="440" height="90" as="geometry"/></mxCell></root></mxGraphModel></diagram></mxfile> | |
| \ No newline at end of file | ||
| 3 | + <mxCell id="e1" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="acp"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e2" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e3" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e4" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e5" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="skills"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e6" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e7" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="tools"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e8" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e9" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e10" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e11" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e12" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="acp" target="acpsdk"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e13" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="acp" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e14" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="compact"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e15" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e16" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e17" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e18" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e19" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e20" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="agent" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e21" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="compact" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e22" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="compact" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e23" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="config" target="yaml"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e24" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="engine" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e25" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="engine" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e26" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="engine" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e27" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="engine" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e28" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="engine" target="openai"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e29" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e30" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e31" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="fileedit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e32" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="skills"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e33" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e34" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e35" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="tools" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e36" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="session" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="mention" value="<b>internal/mention</b><br/><font style="font-size:10px">the @path notation of a question</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="400" width="200" height="60" as="geometry"/></mxCell><mxCell id="e37" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="mention"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e38" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="mention"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="store" value="<b>internal/store</b><br/><font style="font-size:10px">per-project session files (.mm/sessions/<id>.json)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="480" width="200" height="60" as="geometry"/></mxCell><mxCell id="e39" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="store"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e40" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="store"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="tui" value="<b>internal/tui</b><br/><font style="font-size:10px">window front end: one screen on tcell (mm -tui)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1"><mxGeometry x="430" y="160" width="200" height="60" as="geometry"/></mxCell><mxCell id="command" value="<b>internal/command</b><br/><font style="font-size:10px">the one slash-command table, per front end</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="560" width="200" height="60" as="geometry"/></mxCell><mxCell id="tcell" value="<b>github.com/gdamore/tcell/v2</b><br/><font style="font-size:10px">terminal screen, keys, mouse</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="920" y="840" width="200" height="60" as="geometry"/></mxCell><mxCell id="e41" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="tui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e42" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e43" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e44" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="compact"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e45" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="store"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e46" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="mention"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e47" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e48" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e49" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e50" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="command"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e51" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="tcell"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e52" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="command"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e53" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="command"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e54" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="command" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="clipboard" value="<b>internal/clipboard</b><br/><font style="font-size:10px">system clipboard: native tool, then OSC 52</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1140" y="680" width="200" height="60" as="geometry"/></mxCell><mxCell id="e55" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="clipboard"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e56" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="clipboard"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="lsp" value="<b>internal/lsp</b><br/><font style="font-size:10px">language-server client: definition, references, hover, symbols, diagnostics, rename</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1160" y="600" width="220" height="60" as="geometry"/></mxCell><mxCell id="lspdev" value="<b>go.lsp.dev/protocol + jsonrpc2</b><br/><font style="font-size:10px">LSP types and JSON-RPC 2.0 over stdio</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#ffffff;strokeColor=#999999;" vertex="1" parent="1"><mxGeometry x="1160" y="700" width="220" height="60" as="geometry"/></mxCell><mxCell id="e57" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="lsp"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e58" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="lsp"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e59" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="lsp" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e60" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="lsp" target="lspdev"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="legend" value="<b>mm — package dependencies</b><br/>Arrow: "imports". Yellow: entry point. Blue: front ends. Green: shared logic. Grey: leaf packages. Dashed purple: third-party modules.<br/>Regenerate the edge list from: go list -f '{{.ImportPath}}: {{join .Imports " "}}' ./..." style="text;html=1;whiteSpace=wrap;align=left;verticalAlign=top;fillColor=#ffffff;strokeColor=#999999;" vertex="1" parent="1"><mxGeometry x="900" y="40" width="440" height="90" as="geometry"/></mxCell><mxCell id="webmd" value="<b>internal/webmd</b><br/><font style="font-size:10px">web page → Markdown: fetch (robots.txt, charset → UTF-8), clean, convert</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1400" y="460" width="220" height="60" as="geometry"/></mxCell><mxCell id="xnet" value="<b>golang.org/x/net/html (+ x/text)</b><br/><font style="font-size:10px">HTML5 parser, charset lookup</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="1400" y="760" width="220" height="60" as="geometry"/></mxCell><mxCell id="e_main_webmd" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="webmd"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e_tools_webmd" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="webmd"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e_webmd_xnet" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="webmd" target="xnet"><mxGeometry relative="1" as="geometry"/></mxCell></root></mxGraphModel></diagram></mxfile> | |
| \ No newline at end of file | ||
| @@ -1,3 +1,3 @@ | |||
| 1 | -<mxfile host="app.diagrams.net" modified="2026-09-15T00:00:00.000Z" agent="mm docs" version="24.0.0" type="device"><diagram id="packages" name="packages"><mxGraphModel dx="1400" dy="900" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1400" pageHeight="900"><root><mxCell id="0"/><mxCell id="1" parent="0"/><mxCell id="main" value="<b>main (mm)</b><br/><font style="font-size:10px">wiring: config, engine, tools, then one front end</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="1"><mxGeometry x="420" y="40" width="200" height="60" as="geometry"/></mxCell><mxCell id="agent" value="<b>internal/agent</b><br/><font style="font-size:10px">terminal REPL: /quit /abort /compact /new</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1"><mxGeometry x="160" y="180" width="200" height="60" as="geometry"/></mxCell><mxCell id="acp" value="<b>internal/acp</b><br/><font style="font-size:10px">editor front end: Agent Client Protocol on stdio</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1"><mxGeometry x="700" y="180" width="200" height="60" as="geometry"/></mxCell><mxCell id="session" value="<b>internal/session</b><br/><font style="font-size:10px">what a conversation starts from; the /new reset</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="320" width="200" height="60" as="geometry"/></mxCell><mxCell id="compact" value="<b>internal/compact</b><br/><font style="font-size:10px">history compression into a model-written summary</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="40" y="320" width="200" height="60" as="geometry"/></mxCell><mxCell id="engine" value="<b>internal/engine</b><br/><font style="font-size:10px">LLM connection + generation, one Provider per server</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="700" y="320" width="200" height="60" as="geometry"/></mxCell><mxCell id="tools" value="<b>internal/tools</b><br/><font style="font-size:10px">built-in tools: bash, read_skill, file tools, lsp</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="460" width="200" height="60" as="geometry"/></mxCell><mxCell id="detector" value="<b>internal/detector</b><br/><font style="font-size:10px">repeated-action (loop) detection</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="40" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="config" value="<b>internal/config</b><br/><font style="font-size:10px">settings with YAML overrides (agent.yaml)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="260" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="spinner" value="<b>internal/spinner</b><br/><font style="font-size:10px">terminal spinner and styling</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="480" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="ui" value="<b>internal/ui</b><br/><font style="font-size:10px">human output destination + shared event sink</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="700" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="fileedit" value="<b>internal/fileedit</b><br/><font style="font-size:10px">exact-replacement file editing</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="920" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="skills" value="<b>internal/skills</b><br/><font style="font-size:10px">markdown skills catalogue (./skills)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1140" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="genkit" value="<b>github.com/firebase/genkit</b><br/><font style="font-size:10px">ai.Message, tools, model calls</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="160" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="acpsdk" value="<b>github.com/coder/acp-go-sdk</b><br/><font style="font-size:10px">JSON-RPC transport for ACP</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="700" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="openai" value="<b>github.com/openai/openai-go</b><br/><font style="font-size:10px">OpenAI-compatible HTTP client</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="920" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="yaml" value="<b>gopkg.in/yaml.v3</b><br/><font style="font-size:10px">YAML parsing</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="400" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="e0" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="agent"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="version" value="<b>internal/version</b><br/><font style="font-size:10px">which mm this is: build stamp, else the commit</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1380" y="600" width="200" height="60" as="geometry"/></mxCell> | 1 | +<mxfile host="app.diagrams.net" modified="2026-09-15T00:00:00.000Z" agent="mm docs" version="24.0.0" type="device"><diagram id="packages" name="packages"><mxGraphModel dx="1400" dy="900" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1400" pageHeight="900"><root><mxCell id="0"/><mxCell id="1" parent="0"/><mxCell id="main" value="<b>main (mm)</b><br/><font style="font-size:10px">wiring: config, engine, tools, then one front end</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="1"><mxGeometry x="420" y="40" width="200" height="60" as="geometry"/></mxCell><mxCell id="agent" value="<b>internal/agent</b><br/><font style="font-size:10px">terminal REPL: /quit /abort /compact /new</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1"><mxGeometry x="160" y="180" width="200" height="60" as="geometry"/></mxCell><mxCell id="acp" value="<b>internal/acp</b><br/><font style="font-size:10px">editor front end: Agent Client Protocol on stdio</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1"><mxGeometry x="700" y="180" width="200" height="60" as="geometry"/></mxCell><mxCell id="session" value="<b>internal/session</b><br/><font style="font-size:10px">what a conversation starts from; the /new reset</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="320" width="200" height="60" as="geometry"/></mxCell><mxCell id="compact" value="<b>internal/compact</b><br/><font style="font-size:10px">history compression into a model-written summary</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="40" y="320" width="200" height="60" as="geometry"/></mxCell><mxCell id="engine" value="<b>internal/engine</b><br/><font style="font-size:10px">LLM connection + generation, one Provider per server</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="700" y="320" width="200" height="60" as="geometry"/></mxCell><mxCell id="tools" value="<b>internal/tools</b><br/><font style="font-size:10px">built-in tools: bash, read_skill, file tools, lsp, fetch_content</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="460" width="200" height="60" as="geometry"/></mxCell><mxCell id="detector" value="<b>internal/detector</b><br/><font style="font-size:10px">repeated-action (loop) detection</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="40" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="config" value="<b>internal/config</b><br/><font style="font-size:10px">settings with YAML overrides (agent.yaml)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="260" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="spinner" value="<b>internal/spinner</b><br/><font style="font-size:10px">terminal spinner and styling</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="480" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="ui" value="<b>internal/ui</b><br/><font style="font-size:10px">human output destination + shared event sink</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="700" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="fileedit" value="<b>internal/fileedit</b><br/><font style="font-size:10px">exact-replacement file editing</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="920" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="skills" value="<b>internal/skills</b><br/><font style="font-size:10px">markdown skills catalogue (./skills)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1140" y="600" width="200" height="60" as="geometry"/></mxCell><mxCell id="genkit" value="<b>github.com/firebase/genkit</b><br/><font style="font-size:10px">ai.Message, tools, model calls</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="160" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="acpsdk" value="<b>github.com/coder/acp-go-sdk</b><br/><font style="font-size:10px">JSON-RPC transport for ACP</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="700" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="openai" value="<b>github.com/openai/openai-go</b><br/><font style="font-size:10px">OpenAI-compatible HTTP client</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="920" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="yaml" value="<b>gopkg.in/yaml.v3</b><br/><font style="font-size:10px">YAML parsing</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="400" y="760" width="200" height="60" as="geometry"/></mxCell><mxCell id="e0" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="agent"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="version" value="<b>internal/version</b><br/><font style="font-size:10px">which mm this is: build stamp, else the commit</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1380" y="600" width="200" height="60" as="geometry"/></mxCell> |
| 2 | <mxCell id="e61" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="version"><mxGeometry relative="1" as="geometry"/></mxCell> | 2 | <mxCell id="e61" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="version"><mxGeometry relative="1" as="geometry"/></mxCell> |
| 3 | - <mxCell id="e1" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="acp"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e2" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e3" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e4" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e5" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="skills"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e6" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e7" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="tools"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e8" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e9" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e10" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e11" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e12" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="acp" target="acpsdk"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e13" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="acp" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e14" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="compact"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e15" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e16" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e17" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e18" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e19" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e20" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="agent" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e21" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="compact" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e22" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="compact" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e23" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="config" target="yaml"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e24" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="engine" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e25" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="engine" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e26" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="engine" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e27" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="engine" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e28" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="engine" target="openai"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e29" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e30" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e31" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="fileedit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e32" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="skills"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e33" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e34" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e35" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="tools" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e36" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="session" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="mention" value="<b>internal/mention</b><br/><font style="font-size:10px">the @path notation of a question</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="400" width="200" height="60" as="geometry"/></mxCell><mxCell id="e37" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="mention"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e38" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="mention"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="store" value="<b>internal/store</b><br/><font style="font-size:10px">per-project session files (.mm/sessions/<id>.json)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="480" width="200" height="60" as="geometry"/></mxCell><mxCell id="e39" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="store"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e40" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="store"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="tui" value="<b>internal/tui</b><br/><font style="font-size:10px">window front end: one screen on tcell (mm -tui)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1"><mxGeometry x="430" y="160" width="200" height="60" as="geometry"/></mxCell><mxCell id="command" value="<b>internal/command</b><br/><font style="font-size:10px">the one slash-command table, per front end</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="560" width="200" height="60" as="geometry"/></mxCell><mxCell id="tcell" value="<b>github.com/gdamore/tcell/v2</b><br/><font style="font-size:10px">terminal screen, keys, mouse</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="920" y="840" width="200" height="60" as="geometry"/></mxCell><mxCell id="e41" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="tui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e42" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e43" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e44" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="compact"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e45" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="store"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e46" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="mention"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e47" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e48" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e49" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e50" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="command"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e51" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="tcell"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e52" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="command"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e53" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="command"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e54" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="command" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="clipboard" value="<b>internal/clipboard</b><br/><font style="font-size:10px">system clipboard: native tool, then OSC 52</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1140" y="680" width="200" height="60" as="geometry"/></mxCell><mxCell id="e55" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="clipboard"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e56" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="clipboard"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="lsp" value="<b>internal/lsp</b><br/><font style="font-size:10px">language-server client: definition, references, hover, symbols, diagnostics, rename</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1160" y="600" width="220" height="60" as="geometry"/></mxCell><mxCell id="lspdev" value="<b>go.lsp.dev/protocol + jsonrpc2</b><br/><font style="font-size:10px">LSP types and JSON-RPC 2.0 over stdio</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#ffffff;strokeColor=#999999;" vertex="1" parent="1"><mxGeometry x="1160" y="700" width="220" height="60" as="geometry"/></mxCell><mxCell id="e57" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="lsp"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e58" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="lsp"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e59" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="lsp" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e60" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="lsp" target="lspdev"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="legend" value="<b>mm — package dependencies</b><br/>Arrow: "imports". Yellow: entry point. Blue: front ends. Green: shared logic. Grey: leaf packages. Dashed purple: third-party modules.<br/>Regenerate the edge list from: go list -f '{{.ImportPath}}: {{join .Imports " "}}' ./..." style="text;html=1;whiteSpace=wrap;align=left;verticalAlign=top;fillColor=#ffffff;strokeColor=#999999;" vertex="1" parent="1"><mxGeometry x="900" y="40" width="440" height="90" as="geometry"/></mxCell></root></mxGraphModel></diagram></mxfile> | ||
| \ No newline at end of file | \ No newline at end of file | ||
| 3 | + <mxCell id="e1" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="acp"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e2" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e3" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e4" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e5" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="skills"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e6" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e7" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="tools"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e8" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e9" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e10" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e11" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e12" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="acp" target="acpsdk"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e13" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="acp" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e14" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="compact"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e15" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e16" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e17" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e18" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e19" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e20" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="agent" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e21" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="compact" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e22" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="compact" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e23" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="config" target="yaml"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e24" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="engine" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e25" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="engine" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e26" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="engine" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e27" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="engine" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e28" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="engine" target="openai"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e29" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e30" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e31" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="fileedit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e32" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="skills"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e33" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="spinner"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e34" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e35" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="tools" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e36" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="session" target="genkit"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="mention" value="<b>internal/mention</b><br/><font style="font-size:10px">the @path notation of a question</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="400" width="200" height="60" as="geometry"/></mxCell><mxCell id="e37" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="mention"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e38" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="mention"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="store" value="<b>internal/store</b><br/><font style="font-size:10px">per-project session files (.mm/sessions/<id>.json)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="480" width="200" height="60" as="geometry"/></mxCell><mxCell id="e39" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="store"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e40" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="store"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="tui" value="<b>internal/tui</b><br/><font style="font-size:10px">window front end: one screen on tcell (mm -tui)</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1"><mxGeometry x="430" y="160" width="200" height="60" as="geometry"/></mxCell><mxCell id="command" value="<b>internal/command</b><br/><font style="font-size:10px">the one slash-command table, per front end</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1"><mxGeometry x="430" y="560" width="200" height="60" as="geometry"/></mxCell><mxCell id="tcell" value="<b>github.com/gdamore/tcell/v2</b><br/><font style="font-size:10px">terminal screen, keys, mouse</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="920" y="840" width="200" height="60" as="geometry"/></mxCell><mxCell id="e41" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="tui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e42" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="ui"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e43" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="engine"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e44" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="compact"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e45" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="store"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e46" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="mention"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e47" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e48" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="detector"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e49" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e50" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="command"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e51" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="tcell"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e52" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="command"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e53" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="acp" target="command"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e54" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="command" target="session"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="clipboard" value="<b>internal/clipboard</b><br/><font style="font-size:10px">system clipboard: native tool, then OSC 52</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1140" y="680" width="200" height="60" as="geometry"/></mxCell><mxCell id="e55" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="agent" target="clipboard"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e56" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tui" target="clipboard"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="lsp" value="<b>internal/lsp</b><br/><font style="font-size:10px">language-server client: definition, references, hover, symbols, diagnostics, rename</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1160" y="600" width="220" height="60" as="geometry"/></mxCell><mxCell id="lspdev" value="<b>go.lsp.dev/protocol + jsonrpc2</b><br/><font style="font-size:10px">LSP types and JSON-RPC 2.0 over stdio</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#ffffff;strokeColor=#999999;" vertex="1" parent="1"><mxGeometry x="1160" y="700" width="220" height="60" as="geometry"/></mxCell><mxCell id="e57" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="lsp"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e58" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="lsp"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e59" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="lsp" target="config"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e60" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;dashed=1;" edge="1" parent="1" source="lsp" target="lspdev"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="legend" value="<b>mm — package dependencies</b><br/>Arrow: "imports". Yellow: entry point. Blue: front ends. Green: shared logic. Grey: leaf packages. Dashed purple: third-party modules.<br/>Regenerate the edge list from: go list -f '{{.ImportPath}}: {{join .Imports " "}}' ./..." style="text;html=1;whiteSpace=wrap;align=left;verticalAlign=top;fillColor=#ffffff;strokeColor=#999999;" vertex="1" parent="1"><mxGeometry x="900" y="40" width="440" height="90" as="geometry"/></mxCell><mxCell id="webmd" value="<b>internal/webmd</b><br/><font style="font-size:10px">web page → Markdown: fetch (robots.txt, charset → UTF-8), clean, convert</font>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;" vertex="1" parent="1"><mxGeometry x="1400" y="460" width="220" height="60" as="geometry"/></mxCell><mxCell id="xnet" value="<b>golang.org/x/net/html (+ x/text)</b><br/><font style="font-size:10px">HTML5 parser, charset lookup</font>" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1"><mxGeometry x="1400" y="760" width="220" height="60" as="geometry"/></mxCell><mxCell id="e_main_webmd" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="main" target="webmd"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e_tools_webmd" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="tools" target="webmd"><mxGeometry relative="1" as="geometry"/></mxCell><mxCell id="e_webmd_xnet" style="endArrow=block;html=1;rounded=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="webmd" target="xnet"><mxGeometry relative="1" as="geometry"/></mxCell></root></mxGraphModel></diagram></mxfile> | ||
| \ No newline at end of file | \ No newline at end of file | ||
modified
docs/en/README.md +4 -2 | @@ -1,6 +1,6 @@ | ||
| 1 | 1 | # mini-me — documentation (English) |
| 2 | 2 | |
| 3 | -`mm` is a coding agent that runs in your terminal — line by line, or in a single window with `mm -tui` — or inside an editor through the Agent Client Protocol. It talks to a local model server (Docker Model Runner or llama.cpp's `llama-server`) and works with a handful of built-in tools: `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`. | |
| 3 | +`mm` is a coding agent that runs in your terminal — line by line, or in a single window with `mm -tui` — or inside an editor through the Agent Client Protocol. It talks to a local model server (Docker Model Runner or llama.cpp's `llama-server`) and works with a handful of built-in tools: `bash`, `read_skill`, `read_file`, `write_file`, `edit_file` — plus, when turned on, `lsp` and `fetch_content`. | |
| 4 | 4 | |
| 5 | 5 | This documentation is organised along the four [Diátaxis](https://diataxis.fr) quadrants. Pick the one that matches what you need right now. |
| 6 | 6 | |
| @@ -23,6 +23,7 @@ This documentation is organised along the four [Diátaxis](https://diataxis.fr) | ||
| 23 | 23 | - [Use the window mode (`mm -tui`)](how-to/use-the-window-mode.md) |
| 24 | 24 | - [Navigate and refactor code with a language server (`lsp`)](how-to/navigate-code-with-lsp.md) |
| 25 | 25 | - [Run a single prompt and exit (`-p`/`--prompt`)](how-to/run-a-single-prompt.md) |
| 26 | +- [Fetch a web page as Markdown (`fetch_content`)](how-to/fetch-a-web-page.md) — read a page, or save it to a file without showing it | |
| 26 | 27 | |
| 27 | 28 | ### MCP (Model Context Protocol) How-to |
| 28 | 29 | - [Use Context7 for documentation](how-to/mcp-use-context7.md) |
| @@ -34,7 +35,7 @@ This documentation is organised along the four [Diátaxis](https://diataxis.fr) | ||
| 34 | 35 | - [Command line](reference/cli.md) — flags, environment variables, start-up banner |
| 35 | 36 | - [Slash commands](reference/slash-commands.md) — `/quit`, `/abort`, `/compact`, `/new`, `/sessions`, `/resume`, `/help`, keyboard shortcuts, window keys |
| 36 | 37 | - [Configuration file](reference/configuration.md) — every YAML key, its default and its validation rule |
| 37 | -- [Built-in tools](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file` | |
| 38 | +- [Built-in tools](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`, `lsp`, `fetch_content` | |
| 38 | 39 | - [Skill file format](reference/skill-format.md) |
| 39 | 40 | - [Agent Client Protocol surface](reference/acp.md) |
| 40 | 41 | |
| @@ -50,6 +51,7 @@ This documentation is organised along the four [Diátaxis](https://diataxis.fr) | ||
| 50 | 51 | - [Context compression](explanation/context-compression.md) |
| 51 | 52 | - [Persistent sessions](explanation/persistent-sessions.md) — files per project, why a start is a new session, `session/load` versus docker agent's `session/resume` |
| 52 | 53 | - [The `lsp` tool](explanation/lsp-tool.md) — one tool with an `action` field, a `symbol` field instead of columns, files synced from disk, rename gated by permission |
| 54 | +- [The `fetch_content` tool](explanation/fetch-content-tool.md) — why a built-in tool beside MCP `fetch`, a home-made converter, robots.txt, errors as numbered text, known limits | |
| 53 | 55 | - [Thinking models](explanation/thinking-models.md) — where a model's reasoning comes from, why `mm` reads it below Genkit, and why reading it keeps the watchdog alive |
| 54 | 56 | - Package diagram (draw.io source): [`docs/diagrams/packages.drawio`](../diagrams/packages.drawio) |
| 55 | 57 | |
| @@ -1,6 +1,6 @@ | |||
| 1 | # mini-me — documentation (English) | 1 | # mini-me — documentation (English) |
| 2 | 2 | ||
| 3 | -`mm` is a coding agent that runs in your terminal — line by line, or in a single window with `mm -tui` — or inside an editor through the Agent Client Protocol. It talks to a local model server (Docker Model Runner or llama.cpp's `llama-server`) and works with a handful of built-in tools: `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`. | 3 | +`mm` is a coding agent that runs in your terminal — line by line, or in a single window with `mm -tui` — or inside an editor through the Agent Client Protocol. It talks to a local model server (Docker Model Runner or llama.cpp's `llama-server`) and works with a handful of built-in tools: `bash`, `read_skill`, `read_file`, `write_file`, `edit_file` — plus, when turned on, `lsp` and `fetch_content`. |
| 4 | 4 | ||
| 5 | This documentation is organised along the four [Diátaxis](https://diataxis.fr) quadrants. Pick the one that matches what you need right now. | 5 | This documentation is organised along the four [Diátaxis](https://diataxis.fr) quadrants. Pick the one that matches what you need right now. |
| 6 | 6 | ||
| @@ -23,6 +23,7 @@ This documentation is organised along the four [Diátaxis](https://diataxis.fr) | |||
| 23 | - [Use the window mode (`mm -tui`)](how-to/use-the-window-mode.md) | 23 | - [Use the window mode (`mm -tui`)](how-to/use-the-window-mode.md) |
| 24 | - [Navigate and refactor code with a language server (`lsp`)](how-to/navigate-code-with-lsp.md) | 24 | - [Navigate and refactor code with a language server (`lsp`)](how-to/navigate-code-with-lsp.md) |
| 25 | - [Run a single prompt and exit (`-p`/`--prompt`)](how-to/run-a-single-prompt.md) | 25 | - [Run a single prompt and exit (`-p`/`--prompt`)](how-to/run-a-single-prompt.md) |
| 26 | +- [Fetch a web page as Markdown (`fetch_content`)](how-to/fetch-a-web-page.md) — read a page, or save it to a file without showing it | ||
| 26 | 27 | ||
| 27 | ### MCP (Model Context Protocol) How-to | 28 | ### MCP (Model Context Protocol) How-to |
| 28 | - [Use Context7 for documentation](how-to/mcp-use-context7.md) | 29 | - [Use Context7 for documentation](how-to/mcp-use-context7.md) |
| @@ -34,7 +35,7 @@ This documentation is organised along the four [Diátaxis](https://diataxis.fr) | |||
| 34 | - [Command line](reference/cli.md) — flags, environment variables, start-up banner | 35 | - [Command line](reference/cli.md) — flags, environment variables, start-up banner |
| 35 | - [Slash commands](reference/slash-commands.md) — `/quit`, `/abort`, `/compact`, `/new`, `/sessions`, `/resume`, `/help`, keyboard shortcuts, window keys | 36 | - [Slash commands](reference/slash-commands.md) — `/quit`, `/abort`, `/compact`, `/new`, `/sessions`, `/resume`, `/help`, keyboard shortcuts, window keys |
| 36 | - [Configuration file](reference/configuration.md) — every YAML key, its default and its validation rule | 37 | - [Configuration file](reference/configuration.md) — every YAML key, its default and its validation rule |
| 37 | -- [Built-in tools](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file` | 38 | +- [Built-in tools](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`, `lsp`, `fetch_content` |
| 38 | - [Skill file format](reference/skill-format.md) | 39 | - [Skill file format](reference/skill-format.md) |
| 39 | - [Agent Client Protocol surface](reference/acp.md) | 40 | - [Agent Client Protocol surface](reference/acp.md) |
| 40 | 41 | ||
| @@ -50,6 +51,7 @@ This documentation is organised along the four [Diátaxis](https://diataxis.fr) | |||
| 50 | - [Context compression](explanation/context-compression.md) | 51 | - [Context compression](explanation/context-compression.md) |
| 51 | - [Persistent sessions](explanation/persistent-sessions.md) — files per project, why a start is a new session, `session/load` versus docker agent's `session/resume` | 52 | - [Persistent sessions](explanation/persistent-sessions.md) — files per project, why a start is a new session, `session/load` versus docker agent's `session/resume` |
| 52 | - [The `lsp` tool](explanation/lsp-tool.md) — one tool with an `action` field, a `symbol` field instead of columns, files synced from disk, rename gated by permission | 53 | - [The `lsp` tool](explanation/lsp-tool.md) — one tool with an `action` field, a `symbol` field instead of columns, files synced from disk, rename gated by permission |
| 54 | +- [The `fetch_content` tool](explanation/fetch-content-tool.md) — why a built-in tool beside MCP `fetch`, a home-made converter, robots.txt, errors as numbered text, known limits | ||
| 53 | - [Thinking models](explanation/thinking-models.md) — where a model's reasoning comes from, why `mm` reads it below Genkit, and why reading it keeps the watchdog alive | 55 | - [Thinking models](explanation/thinking-models.md) — where a model's reasoning comes from, why `mm` reads it below Genkit, and why reading it keeps the watchdog alive |
| 54 | - Package diagram (draw.io source): [`docs/diagrams/packages.drawio`](../diagrams/packages.drawio) | 56 | - Package diagram (draw.io source): [`docs/diagrams/packages.drawio`](../diagrams/packages.drawio) |
| 55 | 57 | ||
added
docs/en/explanation/fetch-content-tool.md +49 -0 | new file mode 100644 | ||
| @@ -0,0 +1,49 @@ | ||
| 1 | +# The `fetch_content` tool — explanation | |
| 2 | + | |
| 3 | +## What is this about? | |
| 4 | + | |
| 5 | +`fetch_content` gives the model one thing it could only do clumsily before: read a web page. With `bash` alone, a model runs `curl`, gets hundreds of kilobytes of HTML — scripts, styles, menus, tracking code — and either drowns its context window in it or summarises a page it never really saw. `fetch_content` downloads the page, throws away what is not content, and hands back Markdown: the format a language model reads best, and the one a user can keep. | |
| 6 | + | |
| 7 | +The work is split in two. `internal/webmd` is a plain Go package with three stages — `Fetcher.Fetch`, `Clean`, `Convert` — that knows nothing about models. `internal/tools/fetchcontent.go` is the thin tool around it: arguments, display, permission, and the text the model gets back. | |
| 8 | + | |
| 9 | +## Why a built-in tool, and not the MCP `fetch` server | |
| 10 | + | |
| 11 | +An MCP `fetch` server can be wired into `mm` already. The built-in tool exists for what such a server does not guarantee: | |
| 12 | + | |
| 13 | +- **It works with nothing else installed.** No Node or Python runtime, no gateway: the conversion is in the binary. | |
| 14 | +- **It can save without returning.** "Save it to hello.txt, don't show it" is common, and a tool that always returns the page makes the model pay for the whole page in its context just to copy it into a file — which a small local model does badly anyway. `output` writes the file directly and returns one line. | |
| 15 | +- **Its conversion rules are ours.** The specification it follows (see the reference) decides headings, lists, tables, escaping; the tests pin them. | |
| 16 | + | |
| 17 | +The name is `fetch_content`, not `fetch`, so it never collides with an MCP tool of that name. | |
| 18 | + | |
| 19 | +## Why the conversion is written here | |
| 20 | + | |
| 21 | +A library such as html-to-markdown was considered. The specification asks for behaviour that such libraries either do not have or make awkward: complex tables kept as raw HTML *or* flattened on demand, a clean-up pass driven by roles and class words with a guard for layout wrappers, links resolved against the final URL, a strict "one trailing newline, no preamble" output. Bending a general-purpose converter to all of it would have cost about as much as writing the few hundred lines, and would have added a dependency whose defaults change under us. The parsing itself is not ours: `golang.org/x/net/html` is the HTML5 parser, so broken HTML is repaired exactly the way a browser repairs it. | |
| 22 | + | |
| 23 | +## Why robots.txt is honoured | |
| 24 | + | |
| 25 | +`mm` fetches on behalf of a person, one page at a time — closer to a browser than to a crawler. It still honours `robots.txt`, because the request said so and because an agent that ignores a site's stated wishes is a poor citizen however few requests it sends. It identifies itself as `mm/<version>`, so a site can address it by name. | |
| 26 | + | |
| 27 | +The rules follow RFC 9309, including its cautious corners: a site whose `robots.txt` answers with a server error, or cannot be reached, is treated as closed — the RFC's choice, and the only safe reading of "we could not ask". Every redirection is checked too, so a redirect cannot lead into a forbidden path. | |
| 28 | + | |
| 29 | +## Why failures are text with a number | |
| 30 | + | |
| 31 | +Every `mm` tool returns failures as text, so the model can read them and react. `fetch_content` adds the error code of its specification (1 conversion, 2 input, 3 usage) because the three call for different reactions: fix the call (3), give up or try another page (2), or tell the user the thing is not a web page (1). A model that sees `error 2 (input) — … robots.txt does not allow "mm"` has no reason to try `curl` next. | |
| 32 | + | |
| 33 | +## Choices that trade exactness for safety | |
| 34 | + | |
| 35 | +- **Off by default.** It is the only built-in tool that reaches the network; an operator turns it on knowingly. | |
| 36 | +- **Limits.** 30 seconds, 5 MiB, 10 redirections: a tool call must not hang a turn or fill memory. | |
| 37 | +- **Encoding.** The declared encoding wins (BOM, then HTTP header, then `<meta>`); when nothing is declared, valid UTF-8 is taken as UTF-8 and anything else as windows-1252 — the web's historical default. | |
| 38 | +- **Images as `data:` URIs** are reduced to their alt text: kilobytes of base64 are no use to a reader, human or model. | |
| 39 | + | |
| 40 | +## Known limits | |
| 41 | + | |
| 42 | +- The clean-up is heuristic. It removes navigation, ads and banners that are marked as such; an ad without a telling class or role stays, and an unusual class name can, rarely, remove content. Anything wrapping a `main`, an `article` or an `h1` is protected. | |
| 43 | +- Inline elements that a page separates only with CSS come out glued together: tags rendered as `<a>go</a><a>tui</a>` read `gotui`. | |
| 44 | +- Pages built by JavaScript in the browser have little content in their HTML; `fetch_content` does not run scripts. | |
| 45 | +- Only HTML is converted: a PDF or a JSON document is refused with `error 1`. | |
| 46 | + | |
| 47 | +## How it relates to the rest | |
| 48 | + | |
| 49 | +`fetch_content` follows the same contract as the other tools (see [architecture](architecture.md)): one display line in the terminal, an event with a kind (`fetch`) for ACP and the window, a permission request only when it writes a file, and a record in the loop detector. Its output goes through the same `maxOutput` truncation as `bash`. | |
| new file mode 100644 | |||
| @@ -0,0 +1,49 @@ | |||
| 1 | +# The `fetch_content` tool — explanation | ||
| 2 | + | ||
| 3 | +## What is this about? | ||
| 4 | + | ||
| 5 | +`fetch_content` gives the model one thing it could only do clumsily before: read a web page. With `bash` alone, a model runs `curl`, gets hundreds of kilobytes of HTML — scripts, styles, menus, tracking code — and either drowns its context window in it or summarises a page it never really saw. `fetch_content` downloads the page, throws away what is not content, and hands back Markdown: the format a language model reads best, and the one a user can keep. | ||
| 6 | + | ||
| 7 | +The work is split in two. `internal/webmd` is a plain Go package with three stages — `Fetcher.Fetch`, `Clean`, `Convert` — that knows nothing about models. `internal/tools/fetchcontent.go` is the thin tool around it: arguments, display, permission, and the text the model gets back. | ||
| 8 | + | ||
| 9 | +## Why a built-in tool, and not the MCP `fetch` server | ||
| 10 | + | ||
| 11 | +An MCP `fetch` server can be wired into `mm` already. The built-in tool exists for what such a server does not guarantee: | ||
| 12 | + | ||
| 13 | +- **It works with nothing else installed.** No Node or Python runtime, no gateway: the conversion is in the binary. | ||
| 14 | +- **It can save without returning.** "Save it to hello.txt, don't show it" is common, and a tool that always returns the page makes the model pay for the whole page in its context just to copy it into a file — which a small local model does badly anyway. `output` writes the file directly and returns one line. | ||
| 15 | +- **Its conversion rules are ours.** The specification it follows (see the reference) decides headings, lists, tables, escaping; the tests pin them. | ||
| 16 | + | ||
| 17 | +The name is `fetch_content`, not `fetch`, so it never collides with an MCP tool of that name. | ||
| 18 | + | ||
| 19 | +## Why the conversion is written here | ||
| 20 | + | ||
| 21 | +A library such as html-to-markdown was considered. The specification asks for behaviour that such libraries either do not have or make awkward: complex tables kept as raw HTML *or* flattened on demand, a clean-up pass driven by roles and class words with a guard for layout wrappers, links resolved against the final URL, a strict "one trailing newline, no preamble" output. Bending a general-purpose converter to all of it would have cost about as much as writing the few hundred lines, and would have added a dependency whose defaults change under us. The parsing itself is not ours: `golang.org/x/net/html` is the HTML5 parser, so broken HTML is repaired exactly the way a browser repairs it. | ||
| 22 | + | ||
| 23 | +## Why robots.txt is honoured | ||
| 24 | + | ||
| 25 | +`mm` fetches on behalf of a person, one page at a time — closer to a browser than to a crawler. It still honours `robots.txt`, because the request said so and because an agent that ignores a site's stated wishes is a poor citizen however few requests it sends. It identifies itself as `mm/<version>`, so a site can address it by name. | ||
| 26 | + | ||
| 27 | +The rules follow RFC 9309, including its cautious corners: a site whose `robots.txt` answers with a server error, or cannot be reached, is treated as closed — the RFC's choice, and the only safe reading of "we could not ask". Every redirection is checked too, so a redirect cannot lead into a forbidden path. | ||
| 28 | + | ||
| 29 | +## Why failures are text with a number | ||
| 30 | + | ||
| 31 | +Every `mm` tool returns failures as text, so the model can read them and react. `fetch_content` adds the error code of its specification (1 conversion, 2 input, 3 usage) because the three call for different reactions: fix the call (3), give up or try another page (2), or tell the user the thing is not a web page (1). A model that sees `error 2 (input) — … robots.txt does not allow "mm"` has no reason to try `curl` next. | ||
| 32 | + | ||
| 33 | +## Choices that trade exactness for safety | ||
| 34 | + | ||
| 35 | +- **Off by default.** It is the only built-in tool that reaches the network; an operator turns it on knowingly. | ||
| 36 | +- **Limits.** 30 seconds, 5 MiB, 10 redirections: a tool call must not hang a turn or fill memory. | ||
| 37 | +- **Encoding.** The declared encoding wins (BOM, then HTTP header, then `<meta>`); when nothing is declared, valid UTF-8 is taken as UTF-8 and anything else as windows-1252 — the web's historical default. | ||
| 38 | +- **Images as `data:` URIs** are reduced to their alt text: kilobytes of base64 are no use to a reader, human or model. | ||
| 39 | + | ||
| 40 | +## Known limits | ||
| 41 | + | ||
| 42 | +- The clean-up is heuristic. It removes navigation, ads and banners that are marked as such; an ad without a telling class or role stays, and an unusual class name can, rarely, remove content. Anything wrapping a `main`, an `article` or an `h1` is protected. | ||
| 43 | +- Inline elements that a page separates only with CSS come out glued together: tags rendered as `<a>go</a><a>tui</a>` read `gotui`. | ||
| 44 | +- Pages built by JavaScript in the browser have little content in their HTML; `fetch_content` does not run scripts. | ||
| 45 | +- Only HTML is converted: a PDF or a JSON document is refused with `error 1`. | ||
| 46 | + | ||
| 47 | +## How it relates to the rest | ||
| 48 | + | ||
| 49 | +`fetch_content` follows the same contract as the other tools (see [architecture](architecture.md)): one display line in the terminal, an event with a kind (`fetch`) for ACP and the window, a permission request only when it writes a file, and a record in the loop detector. Its output goes through the same `maxOutput` truncation as `bash`. | ||
added
docs/en/how-to/fetch-a-web-page.md +65 -0 | new file mode 100644 | ||
| @@ -0,0 +1,65 @@ | ||
| 1 | +# How to fetch a web page as Markdown | |
| 2 | + | |
| 3 | +This guide shows how to let the model read a web page, or save it to a file, with the built-in `fetch_content` tool. It assumes you already run `mm` with a configuration file. | |
| 4 | + | |
| 5 | +## Steps | |
| 6 | + | |
| 7 | +1. Turn the tool on in your configuration file (it is off by default): | |
| 8 | + | |
| 9 | + ```yaml | |
| 10 | + fetchContentTool: true | |
| 11 | + ``` | |
| 12 | + | |
| 13 | +2. Start `mm` and check that the start-up banner lists `fetch_content` among the tools. | |
| 14 | + | |
| 15 | +3. Ask for the page in plain words, in any language: | |
| 16 | + | |
| 17 | + ```text | |
| 18 | + fetch the content of https://k33g.org | |
| 19 | + ``` | |
| 20 | + | |
| 21 | + ```text | |
| 22 | + récupère le contenu de https://k33g.org | |
| 23 | + ``` | |
| 24 | + | |
| 25 | + The terminal shows `🌐 fetch_content: https://k33g.org`, then the first lines of the Markdown, and the model answers from it. | |
| 26 | + | |
| 27 | +## Save the page to a file instead of showing it | |
| 28 | + | |
| 29 | +Name the file and say not to display it: | |
| 30 | + | |
| 31 | +```text | |
| 32 | +récupère le contenu de https://hello.com mais ne l'affiche pas et sauve le dans hello.txt | |
| 33 | +``` | |
| 34 | + | |
| 35 | +The model passes `output: hello.txt`. The Markdown is written to `hello.txt` (relative to the directory `mm` runs in, or to the project opened in the editor) and the tool returns only one line: | |
| 36 | + | |
| 37 | +```text | |
| 38 | +[fetch_content: saved 9 lines of Markdown (421 bytes) from https://hello.com to hello.txt — the content is not shown] | |
| 39 | +``` | |
| 40 | + | |
| 41 | +The page does not enter the conversation, so a long page does not use up the context window. | |
| 42 | + | |
| 43 | +## Variants | |
| 44 | + | |
| 45 | +- **A URL typed carelessly.** `k33g.org` and `https//k33g.org` both work: a missing scheme becomes `https://`, a missing colon is added back. | |
| 46 | +- **Complex tables.** Tables with merged cells are kept as raw HTML by default. Ask for flattened tables ("…as Markdown tables, flatten them") and the model passes `tables: flatten`. | |
| 47 | +- **Strikethrough and task lists.** Ask for them explicitly ("keep the struck-through text", "keep the checkboxes") and the model passes `strikethrough: true` or `taskList: true`. | |
| 48 | +- **Links relative to another site.** If the page is a mirror, ask to resolve links against the original; the model passes `baseUrl`. | |
| 49 | +- **In an editor (ACP) or with `permissions: ask`.** Reading a page needs no permission; saving it to a file asks first, like `write_file`. | |
| 50 | + | |
| 51 | +## When it fails | |
| 52 | + | |
| 53 | +The model receives `[fetch_content: error N (…) — reason]` and usually tells you why: | |
| 54 | + | |
| 55 | +- `error 2 (input)`: the site is unreachable, returned an HTTP error, or its `robots.txt` forbids robots to read that page. `mm` does not bypass `robots.txt`. | |
| 56 | +- `error 1 (conversion)`: the URL is not an HTML page (a PDF, JSON, an image) or is larger than 5 MiB. | |
| 57 | +- `error 3 (usage)`: the call itself is wrong, for example an `ftp://` URL. | |
| 58 | + | |
| 59 | +If the model uses `curl` through `bash` instead, the tool is probably not declared: check `fetchContentTool: true` and the banner. | |
| 60 | + | |
| 61 | +## See also | |
| 62 | + | |
| 63 | +- Every field, rule and error code: [built-in tools reference](../reference/tools.md#fetch_content) | |
| 64 | +- The configuration key: [configuration reference](../reference/configuration.md) | |
| 65 | +- Why the tool works this way: [the `fetch_content` tool](../explanation/fetch-content-tool.md) | |
| new file mode 100644 | |||
| @@ -0,0 +1,65 @@ | |||
| 1 | +# How to fetch a web page as Markdown | ||
| 2 | + | ||
| 3 | +This guide shows how to let the model read a web page, or save it to a file, with the built-in `fetch_content` tool. It assumes you already run `mm` with a configuration file. | ||
| 4 | + | ||
| 5 | +## Steps | ||
| 6 | + | ||
| 7 | +1. Turn the tool on in your configuration file (it is off by default): | ||
| 8 | + | ||
| 9 | + ```yaml | ||
| 10 | + fetchContentTool: true | ||
| 11 | + ``` | ||
| 12 | + | ||
| 13 | +2. Start `mm` and check that the start-up banner lists `fetch_content` among the tools. | ||
| 14 | + | ||
| 15 | +3. Ask for the page in plain words, in any language: | ||
| 16 | + | ||
| 17 | + ```text | ||
| 18 | + fetch the content of https://k33g.org | ||
| 19 | + ``` | ||
| 20 | + | ||
| 21 | + ```text | ||
| 22 | + récupère le contenu de https://k33g.org | ||
| 23 | + ``` | ||
| 24 | + | ||
| 25 | + The terminal shows `🌐 fetch_content: https://k33g.org`, then the first lines of the Markdown, and the model answers from it. | ||
| 26 | + | ||
| 27 | +## Save the page to a file instead of showing it | ||
| 28 | + | ||
| 29 | +Name the file and say not to display it: | ||
| 30 | + | ||
| 31 | +```text | ||
| 32 | +récupère le contenu de https://hello.com mais ne l'affiche pas et sauve le dans hello.txt | ||
| 33 | +``` | ||
| 34 | + | ||
| 35 | +The model passes `output: hello.txt`. The Markdown is written to `hello.txt` (relative to the directory `mm` runs in, or to the project opened in the editor) and the tool returns only one line: | ||
| 36 | + | ||
| 37 | +```text | ||
| 38 | +[fetch_content: saved 9 lines of Markdown (421 bytes) from https://hello.com to hello.txt — the content is not shown] | ||
| 39 | +``` | ||
| 40 | + | ||
| 41 | +The page does not enter the conversation, so a long page does not use up the context window. | ||
| 42 | + | ||
| 43 | +## Variants | ||
| 44 | + | ||
| 45 | +- **A URL typed carelessly.** `k33g.org` and `https//k33g.org` both work: a missing scheme becomes `https://`, a missing colon is added back. | ||
| 46 | +- **Complex tables.** Tables with merged cells are kept as raw HTML by default. Ask for flattened tables ("…as Markdown tables, flatten them") and the model passes `tables: flatten`. | ||
| 47 | +- **Strikethrough and task lists.** Ask for them explicitly ("keep the struck-through text", "keep the checkboxes") and the model passes `strikethrough: true` or `taskList: true`. | ||
| 48 | +- **Links relative to another site.** If the page is a mirror, ask to resolve links against the original; the model passes `baseUrl`. | ||
| 49 | +- **In an editor (ACP) or with `permissions: ask`.** Reading a page needs no permission; saving it to a file asks first, like `write_file`. | ||
| 50 | + | ||
| 51 | +## When it fails | ||
| 52 | + | ||
| 53 | +The model receives `[fetch_content: error N (…) — reason]` and usually tells you why: | ||
| 54 | + | ||
| 55 | +- `error 2 (input)`: the site is unreachable, returned an HTTP error, or its `robots.txt` forbids robots to read that page. `mm` does not bypass `robots.txt`. | ||
| 56 | +- `error 1 (conversion)`: the URL is not an HTML page (a PDF, JSON, an image) or is larger than 5 MiB. | ||
| 57 | +- `error 3 (usage)`: the call itself is wrong, for example an `ftp://` URL. | ||
| 58 | + | ||
| 59 | +If the model uses `curl` through `bash` instead, the tool is probably not declared: check `fetchContentTool: true` and the banner. | ||
| 60 | + | ||
| 61 | +## See also | ||
| 62 | + | ||
| 63 | +- Every field, rule and error code: [built-in tools reference](../reference/tools.md#fetch_content) | ||
| 64 | +- The configuration key: [configuration reference](../reference/configuration.md) | ||
| 65 | +- Why the tool works this way: [the `fetch_content` tool](../explanation/fetch-content-tool.md) | ||
modified
docs/en/reference/acp.md +1 -0 | @@ -128,6 +128,7 @@ A transport error or a `cancelled` outcome counts as a rejection. | ||
| 128 | 128 | | `execute` | `execute` | `bash` (and any unknown kind) | |
| 129 | 129 | | `read` | `read` | `read_skill`, `read_file` | |
| 130 | 130 | | `edit` | `edit` | `write_file`, `edit_file` | |
| 131 | +| `fetch` | `fetch` | `fetch_content` | | |
| 131 | 132 | |
| 132 | 133 | ## Tool call identifiers |
| 133 | 134 | |
| @@ -128,6 +128,7 @@ A transport error or a `cancelled` outcome counts as a rejection. | |||
| 128 | | `execute` | `execute` | `bash` (and any unknown kind) | | 128 | | `execute` | `execute` | `bash` (and any unknown kind) | |
| 129 | | `read` | `read` | `read_skill`, `read_file` | | 129 | | `read` | `read` | `read_skill`, `read_file` | |
| 130 | | `edit` | `edit` | `write_file`, `edit_file` | | 130 | | `edit` | `edit` | `write_file`, `edit_file` | |
| 131 | +| `fetch` | `fetch` | `fetch_content` | | ||
| 131 | 132 | ||
| 132 | ## Tool call identifiers | 133 | ## Tool call identifiers |
| 133 | 134 | ||
modified
docs/en/reference/configuration.md +1 -0 | @@ -21,6 +21,7 @@ | ||
| 21 | 21 | | `skillsDir` | string | `skills` | Directory holding the skills, as `<name>.md` files or `<name>/SKILL.md` directories. A relative path is resolved from the directory `mm` is started in (terminal mode, or ACP mode without a config file), or next to the config file (ACP mode with one); the banner warns with the absolute path when nothing is found there. | |
| 22 | 22 | | `bashTool` | bool | `true` | Declare `bash`. `false` removes it: the model can still read and edit files (`editTools`) and load skills, but never runs a command. Reword a system prompt that still announces a bash tool. | |
| 23 | 23 | | `editTools` | bool | `true` | Declare `read_file`, `write_file`, `edit_file`. `false` leaves `bash` and `read_skill` only. | |
| 24 | +| `fetchContentTool` | bool | `false` | Declare `fetch_content`, which fetches web pages as Markdown. Off by default: it is the only built-in tool that reaches the network. | | |
| 24 | 25 | | `sampling` | map | `temperature: 0.0`, `top_p: 0.9`, `max_tokens: 4096` | Generation settings, OpenAI API key names, passed to the server. The shipped `agent.yaml` adds `parallel_tool_calls: false`. `max_tokens` also bounds one tool call: a `write_file` of a long document (or a thinking model's reasoning plus the call) can hit it, and the call is then cut off. mm hands it back to the model twice, asking for smaller pieces, then stops with `the model hit max_tokens (N) while writing a tool call`; raise the value if that happens. With streaming, mm also asks the server for token counts (`stream_options.include_usage`). | |
| 25 | 26 | | `watchdogTimeout` | duration | `20s` | Silence tolerated between two streamed tokens before the connection is declared hung. Doubled for summary requests. | |
| 26 | 27 | | `firstTokenTimeout` | duration | `3m` | Silence tolerated before a model call's first token, while the server reads the prompt (every round trip of a turn has one). Never shorter than `watchdogTimeout`; must be >= 0. With a llama.cpp server (`dmr`, `llamacpp`) mm asks for `return_progress`: each progress report restarts this clock, and the spinner shows `Reading the prompt 12.0k/25.3k tokens`. On a timeout the watchdog says `no first token from the server after …` (before the first token) or `connection hang detected after …` (after). | |
| @@ -21,6 +21,7 @@ | |||
| 21 | | `skillsDir` | string | `skills` | Directory holding the skills, as `<name>.md` files or `<name>/SKILL.md` directories. A relative path is resolved from the directory `mm` is started in (terminal mode, or ACP mode without a config file), or next to the config file (ACP mode with one); the banner warns with the absolute path when nothing is found there. | | 21 | | `skillsDir` | string | `skills` | Directory holding the skills, as `<name>.md` files or `<name>/SKILL.md` directories. A relative path is resolved from the directory `mm` is started in (terminal mode, or ACP mode without a config file), or next to the config file (ACP mode with one); the banner warns with the absolute path when nothing is found there. | |
| 22 | | `bashTool` | bool | `true` | Declare `bash`. `false` removes it: the model can still read and edit files (`editTools`) and load skills, but never runs a command. Reword a system prompt that still announces a bash tool. | | 22 | | `bashTool` | bool | `true` | Declare `bash`. `false` removes it: the model can still read and edit files (`editTools`) and load skills, but never runs a command. Reword a system prompt that still announces a bash tool. | |
| 23 | | `editTools` | bool | `true` | Declare `read_file`, `write_file`, `edit_file`. `false` leaves `bash` and `read_skill` only. | | 23 | | `editTools` | bool | `true` | Declare `read_file`, `write_file`, `edit_file`. `false` leaves `bash` and `read_skill` only. | |
| 24 | +| `fetchContentTool` | bool | `false` | Declare `fetch_content`, which fetches web pages as Markdown. Off by default: it is the only built-in tool that reaches the network. | | ||
| 24 | | `sampling` | map | `temperature: 0.0`, `top_p: 0.9`, `max_tokens: 4096` | Generation settings, OpenAI API key names, passed to the server. The shipped `agent.yaml` adds `parallel_tool_calls: false`. `max_tokens` also bounds one tool call: a `write_file` of a long document (or a thinking model's reasoning plus the call) can hit it, and the call is then cut off. mm hands it back to the model twice, asking for smaller pieces, then stops with `the model hit max_tokens (N) while writing a tool call`; raise the value if that happens. With streaming, mm also asks the server for token counts (`stream_options.include_usage`). | | 25 | | `sampling` | map | `temperature: 0.0`, `top_p: 0.9`, `max_tokens: 4096` | Generation settings, OpenAI API key names, passed to the server. The shipped `agent.yaml` adds `parallel_tool_calls: false`. `max_tokens` also bounds one tool call: a `write_file` of a long document (or a thinking model's reasoning plus the call) can hit it, and the call is then cut off. mm hands it back to the model twice, asking for smaller pieces, then stops with `the model hit max_tokens (N) while writing a tool call`; raise the value if that happens. With streaming, mm also asks the server for token counts (`stream_options.include_usage`). | |
| 25 | | `watchdogTimeout` | duration | `20s` | Silence tolerated between two streamed tokens before the connection is declared hung. Doubled for summary requests. | | 26 | | `watchdogTimeout` | duration | `20s` | Silence tolerated between two streamed tokens before the connection is declared hung. Doubled for summary requests. | |
| 26 | | `firstTokenTimeout` | duration | `3m` | Silence tolerated before a model call's first token, while the server reads the prompt (every round trip of a turn has one). Never shorter than `watchdogTimeout`; must be >= 0. With a llama.cpp server (`dmr`, `llamacpp`) mm asks for `return_progress`: each progress report restarts this clock, and the spinner shows `Reading the prompt 12.0k/25.3k tokens`. On a timeout the watchdog says `no first token from the server after …` (before the first token) or `connection hang detected after …` (after). | | 27 | | `firstTokenTimeout` | duration | `3m` | Silence tolerated before a model call's first token, while the server reads the prompt (every round trip of a turn has one). Never shorter than `watchdogTimeout`; must be >= 0. With a llama.cpp server (`dmr`, `llamacpp`) mm asks for `return_progress`: each progress report restarts this clock, and the spinner shows `Reading the prompt 12.0k/25.3k tokens`. On a timeout the watchdog says `no first token from the server after …` (before the first token) or `connection hang detected after …` (after). | |
modified
docs/en/reference/tools.md +39 -0 | @@ -159,6 +159,45 @@ Declared when `lsp.servers` lists at least one server whose command is on the `P | ||
| 159 | 159 | | ACP / window | Kind `read` for navigation, no permission; `rename` is kind `edit` and asks for permission like `edit_file` (`[rename rejected by the user]` when refused). | |
| 160 | 160 | | Counter | `🔎 N lsp call(s)` in the recap line (see below). | |
| 161 | 161 | |
| 162 | +## `fetch_content` | |
| 163 | + | |
| 164 | +Declared only with `fetchContentTool: true`. Downloads a web page and returns it as Markdown, or saves the Markdown to a file. Package `internal/webmd`. | |
| 165 | + | |
| 166 | +| Field | Type | Default | Description | | |
| 167 | +|-------|------|---------|-------------| | |
| 168 | +| `url` | string | — | The page. A missing scheme means `https://` (`k33g.org`); a scheme typed without its colon is repaired (`https//k33g.org`). Only `http` and `https`. | | |
| 169 | +| `output` | string | `""` | File to save the Markdown to, relative to the working directory (the session `cwd` in ACP); parent directories are created. When set, the Markdown is not returned. | | |
| 170 | +| `baseUrl` | string | page URL | Absolute URL against which relative links and image sources are resolved. Default: the page's final URL, after redirections. | | |
| 171 | +| `tables` | string | `html` | Tables with `rowspan`/`colspan` or a nested table: `html` keeps them as raw HTML, `flatten` makes a GFM table and copies a spanned cell into every slot it covers. | | |
| 172 | +| `strikethrough` | bool | `false` | `<del>`, `<s>`, `<strike>` → `~~text~~`; otherwise the plain text. | | |
| 173 | +| `taskList` | bool | `false` | A list item starting with a checkbox → `- [ ] …` / `- [x] …`; otherwise the checkbox is dropped. | | |
| 174 | + | |
| 175 | +| Aspect | Behaviour | | |
| 176 | +|--------|-----------| | |
| 177 | +| Request | `GET` with `User-Agent: mm/<version>`; at most 10 redirections; 30 s for the whole fetch; pages above 5 MiB refused. | | |
| 178 | +| robots.txt | Read (and cached 24 h per origin) before the page and before every redirection target, for the product token `mm`, else the `*` group. `Allow`/`Disallow`, longest match wins, `Allow` wins a tie, `*` and a final `$` supported (RFC 9309). A 4xx robots.txt allows everything; a 5xx or an unreachable one forbids everything. | | |
| 179 | +| Accepted types | `text/html`, `application/xhtml+xml`; no `Content-Type` → sniffed. | | |
| 180 | +| Encoding | In order: byte-order mark, `charset` of the `Content-Type` header, `<meta charset>` or `<meta http-equiv="Content-Type">` in the first 4 KiB, UTF-8 if the bytes are valid UTF-8, else `windows-1252`. An unknown name is skipped. The content is converted to UTF-8. | | |
| 181 | +| Removed | `script`, `style`, `noscript`, `template`, `meta`, `link`, `base`, `title`, `svg`, `canvas`, `iframe`, `object`, `embed`, `nav`, `aside`, `dialog`, `button`, `select`, `textarea`, inputs other than checkboxes, a `footer` outside `article`/`main`, HTML comments, ARIA roles `navigation` `banner` `complementary` `search` `contentinfo` `dialog`, `aria-hidden="true"`, `hidden`, class/id words `ad` `ads` `advert` `sponsor` `promo` `cookie` `consent` `gdpr` `newsletter` `subscribe` `popup` `modal` `share` `social` `breadcrumb` `nav` `navbar` `navigation` `menu` `sidebar` `skiplink` (and a few variants), classes `sr-only` `visually-hidden` `screen-reader-text`. An element matched by class/id is kept when it contains `main`, `article` or `h1`. | | |
| 182 | +| Attributes | All removed except `a[href]`, `img[src,alt]`, `td/th[colspan,rowspan]`, `ol[start]`, `input[type,checked]`, and the `language-*`/`lang-*` words of a `pre`/`code` class. | | |
| 183 | +| Conversion | `h1`–`h6` → `#`–`######`; `p` → paragraph, blank line between blocks; `strong`/`b` → `**…**`; `em`/`i` → `*…*`; inline `code`/`kbd`/`samp`/`tt` → backtick span; `pre` → fenced block (info string from `language-x`), fence longer than any backtick run inside; `br` → hard break `\`; `hr` → `---`; `blockquote` → `> `; `a` → `[text](url)`, the URL when there is no text, text only for `javascript:`; `img` → ``, `` without alt, alt only for a `data:` URI; `ul`/`ol` → `- ` / `N. ` (from `start`), nested lists indented by the parent marker's width; simple tables → GFM, first row as header, `|` escaped, `<br>` for line breaks in a cell, `<caption>` as a paragraph above. | | |
| 184 | +| Unknown element | Tags dropped, text kept; one that contains a block element is treated as a container. | | |
| 185 | +| Empty element | Nothing, no blank line. | | |
| 186 | +| Invalid HTML | Repaired by the HTML5 parser (`golang.org/x/net/html`), then converted. | | |
| 187 | +| Escaping | Backslash, `*`, backtick, `[`, `]`, `<`, and a `_` not inside a word get a backslash; so does a line starting with `#`, `>`, `-`, `+`, `=`, `\|` or `N.`. | | |
| 188 | +| Result | The Markdown, ending with exactly one newline; no title, front matter or preamble. Truncated to `maxOutput` characters. | | |
| 189 | +| With `output` | `[fetch_content: saved N lines of Markdown (B bytes) from <url> to <file> — the content is not shown]`. | | |
| 190 | +| Page with no content left | `[fetch_content: <url> has no text content once scripts, navigation and ads are removed — nothing saved]`; no file written. | | |
| 191 | +| Failure | `[fetch_content: error N (<class>) — <reason>]`, see the codes below. | | |
| 192 | +| Terminal display | `🌐 fetch_content: <url>` (`→ <file>` with `output`), spinner `Fetching`, then the result echoed like a command's output (only the confirmation line with `output`). | | |
| 193 | +| ACP / window | Kind `fetch`. No permission without `output`; with `output`, permission requested like `write_file` (`[fetch_content rejected by the user]` when refused), and the file's absolute path as location. | | |
| 194 | + | |
| 195 | +| Code | Class | Cause | | |
| 196 | +|------|-------|-------| | |
| 197 | +| `1` | `conversion` | Not an HTML page, larger than 5 MiB, undecodable, or the output file cannot be written. | | |
| 198 | +| `2` | `input` | Host unreachable, timeout, HTTP status ≥ 300 after redirections, more than 10 redirections, robots.txt forbids the page or cannot be read. | | |
| 199 | +| `3` | `usage` | Empty or unparsable `url`, a scheme other than `http`/`https`, unknown `tables` value, `baseUrl` not absolute. | | |
| 200 | + | |
| 162 | 201 | ## Loop detection |
| 163 | 202 | |
| 164 | 203 | Every tool records `(tool, input, output)` in the loop detector after each call. |
| @@ -159,6 +159,45 @@ Declared when `lsp.servers` lists at least one server whose command is on the `P | |||
| 159 | | ACP / window | Kind `read` for navigation, no permission; `rename` is kind `edit` and asks for permission like `edit_file` (`[rename rejected by the user]` when refused). | | 159 | | ACP / window | Kind `read` for navigation, no permission; `rename` is kind `edit` and asks for permission like `edit_file` (`[rename rejected by the user]` when refused). | |
| 160 | | Counter | `🔎 N lsp call(s)` in the recap line (see below). | | 160 | | Counter | `🔎 N lsp call(s)` in the recap line (see below). | |
| 161 | 161 | ||
| 162 | +## `fetch_content` | ||
| 163 | + | ||
| 164 | +Declared only with `fetchContentTool: true`. Downloads a web page and returns it as Markdown, or saves the Markdown to a file. Package `internal/webmd`. | ||
| 165 | + | ||
| 166 | +| Field | Type | Default | Description | | ||
| 167 | +|-------|------|---------|-------------| | ||
| 168 | +| `url` | string | — | The page. A missing scheme means `https://` (`k33g.org`); a scheme typed without its colon is repaired (`https//k33g.org`). Only `http` and `https`. | | ||
| 169 | +| `output` | string | `""` | File to save the Markdown to, relative to the working directory (the session `cwd` in ACP); parent directories are created. When set, the Markdown is not returned. | | ||
| 170 | +| `baseUrl` | string | page URL | Absolute URL against which relative links and image sources are resolved. Default: the page's final URL, after redirections. | | ||
| 171 | +| `tables` | string | `html` | Tables with `rowspan`/`colspan` or a nested table: `html` keeps them as raw HTML, `flatten` makes a GFM table and copies a spanned cell into every slot it covers. | | ||
| 172 | +| `strikethrough` | bool | `false` | `<del>`, `<s>`, `<strike>` → `~~text~~`; otherwise the plain text. | | ||
| 173 | +| `taskList` | bool | `false` | A list item starting with a checkbox → `- [ ] …` / `- [x] …`; otherwise the checkbox is dropped. | | ||
| 174 | + | ||
| 175 | +| Aspect | Behaviour | | ||
| 176 | +|--------|-----------| | ||
| 177 | +| Request | `GET` with `User-Agent: mm/<version>`; at most 10 redirections; 30 s for the whole fetch; pages above 5 MiB refused. | | ||
| 178 | +| robots.txt | Read (and cached 24 h per origin) before the page and before every redirection target, for the product token `mm`, else the `*` group. `Allow`/`Disallow`, longest match wins, `Allow` wins a tie, `*` and a final `$` supported (RFC 9309). A 4xx robots.txt allows everything; a 5xx or an unreachable one forbids everything. | | ||
| 179 | +| Accepted types | `text/html`, `application/xhtml+xml`; no `Content-Type` → sniffed. | | ||
| 180 | +| Encoding | In order: byte-order mark, `charset` of the `Content-Type` header, `<meta charset>` or `<meta http-equiv="Content-Type">` in the first 4 KiB, UTF-8 if the bytes are valid UTF-8, else `windows-1252`. An unknown name is skipped. The content is converted to UTF-8. | | ||
| 181 | +| Removed | `script`, `style`, `noscript`, `template`, `meta`, `link`, `base`, `title`, `svg`, `canvas`, `iframe`, `object`, `embed`, `nav`, `aside`, `dialog`, `button`, `select`, `textarea`, inputs other than checkboxes, a `footer` outside `article`/`main`, HTML comments, ARIA roles `navigation` `banner` `complementary` `search` `contentinfo` `dialog`, `aria-hidden="true"`, `hidden`, class/id words `ad` `ads` `advert` `sponsor` `promo` `cookie` `consent` `gdpr` `newsletter` `subscribe` `popup` `modal` `share` `social` `breadcrumb` `nav` `navbar` `navigation` `menu` `sidebar` `skiplink` (and a few variants), classes `sr-only` `visually-hidden` `screen-reader-text`. An element matched by class/id is kept when it contains `main`, `article` or `h1`. | | ||
| 182 | +| Attributes | All removed except `a[href]`, `img[src,alt]`, `td/th[colspan,rowspan]`, `ol[start]`, `input[type,checked]`, and the `language-*`/`lang-*` words of a `pre`/`code` class. | | ||
| 183 | +| Conversion | `h1`–`h6` → `#`–`######`; `p` → paragraph, blank line between blocks; `strong`/`b` → `**…**`; `em`/`i` → `*…*`; inline `code`/`kbd`/`samp`/`tt` → backtick span; `pre` → fenced block (info string from `language-x`), fence longer than any backtick run inside; `br` → hard break `\`; `hr` → `---`; `blockquote` → `> `; `a` → `[text](url)`, the URL when there is no text, text only for `javascript:`; `img` → ``, `` without alt, alt only for a `data:` URI; `ul`/`ol` → `- ` / `N. ` (from `start`), nested lists indented by the parent marker's width; simple tables → GFM, first row as header, `|` escaped, `<br>` for line breaks in a cell, `<caption>` as a paragraph above. | | ||
| 184 | +| Unknown element | Tags dropped, text kept; one that contains a block element is treated as a container. | | ||
| 185 | +| Empty element | Nothing, no blank line. | | ||
| 186 | +| Invalid HTML | Repaired by the HTML5 parser (`golang.org/x/net/html`), then converted. | | ||
| 187 | +| Escaping | Backslash, `*`, backtick, `[`, `]`, `<`, and a `_` not inside a word get a backslash; so does a line starting with `#`, `>`, `-`, `+`, `=`, `\|` or `N.`. | | ||
| 188 | +| Result | The Markdown, ending with exactly one newline; no title, front matter or preamble. Truncated to `maxOutput` characters. | | ||
| 189 | +| With `output` | `[fetch_content: saved N lines of Markdown (B bytes) from <url> to <file> — the content is not shown]`. | | ||
| 190 | +| Page with no content left | `[fetch_content: <url> has no text content once scripts, navigation and ads are removed — nothing saved]`; no file written. | | ||
| 191 | +| Failure | `[fetch_content: error N (<class>) — <reason>]`, see the codes below. | | ||
| 192 | +| Terminal display | `🌐 fetch_content: <url>` (`→ <file>` with `output`), spinner `Fetching`, then the result echoed like a command's output (only the confirmation line with `output`). | | ||
| 193 | +| ACP / window | Kind `fetch`. No permission without `output`; with `output`, permission requested like `write_file` (`[fetch_content rejected by the user]` when refused), and the file's absolute path as location. | | ||
| 194 | + | ||
| 195 | +| Code | Class | Cause | | ||
| 196 | +|------|-------|-------| | ||
| 197 | +| `1` | `conversion` | Not an HTML page, larger than 5 MiB, undecodable, or the output file cannot be written. | | ||
| 198 | +| `2` | `input` | Host unreachable, timeout, HTTP status ≥ 300 after redirections, more than 10 redirections, robots.txt forbids the page or cannot be read. | | ||
| 199 | +| `3` | `usage` | Empty or unparsable `url`, a scheme other than `http`/`https`, unknown `tables` value, `baseUrl` not absolute. | | ||
| 200 | + | ||
| 162 | ## Loop detection | 201 | ## Loop detection |
| 163 | 202 | ||
| 164 | Every tool records `(tool, input, output)` in the loop detector after each call. | 203 | Every tool records `(tool, input, output)` in the loop detector after each call. |
modified
docs/fr/README.md +4 -2 | @@ -1,6 +1,6 @@ | ||
| 1 | 1 | # mini-me — documentation (français) |
| 2 | 2 | |
| 3 | -`mm` est un agent de code qui tourne dans votre terminal — ligne à ligne, ou dans une fenêtre unique avec `mm -tui` — ou dans un éditeur via l'Agent Client Protocol. Il dialogue avec un serveur de modèle local (Docker Model Runner ou `llama-server` de llama.cpp) et travaille avec une poignée d'outils intégrés : `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`. | |
| 3 | +`mm` est un agent de code qui tourne dans votre terminal — ligne à ligne, ou dans une fenêtre unique avec `mm -tui` — ou dans un éditeur via l'Agent Client Protocol. Il dialogue avec un serveur de modèle local (Docker Model Runner ou `llama-server` de llama.cpp) et travaille avec une poignée d'outils intégrés : `bash`, `read_skill`, `read_file`, `write_file`, `edit_file` — et, quand ils sont activés, `lsp` et `fetch_content`. | |
| 4 | 4 | |
| 5 | 5 | Cette documentation est organisée selon les quatre quadrants [Diátaxis](https://diataxis.fr). Choisissez celui qui correspond à votre besoin du moment. |
| 6 | 6 | |
| @@ -23,6 +23,7 @@ Cette documentation est organisée selon les quatre quadrants [Diátaxis](https: | ||
| 23 | 23 | - [Utiliser le mode fenêtre (`mm -tui`)](how-to/use-the-window-mode.md) |
| 24 | 24 | - [Naviguer et refactorer le code avec un serveur de langage (`lsp`)](how-to/navigate-code-with-lsp.md) |
| 25 | 25 | - [Exécuter une invite unique et quitter (`-p`/`--prompt`)](how-to/executer-une-invite-unique.md) |
| 26 | +- [Récupérer une page web en Markdown (`fetch_content`)](how-to/fetch-a-web-page.md) — lire une page, ou l'enregistrer dans un fichier sans l'afficher | |
| 26 | 27 | |
| 27 | 28 | ### Guides pratiques MCP (Model Context Protocol) |
| 28 | 29 | - [Utiliser Context7 pour la documentation](how-to/mcp-utiliser-context7.md) |
| @@ -34,7 +35,7 @@ Cette documentation est organisée selon les quatre quadrants [Diátaxis](https: | ||
| 34 | 35 | - [Ligne de commande](reference/cli.md) — options, variables d'environnement, bannière de démarrage |
| 35 | 36 | - [Commandes slash](reference/slash-commands.md) — `/quit`, `/abort`, `/compact`, `/new`, `/sessions`, `/resume`, `/help`, raccourcis clavier, touches de la fenêtre |
| 36 | 37 | - [Fichier de configuration](reference/configuration.md) — chaque clé YAML, sa valeur par défaut et sa règle de validation |
| 37 | -- [Outils intégrés](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file` | |
| 38 | +- [Outils intégrés](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`, `lsp`, `fetch_content` | |
| 38 | 39 | - [Format d'un fichier de skill](reference/skill-format.md) |
| 39 | 40 | - [Surface Agent Client Protocol](reference/acp.md) |
| 40 | 41 | |
| @@ -50,6 +51,7 @@ Cette documentation est organisée selon les quatre quadrants [Diátaxis](https: | ||
| 50 | 51 | - [Compression du contexte](explanation/context-compression.md) |
| 51 | 52 | - [Les sessions persistantes](explanation/persistent-sessions.md) — des fichiers par projet, pourquoi un lancement est une nouvelle session, `session/load` face au `session/resume` de docker agent |
| 52 | 53 | - [L'outil `lsp`](explanation/lsp-tool.md) — un seul outil avec un champ `action`, un champ `symbol` plutôt que des colonnes, des fichiers synchronisés depuis le disque, un renommage soumis à permission |
| 54 | +- [L'outil `fetch_content`](explanation/fetch-content-tool.md) — pourquoi un outil intégré à côté du `fetch` MCP, un convertisseur maison, le robots.txt, des erreurs numérotées en texte, les limites connues | |
| 53 | 55 | - [Modèles à raisonnement](explanation/thinking-models.md) — d'où vient le raisonnement d'un modèle, pourquoi `mm` le lit sous Genkit, et pourquoi le lire garde le chien de garde en vie |
| 54 | 56 | - Diagramme des paquets (source draw.io) : [`docs/diagrams/packages.drawio`](../diagrams/packages.drawio) |
| 55 | 57 | |
| @@ -1,6 +1,6 @@ | |||
| 1 | # mini-me — documentation (français) | 1 | # mini-me — documentation (français) |
| 2 | 2 | ||
| 3 | -`mm` est un agent de code qui tourne dans votre terminal — ligne à ligne, ou dans une fenêtre unique avec `mm -tui` — ou dans un éditeur via l'Agent Client Protocol. Il dialogue avec un serveur de modèle local (Docker Model Runner ou `llama-server` de llama.cpp) et travaille avec une poignée d'outils intégrés : `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`. | 3 | +`mm` est un agent de code qui tourne dans votre terminal — ligne à ligne, ou dans une fenêtre unique avec `mm -tui` — ou dans un éditeur via l'Agent Client Protocol. Il dialogue avec un serveur de modèle local (Docker Model Runner ou `llama-server` de llama.cpp) et travaille avec une poignée d'outils intégrés : `bash`, `read_skill`, `read_file`, `write_file`, `edit_file` — et, quand ils sont activés, `lsp` et `fetch_content`. |
| 4 | 4 | ||
| 5 | Cette documentation est organisée selon les quatre quadrants [Diátaxis](https://diataxis.fr). Choisissez celui qui correspond à votre besoin du moment. | 5 | Cette documentation est organisée selon les quatre quadrants [Diátaxis](https://diataxis.fr). Choisissez celui qui correspond à votre besoin du moment. |
| 6 | 6 | ||
| @@ -23,6 +23,7 @@ Cette documentation est organisée selon les quatre quadrants [Diátaxis](https: | |||
| 23 | - [Utiliser le mode fenêtre (`mm -tui`)](how-to/use-the-window-mode.md) | 23 | - [Utiliser le mode fenêtre (`mm -tui`)](how-to/use-the-window-mode.md) |
| 24 | - [Naviguer et refactorer le code avec un serveur de langage (`lsp`)](how-to/navigate-code-with-lsp.md) | 24 | - [Naviguer et refactorer le code avec un serveur de langage (`lsp`)](how-to/navigate-code-with-lsp.md) |
| 25 | - [Exécuter une invite unique et quitter (`-p`/`--prompt`)](how-to/executer-une-invite-unique.md) | 25 | - [Exécuter une invite unique et quitter (`-p`/`--prompt`)](how-to/executer-une-invite-unique.md) |
| 26 | +- [Récupérer une page web en Markdown (`fetch_content`)](how-to/fetch-a-web-page.md) — lire une page, ou l'enregistrer dans un fichier sans l'afficher | ||
| 26 | 27 | ||
| 27 | ### Guides pratiques MCP (Model Context Protocol) | 28 | ### Guides pratiques MCP (Model Context Protocol) |
| 28 | - [Utiliser Context7 pour la documentation](how-to/mcp-utiliser-context7.md) | 29 | - [Utiliser Context7 pour la documentation](how-to/mcp-utiliser-context7.md) |
| @@ -34,7 +35,7 @@ Cette documentation est organisée selon les quatre quadrants [Diátaxis](https: | |||
| 34 | - [Ligne de commande](reference/cli.md) — options, variables d'environnement, bannière de démarrage | 35 | - [Ligne de commande](reference/cli.md) — options, variables d'environnement, bannière de démarrage |
| 35 | - [Commandes slash](reference/slash-commands.md) — `/quit`, `/abort`, `/compact`, `/new`, `/sessions`, `/resume`, `/help`, raccourcis clavier, touches de la fenêtre | 36 | - [Commandes slash](reference/slash-commands.md) — `/quit`, `/abort`, `/compact`, `/new`, `/sessions`, `/resume`, `/help`, raccourcis clavier, touches de la fenêtre |
| 36 | - [Fichier de configuration](reference/configuration.md) — chaque clé YAML, sa valeur par défaut et sa règle de validation | 37 | - [Fichier de configuration](reference/configuration.md) — chaque clé YAML, sa valeur par défaut et sa règle de validation |
| 37 | -- [Outils intégrés](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file` | 38 | +- [Outils intégrés](reference/tools.md) — `bash`, `read_skill`, `read_file`, `write_file`, `edit_file`, `lsp`, `fetch_content` |
| 38 | - [Format d'un fichier de skill](reference/skill-format.md) | 39 | - [Format d'un fichier de skill](reference/skill-format.md) |
| 39 | - [Surface Agent Client Protocol](reference/acp.md) | 40 | - [Surface Agent Client Protocol](reference/acp.md) |
| 40 | 41 | ||
| @@ -50,6 +51,7 @@ Cette documentation est organisée selon les quatre quadrants [Diátaxis](https: | |||
| 50 | - [Compression du contexte](explanation/context-compression.md) | 51 | - [Compression du contexte](explanation/context-compression.md) |
| 51 | - [Les sessions persistantes](explanation/persistent-sessions.md) — des fichiers par projet, pourquoi un lancement est une nouvelle session, `session/load` face au `session/resume` de docker agent | 52 | - [Les sessions persistantes](explanation/persistent-sessions.md) — des fichiers par projet, pourquoi un lancement est une nouvelle session, `session/load` face au `session/resume` de docker agent |
| 52 | - [L'outil `lsp`](explanation/lsp-tool.md) — un seul outil avec un champ `action`, un champ `symbol` plutôt que des colonnes, des fichiers synchronisés depuis le disque, un renommage soumis à permission | 53 | - [L'outil `lsp`](explanation/lsp-tool.md) — un seul outil avec un champ `action`, un champ `symbol` plutôt que des colonnes, des fichiers synchronisés depuis le disque, un renommage soumis à permission |
| 54 | +- [L'outil `fetch_content`](explanation/fetch-content-tool.md) — pourquoi un outil intégré à côté du `fetch` MCP, un convertisseur maison, le robots.txt, des erreurs numérotées en texte, les limites connues | ||
| 53 | - [Modèles à raisonnement](explanation/thinking-models.md) — d'où vient le raisonnement d'un modèle, pourquoi `mm` le lit sous Genkit, et pourquoi le lire garde le chien de garde en vie | 55 | - [Modèles à raisonnement](explanation/thinking-models.md) — d'où vient le raisonnement d'un modèle, pourquoi `mm` le lit sous Genkit, et pourquoi le lire garde le chien de garde en vie |
| 54 | - Diagramme des paquets (source draw.io) : [`docs/diagrams/packages.drawio`](../diagrams/packages.drawio) | 56 | - Diagramme des paquets (source draw.io) : [`docs/diagrams/packages.drawio`](../diagrams/packages.drawio) |
| 55 | 57 | ||
added
docs/fr/explanation/fetch-content-tool.md +49 -0 | new file mode 100644 | ||
| @@ -0,0 +1,49 @@ | ||
| 1 | +# L'outil `fetch_content` — explication | |
| 2 | + | |
| 3 | +## De quoi s'agit-il ? | |
| 4 | + | |
| 5 | +`fetch_content` donne au modèle une capacité qu'il n'exerçait jusqu'ici que maladroitement : lire une page web. Avec `bash` seul, un modèle lance `curl`, reçoit des centaines de kilo-octets de HTML (scripts, styles, menus, code de suivi) et soit y noie sa fenêtre de contexte, soit résume une page qu'il n'a jamais vraiment vue. `fetch_content` télécharge la page, jette ce qui n'est pas du contenu et rend du Markdown : le format qu'un modèle de langage lit le mieux, et celui qu'un utilisateur peut conserver. | |
| 6 | + | |
| 7 | +Le travail est coupé en deux. `internal/webmd` est un paquet Go ordinaire en trois étapes — `Fetcher.Fetch`, `Clean`, `Convert` — qui ne sait rien des modèles. `internal/tools/fetchcontent.go` est la fine couche d'outil autour : arguments, affichage, permission, et le texte que reçoit le modèle. | |
| 8 | + | |
| 9 | +## Pourquoi un outil intégré, et pas le serveur MCP `fetch` | |
| 10 | + | |
| 11 | +On peut déjà brancher un serveur MCP `fetch` sur `mm`. L'outil intégré existe pour ce qu'un tel serveur ne garantit pas : | |
| 12 | + | |
| 13 | +- **Il fonctionne sans rien installer d'autre.** Ni Node, ni Python, ni passerelle : la conversion est dans le binaire. | |
| 14 | +- **Il sait enregistrer sans renvoyer.** « Sauve-le dans hello.txt, ne l'affiche pas » est une demande courante, et un outil qui renvoie toujours la page fait payer au modèle toute la page dans son contexte rien que pour la recopier dans un fichier — ce qu'un petit modèle local fait mal de toute façon. `output` écrit le fichier directement et ne renvoie qu'une ligne. | |
| 15 | +- **Ses règles de conversion sont les nôtres.** La spécification qu'il suit (voir la référence) fixe titres, listes, tableaux, échappement ; les tests les verrouillent. | |
| 16 | + | |
| 17 | +Il s'appelle `fetch_content`, pas `fetch`, pour ne jamais entrer en collision avec un outil MCP de ce nom. | |
| 18 | + | |
| 19 | +## Pourquoi la conversion est écrite ici | |
| 20 | + | |
| 21 | +Une bibliothèque comme html-to-markdown a été envisagée. La spécification demande des comportements que ces bibliothèques n'ont pas ou rendent malaisés : tableaux complexes gardés en HTML brut *ou* aplatis à la demande, un nettoyage guidé par les rôles et les mots de classe avec une protection pour les conteneurs de mise en page, des liens résolus contre l'URL finale, une sortie stricte « un seul retour à la ligne final, pas de préambule ». Plier un convertisseur généraliste à tout cela aurait coûté à peu près autant que d'écrire ces quelques centaines de lignes, en ajoutant une dépendance dont les comportements par défaut peuvent changer sous nos pieds. L'analyse elle-même n'est pas la nôtre : `golang.org/x/net/html` est le parseur HTML5, donc un HTML cassé est réparé exactement comme un navigateur le répare. | |
| 22 | + | |
| 23 | +## Pourquoi le robots.txt est respecté | |
| 24 | + | |
| 25 | +`mm` récupère pour le compte d'une personne, une page à la fois — plus proche d'un navigateur que d'un robot d'indexation. Il respecte pourtant le `robots.txt`, parce que la demande l'exigeait et parce qu'un agent qui ignore les souhaits déclarés d'un site se conduit mal, si peu de requêtes qu'il envoie. Il s'annonce comme `mm/<version>`, pour qu'un site puisse le désigner par son nom. | |
| 26 | + | |
| 27 | +Les règles suivent la RFC 9309, y compris ses recoins prudents : un site dont le `robots.txt` répond par une erreur serveur, ou est injoignable, est considéré comme fermé — le choix de la RFC, et la seule lecture sûre de « nous n'avons pas pu demander ». Chaque redirection est vérifiée aussi, pour qu'une redirection ne mène pas vers un chemin interdit. | |
| 28 | + | |
| 29 | +## Pourquoi un échec est un texte avec un numéro | |
| 30 | + | |
| 31 | +Tous les outils de `mm` renvoient leurs échecs en texte, pour que le modèle les lise et réagisse. `fetch_content` y ajoute le code d'erreur de sa spécification (1 conversion, 2 entrée, 3 usage), parce que les trois appellent des réactions différentes : corriger l'appel (3), abandonner ou essayer une autre page (2), ou dire à l'utilisateur que ce n'est pas une page web (1). Un modèle qui lit `error 2 (input) — … robots.txt does not allow "mm"` n'a aucune raison d'essayer `curl` ensuite. | |
| 32 | + | |
| 33 | +## Des choix qui échangent de l'exactitude contre de la sûreté | |
| 34 | + | |
| 35 | +- **Désactivé par défaut.** C'est le seul outil intégré qui accède au réseau ; un opérateur l'active en connaissance de cause. | |
| 36 | +- **Limites.** 30 secondes, 5 Mio, 10 redirections : un appel d'outil ne doit ni bloquer un tour ni remplir la mémoire. | |
| 37 | +- **Encodage.** L'encodage déclaré l'emporte (BOM, puis en-tête HTTP, puis `<meta>`) ; quand rien n'est déclaré, de l'UTF-8 valide est lu comme de l'UTF-8 et tout le reste comme du windows-1252, le défaut historique du web. | |
| 38 | +- **Les images en URI `data:`** sont réduites à leur texte alternatif : des kilo-octets de base64 ne servent à aucun lecteur, humain ou modèle. | |
| 39 | + | |
| 40 | +## Limites connues | |
| 41 | + | |
| 42 | +- Le nettoyage est heuristique. Il retire la navigation, les publicités et les bannières signalées comme telles ; une publicité sans classe ni rôle révélateur reste, et un nom de classe inhabituel peut, rarement, retirer du contenu. Tout ce qui enveloppe un `main`, un `article` ou un `h1` est protégé. | |
| 43 | +- Les éléments en ligne qu'une page ne sépare que par du CSS ressortent collés : des étiquettes rendues par `<a>go</a><a>tui</a>` donnent `gotui`. | |
| 44 | +- Les pages construites par JavaScript dans le navigateur ont peu de contenu dans leur HTML ; `fetch_content` n'exécute pas de script. | |
| 45 | +- Seul le HTML est converti : un PDF ou un document JSON est refusé avec `error 1`. | |
| 46 | + | |
| 47 | +## Comment il s'insère dans le reste | |
| 48 | + | |
| 49 | +`fetch_content` suit le même contrat que les autres outils (voir [l'architecture](architecture.md)) : une ligne d'affichage dans le terminal, un événement avec un genre (`fetch`) pour ACP et la fenêtre, une demande de permission seulement quand il écrit un fichier, et un enregistrement dans le détecteur de boucle. Sa sortie passe par la même troncature `maxOutput` que `bash`. | |
| new file mode 100644 | |||
| @@ -0,0 +1,49 @@ | |||
| 1 | +# L'outil `fetch_content` — explication | ||
| 2 | + | ||
| 3 | +## De quoi s'agit-il ? | ||
| 4 | + | ||
| 5 | +`fetch_content` donne au modèle une capacité qu'il n'exerçait jusqu'ici que maladroitement : lire une page web. Avec `bash` seul, un modèle lance `curl`, reçoit des centaines de kilo-octets de HTML (scripts, styles, menus, code de suivi) et soit y noie sa fenêtre de contexte, soit résume une page qu'il n'a jamais vraiment vue. `fetch_content` télécharge la page, jette ce qui n'est pas du contenu et rend du Markdown : le format qu'un modèle de langage lit le mieux, et celui qu'un utilisateur peut conserver. | ||
| 6 | + | ||
| 7 | +Le travail est coupé en deux. `internal/webmd` est un paquet Go ordinaire en trois étapes — `Fetcher.Fetch`, `Clean`, `Convert` — qui ne sait rien des modèles. `internal/tools/fetchcontent.go` est la fine couche d'outil autour : arguments, affichage, permission, et le texte que reçoit le modèle. | ||
| 8 | + | ||
| 9 | +## Pourquoi un outil intégré, et pas le serveur MCP `fetch` | ||
| 10 | + | ||
| 11 | +On peut déjà brancher un serveur MCP `fetch` sur `mm`. L'outil intégré existe pour ce qu'un tel serveur ne garantit pas : | ||
| 12 | + | ||
| 13 | +- **Il fonctionne sans rien installer d'autre.** Ni Node, ni Python, ni passerelle : la conversion est dans le binaire. | ||
| 14 | +- **Il sait enregistrer sans renvoyer.** « Sauve-le dans hello.txt, ne l'affiche pas » est une demande courante, et un outil qui renvoie toujours la page fait payer au modèle toute la page dans son contexte rien que pour la recopier dans un fichier — ce qu'un petit modèle local fait mal de toute façon. `output` écrit le fichier directement et ne renvoie qu'une ligne. | ||
| 15 | +- **Ses règles de conversion sont les nôtres.** La spécification qu'il suit (voir la référence) fixe titres, listes, tableaux, échappement ; les tests les verrouillent. | ||
| 16 | + | ||
| 17 | +Il s'appelle `fetch_content`, pas `fetch`, pour ne jamais entrer en collision avec un outil MCP de ce nom. | ||
| 18 | + | ||
| 19 | +## Pourquoi la conversion est écrite ici | ||
| 20 | + | ||
| 21 | +Une bibliothèque comme html-to-markdown a été envisagée. La spécification demande des comportements que ces bibliothèques n'ont pas ou rendent malaisés : tableaux complexes gardés en HTML brut *ou* aplatis à la demande, un nettoyage guidé par les rôles et les mots de classe avec une protection pour les conteneurs de mise en page, des liens résolus contre l'URL finale, une sortie stricte « un seul retour à la ligne final, pas de préambule ». Plier un convertisseur généraliste à tout cela aurait coûté à peu près autant que d'écrire ces quelques centaines de lignes, en ajoutant une dépendance dont les comportements par défaut peuvent changer sous nos pieds. L'analyse elle-même n'est pas la nôtre : `golang.org/x/net/html` est le parseur HTML5, donc un HTML cassé est réparé exactement comme un navigateur le répare. | ||
| 22 | + | ||
| 23 | +## Pourquoi le robots.txt est respecté | ||
| 24 | + | ||
| 25 | +`mm` récupère pour le compte d'une personne, une page à la fois — plus proche d'un navigateur que d'un robot d'indexation. Il respecte pourtant le `robots.txt`, parce que la demande l'exigeait et parce qu'un agent qui ignore les souhaits déclarés d'un site se conduit mal, si peu de requêtes qu'il envoie. Il s'annonce comme `mm/<version>`, pour qu'un site puisse le désigner par son nom. | ||
| 26 | + | ||
| 27 | +Les règles suivent la RFC 9309, y compris ses recoins prudents : un site dont le `robots.txt` répond par une erreur serveur, ou est injoignable, est considéré comme fermé — le choix de la RFC, et la seule lecture sûre de « nous n'avons pas pu demander ». Chaque redirection est vérifiée aussi, pour qu'une redirection ne mène pas vers un chemin interdit. | ||
| 28 | + | ||
| 29 | +## Pourquoi un échec est un texte avec un numéro | ||
| 30 | + | ||
| 31 | +Tous les outils de `mm` renvoient leurs échecs en texte, pour que le modèle les lise et réagisse. `fetch_content` y ajoute le code d'erreur de sa spécification (1 conversion, 2 entrée, 3 usage), parce que les trois appellent des réactions différentes : corriger l'appel (3), abandonner ou essayer une autre page (2), ou dire à l'utilisateur que ce n'est pas une page web (1). Un modèle qui lit `error 2 (input) — … robots.txt does not allow "mm"` n'a aucune raison d'essayer `curl` ensuite. | ||
| 32 | + | ||
| 33 | +## Des choix qui échangent de l'exactitude contre de la sûreté | ||
| 34 | + | ||
| 35 | +- **Désactivé par défaut.** C'est le seul outil intégré qui accède au réseau ; un opérateur l'active en connaissance de cause. | ||
| 36 | +- **Limites.** 30 secondes, 5 Mio, 10 redirections : un appel d'outil ne doit ni bloquer un tour ni remplir la mémoire. | ||
| 37 | +- **Encodage.** L'encodage déclaré l'emporte (BOM, puis en-tête HTTP, puis `<meta>`) ; quand rien n'est déclaré, de l'UTF-8 valide est lu comme de l'UTF-8 et tout le reste comme du windows-1252, le défaut historique du web. | ||
| 38 | +- **Les images en URI `data:`** sont réduites à leur texte alternatif : des kilo-octets de base64 ne servent à aucun lecteur, humain ou modèle. | ||
| 39 | + | ||
| 40 | +## Limites connues | ||
| 41 | + | ||
| 42 | +- Le nettoyage est heuristique. Il retire la navigation, les publicités et les bannières signalées comme telles ; une publicité sans classe ni rôle révélateur reste, et un nom de classe inhabituel peut, rarement, retirer du contenu. Tout ce qui enveloppe un `main`, un `article` ou un `h1` est protégé. | ||
| 43 | +- Les éléments en ligne qu'une page ne sépare que par du CSS ressortent collés : des étiquettes rendues par `<a>go</a><a>tui</a>` donnent `gotui`. | ||
| 44 | +- Les pages construites par JavaScript dans le navigateur ont peu de contenu dans leur HTML ; `fetch_content` n'exécute pas de script. | ||
| 45 | +- Seul le HTML est converti : un PDF ou un document JSON est refusé avec `error 1`. | ||
| 46 | + | ||
| 47 | +## Comment il s'insère dans le reste | ||
| 48 | + | ||
| 49 | +`fetch_content` suit le même contrat que les autres outils (voir [l'architecture](architecture.md)) : une ligne d'affichage dans le terminal, un événement avec un genre (`fetch`) pour ACP et la fenêtre, une demande de permission seulement quand il écrit un fichier, et un enregistrement dans le détecteur de boucle. Sa sortie passe par la même troncature `maxOutput` que `bash`. | ||
added
docs/fr/how-to/fetch-a-web-page.md +65 -0 | new file mode 100644 | ||
| @@ -0,0 +1,65 @@ | ||
| 1 | +# Comment récupérer une page web en Markdown | |
| 2 | + | |
| 3 | +Ce guide montre comment faire lire une page web au modèle, ou la faire enregistrer dans un fichier, avec l'outil intégré `fetch_content`. Il suppose que vous lancez déjà `mm` avec un fichier de configuration. | |
| 4 | + | |
| 5 | +## Étapes | |
| 6 | + | |
| 7 | +1. Activez l'outil dans votre fichier de configuration (il est désactivé par défaut) : | |
| 8 | + | |
| 9 | + ```yaml | |
| 10 | + fetchContentTool: true | |
| 11 | + ``` | |
| 12 | + | |
| 13 | +2. Lancez `mm` et vérifiez que la bannière de démarrage cite `fetch_content` parmi les outils. | |
| 14 | + | |
| 15 | +3. Demandez la page avec vos mots, dans la langue de votre choix : | |
| 16 | + | |
| 17 | + ```text | |
| 18 | + récupère le contenu de https://k33g.org | |
| 19 | + ``` | |
| 20 | + | |
| 21 | + ```text | |
| 22 | + fetch the content of https://k33g.org | |
| 23 | + ``` | |
| 24 | + | |
| 25 | + Le terminal affiche `🌐 fetch_content: https://k33g.org`, puis les premières lignes du Markdown, et le modèle répond à partir de ce contenu. | |
| 26 | + | |
| 27 | +## Enregistrer la page dans un fichier sans l'afficher | |
| 28 | + | |
| 29 | +Nommez le fichier et précisez de ne pas l'afficher : | |
| 30 | + | |
| 31 | +```text | |
| 32 | +récupère le contenu de https://hello.com mais ne l'affiche pas et sauve le dans hello.txt | |
| 33 | +``` | |
| 34 | + | |
| 35 | +Le modèle passe `output: hello.txt`. Le Markdown est écrit dans `hello.txt` (relatif au répertoire où tourne `mm`, ou au projet ouvert dans l'éditeur) et l'outil ne renvoie qu'une ligne : | |
| 36 | + | |
| 37 | +```text | |
| 38 | +[fetch_content: saved 9 lines of Markdown (421 bytes) from https://hello.com to hello.txt — the content is not shown] | |
| 39 | +``` | |
| 40 | + | |
| 41 | +La page n'entre pas dans la conversation : une page longue n'épuise donc pas la fenêtre de contexte. | |
| 42 | + | |
| 43 | +## Variantes | |
| 44 | + | |
| 45 | +- **Une URL tapée à la va-vite.** `k33g.org` et `https//k33g.org` fonctionnent : un schéma manquant devient `https://`, les deux-points oubliés sont remis. | |
| 46 | +- **Tableaux complexes.** Les tableaux à cellules fusionnées sont gardés en HTML brut par défaut. Demandez des tableaux aplatis (« …en tableaux Markdown, aplatis-les ») et le modèle passe `tables: flatten`. | |
| 47 | +- **Texte barré et listes de tâches.** Demandez-les explicitement (« garde le texte barré », « garde les cases à cocher ») et le modèle passe `strikethrough: true` ou `taskList: true`. | |
| 48 | +- **Liens relatifs à un autre site.** Si la page est un miroir, demandez de résoudre les liens par rapport à l'original ; le modèle passe `baseUrl`. | |
| 49 | +- **Dans un éditeur (ACP) ou avec `permissions: ask`.** Lire une page ne demande aucune permission ; l'enregistrer dans un fichier la demande d'abord, comme `write_file`. | |
| 50 | + | |
| 51 | +## En cas d'échec | |
| 52 | + | |
| 53 | +Le modèle reçoit `[fetch_content: error N (…) — raison]` et vous explique en général pourquoi : | |
| 54 | + | |
| 55 | +- `error 2 (input)` : le site est injoignable, a renvoyé une erreur HTTP, ou son `robots.txt` interdit aux robots de lire cette page. `mm` ne contourne pas le `robots.txt`. | |
| 56 | +- `error 1 (conversion)` : l'URL n'est pas une page HTML (un PDF, du JSON, une image) ou dépasse 5 Mio. | |
| 57 | +- `error 3 (usage)` : l'appel lui-même est faux, par exemple une URL `ftp://`. | |
| 58 | + | |
| 59 | +Si le modèle utilise `curl` via `bash` à la place, l'outil n'est probablement pas déclaré : vérifiez `fetchContentTool: true` et la bannière. | |
| 60 | + | |
| 61 | +## Voir aussi | |
| 62 | + | |
| 63 | +- Chaque champ, règle et code d'erreur : [référence des outils intégrés](../reference/tools.md#fetch_content) | |
| 64 | +- La clé de configuration : [référence de la configuration](../reference/configuration.md) | |
| 65 | +- Pourquoi l'outil fonctionne ainsi : [l'outil `fetch_content`](../explanation/fetch-content-tool.md) | |
| new file mode 100644 | |||
| @@ -0,0 +1,65 @@ | |||
| 1 | +# Comment récupérer une page web en Markdown | ||
| 2 | + | ||
| 3 | +Ce guide montre comment faire lire une page web au modèle, ou la faire enregistrer dans un fichier, avec l'outil intégré `fetch_content`. Il suppose que vous lancez déjà `mm` avec un fichier de configuration. | ||
| 4 | + | ||
| 5 | +## Étapes | ||
| 6 | + | ||
| 7 | +1. Activez l'outil dans votre fichier de configuration (il est désactivé par défaut) : | ||
| 8 | + | ||
| 9 | + ```yaml | ||
| 10 | + fetchContentTool: true | ||
| 11 | + ``` | ||
| 12 | + | ||
| 13 | +2. Lancez `mm` et vérifiez que la bannière de démarrage cite `fetch_content` parmi les outils. | ||
| 14 | + | ||
| 15 | +3. Demandez la page avec vos mots, dans la langue de votre choix : | ||
| 16 | + | ||
| 17 | + ```text | ||
| 18 | + récupère le contenu de https://k33g.org | ||
| 19 | + ``` | ||
| 20 | + | ||
| 21 | + ```text | ||
| 22 | + fetch the content of https://k33g.org | ||
| 23 | + ``` | ||
| 24 | + | ||
| 25 | + Le terminal affiche `🌐 fetch_content: https://k33g.org`, puis les premières lignes du Markdown, et le modèle répond à partir de ce contenu. | ||
| 26 | + | ||
| 27 | +## Enregistrer la page dans un fichier sans l'afficher | ||
| 28 | + | ||
| 29 | +Nommez le fichier et précisez de ne pas l'afficher : | ||
| 30 | + | ||
| 31 | +```text | ||
| 32 | +récupère le contenu de https://hello.com mais ne l'affiche pas et sauve le dans hello.txt | ||
| 33 | +``` | ||
| 34 | + | ||
| 35 | +Le modèle passe `output: hello.txt`. Le Markdown est écrit dans `hello.txt` (relatif au répertoire où tourne `mm`, ou au projet ouvert dans l'éditeur) et l'outil ne renvoie qu'une ligne : | ||
| 36 | + | ||
| 37 | +```text | ||
| 38 | +[fetch_content: saved 9 lines of Markdown (421 bytes) from https://hello.com to hello.txt — the content is not shown] | ||
| 39 | +``` | ||
| 40 | + | ||
| 41 | +La page n'entre pas dans la conversation : une page longue n'épuise donc pas la fenêtre de contexte. | ||
| 42 | + | ||
| 43 | +## Variantes | ||
| 44 | + | ||
| 45 | +- **Une URL tapée à la va-vite.** `k33g.org` et `https//k33g.org` fonctionnent : un schéma manquant devient `https://`, les deux-points oubliés sont remis. | ||
| 46 | +- **Tableaux complexes.** Les tableaux à cellules fusionnées sont gardés en HTML brut par défaut. Demandez des tableaux aplatis (« …en tableaux Markdown, aplatis-les ») et le modèle passe `tables: flatten`. | ||
| 47 | +- **Texte barré et listes de tâches.** Demandez-les explicitement (« garde le texte barré », « garde les cases à cocher ») et le modèle passe `strikethrough: true` ou `taskList: true`. | ||
| 48 | +- **Liens relatifs à un autre site.** Si la page est un miroir, demandez de résoudre les liens par rapport à l'original ; le modèle passe `baseUrl`. | ||
| 49 | +- **Dans un éditeur (ACP) ou avec `permissions: ask`.** Lire une page ne demande aucune permission ; l'enregistrer dans un fichier la demande d'abord, comme `write_file`. | ||
| 50 | + | ||
| 51 | +## En cas d'échec | ||
| 52 | + | ||
| 53 | +Le modèle reçoit `[fetch_content: error N (…) — raison]` et vous explique en général pourquoi : | ||
| 54 | + | ||
| 55 | +- `error 2 (input)` : le site est injoignable, a renvoyé une erreur HTTP, ou son `robots.txt` interdit aux robots de lire cette page. `mm` ne contourne pas le `robots.txt`. | ||
| 56 | +- `error 1 (conversion)` : l'URL n'est pas une page HTML (un PDF, du JSON, une image) ou dépasse 5 Mio. | ||
| 57 | +- `error 3 (usage)` : l'appel lui-même est faux, par exemple une URL `ftp://`. | ||
| 58 | + | ||
| 59 | +Si le modèle utilise `curl` via `bash` à la place, l'outil n'est probablement pas déclaré : vérifiez `fetchContentTool: true` et la bannière. | ||
| 60 | + | ||
| 61 | +## Voir aussi | ||
| 62 | + | ||
| 63 | +- Chaque champ, règle et code d'erreur : [référence des outils intégrés](../reference/tools.md#fetch_content) | ||
| 64 | +- La clé de configuration : [référence de la configuration](../reference/configuration.md) | ||
| 65 | +- Pourquoi l'outil fonctionne ainsi : [l'outil `fetch_content`](../explanation/fetch-content-tool.md) | ||
modified
docs/fr/reference/acp.md +1 -0 | @@ -128,6 +128,7 @@ Une erreur de transport ou un résultat `cancelled` compte comme un refus. | ||
| 128 | 128 | | `execute` | `execute` | `bash` (et tout genre inconnu) | |
| 129 | 129 | | `read` | `read` | `read_skill`, `read_file` | |
| 130 | 130 | | `edit` | `edit` | `write_file`, `edit_file` | |
| 131 | +| `fetch` | `fetch` | `fetch_content` | | |
| 131 | 132 | |
| 132 | 133 | ## Identifiants d'appel d'outil |
| 133 | 134 | |
| @@ -128,6 +128,7 @@ Une erreur de transport ou un résultat `cancelled` compte comme un refus. | |||
| 128 | | `execute` | `execute` | `bash` (et tout genre inconnu) | | 128 | | `execute` | `execute` | `bash` (et tout genre inconnu) | |
| 129 | | `read` | `read` | `read_skill`, `read_file` | | 129 | | `read` | `read` | `read_skill`, `read_file` | |
| 130 | | `edit` | `edit` | `write_file`, `edit_file` | | 130 | | `edit` | `edit` | `write_file`, `edit_file` | |
| 131 | +| `fetch` | `fetch` | `fetch_content` | | ||
| 131 | 132 | ||
| 132 | ## Identifiants d'appel d'outil | 133 | ## Identifiants d'appel d'outil |
| 133 | 134 | ||
modified
docs/fr/reference/configuration.md +1 -0 | @@ -21,6 +21,7 @@ | ||
| 21 | 21 | | `skillsDir` | chaîne | `skills` | Répertoire des skills, sous forme de fichiers `<nom>.md` ou de répertoires `<nom>/SKILL.md`. Un chemin relatif est résolu depuis le répertoire de lancement de `mm` (mode terminal, ou mode ACP sans fichier de configuration), ou à côté du fichier de configuration (mode ACP avec fichier) ; la bannière avertit avec le chemin absolu quand rien n'y est trouvé. | |
| 22 | 22 | | `bashTool` | booléen | `true` | Déclare `bash`. `false` le retire : le modèle peut encore lire et modifier des fichiers (`editTools`) et charger des skills, mais n'exécute jamais de commande. Reformulez un prompt système qui annonce encore un outil bash. | |
| 23 | 23 | | `editTools` | booléen | `true` | Déclare `read_file`, `write_file`, `edit_file`. `false` ne laisse que `bash` et `read_skill`. | |
| 24 | +| `fetchContentTool` | booléen | `false` | Déclare `fetch_content`, qui récupère des pages web en Markdown. Désactivé par défaut : c'est le seul outil intégré qui accède au réseau. | | |
| 24 | 25 | | `sampling` | dictionnaire | `temperature: 0.0`, `top_p: 0.9`, `max_tokens: 4096` | Paramètres de génération, noms de l'API OpenAI, transmis au serveur. L'`agent.yaml` livré ajoute `parallel_tool_calls: false`. `max_tokens` borne aussi un appel d'outil : un `write_file` de long document (ou le raisonnement d'un modèle « thinking » plus l'appel) peut l'atteindre, et l'appel est alors coupé. mm le rend deux fois au modèle en lui demandant de procéder par morceaux, puis s'arrête avec `the model hit max_tokens (N) while writing a tool call` ; augmentez la valeur si cela arrive. En streaming, mm demande aussi au serveur le compte des tokens (`stream_options.include_usage`). | |
| 25 | 26 | | `watchdogTimeout` | durée | `20s` | Silence toléré entre deux tokens diffusés avant de déclarer la connexion bloquée. Doublé pour les requêtes de résumé. | |
| 26 | 27 | | `firstTokenTimeout` | durée | `3m` | Silence toléré avant le premier token d'un appel au modèle, pendant que le serveur lit le prompt (chaque aller-retour d'un tour en a un). Jamais plus court que `watchdogTimeout` ; doit être >= 0. Avec un serveur llama.cpp (`dmr`, `llamacpp`), mm demande `return_progress` : chaque rapport d'avancement relance ce délai, et le spinner affiche `Reading the prompt 12.0k/25.3k tokens`. À l'expiration, le watchdog dit `no first token from the server after …` (avant le premier token) ou `connection hang detected after …` (après). | |
| @@ -21,6 +21,7 @@ | |||
| 21 | | `skillsDir` | chaîne | `skills` | Répertoire des skills, sous forme de fichiers `<nom>.md` ou de répertoires `<nom>/SKILL.md`. Un chemin relatif est résolu depuis le répertoire de lancement de `mm` (mode terminal, ou mode ACP sans fichier de configuration), ou à côté du fichier de configuration (mode ACP avec fichier) ; la bannière avertit avec le chemin absolu quand rien n'y est trouvé. | | 21 | | `skillsDir` | chaîne | `skills` | Répertoire des skills, sous forme de fichiers `<nom>.md` ou de répertoires `<nom>/SKILL.md`. Un chemin relatif est résolu depuis le répertoire de lancement de `mm` (mode terminal, ou mode ACP sans fichier de configuration), ou à côté du fichier de configuration (mode ACP avec fichier) ; la bannière avertit avec le chemin absolu quand rien n'y est trouvé. | |
| 22 | | `bashTool` | booléen | `true` | Déclare `bash`. `false` le retire : le modèle peut encore lire et modifier des fichiers (`editTools`) et charger des skills, mais n'exécute jamais de commande. Reformulez un prompt système qui annonce encore un outil bash. | | 22 | | `bashTool` | booléen | `true` | Déclare `bash`. `false` le retire : le modèle peut encore lire et modifier des fichiers (`editTools`) et charger des skills, mais n'exécute jamais de commande. Reformulez un prompt système qui annonce encore un outil bash. | |
| 23 | | `editTools` | booléen | `true` | Déclare `read_file`, `write_file`, `edit_file`. `false` ne laisse que `bash` et `read_skill`. | | 23 | | `editTools` | booléen | `true` | Déclare `read_file`, `write_file`, `edit_file`. `false` ne laisse que `bash` et `read_skill`. | |
| 24 | +| `fetchContentTool` | booléen | `false` | Déclare `fetch_content`, qui récupère des pages web en Markdown. Désactivé par défaut : c'est le seul outil intégré qui accède au réseau. | | ||
| 24 | | `sampling` | dictionnaire | `temperature: 0.0`, `top_p: 0.9`, `max_tokens: 4096` | Paramètres de génération, noms de l'API OpenAI, transmis au serveur. L'`agent.yaml` livré ajoute `parallel_tool_calls: false`. `max_tokens` borne aussi un appel d'outil : un `write_file` de long document (ou le raisonnement d'un modèle « thinking » plus l'appel) peut l'atteindre, et l'appel est alors coupé. mm le rend deux fois au modèle en lui demandant de procéder par morceaux, puis s'arrête avec `the model hit max_tokens (N) while writing a tool call` ; augmentez la valeur si cela arrive. En streaming, mm demande aussi au serveur le compte des tokens (`stream_options.include_usage`). | | 25 | | `sampling` | dictionnaire | `temperature: 0.0`, `top_p: 0.9`, `max_tokens: 4096` | Paramètres de génération, noms de l'API OpenAI, transmis au serveur. L'`agent.yaml` livré ajoute `parallel_tool_calls: false`. `max_tokens` borne aussi un appel d'outil : un `write_file` de long document (ou le raisonnement d'un modèle « thinking » plus l'appel) peut l'atteindre, et l'appel est alors coupé. mm le rend deux fois au modèle en lui demandant de procéder par morceaux, puis s'arrête avec `the model hit max_tokens (N) while writing a tool call` ; augmentez la valeur si cela arrive. En streaming, mm demande aussi au serveur le compte des tokens (`stream_options.include_usage`). | |
| 25 | | `watchdogTimeout` | durée | `20s` | Silence toléré entre deux tokens diffusés avant de déclarer la connexion bloquée. Doublé pour les requêtes de résumé. | | 26 | | `watchdogTimeout` | durée | `20s` | Silence toléré entre deux tokens diffusés avant de déclarer la connexion bloquée. Doublé pour les requêtes de résumé. | |
| 26 | | `firstTokenTimeout` | durée | `3m` | Silence toléré avant le premier token d'un appel au modèle, pendant que le serveur lit le prompt (chaque aller-retour d'un tour en a un). Jamais plus court que `watchdogTimeout` ; doit être >= 0. Avec un serveur llama.cpp (`dmr`, `llamacpp`), mm demande `return_progress` : chaque rapport d'avancement relance ce délai, et le spinner affiche `Reading the prompt 12.0k/25.3k tokens`. À l'expiration, le watchdog dit `no first token from the server after …` (avant le premier token) ou `connection hang detected after …` (après). | | 27 | | `firstTokenTimeout` | durée | `3m` | Silence toléré avant le premier token d'un appel au modèle, pendant que le serveur lit le prompt (chaque aller-retour d'un tour en a un). Jamais plus court que `watchdogTimeout` ; doit être >= 0. Avec un serveur llama.cpp (`dmr`, `llamacpp`), mm demande `return_progress` : chaque rapport d'avancement relance ce délai, et le spinner affiche `Reading the prompt 12.0k/25.3k tokens`. À l'expiration, le watchdog dit `no first token from the server after …` (avant le premier token) ou `connection hang detected after …` (après). | |
modified
docs/fr/reference/tools.md +39 -0 | @@ -159,6 +159,45 @@ Déclaré quand `lsp.servers` liste au moins un serveur dont la commande est sur | ||
| 159 | 159 | | ACP / fenêtre | Sorte `read` pour la navigation, sans permission ; `rename` est de sorte `edit` et demande la permission comme `edit_file` (`[rename rejected by the user]` en cas de refus). | |
| 160 | 160 | | Compteur | `🔎 N lsp call(s)` dans la ligne de récapitulatif (voir plus bas). | |
| 161 | 161 | |
| 162 | +## `fetch_content` | |
| 163 | + | |
| 164 | +Déclaré seulement avec `fetchContentTool: true`. Télécharge une page web et la renvoie en Markdown, ou enregistre ce Markdown dans un fichier. Paquet `internal/webmd`. | |
| 165 | + | |
| 166 | +| Champ | Type | Défaut | Description | | |
| 167 | +|-------|------|--------|-------------| | |
| 168 | +| `url` | chaîne | — | La page. Sans schéma, `https://` est ajouté (`k33g.org`) ; un schéma tapé sans ses deux-points est réparé (`https//k33g.org`). Seuls `http` et `https`. | | |
| 169 | +| `output` | chaîne | `""` | Fichier où enregistrer le Markdown, relatif au répertoire de travail (le `cwd` de la session en ACP) ; les répertoires parents sont créés. Renseigné, le Markdown n'est pas renvoyé. | | |
| 170 | +| `baseUrl` | chaîne | URL de la page | URL absolue qui sert à résoudre les liens et images relatifs. Par défaut : l'URL finale de la page, après redirections. | | |
| 171 | +| `tables` | chaîne | `html` | Tableaux avec `rowspan`/`colspan` ou un tableau imbriqué : `html` les garde en HTML brut, `flatten` produit un tableau GFM en recopiant une cellule fusionnée dans chaque case qu'elle couvre. | | |
| 172 | +| `strikethrough` | booléen | `false` | `<del>`, `<s>`, `<strike>` → `~~texte~~` ; sinon le texte seul. | | |
| 173 | +| `taskList` | booléen | `false` | Un élément de liste qui commence par une case à cocher → `- [ ] …` / `- [x] …` ; sinon la case est retirée. | | |
| 174 | + | |
| 175 | +| Aspect | Comportement | | |
| 176 | +|--------|--------------| | |
| 177 | +| Requête | `GET` avec `User-Agent: mm/<version>` ; 10 redirections au plus ; 30 s pour toute la récupération ; pages de plus de 5 Mio refusées. | | |
| 178 | +| robots.txt | Lu (et gardé 24 h par origine) avant la page et avant chaque cible de redirection, pour le jeton `mm`, sinon le groupe `*`. `Allow`/`Disallow`, la correspondance la plus longue l'emporte, `Allow` gagne en cas d'égalité, `*` et `$` final pris en charge (RFC 9309). Un robots.txt en 4xx autorise tout ; en 5xx ou injoignable, il interdit tout. | | |
| 179 | +| Types acceptés | `text/html`, `application/xhtml+xml` ; sans `Content-Type`, le type est deviné. | | |
| 180 | +| Encodage | Dans l'ordre : marque d'ordre des octets, `charset` de l'en-tête `Content-Type`, `<meta charset>` ou `<meta http-equiv="Content-Type">` dans les 4 premiers Kio, UTF-8 si les octets sont de l'UTF-8 valide, sinon `windows-1252`. Un nom inconnu est ignoré. Le contenu est converti en UTF-8. | | |
| 181 | +| Retiré | `script`, `style`, `noscript`, `template`, `meta`, `link`, `base`, `title`, `svg`, `canvas`, `iframe`, `object`, `embed`, `nav`, `aside`, `dialog`, `button`, `select`, `textarea`, les champs autres que les cases à cocher, un `footer` hors de `article`/`main`, les commentaires HTML, les rôles ARIA `navigation` `banner` `complementary` `search` `contentinfo` `dialog`, `aria-hidden="true"`, `hidden`, les mots de class/id `ad` `ads` `advert` `sponsor` `promo` `cookie` `consent` `gdpr` `newsletter` `subscribe` `popup` `modal` `share` `social` `breadcrumb` `nav` `navbar` `navigation` `menu` `sidebar` `skiplink` (et quelques variantes), les classes `sr-only` `visually-hidden` `screen-reader-text`. Un élément repéré par class/id est gardé s'il contient `main`, `article` ou `h1`. | | |
| 182 | +| Attributs | Tous retirés sauf `a[href]`, `img[src,alt]`, `td/th[colspan,rowspan]`, `ol[start]`, `input[type,checked]`, et les mots `language-*`/`lang-*` de la classe d'un `pre`/`code`. | | |
| 183 | +| Conversion | `h1`–`h6` → `#`–`######` ; `p` → paragraphe, ligne vide entre les blocs ; `strong`/`b` → `**…**` ; `em`/`i` → `*…*` ; `code`/`kbd`/`samp`/`tt` en ligne → accents graves ; `pre` → bloc clôturé (langage tiré de `language-x`), clôture plus longue que toute suite d'accents graves du code ; `br` → saut de ligne forcé `\` ; `hr` → `---` ; `blockquote` → `> ` ; `a` → `[texte](url)`, l'URL faute de texte, le texte seul pour `javascript:` ; `img` → ``, `` sans alt, l'alt seul pour une URI `data:` ; `ul`/`ol` → `- ` / `N. ` (depuis `start`), listes imbriquées indentées de la largeur du marqueur parent ; tableaux simples → GFM, première ligne comme en-tête, `|` échappé, `<br>` pour un saut de ligne dans une cellule, `<caption>` en paragraphe au-dessus. | | |
| 184 | +| Élément inconnu | Balises retirées, texte conservé ; s'il contient un bloc, il est traité comme un conteneur. | | |
| 185 | +| Élément vide | Rien, pas de ligne vide. | | |
| 186 | +| HTML invalide | Réparé par le parseur HTML5 (`golang.org/x/net/html`), puis converti. | | |
| 187 | +| Échappement | La barre oblique inverse, `*`, l'accent grave, `[`, `]`, `<` et un `_` hors d'un mot reçoivent une barre oblique inverse ; de même une ligne qui commence par `#`, `>`, `-`, `+`, `=`, `\|` ou `N.`. | | |
| 188 | +| Résultat | Le Markdown, terminé par un unique retour à la ligne ; ni titre, ni front matter, ni préambule. Tronqué à `maxOutput` caractères. | | |
| 189 | +| Avec `output` | `[fetch_content: saved N lines of Markdown (B bytes) from <url> to <fichier> — the content is not shown]`. | | |
| 190 | +| Page vide une fois nettoyée | `[fetch_content: <url> has no text content once scripts, navigation and ads are removed — nothing saved]` ; aucun fichier écrit. | | |
| 191 | +| Échec | `[fetch_content: error N (<classe>) — <raison>]`, voir les codes ci-dessous. | | |
| 192 | +| Affichage terminal | `🌐 fetch_content: <url>` (`→ <fichier>` avec `output`), indicateur `Fetching`, puis le résultat affiché comme la sortie d'une commande (seulement la ligne de confirmation avec `output`). | | |
| 193 | +| ACP / fenêtre | Genre `fetch`. Pas de permission sans `output` ; avec `output`, permission demandée comme pour `write_file` (`[fetch_content rejected by the user]` en cas de refus), et le chemin absolu du fichier comme emplacement. | | |
| 194 | + | |
| 195 | +| Code | Classe | Cause | | |
| 196 | +|------|--------|-------| | |
| 197 | +| `1` | `conversion` | Pas une page HTML, plus de 5 Mio, impossible à décoder, ou fichier de sortie impossible à écrire. | | |
| 198 | +| `2` | `input` | Hôte injoignable, délai dépassé, statut HTTP ≥ 300 après redirections, plus de 10 redirections, page interdite par le robots.txt ou robots.txt illisible. | | |
| 199 | +| `3` | `usage` | `url` vide ou illisible, schéma autre que `http`/`https`, valeur de `tables` inconnue, `baseUrl` non absolue. | | |
| 200 | + | |
| 162 | 201 | ## Détection de boucle |
| 163 | 202 | |
| 164 | 203 | Chaque outil enregistre `(outil, entrée, sortie)` dans le détecteur de boucle après chaque appel. |
| @@ -159,6 +159,45 @@ Déclaré quand `lsp.servers` liste au moins un serveur dont la commande est sur | |||
| 159 | | ACP / fenêtre | Sorte `read` pour la navigation, sans permission ; `rename` est de sorte `edit` et demande la permission comme `edit_file` (`[rename rejected by the user]` en cas de refus). | | 159 | | ACP / fenêtre | Sorte `read` pour la navigation, sans permission ; `rename` est de sorte `edit` et demande la permission comme `edit_file` (`[rename rejected by the user]` en cas de refus). | |
| 160 | | Compteur | `🔎 N lsp call(s)` dans la ligne de récapitulatif (voir plus bas). | | 160 | | Compteur | `🔎 N lsp call(s)` dans la ligne de récapitulatif (voir plus bas). | |
| 161 | 161 | ||
| 162 | +## `fetch_content` | ||
| 163 | + | ||
| 164 | +Déclaré seulement avec `fetchContentTool: true`. Télécharge une page web et la renvoie en Markdown, ou enregistre ce Markdown dans un fichier. Paquet `internal/webmd`. | ||
| 165 | + | ||
| 166 | +| Champ | Type | Défaut | Description | | ||
| 167 | +|-------|------|--------|-------------| | ||
| 168 | +| `url` | chaîne | — | La page. Sans schéma, `https://` est ajouté (`k33g.org`) ; un schéma tapé sans ses deux-points est réparé (`https//k33g.org`). Seuls `http` et `https`. | | ||
| 169 | +| `output` | chaîne | `""` | Fichier où enregistrer le Markdown, relatif au répertoire de travail (le `cwd` de la session en ACP) ; les répertoires parents sont créés. Renseigné, le Markdown n'est pas renvoyé. | | ||
| 170 | +| `baseUrl` | chaîne | URL de la page | URL absolue qui sert à résoudre les liens et images relatifs. Par défaut : l'URL finale de la page, après redirections. | | ||
| 171 | +| `tables` | chaîne | `html` | Tableaux avec `rowspan`/`colspan` ou un tableau imbriqué : `html` les garde en HTML brut, `flatten` produit un tableau GFM en recopiant une cellule fusionnée dans chaque case qu'elle couvre. | | ||
| 172 | +| `strikethrough` | booléen | `false` | `<del>`, `<s>`, `<strike>` → `~~texte~~` ; sinon le texte seul. | | ||
| 173 | +| `taskList` | booléen | `false` | Un élément de liste qui commence par une case à cocher → `- [ ] …` / `- [x] …` ; sinon la case est retirée. | | ||
| 174 | + | ||
| 175 | +| Aspect | Comportement | | ||
| 176 | +|--------|--------------| | ||
| 177 | +| Requête | `GET` avec `User-Agent: mm/<version>` ; 10 redirections au plus ; 30 s pour toute la récupération ; pages de plus de 5 Mio refusées. | | ||
| 178 | +| robots.txt | Lu (et gardé 24 h par origine) avant la page et avant chaque cible de redirection, pour le jeton `mm`, sinon le groupe `*`. `Allow`/`Disallow`, la correspondance la plus longue l'emporte, `Allow` gagne en cas d'égalité, `*` et `$` final pris en charge (RFC 9309). Un robots.txt en 4xx autorise tout ; en 5xx ou injoignable, il interdit tout. | | ||
| 179 | +| Types acceptés | `text/html`, `application/xhtml+xml` ; sans `Content-Type`, le type est deviné. | | ||
| 180 | +| Encodage | Dans l'ordre : marque d'ordre des octets, `charset` de l'en-tête `Content-Type`, `<meta charset>` ou `<meta http-equiv="Content-Type">` dans les 4 premiers Kio, UTF-8 si les octets sont de l'UTF-8 valide, sinon `windows-1252`. Un nom inconnu est ignoré. Le contenu est converti en UTF-8. | | ||
| 181 | +| Retiré | `script`, `style`, `noscript`, `template`, `meta`, `link`, `base`, `title`, `svg`, `canvas`, `iframe`, `object`, `embed`, `nav`, `aside`, `dialog`, `button`, `select`, `textarea`, les champs autres que les cases à cocher, un `footer` hors de `article`/`main`, les commentaires HTML, les rôles ARIA `navigation` `banner` `complementary` `search` `contentinfo` `dialog`, `aria-hidden="true"`, `hidden`, les mots de class/id `ad` `ads` `advert` `sponsor` `promo` `cookie` `consent` `gdpr` `newsletter` `subscribe` `popup` `modal` `share` `social` `breadcrumb` `nav` `navbar` `navigation` `menu` `sidebar` `skiplink` (et quelques variantes), les classes `sr-only` `visually-hidden` `screen-reader-text`. Un élément repéré par class/id est gardé s'il contient `main`, `article` ou `h1`. | | ||
| 182 | +| Attributs | Tous retirés sauf `a[href]`, `img[src,alt]`, `td/th[colspan,rowspan]`, `ol[start]`, `input[type,checked]`, et les mots `language-*`/`lang-*` de la classe d'un `pre`/`code`. | | ||
| 183 | +| Conversion | `h1`–`h6` → `#`–`######` ; `p` → paragraphe, ligne vide entre les blocs ; `strong`/`b` → `**…**` ; `em`/`i` → `*…*` ; `code`/`kbd`/`samp`/`tt` en ligne → accents graves ; `pre` → bloc clôturé (langage tiré de `language-x`), clôture plus longue que toute suite d'accents graves du code ; `br` → saut de ligne forcé `\` ; `hr` → `---` ; `blockquote` → `> ` ; `a` → `[texte](url)`, l'URL faute de texte, le texte seul pour `javascript:` ; `img` → ``, `` sans alt, l'alt seul pour une URI `data:` ; `ul`/`ol` → `- ` / `N. ` (depuis `start`), listes imbriquées indentées de la largeur du marqueur parent ; tableaux simples → GFM, première ligne comme en-tête, `|` échappé, `<br>` pour un saut de ligne dans une cellule, `<caption>` en paragraphe au-dessus. | | ||
| 184 | +| Élément inconnu | Balises retirées, texte conservé ; s'il contient un bloc, il est traité comme un conteneur. | | ||
| 185 | +| Élément vide | Rien, pas de ligne vide. | | ||
| 186 | +| HTML invalide | Réparé par le parseur HTML5 (`golang.org/x/net/html`), puis converti. | | ||
| 187 | +| Échappement | La barre oblique inverse, `*`, l'accent grave, `[`, `]`, `<` et un `_` hors d'un mot reçoivent une barre oblique inverse ; de même une ligne qui commence par `#`, `>`, `-`, `+`, `=`, `\|` ou `N.`. | | ||
| 188 | +| Résultat | Le Markdown, terminé par un unique retour à la ligne ; ni titre, ni front matter, ni préambule. Tronqué à `maxOutput` caractères. | | ||
| 189 | +| Avec `output` | `[fetch_content: saved N lines of Markdown (B bytes) from <url> to <fichier> — the content is not shown]`. | | ||
| 190 | +| Page vide une fois nettoyée | `[fetch_content: <url> has no text content once scripts, navigation and ads are removed — nothing saved]` ; aucun fichier écrit. | | ||
| 191 | +| Échec | `[fetch_content: error N (<classe>) — <raison>]`, voir les codes ci-dessous. | | ||
| 192 | +| Affichage terminal | `🌐 fetch_content: <url>` (`→ <fichier>` avec `output`), indicateur `Fetching`, puis le résultat affiché comme la sortie d'une commande (seulement la ligne de confirmation avec `output`). | | ||
| 193 | +| ACP / fenêtre | Genre `fetch`. Pas de permission sans `output` ; avec `output`, permission demandée comme pour `write_file` (`[fetch_content rejected by the user]` en cas de refus), et le chemin absolu du fichier comme emplacement. | | ||
| 194 | + | ||
| 195 | +| Code | Classe | Cause | | ||
| 196 | +|------|--------|-------| | ||
| 197 | +| `1` | `conversion` | Pas une page HTML, plus de 5 Mio, impossible à décoder, ou fichier de sortie impossible à écrire. | | ||
| 198 | +| `2` | `input` | Hôte injoignable, délai dépassé, statut HTTP ≥ 300 après redirections, plus de 10 redirections, page interdite par le robots.txt ou robots.txt illisible. | | ||
| 199 | +| `3` | `usage` | `url` vide ou illisible, schéma autre que `http`/`https`, valeur de `tables` inconnue, `baseUrl` non absolue. | | ||
| 200 | + | ||
| 162 | ## Détection de boucle | 201 | ## Détection de boucle |
| 163 | 202 | ||
| 164 | Chaque outil enregistre `(outil, entrée, sortie)` dans le détecteur de boucle après chaque appel. | 203 | Chaque outil enregistre `(outil, entrée, sortie)` dans le détecteur de boucle après chaque appel. |
modified
go.mod +6 -5 | @@ -1,14 +1,17 @@ | ||
| 1 | 1 | module mm |
| 2 | 2 | |
| 3 | -go 1.26 | |
| 3 | +go 1.26.0 | |
| 4 | 4 | |
| 5 | 5 | require ( |
| 6 | 6 | github.com/firebase/genkit/go v1.10.0 |
| 7 | 7 | github.com/gdamore/tcell/v2 v2.13.10 |
| 8 | 8 | github.com/openai/openai-go v1.8.2 |
| 9 | + github.com/sashabaranov/go-openai v1.42.1 | |
| 9 | 10 | go.lsp.dev/jsonrpc2 v1.0.1 |
| 10 | 11 | go.lsp.dev/protocol v1.0.1 |
| 11 | 12 | go.lsp.dev/uri v1.0.1 |
| 13 | + golang.org/x/net v0.59.0 | |
| 14 | + golang.org/x/text v0.42.0 | |
| 12 | 15 | gopkg.in/yaml.v3 v3.0.1 |
| 13 | 16 | ) |
| 14 | 17 | |
| @@ -17,9 +20,7 @@ require ( | ||
| 17 | 20 | github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect |
| 18 | 21 | github.com/lucasb-eyer/go-colorful v1.4.0 // indirect |
| 19 | 22 | github.com/rivo/uniseg v0.4.7 // indirect |
| 20 | - github.com/sashabaranov/go-openai v1.42.1 // indirect | |
| 21 | - golang.org/x/term v0.39.0 // indirect | |
| 22 | - golang.org/x/text v0.36.0 // indirect | |
| 23 | + golang.org/x/term v0.46.0 // indirect | |
| 23 | 24 | ) |
| 24 | 25 | |
| 25 | 26 | require ( |
| @@ -49,5 +50,5 @@ require ( | ||
| 49 | 50 | go.opentelemetry.io/otel/metric v1.36.0 // indirect |
| 50 | 51 | go.opentelemetry.io/otel/sdk v1.36.0 // indirect |
| 51 | 52 | go.opentelemetry.io/otel/trace v1.36.0 // indirect |
| 52 | - golang.org/x/sys v0.45.0 // indirect | |
| 53 | + golang.org/x/sys v0.48.0 // indirect | |
| 53 | 54 | ) |
| @@ -1,14 +1,17 @@ | |||
| 1 | module mm | 1 | module mm |
| 2 | 2 | ||
| 3 | -go 1.26 | 3 | +go 1.26.0 |
| 4 | 4 | ||
| 5 | require ( | 5 | require ( |
| 6 | github.com/firebase/genkit/go v1.10.0 | 6 | github.com/firebase/genkit/go v1.10.0 |
| 7 | github.com/gdamore/tcell/v2 v2.13.10 | 7 | github.com/gdamore/tcell/v2 v2.13.10 |
| 8 | github.com/openai/openai-go v1.8.2 | 8 | github.com/openai/openai-go v1.8.2 |
| 9 | + github.com/sashabaranov/go-openai v1.42.1 | ||
| 9 | go.lsp.dev/jsonrpc2 v1.0.1 | 10 | go.lsp.dev/jsonrpc2 v1.0.1 |
| 10 | go.lsp.dev/protocol v1.0.1 | 11 | go.lsp.dev/protocol v1.0.1 |
| 11 | go.lsp.dev/uri v1.0.1 | 12 | go.lsp.dev/uri v1.0.1 |
| 13 | + golang.org/x/net v0.59.0 | ||
| 14 | + golang.org/x/text v0.42.0 | ||
| 12 | gopkg.in/yaml.v3 v3.0.1 | 15 | gopkg.in/yaml.v3 v3.0.1 |
| 13 | ) | 16 | ) |
| 14 | 17 | ||
| @@ -17,9 +20,7 @@ require ( | |||
| 17 | github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect | 20 | github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect |
| 18 | github.com/lucasb-eyer/go-colorful v1.4.0 // indirect | 21 | github.com/lucasb-eyer/go-colorful v1.4.0 // indirect |
| 19 | github.com/rivo/uniseg v0.4.7 // indirect | 22 | github.com/rivo/uniseg v0.4.7 // indirect |
| 20 | - github.com/sashabaranov/go-openai v1.42.1 // indirect | 23 | + golang.org/x/term v0.46.0 // indirect |
| 21 | - golang.org/x/term v0.39.0 // indirect | ||
| 22 | - golang.org/x/text v0.36.0 // indirect | ||
| 23 | ) | 24 | ) |
| 24 | 25 | ||
| 25 | require ( | 26 | require ( |
| @@ -49,5 +50,5 @@ require ( | |||
| 49 | go.opentelemetry.io/otel/metric v1.36.0 // indirect | 50 | go.opentelemetry.io/otel/metric v1.36.0 // indirect |
| 50 | go.opentelemetry.io/otel/sdk v1.36.0 // indirect | 51 | go.opentelemetry.io/otel/sdk v1.36.0 // indirect |
| 51 | go.opentelemetry.io/otel/trace v1.36.0 // indirect | 52 | go.opentelemetry.io/otel/trace v1.36.0 // indirect |
| 52 | - golang.org/x/sys v0.45.0 // indirect | 53 | + golang.org/x/sys v0.48.0 // indirect |
| 53 | ) | 54 | ) |
modified
go.sum +8 -6 | @@ -104,6 +104,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL | ||
| 104 | 104 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= |
| 105 | 105 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= |
| 106 | 106 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= |
| 107 | +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= | |
| 108 | +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= | |
| 107 | 109 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
| 108 | 110 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
| 109 | 111 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
| @@ -113,20 +115,20 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc | ||
| 113 | 115 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= |
| 114 | 116 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= |
| 115 | 117 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= |
| 116 | -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= | |
| 117 | -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= | |
| 118 | +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= | |
| 119 | +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= | |
| 118 | 120 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= |
| 119 | 121 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= |
| 120 | 122 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= |
| 121 | -golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= | |
| 122 | -golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= | |
| 123 | +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= | |
| 124 | +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= | |
| 123 | 125 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= |
| 124 | 126 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= |
| 125 | 127 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= |
| 126 | 128 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= |
| 127 | 129 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= |
| 128 | -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= | |
| 129 | -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= | |
| 130 | +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= | |
| 131 | +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= | |
| 130 | 132 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= |
| 131 | 133 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= |
| 132 | 134 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= |
| @@ -104,6 +104,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL | |||
| 104 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= | 104 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= |
| 105 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= | 105 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= |
| 106 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= | 106 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= |
| 107 | +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= | ||
| 108 | +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= | ||
| 107 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | 109 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
| 108 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | 110 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
| 109 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | 111 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
| @@ -113,20 +115,20 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc | |||
| 113 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= | 115 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= |
| 114 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= | 116 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= |
| 115 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= | 117 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= |
| 116 | -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= | 118 | +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= |
| 117 | -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= | 119 | +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= |
| 118 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= | 120 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= |
| 119 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= | 121 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= |
| 120 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= | 122 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= |
| 121 | -golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= | 123 | +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= |
| 122 | -golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= | 124 | +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= |
| 123 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= | 125 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= |
| 124 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= | 126 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= |
| 125 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= | 127 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= |
| 126 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= | 128 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= |
| 127 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= | 129 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= |
| 128 | -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= | 130 | +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= |
| 129 | -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= | 131 | +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= |
| 130 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= | 132 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= |
| 131 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= | 133 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= |
| 132 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= | 134 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= |
modified
internal/acp/acp_test.go +1 -0 | @@ -40,6 +40,7 @@ func TestToolKindDefaultsToExecute(t *testing.T) { | ||
| 40 | 40 | cases := map[string]sdk.ToolKind{ |
| 41 | 41 | "read": sdk.ToolKindRead, |
| 42 | 42 | "edit": sdk.ToolKindEdit, |
| 43 | + "fetch": sdk.ToolKindFetch, | |
| 43 | 44 | "execute": sdk.ToolKindExecute, |
| 44 | 45 | "": sdk.ToolKindExecute, |
| 45 | 46 | "exotic": sdk.ToolKindExecute, |
| @@ -40,6 +40,7 @@ func TestToolKindDefaultsToExecute(t *testing.T) { | |||
| 40 | cases := map[string]sdk.ToolKind{ | 40 | cases := map[string]sdk.ToolKind{ |
| 41 | "read": sdk.ToolKindRead, | 41 | "read": sdk.ToolKindRead, |
| 42 | "edit": sdk.ToolKindEdit, | 42 | "edit": sdk.ToolKindEdit, |
| 43 | + "fetch": sdk.ToolKindFetch, | ||
| 43 | "execute": sdk.ToolKindExecute, | 44 | "execute": sdk.ToolKindExecute, |
| 44 | "": sdk.ToolKindExecute, | 45 | "": sdk.ToolKindExecute, |
| 45 | "exotic": sdk.ToolKindExecute, | 46 | "exotic": sdk.ToolKindExecute, |
modified
internal/acp/sink.go +2 -0 | @@ -149,6 +149,8 @@ func toolKind(kind string) sdk.ToolKind { | ||
| 149 | 149 | return sdk.ToolKindRead |
| 150 | 150 | case "edit": |
| 151 | 151 | return sdk.ToolKindEdit |
| 152 | + case "fetch": | |
| 153 | + return sdk.ToolKindFetch | |
| 152 | 154 | default: |
| 153 | 155 | return sdk.ToolKindExecute |
| 154 | 156 | } |
| @@ -149,6 +149,8 @@ func toolKind(kind string) sdk.ToolKind { | |||
| 149 | return sdk.ToolKindRead | 149 | return sdk.ToolKindRead |
| 150 | case "edit": | 150 | case "edit": |
| 151 | return sdk.ToolKindEdit | 151 | return sdk.ToolKindEdit |
| 152 | + case "fetch": | ||
| 153 | + return sdk.ToolKindFetch | ||
| 152 | default: | 154 | default: |
| 153 | return sdk.ToolKindExecute | 155 | return sdk.ToolKindExecute |
| 154 | } | 156 | } |
modified
internal/config/config.go +7 -0 | @@ -100,6 +100,12 @@ type Config struct { | ||
| 100 | 100 | // On by default. Off to disable plan creation and tracking via the `plan` tool. |
| 101 | 101 | PlanTool bool `yaml:"planTool"` |
| 102 | 102 | |
| 103 | + // FetchContentTool: declare the built-in `fetch_content` tool, which | |
| 104 | + // downloads a web page and returns it as Markdown. OFF by default: it is | |
| 105 | + // the one built-in tool that reaches the network, so an operator turns it | |
| 106 | + // on knowingly (`fetchContentTool: true`). | |
| 107 | + FetchContentTool bool `yaml:"fetchContentTool"` | |
| 108 | + | |
| 103 | 109 | // Sampling: generation settings (OpenAI API keys; the plugin converts them.) |
| 104 | 110 | Sampling map[string]any `yaml:"sampling"` |
| 105 | 111 | |
| @@ -414,6 +420,7 @@ var Cfg = Config{ | ||
| 414 | 420 | BashTool: true, |
| 415 | 421 | EditTools: true, |
| 416 | 422 | PlanTool: true, |
| 423 | + // FetchContentTool stays false (the zero value): see its field comment. | |
| 417 | 424 | System: `You are a coding agent working in a terminal. |
| 418 | 425 | You have a "bash" tool to run shell commands. |
| 419 | 426 | Use it to explore files, run tests, inspect the repository, etc. |
| @@ -100,6 +100,12 @@ type Config struct { | |||
| 100 | // On by default. Off to disable plan creation and tracking via the `plan` tool. | 100 | // On by default. Off to disable plan creation and tracking via the `plan` tool. |
| 101 | PlanTool bool `yaml:"planTool"` | 101 | PlanTool bool `yaml:"planTool"` |
| 102 | 102 | ||
| 103 | + // FetchContentTool: declare the built-in `fetch_content` tool, which | ||
| 104 | + // downloads a web page and returns it as Markdown. OFF by default: it is | ||
| 105 | + // the one built-in tool that reaches the network, so an operator turns it | ||
| 106 | + // on knowingly (`fetchContentTool: true`). | ||
| 107 | + FetchContentTool bool `yaml:"fetchContentTool"` | ||
| 108 | + | ||
| 103 | // Sampling: generation settings (OpenAI API keys; the plugin converts them.) | 109 | // Sampling: generation settings (OpenAI API keys; the plugin converts them.) |
| 104 | Sampling map[string]any `yaml:"sampling"` | 110 | Sampling map[string]any `yaml:"sampling"` |
| 105 | 111 | ||
| @@ -414,6 +420,7 @@ var Cfg = Config{ | |||
| 414 | BashTool: true, | 420 | BashTool: true, |
| 415 | EditTools: true, | 421 | EditTools: true, |
| 416 | PlanTool: true, | 422 | PlanTool: true, |
| 423 | + // FetchContentTool stays false (the zero value): see its field comment. | ||
| 417 | System: `You are a coding agent working in a terminal. | 424 | System: `You are a coding agent working in a terminal. |
| 418 | You have a "bash" tool to run shell commands. | 425 | You have a "bash" tool to run shell commands. |
| 419 | Use it to explore files, run tests, inspect the repository, etc. | 426 | Use it to explore files, run tests, inspect the repository, etc. |
modified
internal/config/config_test.go +21 -0 | @@ -196,3 +196,24 @@ func TestACPConfig(t *testing.T) { | ||
| 196 | 196 | t.Errorf("Load(bad.yaml) = %v, want an acp.name validation error", err) |
| 197 | 197 | } |
| 198 | 198 | } |
| 199 | + | |
| 200 | +// fetch_content reaches the network, so it is off unless the YAML asks for it. | |
| 201 | +func TestFetchContentToolConfig(t *testing.T) { | |
| 202 | + defaults := Cfg | |
| 203 | + t.Cleanup(func() { Cfg = defaults }) | |
| 204 | + | |
| 205 | + if Cfg.FetchContentTool { | |
| 206 | + t.Fatal("fetchContentTool is on by default, want off") | |
| 207 | + } | |
| 208 | + | |
| 209 | + on := filepath.Join(t.TempDir(), "on.yaml") | |
| 210 | + if err := os.WriteFile(on, []byte("fetchContentTool: true\n"), 0o644); err != nil { | |
| 211 | + t.Fatal(err) | |
| 212 | + } | |
| 213 | + if _, err := Load(on); err != nil { | |
| 214 | + t.Fatal(err) | |
| 215 | + } | |
| 216 | + if !Cfg.FetchContentTool { | |
| 217 | + t.Error("fetchContentTool: true was not read") | |
| 218 | + } | |
| 219 | +} | |
| @@ -196,3 +196,24 @@ func TestACPConfig(t *testing.T) { | |||
| 196 | t.Errorf("Load(bad.yaml) = %v, want an acp.name validation error", err) | 196 | t.Errorf("Load(bad.yaml) = %v, want an acp.name validation error", err) |
| 197 | } | 197 | } |
| 198 | } | 198 | } |
| 199 | + | ||
| 200 | +// fetch_content reaches the network, so it is off unless the YAML asks for it. | ||
| 201 | +func TestFetchContentToolConfig(t *testing.T) { | ||
| 202 | + defaults := Cfg | ||
| 203 | + t.Cleanup(func() { Cfg = defaults }) | ||
| 204 | + | ||
| 205 | + if Cfg.FetchContentTool { | ||
| 206 | + t.Fatal("fetchContentTool is on by default, want off") | ||
| 207 | + } | ||
| 208 | + | ||
| 209 | + on := filepath.Join(t.TempDir(), "on.yaml") | ||
| 210 | + if err := os.WriteFile(on, []byte("fetchContentTool: true\n"), 0o644); err != nil { | ||
| 211 | + t.Fatal(err) | ||
| 212 | + } | ||
| 213 | + if _, err := Load(on); err != nil { | ||
| 214 | + t.Fatal(err) | ||
| 215 | + } | ||
| 216 | + if !Cfg.FetchContentTool { | ||
| 217 | + t.Error("fetchContentTool: true was not read") | ||
| 218 | + } | ||
| 219 | +} | ||
added
internal/tools/fetchcontent.go +200 -0 | new file mode 100644 | ||
| @@ -0,0 +1,200 @@ | ||
| 1 | +package tools | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "context" | |
| 5 | + "fmt" | |
| 6 | + "net/url" | |
| 7 | + "os" | |
| 8 | + "path/filepath" | |
| 9 | + "strings" | |
| 10 | + | |
| 11 | + "mm/internal/detector" | |
| 12 | + "mm/internal/spinner" | |
| 13 | + "mm/internal/ui" | |
| 14 | + "mm/internal/webmd" | |
| 15 | + | |
| 16 | + "github.com/firebase/genkit/go/ai" | |
| 17 | + "github.com/firebase/genkit/go/genkit" | |
| 18 | +) | |
| 19 | + | |
| 20 | +// --- fetch_content ----------------------------------------------------------- | |
| 21 | +// | |
| 22 | +// Downloads a web page and gives it back as Markdown (see internal/webmd for | |
| 23 | +// the conversion rules). Not to be confused with an MCP `fetch` server: this | |
| 24 | +// one is built in, honours robots.txt, and can save the page to a file | |
| 25 | +// instead of returning it — "save it, don't show it" must not cost the | |
| 26 | +// context window a whole page. | |
| 27 | +// | |
| 28 | +// Same contract as the other tools: a failure goes back AS TEXT, carrying | |
| 29 | +// the error code of the fetch-content specification (1 conversion, 2 input, | |
| 30 | +// 3 usage) so the model can tell "fix your call" from "the site is down". | |
| 31 | + | |
| 32 | +type fetchContentInput struct { | |
| 33 | + URL string `json:"url" jsonschema_description:"The web page to fetch, e.g. https://example.com/page. A missing scheme means https."` | |
| 34 | + Output string `json:"output,omitempty" jsonschema_description:"Optional. A file path (relative to the working directory) to SAVE the Markdown to. When set, the Markdown is written to that file and NOT returned: only a one-line confirmation comes back."` | |
| 35 | + BaseURL string `json:"baseUrl,omitempty" jsonschema_description:"Optional. Absolute URL used to resolve relative links and images; defaults to the page's own URL."` | |
| 36 | + Tables string `json:"tables,omitempty" jsonschema_description:"Optional. What to do with tables using rowspan/colspan: \"html\" (default) keeps them as raw HTML, \"flatten\" turns them into Markdown tables."` | |
| 37 | + Strikethrough bool `json:"strikethrough,omitempty" jsonschema_description:"Optional. Render <del>/<s>/<strike> as ~~text~~ (default: plain text)."` | |
| 38 | + TaskList bool `json:"taskList,omitempty" jsonschema_description:"Optional. Render list items starting with a checkbox as - [ ] / - [x] (default: checkbox dropped)."` | |
| 39 | +} | |
| 40 | + | |
| 41 | +// pageFetcher is what fetch_content needs from webmd.Fetcher; tests give | |
| 42 | +// it a fake so that no test touches the network. | |
| 43 | +type pageFetcher interface { | |
| 44 | + Fetch(ctx context.Context, rawURL string) (*webmd.Page, error) | |
| 45 | +} | |
| 46 | + | |
| 47 | +// FetchContent declares the `fetch_content` tool over f. | |
| 48 | +// | |
| 49 | +// tools.FetchContent(g, d, webmd.NewFetcher("mm/"+version.Short())) | |
| 50 | +func FetchContent(g *genkit.Genkit, d *detector.LoopDetector, f pageFetcher) ai.ToolRef { | |
| 51 | + description := `Fetch a web page and return its content as Markdown (headings, lists, links, images, tables, code blocks). Scripts, styles, navigation, ads and cookie banners are removed. | |
| 52 | +Use this tool whenever the user asks to fetch, get, retrieve, download or read the content of a web page or URL, in any language (for example "récupère le contenu de https://example.com") — do not use curl or wget through bash for that. | |
| 53 | +To SAVE the page to a file, set "output" to the file name the user gave: the Markdown is written there and NOT returned. When the user says not to display it, do not show or summarise the content — just confirm where it was saved. | |
| 54 | +Without "output", the Markdown comes back as the result (truncated when very long). | |
| 55 | +The site's robots.txt is honoured: a page it forbids is not fetched. | |
| 56 | +Failures come back as "[fetch_content: error N (…) — reason]": 1 = the page could not be converted (not HTML, too large), 2 = the page could not be fetched (unreachable, HTTP error, forbidden by robots.txt), 3 = invalid argument — fix the call.` | |
| 57 | + | |
| 58 | + return genkit.DefineTool(g, "fetch_content", description, | |
| 59 | + func(tctx *ai.ToolContext, in fetchContentInput) (string, error) { | |
| 60 | + where := in.URL | |
| 61 | + if in.Output != "" { | |
| 62 | + where += " → " + in.Output | |
| 63 | + } | |
| 64 | + if s := ui.ActiveSink(); s != nil { | |
| 65 | + return fetchContentCall{fetcher: f, detector: d}.evented(tctx, s, in, where) | |
| 66 | + } | |
| 67 | + | |
| 68 | + announce("🌐 fetch_content: " + where) | |
| 69 | + // A page can take seconds to come: same reasoning as bash's | |
| 70 | + // "Running" label, a slow site must not look like a hung agent. | |
| 71 | + spinner.Start("Fetching") | |
| 72 | + out, _ := fetchContent(tctx, f, in, ui.Resolve) | |
| 73 | + out = truncate(out) | |
| 74 | + // With `output`, out is the confirmation line: the page itself is | |
| 75 | + // not echoed, which is what "save it, don't show it" asks for. | |
| 76 | + spinner.Suspend(func() { echoOutput(out) }) | |
| 77 | + spinner.Start("Thinking") | |
| 78 | + return record(d, "fetch_content", where, out) | |
| 79 | + }) | |
| 80 | +} | |
| 81 | + | |
| 82 | +// fetchContentCall is what one call of the tool works with. | |
| 83 | +type fetchContentCall struct { | |
| 84 | + fetcher pageFetcher | |
| 85 | + detector *detector.LoopDetector | |
| 86 | +} | |
| 87 | + | |
| 88 | +// evented is `fetch_content` for an event-consuming front end (ACP, the | |
| 89 | +// window). Fetching alone is not gated, like reading a file; saving writes | |
| 90 | +// into the user's project, so it asks first, like write_file. | |
| 91 | +func (call fetchContentCall) evented(ctx context.Context, s ui.Sink, in fetchContentInput, where string) (string, error) { | |
| 92 | + d := call.detector | |
| 93 | + ev := ui.ToolEvent{ | |
| 94 | + ID: ui.NextCallID(), | |
| 95 | + Tool: "fetch_content", | |
| 96 | + Title: "fetch_content: " + where, | |
| 97 | + Kind: "fetch", | |
| 98 | + Input: map[string]any{"url": in.URL, "output": in.Output}, | |
| 99 | + } | |
| 100 | + if in.Output != "" { | |
| 101 | + ev.Path = absolute(ui.Resolve(in.Output)) | |
| 102 | + } | |
| 103 | + s.ToolStart(ev) | |
| 104 | + if in.Output != "" && !s.Allow(ctx, ev) { | |
| 105 | + out := "[fetch_content rejected by the user]" | |
| 106 | + s.ToolEnd(ev.ID, ui.ToolResult{Output: out, Failed: true}) | |
| 107 | + return record(d, "fetch_content", where, out) | |
| 108 | + } | |
| 109 | + s.ToolRunning(ev.ID) | |
| 110 | + | |
| 111 | + out, failed := fetchContent(ctx, call.fetcher, in, ui.Resolve) | |
| 112 | + out = truncate(out) | |
| 113 | + s.ToolEnd(ev.ID, ui.ToolResult{Output: out, Failed: failed}) | |
| 114 | + return record(d, "fetch_content", where, out) | |
| 115 | +} | |
| 116 | + | |
| 117 | +// fetchContent does the work of the tool: fetch, convert, and return the | |
| 118 | +// Markdown or save it. resolve maps the output path to where it is written | |
| 119 | +// (the session's directory in an editor). | |
| 120 | +func fetchContent(ctx context.Context, f pageFetcher, in fetchContentInput, resolve func(string) string) (out string, failed bool) { | |
| 121 | + md, source, err := fetchMarkdown(ctx, f, in) | |
| 122 | + if err != nil { | |
| 123 | + return fetchFailure(err), true | |
| 124 | + } | |
| 125 | + return deliverMarkdown(md, source, in.Output, resolve) | |
| 126 | +} | |
| 127 | + | |
| 128 | +// fetchMarkdown fetches in.URL and converts it with the call's options; | |
| 129 | +// source is the page's final URL. | |
| 130 | +func fetchMarkdown(ctx context.Context, f pageFetcher, in fetchContentInput) (md string, source *url.URL, err error) { | |
| 131 | + opts, err := contentOptions(in) | |
| 132 | + if err != nil { | |
| 133 | + return "", nil, err | |
| 134 | + } | |
| 135 | + page, err := f.Fetch(ctx, in.URL) | |
| 136 | + if err != nil { | |
| 137 | + return "", nil, err | |
| 138 | + } | |
| 139 | + if opts.BaseURL == nil { | |
| 140 | + opts.BaseURL = page.URL | |
| 141 | + } | |
| 142 | + md, err = webmd.Convert(page.HTML, opts) | |
| 143 | + return md, page.URL, err | |
| 144 | +} | |
| 145 | + | |
| 146 | +// deliverMarkdown returns md, or saves it to output and returns a | |
| 147 | +// confirmation that does not carry it. An empty page is reported, not saved. | |
| 148 | +func deliverMarkdown(md string, source *url.URL, output string, resolve func(string) string) (out string, failed bool) { | |
| 149 | + switch { | |
| 150 | + case md == "": | |
| 151 | + out = fmt.Sprintf("[fetch_content: %s has no text content once scripts, navigation and ads are removed — nothing saved]", source) | |
| 152 | + case output == "": | |
| 153 | + out = md | |
| 154 | + default: | |
| 155 | + if err := saveMarkdown(resolve(output), md); err != nil { | |
| 156 | + return fetchFailure(err), true | |
| 157 | + } | |
| 158 | + out = fmt.Sprintf("[fetch_content: saved %d lines of Markdown (%d bytes) from %s to %s — the content is not shown]", | |
| 159 | + strings.Count(md, "\n"), len(md), source, output) | |
| 160 | + } | |
| 161 | + return out, false | |
| 162 | +} | |
| 163 | + | |
| 164 | +// contentOptions validates the optional arguments: an unknown table mode or | |
| 165 | +// a base URL that is not absolute is a usage error (code 3). | |
| 166 | +func contentOptions(in fetchContentInput) (webmd.Options, error) { | |
| 167 | + mode, err := webmd.ParseTableMode(in.Tables) | |
| 168 | + if err != nil { | |
| 169 | + return webmd.Options{}, err | |
| 170 | + } | |
| 171 | + opts := webmd.Options{Tables: mode, Strikethrough: in.Strikethrough, TaskList: in.TaskList} | |
| 172 | + if in.BaseURL != "" { | |
| 173 | + base, err := url.Parse(in.BaseURL) | |
| 174 | + if err != nil || !base.IsAbs() { | |
| 175 | + return webmd.Options{}, &webmd.Error{Code: webmd.CodeUsage, Msg: fmt.Sprintf("baseUrl %q is not an absolute URL", in.BaseURL)} | |
| 176 | + } | |
| 177 | + opts.BaseURL = base | |
| 178 | + } | |
| 179 | + return opts, nil | |
| 180 | +} | |
| 181 | + | |
| 182 | +// saveMarkdown writes md to path, creating the parent directories. A | |
| 183 | +// failure is a processing error (code 1). | |
| 184 | +func saveMarkdown(path, md string) error { | |
| 185 | + if dir := filepath.Dir(path); dir != "." { | |
| 186 | + if err := os.MkdirAll(dir, 0o755); err != nil { | |
| 187 | + return &webmd.Error{Code: webmd.CodeConversion, Msg: "cannot create " + dir, Err: err} | |
| 188 | + } | |
| 189 | + } | |
| 190 | + if err := os.WriteFile(path, []byte(md), 0o644); err != nil { | |
| 191 | + return &webmd.Error{Code: webmd.CodeConversion, Msg: "cannot write " + path, Err: err} | |
| 192 | + } | |
| 193 | + return nil | |
| 194 | +} | |
| 195 | + | |
| 196 | +// fetchFailure is the text a failure goes back to the model as. | |
| 197 | +func fetchFailure(err error) string { | |
| 198 | + code := webmd.CodeOf(err) | |
| 199 | + return fmt.Sprintf("[fetch_content: error %d (%s) — %v]", int(code), code, err) | |
| 200 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,200 @@ | |||
| 1 | +package tools | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "context" | ||
| 5 | + "fmt" | ||
| 6 | + "net/url" | ||
| 7 | + "os" | ||
| 8 | + "path/filepath" | ||
| 9 | + "strings" | ||
| 10 | + | ||
| 11 | + "mm/internal/detector" | ||
| 12 | + "mm/internal/spinner" | ||
| 13 | + "mm/internal/ui" | ||
| 14 | + "mm/internal/webmd" | ||
| 15 | + | ||
| 16 | + "github.com/firebase/genkit/go/ai" | ||
| 17 | + "github.com/firebase/genkit/go/genkit" | ||
| 18 | +) | ||
| 19 | + | ||
| 20 | +// --- fetch_content ----------------------------------------------------------- | ||
| 21 | +// | ||
| 22 | +// Downloads a web page and gives it back as Markdown (see internal/webmd for | ||
| 23 | +// the conversion rules). Not to be confused with an MCP `fetch` server: this | ||
| 24 | +// one is built in, honours robots.txt, and can save the page to a file | ||
| 25 | +// instead of returning it — "save it, don't show it" must not cost the | ||
| 26 | +// context window a whole page. | ||
| 27 | +// | ||
| 28 | +// Same contract as the other tools: a failure goes back AS TEXT, carrying | ||
| 29 | +// the error code of the fetch-content specification (1 conversion, 2 input, | ||
| 30 | +// 3 usage) so the model can tell "fix your call" from "the site is down". | ||
| 31 | + | ||
| 32 | +type fetchContentInput struct { | ||
| 33 | + URL string `json:"url" jsonschema_description:"The web page to fetch, e.g. https://example.com/page. A missing scheme means https."` | ||
| 34 | + Output string `json:"output,omitempty" jsonschema_description:"Optional. A file path (relative to the working directory) to SAVE the Markdown to. When set, the Markdown is written to that file and NOT returned: only a one-line confirmation comes back."` | ||
| 35 | + BaseURL string `json:"baseUrl,omitempty" jsonschema_description:"Optional. Absolute URL used to resolve relative links and images; defaults to the page's own URL."` | ||
| 36 | + Tables string `json:"tables,omitempty" jsonschema_description:"Optional. What to do with tables using rowspan/colspan: \"html\" (default) keeps them as raw HTML, \"flatten\" turns them into Markdown tables."` | ||
| 37 | + Strikethrough bool `json:"strikethrough,omitempty" jsonschema_description:"Optional. Render <del>/<s>/<strike> as ~~text~~ (default: plain text)."` | ||
| 38 | + TaskList bool `json:"taskList,omitempty" jsonschema_description:"Optional. Render list items starting with a checkbox as - [ ] / - [x] (default: checkbox dropped)."` | ||
| 39 | +} | ||
| 40 | + | ||
| 41 | +// pageFetcher is what fetch_content needs from webmd.Fetcher; tests give | ||
| 42 | +// it a fake so that no test touches the network. | ||
| 43 | +type pageFetcher interface { | ||
| 44 | + Fetch(ctx context.Context, rawURL string) (*webmd.Page, error) | ||
| 45 | +} | ||
| 46 | + | ||
| 47 | +// FetchContent declares the `fetch_content` tool over f. | ||
| 48 | +// | ||
| 49 | +// tools.FetchContent(g, d, webmd.NewFetcher("mm/"+version.Short())) | ||
| 50 | +func FetchContent(g *genkit.Genkit, d *detector.LoopDetector, f pageFetcher) ai.ToolRef { | ||
| 51 | + description := `Fetch a web page and return its content as Markdown (headings, lists, links, images, tables, code blocks). Scripts, styles, navigation, ads and cookie banners are removed. | ||
| 52 | +Use this tool whenever the user asks to fetch, get, retrieve, download or read the content of a web page or URL, in any language (for example "récupère le contenu de https://example.com") — do not use curl or wget through bash for that. | ||
| 53 | +To SAVE the page to a file, set "output" to the file name the user gave: the Markdown is written there and NOT returned. When the user says not to display it, do not show or summarise the content — just confirm where it was saved. | ||
| 54 | +Without "output", the Markdown comes back as the result (truncated when very long). | ||
| 55 | +The site's robots.txt is honoured: a page it forbids is not fetched. | ||
| 56 | +Failures come back as "[fetch_content: error N (…) — reason]": 1 = the page could not be converted (not HTML, too large), 2 = the page could not be fetched (unreachable, HTTP error, forbidden by robots.txt), 3 = invalid argument — fix the call.` | ||
| 57 | + | ||
| 58 | + return genkit.DefineTool(g, "fetch_content", description, | ||
| 59 | + func(tctx *ai.ToolContext, in fetchContentInput) (string, error) { | ||
| 60 | + where := in.URL | ||
| 61 | + if in.Output != "" { | ||
| 62 | + where += " → " + in.Output | ||
| 63 | + } | ||
| 64 | + if s := ui.ActiveSink(); s != nil { | ||
| 65 | + return fetchContentCall{fetcher: f, detector: d}.evented(tctx, s, in, where) | ||
| 66 | + } | ||
| 67 | + | ||
| 68 | + announce("🌐 fetch_content: " + where) | ||
| 69 | + // A page can take seconds to come: same reasoning as bash's | ||
| 70 | + // "Running" label, a slow site must not look like a hung agent. | ||
| 71 | + spinner.Start("Fetching") | ||
| 72 | + out, _ := fetchContent(tctx, f, in, ui.Resolve) | ||
| 73 | + out = truncate(out) | ||
| 74 | + // With `output`, out is the confirmation line: the page itself is | ||
| 75 | + // not echoed, which is what "save it, don't show it" asks for. | ||
| 76 | + spinner.Suspend(func() { echoOutput(out) }) | ||
| 77 | + spinner.Start("Thinking") | ||
| 78 | + return record(d, "fetch_content", where, out) | ||
| 79 | + }) | ||
| 80 | +} | ||
| 81 | + | ||
| 82 | +// fetchContentCall is what one call of the tool works with. | ||
| 83 | +type fetchContentCall struct { | ||
| 84 | + fetcher pageFetcher | ||
| 85 | + detector *detector.LoopDetector | ||
| 86 | +} | ||
| 87 | + | ||
| 88 | +// evented is `fetch_content` for an event-consuming front end (ACP, the | ||
| 89 | +// window). Fetching alone is not gated, like reading a file; saving writes | ||
| 90 | +// into the user's project, so it asks first, like write_file. | ||
| 91 | +func (call fetchContentCall) evented(ctx context.Context, s ui.Sink, in fetchContentInput, where string) (string, error) { | ||
| 92 | + d := call.detector | ||
| 93 | + ev := ui.ToolEvent{ | ||
| 94 | + ID: ui.NextCallID(), | ||
| 95 | + Tool: "fetch_content", | ||
| 96 | + Title: "fetch_content: " + where, | ||
| 97 | + Kind: "fetch", | ||
| 98 | + Input: map[string]any{"url": in.URL, "output": in.Output}, | ||
| 99 | + } | ||
| 100 | + if in.Output != "" { | ||
| 101 | + ev.Path = absolute(ui.Resolve(in.Output)) | ||
| 102 | + } | ||
| 103 | + s.ToolStart(ev) | ||
| 104 | + if in.Output != "" && !s.Allow(ctx, ev) { | ||
| 105 | + out := "[fetch_content rejected by the user]" | ||
| 106 | + s.ToolEnd(ev.ID, ui.ToolResult{Output: out, Failed: true}) | ||
| 107 | + return record(d, "fetch_content", where, out) | ||
| 108 | + } | ||
| 109 | + s.ToolRunning(ev.ID) | ||
| 110 | + | ||
| 111 | + out, failed := fetchContent(ctx, call.fetcher, in, ui.Resolve) | ||
| 112 | + out = truncate(out) | ||
| 113 | + s.ToolEnd(ev.ID, ui.ToolResult{Output: out, Failed: failed}) | ||
| 114 | + return record(d, "fetch_content", where, out) | ||
| 115 | +} | ||
| 116 | + | ||
| 117 | +// fetchContent does the work of the tool: fetch, convert, and return the | ||
| 118 | +// Markdown or save it. resolve maps the output path to where it is written | ||
| 119 | +// (the session's directory in an editor). | ||
| 120 | +func fetchContent(ctx context.Context, f pageFetcher, in fetchContentInput, resolve func(string) string) (out string, failed bool) { | ||
| 121 | + md, source, err := fetchMarkdown(ctx, f, in) | ||
| 122 | + if err != nil { | ||
| 123 | + return fetchFailure(err), true | ||
| 124 | + } | ||
| 125 | + return deliverMarkdown(md, source, in.Output, resolve) | ||
| 126 | +} | ||
| 127 | + | ||
| 128 | +// fetchMarkdown fetches in.URL and converts it with the call's options; | ||
| 129 | +// source is the page's final URL. | ||
| 130 | +func fetchMarkdown(ctx context.Context, f pageFetcher, in fetchContentInput) (md string, source *url.URL, err error) { | ||
| 131 | + opts, err := contentOptions(in) | ||
| 132 | + if err != nil { | ||
| 133 | + return "", nil, err | ||
| 134 | + } | ||
| 135 | + page, err := f.Fetch(ctx, in.URL) | ||
| 136 | + if err != nil { | ||
| 137 | + return "", nil, err | ||
| 138 | + } | ||
| 139 | + if opts.BaseURL == nil { | ||
| 140 | + opts.BaseURL = page.URL | ||
| 141 | + } | ||
| 142 | + md, err = webmd.Convert(page.HTML, opts) | ||
| 143 | + return md, page.URL, err | ||
| 144 | +} | ||
| 145 | + | ||
| 146 | +// deliverMarkdown returns md, or saves it to output and returns a | ||
| 147 | +// confirmation that does not carry it. An empty page is reported, not saved. | ||
| 148 | +func deliverMarkdown(md string, source *url.URL, output string, resolve func(string) string) (out string, failed bool) { | ||
| 149 | + switch { | ||
| 150 | + case md == "": | ||
| 151 | + out = fmt.Sprintf("[fetch_content: %s has no text content once scripts, navigation and ads are removed — nothing saved]", source) | ||
| 152 | + case output == "": | ||
| 153 | + out = md | ||
| 154 | + default: | ||
| 155 | + if err := saveMarkdown(resolve(output), md); err != nil { | ||
| 156 | + return fetchFailure(err), true | ||
| 157 | + } | ||
| 158 | + out = fmt.Sprintf("[fetch_content: saved %d lines of Markdown (%d bytes) from %s to %s — the content is not shown]", | ||
| 159 | + strings.Count(md, "\n"), len(md), source, output) | ||
| 160 | + } | ||
| 161 | + return out, false | ||
| 162 | +} | ||
| 163 | + | ||
| 164 | +// contentOptions validates the optional arguments: an unknown table mode or | ||
| 165 | +// a base URL that is not absolute is a usage error (code 3). | ||
| 166 | +func contentOptions(in fetchContentInput) (webmd.Options, error) { | ||
| 167 | + mode, err := webmd.ParseTableMode(in.Tables) | ||
| 168 | + if err != nil { | ||
| 169 | + return webmd.Options{}, err | ||
| 170 | + } | ||
| 171 | + opts := webmd.Options{Tables: mode, Strikethrough: in.Strikethrough, TaskList: in.TaskList} | ||
| 172 | + if in.BaseURL != "" { | ||
| 173 | + base, err := url.Parse(in.BaseURL) | ||
| 174 | + if err != nil || !base.IsAbs() { | ||
| 175 | + return webmd.Options{}, &webmd.Error{Code: webmd.CodeUsage, Msg: fmt.Sprintf("baseUrl %q is not an absolute URL", in.BaseURL)} | ||
| 176 | + } | ||
| 177 | + opts.BaseURL = base | ||
| 178 | + } | ||
| 179 | + return opts, nil | ||
| 180 | +} | ||
| 181 | + | ||
| 182 | +// saveMarkdown writes md to path, creating the parent directories. A | ||
| 183 | +// failure is a processing error (code 1). | ||
| 184 | +func saveMarkdown(path, md string) error { | ||
| 185 | + if dir := filepath.Dir(path); dir != "." { | ||
| 186 | + if err := os.MkdirAll(dir, 0o755); err != nil { | ||
| 187 | + return &webmd.Error{Code: webmd.CodeConversion, Msg: "cannot create " + dir, Err: err} | ||
| 188 | + } | ||
| 189 | + } | ||
| 190 | + if err := os.WriteFile(path, []byte(md), 0o644); err != nil { | ||
| 191 | + return &webmd.Error{Code: webmd.CodeConversion, Msg: "cannot write " + path, Err: err} | ||
| 192 | + } | ||
| 193 | + return nil | ||
| 194 | +} | ||
| 195 | + | ||
| 196 | +// fetchFailure is the text a failure goes back to the model as. | ||
| 197 | +func fetchFailure(err error) string { | ||
| 198 | + code := webmd.CodeOf(err) | ||
| 199 | + return fmt.Sprintf("[fetch_content: error %d (%s) — %v]", int(code), code, err) | ||
| 200 | +} | ||
added
internal/tools/fetchcontent_test.go +191 -0 | new file mode 100644 | ||
| @@ -0,0 +1,191 @@ | ||
| 1 | +package tools | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "context" | |
| 5 | + "net/url" | |
| 6 | + "os" | |
| 7 | + "path/filepath" | |
| 8 | + "strings" | |
| 9 | + "testing" | |
| 10 | + | |
| 11 | + "mm/internal/detector" | |
| 12 | + "mm/internal/ui" | |
| 13 | + "mm/internal/webmd" | |
| 14 | +) | |
| 15 | + | |
| 16 | +// fakeFetcher serves one canned page, or one canned error, and remembers | |
| 17 | +// what it was asked for. | |
| 18 | +type fakeFetcher struct { | |
| 19 | + html string | |
| 20 | + err error | |
| 21 | + asked []string | |
| 22 | +} | |
| 23 | + | |
| 24 | +func (f *fakeFetcher) Fetch(_ context.Context, rawURL string) (*webmd.Page, error) { | |
| 25 | + f.asked = append(f.asked, rawURL) | |
| 26 | + if f.err != nil { | |
| 27 | + return nil, f.err | |
| 28 | + } | |
| 29 | + u, _ := url.Parse("https://example.com/docs/") | |
| 30 | + return &webmd.Page{URL: u, HTML: []byte(f.html), Charset: "utf-8"}, nil | |
| 31 | +} | |
| 32 | + | |
| 33 | +func noResolve(p string) string { return p } | |
| 34 | + | |
| 35 | +func TestFetchContentReturnsMarkdown(t *testing.T) { | |
| 36 | + f := &fakeFetcher{html: `<h1>Hello</h1><p>See <a href="guide.html">the guide</a>.</p>`} | |
| 37 | + out, failed := fetchContent(context.Background(), f, fetchContentInput{URL: "https//example.com/docs/"}, noResolve) | |
| 38 | + if failed { | |
| 39 | + t.Fatalf("failed: %s", out) | |
| 40 | + } | |
| 41 | + if want := "# Hello\n\nSee [the guide](https://example.com/docs/guide.html).\n"; out != want { | |
| 42 | + t.Errorf("got %q, want %q", out, want) | |
| 43 | + } | |
| 44 | + if f.asked[0] != "https//example.com/docs/" { | |
| 45 | + t.Errorf("the URL is handed to the fetcher as typed (it normalises it): %q", f.asked[0]) | |
| 46 | + } | |
| 47 | +} | |
| 48 | + | |
| 49 | +func TestFetchContentSavesWithoutReturningThePage(t *testing.T) { | |
| 50 | + dir := t.TempDir() | |
| 51 | + f := &fakeFetcher{html: "<h1>Hello</h1><p>secret body</p>"} | |
| 52 | + in := fetchContentInput{URL: "https://hello.com", Output: "out/hello.txt"} | |
| 53 | + out, failed := fetchContent(context.Background(), f, in, func(p string) string { return filepath.Join(dir, p) }) | |
| 54 | + if failed { | |
| 55 | + t.Fatalf("failed: %s", out) | |
| 56 | + } | |
| 57 | + saved, err := os.ReadFile(filepath.Join(dir, "out", "hello.txt")) | |
| 58 | + if err != nil { | |
| 59 | + t.Fatal(err) | |
| 60 | + } | |
| 61 | + if string(saved) != "# Hello\n\nsecret body\n" { | |
| 62 | + t.Errorf("saved %q", saved) | |
| 63 | + } | |
| 64 | + if strings.Contains(out, "secret body") || !strings.Contains(out, "saved 3 lines") || !strings.Contains(out, "out/hello.txt") { | |
| 65 | + t.Errorf("confirmation %q must name the file and not carry the page", out) | |
| 66 | + } | |
| 67 | +} | |
| 68 | + | |
| 69 | +func TestFetchContentOptionsReachTheConverter(t *testing.T) { | |
| 70 | + f := &fakeFetcher{html: `<p><del>old</del></p><ul><li><input type="checkbox" checked> done</li></ul><table><tr><td colspan="2">x</td></tr></table>`} | |
| 71 | + in := fetchContentInput{URL: "https://x.org", Strikethrough: true, TaskList: true, Tables: "flatten"} | |
| 72 | + out, _ := fetchContent(context.Background(), f, in, noResolve) | |
| 73 | + for _, want := range []string{"~~old~~", "- [x] done", "| x | x |"} { | |
| 74 | + if !strings.Contains(out, want) { | |
| 75 | + t.Errorf("%q missing from %q", want, out) | |
| 76 | + } | |
| 77 | + } | |
| 78 | +} | |
| 79 | + | |
| 80 | +func TestFetchContentBaseURLOverridesThePageURL(t *testing.T) { | |
| 81 | + f := &fakeFetcher{html: `<img src="a.png" alt="A">`} | |
| 82 | + out, _ := fetchContent(context.Background(), f, fetchContentInput{URL: "https://x.org", BaseURL: "https://cdn.example.net/img/"}, noResolve) | |
| 83 | + if out != "\n" { | |
| 84 | + t.Errorf("got %q", out) | |
| 85 | + } | |
| 86 | +} | |
| 87 | + | |
| 88 | +func TestFetchContentFailuresCarryTheirCode(t *testing.T) { | |
| 89 | + cases := []struct { | |
| 90 | + name string | |
| 91 | + f *fakeFetcher | |
| 92 | + in fetchContentInput | |
| 93 | + want string | |
| 94 | + }{ | |
| 95 | + {"unknown table mode", &fakeFetcher{}, fetchContentInput{URL: "https://x.org", Tables: "grid"}, "error 3 (usage)"}, | |
| 96 | + {"relative base URL", &fakeFetcher{}, fetchContentInput{URL: "https://x.org", BaseURL: "/docs"}, "error 3 (usage)"}, | |
| 97 | + {"unreachable page", &fakeFetcher{err: &webmd.Error{Code: webmd.CodeInput, Msg: "HTTP 404 Not Found for https://x.org"}}, | |
| 98 | + fetchContentInput{URL: "https://x.org"}, "error 2 (input) — HTTP 404"}, | |
| 99 | + {"not HTML", &fakeFetcher{err: &webmd.Error{Code: webmd.CodeConversion, Msg: "not an HTML page"}}, | |
| 100 | + fetchContentInput{URL: "https://x.org"}, "error 1 (conversion)"}, | |
| 101 | + } | |
| 102 | + for _, tc := range cases { | |
| 103 | + t.Run(tc.name, func(t *testing.T) { | |
| 104 | + out, failed := fetchContent(context.Background(), tc.f, tc.in, noResolve) | |
| 105 | + if !failed || !strings.HasPrefix(out, "[fetch_content: ") || !strings.Contains(out, tc.want) { | |
| 106 | + t.Errorf("got %q (failed=%v), want it to contain %q", out, failed, tc.want) | |
| 107 | + } | |
| 108 | + }) | |
| 109 | + } | |
| 110 | +} | |
| 111 | + | |
| 112 | +func TestFetchContentUnwritableOutputIsAConversionError(t *testing.T) { | |
| 113 | + blocker := filepath.Join(t.TempDir(), "file") | |
| 114 | + if err := os.WriteFile(blocker, nil, 0o644); err != nil { | |
| 115 | + t.Fatal(err) | |
| 116 | + } | |
| 117 | + f := &fakeFetcher{html: "<p>x</p>"} | |
| 118 | + out, failed := fetchContent(context.Background(), f, fetchContentInput{URL: "https://x.org", Output: filepath.Join(blocker, "sub", "a.md")}, noResolve) | |
| 119 | + if !failed || !strings.Contains(out, "error 1 (conversion)") { | |
| 120 | + t.Errorf("got %q", out) | |
| 121 | + } | |
| 122 | +} | |
| 123 | + | |
| 124 | +func TestFetchContentEmptyPageSavesNothing(t *testing.T) { | |
| 125 | + dir := t.TempDir() | |
| 126 | + f := &fakeFetcher{html: "<nav>menu</nav><script>x()</script>"} | |
| 127 | + out, failed := fetchContent(context.Background(), f, fetchContentInput{URL: "https://x.org", Output: "e.md"}, func(p string) string { return filepath.Join(dir, p) }) | |
| 128 | + if failed || !strings.Contains(out, "no text content") { | |
| 129 | + t.Errorf("got %q (failed=%v)", out, failed) | |
| 130 | + } | |
| 131 | + if _, err := os.Stat(filepath.Join(dir, "e.md")); !os.IsNotExist(err) { | |
| 132 | + t.Error("an empty page must not create the file") | |
| 133 | + } | |
| 134 | +} | |
| 135 | + | |
| 136 | +// recordingSink is a ui.Sink that answers Allow with allow and remembers | |
| 137 | +// what it was told. | |
| 138 | +type recordingSink struct { | |
| 139 | + allow bool | |
| 140 | + dir string | |
| 141 | + asked int | |
| 142 | + started ui.ToolEvent | |
| 143 | + ended ui.ToolResult | |
| 144 | +} | |
| 145 | + | |
| 146 | +func (k *recordingSink) Text(string) {} | |
| 147 | +func (k *recordingSink) Reasoning(string) {} | |
| 148 | +func (k *recordingSink) ToolStart(ev ui.ToolEvent) { k.started = ev } | |
| 149 | +func (k *recordingSink) ToolRunning(string) {} | |
| 150 | +func (k *recordingSink) ToolEnd(_ string, r ui.ToolResult) { k.ended = r } | |
| 151 | +func (k *recordingSink) WorkDir() string { return k.dir } | |
| 152 | +func (k *recordingSink) Allow(context.Context, ui.ToolEvent) bool { | |
| 153 | + k.asked++ | |
| 154 | + return k.allow | |
| 155 | +} | |
| 156 | + | |
| 157 | +func TestFetchContentEventedAsksBeforeSaving(t *testing.T) { | |
| 158 | + dir := t.TempDir() | |
| 159 | + sink := &recordingSink{allow: false, dir: dir} | |
| 160 | + ui.SetActive(sink) | |
| 161 | + t.Cleanup(func() { ui.SetActive(nil) }) | |
| 162 | + call := fetchContentCall{fetcher: &fakeFetcher{html: "<p>x</p>"}, detector: detector.NewLoopDetector(10, 3)} | |
| 163 | + | |
| 164 | + out, err := call.evented(context.Background(), sink, fetchContentInput{URL: "https://x.org", Output: "x.md"}, "https://x.org → x.md") | |
| 165 | + if err != nil || out != "[fetch_content rejected by the user]" || !sink.ended.Failed { | |
| 166 | + t.Fatalf("rejected save: %q %v %+v", out, err, sink.ended) | |
| 167 | + } | |
| 168 | + if _, statErr := os.Stat(filepath.Join(dir, "x.md")); !os.IsNotExist(statErr) { | |
| 169 | + t.Error("a rejected save wrote the file") | |
| 170 | + } | |
| 171 | + if sink.started.Kind != "fetch" || sink.started.Path != filepath.Join(dir, "x.md") { | |
| 172 | + t.Errorf("event %+v", sink.started) | |
| 173 | + } | |
| 174 | + | |
| 175 | + sink.allow = true | |
| 176 | + if out, _ := call.evented(context.Background(), sink, fetchContentInput{URL: "https://x.org", Output: "x.md"}, "save"); !strings.Contains(out, "saved") { | |
| 177 | + t.Errorf("allowed save: %q", out) | |
| 178 | + } | |
| 179 | + if _, statErr := os.Stat(filepath.Join(dir, "x.md")); statErr != nil { | |
| 180 | + t.Errorf("the file was not written in the session's directory: %v", statErr) | |
| 181 | + } | |
| 182 | +} | |
| 183 | + | |
| 184 | +func TestFetchContentEventedDoesNotAskToRead(t *testing.T) { | |
| 185 | + sink := &recordingSink{} | |
| 186 | + call := fetchContentCall{fetcher: &fakeFetcher{html: "<p>x</p>"}, detector: detector.NewLoopDetector(10, 3)} | |
| 187 | + out, _ := call.evented(context.Background(), sink, fetchContentInput{URL: "https://x.org"}, "https://x.org") | |
| 188 | + if sink.asked != 0 || out != "x\n" || sink.ended.Output != "x\n" { | |
| 189 | + t.Errorf("asked %d, out %q, ended %+v", sink.asked, out, sink.ended) | |
| 190 | + } | |
| 191 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,191 @@ | |||
| 1 | +package tools | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "context" | ||
| 5 | + "net/url" | ||
| 6 | + "os" | ||
| 7 | + "path/filepath" | ||
| 8 | + "strings" | ||
| 9 | + "testing" | ||
| 10 | + | ||
| 11 | + "mm/internal/detector" | ||
| 12 | + "mm/internal/ui" | ||
| 13 | + "mm/internal/webmd" | ||
| 14 | +) | ||
| 15 | + | ||
| 16 | +// fakeFetcher serves one canned page, or one canned error, and remembers | ||
| 17 | +// what it was asked for. | ||
| 18 | +type fakeFetcher struct { | ||
| 19 | + html string | ||
| 20 | + err error | ||
| 21 | + asked []string | ||
| 22 | +} | ||
| 23 | + | ||
| 24 | +func (f *fakeFetcher) Fetch(_ context.Context, rawURL string) (*webmd.Page, error) { | ||
| 25 | + f.asked = append(f.asked, rawURL) | ||
| 26 | + if f.err != nil { | ||
| 27 | + return nil, f.err | ||
| 28 | + } | ||
| 29 | + u, _ := url.Parse("https://example.com/docs/") | ||
| 30 | + return &webmd.Page{URL: u, HTML: []byte(f.html), Charset: "utf-8"}, nil | ||
| 31 | +} | ||
| 32 | + | ||
| 33 | +func noResolve(p string) string { return p } | ||
| 34 | + | ||
| 35 | +func TestFetchContentReturnsMarkdown(t *testing.T) { | ||
| 36 | + f := &fakeFetcher{html: `<h1>Hello</h1><p>See <a href="guide.html">the guide</a>.</p>`} | ||
| 37 | + out, failed := fetchContent(context.Background(), f, fetchContentInput{URL: "https//example.com/docs/"}, noResolve) | ||
| 38 | + if failed { | ||
| 39 | + t.Fatalf("failed: %s", out) | ||
| 40 | + } | ||
| 41 | + if want := "# Hello\n\nSee [the guide](https://example.com/docs/guide.html).\n"; out != want { | ||
| 42 | + t.Errorf("got %q, want %q", out, want) | ||
| 43 | + } | ||
| 44 | + if f.asked[0] != "https//example.com/docs/" { | ||
| 45 | + t.Errorf("the URL is handed to the fetcher as typed (it normalises it): %q", f.asked[0]) | ||
| 46 | + } | ||
| 47 | +} | ||
| 48 | + | ||
| 49 | +func TestFetchContentSavesWithoutReturningThePage(t *testing.T) { | ||
| 50 | + dir := t.TempDir() | ||
| 51 | + f := &fakeFetcher{html: "<h1>Hello</h1><p>secret body</p>"} | ||
| 52 | + in := fetchContentInput{URL: "https://hello.com", Output: "out/hello.txt"} | ||
| 53 | + out, failed := fetchContent(context.Background(), f, in, func(p string) string { return filepath.Join(dir, p) }) | ||
| 54 | + if failed { | ||
| 55 | + t.Fatalf("failed: %s", out) | ||
| 56 | + } | ||
| 57 | + saved, err := os.ReadFile(filepath.Join(dir, "out", "hello.txt")) | ||
| 58 | + if err != nil { | ||
| 59 | + t.Fatal(err) | ||
| 60 | + } | ||
| 61 | + if string(saved) != "# Hello\n\nsecret body\n" { | ||
| 62 | + t.Errorf("saved %q", saved) | ||
| 63 | + } | ||
| 64 | + if strings.Contains(out, "secret body") || !strings.Contains(out, "saved 3 lines") || !strings.Contains(out, "out/hello.txt") { | ||
| 65 | + t.Errorf("confirmation %q must name the file and not carry the page", out) | ||
| 66 | + } | ||
| 67 | +} | ||
| 68 | + | ||
| 69 | +func TestFetchContentOptionsReachTheConverter(t *testing.T) { | ||
| 70 | + f := &fakeFetcher{html: `<p><del>old</del></p><ul><li><input type="checkbox" checked> done</li></ul><table><tr><td colspan="2">x</td></tr></table>`} | ||
| 71 | + in := fetchContentInput{URL: "https://x.org", Strikethrough: true, TaskList: true, Tables: "flatten"} | ||
| 72 | + out, _ := fetchContent(context.Background(), f, in, noResolve) | ||
| 73 | + for _, want := range []string{"~~old~~", "- [x] done", "| x | x |"} { | ||
| 74 | + if !strings.Contains(out, want) { | ||
| 75 | + t.Errorf("%q missing from %q", want, out) | ||
| 76 | + } | ||
| 77 | + } | ||
| 78 | +} | ||
| 79 | + | ||
| 80 | +func TestFetchContentBaseURLOverridesThePageURL(t *testing.T) { | ||
| 81 | + f := &fakeFetcher{html: `<img src="a.png" alt="A">`} | ||
| 82 | + out, _ := fetchContent(context.Background(), f, fetchContentInput{URL: "https://x.org", BaseURL: "https://cdn.example.net/img/"}, noResolve) | ||
| 83 | + if out != "\n" { | ||
| 84 | + t.Errorf("got %q", out) | ||
| 85 | + } | ||
| 86 | +} | ||
| 87 | + | ||
| 88 | +func TestFetchContentFailuresCarryTheirCode(t *testing.T) { | ||
| 89 | + cases := []struct { | ||
| 90 | + name string | ||
| 91 | + f *fakeFetcher | ||
| 92 | + in fetchContentInput | ||
| 93 | + want string | ||
| 94 | + }{ | ||
| 95 | + {"unknown table mode", &fakeFetcher{}, fetchContentInput{URL: "https://x.org", Tables: "grid"}, "error 3 (usage)"}, | ||
| 96 | + {"relative base URL", &fakeFetcher{}, fetchContentInput{URL: "https://x.org", BaseURL: "/docs"}, "error 3 (usage)"}, | ||
| 97 | + {"unreachable page", &fakeFetcher{err: &webmd.Error{Code: webmd.CodeInput, Msg: "HTTP 404 Not Found for https://x.org"}}, | ||
| 98 | + fetchContentInput{URL: "https://x.org"}, "error 2 (input) — HTTP 404"}, | ||
| 99 | + {"not HTML", &fakeFetcher{err: &webmd.Error{Code: webmd.CodeConversion, Msg: "not an HTML page"}}, | ||
| 100 | + fetchContentInput{URL: "https://x.org"}, "error 1 (conversion)"}, | ||
| 101 | + } | ||
| 102 | + for _, tc := range cases { | ||
| 103 | + t.Run(tc.name, func(t *testing.T) { | ||
| 104 | + out, failed := fetchContent(context.Background(), tc.f, tc.in, noResolve) | ||
| 105 | + if !failed || !strings.HasPrefix(out, "[fetch_content: ") || !strings.Contains(out, tc.want) { | ||
| 106 | + t.Errorf("got %q (failed=%v), want it to contain %q", out, failed, tc.want) | ||
| 107 | + } | ||
| 108 | + }) | ||
| 109 | + } | ||
| 110 | +} | ||
| 111 | + | ||
| 112 | +func TestFetchContentUnwritableOutputIsAConversionError(t *testing.T) { | ||
| 113 | + blocker := filepath.Join(t.TempDir(), "file") | ||
| 114 | + if err := os.WriteFile(blocker, nil, 0o644); err != nil { | ||
| 115 | + t.Fatal(err) | ||
| 116 | + } | ||
| 117 | + f := &fakeFetcher{html: "<p>x</p>"} | ||
| 118 | + out, failed := fetchContent(context.Background(), f, fetchContentInput{URL: "https://x.org", Output: filepath.Join(blocker, "sub", "a.md")}, noResolve) | ||
| 119 | + if !failed || !strings.Contains(out, "error 1 (conversion)") { | ||
| 120 | + t.Errorf("got %q", out) | ||
| 121 | + } | ||
| 122 | +} | ||
| 123 | + | ||
| 124 | +func TestFetchContentEmptyPageSavesNothing(t *testing.T) { | ||
| 125 | + dir := t.TempDir() | ||
| 126 | + f := &fakeFetcher{html: "<nav>menu</nav><script>x()</script>"} | ||
| 127 | + out, failed := fetchContent(context.Background(), f, fetchContentInput{URL: "https://x.org", Output: "e.md"}, func(p string) string { return filepath.Join(dir, p) }) | ||
| 128 | + if failed || !strings.Contains(out, "no text content") { | ||
| 129 | + t.Errorf("got %q (failed=%v)", out, failed) | ||
| 130 | + } | ||
| 131 | + if _, err := os.Stat(filepath.Join(dir, "e.md")); !os.IsNotExist(err) { | ||
| 132 | + t.Error("an empty page must not create the file") | ||
| 133 | + } | ||
| 134 | +} | ||
| 135 | + | ||
| 136 | +// recordingSink is a ui.Sink that answers Allow with allow and remembers | ||
| 137 | +// what it was told. | ||
| 138 | +type recordingSink struct { | ||
| 139 | + allow bool | ||
| 140 | + dir string | ||
| 141 | + asked int | ||
| 142 | + started ui.ToolEvent | ||
| 143 | + ended ui.ToolResult | ||
| 144 | +} | ||
| 145 | + | ||
| 146 | +func (k *recordingSink) Text(string) {} | ||
| 147 | +func (k *recordingSink) Reasoning(string) {} | ||
| 148 | +func (k *recordingSink) ToolStart(ev ui.ToolEvent) { k.started = ev } | ||
| 149 | +func (k *recordingSink) ToolRunning(string) {} | ||
| 150 | +func (k *recordingSink) ToolEnd(_ string, r ui.ToolResult) { k.ended = r } | ||
| 151 | +func (k *recordingSink) WorkDir() string { return k.dir } | ||
| 152 | +func (k *recordingSink) Allow(context.Context, ui.ToolEvent) bool { | ||
| 153 | + k.asked++ | ||
| 154 | + return k.allow | ||
| 155 | +} | ||
| 156 | + | ||
| 157 | +func TestFetchContentEventedAsksBeforeSaving(t *testing.T) { | ||
| 158 | + dir := t.TempDir() | ||
| 159 | + sink := &recordingSink{allow: false, dir: dir} | ||
| 160 | + ui.SetActive(sink) | ||
| 161 | + t.Cleanup(func() { ui.SetActive(nil) }) | ||
| 162 | + call := fetchContentCall{fetcher: &fakeFetcher{html: "<p>x</p>"}, detector: detector.NewLoopDetector(10, 3)} | ||
| 163 | + | ||
| 164 | + out, err := call.evented(context.Background(), sink, fetchContentInput{URL: "https://x.org", Output: "x.md"}, "https://x.org → x.md") | ||
| 165 | + if err != nil || out != "[fetch_content rejected by the user]" || !sink.ended.Failed { | ||
| 166 | + t.Fatalf("rejected save: %q %v %+v", out, err, sink.ended) | ||
| 167 | + } | ||
| 168 | + if _, statErr := os.Stat(filepath.Join(dir, "x.md")); !os.IsNotExist(statErr) { | ||
| 169 | + t.Error("a rejected save wrote the file") | ||
| 170 | + } | ||
| 171 | + if sink.started.Kind != "fetch" || sink.started.Path != filepath.Join(dir, "x.md") { | ||
| 172 | + t.Errorf("event %+v", sink.started) | ||
| 173 | + } | ||
| 174 | + | ||
| 175 | + sink.allow = true | ||
| 176 | + if out, _ := call.evented(context.Background(), sink, fetchContentInput{URL: "https://x.org", Output: "x.md"}, "save"); !strings.Contains(out, "saved") { | ||
| 177 | + t.Errorf("allowed save: %q", out) | ||
| 178 | + } | ||
| 179 | + if _, statErr := os.Stat(filepath.Join(dir, "x.md")); statErr != nil { | ||
| 180 | + t.Errorf("the file was not written in the session's directory: %v", statErr) | ||
| 181 | + } | ||
| 182 | +} | ||
| 183 | + | ||
| 184 | +func TestFetchContentEventedDoesNotAskToRead(t *testing.T) { | ||
| 185 | + sink := &recordingSink{} | ||
| 186 | + call := fetchContentCall{fetcher: &fakeFetcher{html: "<p>x</p>"}, detector: detector.NewLoopDetector(10, 3)} | ||
| 187 | + out, _ := call.evented(context.Background(), sink, fetchContentInput{URL: "https://x.org"}, "https://x.org") | ||
| 188 | + if sink.asked != 0 || out != "x\n" || sink.ended.Output != "x\n" { | ||
| 189 | + t.Errorf("asked %d, out %q, ended %+v", sink.asked, out, sink.ended) | ||
| 190 | + } | ||
| 191 | +} | ||
added
internal/webmd/blocks.go +86 -0 | new file mode 100644 | ||
| @@ -0,0 +1,86 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "strings" | |
| 5 | + | |
| 6 | + "golang.org/x/net/html" | |
| 7 | + "golang.org/x/net/html/atom" | |
| 8 | +) | |
| 9 | + | |
| 10 | +// Renderers of the block elements that need more than their children. | |
| 11 | + | |
| 12 | +func heading(level int, text string) string { | |
| 13 | + text = strings.TrimSpace(strings.ReplaceAll(text, "\n", " ")) | |
| 14 | + if text == "" { | |
| 15 | + return "" | |
| 16 | + } | |
| 17 | + return strings.Repeat("#", level) + " " + text | |
| 18 | +} | |
| 19 | + | |
| 20 | +// codeBlock renders <pre> as a fenced block, its text untouched. The fence | |
| 21 | +// is longer than any backtick run inside, and its info string is the | |
| 22 | +// language of a "language-x" / "lang-x" class on the <pre> or its <code>. | |
| 23 | +func codeBlock(pre *html.Node) string { | |
| 24 | + code := strings.TrimRight(strings.TrimPrefix(textContent(pre), "\n"), "\n") | |
| 25 | + if strings.TrimSpace(code) == "" { | |
| 26 | + return "" | |
| 27 | + } | |
| 28 | + fence := strings.Repeat("`", max(3, longestRun(code, '`')+1)) | |
| 29 | + return fence + language(pre) + "\n" + code + "\n" + fence | |
| 30 | +} | |
| 31 | + | |
| 32 | +func language(pre *html.Node) string { | |
| 33 | + nodes := []*html.Node{pre} | |
| 34 | + for ch := pre.FirstChild; ch != nil; ch = ch.NextSibling { | |
| 35 | + if ch.DataAtom == atom.Code { | |
| 36 | + nodes = append(nodes, ch) | |
| 37 | + } | |
| 38 | + } | |
| 39 | + for _, n := range nodes { | |
| 40 | + for _, word := range strings.Fields(attr(n, "class")) { | |
| 41 | + if lang, ok := strings.CutPrefix(word, "language-"); ok { | |
| 42 | + return lang | |
| 43 | + } | |
| 44 | + if lang, ok := strings.CutPrefix(word, "lang-"); ok { | |
| 45 | + return lang | |
| 46 | + } | |
| 47 | + } | |
| 48 | + } | |
| 49 | + return "" | |
| 50 | +} | |
| 51 | + | |
| 52 | +// quote prefixes every line of md with "> ". | |
| 53 | +func quote(md string) string { | |
| 54 | + if md == "" { | |
| 55 | + return "" | |
| 56 | + } | |
| 57 | + lines := strings.Split(md, "\n") | |
| 58 | + for i, line := range lines { | |
| 59 | + if line == "" { | |
| 60 | + lines[i] = ">" | |
| 61 | + } else { | |
| 62 | + lines[i] = "> " + line | |
| 63 | + } | |
| 64 | + } | |
| 65 | + return strings.Join(lines, "\n") | |
| 66 | +} | |
| 67 | + | |
| 68 | +// textContent is the raw text under n — no escaping, no whitespace | |
| 69 | +// collapsing — with <br> as a newline. It is what code keeps. | |
| 70 | +func textContent(n *html.Node) string { | |
| 71 | + var b strings.Builder | |
| 72 | + var walk func(*html.Node) | |
| 73 | + walk = func(n *html.Node) { | |
| 74 | + switch { | |
| 75 | + case n.Type == html.TextNode: | |
| 76 | + b.WriteString(n.Data) | |
| 77 | + case n.DataAtom == atom.Br: | |
| 78 | + b.WriteString("\n") | |
| 79 | + } | |
| 80 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | |
| 81 | + walk(ch) | |
| 82 | + } | |
| 83 | + } | |
| 84 | + walk(n) | |
| 85 | + return b.String() | |
| 86 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,86 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "strings" | ||
| 5 | + | ||
| 6 | + "golang.org/x/net/html" | ||
| 7 | + "golang.org/x/net/html/atom" | ||
| 8 | +) | ||
| 9 | + | ||
| 10 | +// Renderers of the block elements that need more than their children. | ||
| 11 | + | ||
| 12 | +func heading(level int, text string) string { | ||
| 13 | + text = strings.TrimSpace(strings.ReplaceAll(text, "\n", " ")) | ||
| 14 | + if text == "" { | ||
| 15 | + return "" | ||
| 16 | + } | ||
| 17 | + return strings.Repeat("#", level) + " " + text | ||
| 18 | +} | ||
| 19 | + | ||
| 20 | +// codeBlock renders <pre> as a fenced block, its text untouched. The fence | ||
| 21 | +// is longer than any backtick run inside, and its info string is the | ||
| 22 | +// language of a "language-x" / "lang-x" class on the <pre> or its <code>. | ||
| 23 | +func codeBlock(pre *html.Node) string { | ||
| 24 | + code := strings.TrimRight(strings.TrimPrefix(textContent(pre), "\n"), "\n") | ||
| 25 | + if strings.TrimSpace(code) == "" { | ||
| 26 | + return "" | ||
| 27 | + } | ||
| 28 | + fence := strings.Repeat("`", max(3, longestRun(code, '`')+1)) | ||
| 29 | + return fence + language(pre) + "\n" + code + "\n" + fence | ||
| 30 | +} | ||
| 31 | + | ||
| 32 | +func language(pre *html.Node) string { | ||
| 33 | + nodes := []*html.Node{pre} | ||
| 34 | + for ch := pre.FirstChild; ch != nil; ch = ch.NextSibling { | ||
| 35 | + if ch.DataAtom == atom.Code { | ||
| 36 | + nodes = append(nodes, ch) | ||
| 37 | + } | ||
| 38 | + } | ||
| 39 | + for _, n := range nodes { | ||
| 40 | + for _, word := range strings.Fields(attr(n, "class")) { | ||
| 41 | + if lang, ok := strings.CutPrefix(word, "language-"); ok { | ||
| 42 | + return lang | ||
| 43 | + } | ||
| 44 | + if lang, ok := strings.CutPrefix(word, "lang-"); ok { | ||
| 45 | + return lang | ||
| 46 | + } | ||
| 47 | + } | ||
| 48 | + } | ||
| 49 | + return "" | ||
| 50 | +} | ||
| 51 | + | ||
| 52 | +// quote prefixes every line of md with "> ". | ||
| 53 | +func quote(md string) string { | ||
| 54 | + if md == "" { | ||
| 55 | + return "" | ||
| 56 | + } | ||
| 57 | + lines := strings.Split(md, "\n") | ||
| 58 | + for i, line := range lines { | ||
| 59 | + if line == "" { | ||
| 60 | + lines[i] = ">" | ||
| 61 | + } else { | ||
| 62 | + lines[i] = "> " + line | ||
| 63 | + } | ||
| 64 | + } | ||
| 65 | + return strings.Join(lines, "\n") | ||
| 66 | +} | ||
| 67 | + | ||
| 68 | +// textContent is the raw text under n — no escaping, no whitespace | ||
| 69 | +// collapsing — with <br> as a newline. It is what code keeps. | ||
| 70 | +func textContent(n *html.Node) string { | ||
| 71 | + var b strings.Builder | ||
| 72 | + var walk func(*html.Node) | ||
| 73 | + walk = func(n *html.Node) { | ||
| 74 | + switch { | ||
| 75 | + case n.Type == html.TextNode: | ||
| 76 | + b.WriteString(n.Data) | ||
| 77 | + case n.DataAtom == atom.Br: | ||
| 78 | + b.WriteString("\n") | ||
| 79 | + } | ||
| 80 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | ||
| 81 | + walk(ch) | ||
| 82 | + } | ||
| 83 | + } | ||
| 84 | + walk(n) | ||
| 85 | + return b.String() | ||
| 86 | +} | ||
added
internal/webmd/charset.go +131 -0 | new file mode 100644 | ||
| @@ -0,0 +1,131 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "bytes" | |
| 5 | + "mime" | |
| 6 | + "strings" | |
| 7 | + "unicode/utf8" | |
| 8 | + | |
| 9 | + "golang.org/x/net/html" | |
| 10 | + "golang.org/x/net/html/charset" | |
| 11 | + "golang.org/x/text/encoding" | |
| 12 | +) | |
| 13 | + | |
| 14 | +// metaScanBytes is how far into the page a <meta charset> is looked for. | |
| 15 | +// HTML requires it within the first 1024 bytes; real pages are sloppier. | |
| 16 | +const metaScanBytes = 4096 | |
| 17 | + | |
| 18 | +// toUTF8 decodes body to UTF-8 and names the encoding it came from. The | |
| 19 | +// encoding is, in order: a byte-order mark; the charset of the | |
| 20 | +// Content-Type header; a <meta charset> or <meta http-equiv> in the page; | |
| 21 | +// UTF-8 when the bytes are valid UTF-8; windows-1252 — the web's historical | |
| 22 | +// default — otherwise. An encoding name nobody knows is skipped. | |
| 23 | +func toUTF8(body []byte, contentType string) ([]byte, string, error) { | |
| 24 | + enc, name, body := detectEncoding(body, contentType) | |
| 25 | + if name == "utf-8" { | |
| 26 | + return body, name, nil | |
| 27 | + } | |
| 28 | + decoded, err := enc.NewDecoder().Bytes(body) | |
| 29 | + if err != nil { | |
| 30 | + return nil, "", errorf(CodeConversion, err, "cannot decode the page from %s", name) | |
| 31 | + } | |
| 32 | + return decoded, name, nil | |
| 33 | +} | |
| 34 | + | |
| 35 | +// detectEncoding returns the encoding of body, and body without its BOM. | |
| 36 | +func detectEncoding(body []byte, contentType string) (encoding.Encoding, string, []byte) { | |
| 37 | + label, bomSize := byteOrderMark(body) | |
| 38 | + if label == "" { | |
| 39 | + label = firstKnown(headerCharset(contentType), metaCharset(body)) | |
| 40 | + } | |
| 41 | + if label == "" { | |
| 42 | + label = "windows-1252" | |
| 43 | + if utf8.Valid(body) { | |
| 44 | + label = "utf-8" | |
| 45 | + } | |
| 46 | + } | |
| 47 | + enc, name := charset.Lookup(label) | |
| 48 | + return enc, name, body[bomSize:] | |
| 49 | +} | |
| 50 | + | |
| 51 | +// byteOrderMarks are checked in order; the UTF-8 one is not a prefix of | |
| 52 | +// the others, so the order only matters for readability. | |
| 53 | +var byteOrderMarks = []struct { | |
| 54 | + mark []byte | |
| 55 | + label string | |
| 56 | +}{ | |
| 57 | + {[]byte{0xEF, 0xBB, 0xBF}, "utf-8"}, | |
| 58 | + {[]byte{0xFF, 0xFE}, "utf-16le"}, | |
| 59 | + {[]byte{0xFE, 0xFF}, "utf-16be"}, | |
| 60 | +} | |
| 61 | + | |
| 62 | +// byteOrderMark returns the encoding a BOM announces and the BOM's size, | |
| 63 | +// or "" and 0 when body starts with none. | |
| 64 | +func byteOrderMark(body []byte) (string, int) { | |
| 65 | + for _, bom := range byteOrderMarks { | |
| 66 | + if bytes.HasPrefix(body, bom.mark) { | |
| 67 | + return bom.label, len(bom.mark) | |
| 68 | + } | |
| 69 | + } | |
| 70 | + return "", 0 | |
| 71 | +} | |
| 72 | + | |
| 73 | +// firstKnown returns the first label that names an encoding, "" if none does. | |
| 74 | +func firstKnown(labels ...string) string { | |
| 75 | + for _, label := range labels { | |
| 76 | + if enc, _ := charset.Lookup(label); enc != nil { | |
| 77 | + return label | |
| 78 | + } | |
| 79 | + } | |
| 80 | + return "" | |
| 81 | +} | |
| 82 | + | |
| 83 | +// headerCharset is the charset parameter of a Content-Type, "" if none. | |
| 84 | +func headerCharset(contentType string) string { | |
| 85 | + _, params, err := mime.ParseMediaType(contentType) | |
| 86 | + if err != nil { | |
| 87 | + return "" | |
| 88 | + } | |
| 89 | + return params["charset"] | |
| 90 | +} | |
| 91 | + | |
| 92 | +// metaCharset is the encoding the page declares for itself, in | |
| 93 | +// <meta charset="…"> or <meta http-equiv="Content-Type" content="…; charset=…">. | |
| 94 | +func metaCharset(body []byte) string { | |
| 95 | + head := body[:min(len(body), metaScanBytes)] | |
| 96 | + z := html.NewTokenizer(bytes.NewReader(head)) | |
| 97 | + for { | |
| 98 | + switch z.Next() { | |
| 99 | + case html.ErrorToken: | |
| 100 | + return "" | |
| 101 | + case html.StartTagToken, html.SelfClosingTagToken: | |
| 102 | + tok := z.Token() | |
| 103 | + if tok.Data == "body" { | |
| 104 | + return "" | |
| 105 | + } | |
| 106 | + if tok.Data == "meta" { | |
| 107 | + if label := metaTagCharset(tok.Attr); label != "" { | |
| 108 | + return label | |
| 109 | + } | |
| 110 | + } | |
| 111 | + } | |
| 112 | + } | |
| 113 | +} | |
| 114 | + | |
| 115 | +func metaTagCharset(attrs []html.Attribute) string { | |
| 116 | + var httpEquiv, content string | |
| 117 | + for _, a := range attrs { | |
| 118 | + switch strings.ToLower(a.Key) { | |
| 119 | + case "charset": | |
| 120 | + return strings.TrimSpace(a.Val) | |
| 121 | + case "http-equiv": | |
| 122 | + httpEquiv = strings.ToLower(strings.TrimSpace(a.Val)) | |
| 123 | + case "content": | |
| 124 | + content = a.Val | |
| 125 | + } | |
| 126 | + } | |
| 127 | + if httpEquiv != "content-type" { | |
| 128 | + return "" | |
| 129 | + } | |
| 130 | + return headerCharset(content) | |
| 131 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,131 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "bytes" | ||
| 5 | + "mime" | ||
| 6 | + "strings" | ||
| 7 | + "unicode/utf8" | ||
| 8 | + | ||
| 9 | + "golang.org/x/net/html" | ||
| 10 | + "golang.org/x/net/html/charset" | ||
| 11 | + "golang.org/x/text/encoding" | ||
| 12 | +) | ||
| 13 | + | ||
| 14 | +// metaScanBytes is how far into the page a <meta charset> is looked for. | ||
| 15 | +// HTML requires it within the first 1024 bytes; real pages are sloppier. | ||
| 16 | +const metaScanBytes = 4096 | ||
| 17 | + | ||
| 18 | +// toUTF8 decodes body to UTF-8 and names the encoding it came from. The | ||
| 19 | +// encoding is, in order: a byte-order mark; the charset of the | ||
| 20 | +// Content-Type header; a <meta charset> or <meta http-equiv> in the page; | ||
| 21 | +// UTF-8 when the bytes are valid UTF-8; windows-1252 — the web's historical | ||
| 22 | +// default — otherwise. An encoding name nobody knows is skipped. | ||
| 23 | +func toUTF8(body []byte, contentType string) ([]byte, string, error) { | ||
| 24 | + enc, name, body := detectEncoding(body, contentType) | ||
| 25 | + if name == "utf-8" { | ||
| 26 | + return body, name, nil | ||
| 27 | + } | ||
| 28 | + decoded, err := enc.NewDecoder().Bytes(body) | ||
| 29 | + if err != nil { | ||
| 30 | + return nil, "", errorf(CodeConversion, err, "cannot decode the page from %s", name) | ||
| 31 | + } | ||
| 32 | + return decoded, name, nil | ||
| 33 | +} | ||
| 34 | + | ||
| 35 | +// detectEncoding returns the encoding of body, and body without its BOM. | ||
| 36 | +func detectEncoding(body []byte, contentType string) (encoding.Encoding, string, []byte) { | ||
| 37 | + label, bomSize := byteOrderMark(body) | ||
| 38 | + if label == "" { | ||
| 39 | + label = firstKnown(headerCharset(contentType), metaCharset(body)) | ||
| 40 | + } | ||
| 41 | + if label == "" { | ||
| 42 | + label = "windows-1252" | ||
| 43 | + if utf8.Valid(body) { | ||
| 44 | + label = "utf-8" | ||
| 45 | + } | ||
| 46 | + } | ||
| 47 | + enc, name := charset.Lookup(label) | ||
| 48 | + return enc, name, body[bomSize:] | ||
| 49 | +} | ||
| 50 | + | ||
| 51 | +// byteOrderMarks are checked in order; the UTF-8 one is not a prefix of | ||
| 52 | +// the others, so the order only matters for readability. | ||
| 53 | +var byteOrderMarks = []struct { | ||
| 54 | + mark []byte | ||
| 55 | + label string | ||
| 56 | +}{ | ||
| 57 | + {[]byte{0xEF, 0xBB, 0xBF}, "utf-8"}, | ||
| 58 | + {[]byte{0xFF, 0xFE}, "utf-16le"}, | ||
| 59 | + {[]byte{0xFE, 0xFF}, "utf-16be"}, | ||
| 60 | +} | ||
| 61 | + | ||
| 62 | +// byteOrderMark returns the encoding a BOM announces and the BOM's size, | ||
| 63 | +// or "" and 0 when body starts with none. | ||
| 64 | +func byteOrderMark(body []byte) (string, int) { | ||
| 65 | + for _, bom := range byteOrderMarks { | ||
| 66 | + if bytes.HasPrefix(body, bom.mark) { | ||
| 67 | + return bom.label, len(bom.mark) | ||
| 68 | + } | ||
| 69 | + } | ||
| 70 | + return "", 0 | ||
| 71 | +} | ||
| 72 | + | ||
| 73 | +// firstKnown returns the first label that names an encoding, "" if none does. | ||
| 74 | +func firstKnown(labels ...string) string { | ||
| 75 | + for _, label := range labels { | ||
| 76 | + if enc, _ := charset.Lookup(label); enc != nil { | ||
| 77 | + return label | ||
| 78 | + } | ||
| 79 | + } | ||
| 80 | + return "" | ||
| 81 | +} | ||
| 82 | + | ||
| 83 | +// headerCharset is the charset parameter of a Content-Type, "" if none. | ||
| 84 | +func headerCharset(contentType string) string { | ||
| 85 | + _, params, err := mime.ParseMediaType(contentType) | ||
| 86 | + if err != nil { | ||
| 87 | + return "" | ||
| 88 | + } | ||
| 89 | + return params["charset"] | ||
| 90 | +} | ||
| 91 | + | ||
| 92 | +// metaCharset is the encoding the page declares for itself, in | ||
| 93 | +// <meta charset="…"> or <meta http-equiv="Content-Type" content="…; charset=…">. | ||
| 94 | +func metaCharset(body []byte) string { | ||
| 95 | + head := body[:min(len(body), metaScanBytes)] | ||
| 96 | + z := html.NewTokenizer(bytes.NewReader(head)) | ||
| 97 | + for { | ||
| 98 | + switch z.Next() { | ||
| 99 | + case html.ErrorToken: | ||
| 100 | + return "" | ||
| 101 | + case html.StartTagToken, html.SelfClosingTagToken: | ||
| 102 | + tok := z.Token() | ||
| 103 | + if tok.Data == "body" { | ||
| 104 | + return "" | ||
| 105 | + } | ||
| 106 | + if tok.Data == "meta" { | ||
| 107 | + if label := metaTagCharset(tok.Attr); label != "" { | ||
| 108 | + return label | ||
| 109 | + } | ||
| 110 | + } | ||
| 111 | + } | ||
| 112 | + } | ||
| 113 | +} | ||
| 114 | + | ||
| 115 | +func metaTagCharset(attrs []html.Attribute) string { | ||
| 116 | + var httpEquiv, content string | ||
| 117 | + for _, a := range attrs { | ||
| 118 | + switch strings.ToLower(a.Key) { | ||
| 119 | + case "charset": | ||
| 120 | + return strings.TrimSpace(a.Val) | ||
| 121 | + case "http-equiv": | ||
| 122 | + httpEquiv = strings.ToLower(strings.TrimSpace(a.Val)) | ||
| 123 | + case "content": | ||
| 124 | + content = a.Val | ||
| 125 | + } | ||
| 126 | + } | ||
| 127 | + if httpEquiv != "content-type" { | ||
| 128 | + return "" | ||
| 129 | + } | ||
| 130 | + return headerCharset(content) | ||
| 131 | +} | ||
added
internal/webmd/chrome.go +143 -0 | new file mode 100644 | ||
| @@ -0,0 +1,143 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "strings" | |
| 5 | + | |
| 6 | + "golang.org/x/net/html" | |
| 7 | + "golang.org/x/net/html/atom" | |
| 8 | +) | |
| 9 | + | |
| 10 | +// What Clean removes as page chrome, and how it recognises it. | |
| 11 | + | |
| 12 | +// droppedTags are removed with everything they contain: they are code, | |
| 13 | +// metadata, page chrome, or controls — never the content a reader came for. | |
| 14 | +var droppedTags = map[atom.Atom]bool{ | |
| 15 | + atom.Script: true, atom.Style: true, atom.Noscript: true, atom.Template: true, | |
| 16 | + atom.Meta: true, atom.Link: true, atom.Base: true, atom.Title: true, | |
| 17 | + atom.Svg: true, atom.Canvas: true, atom.Iframe: true, atom.Object: true, atom.Embed: true, | |
| 18 | + atom.Nav: true, atom.Aside: true, atom.Dialog: true, | |
| 19 | + atom.Button: true, atom.Select: true, atom.Textarea: true, | |
| 20 | +} | |
| 21 | + | |
| 22 | +// droppedRoles are the ARIA landmarks of page chrome: the site banner, its | |
| 23 | +// navigation, the side column, the search box and the site footer. | |
| 24 | +var droppedRoles = map[string]bool{ | |
| 25 | + "navigation": true, "banner": true, "complementary": true, | |
| 26 | + "search": true, "contentinfo": true, "dialog": true, | |
| 27 | +} | |
| 28 | + | |
| 29 | +// decorativeWords are class or id words that name page chrome. A class is | |
| 30 | +// split on spaces, hyphens and underscores, and matched word by word, so | |
| 31 | +// "post-ad" matches "ad" but "shadow" or "add-to-cart" do not. | |
| 32 | +var decorativeWords = map[string]bool{ | |
| 33 | + "ad": true, "ads": true, "advert": true, "adverts": true, "advertisement": true, | |
| 34 | + "advertising": true, "adsbygoogle": true, "sponsor": true, "sponsored": true, | |
| 35 | + "promo": true, "cookie": true, "cookies": true, "consent": true, "gdpr": true, | |
| 36 | + "newsletter": true, "subscribe": true, "popup": true, "modal": true, | |
| 37 | + "share": true, "sharing": true, "social": true, "breadcrumb": true, "breadcrumbs": true, | |
| 38 | + "nav": true, "navbar": true, "navigation": true, "menu": true, "sidebar": true, | |
| 39 | + "skiplink": true, | |
| 40 | +} | |
| 41 | + | |
| 42 | +// hiddenClasses are whole class names meaning "for screen readers only": | |
| 43 | +// "Skip to content" links and the like, noise in a Markdown copy. | |
| 44 | +var hiddenClasses = map[string]bool{ | |
| 45 | + "sr-only": true, "visually-hidden": true, "screen-reader-text": true, | |
| 46 | +} | |
| 47 | + | |
| 48 | +// isChrome reports whether n is to be removed with its subtree. | |
| 49 | +func isChrome(n *html.Node) bool { | |
| 50 | + switch n.Type { | |
| 51 | + case html.CommentNode: | |
| 52 | + return true | |
| 53 | + case html.ElementNode: | |
| 54 | + return isChromeElement(n) | |
| 55 | + } | |
| 56 | + return false | |
| 57 | +} | |
| 58 | + | |
| 59 | +// isChromeElement applies the rules in order of certainty: tags that are | |
| 60 | +// never content, then landmarks and hidden elements, then the class/id | |
| 61 | +// heuristic. <html>, <body>, <main> and <article> are never removed by the | |
| 62 | +// last two, whatever their attributes say. | |
| 63 | +func isChromeElement(n *html.Node) bool { | |
| 64 | + if droppedTags[n.DataAtom] || isNonCheckboxInput(n) || isSiteFooter(n) { | |
| 65 | + return true | |
| 66 | + } | |
| 67 | + if protectedTags[n.DataAtom] { | |
| 68 | + return false | |
| 69 | + } | |
| 70 | + return isHiddenOrLandmark(n) || (looksDecorative(n) && !holdsMainContent(n)) | |
| 71 | +} | |
| 72 | + | |
| 73 | +// protectedTags hold the page itself: a class like "has-sidebar" on them | |
| 74 | +// says nothing about their content. | |
| 75 | +var protectedTags = map[atom.Atom]bool{ | |
| 76 | + atom.Html: true, atom.Body: true, atom.Main: true, atom.Article: true, | |
| 77 | +} | |
| 78 | + | |
| 79 | +// isNonCheckboxInput: form controls go, except the checkboxes a task list | |
| 80 | +// is made of. | |
| 81 | +func isNonCheckboxInput(n *html.Node) bool { | |
| 82 | + return n.DataAtom == atom.Input && attr(n, "type") != "checkbox" | |
| 83 | +} | |
| 84 | + | |
| 85 | +// isSiteFooter: a <footer> outside any <article> or <main> is the site's, | |
| 86 | +// not the article's byline. | |
| 87 | +func isSiteFooter(n *html.Node) bool { | |
| 88 | + return n.DataAtom == atom.Footer && !hasAncestor(n, atom.Article, atom.Main) | |
| 89 | +} | |
| 90 | + | |
| 91 | +func isHiddenOrLandmark(n *html.Node) bool { | |
| 92 | + return droppedRoles[attr(n, "role")] || attr(n, "aria-hidden") == "true" || hasAttr(n, "hidden") | |
| 93 | +} | |
| 94 | + | |
| 95 | +// looksDecorative applies the class/id word heuristic. | |
| 96 | +func looksDecorative(n *html.Node) bool { | |
| 97 | + for _, class := range strings.Fields(attr(n, "class")) { | |
| 98 | + if hiddenClasses[strings.ToLower(class)] || hasDecorativeWord(class) { | |
| 99 | + return true | |
| 100 | + } | |
| 101 | + } | |
| 102 | + return hasDecorativeWord(attr(n, "id")) | |
| 103 | +} | |
| 104 | + | |
| 105 | +func hasDecorativeWord(name string) bool { | |
| 106 | + words := strings.FieldsFunc(strings.ToLower(name), func(r rune) bool { | |
| 107 | + return r == '-' || r == '_' || r == ' ' | |
| 108 | + }) | |
| 109 | + for _, w := range words { | |
| 110 | + if decorativeWords[w] { | |
| 111 | + return true | |
| 112 | + } | |
| 113 | + } | |
| 114 | + return false | |
| 115 | +} | |
| 116 | + | |
| 117 | +// holdsMainContent reports whether n contains a <main>, an <article> or an | |
| 118 | +// <h1>, the signs of a wrapper around the page's content. | |
| 119 | +func holdsMainContent(n *html.Node) bool { | |
| 120 | + for c := n.FirstChild; c != nil; c = c.NextSibling { | |
| 121 | + if c.Type != html.ElementNode { | |
| 122 | + continue | |
| 123 | + } | |
| 124 | + if c.DataAtom == atom.Main || c.DataAtom == atom.Article || c.DataAtom == atom.H1 { | |
| 125 | + return true | |
| 126 | + } | |
| 127 | + if holdsMainContent(c) { | |
| 128 | + return true | |
| 129 | + } | |
| 130 | + } | |
| 131 | + return false | |
| 132 | +} | |
| 133 | + | |
| 134 | +func hasAncestor(n *html.Node, tags ...atom.Atom) bool { | |
| 135 | + for p := n.Parent; p != nil; p = p.Parent { | |
| 136 | + for _, t := range tags { | |
| 137 | + if p.DataAtom == t { | |
| 138 | + return true | |
| 139 | + } | |
| 140 | + } | |
| 141 | + } | |
| 142 | + return false | |
| 143 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,143 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "strings" | ||
| 5 | + | ||
| 6 | + "golang.org/x/net/html" | ||
| 7 | + "golang.org/x/net/html/atom" | ||
| 8 | +) | ||
| 9 | + | ||
| 10 | +// What Clean removes as page chrome, and how it recognises it. | ||
| 11 | + | ||
| 12 | +// droppedTags are removed with everything they contain: they are code, | ||
| 13 | +// metadata, page chrome, or controls — never the content a reader came for. | ||
| 14 | +var droppedTags = map[atom.Atom]bool{ | ||
| 15 | + atom.Script: true, atom.Style: true, atom.Noscript: true, atom.Template: true, | ||
| 16 | + atom.Meta: true, atom.Link: true, atom.Base: true, atom.Title: true, | ||
| 17 | + atom.Svg: true, atom.Canvas: true, atom.Iframe: true, atom.Object: true, atom.Embed: true, | ||
| 18 | + atom.Nav: true, atom.Aside: true, atom.Dialog: true, | ||
| 19 | + atom.Button: true, atom.Select: true, atom.Textarea: true, | ||
| 20 | +} | ||
| 21 | + | ||
| 22 | +// droppedRoles are the ARIA landmarks of page chrome: the site banner, its | ||
| 23 | +// navigation, the side column, the search box and the site footer. | ||
| 24 | +var droppedRoles = map[string]bool{ | ||
| 25 | + "navigation": true, "banner": true, "complementary": true, | ||
| 26 | + "search": true, "contentinfo": true, "dialog": true, | ||
| 27 | +} | ||
| 28 | + | ||
| 29 | +// decorativeWords are class or id words that name page chrome. A class is | ||
| 30 | +// split on spaces, hyphens and underscores, and matched word by word, so | ||
| 31 | +// "post-ad" matches "ad" but "shadow" or "add-to-cart" do not. | ||
| 32 | +var decorativeWords = map[string]bool{ | ||
| 33 | + "ad": true, "ads": true, "advert": true, "adverts": true, "advertisement": true, | ||
| 34 | + "advertising": true, "adsbygoogle": true, "sponsor": true, "sponsored": true, | ||
| 35 | + "promo": true, "cookie": true, "cookies": true, "consent": true, "gdpr": true, | ||
| 36 | + "newsletter": true, "subscribe": true, "popup": true, "modal": true, | ||
| 37 | + "share": true, "sharing": true, "social": true, "breadcrumb": true, "breadcrumbs": true, | ||
| 38 | + "nav": true, "navbar": true, "navigation": true, "menu": true, "sidebar": true, | ||
| 39 | + "skiplink": true, | ||
| 40 | +} | ||
| 41 | + | ||
| 42 | +// hiddenClasses are whole class names meaning "for screen readers only": | ||
| 43 | +// "Skip to content" links and the like, noise in a Markdown copy. | ||
| 44 | +var hiddenClasses = map[string]bool{ | ||
| 45 | + "sr-only": true, "visually-hidden": true, "screen-reader-text": true, | ||
| 46 | +} | ||
| 47 | + | ||
| 48 | +// isChrome reports whether n is to be removed with its subtree. | ||
| 49 | +func isChrome(n *html.Node) bool { | ||
| 50 | + switch n.Type { | ||
| 51 | + case html.CommentNode: | ||
| 52 | + return true | ||
| 53 | + case html.ElementNode: | ||
| 54 | + return isChromeElement(n) | ||
| 55 | + } | ||
| 56 | + return false | ||
| 57 | +} | ||
| 58 | + | ||
| 59 | +// isChromeElement applies the rules in order of certainty: tags that are | ||
| 60 | +// never content, then landmarks and hidden elements, then the class/id | ||
| 61 | +// heuristic. <html>, <body>, <main> and <article> are never removed by the | ||
| 62 | +// last two, whatever their attributes say. | ||
| 63 | +func isChromeElement(n *html.Node) bool { | ||
| 64 | + if droppedTags[n.DataAtom] || isNonCheckboxInput(n) || isSiteFooter(n) { | ||
| 65 | + return true | ||
| 66 | + } | ||
| 67 | + if protectedTags[n.DataAtom] { | ||
| 68 | + return false | ||
| 69 | + } | ||
| 70 | + return isHiddenOrLandmark(n) || (looksDecorative(n) && !holdsMainContent(n)) | ||
| 71 | +} | ||
| 72 | + | ||
| 73 | +// protectedTags hold the page itself: a class like "has-sidebar" on them | ||
| 74 | +// says nothing about their content. | ||
| 75 | +var protectedTags = map[atom.Atom]bool{ | ||
| 76 | + atom.Html: true, atom.Body: true, atom.Main: true, atom.Article: true, | ||
| 77 | +} | ||
| 78 | + | ||
| 79 | +// isNonCheckboxInput: form controls go, except the checkboxes a task list | ||
| 80 | +// is made of. | ||
| 81 | +func isNonCheckboxInput(n *html.Node) bool { | ||
| 82 | + return n.DataAtom == atom.Input && attr(n, "type") != "checkbox" | ||
| 83 | +} | ||
| 84 | + | ||
| 85 | +// isSiteFooter: a <footer> outside any <article> or <main> is the site's, | ||
| 86 | +// not the article's byline. | ||
| 87 | +func isSiteFooter(n *html.Node) bool { | ||
| 88 | + return n.DataAtom == atom.Footer && !hasAncestor(n, atom.Article, atom.Main) | ||
| 89 | +} | ||
| 90 | + | ||
| 91 | +func isHiddenOrLandmark(n *html.Node) bool { | ||
| 92 | + return droppedRoles[attr(n, "role")] || attr(n, "aria-hidden") == "true" || hasAttr(n, "hidden") | ||
| 93 | +} | ||
| 94 | + | ||
| 95 | +// looksDecorative applies the class/id word heuristic. | ||
| 96 | +func looksDecorative(n *html.Node) bool { | ||
| 97 | + for _, class := range strings.Fields(attr(n, "class")) { | ||
| 98 | + if hiddenClasses[strings.ToLower(class)] || hasDecorativeWord(class) { | ||
| 99 | + return true | ||
| 100 | + } | ||
| 101 | + } | ||
| 102 | + return hasDecorativeWord(attr(n, "id")) | ||
| 103 | +} | ||
| 104 | + | ||
| 105 | +func hasDecorativeWord(name string) bool { | ||
| 106 | + words := strings.FieldsFunc(strings.ToLower(name), func(r rune) bool { | ||
| 107 | + return r == '-' || r == '_' || r == ' ' | ||
| 108 | + }) | ||
| 109 | + for _, w := range words { | ||
| 110 | + if decorativeWords[w] { | ||
| 111 | + return true | ||
| 112 | + } | ||
| 113 | + } | ||
| 114 | + return false | ||
| 115 | +} | ||
| 116 | + | ||
| 117 | +// holdsMainContent reports whether n contains a <main>, an <article> or an | ||
| 118 | +// <h1>, the signs of a wrapper around the page's content. | ||
| 119 | +func holdsMainContent(n *html.Node) bool { | ||
| 120 | + for c := n.FirstChild; c != nil; c = c.NextSibling { | ||
| 121 | + if c.Type != html.ElementNode { | ||
| 122 | + continue | ||
| 123 | + } | ||
| 124 | + if c.DataAtom == atom.Main || c.DataAtom == atom.Article || c.DataAtom == atom.H1 { | ||
| 125 | + return true | ||
| 126 | + } | ||
| 127 | + if holdsMainContent(c) { | ||
| 128 | + return true | ||
| 129 | + } | ||
| 130 | + } | ||
| 131 | + return false | ||
| 132 | +} | ||
| 133 | + | ||
| 134 | +func hasAncestor(n *html.Node, tags ...atom.Atom) bool { | ||
| 135 | + for p := n.Parent; p != nil; p = p.Parent { | ||
| 136 | + for _, t := range tags { | ||
| 137 | + if p.DataAtom == t { | ||
| 138 | + return true | ||
| 139 | + } | ||
| 140 | + } | ||
| 141 | + } | ||
| 142 | + return false | ||
| 143 | +} | ||
added
internal/webmd/clean.go +107 -0 | new file mode 100644 | ||
| @@ -0,0 +1,107 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "strings" | |
| 5 | + | |
| 6 | + "golang.org/x/net/html" | |
| 7 | + "golang.org/x/net/html/atom" | |
| 8 | +) | |
| 9 | + | |
| 10 | +// keptAttributes is, per element, the only attributes the conversion reads. | |
| 11 | +// Everything else — style, class, id, data-*, event handlers — goes. | |
| 12 | +var keptAttributes = map[atom.Atom][]string{ | |
| 13 | + atom.A: {"href"}, | |
| 14 | + atom.Img: {"src", "alt"}, | |
| 15 | + atom.Td: {"colspan", "rowspan"}, | |
| 16 | + atom.Th: {"colspan", "rowspan"}, | |
| 17 | + atom.Ol: {"start"}, | |
| 18 | + atom.Input: {"type", "checked"}, | |
| 19 | + // class survives on code blocks for one reason: "language-go" is where | |
| 20 | + // the fence's info string comes from. Only those words are kept. | |
| 21 | + atom.Pre: {"class"}, | |
| 22 | + atom.Code: {"class"}, | |
| 23 | +} | |
| 24 | + | |
| 25 | +// Clean removes, in place, everything under root that is not content (see | |
| 26 | +// droppedTags, droppedRoles, decorativeWords), every HTML comment, and every | |
| 27 | +// attribute the conversion does not need. | |
| 28 | +// | |
| 29 | +// An element found by the class/id heuristic is kept when it holds the | |
| 30 | +// page's main content (a <main>, an <article> or an <h1>): a layout wrapper | |
| 31 | +// called "page-with-sidebar" must not take the whole article with it. | |
| 32 | +// | |
| 33 | +// doc, _ := html.Parse(strings.NewReader(page)) | |
| 34 | +// webmd.Clean(doc) | |
| 35 | +func Clean(root *html.Node) { | |
| 36 | + for child := root.FirstChild; child != nil; { | |
| 37 | + next := child.NextSibling | |
| 38 | + if isChrome(child) { | |
| 39 | + root.RemoveChild(child) | |
| 40 | + } else { | |
| 41 | + if child.Type == html.ElementNode { | |
| 42 | + keepAttributes(child) | |
| 43 | + } | |
| 44 | + Clean(child) | |
| 45 | + } | |
| 46 | + child = next | |
| 47 | + } | |
| 48 | +} | |
| 49 | + | |
| 50 | +// keepAttributes drops every attribute of n not listed in keptAttributes, | |
| 51 | +// and reduces a code block's class to its language words. | |
| 52 | +func keepAttributes(n *html.Node) { | |
| 53 | + allowed := keptAttributes[n.DataAtom] | |
| 54 | + kept := n.Attr[:0] | |
| 55 | + for _, a := range n.Attr { | |
| 56 | + if !contains(allowed, a.Key) { | |
| 57 | + continue | |
| 58 | + } | |
| 59 | + if a.Key == "class" { | |
| 60 | + a.Val = languageClasses(a.Val) | |
| 61 | + if a.Val == "" { | |
| 62 | + continue | |
| 63 | + } | |
| 64 | + } | |
| 65 | + kept = append(kept, a) | |
| 66 | + } | |
| 67 | + n.Attr = kept | |
| 68 | +} | |
| 69 | + | |
| 70 | +// languageClasses keeps the "language-x" / "lang-x" words of a class. | |
| 71 | +func languageClasses(class string) string { | |
| 72 | + var kept []string | |
| 73 | + for _, word := range strings.Fields(class) { | |
| 74 | + if strings.HasPrefix(word, "language-") || strings.HasPrefix(word, "lang-") { | |
| 75 | + kept = append(kept, word) | |
| 76 | + } | |
| 77 | + } | |
| 78 | + return strings.Join(kept, " ") | |
| 79 | +} | |
| 80 | + | |
| 81 | +func contains(list []string, s string) bool { | |
| 82 | + for _, item := range list { | |
| 83 | + if item == s { | |
| 84 | + return true | |
| 85 | + } | |
| 86 | + } | |
| 87 | + return false | |
| 88 | +} | |
| 89 | + | |
| 90 | +// attr returns the value of n's attribute key, "" when absent. | |
| 91 | +func attr(n *html.Node, key string) string { | |
| 92 | + for _, a := range n.Attr { | |
| 93 | + if a.Key == key { | |
| 94 | + return a.Val | |
| 95 | + } | |
| 96 | + } | |
| 97 | + return "" | |
| 98 | +} | |
| 99 | + | |
| 100 | +func hasAttr(n *html.Node, key string) bool { | |
| 101 | + for _, a := range n.Attr { | |
| 102 | + if a.Key == key { | |
| 103 | + return true | |
| 104 | + } | |
| 105 | + } | |
| 106 | + return false | |
| 107 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,107 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "strings" | ||
| 5 | + | ||
| 6 | + "golang.org/x/net/html" | ||
| 7 | + "golang.org/x/net/html/atom" | ||
| 8 | +) | ||
| 9 | + | ||
| 10 | +// keptAttributes is, per element, the only attributes the conversion reads. | ||
| 11 | +// Everything else — style, class, id, data-*, event handlers — goes. | ||
| 12 | +var keptAttributes = map[atom.Atom][]string{ | ||
| 13 | + atom.A: {"href"}, | ||
| 14 | + atom.Img: {"src", "alt"}, | ||
| 15 | + atom.Td: {"colspan", "rowspan"}, | ||
| 16 | + atom.Th: {"colspan", "rowspan"}, | ||
| 17 | + atom.Ol: {"start"}, | ||
| 18 | + atom.Input: {"type", "checked"}, | ||
| 19 | + // class survives on code blocks for one reason: "language-go" is where | ||
| 20 | + // the fence's info string comes from. Only those words are kept. | ||
| 21 | + atom.Pre: {"class"}, | ||
| 22 | + atom.Code: {"class"}, | ||
| 23 | +} | ||
| 24 | + | ||
| 25 | +// Clean removes, in place, everything under root that is not content (see | ||
| 26 | +// droppedTags, droppedRoles, decorativeWords), every HTML comment, and every | ||
| 27 | +// attribute the conversion does not need. | ||
| 28 | +// | ||
| 29 | +// An element found by the class/id heuristic is kept when it holds the | ||
| 30 | +// page's main content (a <main>, an <article> or an <h1>): a layout wrapper | ||
| 31 | +// called "page-with-sidebar" must not take the whole article with it. | ||
| 32 | +// | ||
| 33 | +// doc, _ := html.Parse(strings.NewReader(page)) | ||
| 34 | +// webmd.Clean(doc) | ||
| 35 | +func Clean(root *html.Node) { | ||
| 36 | + for child := root.FirstChild; child != nil; { | ||
| 37 | + next := child.NextSibling | ||
| 38 | + if isChrome(child) { | ||
| 39 | + root.RemoveChild(child) | ||
| 40 | + } else { | ||
| 41 | + if child.Type == html.ElementNode { | ||
| 42 | + keepAttributes(child) | ||
| 43 | + } | ||
| 44 | + Clean(child) | ||
| 45 | + } | ||
| 46 | + child = next | ||
| 47 | + } | ||
| 48 | +} | ||
| 49 | + | ||
| 50 | +// keepAttributes drops every attribute of n not listed in keptAttributes, | ||
| 51 | +// and reduces a code block's class to its language words. | ||
| 52 | +func keepAttributes(n *html.Node) { | ||
| 53 | + allowed := keptAttributes[n.DataAtom] | ||
| 54 | + kept := n.Attr[:0] | ||
| 55 | + for _, a := range n.Attr { | ||
| 56 | + if !contains(allowed, a.Key) { | ||
| 57 | + continue | ||
| 58 | + } | ||
| 59 | + if a.Key == "class" { | ||
| 60 | + a.Val = languageClasses(a.Val) | ||
| 61 | + if a.Val == "" { | ||
| 62 | + continue | ||
| 63 | + } | ||
| 64 | + } | ||
| 65 | + kept = append(kept, a) | ||
| 66 | + } | ||
| 67 | + n.Attr = kept | ||
| 68 | +} | ||
| 69 | + | ||
| 70 | +// languageClasses keeps the "language-x" / "lang-x" words of a class. | ||
| 71 | +func languageClasses(class string) string { | ||
| 72 | + var kept []string | ||
| 73 | + for _, word := range strings.Fields(class) { | ||
| 74 | + if strings.HasPrefix(word, "language-") || strings.HasPrefix(word, "lang-") { | ||
| 75 | + kept = append(kept, word) | ||
| 76 | + } | ||
| 77 | + } | ||
| 78 | + return strings.Join(kept, " ") | ||
| 79 | +} | ||
| 80 | + | ||
| 81 | +func contains(list []string, s string) bool { | ||
| 82 | + for _, item := range list { | ||
| 83 | + if item == s { | ||
| 84 | + return true | ||
| 85 | + } | ||
| 86 | + } | ||
| 87 | + return false | ||
| 88 | +} | ||
| 89 | + | ||
| 90 | +// attr returns the value of n's attribute key, "" when absent. | ||
| 91 | +func attr(n *html.Node, key string) string { | ||
| 92 | + for _, a := range n.Attr { | ||
| 93 | + if a.Key == key { | ||
| 94 | + return a.Val | ||
| 95 | + } | ||
| 96 | + } | ||
| 97 | + return "" | ||
| 98 | +} | ||
| 99 | + | ||
| 100 | +func hasAttr(n *html.Node, key string) bool { | ||
| 101 | + for _, a := range n.Attr { | ||
| 102 | + if a.Key == key { | ||
| 103 | + return true | ||
| 104 | + } | ||
| 105 | + } | ||
| 106 | + return false | ||
| 107 | +} | ||
added
internal/webmd/convert.go +187 -0 | new file mode 100644 | ||
| @@ -0,0 +1,187 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "bytes" | |
| 5 | + "net/url" | |
| 6 | + "strings" | |
| 7 | + | |
| 8 | + "golang.org/x/net/html" | |
| 9 | + "golang.org/x/net/html/atom" | |
| 10 | +) | |
| 11 | + | |
| 12 | +// blockTags are the elements rendered as blocks of their own. Any other | |
| 13 | +// element is inline — unless it contains one of these, in which case it is | |
| 14 | +// a transparent container (see converter.isBlock). | |
| 15 | +var blockTags = map[atom.Atom]bool{ | |
| 16 | + atom.Html: true, atom.Body: true, atom.Main: true, atom.Article: true, atom.Section: true, | |
| 17 | + atom.Header: true, atom.Footer: true, atom.Div: true, atom.P: true, atom.Address: true, | |
| 18 | + atom.H1: true, atom.H2: true, atom.H3: true, atom.H4: true, atom.H5: true, atom.H6: true, | |
| 19 | + atom.Ul: true, atom.Ol: true, atom.Li: true, atom.Dl: true, atom.Dt: true, atom.Dd: true, | |
| 20 | + atom.Pre: true, atom.Blockquote: true, atom.Hr: true, atom.Table: true, atom.Caption: true, | |
| 21 | + atom.Figure: true, atom.Figcaption: true, atom.Details: true, atom.Summary: true, | |
| 22 | + atom.Form: true, atom.Fieldset: true, atom.Legend: true, atom.Center: true, | |
| 23 | +} | |
| 24 | + | |
| 25 | +var headingLevels = map[atom.Atom]int{ | |
| 26 | + atom.H1: 1, atom.H2: 2, atom.H3: 3, atom.H4: 4, atom.H5: 5, atom.H6: 6, | |
| 27 | +} | |
| 28 | + | |
| 29 | +// Convert parses an HTML document — invalid or partial HTML included: the | |
| 30 | +// parser repairs it the way a browser does — cleans it (see Clean) and | |
| 31 | +// renders its body as Markdown. | |
| 32 | +// | |
| 33 | +// The result ends with exactly one newline, or is "" when the page has no | |
| 34 | +// content left once cleaned. Nothing is added: no title, no front matter. | |
| 35 | +// | |
| 36 | +// md, err := webmd.Convert([]byte("<h1>Hi</h1><p>A <b>bold</b> move.</p>"), webmd.Options{}) | |
| 37 | +// // md == "# Hi\n\nA **bold** move.\n" | |
| 38 | +func Convert(page []byte, opts Options) (string, error) { | |
| 39 | + doc, err := html.Parse(bytes.NewReader(page)) | |
| 40 | + if err != nil { | |
| 41 | + return "", errorf(CodeConversion, err, "cannot parse the HTML") | |
| 42 | + } | |
| 43 | + Clean(doc) | |
| 44 | + if opts.BaseURL != nil { | |
| 45 | + resolveURLs(doc, opts.BaseURL) | |
| 46 | + } | |
| 47 | + if opts.Tables == "" { | |
| 48 | + opts.Tables = TablesHTML | |
| 49 | + } | |
| 50 | + | |
| 51 | + c := &converter{opts: opts, holdsBlock: map[*html.Node]bool{}} | |
| 52 | + md := strings.Trim(joinBlocks(c.blocks(findBody(doc))), "\n") | |
| 53 | + if strings.TrimSpace(md) == "" { | |
| 54 | + return "", nil | |
| 55 | + } | |
| 56 | + return md + "\n", nil | |
| 57 | +} | |
| 58 | + | |
| 59 | +// converter carries the options and a memo of which inline elements hold a | |
| 60 | +// block, so that deciding it stays linear on deeply nested pages. | |
| 61 | +type converter struct { | |
| 62 | + opts Options | |
| 63 | + holdsBlock map[*html.Node]bool | |
| 64 | +} | |
| 65 | + | |
| 66 | +// block is one rendered Markdown block. list tells a list item whether its | |
| 67 | +// parts can be packed tightly (text followed by a nested list) or need blank | |
| 68 | +// lines between them. | |
| 69 | +type block struct { | |
| 70 | + md string | |
| 71 | + list bool | |
| 72 | +} | |
| 73 | + | |
| 74 | +func joinBlocks(parts []block) string { | |
| 75 | + texts := make([]string, len(parts)) | |
| 76 | + for i, p := range parts { | |
| 77 | + texts[i] = p.md | |
| 78 | + } | |
| 79 | + return strings.Join(texts, "\n\n") | |
| 80 | +} | |
| 81 | + | |
| 82 | +// isBlock reports whether n renders as a block: a block tag, or any other | |
| 83 | +// element wrapping one (<a><div>…</div></a>, an unknown <my-card>). | |
| 84 | +func (c *converter) isBlock(n *html.Node) bool { | |
| 85 | + if n.Type != html.ElementNode { | |
| 86 | + return false | |
| 87 | + } | |
| 88 | + if blockTags[n.DataAtom] { | |
| 89 | + return true | |
| 90 | + } | |
| 91 | + return c.containsBlock(n) | |
| 92 | +} | |
| 93 | + | |
| 94 | +func (c *converter) containsBlock(n *html.Node) bool { | |
| 95 | + if known, ok := c.holdsBlock[n]; ok { | |
| 96 | + return known | |
| 97 | + } | |
| 98 | + found := false | |
| 99 | + for ch := n.FirstChild; ch != nil && !found; ch = ch.NextSibling { | |
| 100 | + found = ch.Type == html.ElementNode && (blockTags[ch.DataAtom] || c.containsBlock(ch)) | |
| 101 | + } | |
| 102 | + c.holdsBlock[n] = found | |
| 103 | + return found | |
| 104 | +} | |
| 105 | + | |
| 106 | +// blocks renders the children of n: each block child becomes a block, and | |
| 107 | +// each run of inline children between them becomes a paragraph. Empty | |
| 108 | +// results are dropped, so an empty element leaves no blank line behind. | |
| 109 | +func (c *converter) blocks(n *html.Node) []block { | |
| 110 | + var out []block | |
| 111 | + var run []string | |
| 112 | + flush := func() { | |
| 113 | + if p := paragraph(joinInline(run)); p != "" { | |
| 114 | + out = append(out, block{md: p}) | |
| 115 | + } | |
| 116 | + run = run[:0] | |
| 117 | + } | |
| 118 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | |
| 119 | + if !c.isBlock(ch) { | |
| 120 | + run = append(run, c.inline(ch)) | |
| 121 | + continue | |
| 122 | + } | |
| 123 | + flush() | |
| 124 | + if md := c.block(ch); md != "" { | |
| 125 | + out = append(out, block{md: md, list: ch.DataAtom == atom.Ul || ch.DataAtom == atom.Ol}) | |
| 126 | + } | |
| 127 | + } | |
| 128 | + flush() | |
| 129 | + return out | |
| 130 | +} | |
| 131 | + | |
| 132 | +// resolveURLs makes every link and image source under n absolute. | |
| 133 | +// A value that does not parse is left as it is. | |
| 134 | +func resolveURLs(n *html.Node, base *url.URL) { | |
| 135 | + if n.Type == html.ElementNode { | |
| 136 | + for i, a := range n.Attr { | |
| 137 | + if (n.DataAtom == atom.A && a.Key == "href") || (n.DataAtom == atom.Img && a.Key == "src") { | |
| 138 | + if u, err := base.Parse(strings.TrimSpace(a.Val)); err == nil { | |
| 139 | + n.Attr[i].Val = u.String() | |
| 140 | + } | |
| 141 | + } | |
| 142 | + } | |
| 143 | + } | |
| 144 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | |
| 145 | + resolveURLs(ch, base) | |
| 146 | + } | |
| 147 | +} | |
| 148 | + | |
| 149 | +// findBody returns the <body> of a parsed document (the parser always | |
| 150 | +// creates one), or the document itself. | |
| 151 | +func findBody(n *html.Node) *html.Node { | |
| 152 | + if n.DataAtom == atom.Body { | |
| 153 | + return n | |
| 154 | + } | |
| 155 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | |
| 156 | + if body := findBody(ch); body != nil { | |
| 157 | + return body | |
| 158 | + } | |
| 159 | + } | |
| 160 | + if n.Parent == nil { | |
| 161 | + return n | |
| 162 | + } | |
| 163 | + return nil | |
| 164 | +} | |
| 165 | + | |
| 166 | +// block renders one block element. | |
| 167 | +func (c *converter) block(n *html.Node) string { | |
| 168 | + if level, ok := headingLevels[n.DataAtom]; ok { | |
| 169 | + return heading(level, c.children(n)) | |
| 170 | + } | |
| 171 | + var md string | |
| 172 | + switch n.DataAtom { | |
| 173 | + case atom.Hr: | |
| 174 | + md = "---" | |
| 175 | + case atom.Pre: | |
| 176 | + md = codeBlock(n) | |
| 177 | + case atom.Blockquote: | |
| 178 | + md = quote(joinBlocks(c.blocks(n))) | |
| 179 | + case atom.Ul, atom.Ol: | |
| 180 | + md = c.list(n, n.DataAtom == atom.Ol) | |
| 181 | + case atom.Table: | |
| 182 | + md = c.table(n) | |
| 183 | + default: | |
| 184 | + md = joinBlocks(c.blocks(n)) | |
| 185 | + } | |
| 186 | + return md | |
| 187 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,187 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "bytes" | ||
| 5 | + "net/url" | ||
| 6 | + "strings" | ||
| 7 | + | ||
| 8 | + "golang.org/x/net/html" | ||
| 9 | + "golang.org/x/net/html/atom" | ||
| 10 | +) | ||
| 11 | + | ||
| 12 | +// blockTags are the elements rendered as blocks of their own. Any other | ||
| 13 | +// element is inline — unless it contains one of these, in which case it is | ||
| 14 | +// a transparent container (see converter.isBlock). | ||
| 15 | +var blockTags = map[atom.Atom]bool{ | ||
| 16 | + atom.Html: true, atom.Body: true, atom.Main: true, atom.Article: true, atom.Section: true, | ||
| 17 | + atom.Header: true, atom.Footer: true, atom.Div: true, atom.P: true, atom.Address: true, | ||
| 18 | + atom.H1: true, atom.H2: true, atom.H3: true, atom.H4: true, atom.H5: true, atom.H6: true, | ||
| 19 | + atom.Ul: true, atom.Ol: true, atom.Li: true, atom.Dl: true, atom.Dt: true, atom.Dd: true, | ||
| 20 | + atom.Pre: true, atom.Blockquote: true, atom.Hr: true, atom.Table: true, atom.Caption: true, | ||
| 21 | + atom.Figure: true, atom.Figcaption: true, atom.Details: true, atom.Summary: true, | ||
| 22 | + atom.Form: true, atom.Fieldset: true, atom.Legend: true, atom.Center: true, | ||
| 23 | +} | ||
| 24 | + | ||
| 25 | +var headingLevels = map[atom.Atom]int{ | ||
| 26 | + atom.H1: 1, atom.H2: 2, atom.H3: 3, atom.H4: 4, atom.H5: 5, atom.H6: 6, | ||
| 27 | +} | ||
| 28 | + | ||
| 29 | +// Convert parses an HTML document — invalid or partial HTML included: the | ||
| 30 | +// parser repairs it the way a browser does — cleans it (see Clean) and | ||
| 31 | +// renders its body as Markdown. | ||
| 32 | +// | ||
| 33 | +// The result ends with exactly one newline, or is "" when the page has no | ||
| 34 | +// content left once cleaned. Nothing is added: no title, no front matter. | ||
| 35 | +// | ||
| 36 | +// md, err := webmd.Convert([]byte("<h1>Hi</h1><p>A <b>bold</b> move.</p>"), webmd.Options{}) | ||
| 37 | +// // md == "# Hi\n\nA **bold** move.\n" | ||
| 38 | +func Convert(page []byte, opts Options) (string, error) { | ||
| 39 | + doc, err := html.Parse(bytes.NewReader(page)) | ||
| 40 | + if err != nil { | ||
| 41 | + return "", errorf(CodeConversion, err, "cannot parse the HTML") | ||
| 42 | + } | ||
| 43 | + Clean(doc) | ||
| 44 | + if opts.BaseURL != nil { | ||
| 45 | + resolveURLs(doc, opts.BaseURL) | ||
| 46 | + } | ||
| 47 | + if opts.Tables == "" { | ||
| 48 | + opts.Tables = TablesHTML | ||
| 49 | + } | ||
| 50 | + | ||
| 51 | + c := &converter{opts: opts, holdsBlock: map[*html.Node]bool{}} | ||
| 52 | + md := strings.Trim(joinBlocks(c.blocks(findBody(doc))), "\n") | ||
| 53 | + if strings.TrimSpace(md) == "" { | ||
| 54 | + return "", nil | ||
| 55 | + } | ||
| 56 | + return md + "\n", nil | ||
| 57 | +} | ||
| 58 | + | ||
| 59 | +// converter carries the options and a memo of which inline elements hold a | ||
| 60 | +// block, so that deciding it stays linear on deeply nested pages. | ||
| 61 | +type converter struct { | ||
| 62 | + opts Options | ||
| 63 | + holdsBlock map[*html.Node]bool | ||
| 64 | +} | ||
| 65 | + | ||
| 66 | +// block is one rendered Markdown block. list tells a list item whether its | ||
| 67 | +// parts can be packed tightly (text followed by a nested list) or need blank | ||
| 68 | +// lines between them. | ||
| 69 | +type block struct { | ||
| 70 | + md string | ||
| 71 | + list bool | ||
| 72 | +} | ||
| 73 | + | ||
| 74 | +func joinBlocks(parts []block) string { | ||
| 75 | + texts := make([]string, len(parts)) | ||
| 76 | + for i, p := range parts { | ||
| 77 | + texts[i] = p.md | ||
| 78 | + } | ||
| 79 | + return strings.Join(texts, "\n\n") | ||
| 80 | +} | ||
| 81 | + | ||
| 82 | +// isBlock reports whether n renders as a block: a block tag, or any other | ||
| 83 | +// element wrapping one (<a><div>…</div></a>, an unknown <my-card>). | ||
| 84 | +func (c *converter) isBlock(n *html.Node) bool { | ||
| 85 | + if n.Type != html.ElementNode { | ||
| 86 | + return false | ||
| 87 | + } | ||
| 88 | + if blockTags[n.DataAtom] { | ||
| 89 | + return true | ||
| 90 | + } | ||
| 91 | + return c.containsBlock(n) | ||
| 92 | +} | ||
| 93 | + | ||
| 94 | +func (c *converter) containsBlock(n *html.Node) bool { | ||
| 95 | + if known, ok := c.holdsBlock[n]; ok { | ||
| 96 | + return known | ||
| 97 | + } | ||
| 98 | + found := false | ||
| 99 | + for ch := n.FirstChild; ch != nil && !found; ch = ch.NextSibling { | ||
| 100 | + found = ch.Type == html.ElementNode && (blockTags[ch.DataAtom] || c.containsBlock(ch)) | ||
| 101 | + } | ||
| 102 | + c.holdsBlock[n] = found | ||
| 103 | + return found | ||
| 104 | +} | ||
| 105 | + | ||
| 106 | +// blocks renders the children of n: each block child becomes a block, and | ||
| 107 | +// each run of inline children between them becomes a paragraph. Empty | ||
| 108 | +// results are dropped, so an empty element leaves no blank line behind. | ||
| 109 | +func (c *converter) blocks(n *html.Node) []block { | ||
| 110 | + var out []block | ||
| 111 | + var run []string | ||
| 112 | + flush := func() { | ||
| 113 | + if p := paragraph(joinInline(run)); p != "" { | ||
| 114 | + out = append(out, block{md: p}) | ||
| 115 | + } | ||
| 116 | + run = run[:0] | ||
| 117 | + } | ||
| 118 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | ||
| 119 | + if !c.isBlock(ch) { | ||
| 120 | + run = append(run, c.inline(ch)) | ||
| 121 | + continue | ||
| 122 | + } | ||
| 123 | + flush() | ||
| 124 | + if md := c.block(ch); md != "" { | ||
| 125 | + out = append(out, block{md: md, list: ch.DataAtom == atom.Ul || ch.DataAtom == atom.Ol}) | ||
| 126 | + } | ||
| 127 | + } | ||
| 128 | + flush() | ||
| 129 | + return out | ||
| 130 | +} | ||
| 131 | + | ||
| 132 | +// resolveURLs makes every link and image source under n absolute. | ||
| 133 | +// A value that does not parse is left as it is. | ||
| 134 | +func resolveURLs(n *html.Node, base *url.URL) { | ||
| 135 | + if n.Type == html.ElementNode { | ||
| 136 | + for i, a := range n.Attr { | ||
| 137 | + if (n.DataAtom == atom.A && a.Key == "href") || (n.DataAtom == atom.Img && a.Key == "src") { | ||
| 138 | + if u, err := base.Parse(strings.TrimSpace(a.Val)); err == nil { | ||
| 139 | + n.Attr[i].Val = u.String() | ||
| 140 | + } | ||
| 141 | + } | ||
| 142 | + } | ||
| 143 | + } | ||
| 144 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | ||
| 145 | + resolveURLs(ch, base) | ||
| 146 | + } | ||
| 147 | +} | ||
| 148 | + | ||
| 149 | +// findBody returns the <body> of a parsed document (the parser always | ||
| 150 | +// creates one), or the document itself. | ||
| 151 | +func findBody(n *html.Node) *html.Node { | ||
| 152 | + if n.DataAtom == atom.Body { | ||
| 153 | + return n | ||
| 154 | + } | ||
| 155 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | ||
| 156 | + if body := findBody(ch); body != nil { | ||
| 157 | + return body | ||
| 158 | + } | ||
| 159 | + } | ||
| 160 | + if n.Parent == nil { | ||
| 161 | + return n | ||
| 162 | + } | ||
| 163 | + return nil | ||
| 164 | +} | ||
| 165 | + | ||
| 166 | +// block renders one block element. | ||
| 167 | +func (c *converter) block(n *html.Node) string { | ||
| 168 | + if level, ok := headingLevels[n.DataAtom]; ok { | ||
| 169 | + return heading(level, c.children(n)) | ||
| 170 | + } | ||
| 171 | + var md string | ||
| 172 | + switch n.DataAtom { | ||
| 173 | + case atom.Hr: | ||
| 174 | + md = "---" | ||
| 175 | + case atom.Pre: | ||
| 176 | + md = codeBlock(n) | ||
| 177 | + case atom.Blockquote: | ||
| 178 | + md = quote(joinBlocks(c.blocks(n))) | ||
| 179 | + case atom.Ul, atom.Ol: | ||
| 180 | + md = c.list(n, n.DataAtom == atom.Ol) | ||
| 181 | + case atom.Table: | ||
| 182 | + md = c.table(n) | ||
| 183 | + default: | ||
| 184 | + md = joinBlocks(c.blocks(n)) | ||
| 185 | + } | ||
| 186 | + return md | ||
| 187 | +} | ||
added
internal/webmd/convert_test.go +168 -0 | new file mode 100644 | ||
| @@ -0,0 +1,168 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "net/url" | |
| 5 | + "strings" | |
| 6 | + "testing" | |
| 7 | +) | |
| 8 | + | |
| 9 | +func convert(t *testing.T, page string, opts Options) string { | |
| 10 | + t.Helper() | |
| 11 | + md, err := Convert([]byte(page), opts) | |
| 12 | + if err != nil { | |
| 13 | + t.Fatalf("Convert: %v", err) | |
| 14 | + } | |
| 15 | + return md | |
| 16 | +} | |
| 17 | + | |
| 18 | +type convertCase struct { | |
| 19 | + name, html, want string | |
| 20 | + opts Options | |
| 21 | +} | |
| 22 | + | |
| 23 | +func runCases(t *testing.T, cases []convertCase) { | |
| 24 | + t.Helper() | |
| 25 | + for _, tc := range cases { | |
| 26 | + t.Run(tc.name, func(t *testing.T) { | |
| 27 | + if got := convert(t, tc.html, tc.opts); got != tc.want { | |
| 28 | + t.Errorf("got:\n%q\nwant:\n%q", got, tc.want) | |
| 29 | + } | |
| 30 | + }) | |
| 31 | + } | |
| 32 | +} | |
| 33 | + | |
| 34 | +func TestConvertTextAndFormatting(t *testing.T) { | |
| 35 | + runCases(t, []convertCase{ | |
| 36 | + {name: "headings h1 to h6", html: "<h1>One</h1><h2>Two</h2><h3>3</h3><h4>4</h4><h5>5</h5><h6>6</h6>", | |
| 37 | + want: "# One\n\n## Two\n\n### 3\n\n#### 4\n\n##### 5\n\n###### 6\n"}, | |
| 38 | + {name: "paragraphs are separated by a blank line", html: "<p>First</p>\n\n<p>Second\nline</p>", | |
| 39 | + want: "First\n\nSecond line\n"}, | |
| 40 | + {name: "strong and b", html: "<p>a<strong> bold </strong>b and <b>c</b></p>", want: "a **bold** b and **c**\n"}, | |
| 41 | + {name: "em and i", html: "<p><em>x</em> <i>y</i></p>", want: "*x* *y*\n"}, | |
| 42 | + {name: "inline code keeps its text", html: "<p>run <code>go test ./...</code></p>", want: "run `go test ./...`\n"}, | |
| 43 | + {name: "inline code with backticks", html: "<p><code>a`b</code></p>", want: "``a`b``\n"}, | |
| 44 | + {name: "pre code becomes a fenced block with its language", | |
| 45 | + html: "<pre><code class=\"language-go\">func main() {\n\tfmt.Println(\"*hi*\")\n}\n</code></pre>", | |
| 46 | + want: "```go\nfunc main() {\n\tfmt.Println(\"*hi*\")\n}\n```\n"}, | |
| 47 | + {name: "fence longer than a backtick run in the code", html: "<pre>```\nx\n```</pre>", want: "````\n```\nx\n```\n````\n"}, | |
| 48 | + {name: "br is a hard break", html: "<p>line one<br>line two</p>", want: "line one\\\nline two\n"}, | |
| 49 | + {name: "trailing br adds nothing", html: "<p>alone<br><br></p>", want: "alone\n"}, | |
| 50 | + {name: "hr", html: "<p>a</p><hr><p>b</p>", want: "a\n\n---\n\nb\n"}, | |
| 51 | + {name: "blockquote", html: "<blockquote><p>one</p><p>two</p></blockquote>", want: "> one\n>\n> two\n"}, | |
| 52 | + {name: "strikethrough off keeps the text", html: "<p><del>old</del> <s>x</s> <strike>y</strike></p>", want: "old x y\n"}, | |
| 53 | + {name: "strikethrough on", html: "<p><del>old</del> new</p>", opts: Options{Strikethrough: true}, want: "~~old~~ new\n"}, | |
| 54 | + {name: "markdown characters in text are escaped", html: "<p>2 * 3 = [six] and a_b _c_</p>", want: "2 \\* 3 = \\[six\\] and a_b \\_c\\_\n"}, | |
| 55 | + {name: "a line looking like a heading is escaped", html: "<p># not a title</p><p>1. not a list</p>", want: "\\# not a title\n\n1\\. not a list\n"}, | |
| 56 | + }) | |
| 57 | +} | |
| 58 | + | |
| 59 | +func TestConvertFallbacks(t *testing.T) { | |
| 60 | + runCases(t, []convertCase{ | |
| 61 | + {name: "invalid HTML is repaired", html: "<p>open <b>bold <i>both</p><p>next", want: "open **bold *both***\n\n***next***\n"}, | |
| 62 | + {name: "unknown element keeps its text", html: "<p>a <x-widget>custom</x-widget> b</p>", want: "a custom b\n"}, | |
| 63 | + {name: "unknown element holding blocks is a container", html: "<my-card><h2>T</h2><p>body</p></my-card>", want: "## T\n\nbody\n"}, | |
| 64 | + {name: "empty elements leave no blank line", html: "<p>a</p><p> </p><div><span></span></div><h2></h2><ul><li></li></ul><p>b</p>", want: "a\n\nb\n"}, | |
| 65 | + {name: "empty page", html: "<html><head><title>t</title></head><body> </body></html>", want: ""}, | |
| 66 | + {name: "no metadata is added", html: "<html><head><title>Title</title><meta name=\"author\" content=\"me\"></head><body><p>x</p></body></html>", want: "x\n"}, | |
| 67 | + }) | |
| 68 | +} | |
| 69 | + | |
| 70 | +func TestConvertLinksAndImages(t *testing.T) { | |
| 71 | + base, _ := url.Parse("https://example.com/docs/page.html") | |
| 72 | + runCases(t, []convertCase{ | |
| 73 | + {name: "link", html: `<a href="https://go.dev/">Go</a>`, want: "[Go](https://go.dev/)\n"}, | |
| 74 | + {name: "link without text falls back to its URL", html: `<a href="https://go.dev/"></a>`, want: "[https://go.dev/](https://go.dev/)\n"}, | |
| 75 | + {name: "relative link resolved against the base", html: `<a href="../img/x.html">x</a>`, opts: Options{BaseURL: base}, | |
| 76 | + want: "[x](https://example.com/img/x.html)\n"}, | |
| 77 | + {name: "relative link kept without a base", html: `<a href="x.html">x</a>`, want: "[x](x.html)\n"}, | |
| 78 | + {name: "spaces and parentheses are encoded", html: `<a href="a b(1).html">x</a>`, want: "[x](a%20b%281%29.html)\n"}, | |
| 79 | + {name: "javascript link keeps only its text", html: `<a href="javascript:void(0)">click</a>`, want: "click\n"}, | |
| 80 | + {name: "image", html: `<img src="/logo.png" alt="Logo">`, opts: Options{BaseURL: base}, want: "\n"}, | |
| 81 | + {name: "image without alt", html: `<img src="https://x.org/a.png">`, want: "\n"}, | |
| 82 | + {name: "data URI image is reduced to its alt", html: `<p>see <img src="data:image/png;base64,AAAA" alt="chart"></p>`, want: "see chart\n"}, | |
| 83 | + {name: "image inside a link", html: `<a href="https://x.org/"><img src="https://x.org/a.png" alt="A"></a>`, | |
| 84 | + want: "[](https://x.org/)\n"}, | |
| 85 | + }) | |
| 86 | +} | |
| 87 | + | |
| 88 | +func TestConvertLists(t *testing.T) { | |
| 89 | + runCases(t, []convertCase{ | |
| 90 | + {name: "unordered", html: "<ul><li>a</li><li>b</li></ul>", want: "- a\n- b\n"}, | |
| 91 | + {name: "ordered with start", html: `<ol start="3"><li>c</li><li>d</li></ol>`, want: "3. c\n4. d\n"}, | |
| 92 | + {name: "nested lists are indented under their item", | |
| 93 | + html: "<ul><li>a<ul><li>a1<ol><li>deep</li></ol></li></ul></li><li>b</li></ul>", | |
| 94 | + want: "- a\n - a1\n 1. deep\n- b\n"}, | |
| 95 | + {name: "item with two paragraphs is loose", html: "<ul><li><p>one</p><p>two</p></li></ul>", want: "- one\n\n two\n"}, | |
| 96 | + {name: "stray nested list belongs to the previous item", html: "<ul><li>a</li><ul><li>b</li></ul></ul>", want: "- a\n - b\n"}, | |
| 97 | + {name: "checkboxes dropped without task-list", html: `<ul><li><input type="checkbox" checked> done</li></ul>`, want: "- done\n"}, | |
| 98 | + {name: "task list", opts: Options{TaskList: true}, | |
| 99 | + html: `<ul><li><input type="checkbox" checked> done</li><li><label><input type="checkbox"> todo</label></li><li>plain</li></ul>`, | |
| 100 | + want: "- [x] done\n- [ ] todo\n- plain\n"}, | |
| 101 | + }) | |
| 102 | +} | |
| 103 | + | |
| 104 | +func TestConvertTables(t *testing.T) { | |
| 105 | + simple := "<table><thead><tr><th>A</th><th>B</th></tr></thead><tbody><tr><td>1</td><td>x | y</td></tr><tr><td>2</td></tr></tbody></table>" | |
| 106 | + spanned := `<table><tr><th>K</th><th>V</th></tr><tr><td rowspan="2" style="color:red">k</td><td colspan="1">v1</td></tr><tr><td>v2</td></tr><tr><td colspan="2">both</td></tr></table>` | |
| 107 | + runCases(t, []convertCase{ | |
| 108 | + {name: "simple table becomes GFM", html: simple, | |
| 109 | + want: "| A | B |\n|---|---|\n| 1 | x \\| y |\n| 2 | |\n"}, | |
| 110 | + {name: "first row is the header even without th", html: "<table><tr><td>a</td><td>b</td></tr><tr><td>c</td><td>d</td></tr></table>", | |
| 111 | + want: "| a | b |\n|---|---|\n| c | d |\n"}, | |
| 112 | + {name: "caption goes above", html: "<table><caption>Prices</caption><tr><td>a</td></tr></table>", want: "Prices\n\n| a |\n|---|\n"}, | |
| 113 | + {name: "br and paragraphs in a cell", html: "<table><tr><td><p>one</p><p>two<br>three</p></td></tr></table>", | |
| 114 | + want: "| one<br>two<br>three |\n|---|\n"}, | |
| 115 | + {name: "spans kept as raw HTML by default", html: spanned, | |
| 116 | + want: "<table><tbody><tr><th>K</th><th>V</th></tr><tr><td rowspan=\"2\">k</td><td colspan=\"1\">v1</td></tr><tr><td>v2</td></tr><tr><td colspan=\"2\">both</td></tr></tbody></table>\n"}, | |
| 117 | + {name: "spans flattened on request", html: spanned, opts: Options{Tables: TablesFlatten}, | |
| 118 | + want: "| K | V |\n|---|---|\n| k | v1 |\n| k | v2 |\n| both | both |\n"}, | |
| 119 | + {name: "nested table is complex", html: "<table><tr><td><table><tr><td>in</td></tr></table></td></tr></table>", | |
| 120 | + want: "<table><tbody><tr><td><table><tbody><tr><td>in</td></tr></tbody></table></td></tr></tbody></table>\n"}, | |
| 121 | + {name: "empty table", html: "<table><tr><td></td></tr></table>", want: ""}, | |
| 122 | + }) | |
| 123 | +} | |
| 124 | + | |
| 125 | +func TestConvertRemovesChrome(t *testing.T) { | |
| 126 | + page := `<html><head><style>p{}</style><script>alert(1)</script><link rel="stylesheet" href="a.css"><base href="/"></head> | |
| 127 | +<body> | |
| 128 | +<!-- a comment --> | |
| 129 | +<nav><a href="/">Home</a></nav> | |
| 130 | +<div role="navigation">menu</div> | |
| 131 | +<div class="cookie-banner">We use cookies</div> | |
| 132 | +<div id="sidebar">side</div> | |
| 133 | +<div class="ad-slot">BUY</div> | |
| 134 | +<span class="sr-only">Skip to content</span> | |
| 135 | +<div class="page-with-sidebar"><article><h1>Title</h1><p style="color:red" data-x="1" class="lead" id="p1">Text</p><noscript>enable js</noscript></article></div> | |
| 136 | +<footer>© site</footer> | |
| 137 | +<script>track()</script> | |
| 138 | +</body></html>` | |
| 139 | + got := convert(t, page, Options{}) | |
| 140 | + if want := "# Title\n\nText\n"; got != want { | |
| 141 | + t.Errorf("got %q, want %q", got, want) | |
| 142 | + } | |
| 143 | +} | |
| 144 | + | |
| 145 | +func TestConvertKeepsFooterInsideArticle(t *testing.T) { | |
| 146 | + got := convert(t, "<article><p>body</p><footer>by Ann</footer></article>", Options{}) | |
| 147 | + if !strings.Contains(got, "by Ann") { | |
| 148 | + t.Errorf("article footer removed: %q", got) | |
| 149 | + } | |
| 150 | +} | |
| 151 | + | |
| 152 | +func TestConvertEndsWithOneNewline(t *testing.T) { | |
| 153 | + got := convert(t, "<p>x</p>\n\n\n<pre>code\n\n\n</pre>\n\n", Options{}) | |
| 154 | + if !strings.HasSuffix(got, "```\n") || strings.HasSuffix(got, "\n\n") { | |
| 155 | + t.Errorf("ending: %q", got) | |
| 156 | + } | |
| 157 | +} | |
| 158 | + | |
| 159 | +func TestParseTableMode(t *testing.T) { | |
| 160 | + for in, want := range map[string]TableMode{"": TablesHTML, "html": TablesHTML, "flatten": TablesFlatten} { | |
| 161 | + if got, err := ParseTableMode(in); err != nil || got != want { | |
| 162 | + t.Errorf("ParseTableMode(%q) = %q, %v", in, got, err) | |
| 163 | + } | |
| 164 | + } | |
| 165 | + if _, err := ParseTableMode("markdown"); CodeOf(err) != CodeUsage { | |
| 166 | + t.Errorf("ParseTableMode(markdown) error code = %v, want usage", CodeOf(err)) | |
| 167 | + } | |
| 168 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,168 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "net/url" | ||
| 5 | + "strings" | ||
| 6 | + "testing" | ||
| 7 | +) | ||
| 8 | + | ||
| 9 | +func convert(t *testing.T, page string, opts Options) string { | ||
| 10 | + t.Helper() | ||
| 11 | + md, err := Convert([]byte(page), opts) | ||
| 12 | + if err != nil { | ||
| 13 | + t.Fatalf("Convert: %v", err) | ||
| 14 | + } | ||
| 15 | + return md | ||
| 16 | +} | ||
| 17 | + | ||
| 18 | +type convertCase struct { | ||
| 19 | + name, html, want string | ||
| 20 | + opts Options | ||
| 21 | +} | ||
| 22 | + | ||
| 23 | +func runCases(t *testing.T, cases []convertCase) { | ||
| 24 | + t.Helper() | ||
| 25 | + for _, tc := range cases { | ||
| 26 | + t.Run(tc.name, func(t *testing.T) { | ||
| 27 | + if got := convert(t, tc.html, tc.opts); got != tc.want { | ||
| 28 | + t.Errorf("got:\n%q\nwant:\n%q", got, tc.want) | ||
| 29 | + } | ||
| 30 | + }) | ||
| 31 | + } | ||
| 32 | +} | ||
| 33 | + | ||
| 34 | +func TestConvertTextAndFormatting(t *testing.T) { | ||
| 35 | + runCases(t, []convertCase{ | ||
| 36 | + {name: "headings h1 to h6", html: "<h1>One</h1><h2>Two</h2><h3>3</h3><h4>4</h4><h5>5</h5><h6>6</h6>", | ||
| 37 | + want: "# One\n\n## Two\n\n### 3\n\n#### 4\n\n##### 5\n\n###### 6\n"}, | ||
| 38 | + {name: "paragraphs are separated by a blank line", html: "<p>First</p>\n\n<p>Second\nline</p>", | ||
| 39 | + want: "First\n\nSecond line\n"}, | ||
| 40 | + {name: "strong and b", html: "<p>a<strong> bold </strong>b and <b>c</b></p>", want: "a **bold** b and **c**\n"}, | ||
| 41 | + {name: "em and i", html: "<p><em>x</em> <i>y</i></p>", want: "*x* *y*\n"}, | ||
| 42 | + {name: "inline code keeps its text", html: "<p>run <code>go test ./...</code></p>", want: "run `go test ./...`\n"}, | ||
| 43 | + {name: "inline code with backticks", html: "<p><code>a`b</code></p>", want: "``a`b``\n"}, | ||
| 44 | + {name: "pre code becomes a fenced block with its language", | ||
| 45 | + html: "<pre><code class=\"language-go\">func main() {\n\tfmt.Println(\"*hi*\")\n}\n</code></pre>", | ||
| 46 | + want: "```go\nfunc main() {\n\tfmt.Println(\"*hi*\")\n}\n```\n"}, | ||
| 47 | + {name: "fence longer than a backtick run in the code", html: "<pre>```\nx\n```</pre>", want: "````\n```\nx\n```\n````\n"}, | ||
| 48 | + {name: "br is a hard break", html: "<p>line one<br>line two</p>", want: "line one\\\nline two\n"}, | ||
| 49 | + {name: "trailing br adds nothing", html: "<p>alone<br><br></p>", want: "alone\n"}, | ||
| 50 | + {name: "hr", html: "<p>a</p><hr><p>b</p>", want: "a\n\n---\n\nb\n"}, | ||
| 51 | + {name: "blockquote", html: "<blockquote><p>one</p><p>two</p></blockquote>", want: "> one\n>\n> two\n"}, | ||
| 52 | + {name: "strikethrough off keeps the text", html: "<p><del>old</del> <s>x</s> <strike>y</strike></p>", want: "old x y\n"}, | ||
| 53 | + {name: "strikethrough on", html: "<p><del>old</del> new</p>", opts: Options{Strikethrough: true}, want: "~~old~~ new\n"}, | ||
| 54 | + {name: "markdown characters in text are escaped", html: "<p>2 * 3 = [six] and a_b _c_</p>", want: "2 \\* 3 = \\[six\\] and a_b \\_c\\_\n"}, | ||
| 55 | + {name: "a line looking like a heading is escaped", html: "<p># not a title</p><p>1. not a list</p>", want: "\\# not a title\n\n1\\. not a list\n"}, | ||
| 56 | + }) | ||
| 57 | +} | ||
| 58 | + | ||
| 59 | +func TestConvertFallbacks(t *testing.T) { | ||
| 60 | + runCases(t, []convertCase{ | ||
| 61 | + {name: "invalid HTML is repaired", html: "<p>open <b>bold <i>both</p><p>next", want: "open **bold *both***\n\n***next***\n"}, | ||
| 62 | + {name: "unknown element keeps its text", html: "<p>a <x-widget>custom</x-widget> b</p>", want: "a custom b\n"}, | ||
| 63 | + {name: "unknown element holding blocks is a container", html: "<my-card><h2>T</h2><p>body</p></my-card>", want: "## T\n\nbody\n"}, | ||
| 64 | + {name: "empty elements leave no blank line", html: "<p>a</p><p> </p><div><span></span></div><h2></h2><ul><li></li></ul><p>b</p>", want: "a\n\nb\n"}, | ||
| 65 | + {name: "empty page", html: "<html><head><title>t</title></head><body> </body></html>", want: ""}, | ||
| 66 | + {name: "no metadata is added", html: "<html><head><title>Title</title><meta name=\"author\" content=\"me\"></head><body><p>x</p></body></html>", want: "x\n"}, | ||
| 67 | + }) | ||
| 68 | +} | ||
| 69 | + | ||
| 70 | +func TestConvertLinksAndImages(t *testing.T) { | ||
| 71 | + base, _ := url.Parse("https://example.com/docs/page.html") | ||
| 72 | + runCases(t, []convertCase{ | ||
| 73 | + {name: "link", html: `<a href="https://go.dev/">Go</a>`, want: "[Go](https://go.dev/)\n"}, | ||
| 74 | + {name: "link without text falls back to its URL", html: `<a href="https://go.dev/"></a>`, want: "[https://go.dev/](https://go.dev/)\n"}, | ||
| 75 | + {name: "relative link resolved against the base", html: `<a href="../img/x.html">x</a>`, opts: Options{BaseURL: base}, | ||
| 76 | + want: "[x](https://example.com/img/x.html)\n"}, | ||
| 77 | + {name: "relative link kept without a base", html: `<a href="x.html">x</a>`, want: "[x](x.html)\n"}, | ||
| 78 | + {name: "spaces and parentheses are encoded", html: `<a href="a b(1).html">x</a>`, want: "[x](a%20b%281%29.html)\n"}, | ||
| 79 | + {name: "javascript link keeps only its text", html: `<a href="javascript:void(0)">click</a>`, want: "click\n"}, | ||
| 80 | + {name: "image", html: `<img src="/logo.png" alt="Logo">`, opts: Options{BaseURL: base}, want: "\n"}, | ||
| 81 | + {name: "image without alt", html: `<img src="https://x.org/a.png">`, want: "\n"}, | ||
| 82 | + {name: "data URI image is reduced to its alt", html: `<p>see <img src="data:image/png;base64,AAAA" alt="chart"></p>`, want: "see chart\n"}, | ||
| 83 | + {name: "image inside a link", html: `<a href="https://x.org/"><img src="https://x.org/a.png" alt="A"></a>`, | ||
| 84 | + want: "[](https://x.org/)\n"}, | ||
| 85 | + }) | ||
| 86 | +} | ||
| 87 | + | ||
| 88 | +func TestConvertLists(t *testing.T) { | ||
| 89 | + runCases(t, []convertCase{ | ||
| 90 | + {name: "unordered", html: "<ul><li>a</li><li>b</li></ul>", want: "- a\n- b\n"}, | ||
| 91 | + {name: "ordered with start", html: `<ol start="3"><li>c</li><li>d</li></ol>`, want: "3. c\n4. d\n"}, | ||
| 92 | + {name: "nested lists are indented under their item", | ||
| 93 | + html: "<ul><li>a<ul><li>a1<ol><li>deep</li></ol></li></ul></li><li>b</li></ul>", | ||
| 94 | + want: "- a\n - a1\n 1. deep\n- b\n"}, | ||
| 95 | + {name: "item with two paragraphs is loose", html: "<ul><li><p>one</p><p>two</p></li></ul>", want: "- one\n\n two\n"}, | ||
| 96 | + {name: "stray nested list belongs to the previous item", html: "<ul><li>a</li><ul><li>b</li></ul></ul>", want: "- a\n - b\n"}, | ||
| 97 | + {name: "checkboxes dropped without task-list", html: `<ul><li><input type="checkbox" checked> done</li></ul>`, want: "- done\n"}, | ||
| 98 | + {name: "task list", opts: Options{TaskList: true}, | ||
| 99 | + html: `<ul><li><input type="checkbox" checked> done</li><li><label><input type="checkbox"> todo</label></li><li>plain</li></ul>`, | ||
| 100 | + want: "- [x] done\n- [ ] todo\n- plain\n"}, | ||
| 101 | + }) | ||
| 102 | +} | ||
| 103 | + | ||
| 104 | +func TestConvertTables(t *testing.T) { | ||
| 105 | + simple := "<table><thead><tr><th>A</th><th>B</th></tr></thead><tbody><tr><td>1</td><td>x | y</td></tr><tr><td>2</td></tr></tbody></table>" | ||
| 106 | + spanned := `<table><tr><th>K</th><th>V</th></tr><tr><td rowspan="2" style="color:red">k</td><td colspan="1">v1</td></tr><tr><td>v2</td></tr><tr><td colspan="2">both</td></tr></table>` | ||
| 107 | + runCases(t, []convertCase{ | ||
| 108 | + {name: "simple table becomes GFM", html: simple, | ||
| 109 | + want: "| A | B |\n|---|---|\n| 1 | x \\| y |\n| 2 | |\n"}, | ||
| 110 | + {name: "first row is the header even without th", html: "<table><tr><td>a</td><td>b</td></tr><tr><td>c</td><td>d</td></tr></table>", | ||
| 111 | + want: "| a | b |\n|---|---|\n| c | d |\n"}, | ||
| 112 | + {name: "caption goes above", html: "<table><caption>Prices</caption><tr><td>a</td></tr></table>", want: "Prices\n\n| a |\n|---|\n"}, | ||
| 113 | + {name: "br and paragraphs in a cell", html: "<table><tr><td><p>one</p><p>two<br>three</p></td></tr></table>", | ||
| 114 | + want: "| one<br>two<br>three |\n|---|\n"}, | ||
| 115 | + {name: "spans kept as raw HTML by default", html: spanned, | ||
| 116 | + want: "<table><tbody><tr><th>K</th><th>V</th></tr><tr><td rowspan=\"2\">k</td><td colspan=\"1\">v1</td></tr><tr><td>v2</td></tr><tr><td colspan=\"2\">both</td></tr></tbody></table>\n"}, | ||
| 117 | + {name: "spans flattened on request", html: spanned, opts: Options{Tables: TablesFlatten}, | ||
| 118 | + want: "| K | V |\n|---|---|\n| k | v1 |\n| k | v2 |\n| both | both |\n"}, | ||
| 119 | + {name: "nested table is complex", html: "<table><tr><td><table><tr><td>in</td></tr></table></td></tr></table>", | ||
| 120 | + want: "<table><tbody><tr><td><table><tbody><tr><td>in</td></tr></tbody></table></td></tr></tbody></table>\n"}, | ||
| 121 | + {name: "empty table", html: "<table><tr><td></td></tr></table>", want: ""}, | ||
| 122 | + }) | ||
| 123 | +} | ||
| 124 | + | ||
| 125 | +func TestConvertRemovesChrome(t *testing.T) { | ||
| 126 | + page := `<html><head><style>p{}</style><script>alert(1)</script><link rel="stylesheet" href="a.css"><base href="/"></head> | ||
| 127 | +<body> | ||
| 128 | +<!-- a comment --> | ||
| 129 | +<nav><a href="/">Home</a></nav> | ||
| 130 | +<div role="navigation">menu</div> | ||
| 131 | +<div class="cookie-banner">We use cookies</div> | ||
| 132 | +<div id="sidebar">side</div> | ||
| 133 | +<div class="ad-slot">BUY</div> | ||
| 134 | +<span class="sr-only">Skip to content</span> | ||
| 135 | +<div class="page-with-sidebar"><article><h1>Title</h1><p style="color:red" data-x="1" class="lead" id="p1">Text</p><noscript>enable js</noscript></article></div> | ||
| 136 | +<footer>© site</footer> | ||
| 137 | +<script>track()</script> | ||
| 138 | +</body></html>` | ||
| 139 | + got := convert(t, page, Options{}) | ||
| 140 | + if want := "# Title\n\nText\n"; got != want { | ||
| 141 | + t.Errorf("got %q, want %q", got, want) | ||
| 142 | + } | ||
| 143 | +} | ||
| 144 | + | ||
| 145 | +func TestConvertKeepsFooterInsideArticle(t *testing.T) { | ||
| 146 | + got := convert(t, "<article><p>body</p><footer>by Ann</footer></article>", Options{}) | ||
| 147 | + if !strings.Contains(got, "by Ann") { | ||
| 148 | + t.Errorf("article footer removed: %q", got) | ||
| 149 | + } | ||
| 150 | +} | ||
| 151 | + | ||
| 152 | +func TestConvertEndsWithOneNewline(t *testing.T) { | ||
| 153 | + got := convert(t, "<p>x</p>\n\n\n<pre>code\n\n\n</pre>\n\n", Options{}) | ||
| 154 | + if !strings.HasSuffix(got, "```\n") || strings.HasSuffix(got, "\n\n") { | ||
| 155 | + t.Errorf("ending: %q", got) | ||
| 156 | + } | ||
| 157 | +} | ||
| 158 | + | ||
| 159 | +func TestParseTableMode(t *testing.T) { | ||
| 160 | + for in, want := range map[string]TableMode{"": TablesHTML, "html": TablesHTML, "flatten": TablesFlatten} { | ||
| 161 | + if got, err := ParseTableMode(in); err != nil || got != want { | ||
| 162 | + t.Errorf("ParseTableMode(%q) = %q, %v", in, got, err) | ||
| 163 | + } | ||
| 164 | + } | ||
| 165 | + if _, err := ParseTableMode("markdown"); CodeOf(err) != CodeUsage { | ||
| 166 | + t.Errorf("ParseTableMode(markdown) error code = %v, want usage", CodeOf(err)) | ||
| 167 | + } | ||
| 168 | +} | ||
added
internal/webmd/errors.go +77 -0 | new file mode 100644 | ||
| @@ -0,0 +1,77 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "errors" | |
| 5 | + "fmt" | |
| 6 | +) | |
| 7 | + | |
| 8 | +// Code classifies a failure. The values are the exit codes of the | |
| 9 | +// fetch-content specification, so a front end can report them as they are. | |
| 10 | +type Code int | |
| 11 | + | |
| 12 | +const ( | |
| 13 | + // CodeOK is success. | |
| 14 | + CodeOK Code = 0 | |
| 15 | + // CodeConversion is a failure while processing the page: not HTML, | |
| 16 | + // too large, unparsable. | |
| 17 | + CodeConversion Code = 1 | |
| 18 | + // CodeInput is a failure to get the page: unreachable host, HTTP | |
| 19 | + // error status, or a robots.txt that forbids it. | |
| 20 | + CodeInput Code = 2 | |
| 21 | + // CodeUsage is an invalid argument: a URL that cannot be parsed, a | |
| 22 | + // scheme other than http(s), an unknown table mode. | |
| 23 | + CodeUsage Code = 3 | |
| 24 | +) | |
| 25 | + | |
| 26 | +// String names the class of failure, for messages. | |
| 27 | +func (c Code) String() string { | |
| 28 | + switch c { | |
| 29 | + case CodeOK: | |
| 30 | + return "ok" | |
| 31 | + case CodeConversion: | |
| 32 | + return "conversion" | |
| 33 | + case CodeInput: | |
| 34 | + return "input" | |
| 35 | + case CodeUsage: | |
| 36 | + return "usage" | |
| 37 | + } | |
| 38 | + return fmt.Sprintf("code %d", int(c)) | |
| 39 | +} | |
| 40 | + | |
| 41 | +// Error is the only error type this package returns. | |
| 42 | +// | |
| 43 | +// var e *webmd.Error | |
| 44 | +// if errors.As(err, &e) && e.Code == webmd.CodeInput { … } | |
| 45 | +type Error struct { | |
| 46 | + Code Code | |
| 47 | + Msg string | |
| 48 | + Err error // the underlying cause, when there is one | |
| 49 | +} | |
| 50 | + | |
| 51 | +func (e *Error) Error() string { | |
| 52 | + if e.Err != nil { | |
| 53 | + return e.Msg + ": " + e.Err.Error() | |
| 54 | + } | |
| 55 | + return e.Msg | |
| 56 | +} | |
| 57 | + | |
| 58 | +func (e *Error) Unwrap() error { return e.Err } | |
| 59 | + | |
| 60 | +// CodeOf returns the Code of err: CodeOK for nil, the Code of an *Error | |
| 61 | +// anywhere in its chain, CodeConversion for anything else. | |
| 62 | +// | |
| 63 | +// os.Exit(int(webmd.CodeOf(err))) | |
| 64 | +func CodeOf(err error) Code { | |
| 65 | + if err == nil { | |
| 66 | + return CodeOK | |
| 67 | + } | |
| 68 | + var e *Error | |
| 69 | + if errors.As(err, &e) { | |
| 70 | + return e.Code | |
| 71 | + } | |
| 72 | + return CodeConversion | |
| 73 | +} | |
| 74 | + | |
| 75 | +func errorf(code Code, cause error, format string, args ...any) *Error { | |
| 76 | + return &Error{Code: code, Msg: fmt.Sprintf(format, args...), Err: cause} | |
| 77 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,77 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "errors" | ||
| 5 | + "fmt" | ||
| 6 | +) | ||
| 7 | + | ||
| 8 | +// Code classifies a failure. The values are the exit codes of the | ||
| 9 | +// fetch-content specification, so a front end can report them as they are. | ||
| 10 | +type Code int | ||
| 11 | + | ||
| 12 | +const ( | ||
| 13 | + // CodeOK is success. | ||
| 14 | + CodeOK Code = 0 | ||
| 15 | + // CodeConversion is a failure while processing the page: not HTML, | ||
| 16 | + // too large, unparsable. | ||
| 17 | + CodeConversion Code = 1 | ||
| 18 | + // CodeInput is a failure to get the page: unreachable host, HTTP | ||
| 19 | + // error status, or a robots.txt that forbids it. | ||
| 20 | + CodeInput Code = 2 | ||
| 21 | + // CodeUsage is an invalid argument: a URL that cannot be parsed, a | ||
| 22 | + // scheme other than http(s), an unknown table mode. | ||
| 23 | + CodeUsage Code = 3 | ||
| 24 | +) | ||
| 25 | + | ||
| 26 | +// String names the class of failure, for messages. | ||
| 27 | +func (c Code) String() string { | ||
| 28 | + switch c { | ||
| 29 | + case CodeOK: | ||
| 30 | + return "ok" | ||
| 31 | + case CodeConversion: | ||
| 32 | + return "conversion" | ||
| 33 | + case CodeInput: | ||
| 34 | + return "input" | ||
| 35 | + case CodeUsage: | ||
| 36 | + return "usage" | ||
| 37 | + } | ||
| 38 | + return fmt.Sprintf("code %d", int(c)) | ||
| 39 | +} | ||
| 40 | + | ||
| 41 | +// Error is the only error type this package returns. | ||
| 42 | +// | ||
| 43 | +// var e *webmd.Error | ||
| 44 | +// if errors.As(err, &e) && e.Code == webmd.CodeInput { … } | ||
| 45 | +type Error struct { | ||
| 46 | + Code Code | ||
| 47 | + Msg string | ||
| 48 | + Err error // the underlying cause, when there is one | ||
| 49 | +} | ||
| 50 | + | ||
| 51 | +func (e *Error) Error() string { | ||
| 52 | + if e.Err != nil { | ||
| 53 | + return e.Msg + ": " + e.Err.Error() | ||
| 54 | + } | ||
| 55 | + return e.Msg | ||
| 56 | +} | ||
| 57 | + | ||
| 58 | +func (e *Error) Unwrap() error { return e.Err } | ||
| 59 | + | ||
| 60 | +// CodeOf returns the Code of err: CodeOK for nil, the Code of an *Error | ||
| 61 | +// anywhere in its chain, CodeConversion for anything else. | ||
| 62 | +// | ||
| 63 | +// os.Exit(int(webmd.CodeOf(err))) | ||
| 64 | +func CodeOf(err error) Code { | ||
| 65 | + if err == nil { | ||
| 66 | + return CodeOK | ||
| 67 | + } | ||
| 68 | + var e *Error | ||
| 69 | + if errors.As(err, &e) { | ||
| 70 | + return e.Code | ||
| 71 | + } | ||
| 72 | + return CodeConversion | ||
| 73 | +} | ||
| 74 | + | ||
| 75 | +func errorf(code Code, cause error, format string, args ...any) *Error { | ||
| 76 | + return &Error{Code: code, Msg: fmt.Sprintf(format, args...), Err: cause} | ||
| 77 | +} | ||
added
internal/webmd/fetch.go +325 -0 | new file mode 100644 | ||
| @@ -0,0 +1,325 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "context" | |
| 5 | + "errors" | |
| 6 | + "fmt" | |
| 7 | + "io" | |
| 8 | + "mime" | |
| 9 | + "net/http" | |
| 10 | + "net/url" | |
| 11 | + "regexp" | |
| 12 | + "strings" | |
| 13 | + "sync" | |
| 14 | + "time" | |
| 15 | +) | |
| 16 | + | |
| 17 | +const ( | |
| 18 | + // DefaultTimeout bounds a whole fetch: robots.txt, redirects and body. | |
| 19 | + DefaultTimeout = 30 * time.Second | |
| 20 | + // DefaultMaxBytes is the largest page Fetch accepts. | |
| 21 | + DefaultMaxBytes = 5 << 20 | |
| 22 | + // maxRedirects is how many redirections a page may go through. | |
| 23 | + maxRedirects = 10 | |
| 24 | + // robotsMaxBytes is what RFC 9309 asks a crawler to read at least. | |
| 25 | + robotsMaxBytes = 500 << 10 | |
| 26 | + // robotsTTL is how long a site's robots.txt is trusted (RFC 9309 §2.4). | |
| 27 | + robotsTTL = 24 * time.Hour | |
| 28 | +) | |
| 29 | + | |
| 30 | +// Page is a downloaded HTML page. | |
| 31 | +type Page struct { | |
| 32 | + // URL is where the page was found, after redirections: the base its | |
| 33 | + // relative links resolve against. | |
| 34 | + URL *url.URL | |
| 35 | + // HTML is the page body, converted to UTF-8. | |
| 36 | + HTML []byte | |
| 37 | + // Charset is the encoding the page was decoded from ("utf-8", | |
| 38 | + // "windows-1252", "shift_jis"…). | |
| 39 | + Charset string | |
| 40 | +} | |
| 41 | + | |
| 42 | +// Fetcher downloads pages while honouring the sites' robots.txt, which it | |
| 43 | +// caches per origin. The zero value is not usable: call NewFetcher. A | |
| 44 | +// Fetcher is safe for concurrent use. | |
| 45 | +type Fetcher struct { | |
| 46 | + // UserAgent is sent with every request. Its product token — what comes | |
| 47 | + // before the first "/" — is the name looked up in robots.txt. | |
| 48 | + UserAgent string | |
| 49 | + // Client performs the requests. Its CheckRedirect is replaced for page | |
| 50 | + // requests, so that every hop is checked against robots.txt. | |
| 51 | + Client *http.Client | |
| 52 | + // MaxBytes is the largest page accepted. | |
| 53 | + MaxBytes int64 | |
| 54 | + | |
| 55 | + mu sync.Mutex | |
| 56 | + robots map[string]robotsEntry | |
| 57 | + now func() time.Time | |
| 58 | +} | |
| 59 | + | |
| 60 | +type robotsEntry struct { | |
| 61 | + rules *robotsRules | |
| 62 | + fetched time.Time | |
| 63 | +} | |
| 64 | + | |
| 65 | +// NewFetcher returns a Fetcher identifying itself as userAgent, with the | |
| 66 | +// default timeout and size limit. | |
| 67 | +// | |
| 68 | +// f := webmd.NewFetcher("mm/0.3.0") | |
| 69 | +// page, err := f.Fetch(ctx, "https://example.com/") | |
| 70 | +func NewFetcher(userAgent string) *Fetcher { | |
| 71 | + return &Fetcher{ | |
| 72 | + UserAgent: userAgent, | |
| 73 | + Client: &http.Client{Timeout: DefaultTimeout}, | |
| 74 | + MaxBytes: DefaultMaxBytes, | |
| 75 | + robots: map[string]robotsEntry{}, | |
| 76 | + now: time.Now, | |
| 77 | + } | |
| 78 | +} | |
| 79 | + | |
| 80 | +// NormalizeURL turns what a user typed into an absolute http(s) URL: a | |
| 81 | +// missing scheme means https ("example.com"), and a scheme typed without | |
| 82 | +// its colon ("https//example.com") is repaired. Anything else that is not | |
| 83 | +// http or https is a CodeUsage error. | |
| 84 | +// | |
| 85 | +// u, _ := webmd.NormalizeURL("https//k33g.org") // https://k33g.org | |
| 86 | +func NormalizeURL(raw string) (*url.URL, error) { | |
| 87 | + s := strings.TrimSpace(raw) | |
| 88 | + if s == "" { | |
| 89 | + return nil, errorf(CodeUsage, nil, "no URL given") | |
| 90 | + } | |
| 91 | + s = repairScheme(s) | |
| 92 | + if !hasScheme(s) { | |
| 93 | + s = "https://" + s | |
| 94 | + } | |
| 95 | + u, err := url.Parse(s) | |
| 96 | + if err != nil { | |
| 97 | + return nil, errorf(CodeUsage, err, "invalid URL %q", raw) | |
| 98 | + } | |
| 99 | + if u.Scheme != "http" && u.Scheme != "https" { | |
| 100 | + return nil, errorf(CodeUsage, nil, "unsupported URL scheme %q in %q: only http and https are fetched", u.Scheme, raw) | |
| 101 | + } | |
| 102 | + if u.Host == "" { | |
| 103 | + return nil, errorf(CodeUsage, nil, "invalid URL %q: no host", raw) | |
| 104 | + } | |
| 105 | + return u, nil | |
| 106 | +} | |
| 107 | + | |
| 108 | +// explicitScheme matches "scheme:" followed by something that is not a | |
| 109 | +// port number: "mailto:x" has a scheme, "localhost:8080/x" does not. | |
| 110 | +var explicitScheme = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.-]*:(//|[^0-9]|$)`) | |
| 111 | + | |
| 112 | +func hasScheme(s string) bool { | |
| 113 | + return explicitScheme.MatchString(s) | |
| 114 | +} | |
| 115 | + | |
| 116 | +// repairScheme turns "https//host" into "https://host". | |
| 117 | +func repairScheme(s string) string { | |
| 118 | + lower := strings.ToLower(s) | |
| 119 | + for _, scheme := range []string{"https", "http"} { | |
| 120 | + if strings.HasPrefix(lower, scheme+"//") { | |
| 121 | + return s[:len(scheme)] + ":" + s[len(scheme):] | |
| 122 | + } | |
| 123 | + } | |
| 124 | + return s | |
| 125 | +} | |
| 126 | + | |
| 127 | +// Fetch downloads the HTML page at rawURL (see NormalizeURL for what is | |
| 128 | +// accepted) and returns it in UTF-8. | |
| 129 | +// | |
| 130 | +// Errors are *Error values: CodeUsage for a bad URL; CodeInput when the | |
| 131 | +// page cannot be had — unreachable host, HTTP status ≥ 300 once redirections | |
| 132 | +// are followed, too many redirections, or a robots.txt (of the first host or | |
| 133 | +// of any redirection target) that forbids it; CodeConversion when what came | |
| 134 | +// back is not an HTML page or is larger than MaxBytes. | |
| 135 | +func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Page, error) { | |
| 136 | + u, err := NormalizeURL(rawURL) | |
| 137 | + if err != nil { | |
| 138 | + return nil, err | |
| 139 | + } | |
| 140 | + if err := f.checkRobots(ctx, u); err != nil { | |
| 141 | + return nil, err | |
| 142 | + } | |
| 143 | + | |
| 144 | + resp, err := f.get(ctx, u) | |
| 145 | + if err != nil { | |
| 146 | + return nil, err | |
| 147 | + } | |
| 148 | + defer closeQuietly(resp.Body) | |
| 149 | + return f.readPage(resp) | |
| 150 | +} | |
| 151 | + | |
| 152 | +// readPage turns a response into a Page, or into the error its status, | |
| 153 | +// size or type calls for. | |
| 154 | +func (f *Fetcher) readPage(resp *http.Response) (*Page, error) { | |
| 155 | + if resp.StatusCode >= 300 { | |
| 156 | + return nil, errorf(CodeInput, nil, "HTTP %s for %s", resp.Status, resp.Request.URL) | |
| 157 | + } | |
| 158 | + body, err := readAtMost(resp.Body, f.MaxBytes) | |
| 159 | + if err != nil { | |
| 160 | + return nil, err | |
| 161 | + } | |
| 162 | + contentType := resp.Header.Get("Content-Type") | |
| 163 | + if !isHTML(contentType, body) { | |
| 164 | + return nil, errorf(CodeConversion, nil, "%s is not an HTML page (Content-Type: %q)", resp.Request.URL, contentType) | |
| 165 | + } | |
| 166 | + text, name, err := toUTF8(body, contentType) | |
| 167 | + if err != nil { | |
| 168 | + return nil, err | |
| 169 | + } | |
| 170 | + return &Page{URL: resp.Request.URL, HTML: text, Charset: name}, nil | |
| 171 | +} | |
| 172 | + | |
| 173 | +// get sends the page request, checking every redirection against the | |
| 174 | +// target's robots.txt. | |
| 175 | +func (f *Fetcher) get(ctx context.Context, u *url.URL) (*http.Response, error) { | |
| 176 | + client := *f.Client | |
| 177 | + client.CheckRedirect = f.redirectPolicy(u) | |
| 178 | + | |
| 179 | + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) | |
| 180 | + if err != nil { | |
| 181 | + return nil, errorf(CodeUsage, err, "invalid URL %s", u) | |
| 182 | + } | |
| 183 | + req.Header.Set("User-Agent", f.UserAgent) | |
| 184 | + req.Header.Set("Accept", "text/html,application/xhtml+xml;q=0.9,*/*;q=0.1") | |
| 185 | + | |
| 186 | + resp, err := client.Do(req) | |
| 187 | + if err != nil { | |
| 188 | + var own *Error | |
| 189 | + if errors.As(err, &own) { | |
| 190 | + return nil, own | |
| 191 | + } | |
| 192 | + return nil, errorf(CodeInput, unwrapURLError(err), "cannot fetch %s", u) | |
| 193 | + } | |
| 194 | + return resp, nil | |
| 195 | +} | |
| 196 | + | |
| 197 | +// redirectPolicy follows at most maxRedirects redirections from u, each | |
| 198 | +// one only if the target's robots.txt allows it. | |
| 199 | +func (f *Fetcher) redirectPolicy(u *url.URL) func(*http.Request, []*http.Request) error { | |
| 200 | + return func(req *http.Request, via []*http.Request) error { | |
| 201 | + if len(via) >= maxRedirects { | |
| 202 | + return errorf(CodeInput, nil, "more than %d redirections from %s", maxRedirects, u) | |
| 203 | + } | |
| 204 | + return f.checkRobots(req.Context(), req.URL) | |
| 205 | + } | |
| 206 | +} | |
| 207 | + | |
| 208 | +// closeQuietly closes a response body. There is nothing to do about a | |
| 209 | +// failure to close a body that has been read: the answer is already in hand. | |
| 210 | +func closeQuietly(body io.Closer) { | |
| 211 | + _ = body.Close() | |
| 212 | +} | |
| 213 | + | |
| 214 | +// unwrapURLError drops the *url.Error wrapper, whose message repeats the | |
| 215 | +// method and the URL the caller already names. | |
| 216 | +func unwrapURLError(err error) error { | |
| 217 | + var urlErr *url.Error | |
| 218 | + if errors.As(err, &urlErr) { | |
| 219 | + return urlErr.Err | |
| 220 | + } | |
| 221 | + return err | |
| 222 | +} | |
| 223 | + | |
| 224 | +// readAtMost reads body, refusing one larger than limit. | |
| 225 | +func readAtMost(body io.Reader, limit int64) ([]byte, error) { | |
| 226 | + data, err := io.ReadAll(io.LimitReader(body, limit+1)) | |
| 227 | + if err != nil { | |
| 228 | + return nil, errorf(CodeInput, err, "reading the page failed") | |
| 229 | + } | |
| 230 | + if int64(len(data)) > limit { | |
| 231 | + return nil, errorf(CodeConversion, nil, "the page is larger than %d bytes", limit) | |
| 232 | + } | |
| 233 | + return data, nil | |
| 234 | +} | |
| 235 | + | |
| 236 | +// isHTML accepts text/html and XHTML; a missing Content-Type is sniffed. | |
| 237 | +func isHTML(contentType string, body []byte) bool { | |
| 238 | + if strings.TrimSpace(contentType) == "" { | |
| 239 | + return strings.HasPrefix(http.DetectContentType(body), "text/html") | |
| 240 | + } | |
| 241 | + media, _, err := mime.ParseMediaType(contentType) | |
| 242 | + if err != nil { | |
| 243 | + return false | |
| 244 | + } | |
| 245 | + return media == "text/html" || media == "application/xhtml+xml" | |
| 246 | +} | |
| 247 | + | |
| 248 | +// checkRobots returns a CodeInput error when the robots.txt of u's origin | |
| 249 | +// forbids this agent to fetch u. | |
| 250 | +func (f *Fetcher) checkRobots(ctx context.Context, u *url.URL) error { | |
| 251 | + rules, err := f.robotsFor(ctx, u) | |
| 252 | + if err != nil { | |
| 253 | + return errorf(CodeInput, err, "cannot read %s://%s/robots.txt, so the site is treated as closed to robots", u.Scheme, u.Host) | |
| 254 | + } | |
| 255 | + path := u.EscapedPath() | |
| 256 | + if path == "" { | |
| 257 | + path = "/" | |
| 258 | + } | |
| 259 | + if u.RawQuery != "" { | |
| 260 | + path += "?" + u.RawQuery | |
| 261 | + } | |
| 262 | + if !rules.allows(path) { | |
| 263 | + return errorf(CodeInput, nil, "%s://%s/robots.txt does not allow %q to fetch %s", u.Scheme, u.Host, f.agent(), path) | |
| 264 | + } | |
| 265 | + return nil | |
| 266 | +} | |
| 267 | + | |
| 268 | +// agent is the product token of the User-Agent: "mm" for "mm/0.3.0". | |
| 269 | +func (f *Fetcher) agent() string { | |
| 270 | + token, _, _ := strings.Cut(f.UserAgent, "/") | |
| 271 | + return strings.TrimSpace(token) | |
| 272 | +} | |
| 273 | + | |
| 274 | +// robotsFor returns the cached rules of u's origin, or loads them. Only a | |
| 275 | +// definite answer is cached: a network failure is retried next time. | |
| 276 | +func (f *Fetcher) robotsFor(ctx context.Context, u *url.URL) (*robotsRules, error) { | |
| 277 | + origin := u.Scheme + "://" + u.Host | |
| 278 | + f.mu.Lock() | |
| 279 | + entry, ok := f.robots[origin] | |
| 280 | + f.mu.Unlock() | |
| 281 | + if ok && f.now().Sub(entry.fetched) < robotsTTL { | |
| 282 | + return entry.rules, nil | |
| 283 | + } | |
| 284 | + | |
| 285 | + rules, err := f.loadRobots(ctx, origin) | |
| 286 | + if err != nil { | |
| 287 | + return nil, err | |
| 288 | + } | |
| 289 | + f.mu.Lock() | |
| 290 | + f.robots[origin] = robotsEntry{rules: rules, fetched: f.now()} | |
| 291 | + f.mu.Unlock() | |
| 292 | + return rules, nil | |
| 293 | +} | |
| 294 | + | |
| 295 | +// loadRobots downloads and parses origin's robots.txt: none (a 4xx status) | |
| 296 | +// allows everything; a server error or an unreachable host is an error, | |
| 297 | +// which the caller turns into "disallowed" as RFC 9309 asks. | |
| 298 | +func (f *Fetcher) loadRobots(ctx context.Context, origin string) (*robotsRules, error) { | |
| 299 | + req, err := http.NewRequestWithContext(ctx, http.MethodGet, origin+"/robots.txt", nil) | |
| 300 | + if err != nil { | |
| 301 | + return nil, err | |
| 302 | + } | |
| 303 | + req.Header.Set("User-Agent", f.UserAgent) | |
| 304 | + resp, err := f.Client.Do(req) | |
| 305 | + if err != nil { | |
| 306 | + return nil, unwrapURLError(err) | |
| 307 | + } | |
| 308 | + defer closeQuietly(resp.Body) | |
| 309 | + return f.robotsFromResponse(resp) | |
| 310 | +} | |
| 311 | + | |
| 312 | +// robotsFromResponse reads the rules out of a robots.txt response. | |
| 313 | +func (f *Fetcher) robotsFromResponse(resp *http.Response) (*robotsRules, error) { | |
| 314 | + switch { | |
| 315 | + case resp.StatusCode >= 200 && resp.StatusCode < 300: | |
| 316 | + text, err := io.ReadAll(io.LimitReader(resp.Body, robotsMaxBytes)) | |
| 317 | + if err != nil { | |
| 318 | + return nil, err | |
| 319 | + } | |
| 320 | + return parseRobots(string(text), f.agent()), nil | |
| 321 | + case resp.StatusCode >= 400 && resp.StatusCode < 500: | |
| 322 | + return allowAll, nil | |
| 323 | + } | |
| 324 | + return nil, fmt.Errorf("HTTP %s", resp.Status) | |
| 325 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,325 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "context" | ||
| 5 | + "errors" | ||
| 6 | + "fmt" | ||
| 7 | + "io" | ||
| 8 | + "mime" | ||
| 9 | + "net/http" | ||
| 10 | + "net/url" | ||
| 11 | + "regexp" | ||
| 12 | + "strings" | ||
| 13 | + "sync" | ||
| 14 | + "time" | ||
| 15 | +) | ||
| 16 | + | ||
| 17 | +const ( | ||
| 18 | + // DefaultTimeout bounds a whole fetch: robots.txt, redirects and body. | ||
| 19 | + DefaultTimeout = 30 * time.Second | ||
| 20 | + // DefaultMaxBytes is the largest page Fetch accepts. | ||
| 21 | + DefaultMaxBytes = 5 << 20 | ||
| 22 | + // maxRedirects is how many redirections a page may go through. | ||
| 23 | + maxRedirects = 10 | ||
| 24 | + // robotsMaxBytes is what RFC 9309 asks a crawler to read at least. | ||
| 25 | + robotsMaxBytes = 500 << 10 | ||
| 26 | + // robotsTTL is how long a site's robots.txt is trusted (RFC 9309 §2.4). | ||
| 27 | + robotsTTL = 24 * time.Hour | ||
| 28 | +) | ||
| 29 | + | ||
| 30 | +// Page is a downloaded HTML page. | ||
| 31 | +type Page struct { | ||
| 32 | + // URL is where the page was found, after redirections: the base its | ||
| 33 | + // relative links resolve against. | ||
| 34 | + URL *url.URL | ||
| 35 | + // HTML is the page body, converted to UTF-8. | ||
| 36 | + HTML []byte | ||
| 37 | + // Charset is the encoding the page was decoded from ("utf-8", | ||
| 38 | + // "windows-1252", "shift_jis"…). | ||
| 39 | + Charset string | ||
| 40 | +} | ||
| 41 | + | ||
| 42 | +// Fetcher downloads pages while honouring the sites' robots.txt, which it | ||
| 43 | +// caches per origin. The zero value is not usable: call NewFetcher. A | ||
| 44 | +// Fetcher is safe for concurrent use. | ||
| 45 | +type Fetcher struct { | ||
| 46 | + // UserAgent is sent with every request. Its product token — what comes | ||
| 47 | + // before the first "/" — is the name looked up in robots.txt. | ||
| 48 | + UserAgent string | ||
| 49 | + // Client performs the requests. Its CheckRedirect is replaced for page | ||
| 50 | + // requests, so that every hop is checked against robots.txt. | ||
| 51 | + Client *http.Client | ||
| 52 | + // MaxBytes is the largest page accepted. | ||
| 53 | + MaxBytes int64 | ||
| 54 | + | ||
| 55 | + mu sync.Mutex | ||
| 56 | + robots map[string]robotsEntry | ||
| 57 | + now func() time.Time | ||
| 58 | +} | ||
| 59 | + | ||
| 60 | +type robotsEntry struct { | ||
| 61 | + rules *robotsRules | ||
| 62 | + fetched time.Time | ||
| 63 | +} | ||
| 64 | + | ||
| 65 | +// NewFetcher returns a Fetcher identifying itself as userAgent, with the | ||
| 66 | +// default timeout and size limit. | ||
| 67 | +// | ||
| 68 | +// f := webmd.NewFetcher("mm/0.3.0") | ||
| 69 | +// page, err := f.Fetch(ctx, "https://example.com/") | ||
| 70 | +func NewFetcher(userAgent string) *Fetcher { | ||
| 71 | + return &Fetcher{ | ||
| 72 | + UserAgent: userAgent, | ||
| 73 | + Client: &http.Client{Timeout: DefaultTimeout}, | ||
| 74 | + MaxBytes: DefaultMaxBytes, | ||
| 75 | + robots: map[string]robotsEntry{}, | ||
| 76 | + now: time.Now, | ||
| 77 | + } | ||
| 78 | +} | ||
| 79 | + | ||
| 80 | +// NormalizeURL turns what a user typed into an absolute http(s) URL: a | ||
| 81 | +// missing scheme means https ("example.com"), and a scheme typed without | ||
| 82 | +// its colon ("https//example.com") is repaired. Anything else that is not | ||
| 83 | +// http or https is a CodeUsage error. | ||
| 84 | +// | ||
| 85 | +// u, _ := webmd.NormalizeURL("https//k33g.org") // https://k33g.org | ||
| 86 | +func NormalizeURL(raw string) (*url.URL, error) { | ||
| 87 | + s := strings.TrimSpace(raw) | ||
| 88 | + if s == "" { | ||
| 89 | + return nil, errorf(CodeUsage, nil, "no URL given") | ||
| 90 | + } | ||
| 91 | + s = repairScheme(s) | ||
| 92 | + if !hasScheme(s) { | ||
| 93 | + s = "https://" + s | ||
| 94 | + } | ||
| 95 | + u, err := url.Parse(s) | ||
| 96 | + if err != nil { | ||
| 97 | + return nil, errorf(CodeUsage, err, "invalid URL %q", raw) | ||
| 98 | + } | ||
| 99 | + if u.Scheme != "http" && u.Scheme != "https" { | ||
| 100 | + return nil, errorf(CodeUsage, nil, "unsupported URL scheme %q in %q: only http and https are fetched", u.Scheme, raw) | ||
| 101 | + } | ||
| 102 | + if u.Host == "" { | ||
| 103 | + return nil, errorf(CodeUsage, nil, "invalid URL %q: no host", raw) | ||
| 104 | + } | ||
| 105 | + return u, nil | ||
| 106 | +} | ||
| 107 | + | ||
| 108 | +// explicitScheme matches "scheme:" followed by something that is not a | ||
| 109 | +// port number: "mailto:x" has a scheme, "localhost:8080/x" does not. | ||
| 110 | +var explicitScheme = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.-]*:(//|[^0-9]|$)`) | ||
| 111 | + | ||
| 112 | +func hasScheme(s string) bool { | ||
| 113 | + return explicitScheme.MatchString(s) | ||
| 114 | +} | ||
| 115 | + | ||
| 116 | +// repairScheme turns "https//host" into "https://host". | ||
| 117 | +func repairScheme(s string) string { | ||
| 118 | + lower := strings.ToLower(s) | ||
| 119 | + for _, scheme := range []string{"https", "http"} { | ||
| 120 | + if strings.HasPrefix(lower, scheme+"//") { | ||
| 121 | + return s[:len(scheme)] + ":" + s[len(scheme):] | ||
| 122 | + } | ||
| 123 | + } | ||
| 124 | + return s | ||
| 125 | +} | ||
| 126 | + | ||
| 127 | +// Fetch downloads the HTML page at rawURL (see NormalizeURL for what is | ||
| 128 | +// accepted) and returns it in UTF-8. | ||
| 129 | +// | ||
| 130 | +// Errors are *Error values: CodeUsage for a bad URL; CodeInput when the | ||
| 131 | +// page cannot be had — unreachable host, HTTP status ≥ 300 once redirections | ||
| 132 | +// are followed, too many redirections, or a robots.txt (of the first host or | ||
| 133 | +// of any redirection target) that forbids it; CodeConversion when what came | ||
| 134 | +// back is not an HTML page or is larger than MaxBytes. | ||
| 135 | +func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Page, error) { | ||
| 136 | + u, err := NormalizeURL(rawURL) | ||
| 137 | + if err != nil { | ||
| 138 | + return nil, err | ||
| 139 | + } | ||
| 140 | + if err := f.checkRobots(ctx, u); err != nil { | ||
| 141 | + return nil, err | ||
| 142 | + } | ||
| 143 | + | ||
| 144 | + resp, err := f.get(ctx, u) | ||
| 145 | + if err != nil { | ||
| 146 | + return nil, err | ||
| 147 | + } | ||
| 148 | + defer closeQuietly(resp.Body) | ||
| 149 | + return f.readPage(resp) | ||
| 150 | +} | ||
| 151 | + | ||
| 152 | +// readPage turns a response into a Page, or into the error its status, | ||
| 153 | +// size or type calls for. | ||
| 154 | +func (f *Fetcher) readPage(resp *http.Response) (*Page, error) { | ||
| 155 | + if resp.StatusCode >= 300 { | ||
| 156 | + return nil, errorf(CodeInput, nil, "HTTP %s for %s", resp.Status, resp.Request.URL) | ||
| 157 | + } | ||
| 158 | + body, err := readAtMost(resp.Body, f.MaxBytes) | ||
| 159 | + if err != nil { | ||
| 160 | + return nil, err | ||
| 161 | + } | ||
| 162 | + contentType := resp.Header.Get("Content-Type") | ||
| 163 | + if !isHTML(contentType, body) { | ||
| 164 | + return nil, errorf(CodeConversion, nil, "%s is not an HTML page (Content-Type: %q)", resp.Request.URL, contentType) | ||
| 165 | + } | ||
| 166 | + text, name, err := toUTF8(body, contentType) | ||
| 167 | + if err != nil { | ||
| 168 | + return nil, err | ||
| 169 | + } | ||
| 170 | + return &Page{URL: resp.Request.URL, HTML: text, Charset: name}, nil | ||
| 171 | +} | ||
| 172 | + | ||
| 173 | +// get sends the page request, checking every redirection against the | ||
| 174 | +// target's robots.txt. | ||
| 175 | +func (f *Fetcher) get(ctx context.Context, u *url.URL) (*http.Response, error) { | ||
| 176 | + client := *f.Client | ||
| 177 | + client.CheckRedirect = f.redirectPolicy(u) | ||
| 178 | + | ||
| 179 | + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) | ||
| 180 | + if err != nil { | ||
| 181 | + return nil, errorf(CodeUsage, err, "invalid URL %s", u) | ||
| 182 | + } | ||
| 183 | + req.Header.Set("User-Agent", f.UserAgent) | ||
| 184 | + req.Header.Set("Accept", "text/html,application/xhtml+xml;q=0.9,*/*;q=0.1") | ||
| 185 | + | ||
| 186 | + resp, err := client.Do(req) | ||
| 187 | + if err != nil { | ||
| 188 | + var own *Error | ||
| 189 | + if errors.As(err, &own) { | ||
| 190 | + return nil, own | ||
| 191 | + } | ||
| 192 | + return nil, errorf(CodeInput, unwrapURLError(err), "cannot fetch %s", u) | ||
| 193 | + } | ||
| 194 | + return resp, nil | ||
| 195 | +} | ||
| 196 | + | ||
| 197 | +// redirectPolicy follows at most maxRedirects redirections from u, each | ||
| 198 | +// one only if the target's robots.txt allows it. | ||
| 199 | +func (f *Fetcher) redirectPolicy(u *url.URL) func(*http.Request, []*http.Request) error { | ||
| 200 | + return func(req *http.Request, via []*http.Request) error { | ||
| 201 | + if len(via) >= maxRedirects { | ||
| 202 | + return errorf(CodeInput, nil, "more than %d redirections from %s", maxRedirects, u) | ||
| 203 | + } | ||
| 204 | + return f.checkRobots(req.Context(), req.URL) | ||
| 205 | + } | ||
| 206 | +} | ||
| 207 | + | ||
| 208 | +// closeQuietly closes a response body. There is nothing to do about a | ||
| 209 | +// failure to close a body that has been read: the answer is already in hand. | ||
| 210 | +func closeQuietly(body io.Closer) { | ||
| 211 | + _ = body.Close() | ||
| 212 | +} | ||
| 213 | + | ||
| 214 | +// unwrapURLError drops the *url.Error wrapper, whose message repeats the | ||
| 215 | +// method and the URL the caller already names. | ||
| 216 | +func unwrapURLError(err error) error { | ||
| 217 | + var urlErr *url.Error | ||
| 218 | + if errors.As(err, &urlErr) { | ||
| 219 | + return urlErr.Err | ||
| 220 | + } | ||
| 221 | + return err | ||
| 222 | +} | ||
| 223 | + | ||
| 224 | +// readAtMost reads body, refusing one larger than limit. | ||
| 225 | +func readAtMost(body io.Reader, limit int64) ([]byte, error) { | ||
| 226 | + data, err := io.ReadAll(io.LimitReader(body, limit+1)) | ||
| 227 | + if err != nil { | ||
| 228 | + return nil, errorf(CodeInput, err, "reading the page failed") | ||
| 229 | + } | ||
| 230 | + if int64(len(data)) > limit { | ||
| 231 | + return nil, errorf(CodeConversion, nil, "the page is larger than %d bytes", limit) | ||
| 232 | + } | ||
| 233 | + return data, nil | ||
| 234 | +} | ||
| 235 | + | ||
| 236 | +// isHTML accepts text/html and XHTML; a missing Content-Type is sniffed. | ||
| 237 | +func isHTML(contentType string, body []byte) bool { | ||
| 238 | + if strings.TrimSpace(contentType) == "" { | ||
| 239 | + return strings.HasPrefix(http.DetectContentType(body), "text/html") | ||
| 240 | + } | ||
| 241 | + media, _, err := mime.ParseMediaType(contentType) | ||
| 242 | + if err != nil { | ||
| 243 | + return false | ||
| 244 | + } | ||
| 245 | + return media == "text/html" || media == "application/xhtml+xml" | ||
| 246 | +} | ||
| 247 | + | ||
| 248 | +// checkRobots returns a CodeInput error when the robots.txt of u's origin | ||
| 249 | +// forbids this agent to fetch u. | ||
| 250 | +func (f *Fetcher) checkRobots(ctx context.Context, u *url.URL) error { | ||
| 251 | + rules, err := f.robotsFor(ctx, u) | ||
| 252 | + if err != nil { | ||
| 253 | + return errorf(CodeInput, err, "cannot read %s://%s/robots.txt, so the site is treated as closed to robots", u.Scheme, u.Host) | ||
| 254 | + } | ||
| 255 | + path := u.EscapedPath() | ||
| 256 | + if path == "" { | ||
| 257 | + path = "/" | ||
| 258 | + } | ||
| 259 | + if u.RawQuery != "" { | ||
| 260 | + path += "?" + u.RawQuery | ||
| 261 | + } | ||
| 262 | + if !rules.allows(path) { | ||
| 263 | + return errorf(CodeInput, nil, "%s://%s/robots.txt does not allow %q to fetch %s", u.Scheme, u.Host, f.agent(), path) | ||
| 264 | + } | ||
| 265 | + return nil | ||
| 266 | +} | ||
| 267 | + | ||
| 268 | +// agent is the product token of the User-Agent: "mm" for "mm/0.3.0". | ||
| 269 | +func (f *Fetcher) agent() string { | ||
| 270 | + token, _, _ := strings.Cut(f.UserAgent, "/") | ||
| 271 | + return strings.TrimSpace(token) | ||
| 272 | +} | ||
| 273 | + | ||
| 274 | +// robotsFor returns the cached rules of u's origin, or loads them. Only a | ||
| 275 | +// definite answer is cached: a network failure is retried next time. | ||
| 276 | +func (f *Fetcher) robotsFor(ctx context.Context, u *url.URL) (*robotsRules, error) { | ||
| 277 | + origin := u.Scheme + "://" + u.Host | ||
| 278 | + f.mu.Lock() | ||
| 279 | + entry, ok := f.robots[origin] | ||
| 280 | + f.mu.Unlock() | ||
| 281 | + if ok && f.now().Sub(entry.fetched) < robotsTTL { | ||
| 282 | + return entry.rules, nil | ||
| 283 | + } | ||
| 284 | + | ||
| 285 | + rules, err := f.loadRobots(ctx, origin) | ||
| 286 | + if err != nil { | ||
| 287 | + return nil, err | ||
| 288 | + } | ||
| 289 | + f.mu.Lock() | ||
| 290 | + f.robots[origin] = robotsEntry{rules: rules, fetched: f.now()} | ||
| 291 | + f.mu.Unlock() | ||
| 292 | + return rules, nil | ||
| 293 | +} | ||
| 294 | + | ||
| 295 | +// loadRobots downloads and parses origin's robots.txt: none (a 4xx status) | ||
| 296 | +// allows everything; a server error or an unreachable host is an error, | ||
| 297 | +// which the caller turns into "disallowed" as RFC 9309 asks. | ||
| 298 | +func (f *Fetcher) loadRobots(ctx context.Context, origin string) (*robotsRules, error) { | ||
| 299 | + req, err := http.NewRequestWithContext(ctx, http.MethodGet, origin+"/robots.txt", nil) | ||
| 300 | + if err != nil { | ||
| 301 | + return nil, err | ||
| 302 | + } | ||
| 303 | + req.Header.Set("User-Agent", f.UserAgent) | ||
| 304 | + resp, err := f.Client.Do(req) | ||
| 305 | + if err != nil { | ||
| 306 | + return nil, unwrapURLError(err) | ||
| 307 | + } | ||
| 308 | + defer closeQuietly(resp.Body) | ||
| 309 | + return f.robotsFromResponse(resp) | ||
| 310 | +} | ||
| 311 | + | ||
| 312 | +// robotsFromResponse reads the rules out of a robots.txt response. | ||
| 313 | +func (f *Fetcher) robotsFromResponse(resp *http.Response) (*robotsRules, error) { | ||
| 314 | + switch { | ||
| 315 | + case resp.StatusCode >= 200 && resp.StatusCode < 300: | ||
| 316 | + text, err := io.ReadAll(io.LimitReader(resp.Body, robotsMaxBytes)) | ||
| 317 | + if err != nil { | ||
| 318 | + return nil, err | ||
| 319 | + } | ||
| 320 | + return parseRobots(string(text), f.agent()), nil | ||
| 321 | + case resp.StatusCode >= 400 && resp.StatusCode < 500: | ||
| 322 | + return allowAll, nil | ||
| 323 | + } | ||
| 324 | + return nil, fmt.Errorf("HTTP %s", resp.Status) | ||
| 325 | +} | ||
added
internal/webmd/fetch_test.go +237 -0 | new file mode 100644 | ||
| @@ -0,0 +1,237 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "context" | |
| 5 | + "net/http" | |
| 6 | + "net/http/httptest" | |
| 7 | + "strings" | |
| 8 | + "testing" | |
| 9 | + "time" | |
| 10 | + | |
| 11 | + "golang.org/x/text/encoding/charmap" | |
| 12 | + "golang.org/x/text/encoding/japanese" | |
| 13 | +) | |
| 14 | + | |
| 15 | +// site is a fake web site: path → handler, with /robots.txt answered by | |
| 16 | +// robots (404 when empty) and every request counted. | |
| 17 | +type site struct { | |
| 18 | + robots string | |
| 19 | + robotsTo int // status for /robots.txt, when not 200/404 | |
| 20 | + pages map[string]http.HandlerFunc | |
| 21 | + hits map[string]int | |
| 22 | +} | |
| 23 | + | |
| 24 | +func newSite(t *testing.T, s *site) (*httptest.Server, *Fetcher) { | |
| 25 | + t.Helper() | |
| 26 | + s.hits = map[string]int{} | |
| 27 | + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| 28 | + s.hits[r.URL.Path]++ | |
| 29 | + if r.URL.Path == "/robots.txt" { | |
| 30 | + switch { | |
| 31 | + case s.robotsTo != 0: | |
| 32 | + w.WriteHeader(s.robotsTo) | |
| 33 | + case s.robots == "": | |
| 34 | + http.NotFound(w, r) | |
| 35 | + default: | |
| 36 | + _, _ = w.Write([]byte(s.robots)) | |
| 37 | + } | |
| 38 | + return | |
| 39 | + } | |
| 40 | + if h, ok := s.pages[r.URL.Path]; ok { | |
| 41 | + h(w, r) | |
| 42 | + return | |
| 43 | + } | |
| 44 | + http.NotFound(w, r) | |
| 45 | + })) | |
| 46 | + t.Cleanup(srv.Close) | |
| 47 | + f := NewFetcher("mm/test") | |
| 48 | + f.Client = srv.Client() | |
| 49 | + return srv, f | |
| 50 | +} | |
| 51 | + | |
| 52 | +func html200(contentType string, body []byte) http.HandlerFunc { | |
| 53 | + return func(w http.ResponseWriter, _ *http.Request) { | |
| 54 | + if contentType != "" { | |
| 55 | + w.Header().Set("Content-Type", contentType) | |
| 56 | + } | |
| 57 | + _, _ = w.Write(body) | |
| 58 | + } | |
| 59 | +} | |
| 60 | + | |
| 61 | +func TestFetchReturnsThePageAndItsFinalURL(t *testing.T) { | |
| 62 | + s := &site{pages: map[string]http.HandlerFunc{ | |
| 63 | + "/old": func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/new", http.StatusMovedPermanently) }, | |
| 64 | + "/new": html200("text/html; charset=utf-8", []byte("<p>café</p>")), | |
| 65 | + }} | |
| 66 | + srv, f := newSite(t, s) | |
| 67 | + | |
| 68 | + page, err := f.Fetch(context.Background(), srv.URL+"/old") | |
| 69 | + if err != nil { | |
| 70 | + t.Fatal(err) | |
| 71 | + } | |
| 72 | + if string(page.HTML) != "<p>café</p>" || page.Charset != "utf-8" || page.URL.Path != "/new" { | |
| 73 | + t.Errorf("page = %q %s %s", page.HTML, page.Charset, page.URL) | |
| 74 | + } | |
| 75 | +} | |
| 76 | + | |
| 77 | +func TestFetchSendsTheUserAgent(t *testing.T) { | |
| 78 | + var got string | |
| 79 | + s := &site{pages: map[string]http.HandlerFunc{"/": func(w http.ResponseWriter, r *http.Request) { | |
| 80 | + got = r.UserAgent() | |
| 81 | + html200("text/html", []byte("x"))(w, r) | |
| 82 | + }}} | |
| 83 | + srv, f := newSite(t, s) | |
| 84 | + if _, err := f.Fetch(context.Background(), srv.URL+"/"); err != nil { | |
| 85 | + t.Fatal(err) | |
| 86 | + } | |
| 87 | + if got != "mm/test" { | |
| 88 | + t.Errorf("User-Agent = %q", got) | |
| 89 | + } | |
| 90 | +} | |
| 91 | + | |
| 92 | +func TestFetchDecodesDeclaredEncodings(t *testing.T) { | |
| 93 | + latin1, _ := charmap.ISO8859_1.NewEncoder().String("<p>déjà vu</p>") | |
| 94 | + sjis, _ := japanese.ShiftJIS.NewEncoder().String(`<html><head><meta charset="Shift_JIS"></head><body><p>日本語</p></body></html>`) | |
| 95 | + httpEquiv, _ := charmap.Windows1252.NewEncoder().String(`<meta http-equiv="Content-Type" content="text/html; charset=windows-1252"><p>€uro</p>`) | |
| 96 | + undeclared, _ := charmap.Windows1252.NewEncoder().String("<p>naïve</p>") | |
| 97 | + | |
| 98 | + cases := []struct { | |
| 99 | + name, contentType, body, want, charset string | |
| 100 | + }{ | |
| 101 | + {"charset from the HTTP header", "text/html; charset=ISO-8859-1", latin1, "déjà vu", "windows-1252"}, | |
| 102 | + {"charset from meta charset", "text/html", sjis, "日本語", "shift_jis"}, | |
| 103 | + {"charset from meta http-equiv", "text/html", httpEquiv, "€uro", "windows-1252"}, | |
| 104 | + {"the header wins over the meta", "text/html; charset=utf-8", `<meta charset="iso-8859-1"><p>é</p>`, "é", "utf-8"}, | |
| 105 | + {"undeclared valid UTF-8", "text/html", "<p>ok ✓</p>", "ok ✓", "utf-8"}, | |
| 106 | + {"undeclared and not UTF-8 falls back to windows-1252", "text/html", undeclared, "naïve", "windows-1252"}, | |
| 107 | + {"UTF-8 BOM is removed", "text/html", "\xEF\xBB\xBF<p>bom</p>", "<p>bom</p>", "utf-8"}, | |
| 108 | + {"unknown declared charset is skipped", "text/html; charset=klingon", "<p>é</p>", "é", "utf-8"}, | |
| 109 | + } | |
| 110 | + for _, tc := range cases { | |
| 111 | + t.Run(tc.name, func(t *testing.T) { | |
| 112 | + s := &site{pages: map[string]http.HandlerFunc{"/": html200(tc.contentType, []byte(tc.body))}} | |
| 113 | + srv, f := newSite(t, s) | |
| 114 | + page, err := f.Fetch(context.Background(), srv.URL+"/") | |
| 115 | + if err != nil { | |
| 116 | + t.Fatal(err) | |
| 117 | + } | |
| 118 | + if !strings.Contains(string(page.HTML), tc.want) || page.Charset != tc.charset { | |
| 119 | + t.Errorf("got %q (%s), want %q (%s)", page.HTML, page.Charset, tc.want, tc.charset) | |
| 120 | + } | |
| 121 | + }) | |
| 122 | + } | |
| 123 | +} | |
| 124 | + | |
| 125 | +func TestFetchErrorCodes(t *testing.T) { | |
| 126 | + s := &site{pages: map[string]http.HandlerFunc{ | |
| 127 | + "/gone": func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "gone", http.StatusGone) }, | |
| 128 | + "/json": html200("application/json", []byte(`{"a":1}`)), | |
| 129 | + "/big": html200("text/html", []byte("<p>"+strings.Repeat("x", 2000)+"</p>")), | |
| 130 | + "/sniff": html200("", []byte("<!DOCTYPE html><html><body>x</body></html>")), | |
| 131 | + "/loop": func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/loop", http.StatusFound) }, | |
| 132 | + }} | |
| 133 | + srv, f := newSite(t, s) | |
| 134 | + f.MaxBytes = 1000 | |
| 135 | + | |
| 136 | + cases := []struct { | |
| 137 | + url string | |
| 138 | + want Code | |
| 139 | + }{ | |
| 140 | + {srv.URL + "/gone", CodeInput}, | |
| 141 | + {srv.URL + "/missing", CodeInput}, | |
| 142 | + {srv.URL + "/loop", CodeInput}, | |
| 143 | + {srv.URL + "/json", CodeConversion}, | |
| 144 | + {srv.URL + "/big", CodeConversion}, | |
| 145 | + {srv.URL + "/sniff", CodeOK}, | |
| 146 | + {"ftp://example.com/x", CodeUsage}, | |
| 147 | + {" ", CodeUsage}, | |
| 148 | + {"http://127.0.0.1:1/", CodeInput}, // nothing listens on port 1 | |
| 149 | + } | |
| 150 | + for _, tc := range cases { | |
| 151 | + _, err := f.Fetch(context.Background(), tc.url) | |
| 152 | + if got := CodeOf(err); got != tc.want { | |
| 153 | + t.Errorf("Fetch(%s): code %v (%v), want %v", tc.url, got, err, tc.want) | |
| 154 | + } | |
| 155 | + } | |
| 156 | +} | |
| 157 | + | |
| 158 | +func TestFetchHonoursRobotsTxt(t *testing.T) { | |
| 159 | + page := html200("text/html", []byte("<p>x</p>")) | |
| 160 | + s := &site{ | |
| 161 | + robots: "User-agent: *\nDisallow: /private\n\nUser-agent: other\nDisallow: /", | |
| 162 | + pages: map[string]http.HandlerFunc{ | |
| 163 | + "/public": page, | |
| 164 | + "/private": page, | |
| 165 | + "/sneaky": func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/private", http.StatusFound) }, | |
| 166 | + }, | |
| 167 | + } | |
| 168 | + srv, f := newSite(t, s) | |
| 169 | + ctx := context.Background() | |
| 170 | + | |
| 171 | + if _, err := f.Fetch(ctx, srv.URL+"/public"); err != nil { | |
| 172 | + t.Errorf("public: %v", err) | |
| 173 | + } | |
| 174 | + _, err := f.Fetch(ctx, srv.URL+"/private") | |
| 175 | + if CodeOf(err) != CodeInput || !strings.Contains(err.Error(), "robots.txt") { | |
| 176 | + t.Errorf("private: %v, want a robots.txt refusal", err) | |
| 177 | + } | |
| 178 | + if _, err := f.Fetch(ctx, srv.URL+"/sneaky"); CodeOf(err) != CodeInput { | |
| 179 | + t.Errorf("redirect into a disallowed path: %v", err) | |
| 180 | + } | |
| 181 | + if s.hits["/private"] != 0 { | |
| 182 | + t.Errorf("a disallowed page was requested %d time(s)", s.hits["/private"]) | |
| 183 | + } | |
| 184 | + if s.hits["/robots.txt"] != 1 { | |
| 185 | + t.Errorf("robots.txt fetched %d times, want 1 (cached)", s.hits["/robots.txt"]) | |
| 186 | + } | |
| 187 | +} | |
| 188 | + | |
| 189 | +func TestFetchRobotsTxtCacheExpires(t *testing.T) { | |
| 190 | + s := &site{robots: "User-agent: *\nAllow: /", pages: map[string]http.HandlerFunc{"/": html200("text/html", []byte("x"))}} | |
| 191 | + srv, f := newSite(t, s) | |
| 192 | + clock := time.Date(2026, 9, 29, 0, 0, 0, 0, time.UTC) | |
| 193 | + f.now = func() time.Time { return clock } | |
| 194 | + | |
| 195 | + _, _ = f.Fetch(context.Background(), srv.URL+"/") | |
| 196 | + clock = clock.Add(robotsTTL + time.Minute) | |
| 197 | + _, _ = f.Fetch(context.Background(), srv.URL+"/") | |
| 198 | + if s.hits["/robots.txt"] != 2 { | |
| 199 | + t.Errorf("robots.txt fetched %d times, want 2 after expiry", s.hits["/robots.txt"]) | |
| 200 | + } | |
| 201 | +} | |
| 202 | + | |
| 203 | +func TestFetchRobotsTxtServerErrorClosesTheSite(t *testing.T) { | |
| 204 | + s := &site{robotsTo: http.StatusServiceUnavailable, pages: map[string]http.HandlerFunc{"/": html200("text/html", []byte("x"))}} | |
| 205 | + srv, f := newSite(t, s) | |
| 206 | + _, err := f.Fetch(context.Background(), srv.URL+"/") | |
| 207 | + if CodeOf(err) != CodeInput || s.hits["/"] != 0 { | |
| 208 | + t.Errorf("err %v, page hits %d: want a refusal before the page is requested", err, s.hits["/"]) | |
| 209 | + } | |
| 210 | +} | |
| 211 | + | |
| 212 | +func TestNormalizeURL(t *testing.T) { | |
| 213 | + cases := map[string]string{ | |
| 214 | + "https//k33g.org": "https://k33g.org", | |
| 215 | + "HTTP//Example.com/a": "HTTP://Example.com/a", | |
| 216 | + "k33g.org": "https://k33g.org", | |
| 217 | + " https://hello.com/x?y=1": "https://hello.com/x?y=1", | |
| 218 | + "http://example.com": "http://example.com", | |
| 219 | + "localhost:8080/x": "https://localhost:8080/x", | |
| 220 | + } | |
| 221 | + for in, want := range cases { | |
| 222 | + u, err := NormalizeURL(in) | |
| 223 | + if err != nil { | |
| 224 | + t.Errorf("NormalizeURL(%q): %v", in, err) | |
| 225 | + continue | |
| 226 | + } | |
| 227 | + // url.Parse lower-cases the scheme; compare case-insensitively on it. | |
| 228 | + if !strings.EqualFold(u.String(), want) { | |
| 229 | + t.Errorf("NormalizeURL(%q) = %q, want %q", in, u, want) | |
| 230 | + } | |
| 231 | + } | |
| 232 | + for _, bad := range []string{"", "mailto:a@b.c", "file:///etc/passwd", "https://"} { | |
| 233 | + if _, err := NormalizeURL(bad); CodeOf(err) != CodeUsage { | |
| 234 | + t.Errorf("NormalizeURL(%q) = %v, want a usage error", bad, err) | |
| 235 | + } | |
| 236 | + } | |
| 237 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,237 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "context" | ||
| 5 | + "net/http" | ||
| 6 | + "net/http/httptest" | ||
| 7 | + "strings" | ||
| 8 | + "testing" | ||
| 9 | + "time" | ||
| 10 | + | ||
| 11 | + "golang.org/x/text/encoding/charmap" | ||
| 12 | + "golang.org/x/text/encoding/japanese" | ||
| 13 | +) | ||
| 14 | + | ||
| 15 | +// site is a fake web site: path → handler, with /robots.txt answered by | ||
| 16 | +// robots (404 when empty) and every request counted. | ||
| 17 | +type site struct { | ||
| 18 | + robots string | ||
| 19 | + robotsTo int // status for /robots.txt, when not 200/404 | ||
| 20 | + pages map[string]http.HandlerFunc | ||
| 21 | + hits map[string]int | ||
| 22 | +} | ||
| 23 | + | ||
| 24 | +func newSite(t *testing.T, s *site) (*httptest.Server, *Fetcher) { | ||
| 25 | + t.Helper() | ||
| 26 | + s.hits = map[string]int{} | ||
| 27 | + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| 28 | + s.hits[r.URL.Path]++ | ||
| 29 | + if r.URL.Path == "/robots.txt" { | ||
| 30 | + switch { | ||
| 31 | + case s.robotsTo != 0: | ||
| 32 | + w.WriteHeader(s.robotsTo) | ||
| 33 | + case s.robots == "": | ||
| 34 | + http.NotFound(w, r) | ||
| 35 | + default: | ||
| 36 | + _, _ = w.Write([]byte(s.robots)) | ||
| 37 | + } | ||
| 38 | + return | ||
| 39 | + } | ||
| 40 | + if h, ok := s.pages[r.URL.Path]; ok { | ||
| 41 | + h(w, r) | ||
| 42 | + return | ||
| 43 | + } | ||
| 44 | + http.NotFound(w, r) | ||
| 45 | + })) | ||
| 46 | + t.Cleanup(srv.Close) | ||
| 47 | + f := NewFetcher("mm/test") | ||
| 48 | + f.Client = srv.Client() | ||
| 49 | + return srv, f | ||
| 50 | +} | ||
| 51 | + | ||
| 52 | +func html200(contentType string, body []byte) http.HandlerFunc { | ||
| 53 | + return func(w http.ResponseWriter, _ *http.Request) { | ||
| 54 | + if contentType != "" { | ||
| 55 | + w.Header().Set("Content-Type", contentType) | ||
| 56 | + } | ||
| 57 | + _, _ = w.Write(body) | ||
| 58 | + } | ||
| 59 | +} | ||
| 60 | + | ||
| 61 | +func TestFetchReturnsThePageAndItsFinalURL(t *testing.T) { | ||
| 62 | + s := &site{pages: map[string]http.HandlerFunc{ | ||
| 63 | + "/old": func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/new", http.StatusMovedPermanently) }, | ||
| 64 | + "/new": html200("text/html; charset=utf-8", []byte("<p>café</p>")), | ||
| 65 | + }} | ||
| 66 | + srv, f := newSite(t, s) | ||
| 67 | + | ||
| 68 | + page, err := f.Fetch(context.Background(), srv.URL+"/old") | ||
| 69 | + if err != nil { | ||
| 70 | + t.Fatal(err) | ||
| 71 | + } | ||
| 72 | + if string(page.HTML) != "<p>café</p>" || page.Charset != "utf-8" || page.URL.Path != "/new" { | ||
| 73 | + t.Errorf("page = %q %s %s", page.HTML, page.Charset, page.URL) | ||
| 74 | + } | ||
| 75 | +} | ||
| 76 | + | ||
| 77 | +func TestFetchSendsTheUserAgent(t *testing.T) { | ||
| 78 | + var got string | ||
| 79 | + s := &site{pages: map[string]http.HandlerFunc{"/": func(w http.ResponseWriter, r *http.Request) { | ||
| 80 | + got = r.UserAgent() | ||
| 81 | + html200("text/html", []byte("x"))(w, r) | ||
| 82 | + }}} | ||
| 83 | + srv, f := newSite(t, s) | ||
| 84 | + if _, err := f.Fetch(context.Background(), srv.URL+"/"); err != nil { | ||
| 85 | + t.Fatal(err) | ||
| 86 | + } | ||
| 87 | + if got != "mm/test" { | ||
| 88 | + t.Errorf("User-Agent = %q", got) | ||
| 89 | + } | ||
| 90 | +} | ||
| 91 | + | ||
| 92 | +func TestFetchDecodesDeclaredEncodings(t *testing.T) { | ||
| 93 | + latin1, _ := charmap.ISO8859_1.NewEncoder().String("<p>déjà vu</p>") | ||
| 94 | + sjis, _ := japanese.ShiftJIS.NewEncoder().String(`<html><head><meta charset="Shift_JIS"></head><body><p>日本語</p></body></html>`) | ||
| 95 | + httpEquiv, _ := charmap.Windows1252.NewEncoder().String(`<meta http-equiv="Content-Type" content="text/html; charset=windows-1252"><p>€uro</p>`) | ||
| 96 | + undeclared, _ := charmap.Windows1252.NewEncoder().String("<p>naïve</p>") | ||
| 97 | + | ||
| 98 | + cases := []struct { | ||
| 99 | + name, contentType, body, want, charset string | ||
| 100 | + }{ | ||
| 101 | + {"charset from the HTTP header", "text/html; charset=ISO-8859-1", latin1, "déjà vu", "windows-1252"}, | ||
| 102 | + {"charset from meta charset", "text/html", sjis, "日本語", "shift_jis"}, | ||
| 103 | + {"charset from meta http-equiv", "text/html", httpEquiv, "€uro", "windows-1252"}, | ||
| 104 | + {"the header wins over the meta", "text/html; charset=utf-8", `<meta charset="iso-8859-1"><p>é</p>`, "é", "utf-8"}, | ||
| 105 | + {"undeclared valid UTF-8", "text/html", "<p>ok ✓</p>", "ok ✓", "utf-8"}, | ||
| 106 | + {"undeclared and not UTF-8 falls back to windows-1252", "text/html", undeclared, "naïve", "windows-1252"}, | ||
| 107 | + {"UTF-8 BOM is removed", "text/html", "\xEF\xBB\xBF<p>bom</p>", "<p>bom</p>", "utf-8"}, | ||
| 108 | + {"unknown declared charset is skipped", "text/html; charset=klingon", "<p>é</p>", "é", "utf-8"}, | ||
| 109 | + } | ||
| 110 | + for _, tc := range cases { | ||
| 111 | + t.Run(tc.name, func(t *testing.T) { | ||
| 112 | + s := &site{pages: map[string]http.HandlerFunc{"/": html200(tc.contentType, []byte(tc.body))}} | ||
| 113 | + srv, f := newSite(t, s) | ||
| 114 | + page, err := f.Fetch(context.Background(), srv.URL+"/") | ||
| 115 | + if err != nil { | ||
| 116 | + t.Fatal(err) | ||
| 117 | + } | ||
| 118 | + if !strings.Contains(string(page.HTML), tc.want) || page.Charset != tc.charset { | ||
| 119 | + t.Errorf("got %q (%s), want %q (%s)", page.HTML, page.Charset, tc.want, tc.charset) | ||
| 120 | + } | ||
| 121 | + }) | ||
| 122 | + } | ||
| 123 | +} | ||
| 124 | + | ||
| 125 | +func TestFetchErrorCodes(t *testing.T) { | ||
| 126 | + s := &site{pages: map[string]http.HandlerFunc{ | ||
| 127 | + "/gone": func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "gone", http.StatusGone) }, | ||
| 128 | + "/json": html200("application/json", []byte(`{"a":1}`)), | ||
| 129 | + "/big": html200("text/html", []byte("<p>"+strings.Repeat("x", 2000)+"</p>")), | ||
| 130 | + "/sniff": html200("", []byte("<!DOCTYPE html><html><body>x</body></html>")), | ||
| 131 | + "/loop": func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/loop", http.StatusFound) }, | ||
| 132 | + }} | ||
| 133 | + srv, f := newSite(t, s) | ||
| 134 | + f.MaxBytes = 1000 | ||
| 135 | + | ||
| 136 | + cases := []struct { | ||
| 137 | + url string | ||
| 138 | + want Code | ||
| 139 | + }{ | ||
| 140 | + {srv.URL + "/gone", CodeInput}, | ||
| 141 | + {srv.URL + "/missing", CodeInput}, | ||
| 142 | + {srv.URL + "/loop", CodeInput}, | ||
| 143 | + {srv.URL + "/json", CodeConversion}, | ||
| 144 | + {srv.URL + "/big", CodeConversion}, | ||
| 145 | + {srv.URL + "/sniff", CodeOK}, | ||
| 146 | + {"ftp://example.com/x", CodeUsage}, | ||
| 147 | + {" ", CodeUsage}, | ||
| 148 | + {"http://127.0.0.1:1/", CodeInput}, // nothing listens on port 1 | ||
| 149 | + } | ||
| 150 | + for _, tc := range cases { | ||
| 151 | + _, err := f.Fetch(context.Background(), tc.url) | ||
| 152 | + if got := CodeOf(err); got != tc.want { | ||
| 153 | + t.Errorf("Fetch(%s): code %v (%v), want %v", tc.url, got, err, tc.want) | ||
| 154 | + } | ||
| 155 | + } | ||
| 156 | +} | ||
| 157 | + | ||
| 158 | +func TestFetchHonoursRobotsTxt(t *testing.T) { | ||
| 159 | + page := html200("text/html", []byte("<p>x</p>")) | ||
| 160 | + s := &site{ | ||
| 161 | + robots: "User-agent: *\nDisallow: /private\n\nUser-agent: other\nDisallow: /", | ||
| 162 | + pages: map[string]http.HandlerFunc{ | ||
| 163 | + "/public": page, | ||
| 164 | + "/private": page, | ||
| 165 | + "/sneaky": func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/private", http.StatusFound) }, | ||
| 166 | + }, | ||
| 167 | + } | ||
| 168 | + srv, f := newSite(t, s) | ||
| 169 | + ctx := context.Background() | ||
| 170 | + | ||
| 171 | + if _, err := f.Fetch(ctx, srv.URL+"/public"); err != nil { | ||
| 172 | + t.Errorf("public: %v", err) | ||
| 173 | + } | ||
| 174 | + _, err := f.Fetch(ctx, srv.URL+"/private") | ||
| 175 | + if CodeOf(err) != CodeInput || !strings.Contains(err.Error(), "robots.txt") { | ||
| 176 | + t.Errorf("private: %v, want a robots.txt refusal", err) | ||
| 177 | + } | ||
| 178 | + if _, err := f.Fetch(ctx, srv.URL+"/sneaky"); CodeOf(err) != CodeInput { | ||
| 179 | + t.Errorf("redirect into a disallowed path: %v", err) | ||
| 180 | + } | ||
| 181 | + if s.hits["/private"] != 0 { | ||
| 182 | + t.Errorf("a disallowed page was requested %d time(s)", s.hits["/private"]) | ||
| 183 | + } | ||
| 184 | + if s.hits["/robots.txt"] != 1 { | ||
| 185 | + t.Errorf("robots.txt fetched %d times, want 1 (cached)", s.hits["/robots.txt"]) | ||
| 186 | + } | ||
| 187 | +} | ||
| 188 | + | ||
| 189 | +func TestFetchRobotsTxtCacheExpires(t *testing.T) { | ||
| 190 | + s := &site{robots: "User-agent: *\nAllow: /", pages: map[string]http.HandlerFunc{"/": html200("text/html", []byte("x"))}} | ||
| 191 | + srv, f := newSite(t, s) | ||
| 192 | + clock := time.Date(2026, 9, 29, 0, 0, 0, 0, time.UTC) | ||
| 193 | + f.now = func() time.Time { return clock } | ||
| 194 | + | ||
| 195 | + _, _ = f.Fetch(context.Background(), srv.URL+"/") | ||
| 196 | + clock = clock.Add(robotsTTL + time.Minute) | ||
| 197 | + _, _ = f.Fetch(context.Background(), srv.URL+"/") | ||
| 198 | + if s.hits["/robots.txt"] != 2 { | ||
| 199 | + t.Errorf("robots.txt fetched %d times, want 2 after expiry", s.hits["/robots.txt"]) | ||
| 200 | + } | ||
| 201 | +} | ||
| 202 | + | ||
| 203 | +func TestFetchRobotsTxtServerErrorClosesTheSite(t *testing.T) { | ||
| 204 | + s := &site{robotsTo: http.StatusServiceUnavailable, pages: map[string]http.HandlerFunc{"/": html200("text/html", []byte("x"))}} | ||
| 205 | + srv, f := newSite(t, s) | ||
| 206 | + _, err := f.Fetch(context.Background(), srv.URL+"/") | ||
| 207 | + if CodeOf(err) != CodeInput || s.hits["/"] != 0 { | ||
| 208 | + t.Errorf("err %v, page hits %d: want a refusal before the page is requested", err, s.hits["/"]) | ||
| 209 | + } | ||
| 210 | +} | ||
| 211 | + | ||
| 212 | +func TestNormalizeURL(t *testing.T) { | ||
| 213 | + cases := map[string]string{ | ||
| 214 | + "https//k33g.org": "https://k33g.org", | ||
| 215 | + "HTTP//Example.com/a": "HTTP://Example.com/a", | ||
| 216 | + "k33g.org": "https://k33g.org", | ||
| 217 | + " https://hello.com/x?y=1": "https://hello.com/x?y=1", | ||
| 218 | + "http://example.com": "http://example.com", | ||
| 219 | + "localhost:8080/x": "https://localhost:8080/x", | ||
| 220 | + } | ||
| 221 | + for in, want := range cases { | ||
| 222 | + u, err := NormalizeURL(in) | ||
| 223 | + if err != nil { | ||
| 224 | + t.Errorf("NormalizeURL(%q): %v", in, err) | ||
| 225 | + continue | ||
| 226 | + } | ||
| 227 | + // url.Parse lower-cases the scheme; compare case-insensitively on it. | ||
| 228 | + if !strings.EqualFold(u.String(), want) { | ||
| 229 | + t.Errorf("NormalizeURL(%q) = %q, want %q", in, u, want) | ||
| 230 | + } | ||
| 231 | + } | ||
| 232 | + for _, bad := range []string{"", "mailto:a@b.c", "file:///etc/passwd", "https://"} { | ||
| 233 | + if _, err := NormalizeURL(bad); CodeOf(err) != CodeUsage { | ||
| 234 | + t.Errorf("NormalizeURL(%q) = %v, want a usage error", bad, err) | ||
| 235 | + } | ||
| 236 | + } | ||
| 237 | +} | ||
added
internal/webmd/grid.go +89 -0 | new file mode 100644 | ||
| @@ -0,0 +1,89 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "bytes" | |
| 5 | + "strings" | |
| 6 | + | |
| 7 | + "golang.org/x/net/html" | |
| 8 | +) | |
| 9 | + | |
| 10 | +// Laying a table out: as a GFM grid, or as raw HTML. | |
| 11 | + | |
| 12 | +// grid lays the cells out on a rectangle, copying a spanned cell's text | |
| 13 | +// into every slot it covers. Without spans, it is the rows as they are. | |
| 14 | +func (c *converter) grid(rows [][]tableCell) [][]string { | |
| 15 | + grid := make([][]string, len(rows)) | |
| 16 | + filled := make([][]bool, len(rows)) | |
| 17 | + set := func(r, col int, text string) { | |
| 18 | + for len(grid[r]) <= col { | |
| 19 | + grid[r] = append(grid[r], "") | |
| 20 | + filled[r] = append(filled[r], false) | |
| 21 | + } | |
| 22 | + grid[r][col], filled[r][col] = text, true | |
| 23 | + } | |
| 24 | + for r, row := range rows { | |
| 25 | + col := 0 | |
| 26 | + for _, cell := range row { | |
| 27 | + for col < len(filled[r]) && filled[r][col] { | |
| 28 | + col++ | |
| 29 | + } | |
| 30 | + text := c.cellText(cell.node) | |
| 31 | + for dr := 0; dr < cell.rowspan && r+dr < len(rows); dr++ { | |
| 32 | + for dc := 0; dc < cell.colspan; dc++ { | |
| 33 | + set(r+dr, col+dc, text) | |
| 34 | + } | |
| 35 | + } | |
| 36 | + col += cell.colspan | |
| 37 | + } | |
| 38 | + } | |
| 39 | + return grid | |
| 40 | +} | |
| 41 | + | |
| 42 | +// cellText renders a cell on one line: its blocks and hard breaks become | |
| 43 | +// <br>, and a pipe is escaped so it does not end the cell. | |
| 44 | +func (c *converter) cellText(n *html.Node) string { | |
| 45 | + text := joinBlocks(c.blocks(n)) | |
| 46 | + text = strings.ReplaceAll(text, "\\\n", "\n") | |
| 47 | + text = strings.ReplaceAll(text, "\n\n", "\n") | |
| 48 | + text = strings.ReplaceAll(text, "\n", "<br>") | |
| 49 | + return strings.ReplaceAll(text, "|", `\|`) | |
| 50 | +} | |
| 51 | + | |
| 52 | +func gfmTable(grid [][]string) string { | |
| 53 | + width := 0 | |
| 54 | + empty := true | |
| 55 | + for _, row := range grid { | |
| 56 | + width = max(width, len(row)) | |
| 57 | + for _, text := range row { | |
| 58 | + empty = empty && text == "" | |
| 59 | + } | |
| 60 | + } | |
| 61 | + if width == 0 || empty { | |
| 62 | + return "" | |
| 63 | + } | |
| 64 | + lines := make([]string, 0, len(grid)+1) | |
| 65 | + for i, row := range grid { | |
| 66 | + cells := append(row[:len(row):len(row)], make([]string, width-len(row))...) | |
| 67 | + lines = append(lines, "| "+strings.Join(cells, " | ")+" |") | |
| 68 | + if i == 0 { | |
| 69 | + lines = append(lines, "|"+strings.Repeat("---|", width)) | |
| 70 | + } | |
| 71 | + } | |
| 72 | + return strings.Join(lines, "\n") | |
| 73 | +} | |
| 74 | + | |
| 75 | +// rawHTML renders the cleaned table as HTML, without the blank lines that | |
| 76 | +// would end a Markdown HTML block halfway through it. | |
| 77 | +func rawHTML(n *html.Node) string { | |
| 78 | + var buf bytes.Buffer | |
| 79 | + if err := html.Render(&buf, n); err != nil { | |
| 80 | + return "" | |
| 81 | + } | |
| 82 | + var lines []string | |
| 83 | + for _, line := range strings.Split(buf.String(), "\n") { | |
| 84 | + if strings.TrimSpace(line) != "" { | |
| 85 | + lines = append(lines, line) | |
| 86 | + } | |
| 87 | + } | |
| 88 | + return strings.Join(lines, "\n") | |
| 89 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,89 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "bytes" | ||
| 5 | + "strings" | ||
| 6 | + | ||
| 7 | + "golang.org/x/net/html" | ||
| 8 | +) | ||
| 9 | + | ||
| 10 | +// Laying a table out: as a GFM grid, or as raw HTML. | ||
| 11 | + | ||
| 12 | +// grid lays the cells out on a rectangle, copying a spanned cell's text | ||
| 13 | +// into every slot it covers. Without spans, it is the rows as they are. | ||
| 14 | +func (c *converter) grid(rows [][]tableCell) [][]string { | ||
| 15 | + grid := make([][]string, len(rows)) | ||
| 16 | + filled := make([][]bool, len(rows)) | ||
| 17 | + set := func(r, col int, text string) { | ||
| 18 | + for len(grid[r]) <= col { | ||
| 19 | + grid[r] = append(grid[r], "") | ||
| 20 | + filled[r] = append(filled[r], false) | ||
| 21 | + } | ||
| 22 | + grid[r][col], filled[r][col] = text, true | ||
| 23 | + } | ||
| 24 | + for r, row := range rows { | ||
| 25 | + col := 0 | ||
| 26 | + for _, cell := range row { | ||
| 27 | + for col < len(filled[r]) && filled[r][col] { | ||
| 28 | + col++ | ||
| 29 | + } | ||
| 30 | + text := c.cellText(cell.node) | ||
| 31 | + for dr := 0; dr < cell.rowspan && r+dr < len(rows); dr++ { | ||
| 32 | + for dc := 0; dc < cell.colspan; dc++ { | ||
| 33 | + set(r+dr, col+dc, text) | ||
| 34 | + } | ||
| 35 | + } | ||
| 36 | + col += cell.colspan | ||
| 37 | + } | ||
| 38 | + } | ||
| 39 | + return grid | ||
| 40 | +} | ||
| 41 | + | ||
| 42 | +// cellText renders a cell on one line: its blocks and hard breaks become | ||
| 43 | +// <br>, and a pipe is escaped so it does not end the cell. | ||
| 44 | +func (c *converter) cellText(n *html.Node) string { | ||
| 45 | + text := joinBlocks(c.blocks(n)) | ||
| 46 | + text = strings.ReplaceAll(text, "\\\n", "\n") | ||
| 47 | + text = strings.ReplaceAll(text, "\n\n", "\n") | ||
| 48 | + text = strings.ReplaceAll(text, "\n", "<br>") | ||
| 49 | + return strings.ReplaceAll(text, "|", `\|`) | ||
| 50 | +} | ||
| 51 | + | ||
| 52 | +func gfmTable(grid [][]string) string { | ||
| 53 | + width := 0 | ||
| 54 | + empty := true | ||
| 55 | + for _, row := range grid { | ||
| 56 | + width = max(width, len(row)) | ||
| 57 | + for _, text := range row { | ||
| 58 | + empty = empty && text == "" | ||
| 59 | + } | ||
| 60 | + } | ||
| 61 | + if width == 0 || empty { | ||
| 62 | + return "" | ||
| 63 | + } | ||
| 64 | + lines := make([]string, 0, len(grid)+1) | ||
| 65 | + for i, row := range grid { | ||
| 66 | + cells := append(row[:len(row):len(row)], make([]string, width-len(row))...) | ||
| 67 | + lines = append(lines, "| "+strings.Join(cells, " | ")+" |") | ||
| 68 | + if i == 0 { | ||
| 69 | + lines = append(lines, "|"+strings.Repeat("---|", width)) | ||
| 70 | + } | ||
| 71 | + } | ||
| 72 | + return strings.Join(lines, "\n") | ||
| 73 | +} | ||
| 74 | + | ||
| 75 | +// rawHTML renders the cleaned table as HTML, without the blank lines that | ||
| 76 | +// would end a Markdown HTML block halfway through it. | ||
| 77 | +func rawHTML(n *html.Node) string { | ||
| 78 | + var buf bytes.Buffer | ||
| 79 | + if err := html.Render(&buf, n); err != nil { | ||
| 80 | + return "" | ||
| 81 | + } | ||
| 82 | + var lines []string | ||
| 83 | + for _, line := range strings.Split(buf.String(), "\n") { | ||
| 84 | + if strings.TrimSpace(line) != "" { | ||
| 85 | + lines = append(lines, line) | ||
| 86 | + } | ||
| 87 | + } | ||
| 88 | + return strings.Join(lines, "\n") | ||
| 89 | +} | ||
added
internal/webmd/inline.go +95 -0 | new file mode 100644 | ||
| @@ -0,0 +1,95 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "strings" | |
| 5 | + | |
| 6 | + "golang.org/x/net/html" | |
| 7 | + "golang.org/x/net/html/atom" | |
| 8 | +) | |
| 9 | + | |
| 10 | +// inline renders a text node or an inline element. | |
| 11 | +func (c *converter) inline(n *html.Node) string { | |
| 12 | + switch n.Type { | |
| 13 | + case html.TextNode: | |
| 14 | + return escapeText(collapseSpace(n.Data)) | |
| 15 | + case html.ElementNode: | |
| 16 | + return c.inlineElement(n) | |
| 17 | + } | |
| 18 | + return "" | |
| 19 | +} | |
| 20 | + | |
| 21 | +// inlineElement renders an inline element. An unknown or purely | |
| 22 | +// presentational one loses its tags and keeps its text. | |
| 23 | +func (c *converter) inlineElement(n *html.Node) string { | |
| 24 | + var md string | |
| 25 | + switch n.DataAtom { | |
| 26 | + case atom.Br: | |
| 27 | + // A raw newline: paragraph() turns it into a Markdown hard break, | |
| 28 | + // headings and table cells into what they can hold. | |
| 29 | + md = "\n" | |
| 30 | + case atom.Strong, atom.B: | |
| 31 | + md = emphasis(c.children(n), "**") | |
| 32 | + case atom.Em, atom.I: | |
| 33 | + md = emphasis(c.children(n), "*") | |
| 34 | + case atom.Code, atom.Kbd, atom.Samp, atom.Tt: | |
| 35 | + md = codeSpan(textContent(n)) | |
| 36 | + case atom.Del, atom.S, atom.Strike: | |
| 37 | + md = c.strikethrough(n) | |
| 38 | + case atom.A: | |
| 39 | + md = c.link(n) | |
| 40 | + case atom.Img: | |
| 41 | + md = image(n) | |
| 42 | + case atom.Input: | |
| 43 | + // A checkbox reaching this point is not the head of a task-list | |
| 44 | + // item (or task lists are off): it carries no text. | |
| 45 | + md = "" | |
| 46 | + case atom.Q: | |
| 47 | + md = `"` + c.children(n) + `"` | |
| 48 | + default: | |
| 49 | + md = c.children(n) | |
| 50 | + } | |
| 51 | + return md | |
| 52 | +} | |
| 53 | + | |
| 54 | +// strikethrough renders deleted text as ~~text~~ when the option is on, | |
| 55 | +// as plain text otherwise. | |
| 56 | +func (c *converter) strikethrough(n *html.Node) string { | |
| 57 | + if c.opts.Strikethrough { | |
| 58 | + return emphasis(c.children(n), "~~") | |
| 59 | + } | |
| 60 | + return c.children(n) | |
| 61 | +} | |
| 62 | + | |
| 63 | +// children renders the children of n inline, whatever they are. | |
| 64 | +func (c *converter) children(n *html.Node) string { | |
| 65 | + var parts []string | |
| 66 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | |
| 67 | + parts = append(parts, c.inline(ch)) | |
| 68 | + } | |
| 69 | + return joinInline(parts) | |
| 70 | +} | |
| 71 | + | |
| 72 | +// link renders <a href> as [text](url); a link without text shows its URL. | |
| 73 | +func (c *converter) link(n *html.Node) string { | |
| 74 | + text := c.children(n) | |
| 75 | + href := strings.TrimSpace(attr(n, "href")) | |
| 76 | + if href == "" || strings.HasPrefix(strings.ToLower(href), "javascript:") { | |
| 77 | + return text | |
| 78 | + } | |
| 79 | + lead, core, trail := splitSpace(strings.ReplaceAll(text, "\n", " ")) | |
| 80 | + if core == "" { | |
| 81 | + core = escapeText(href) | |
| 82 | + } | |
| 83 | + return lead + "[" + core + "](" + destination(href) + ")" + trail | |
| 84 | +} | |
| 85 | + | |
| 86 | +// image renders <img> as . An inline data: URI would put | |
| 87 | +// kilobytes of base64 in the text: such an image is reduced to its alt. | |
| 88 | +func image(n *html.Node) string { | |
| 89 | + src := strings.TrimSpace(attr(n, "src")) | |
| 90 | + alt := escapeText(collapseSpace(attr(n, "alt"))) | |
| 91 | + if src == "" || strings.HasPrefix(strings.ToLower(src), "data:") { | |
| 92 | + return alt | |
| 93 | + } | |
| 94 | + return "![" + strings.TrimSpace(alt) + "](" + destination(src) + ")" | |
| 95 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,95 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "strings" | ||
| 5 | + | ||
| 6 | + "golang.org/x/net/html" | ||
| 7 | + "golang.org/x/net/html/atom" | ||
| 8 | +) | ||
| 9 | + | ||
| 10 | +// inline renders a text node or an inline element. | ||
| 11 | +func (c *converter) inline(n *html.Node) string { | ||
| 12 | + switch n.Type { | ||
| 13 | + case html.TextNode: | ||
| 14 | + return escapeText(collapseSpace(n.Data)) | ||
| 15 | + case html.ElementNode: | ||
| 16 | + return c.inlineElement(n) | ||
| 17 | + } | ||
| 18 | + return "" | ||
| 19 | +} | ||
| 20 | + | ||
| 21 | +// inlineElement renders an inline element. An unknown or purely | ||
| 22 | +// presentational one loses its tags and keeps its text. | ||
| 23 | +func (c *converter) inlineElement(n *html.Node) string { | ||
| 24 | + var md string | ||
| 25 | + switch n.DataAtom { | ||
| 26 | + case atom.Br: | ||
| 27 | + // A raw newline: paragraph() turns it into a Markdown hard break, | ||
| 28 | + // headings and table cells into what they can hold. | ||
| 29 | + md = "\n" | ||
| 30 | + case atom.Strong, atom.B: | ||
| 31 | + md = emphasis(c.children(n), "**") | ||
| 32 | + case atom.Em, atom.I: | ||
| 33 | + md = emphasis(c.children(n), "*") | ||
| 34 | + case atom.Code, atom.Kbd, atom.Samp, atom.Tt: | ||
| 35 | + md = codeSpan(textContent(n)) | ||
| 36 | + case atom.Del, atom.S, atom.Strike: | ||
| 37 | + md = c.strikethrough(n) | ||
| 38 | + case atom.A: | ||
| 39 | + md = c.link(n) | ||
| 40 | + case atom.Img: | ||
| 41 | + md = image(n) | ||
| 42 | + case atom.Input: | ||
| 43 | + // A checkbox reaching this point is not the head of a task-list | ||
| 44 | + // item (or task lists are off): it carries no text. | ||
| 45 | + md = "" | ||
| 46 | + case atom.Q: | ||
| 47 | + md = `"` + c.children(n) + `"` | ||
| 48 | + default: | ||
| 49 | + md = c.children(n) | ||
| 50 | + } | ||
| 51 | + return md | ||
| 52 | +} | ||
| 53 | + | ||
| 54 | +// strikethrough renders deleted text as ~~text~~ when the option is on, | ||
| 55 | +// as plain text otherwise. | ||
| 56 | +func (c *converter) strikethrough(n *html.Node) string { | ||
| 57 | + if c.opts.Strikethrough { | ||
| 58 | + return emphasis(c.children(n), "~~") | ||
| 59 | + } | ||
| 60 | + return c.children(n) | ||
| 61 | +} | ||
| 62 | + | ||
| 63 | +// children renders the children of n inline, whatever they are. | ||
| 64 | +func (c *converter) children(n *html.Node) string { | ||
| 65 | + var parts []string | ||
| 66 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | ||
| 67 | + parts = append(parts, c.inline(ch)) | ||
| 68 | + } | ||
| 69 | + return joinInline(parts) | ||
| 70 | +} | ||
| 71 | + | ||
| 72 | +// link renders <a href> as [text](url); a link without text shows its URL. | ||
| 73 | +func (c *converter) link(n *html.Node) string { | ||
| 74 | + text := c.children(n) | ||
| 75 | + href := strings.TrimSpace(attr(n, "href")) | ||
| 76 | + if href == "" || strings.HasPrefix(strings.ToLower(href), "javascript:") { | ||
| 77 | + return text | ||
| 78 | + } | ||
| 79 | + lead, core, trail := splitSpace(strings.ReplaceAll(text, "\n", " ")) | ||
| 80 | + if core == "" { | ||
| 81 | + core = escapeText(href) | ||
| 82 | + } | ||
| 83 | + return lead + "[" + core + "](" + destination(href) + ")" + trail | ||
| 84 | +} | ||
| 85 | + | ||
| 86 | +// image renders <img> as . An inline data: URI would put | ||
| 87 | +// kilobytes of base64 in the text: such an image is reduced to its alt. | ||
| 88 | +func image(n *html.Node) string { | ||
| 89 | + src := strings.TrimSpace(attr(n, "src")) | ||
| 90 | + alt := escapeText(collapseSpace(attr(n, "alt"))) | ||
| 91 | + if src == "" || strings.HasPrefix(strings.ToLower(src), "data:") { | ||
| 92 | + return alt | ||
| 93 | + } | ||
| 94 | + return "![" + strings.TrimSpace(alt) + "](" + destination(src) + ")" | ||
| 95 | +} | ||
added
internal/webmd/lists.go +108 -0 | new file mode 100644 | ||
| @@ -0,0 +1,108 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "strconv" | |
| 5 | + "strings" | |
| 6 | + | |
| 7 | + "golang.org/x/net/html" | |
| 8 | + "golang.org/x/net/html/atom" | |
| 9 | +) | |
| 10 | + | |
| 11 | +// list renders <ul> as "- " items and <ol> as "1. " items (from the | |
| 12 | +// start attribute when there is one). A nested list is indented by the | |
| 13 | +// width of its parent's marker, which is what CommonMark needs to keep it | |
| 14 | +// inside the item — the indentation is therefore the same at every level | |
| 15 | +// for "- " lists. | |
| 16 | +func (c *converter) list(n *html.Node, ordered bool) string { | |
| 17 | + number := 1 | |
| 18 | + if ordered { | |
| 19 | + if start, err := strconv.Atoi(strings.TrimSpace(attr(n, "start"))); err == nil { | |
| 20 | + number = start | |
| 21 | + } | |
| 22 | + } | |
| 23 | + var items []string | |
| 24 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | |
| 25 | + switch { | |
| 26 | + case ch.DataAtom == atom.Li: | |
| 27 | + marker := "- " | |
| 28 | + if ordered { | |
| 29 | + marker = strconv.Itoa(number) + ". " | |
| 30 | + number++ | |
| 31 | + } | |
| 32 | + if item := c.listItem(ch, marker); item != "" { | |
| 33 | + items = append(items, item) | |
| 34 | + } | |
| 35 | + case (ch.DataAtom == atom.Ul || ch.DataAtom == atom.Ol) && len(items) > 0: | |
| 36 | + // <ul><li>a</li><ul>…</ul></ul> is invalid but common: the | |
| 37 | + // stray list belongs to the item before it. | |
| 38 | + nested := c.list(ch, ch.DataAtom == atom.Ol) | |
| 39 | + items[len(items)-1] += "\n" + indent(nested, " ") | |
| 40 | + } | |
| 41 | + } | |
| 42 | + return strings.Join(items, "\n") | |
| 43 | +} | |
| 44 | + | |
| 45 | +// listItem renders one <li> behind its marker. Text followed by nested | |
| 46 | +// lists stays tight; several paragraphs (or a paragraph and a code block, | |
| 47 | +// a table…) are separated by blank lines, or Markdown would merge them. | |
| 48 | +func (c *converter) listItem(li *html.Node, marker string) string { | |
| 49 | + box := "" | |
| 50 | + if c.opts.TaskList { | |
| 51 | + if input := leadingCheckbox(li); input != nil { | |
| 52 | + box = "[ ] " | |
| 53 | + if hasAttr(input, "checked") { | |
| 54 | + box = "[x] " | |
| 55 | + } | |
| 56 | + input.Parent.RemoveChild(input) | |
| 57 | + } | |
| 58 | + } | |
| 59 | + | |
| 60 | + parts := c.blocks(li) | |
| 61 | + separator := "\n" | |
| 62 | + for _, p := range parts[min(1, len(parts)):] { | |
| 63 | + if !p.list { | |
| 64 | + separator = "\n\n" | |
| 65 | + } | |
| 66 | + } | |
| 67 | + texts := make([]string, len(parts)) | |
| 68 | + for i, p := range parts { | |
| 69 | + texts[i] = p.md | |
| 70 | + } | |
| 71 | + body := box + strings.Join(texts, separator) | |
| 72 | + if strings.TrimSpace(body) == "" { | |
| 73 | + return "" | |
| 74 | + } | |
| 75 | + return marker + indent(body, strings.Repeat(" ", len(marker)))[len(marker):] | |
| 76 | +} | |
| 77 | + | |
| 78 | +// indent prefixes every non-empty line of s with pad. | |
| 79 | +func indent(s, pad string) string { | |
| 80 | + lines := strings.Split(s, "\n") | |
| 81 | + for i, line := range lines { | |
| 82 | + if line != "" { | |
| 83 | + lines[i] = pad + line | |
| 84 | + } | |
| 85 | + } | |
| 86 | + return strings.Join(lines, "\n") | |
| 87 | +} | |
| 88 | + | |
| 89 | +// leadingCheckbox returns the checkbox an item starts with — directly or | |
| 90 | +// inside a leading <p>, <label> or <span> — or nil when text comes first. | |
| 91 | +func leadingCheckbox(n *html.Node) *html.Node { | |
| 92 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | |
| 93 | + switch { | |
| 94 | + case ch.Type == html.TextNode && strings.TrimSpace(ch.Data) == "": | |
| 95 | + continue | |
| 96 | + case ch.Type != html.ElementNode: | |
| 97 | + return nil | |
| 98 | + case ch.DataAtom == atom.Input: | |
| 99 | + if attr(ch, "type") == "checkbox" { | |
| 100 | + return ch | |
| 101 | + } | |
| 102 | + return nil | |
| 103 | + default: | |
| 104 | + return leadingCheckbox(ch) | |
| 105 | + } | |
| 106 | + } | |
| 107 | + return nil | |
| 108 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,108 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "strconv" | ||
| 5 | + "strings" | ||
| 6 | + | ||
| 7 | + "golang.org/x/net/html" | ||
| 8 | + "golang.org/x/net/html/atom" | ||
| 9 | +) | ||
| 10 | + | ||
| 11 | +// list renders <ul> as "- " items and <ol> as "1. " items (from the | ||
| 12 | +// start attribute when there is one). A nested list is indented by the | ||
| 13 | +// width of its parent's marker, which is what CommonMark needs to keep it | ||
| 14 | +// inside the item — the indentation is therefore the same at every level | ||
| 15 | +// for "- " lists. | ||
| 16 | +func (c *converter) list(n *html.Node, ordered bool) string { | ||
| 17 | + number := 1 | ||
| 18 | + if ordered { | ||
| 19 | + if start, err := strconv.Atoi(strings.TrimSpace(attr(n, "start"))); err == nil { | ||
| 20 | + number = start | ||
| 21 | + } | ||
| 22 | + } | ||
| 23 | + var items []string | ||
| 24 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | ||
| 25 | + switch { | ||
| 26 | + case ch.DataAtom == atom.Li: | ||
| 27 | + marker := "- " | ||
| 28 | + if ordered { | ||
| 29 | + marker = strconv.Itoa(number) + ". " | ||
| 30 | + number++ | ||
| 31 | + } | ||
| 32 | + if item := c.listItem(ch, marker); item != "" { | ||
| 33 | + items = append(items, item) | ||
| 34 | + } | ||
| 35 | + case (ch.DataAtom == atom.Ul || ch.DataAtom == atom.Ol) && len(items) > 0: | ||
| 36 | + // <ul><li>a</li><ul>…</ul></ul> is invalid but common: the | ||
| 37 | + // stray list belongs to the item before it. | ||
| 38 | + nested := c.list(ch, ch.DataAtom == atom.Ol) | ||
| 39 | + items[len(items)-1] += "\n" + indent(nested, " ") | ||
| 40 | + } | ||
| 41 | + } | ||
| 42 | + return strings.Join(items, "\n") | ||
| 43 | +} | ||
| 44 | + | ||
| 45 | +// listItem renders one <li> behind its marker. Text followed by nested | ||
| 46 | +// lists stays tight; several paragraphs (or a paragraph and a code block, | ||
| 47 | +// a table…) are separated by blank lines, or Markdown would merge them. | ||
| 48 | +func (c *converter) listItem(li *html.Node, marker string) string { | ||
| 49 | + box := "" | ||
| 50 | + if c.opts.TaskList { | ||
| 51 | + if input := leadingCheckbox(li); input != nil { | ||
| 52 | + box = "[ ] " | ||
| 53 | + if hasAttr(input, "checked") { | ||
| 54 | + box = "[x] " | ||
| 55 | + } | ||
| 56 | + input.Parent.RemoveChild(input) | ||
| 57 | + } | ||
| 58 | + } | ||
| 59 | + | ||
| 60 | + parts := c.blocks(li) | ||
| 61 | + separator := "\n" | ||
| 62 | + for _, p := range parts[min(1, len(parts)):] { | ||
| 63 | + if !p.list { | ||
| 64 | + separator = "\n\n" | ||
| 65 | + } | ||
| 66 | + } | ||
| 67 | + texts := make([]string, len(parts)) | ||
| 68 | + for i, p := range parts { | ||
| 69 | + texts[i] = p.md | ||
| 70 | + } | ||
| 71 | + body := box + strings.Join(texts, separator) | ||
| 72 | + if strings.TrimSpace(body) == "" { | ||
| 73 | + return "" | ||
| 74 | + } | ||
| 75 | + return marker + indent(body, strings.Repeat(" ", len(marker)))[len(marker):] | ||
| 76 | +} | ||
| 77 | + | ||
| 78 | +// indent prefixes every non-empty line of s with pad. | ||
| 79 | +func indent(s, pad string) string { | ||
| 80 | + lines := strings.Split(s, "\n") | ||
| 81 | + for i, line := range lines { | ||
| 82 | + if line != "" { | ||
| 83 | + lines[i] = pad + line | ||
| 84 | + } | ||
| 85 | + } | ||
| 86 | + return strings.Join(lines, "\n") | ||
| 87 | +} | ||
| 88 | + | ||
| 89 | +// leadingCheckbox returns the checkbox an item starts with — directly or | ||
| 90 | +// inside a leading <p>, <label> or <span> — or nil when text comes first. | ||
| 91 | +func leadingCheckbox(n *html.Node) *html.Node { | ||
| 92 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | ||
| 93 | + switch { | ||
| 94 | + case ch.Type == html.TextNode && strings.TrimSpace(ch.Data) == "": | ||
| 95 | + continue | ||
| 96 | + case ch.Type != html.ElementNode: | ||
| 97 | + return nil | ||
| 98 | + case ch.DataAtom == atom.Input: | ||
| 99 | + if attr(ch, "type") == "checkbox" { | ||
| 100 | + return ch | ||
| 101 | + } | ||
| 102 | + return nil | ||
| 103 | + default: | ||
| 104 | + return leadingCheckbox(ch) | ||
| 105 | + } | ||
| 106 | + } | ||
| 107 | + return nil | ||
| 108 | +} | ||
added
internal/webmd/robots.go +125 -0 | new file mode 100644 | ||
| @@ -0,0 +1,125 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "bufio" | |
| 5 | + "regexp" | |
| 6 | + "strings" | |
| 7 | +) | |
| 8 | + | |
| 9 | +// robotsRules is the part of a robots.txt (RFC 9309) that applies to one | |
| 10 | +// user agent: the Allow and Disallow lines of the groups naming it, or of | |
| 11 | +// the "*" group when none does. | |
| 12 | +type robotsRules struct { | |
| 13 | + allow []string | |
| 14 | + disallow []string | |
| 15 | +} | |
| 16 | + | |
| 17 | +// allowAll is the answer when a site has no robots.txt. | |
| 18 | +var allowAll = &robotsRules{} | |
| 19 | + | |
| 20 | +// disallowAll is the answer when the robots.txt cannot be read (a 5xx or a | |
| 21 | +// network failure): RFC 9309 §2.3.1.4 asks crawlers to assume the worst. | |
| 22 | +var disallowAll = &robotsRules{disallow: []string{"/"}} | |
| 23 | + | |
| 24 | +// parseRobots extracts the rules for agent (a product token such as "mm", | |
| 25 | +// matched without regard to case) from the text of a robots.txt. | |
| 26 | +func parseRobots(text, agent string) *robotsRules { | |
| 27 | + agent = strings.ToLower(agent) | |
| 28 | + var mine, star robotsRules | |
| 29 | + foundMine := false | |
| 30 | + | |
| 31 | + // A group is a run of user-agent lines followed by rules; a user-agent | |
| 32 | + // line after a rule starts a new group. | |
| 33 | + var agents []string | |
| 34 | + inRules := false | |
| 35 | + scanner := bufio.NewScanner(strings.NewReader(text)) | |
| 36 | + for scanner.Scan() { | |
| 37 | + key, value, ok := robotsLine(scanner.Text()) | |
| 38 | + if !ok { | |
| 39 | + continue | |
| 40 | + } | |
| 41 | + switch key { | |
| 42 | + case "user-agent": | |
| 43 | + if inRules { | |
| 44 | + agents, inRules = nil, false | |
| 45 | + } | |
| 46 | + agents = append(agents, strings.ToLower(value)) | |
| 47 | + case "allow", "disallow": | |
| 48 | + inRules = true | |
| 49 | + for _, a := range agents { | |
| 50 | + switch a { | |
| 51 | + case agent: | |
| 52 | + foundMine = true | |
| 53 | + mine.add(key, value) | |
| 54 | + case "*": | |
| 55 | + star.add(key, value) | |
| 56 | + } | |
| 57 | + } | |
| 58 | + } | |
| 59 | + } | |
| 60 | + if foundMine { | |
| 61 | + return &mine | |
| 62 | + } | |
| 63 | + return &star | |
| 64 | +} | |
| 65 | + | |
| 66 | +// robotsLine splits "Key: value # comment" into a lower-case key and a | |
| 67 | +// value; ok is false for blank lines, comments and malformed lines. | |
| 68 | +func robotsLine(line string) (key, value string, ok bool) { | |
| 69 | + if i := strings.IndexByte(line, '#'); i >= 0 { | |
| 70 | + line = line[:i] | |
| 71 | + } | |
| 72 | + key, value, ok = strings.Cut(line, ":") | |
| 73 | + if !ok { | |
| 74 | + return "", "", false | |
| 75 | + } | |
| 76 | + return strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value), true | |
| 77 | +} | |
| 78 | + | |
| 79 | +func (r *robotsRules) add(key, pattern string) { | |
| 80 | + // An empty Disallow allows everything; an empty Allow says nothing. | |
| 81 | + if pattern == "" { | |
| 82 | + return | |
| 83 | + } | |
| 84 | + if key == "allow" { | |
| 85 | + r.allow = append(r.allow, pattern) | |
| 86 | + } else { | |
| 87 | + r.disallow = append(r.disallow, pattern) | |
| 88 | + } | |
| 89 | +} | |
| 90 | + | |
| 91 | +// allows reports whether path (with its query) may be fetched: the | |
| 92 | +// longest matching pattern wins, and Allow wins a tie. | |
| 93 | +func (r *robotsRules) allows(path string) bool { | |
| 94 | + longestAllow := longestMatch(r.allow, path) | |
| 95 | + longestDisallow := longestMatch(r.disallow, path) | |
| 96 | + return longestDisallow < 0 || longestAllow >= longestDisallow | |
| 97 | +} | |
| 98 | + | |
| 99 | +// longestMatch returns the length of the longest pattern matching path, | |
| 100 | +// -1 when none does. | |
| 101 | +func longestMatch(patterns []string, path string) int { | |
| 102 | + longest := -1 | |
| 103 | + for _, p := range patterns { | |
| 104 | + if len(p) > longest && robotsPattern(p).MatchString(path) { | |
| 105 | + longest = len(p) | |
| 106 | + } | |
| 107 | + } | |
| 108 | + return longest | |
| 109 | +} | |
| 110 | + | |
| 111 | +// robotsPattern compiles a robots.txt path pattern: a prefix match where | |
| 112 | +// "*" is any sequence of characters and a final "$" anchors the end. | |
| 113 | +func robotsPattern(p string) *regexp.Regexp { | |
| 114 | + anchored := strings.HasSuffix(p, "$") | |
| 115 | + p = strings.TrimSuffix(p, "$") | |
| 116 | + parts := strings.Split(p, "*") | |
| 117 | + for i, part := range parts { | |
| 118 | + parts[i] = regexp.QuoteMeta(part) | |
| 119 | + } | |
| 120 | + expr := "^" + strings.Join(parts, ".*") | |
| 121 | + if anchored { | |
| 122 | + expr += "$" | |
| 123 | + } | |
| 124 | + return regexp.MustCompile(expr) | |
| 125 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,125 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "bufio" | ||
| 5 | + "regexp" | ||
| 6 | + "strings" | ||
| 7 | +) | ||
| 8 | + | ||
| 9 | +// robotsRules is the part of a robots.txt (RFC 9309) that applies to one | ||
| 10 | +// user agent: the Allow and Disallow lines of the groups naming it, or of | ||
| 11 | +// the "*" group when none does. | ||
| 12 | +type robotsRules struct { | ||
| 13 | + allow []string | ||
| 14 | + disallow []string | ||
| 15 | +} | ||
| 16 | + | ||
| 17 | +// allowAll is the answer when a site has no robots.txt. | ||
| 18 | +var allowAll = &robotsRules{} | ||
| 19 | + | ||
| 20 | +// disallowAll is the answer when the robots.txt cannot be read (a 5xx or a | ||
| 21 | +// network failure): RFC 9309 §2.3.1.4 asks crawlers to assume the worst. | ||
| 22 | +var disallowAll = &robotsRules{disallow: []string{"/"}} | ||
| 23 | + | ||
| 24 | +// parseRobots extracts the rules for agent (a product token such as "mm", | ||
| 25 | +// matched without regard to case) from the text of a robots.txt. | ||
| 26 | +func parseRobots(text, agent string) *robotsRules { | ||
| 27 | + agent = strings.ToLower(agent) | ||
| 28 | + var mine, star robotsRules | ||
| 29 | + foundMine := false | ||
| 30 | + | ||
| 31 | + // A group is a run of user-agent lines followed by rules; a user-agent | ||
| 32 | + // line after a rule starts a new group. | ||
| 33 | + var agents []string | ||
| 34 | + inRules := false | ||
| 35 | + scanner := bufio.NewScanner(strings.NewReader(text)) | ||
| 36 | + for scanner.Scan() { | ||
| 37 | + key, value, ok := robotsLine(scanner.Text()) | ||
| 38 | + if !ok { | ||
| 39 | + continue | ||
| 40 | + } | ||
| 41 | + switch key { | ||
| 42 | + case "user-agent": | ||
| 43 | + if inRules { | ||
| 44 | + agents, inRules = nil, false | ||
| 45 | + } | ||
| 46 | + agents = append(agents, strings.ToLower(value)) | ||
| 47 | + case "allow", "disallow": | ||
| 48 | + inRules = true | ||
| 49 | + for _, a := range agents { | ||
| 50 | + switch a { | ||
| 51 | + case agent: | ||
| 52 | + foundMine = true | ||
| 53 | + mine.add(key, value) | ||
| 54 | + case "*": | ||
| 55 | + star.add(key, value) | ||
| 56 | + } | ||
| 57 | + } | ||
| 58 | + } | ||
| 59 | + } | ||
| 60 | + if foundMine { | ||
| 61 | + return &mine | ||
| 62 | + } | ||
| 63 | + return &star | ||
| 64 | +} | ||
| 65 | + | ||
| 66 | +// robotsLine splits "Key: value # comment" into a lower-case key and a | ||
| 67 | +// value; ok is false for blank lines, comments and malformed lines. | ||
| 68 | +func robotsLine(line string) (key, value string, ok bool) { | ||
| 69 | + if i := strings.IndexByte(line, '#'); i >= 0 { | ||
| 70 | + line = line[:i] | ||
| 71 | + } | ||
| 72 | + key, value, ok = strings.Cut(line, ":") | ||
| 73 | + if !ok { | ||
| 74 | + return "", "", false | ||
| 75 | + } | ||
| 76 | + return strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value), true | ||
| 77 | +} | ||
| 78 | + | ||
| 79 | +func (r *robotsRules) add(key, pattern string) { | ||
| 80 | + // An empty Disallow allows everything; an empty Allow says nothing. | ||
| 81 | + if pattern == "" { | ||
| 82 | + return | ||
| 83 | + } | ||
| 84 | + if key == "allow" { | ||
| 85 | + r.allow = append(r.allow, pattern) | ||
| 86 | + } else { | ||
| 87 | + r.disallow = append(r.disallow, pattern) | ||
| 88 | + } | ||
| 89 | +} | ||
| 90 | + | ||
| 91 | +// allows reports whether path (with its query) may be fetched: the | ||
| 92 | +// longest matching pattern wins, and Allow wins a tie. | ||
| 93 | +func (r *robotsRules) allows(path string) bool { | ||
| 94 | + longestAllow := longestMatch(r.allow, path) | ||
| 95 | + longestDisallow := longestMatch(r.disallow, path) | ||
| 96 | + return longestDisallow < 0 || longestAllow >= longestDisallow | ||
| 97 | +} | ||
| 98 | + | ||
| 99 | +// longestMatch returns the length of the longest pattern matching path, | ||
| 100 | +// -1 when none does. | ||
| 101 | +func longestMatch(patterns []string, path string) int { | ||
| 102 | + longest := -1 | ||
| 103 | + for _, p := range patterns { | ||
| 104 | + if len(p) > longest && robotsPattern(p).MatchString(path) { | ||
| 105 | + longest = len(p) | ||
| 106 | + } | ||
| 107 | + } | ||
| 108 | + return longest | ||
| 109 | +} | ||
| 110 | + | ||
| 111 | +// robotsPattern compiles a robots.txt path pattern: a prefix match where | ||
| 112 | +// "*" is any sequence of characters and a final "$" anchors the end. | ||
| 113 | +func robotsPattern(p string) *regexp.Regexp { | ||
| 114 | + anchored := strings.HasSuffix(p, "$") | ||
| 115 | + p = strings.TrimSuffix(p, "$") | ||
| 116 | + parts := strings.Split(p, "*") | ||
| 117 | + for i, part := range parts { | ||
| 118 | + parts[i] = regexp.QuoteMeta(part) | ||
| 119 | + } | ||
| 120 | + expr := "^" + strings.Join(parts, ".*") | ||
| 121 | + if anchored { | ||
| 122 | + expr += "$" | ||
| 123 | + } | ||
| 124 | + return regexp.MustCompile(expr) | ||
| 125 | +} | ||
added
internal/webmd/robots_test.go +70 -0 | new file mode 100644 | ||
| @@ -0,0 +1,70 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "errors" | |
| 5 | + "testing" | |
| 6 | +) | |
| 7 | + | |
| 8 | +func TestRobotsRules(t *testing.T) { | |
| 9 | + text := `# comment | |
| 10 | +User-agent: Googlebot | |
| 11 | +Disallow: / | |
| 12 | + | |
| 13 | +User-agent: MM | |
| 14 | +User-agent: somebot | |
| 15 | +Disallow: /tmp/ | |
| 16 | +Allow: /tmp/public | |
| 17 | +Disallow: /*.pdf$ | |
| 18 | +Disallow: /search? | |
| 19 | + | |
| 20 | +User-agent: * | |
| 21 | +Disallow: /everything-for-others | |
| 22 | +Disallow: | |
| 23 | +` | |
| 24 | + mm := parseRobots(text, "mm") | |
| 25 | + for path, want := range map[string]bool{ | |
| 26 | + "/": true, | |
| 27 | + "/tmp/x": false, | |
| 28 | + "/tmp/public/page": true, // longer Allow wins | |
| 29 | + "/docs/a.pdf": false, // * and $ patterns | |
| 30 | + "/docs/a.pdf.html": true, // $ anchors the end | |
| 31 | + "/search?q=go": false, | |
| 32 | + "/everything-for-others": true, // not our group | |
| 33 | + } { | |
| 34 | + if got := mm.allows(path); got != want { | |
| 35 | + t.Errorf("mm allows(%q) = %v, want %v", path, got, want) | |
| 36 | + } | |
| 37 | + } | |
| 38 | + | |
| 39 | + other := parseRobots(text, "unknownbot") | |
| 40 | + if other.allows("/everything-for-others") || !other.allows("/tmp/x") { | |
| 41 | + t.Error("an agent without a group of its own must get the * group") | |
| 42 | + } | |
| 43 | +} | |
| 44 | + | |
| 45 | +func TestRobotsTieGoesToAllow(t *testing.T) { | |
| 46 | + r := parseRobots("User-agent: *\nDisallow: /page\nAllow: /page\n", "mm") | |
| 47 | + if !r.allows("/page") { | |
| 48 | + t.Error("equal-length Allow and Disallow: Allow must win") | |
| 49 | + } | |
| 50 | +} | |
| 51 | + | |
| 52 | +func TestRobotsEmptyFileAllowsEverything(t *testing.T) { | |
| 53 | + if !parseRobots("", "mm").allows("/anything") || !allowAll.allows("/x") || disallowAll.allows("/x") { | |
| 54 | + t.Error("empty robots.txt, allowAll and disallowAll") | |
| 55 | + } | |
| 56 | +} | |
| 57 | + | |
| 58 | +func TestCodeOf(t *testing.T) { | |
| 59 | + if CodeOf(nil) != CodeOK || CodeOf(errors.New("x")) != CodeConversion { | |
| 60 | + t.Error("nil and foreign errors") | |
| 61 | + } | |
| 62 | + wrapped := errors.Join(errors.New("ctx"), errorf(CodeUsage, nil, "bad")) | |
| 63 | + if CodeOf(wrapped) != CodeUsage { | |
| 64 | + t.Error("the code of a wrapped *Error") | |
| 65 | + } | |
| 66 | + e := errorf(CodeInput, errors.New("refused"), "cannot fetch") | |
| 67 | + if e.Error() != "cannot fetch: refused" || CodeInput.String() != "input" { | |
| 68 | + t.Errorf("message %q", e.Error()) | |
| 69 | + } | |
| 70 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,70 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "errors" | ||
| 5 | + "testing" | ||
| 6 | +) | ||
| 7 | + | ||
| 8 | +func TestRobotsRules(t *testing.T) { | ||
| 9 | + text := `# comment | ||
| 10 | +User-agent: Googlebot | ||
| 11 | +Disallow: / | ||
| 12 | + | ||
| 13 | +User-agent: MM | ||
| 14 | +User-agent: somebot | ||
| 15 | +Disallow: /tmp/ | ||
| 16 | +Allow: /tmp/public | ||
| 17 | +Disallow: /*.pdf$ | ||
| 18 | +Disallow: /search? | ||
| 19 | + | ||
| 20 | +User-agent: * | ||
| 21 | +Disallow: /everything-for-others | ||
| 22 | +Disallow: | ||
| 23 | +` | ||
| 24 | + mm := parseRobots(text, "mm") | ||
| 25 | + for path, want := range map[string]bool{ | ||
| 26 | + "/": true, | ||
| 27 | + "/tmp/x": false, | ||
| 28 | + "/tmp/public/page": true, // longer Allow wins | ||
| 29 | + "/docs/a.pdf": false, // * and $ patterns | ||
| 30 | + "/docs/a.pdf.html": true, // $ anchors the end | ||
| 31 | + "/search?q=go": false, | ||
| 32 | + "/everything-for-others": true, // not our group | ||
| 33 | + } { | ||
| 34 | + if got := mm.allows(path); got != want { | ||
| 35 | + t.Errorf("mm allows(%q) = %v, want %v", path, got, want) | ||
| 36 | + } | ||
| 37 | + } | ||
| 38 | + | ||
| 39 | + other := parseRobots(text, "unknownbot") | ||
| 40 | + if other.allows("/everything-for-others") || !other.allows("/tmp/x") { | ||
| 41 | + t.Error("an agent without a group of its own must get the * group") | ||
| 42 | + } | ||
| 43 | +} | ||
| 44 | + | ||
| 45 | +func TestRobotsTieGoesToAllow(t *testing.T) { | ||
| 46 | + r := parseRobots("User-agent: *\nDisallow: /page\nAllow: /page\n", "mm") | ||
| 47 | + if !r.allows("/page") { | ||
| 48 | + t.Error("equal-length Allow and Disallow: Allow must win") | ||
| 49 | + } | ||
| 50 | +} | ||
| 51 | + | ||
| 52 | +func TestRobotsEmptyFileAllowsEverything(t *testing.T) { | ||
| 53 | + if !parseRobots("", "mm").allows("/anything") || !allowAll.allows("/x") || disallowAll.allows("/x") { | ||
| 54 | + t.Error("empty robots.txt, allowAll and disallowAll") | ||
| 55 | + } | ||
| 56 | +} | ||
| 57 | + | ||
| 58 | +func TestCodeOf(t *testing.T) { | ||
| 59 | + if CodeOf(nil) != CodeOK || CodeOf(errors.New("x")) != CodeConversion { | ||
| 60 | + t.Error("nil and foreign errors") | ||
| 61 | + } | ||
| 62 | + wrapped := errors.Join(errors.New("ctx"), errorf(CodeUsage, nil, "bad")) | ||
| 63 | + if CodeOf(wrapped) != CodeUsage { | ||
| 64 | + t.Error("the code of a wrapped *Error") | ||
| 65 | + } | ||
| 66 | + e := errorf(CodeInput, errors.New("refused"), "cannot fetch") | ||
| 67 | + if e.Error() != "cannot fetch: refused" || CodeInput.String() != "input" { | ||
| 68 | + t.Errorf("message %q", e.Error()) | ||
| 69 | + } | ||
| 70 | +} | ||
added
internal/webmd/tables.go +110 -0 | new file mode 100644 | ||
| @@ -0,0 +1,110 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "strconv" | |
| 5 | + "strings" | |
| 6 | + | |
| 7 | + "golang.org/x/net/html" | |
| 8 | + "golang.org/x/net/html/atom" | |
| 9 | +) | |
| 10 | + | |
| 11 | +// maxSpan caps a rowspan or colspan: a page asking for colspan="100000" | |
| 12 | +// must not make the flattened table explode. | |
| 13 | +const maxSpan = 64 | |
| 14 | + | |
| 15 | +type tableCell struct { | |
| 16 | + node *html.Node | |
| 17 | + colspan int | |
| 18 | + rowspan int | |
| 19 | +} | |
| 20 | + | |
| 21 | +// table renders a <table> as a GitHub-Flavored Markdown table, its | |
| 22 | +// <caption> as a paragraph above it. | |
| 23 | +// | |
| 24 | +// A table Markdown cannot express — a rowspan or colspan, or another table | |
| 25 | +// inside a cell — is kept as raw HTML with TablesHTML (the default), or | |
| 26 | +// flattened with TablesFlatten. GFM needs a header row: the first row is | |
| 27 | +// used as one, whether it is made of <th> or not. | |
| 28 | +func (c *converter) table(n *html.Node) string { | |
| 29 | + rows, caption := tableRows(n) | |
| 30 | + var md string | |
| 31 | + if isComplex(rows) && c.opts.Tables == TablesHTML { | |
| 32 | + md = rawHTML(n) | |
| 33 | + } else { | |
| 34 | + md = gfmTable(c.grid(rows)) | |
| 35 | + } | |
| 36 | + if caption == nil || md == "" { | |
| 37 | + return md | |
| 38 | + } | |
| 39 | + if title := joinBlocks(c.blocks(caption)); title != "" { | |
| 40 | + return title + "\n\n" + md | |
| 41 | + } | |
| 42 | + return md | |
| 43 | +} | |
| 44 | + | |
| 45 | +// isComplex reports whether GFM cannot express the table as it is: a cell | |
| 46 | +// spans rows or columns, or holds another table. | |
| 47 | +func isComplex(rows [][]tableCell) bool { | |
| 48 | + for _, row := range rows { | |
| 49 | + for _, cell := range row { | |
| 50 | + if cell.colspan > 1 || cell.rowspan > 1 || hasDescendant(cell.node, atom.Table) { | |
| 51 | + return true | |
| 52 | + } | |
| 53 | + } | |
| 54 | + } | |
| 55 | + return false | |
| 56 | +} | |
| 57 | + | |
| 58 | +// tableRows collects the rows of n — directly, or in its thead, tbody and | |
| 59 | +// tfoot — without descending into nested tables. | |
| 60 | +func tableRows(n *html.Node) (rows [][]tableCell, caption *html.Node) { | |
| 61 | + var walk func(*html.Node) | |
| 62 | + walk = func(n *html.Node) { | |
| 63 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | |
| 64 | + switch ch.DataAtom { | |
| 65 | + case atom.Caption: | |
| 66 | + caption = ch | |
| 67 | + case atom.Thead, atom.Tbody, atom.Tfoot: | |
| 68 | + walk(ch) | |
| 69 | + case atom.Tr: | |
| 70 | + if row := rowCells(ch); len(row) > 0 { | |
| 71 | + rows = append(rows, row) | |
| 72 | + } | |
| 73 | + } | |
| 74 | + } | |
| 75 | + } | |
| 76 | + walk(n) | |
| 77 | + return rows, caption | |
| 78 | +} | |
| 79 | + | |
| 80 | +func rowCells(tr *html.Node) []tableCell { | |
| 81 | + var row []tableCell | |
| 82 | + for ch := tr.FirstChild; ch != nil; ch = ch.NextSibling { | |
| 83 | + if ch.DataAtom != atom.Td && ch.DataAtom != atom.Th { | |
| 84 | + continue | |
| 85 | + } | |
| 86 | + row = append(row, tableCell{ | |
| 87 | + node: ch, | |
| 88 | + colspan: span(attr(ch, "colspan")), | |
| 89 | + rowspan: span(attr(ch, "rowspan")), | |
| 90 | + }) | |
| 91 | + } | |
| 92 | + return row | |
| 93 | +} | |
| 94 | + | |
| 95 | +func span(value string) int { | |
| 96 | + n, err := strconv.Atoi(strings.TrimSpace(value)) | |
| 97 | + if err != nil || n < 1 { | |
| 98 | + return 1 | |
| 99 | + } | |
| 100 | + return min(n, maxSpan) | |
| 101 | +} | |
| 102 | + | |
| 103 | +func hasDescendant(n *html.Node, tag atom.Atom) bool { | |
| 104 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | |
| 105 | + if ch.DataAtom == tag || hasDescendant(ch, tag) { | |
| 106 | + return true | |
| 107 | + } | |
| 108 | + } | |
| 109 | + return false | |
| 110 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,110 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "strconv" | ||
| 5 | + "strings" | ||
| 6 | + | ||
| 7 | + "golang.org/x/net/html" | ||
| 8 | + "golang.org/x/net/html/atom" | ||
| 9 | +) | ||
| 10 | + | ||
| 11 | +// maxSpan caps a rowspan or colspan: a page asking for colspan="100000" | ||
| 12 | +// must not make the flattened table explode. | ||
| 13 | +const maxSpan = 64 | ||
| 14 | + | ||
| 15 | +type tableCell struct { | ||
| 16 | + node *html.Node | ||
| 17 | + colspan int | ||
| 18 | + rowspan int | ||
| 19 | +} | ||
| 20 | + | ||
| 21 | +// table renders a <table> as a GitHub-Flavored Markdown table, its | ||
| 22 | +// <caption> as a paragraph above it. | ||
| 23 | +// | ||
| 24 | +// A table Markdown cannot express — a rowspan or colspan, or another table | ||
| 25 | +// inside a cell — is kept as raw HTML with TablesHTML (the default), or | ||
| 26 | +// flattened with TablesFlatten. GFM needs a header row: the first row is | ||
| 27 | +// used as one, whether it is made of <th> or not. | ||
| 28 | +func (c *converter) table(n *html.Node) string { | ||
| 29 | + rows, caption := tableRows(n) | ||
| 30 | + var md string | ||
| 31 | + if isComplex(rows) && c.opts.Tables == TablesHTML { | ||
| 32 | + md = rawHTML(n) | ||
| 33 | + } else { | ||
| 34 | + md = gfmTable(c.grid(rows)) | ||
| 35 | + } | ||
| 36 | + if caption == nil || md == "" { | ||
| 37 | + return md | ||
| 38 | + } | ||
| 39 | + if title := joinBlocks(c.blocks(caption)); title != "" { | ||
| 40 | + return title + "\n\n" + md | ||
| 41 | + } | ||
| 42 | + return md | ||
| 43 | +} | ||
| 44 | + | ||
| 45 | +// isComplex reports whether GFM cannot express the table as it is: a cell | ||
| 46 | +// spans rows or columns, or holds another table. | ||
| 47 | +func isComplex(rows [][]tableCell) bool { | ||
| 48 | + for _, row := range rows { | ||
| 49 | + for _, cell := range row { | ||
| 50 | + if cell.colspan > 1 || cell.rowspan > 1 || hasDescendant(cell.node, atom.Table) { | ||
| 51 | + return true | ||
| 52 | + } | ||
| 53 | + } | ||
| 54 | + } | ||
| 55 | + return false | ||
| 56 | +} | ||
| 57 | + | ||
| 58 | +// tableRows collects the rows of n — directly, or in its thead, tbody and | ||
| 59 | +// tfoot — without descending into nested tables. | ||
| 60 | +func tableRows(n *html.Node) (rows [][]tableCell, caption *html.Node) { | ||
| 61 | + var walk func(*html.Node) | ||
| 62 | + walk = func(n *html.Node) { | ||
| 63 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | ||
| 64 | + switch ch.DataAtom { | ||
| 65 | + case atom.Caption: | ||
| 66 | + caption = ch | ||
| 67 | + case atom.Thead, atom.Tbody, atom.Tfoot: | ||
| 68 | + walk(ch) | ||
| 69 | + case atom.Tr: | ||
| 70 | + if row := rowCells(ch); len(row) > 0 { | ||
| 71 | + rows = append(rows, row) | ||
| 72 | + } | ||
| 73 | + } | ||
| 74 | + } | ||
| 75 | + } | ||
| 76 | + walk(n) | ||
| 77 | + return rows, caption | ||
| 78 | +} | ||
| 79 | + | ||
| 80 | +func rowCells(tr *html.Node) []tableCell { | ||
| 81 | + var row []tableCell | ||
| 82 | + for ch := tr.FirstChild; ch != nil; ch = ch.NextSibling { | ||
| 83 | + if ch.DataAtom != atom.Td && ch.DataAtom != atom.Th { | ||
| 84 | + continue | ||
| 85 | + } | ||
| 86 | + row = append(row, tableCell{ | ||
| 87 | + node: ch, | ||
| 88 | + colspan: span(attr(ch, "colspan")), | ||
| 89 | + rowspan: span(attr(ch, "rowspan")), | ||
| 90 | + }) | ||
| 91 | + } | ||
| 92 | + return row | ||
| 93 | +} | ||
| 94 | + | ||
| 95 | +func span(value string) int { | ||
| 96 | + n, err := strconv.Atoi(strings.TrimSpace(value)) | ||
| 97 | + if err != nil || n < 1 { | ||
| 98 | + return 1 | ||
| 99 | + } | ||
| 100 | + return min(n, maxSpan) | ||
| 101 | +} | ||
| 102 | + | ||
| 103 | +func hasDescendant(n *html.Node, tag atom.Atom) bool { | ||
| 104 | + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { | ||
| 105 | + if ch.DataAtom == tag || hasDescendant(ch, tag) { | ||
| 106 | + return true | ||
| 107 | + } | ||
| 108 | + } | ||
| 109 | + return false | ||
| 110 | +} | ||
added
internal/webmd/text.go +149 -0 | new file mode 100644 | ||
| @@ -0,0 +1,149 @@ | ||
| 1 | +package webmd | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "regexp" | |
| 5 | + "strings" | |
| 6 | + "unicode" | |
| 7 | +) | |
| 8 | + | |
| 9 | +// htmlSpace matches a run of HTML whitespace, which a browser shows as | |
| 10 | +// one space outside <pre>. | |
| 11 | +var htmlSpace = regexp.MustCompile(`[ \t\n\r\f]+`) | |
| 12 | + | |
| 13 | +func collapseSpace(s string) string { | |
| 14 | + return htmlSpace.ReplaceAllString(s, " ") | |
| 15 | +} | |
| 16 | + | |
| 17 | +// escapeText backslash-escapes what Markdown would otherwise read as | |
| 18 | +// syntax: \ * ` [ ] < and an underscore that is not inside a word | |
| 19 | +// (GFM ignores snake_case, so escaping it would only add noise). | |
| 20 | +func escapeText(s string) string { | |
| 21 | + runes := []rune(s) | |
| 22 | + var b strings.Builder | |
| 23 | + for i, r := range runes { | |
| 24 | + switch r { | |
| 25 | + case '\\', '*', '`', '[', ']', '<': | |
| 26 | + b.WriteRune('\\') | |
| 27 | + case '_': | |
| 28 | + if !inWord(runes, i) { | |
| 29 | + b.WriteRune('\\') | |
| 30 | + } | |
| 31 | + } | |
| 32 | + b.WriteRune(r) | |
| 33 | + } | |
| 34 | + return b.String() | |
| 35 | +} | |
| 36 | + | |
| 37 | +func inWord(runes []rune, i int) bool { | |
| 38 | + isWord := func(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) } | |
| 39 | + return i > 0 && i < len(runes)-1 && isWord(runes[i-1]) && isWord(runes[i+1]) | |
| 40 | +} | |
| 41 | + | |
| 42 | +// joinInline concatenates inline fragments, dropping a space that would | |
| 43 | +// double one already there — two adjacent text nodes each ending and | |
| 44 | +// starting with whitespace render as one space in a browser. | |
| 45 | +func joinInline(parts []string) string { | |
| 46 | + var b strings.Builder | |
| 47 | + // A leading space is kept: inside <b> or <a> it must move outside the | |
| 48 | + // marker (see emphasis), and paragraph() trims what starts a block. | |
| 49 | + afterSpace := false | |
| 50 | + for _, p := range parts { | |
| 51 | + if afterSpace { | |
| 52 | + p = strings.TrimLeft(p, " ") | |
| 53 | + } | |
| 54 | + if p == "" { | |
| 55 | + continue | |
| 56 | + } | |
| 57 | + b.WriteString(p) | |
| 58 | + afterSpace = p[len(p)-1] == ' ' || p[len(p)-1] == '\n' | |
| 59 | + } | |
| 60 | + return b.String() | |
| 61 | +} | |
| 62 | + | |
| 63 | +// splitSpace separates the leading and trailing spaces of s from its core. | |
| 64 | +func splitSpace(s string) (lead, core, trail string) { | |
| 65 | + core = strings.TrimSpace(s) | |
| 66 | + if core == "" { | |
| 67 | + return "", "", "" | |
| 68 | + } | |
| 69 | + if s[0] == ' ' { | |
| 70 | + lead = " " | |
| 71 | + } | |
| 72 | + if s[len(s)-1] == ' ' { | |
| 73 | + trail = " " | |
| 74 | + } | |
| 75 | + return lead, core, trail | |
| 76 | +} | |
| 77 | + | |
| 78 | +// emphasis wraps text in marker, keeping its surrounding spaces outside | |
| 79 | +// ("a<b> bold </b>b" must give "a **bold** b", not "a** bold **b"). | |
| 80 | +func emphasis(text, marker string) string { | |
| 81 | + lead, core, trail := splitSpace(text) | |
| 82 | + if core == "" { | |
| 83 | + return lead | |
| 84 | + } | |
| 85 | + return lead + marker + core + marker + trail | |
| 86 | +} | |
| 87 | + | |
| 88 | +// codeSpan renders inline code with a backtick fence longer than any run | |
| 89 | +// inside it, padded when the code itself starts or ends with a backtick. | |
| 90 | +func codeSpan(code string) string { | |
| 91 | + code = strings.ReplaceAll(code, "\n", " ") | |
| 92 | + if strings.TrimSpace(code) == "" { | |
| 93 | + return "" | |
| 94 | + } | |
| 95 | + fence := strings.Repeat("`", longestRun(code, '`')+1) | |
| 96 | + if strings.HasPrefix(code, "`") || strings.HasSuffix(code, "`") { | |
| 97 | + code = " " + code + " " | |
| 98 | + } | |
| 99 | + return fence + code + fence | |
| 100 | +} | |
| 101 | + | |
| 102 | +func longestRun(s string, r rune) int { | |
| 103 | + longest, current := 0, 0 | |
| 104 | + for _, c := range s { | |
| 105 | + if c == r { | |
| 106 | + current++ | |
| 107 | + longest = max(longest, current) | |
| 108 | + } else { | |
| 109 | + current = 0 | |
| 110 | + } | |
| 111 | + } | |
| 112 | + return longest | |
| 113 | +} | |
| 114 | + | |
| 115 | +// paragraph turns a run of inline Markdown into a paragraph: each line | |
| 116 | +// (a <br> made it) is trimmed, empty ones dropped, and the rest joined by | |
| 117 | +// hard breaks. A line starting with a character Markdown would read as a | |
| 118 | +// block marker is escaped. | |
| 119 | +func paragraph(inline string) string { | |
| 120 | + var lines []string | |
| 121 | + for _, line := range strings.Split(inline, "\n") { | |
| 122 | + if line = strings.TrimSpace(line); line != "" { | |
| 123 | + lines = append(lines, escapeLineStart(line)) | |
| 124 | + } | |
| 125 | + } | |
| 126 | + return strings.Join(lines, "\\\n") | |
| 127 | +} | |
| 128 | + | |
| 129 | +// orderedMarker matches the start of a line that would begin an ordered list. | |
| 130 | +var orderedMarker = regexp.MustCompile(`^(\d+)([.)])(\s|$)`) | |
| 131 | + | |
| 132 | +func escapeLineStart(line string) string { | |
| 133 | + if m := orderedMarker.FindStringSubmatchIndex(line); m != nil { | |
| 134 | + return line[:m[3]] + `\` + line[m[3]:] | |
| 135 | + } | |
| 136 | + switch line[0] { | |
| 137 | + case '#', '>', '-', '+', '=', '|': | |
| 138 | + return `\` + line | |
| 139 | + } | |
| 140 | + return line | |
| 141 | +} | |
| 142 | + | |
| 143 | +// destination makes a URL safe inside Markdown's (…): spaces, parentheses | |
| 144 | +// and angle brackets are percent-encoded. | |
| 145 | +var destinationEscaper = strings.NewReplacer(" ", "%20", "(", "%28", ")", "%29", "<", "%3C", ">", "%3E") | |
| 146 | + | |
| 147 | +func destination(u string) string { | |
| 148 | + return destinationEscaper.Replace(u) | |
| 149 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,149 @@ | |||
| 1 | +package webmd | ||
| 2 | + | ||
| 3 | +import ( | ||
| 4 | + "regexp" | ||
| 5 | + "strings" | ||
| 6 | + "unicode" | ||
| 7 | +) | ||
| 8 | + | ||
| 9 | +// htmlSpace matches a run of HTML whitespace, which a browser shows as | ||
| 10 | +// one space outside <pre>. | ||
| 11 | +var htmlSpace = regexp.MustCompile(`[ \t\n\r\f]+`) | ||
| 12 | + | ||
| 13 | +func collapseSpace(s string) string { | ||
| 14 | + return htmlSpace.ReplaceAllString(s, " ") | ||
| 15 | +} | ||
| 16 | + | ||
| 17 | +// escapeText backslash-escapes what Markdown would otherwise read as | ||
| 18 | +// syntax: \ * ` [ ] < and an underscore that is not inside a word | ||
| 19 | +// (GFM ignores snake_case, so escaping it would only add noise). | ||
| 20 | +func escapeText(s string) string { | ||
| 21 | + runes := []rune(s) | ||
| 22 | + var b strings.Builder | ||
| 23 | + for i, r := range runes { | ||
| 24 | + switch r { | ||
| 25 | + case '\\', '*', '`', '[', ']', '<': | ||
| 26 | + b.WriteRune('\\') | ||
| 27 | + case '_': | ||
| 28 | + if !inWord(runes, i) { | ||
| 29 | + b.WriteRune('\\') | ||
| 30 | + } | ||
| 31 | + } | ||
| 32 | + b.WriteRune(r) | ||
| 33 | + } | ||
| 34 | + return b.String() | ||
| 35 | +} | ||
| 36 | + | ||
| 37 | +func inWord(runes []rune, i int) bool { | ||
| 38 | + isWord := func(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) } | ||
| 39 | + return i > 0 && i < len(runes)-1 && isWord(runes[i-1]) && isWord(runes[i+1]) | ||
| 40 | +} | ||
| 41 | + | ||
| 42 | +// joinInline concatenates inline fragments, dropping a space that would | ||
| 43 | +// double one already there — two adjacent text nodes each ending and | ||
| 44 | +// starting with whitespace render as one space in a browser. | ||
| 45 | +func joinInline(parts []string) string { | ||
| 46 | + var b strings.Builder | ||
| 47 | + // A leading space is kept: inside <b> or <a> it must move outside the | ||
| 48 | + // marker (see emphasis), and paragraph() trims what starts a block. | ||
| 49 | + afterSpace := false | ||
| 50 | + for _, p := range parts { | ||
| 51 | + if afterSpace { | ||
| 52 | + p = strings.TrimLeft(p, " ") | ||
| 53 | + } | ||
| 54 | + if p == "" { | ||
| 55 | + continue | ||
| 56 | + } | ||
| 57 | + b.WriteString(p) | ||
| 58 | + afterSpace = p[len(p)-1] == ' ' || p[len(p)-1] == '\n' | ||
| 59 | + } | ||
| 60 | + return b.String() | ||
| 61 | +} | ||
| 62 | + | ||
| 63 | +// splitSpace separates the leading and trailing spaces of s from its core. | ||
| 64 | +func splitSpace(s string) (lead, core, trail string) { | ||
| 65 | + core = strings.TrimSpace(s) | ||
| 66 | + if core == "" { | ||
| 67 | + return "", "", "" | ||
| 68 | + } | ||
| 69 | + if s[0] == ' ' { | ||
| 70 | + lead = " " | ||
| 71 | + } | ||
| 72 | + if s[len(s)-1] == ' ' { | ||
| 73 | + trail = " " | ||
| 74 | + } | ||
| 75 | + return lead, core, trail | ||
| 76 | +} | ||
| 77 | + | ||
| 78 | +// emphasis wraps text in marker, keeping its surrounding spaces outside | ||
| 79 | +// ("a<b> bold </b>b" must give "a **bold** b", not "a** bold **b"). | ||
| 80 | +func emphasis(text, marker string) string { | ||
| 81 | + lead, core, trail := splitSpace(text) | ||
| 82 | + if core == "" { | ||
| 83 | + return lead | ||
| 84 | + } | ||
| 85 | + return lead + marker + core + marker + trail | ||
| 86 | +} | ||
| 87 | + | ||
| 88 | +// codeSpan renders inline code with a backtick fence longer than any run | ||
| 89 | +// inside it, padded when the code itself starts or ends with a backtick. | ||
| 90 | +func codeSpan(code string) string { | ||
| 91 | + code = strings.ReplaceAll(code, "\n", " ") | ||
| 92 | + if strings.TrimSpace(code) == "" { | ||
| 93 | + return "" | ||
| 94 | + } | ||
| 95 | + fence := strings.Repeat("`", longestRun(code, '`')+1) | ||
| 96 | + if strings.HasPrefix(code, "`") || strings.HasSuffix(code, "`") { | ||
| 97 | + code = " " + code + " " | ||
| 98 | + } | ||
| 99 | + return fence + code + fence | ||
| 100 | +} | ||
| 101 | + | ||
| 102 | +func longestRun(s string, r rune) int { | ||
| 103 | + longest, current := 0, 0 | ||
| 104 | + for _, c := range s { | ||
| 105 | + if c == r { | ||
| 106 | + current++ | ||
| 107 | + longest = max(longest, current) | ||
| 108 | + } else { | ||
| 109 | + current = 0 | ||
| 110 | + } | ||
| 111 | + } | ||
| 112 | + return longest | ||
| 113 | +} | ||
| 114 | + | ||
| 115 | +// paragraph turns a run of inline Markdown into a paragraph: each line | ||
| 116 | +// (a <br> made it) is trimmed, empty ones dropped, and the rest joined by | ||
| 117 | +// hard breaks. A line starting with a character Markdown would read as a | ||
| 118 | +// block marker is escaped. | ||
| 119 | +func paragraph(inline string) string { | ||
| 120 | + var lines []string | ||
| 121 | + for _, line := range strings.Split(inline, "\n") { | ||
| 122 | + if line = strings.TrimSpace(line); line != "" { | ||
| 123 | + lines = append(lines, escapeLineStart(line)) | ||
| 124 | + } | ||
| 125 | + } | ||
| 126 | + return strings.Join(lines, "\\\n") | ||
| 127 | +} | ||
| 128 | + | ||
| 129 | +// orderedMarker matches the start of a line that would begin an ordered list. | ||
| 130 | +var orderedMarker = regexp.MustCompile(`^(\d+)([.)])(\s|$)`) | ||
| 131 | + | ||
| 132 | +func escapeLineStart(line string) string { | ||
| 133 | + if m := orderedMarker.FindStringSubmatchIndex(line); m != nil { | ||
| 134 | + return line[:m[3]] + `\` + line[m[3]:] | ||
| 135 | + } | ||
| 136 | + switch line[0] { | ||
| 137 | + case '#', '>', '-', '+', '=', '|': | ||
| 138 | + return `\` + line | ||
| 139 | + } | ||
| 140 | + return line | ||
| 141 | +} | ||
| 142 | + | ||
| 143 | +// destination makes a URL safe inside Markdown's (…): spaces, parentheses | ||
| 144 | +// and angle brackets are percent-encoded. | ||
| 145 | +var destinationEscaper = strings.NewReplacer(" ", "%20", "(", "%28", ")", "%29", "<", "%3C", ">", "%3E") | ||
| 146 | + | ||
| 147 | +func destination(u string) string { | ||
| 148 | + return destinationEscaper.Replace(u) | ||
| 149 | +} | ||
added
internal/webmd/webmd.go +73 -0 | new file mode 100644 | ||
| @@ -0,0 +1,73 @@ | ||
| 1 | +// Package webmd downloads a web page and turns it into Markdown. | |
| 2 | +// | |
| 3 | +// It is the engine behind the `fetch_content` tool, split in three stages | |
| 4 | +// that can be used — and tested — on their own: | |
| 5 | +// | |
| 6 | +// - Fetcher.Fetch downloads a page over HTTP(S), honouring the site's | |
| 7 | +// robots.txt, and returns its HTML normalised to UTF-8 (the encoding is | |
| 8 | +// read from the BOM, the Content-Type header or a <meta charset>). | |
| 9 | +// - Clean strips what is not content: scripts, styles, comments, meta | |
| 10 | +// tags, navigation, advertisements, decorative elements and | |
| 11 | +// presentation attributes. | |
| 12 | +// - Convert renders what remains as GitHub-Flavored Markdown. | |
| 13 | +// | |
| 14 | +// Every error this package returns is an *Error carrying a Code (see | |
| 15 | +// errors.go), so a caller can map a failure to the documented exit codes | |
| 16 | +// without parsing messages. | |
| 17 | +// | |
| 18 | +// Example — the whole pipeline: | |
| 19 | +// | |
| 20 | +// f := webmd.NewFetcher("mm/0.3.0") | |
| 21 | +// page, err := f.Fetch(ctx, "https://example.com/") | |
| 22 | +// if err != nil { | |
| 23 | +// return webmd.CodeOf(err), err | |
| 24 | +// } | |
| 25 | +// md, err := webmd.Convert(page.HTML, webmd.Options{BaseURL: page.URL}) | |
| 26 | +package webmd | |
| 27 | + | |
| 28 | +import ( | |
| 29 | + "net/url" | |
| 30 | +) | |
| 31 | + | |
| 32 | +// TableMode says what happens to a table that Markdown cannot express — | |
| 33 | +// one with a rowspan or a colspan, or a table nested in another. | |
| 34 | +type TableMode string | |
| 35 | + | |
| 36 | +const ( | |
| 37 | + // TablesHTML keeps such a table as raw (cleaned) HTML. It is the | |
| 38 | + // default: nothing is lost, and GFM renderers display inline HTML. | |
| 39 | + TablesHTML TableMode = "html" | |
| 40 | + // TablesFlatten turns such a table into a GFM table anyway, copying a | |
| 41 | + // spanned cell's content into every slot it covers so that each row | |
| 42 | + // stays readable on its own. | |
| 43 | + TablesFlatten TableMode = "flatten" | |
| 44 | +) | |
| 45 | + | |
| 46 | +// ParseTableMode validates a user-supplied table mode; "" means the default. | |
| 47 | +// | |
| 48 | +// mode, err := webmd.ParseTableMode("flatten") // TablesFlatten, nil | |
| 49 | +func ParseTableMode(s string) (TableMode, error) { | |
| 50 | + switch TableMode(s) { | |
| 51 | + case "", TablesHTML: | |
| 52 | + return TablesHTML, nil | |
| 53 | + case TablesFlatten: | |
| 54 | + return TablesFlatten, nil | |
| 55 | + } | |
| 56 | + return "", errorf(CodeUsage, nil, "tables must be %q or %q, got %q", TablesHTML, TablesFlatten, s) | |
| 57 | +} | |
| 58 | + | |
| 59 | +// Options tunes the conversion. The zero value is valid: no URL | |
| 60 | +// resolution, complex tables kept as HTML, no strikethrough, no task lists. | |
| 61 | +type Options struct { | |
| 62 | + // BaseURL resolves relative links and image sources. Nil leaves them | |
| 63 | + // as written in the page. | |
| 64 | + BaseURL *url.URL | |
| 65 | + // Tables is what to do with tables Markdown cannot express. | |
| 66 | + Tables TableMode | |
| 67 | + // Strikethrough renders <del>, <s> and <strike> as ~~text~~; off, the | |
| 68 | + // text is kept without the marker. | |
| 69 | + Strikethrough bool | |
| 70 | + // TaskList renders a list item that starts with a checkbox as | |
| 71 | + // "- [ ] …" or "- [x] …"; off, the checkbox is dropped. | |
| 72 | + TaskList bool | |
| 73 | +} | |
| new file mode 100644 | |||
| @@ -0,0 +1,73 @@ | |||
| 1 | +// Package webmd downloads a web page and turns it into Markdown. | ||
| 2 | +// | ||
| 3 | +// It is the engine behind the `fetch_content` tool, split in three stages | ||
| 4 | +// that can be used — and tested — on their own: | ||
| 5 | +// | ||
| 6 | +// - Fetcher.Fetch downloads a page over HTTP(S), honouring the site's | ||
| 7 | +// robots.txt, and returns its HTML normalised to UTF-8 (the encoding is | ||
| 8 | +// read from the BOM, the Content-Type header or a <meta charset>). | ||
| 9 | +// - Clean strips what is not content: scripts, styles, comments, meta | ||
| 10 | +// tags, navigation, advertisements, decorative elements and | ||
| 11 | +// presentation attributes. | ||
| 12 | +// - Convert renders what remains as GitHub-Flavored Markdown. | ||
| 13 | +// | ||
| 14 | +// Every error this package returns is an *Error carrying a Code (see | ||
| 15 | +// errors.go), so a caller can map a failure to the documented exit codes | ||
| 16 | +// without parsing messages. | ||
| 17 | +// | ||
| 18 | +// Example — the whole pipeline: | ||
| 19 | +// | ||
| 20 | +// f := webmd.NewFetcher("mm/0.3.0") | ||
| 21 | +// page, err := f.Fetch(ctx, "https://example.com/") | ||
| 22 | +// if err != nil { | ||
| 23 | +// return webmd.CodeOf(err), err | ||
| 24 | +// } | ||
| 25 | +// md, err := webmd.Convert(page.HTML, webmd.Options{BaseURL: page.URL}) | ||
| 26 | +package webmd | ||
| 27 | + | ||
| 28 | +import ( | ||
| 29 | + "net/url" | ||
| 30 | +) | ||
| 31 | + | ||
| 32 | +// TableMode says what happens to a table that Markdown cannot express — | ||
| 33 | +// one with a rowspan or a colspan, or a table nested in another. | ||
| 34 | +type TableMode string | ||
| 35 | + | ||
| 36 | +const ( | ||
| 37 | + // TablesHTML keeps such a table as raw (cleaned) HTML. It is the | ||
| 38 | + // default: nothing is lost, and GFM renderers display inline HTML. | ||
| 39 | + TablesHTML TableMode = "html" | ||
| 40 | + // TablesFlatten turns such a table into a GFM table anyway, copying a | ||
| 41 | + // spanned cell's content into every slot it covers so that each row | ||
| 42 | + // stays readable on its own. | ||
| 43 | + TablesFlatten TableMode = "flatten" | ||
| 44 | +) | ||
| 45 | + | ||
| 46 | +// ParseTableMode validates a user-supplied table mode; "" means the default. | ||
| 47 | +// | ||
| 48 | +// mode, err := webmd.ParseTableMode("flatten") // TablesFlatten, nil | ||
| 49 | +func ParseTableMode(s string) (TableMode, error) { | ||
| 50 | + switch TableMode(s) { | ||
| 51 | + case "", TablesHTML: | ||
| 52 | + return TablesHTML, nil | ||
| 53 | + case TablesFlatten: | ||
| 54 | + return TablesFlatten, nil | ||
| 55 | + } | ||
| 56 | + return "", errorf(CodeUsage, nil, "tables must be %q or %q, got %q", TablesHTML, TablesFlatten, s) | ||
| 57 | +} | ||
| 58 | + | ||
| 59 | +// Options tunes the conversion. The zero value is valid: no URL | ||
| 60 | +// resolution, complex tables kept as HTML, no strikethrough, no task lists. | ||
| 61 | +type Options struct { | ||
| 62 | + // BaseURL resolves relative links and image sources. Nil leaves them | ||
| 63 | + // as written in the page. | ||
| 64 | + BaseURL *url.URL | ||
| 65 | + // Tables is what to do with tables Markdown cannot express. | ||
| 66 | + Tables TableMode | ||
| 67 | + // Strikethrough renders <del>, <s> and <strike> as ~~text~~; off, the | ||
| 68 | + // text is kept without the marker. | ||
| 69 | + Strikethrough bool | ||
| 70 | + // TaskList renders a list item that starts with a checkbox as | ||
| 71 | + // "- [ ] …" or "- [x] …"; off, the checkbox is dropped. | ||
| 72 | + TaskList bool | ||
| 73 | +} | ||
modified
main.go +10 -2 | @@ -49,6 +49,7 @@ import ( | ||
| 49 | 49 | "mm/internal/tui" |
| 50 | 50 | "mm/internal/ui" |
| 51 | 51 | "mm/internal/version" |
| 52 | + "mm/internal/webmd" | |
| 52 | 53 | |
| 53 | 54 | "github.com/firebase/genkit/go/ai" |
| 54 | 55 | ) |
| @@ -256,8 +257,9 @@ type toolSet struct { | ||
| 256 | 257 | // description carries the catalogue, so declaring it empty would advertise a |
| 257 | 258 | // tool that can do nothing); the file tools behind `editTools` — the same |
| 258 | 259 | // binary must run as part 09 did (bash + the `edit` CLI) and as this part |
| 259 | -// (built-in tools), so the two can be compared on the same prompts; and | |
| 260 | -// `lsp` when `lsp.servers` names at least one server whose command is on | |
| 260 | +// (built-in tools), so the two can be compared on the same prompts; | |
| 261 | +// `fetch_content` behind `fetchContentTool`, off by default because it is the | |
| 262 | +// one built-in tool that reaches the network; and `lsp` when `lsp.servers` names at least one server whose command is on | |
| 261 | 263 | // the PATH — a missing server is a warning, never a refusal to start. |
| 262 | 264 | func assembleTools(e *engine.Engine, d *detector.LoopDetector) toolSet { |
| 263 | 265 | set := toolSet{close: func() {}} |
| @@ -287,6 +289,12 @@ func assembleTools(e *engine.Engine, d *detector.LoopDetector) toolSet { | ||
| 287 | 289 | set.tools = append(set.tools, tools.PlanTool(e.G, d)) |
| 288 | 290 | set.names = append(set.names, "plan") |
| 289 | 291 | } |
| 292 | + if config.Cfg.FetchContentTool { | |
| 293 | + // The product token "mm" is what the sites' robots.txt are read for. | |
| 294 | + fetcher := webmd.NewFetcher("mm/" + version.Short()) | |
| 295 | + set.tools = append(set.tools, tools.FetchContent(e.G, d, fetcher)) | |
| 296 | + set.names = append(set.names, "fetch_content") | |
| 297 | + } | |
| 290 | 298 | assembleLSP(e, d, &set) |
| 291 | 299 | assembleMCP(e, d, &set) |
| 292 | 300 | return set |
| @@ -49,6 +49,7 @@ import ( | |||
| 49 | "mm/internal/tui" | 49 | "mm/internal/tui" |
| 50 | "mm/internal/ui" | 50 | "mm/internal/ui" |
| 51 | "mm/internal/version" | 51 | "mm/internal/version" |
| 52 | + "mm/internal/webmd" | ||
| 52 | 53 | ||
| 53 | "github.com/firebase/genkit/go/ai" | 54 | "github.com/firebase/genkit/go/ai" |
| 54 | ) | 55 | ) |
| @@ -256,8 +257,9 @@ type toolSet struct { | |||
| 256 | // description carries the catalogue, so declaring it empty would advertise a | 257 | // description carries the catalogue, so declaring it empty would advertise a |
| 257 | // tool that can do nothing); the file tools behind `editTools` — the same | 258 | // tool that can do nothing); the file tools behind `editTools` — the same |
| 258 | // binary must run as part 09 did (bash + the `edit` CLI) and as this part | 259 | // binary must run as part 09 did (bash + the `edit` CLI) and as this part |
| 259 | -// (built-in tools), so the two can be compared on the same prompts; and | 260 | +// (built-in tools), so the two can be compared on the same prompts; |
| 260 | -// `lsp` when `lsp.servers` names at least one server whose command is on | 261 | +// `fetch_content` behind `fetchContentTool`, off by default because it is the |
| 262 | +// one built-in tool that reaches the network; and `lsp` when `lsp.servers` names at least one server whose command is on | ||
| 261 | // the PATH — a missing server is a warning, never a refusal to start. | 263 | // the PATH — a missing server is a warning, never a refusal to start. |
| 262 | func assembleTools(e *engine.Engine, d *detector.LoopDetector) toolSet { | 264 | func assembleTools(e *engine.Engine, d *detector.LoopDetector) toolSet { |
| 263 | set := toolSet{close: func() {}} | 265 | set := toolSet{close: func() {}} |
| @@ -287,6 +289,12 @@ func assembleTools(e *engine.Engine, d *detector.LoopDetector) toolSet { | |||
| 287 | set.tools = append(set.tools, tools.PlanTool(e.G, d)) | 289 | set.tools = append(set.tools, tools.PlanTool(e.G, d)) |
| 288 | set.names = append(set.names, "plan") | 290 | set.names = append(set.names, "plan") |
| 289 | } | 291 | } |
| 292 | + if config.Cfg.FetchContentTool { | ||
| 293 | + // The product token "mm" is what the sites' robots.txt are read for. | ||
| 294 | + fetcher := webmd.NewFetcher("mm/" + version.Short()) | ||
| 295 | + set.tools = append(set.tools, tools.FetchContent(e.G, d, fetcher)) | ||
| 296 | + set.names = append(set.names, "fetch_content") | ||
| 297 | + } | ||
| 290 | assembleLSP(e, d, &set) | 298 | assembleLSP(e, d, &set) |
| 291 | assembleMCP(e, d, &set) | 299 | assembleMCP(e, d, &set) |
| 292 | return set | 300 | return set |