1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

"""A scale-to-zero ClickHouse HTTP endpoint for low-volume atop data.

Deploy with:

    modal deploy modal_clickhouse.py

The endpoint requires a Modal Proxy Token.  It starts ClickHouse when a request
arrives, retains data in the named Modal Volume, and can scale to zero after it
has been idle.  Keep max_containers at one: a Modal Volume is not a suitable
shared filesystem for multiple ClickHouse writers.
"""

import socket
import subprocess
import time

import modal

APP_NAME = "atop-clickhouse"
DATA_VOLUME_NAME = "atop-clickhouse-data"
IDLE_SECONDS = 300

app = modal.App(APP_NAME)
data_volume = modal.Volume.from_name(DATA_VOLUME_NAME, create_if_missing=True)
image = modal.Image.from_registry(
    "clickhouse/clickhouse-server:25.8", add_python="3.12"
)


@app.function(
    image=image,
    volumes={"/var/lib/clickhouse": data_volume},
    min_containers=0,
    max_containers=1,
    scaledown_window=IDLE_SECONDS,
    timeout=10 * 60,
)
@modal.web_server(8123, startup_timeout=240, requires_proxy_auth=True)
def clickhouse() -> None:
    """Start ClickHouse's native HTTP API; Modal proxies requests to port 8123."""
    process = subprocess.Popen(
        ["clickhouse-server", "--config-file=/etc/clickhouse-server/config.xml"]
    )
    deadline = time.monotonic() + 210
    while time.monotonic() < deadline:
        if process.poll() is not None:
            raise RuntimeError(f"ClickHouse exited during startup ({process.returncode})")
        with socket.socket() as sock:
            sock.settimeout(0.2)
            if sock.connect_ex(("127.0.0.1", 8123)) == 0:
                return
        time.sleep(0.2)
    process.terminate()
    raise RuntimeError("ClickHouse did not open port 8123 within 210 seconds")