Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Code

395 commit(s) on main

Compare History
update readme Keenan Nemetz · 24m ago 32290114
history blame raw 12.9 KiB
1#!/usr/bin/env python3
2"""Fail when a file under deploy/ would break someone's first deployment.
3
4The deployment files are the ones nobody runs before a user does: a cloud-init
5template with a stray placeholder, or a boot script with a shell syntax error,
6fails on somebody's first boot, minutes after they spent money on a machine.
7
8What is checked:
9
10- every YAML file under deploy/ parses;
11- each cloud-init template renders with the placeholder list that
12 deploy/cloud-init/render.sh uses for it, and no placeholder is left behind;
13- the rendered user data is a cloud-config document with the files it promises,
14 and its boot script is valid `sh`;
15- the packaged runner unit, its argument file, and the template that configures
16 them agree on the names, because three files agreeing by memory is how a
17 runner ends up enabled but idle.
18
19It needs python3 and PyYAML, and nothing else: no container runtime, no cloud
20credentials, and no network.
21"""
22
23import os
24import re
25import subprocess
26import sys
27
28import yaml
29
30ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
31DEPLOY = os.path.join(ROOT, "deploy")
32
33# Values with the right shapes: a bare name, a real looking ed25519 key, an
34# https URL, and a plain instance name. They are never used against anything.
35SAMPLE = {
36 "hostname": "forge.example.com",
37 "admin_account": "admin",
38 "admin_ssh_key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl admin@example.com",
39 "admin_email": "admin@example.com",
40 "instance_name": "Example Forge",
41 "tls_mode": "acme",
42 "vole_package_url": "https://rodent.software/vole/releases/v0.2.0/artifacts/vole_0.2.0-1_amd64.deb",
43 "runner_hostname": "vole-runner",
44 "forge_host": "forge.example.com",
45 "runner_runtime": "podman",
46 "runner_network": "bridge",
47 "runner_dir": "/var/lib/vole-runner",
48 "runner_jobs": "--jobs 2",
49 "runner_memory": "4G",
50 "runner_pids": "2048",
51 "runner_tmpfs": "1G",
52 "runner_disk_reserve": "2G",
53 "runner_labels": "--label linux --label amd64",
54}
55
56# What each template has to produce. The files are the contract between the
57# template and the script that installs them.
58TEMPLATES = {
59 "vole.yaml": {
60 "files": [
61 "/opt/vole-deploy/vole.acme.toml",
62 "/opt/vole-deploy/vole.proxy.toml",
63 "/opt/vole-deploy/bind.conf",
64 "/opt/vole-deploy/admin.pub",
65 "/usr/local/sbin/vole-first-boot",
66 ],
67 "scripts": ["/usr/local/sbin/vole-first-boot"],
68 },
69 "runner.yaml": {
70 "files": [
71 "/opt/vole-runner/runner.env",
72 "/usr/local/sbin/vole-runner-first-boot",
73 ],
74 "scripts": ["/usr/local/sbin/vole-runner-first-boot"],
75 },
76}
77
78
79def fail(message, where=""):
80 print(f"{where}: {message}" if where else message, file=sys.stderr)
81 return 1
82
83
84def check_yaml_files():
85 bad = 0
86 for root, dirs, files in os.walk(DEPLOY):
87 for name in files:
88 if not name.endswith((".yaml", ".yml")):
89 continue
90 path = os.path.join(root, name)
91 with open(path, encoding="utf-8") as f:
92 try:
93 yaml.safe_load(f)
94 except yaml.YAMLError as err:
95 bad |= fail(f"not valid YAML: {err}", os.path.relpath(path, ROOT))
96 return bad
97
98
99def placeholder_lists():
100 """The placeholders per template, taken from the one place that lists them.
101
102 render.sh names the template and then the placeholders it substitutes, so
103 that line is the contract between the script and the file.
104 """
105 path = os.path.join(DEPLOY, "cloud-init", "render.sh")
106 with open(path, encoding="utf-8") as f:
107 text = f.read()
108 lists = {}
109 for match in re.finditer(r"^\s*render (\S+\.yaml) (.+)$", text, re.MULTILINE):
110 lists[match.group(1)] = match.group(2).split()
111 if not lists:
112 raise SystemExit(f"{path}: no render lines naming a template")
113 return lists
114
115
116def render(template, names):
117 missing = [n for n in names if n not in SAMPLE]
118 if missing:
119 raise SystemExit(f"render.sh names placeholders with no sample value: {missing}")
120 path = os.path.join(DEPLOY, "cloud-init", template)
121 with open(path, encoding="utf-8") as f:
122 text = f.read()
123 for name in names:
124 text = text.replace("${" + name + "}", SAMPLE[name])
125 return text
126
127
128def check_cloud_init(template, names):
129 bad = 0
130 where = f"deploy/cloud-init/{template}"
131 rendered = render(template, names)
132
133 if not rendered.startswith("#cloud-config"):
134 bad |= fail("the first line must be #cloud-config", where)
135
136 left = re.findall(r"\$\{\w+\}", rendered)
137 if left:
138 bad |= fail(f"placeholders left after rendering: {sorted(set(left))}", where)
139
140 doc = yaml.safe_load(rendered)
141 if not isinstance(doc, dict):
142 return bad | fail("the document is not a mapping", where)
143
144 for key in ("hostname", "packages", "write_files", "runcmd"):
145 if key not in doc:
146 bad |= fail(f"missing {key}", where)
147
148 if doc.get("hostname") not in (SAMPLE["hostname"], SAMPLE["runner_hostname"]):
149 bad |= fail("hostname was not substituted", where)
150
151 files = {f.get("path"): f for f in doc.get("write_files") or []}
152 for wanted in TEMPLATES[template]["files"]:
153 if wanted not in files:
154 bad |= fail(f"no write_files entry for {wanted}", where)
155
156 for path in TEMPLATES[template]["files"]:
157 mode = files.get(path, {}).get("permissions")
158 if not isinstance(mode, str) or not re.fullmatch(r"0[0-7]{3}", mode):
159 bad |= fail(f"{path} permissions {mode!r} is not a literal mode", where)
160
161 for path in TEMPLATES[template]["scripts"]:
162 script = files.get(path, {}).get("content")
163 if not script:
164 bad |= fail(f"{path} has no content", where)
165 continue
166 bad |= check_sh(script, f"{where} ({os.path.basename(path)})")
167
168 return bad
169
170
171def check_vole_template(names):
172 """The forge template's two configurations, and which one the script installs."""
173 bad = 0
174 where = "deploy/cloud-init/vole.yaml"
175 doc = yaml.safe_load(render("vole.yaml", names))
176 files = {f.get("path"): f for f in doc.get("write_files") or []}
177
178 # A public key is not a secret, and the boot script reads it as the vole user.
179 if files.get("/opt/vole-deploy/admin.pub", {}).get("permissions") not in ("0644", "0444", "0640", "0440"):
180 bad |= fail("admin.pub must be readable by the account that registers it", where)
181
182 script = files.get("/usr/local/sbin/vole-first-boot", {}).get("content", "")
183 # The configuration the script installs has to be chosen by the mode, not
184 # hardcoded to one of them.
185 if 'vole.$TLS_MODE.toml" /etc/vole/vole.toml' not in script:
186 bad |= fail("the boot script does not install the configuration the mode names", where)
187 if "TLS_MODE='" not in script:
188 bad |= fail("the boot script does not assign TLS_MODE", where)
189 return bad
190
191
192def check_runner_template(names):
193 """The runner template, the packaged unit, and the argument file agree.
194
195 Three files describe one runner by hand: the template writes its arguments,
196 the unit reads them, and the package ships the example the operator copies.
197 A name that drifts between them leaves a machine enabled and idle. The same
198 goes for the two runtime drop-ins and the one sandbox setting that lets a
199 rootless runtime map the caller into a container: a machine that loses any of
200 them takes a job and fails it before a step runs.
201 """
202 bad = 0
203 where = "deploy/cloud-init/runner.yaml"
204 doc = yaml.safe_load(render("runner.yaml", names))
205 files = {f.get("path"): f for f in doc.get("write_files") or []}
206
207 env = files.get("/opt/vole-runner/runner.env", {}).get("content", "")
208 args = [line for line in env.splitlines() if line.startswith("VOLE_RUNNER_ARGS=")]
209 if len(args) != 1:
210 return fail("runner.env must set VOLE_RUNNER_ARGS exactly once", where)
211 if f"--remote ssh://git@{SAMPLE['forge_host']}:2222" not in args[0]:
212 bad |= fail("VOLE_RUNNER_ARGS does not point at the instance", where)
213 if SAMPLE["runner_labels"] not in args[0]:
214 bad |= fail("VOLE_RUNNER_ARGS does not carry the labels", where)
215 if SAMPLE["runner_jobs"] not in args[0]:
216 bad |= fail("VOLE_RUNNER_ARGS does not carry the job count", where)
217 # A runner built from this file must come up with finite limits. The container
218 # limits are the operator's settings, and a machine with none is one job away from
219 # being out of service for every repository it serves (#211).
220 for flag, value in (("--container-memory", "4G"), ("--container-pids", "2048"),
221 ("--container-tmpfs", "1G"), ("--disk-reserve", "2G")):
222 if f"{flag} {value}" not in args[0]:
223 bad |= fail(f"VOLE_RUNNER_ARGS does not set {flag}", where)
224
225 script = files.get("/usr/local/sbin/vole-runner-first-boot", {}).get("content", "")
226 for needed in ("install -m 0640 /opt/vole-runner/runner.env /etc/vole/runner.env",
227 "systemctl enable vole-runner",
228 "/etc/vole/runner_known_hosts"):
229 if needed not in script:
230 bad |= fail(f"the boot script never mentions {needed!r}", where)
231 if "--known-hosts /etc/vole/runner_known_hosts" not in args[0]:
232 bad |= fail("the runner checks a different known_hosts than the script pins", where)
233
234 unit = read("packaging/systemd/vole-runner.service")
235 if "ExecStart=/usr/bin/vole runner $VOLE_RUNNER_ARGS" not in unit:
236 bad |= fail("the unit does not run the runner with VOLE_RUNNER_ARGS",
237 "packaging/systemd/vole-runner.service")
238 if "EnvironmentFile=-/etc/vole/runner.env" not in unit:
239 bad |= fail("the unit does not read /etc/vole/runner.env",
240 "packaging/systemd/vole-runner.service")
241
242 example = read("packaging/runner.env")
243 if "VOLE_RUNNER_ARGS=--remote ssh://git@" not in example:
244 bad |= fail("the shipped example has no working argument line", "packaging/runner.env")
245
246 package = read("packaging/nfpm.yaml")
247 for needed in ("packaging/systemd/vole-runner.service", "packaging/runner.env"):
248 if needed not in package:
249 bad |= fail(f"{needed} is not in the package", "packaging/nfpm.yaml")
250
251 # What makes a fresh Debian or Ubuntu machine able to create a container at
252 # all: no unit sandbox that defeats setuid newuidmap, a registry to resolve a
253 # short image name against, and a cgroup manager that needs no user session.
254 # Missing any of them reads on the instance as a repository whose build
255 # failed, so each is checked by name and by the setting it exists for.
256 if "NoNewPrivileges=false" not in unit:
257 bad |= fail("the unit stops rootless runtimes from mapping the caller into a container",
258 "packaging/systemd/vole-runner.service")
259 dropins = (
260 ("packaging/containers/registries.conf",
261 "/etc/containers/registries.conf.d/50-vole-runner.conf",
262 "unqualified-search-registries"),
263 ("packaging/containers/containers.conf",
264 "/etc/containers/containers.conf.d/50-vole-runner.conf",
265 "cgroup_manager"),
266 )
267 for src, dst, setting in dropins:
268 if not os.path.exists(os.path.join(ROOT, src)):
269 bad |= fail("missing", src)
270 continue
271 if setting not in read(src):
272 bad |= fail(f"does not set {setting}", src)
273 if f"src: {src}" not in package or f"dst: {dst}" not in package:
274 bad |= fail(f"{src} is not installed as {dst}", "packaging/nfpm.yaml")
275 return bad
276
277
278def read(rel):
279 with open(os.path.join(ROOT, rel), encoding="utf-8") as f:
280 return f.read()
281
282
283def check_sh(text, where):
284 result = subprocess.run(["sh", "-n"], input=text, text=True, capture_output=True)
285 if result.returncode != 0:
286 return fail(result.stderr.strip(), where)
287 return 0
288
289
290def check_scripts():
291 bad = 0
292 for rel in ("cloud-init/render.sh",):
293 path = os.path.join(DEPLOY, rel)
294 if not os.path.exists(path):
295 bad |= fail("missing", f"deploy/{rel}")
296 continue
297 if not os.access(path, os.X_OK):
298 bad |= fail("is not executable", f"deploy/{rel}")
299 with open(path, encoding="utf-8") as f:
300 bad |= check_sh(f.read(), f"deploy/{rel}")
301 return bad
302
303
304def main():
305 bad = 0
306 bad |= check_yaml_files()
307
308 lists = placeholder_lists()
309 for template in TEMPLATES:
310 if template not in lists:
311 bad |= fail(f"render.sh does not render it", f"deploy/cloud-init/{template}")
312 continue
313 bad |= check_cloud_init(template, lists[template])
314
315 if "vole.yaml" in lists:
316 bad |= check_vole_template(lists["vole.yaml"])
317 if "runner.yaml" in lists:
318 bad |= check_runner_template(lists["runner.yaml"])
319
320 bad |= check_scripts()
321 return 1 if bad else 0
322
323
324if __name__ == "__main__":
325 sys.exit(main())
326