vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Document runners, and check the three files that describe one agree
e21f6c5e26cc49d5a08d337cb1c7b70d373a8cce · Keenan Nemetz · 3d ago · parent d0d4239f
Document runners, and check the three files that describe one agree A runner is a second decision after the forge, not part of its first boot: it executes repository code, it can read every secret a job declares, and the allowlist that lets it take work is per repository. The guide says what the deployment creates, how to set one up on a machine you already have, the one step only an admin can do, what to know before giving it a repository, and how to upgrade it. render.sh now renders either first-boot file, taking its placeholder list from the render line it uses, so the checker reads the same line instead of keeping a second copy. check-deploy.py renders both templates, requires each one's promised files with literal modes, and holds the runner's three descriptions together: the template writes the arguments, the unit reads them, and the package ships the example an operator copies. A name that drifts leaves a machine enabled and idle, which looks exactly like a runner that is working.
scripts/check-deploy.py +157−39
| @@ -9,11 +9,14 @@ boot, minutes after they spent money on a machine. | |||
| 9 | 9 | What is checked: | |
| 10 | 10 | ||
| 11 | 11 | - every YAML file under deploy/ parses; | |
| 12 | - deploy/cloud-init/vole.yaml renders with the placeholder list that | ||
| 13 | deploy/cloud-init/render.sh uses, and no placeholder is left behind; | ||
| 14 | - the rendered user data is a cloud-config document with the files it promises; | ||
| 15 | - the boot script, the container entrypoint, and the render script are valid | ||
| 16 | `sh`, and the scripts are executable. | ||
| 12 | - each cloud-init template renders with the placeholder list that | ||
| 13 | deploy/cloud-init/render.sh uses for it, and no placeholder is left behind; | ||
| 14 | - the rendered user data is a cloud-config document with the files it promises, | ||
| 15 | and its boot script is valid `sh`; | ||
| 16 | - deploy/docker/compose.yaml builds a Dockerfile whose COPY sources exist; | ||
| 17 | - the packaged runner unit, its argument file, and the template that configures | ||
| 18 | them agree on the names, because three files agreeing by memory is how a | ||
| 19 | runner ends up enabled but idle. | ||
| 17 | 20 | ||
| 18 | 21 | Terraform is checked with `terraform fmt -check` in the workflow, because it needs | |
| 19 | 22 | the tool; this file only needs python3 and PyYAML. | |
| @@ -39,6 +42,34 @@ SAMPLE = { | |||
| 39 | 42 | "instance_name": "Example Forge", | |
| 40 | 43 | "tls_mode": "acme", | |
| 41 | 44 | "vole_package_url": "https://forge.example.com/r/vole/releases/v0.2.0/artifacts/vole_0.2.0-1_amd64.deb", | |
| 45 | "runner_hostname": "vole-runner", | ||
| 46 | "forge_host": "forge.example.com", | ||
| 47 | "runner_runtime": "podman", | ||
| 48 | "runner_network": "bridge", | ||
| 49 | "runner_dir": "/var/lib/vole-runner", | ||
| 50 | "runner_labels": "--label linux --label amd64", | ||
| 51 | } | ||
| 52 | |||
| 53 | # What each template has to produce. The files are the contract between the | ||
| 54 | # template and the script that installs them. | ||
| 55 | TEMPLATES = { | ||
| 56 | "vole.yaml": { | ||
| 57 | "files": [ | ||
| 58 | "/opt/vole-deploy/vole.acme.toml", | ||
| 59 | "/opt/vole-deploy/vole.proxy.toml", | ||
| 60 | "/opt/vole-deploy/bind.conf", | ||
| 61 | "/opt/vole-deploy/admin.pub", | ||
| 62 | "/usr/local/sbin/vole-first-boot", | ||
| 63 | ], | ||
| 64 | "scripts": ["/usr/local/sbin/vole-first-boot"], | ||
| 65 | }, | ||
| 66 | "runner.yaml": { | ||
| 67 | "files": [ | ||
| 68 | "/opt/vole-runner/runner.env", | ||
| 69 | "/usr/local/sbin/vole-runner-first-boot", | ||
| 70 | ], | ||
| 71 | "scripts": ["/usr/local/sbin/vole-runner-first-boot"], | ||
| 72 | }, | ||
| 42 | 73 | } | |
| 43 | 74 | ||
| 44 | 75 | ||
| @@ -63,23 +94,28 @@ def check_yaml_files(): | |||
| 63 | 94 | return bad | |
| 64 | 95 | ||
| 65 | 96 | ||
| 66 | def placeholders(): | ||
| 67 | """The placeholder names, taken from the one place that lists them.""" | ||
| 97 | def placeholder_lists(): | ||
| 98 | """The placeholders per template, taken from the one place that lists them. | ||
| 99 | |||
| 100 | render.sh names the template and then the placeholders it substitutes, so | ||
| 101 | that line is the contract between the script and the file. | ||
| 102 | """ | ||
| 68 | 103 | path = os.path.join(DEPLOY, "cloud-init", "render.sh") | |
| 69 | 104 | with open(path, encoding="utf-8") as f: | |
| 70 | 105 | text = f.read() | |
| 71 | match = re.search(r"^placeholder_list='([^']*)'", text, re.MULTILINE) | ||
| 72 | if not match: | ||
| 73 | raise SystemExit(f"{path}: no placeholder_list line") | ||
| 74 | return re.findall(r"\$\{(\w+)\}", match.group(1)) | ||
| 106 | lists = {} | ||
| 107 | for match in re.finditer(r"^\s*render (\S+\.yaml) (.+)$", text, re.MULTILINE): | ||
| 108 | lists[match.group(1)] = match.group(2).split() | ||
| 109 | if not lists: | ||
| 110 | raise SystemExit(f"{path}: no render lines naming a template") | ||
| 111 | return lists | ||
| 75 | 112 | ||
| 76 | 113 | ||
| 77 | def render(): | ||
| 78 | names = placeholders() | ||
| 114 | def render(template, names): | ||
| 79 | 115 | missing = [n for n in names if n not in SAMPLE] | |
| 80 | 116 | if missing: | |
| 81 | 117 | raise SystemExit(f"render.sh names placeholders with no sample value: {missing}") | |
| 82 | path = os.path.join(DEPLOY, "cloud-init", "vole.yaml") | ||
| 118 | path = os.path.join(DEPLOY, "cloud-init", template) | ||
| 83 | 119 | with open(path, encoding="utf-8") as f: | |
| 84 | 120 | text = f.read() | |
| 85 | 121 | for name in names: | |
| @@ -87,54 +123,124 @@ def render(): | |||
| 87 | 123 | return text | |
| 88 | 124 | ||
| 89 | 125 | ||
| 90 | def check_cloud_init(): | ||
| 126 | def check_cloud_init(template, names): | ||
| 91 | 127 | bad = 0 | |
| 92 | rendered = render() | ||
| 128 | where = f"deploy/cloud-init/{template}" | ||
| 129 | rendered = render(template, names) | ||
| 130 | |||
| 93 | 131 | if not rendered.startswith("#cloud-config"): | |
| 94 | bad |= fail("the first line must be #cloud-config", "deploy/cloud-init/vole.yaml") | ||
| 132 | bad |= fail("the first line must be #cloud-config", where) | ||
| 95 | 133 | ||
| 96 | 134 | left = re.findall(r"\$\{\w+\}", rendered) | |
| 97 | 135 | if left: | |
| 98 | bad |= fail(f"placeholders left after rendering: {sorted(set(left))}", "deploy/cloud-init/vole.yaml") | ||
| 136 | bad |= fail(f"placeholders left after rendering: {sorted(set(left))}", where) | ||
| 99 | 137 | ||
| 100 | 138 | doc = yaml.safe_load(rendered) | |
| 101 | 139 | if not isinstance(doc, dict): | |
| 102 | return bad | fail("the document is not a mapping", "deploy/cloud-init/vole.yaml") | ||
| 140 | return bad | fail("the document is not a mapping", where) | ||
| 103 | 141 | ||
| 104 | 142 | for key in ("hostname", "packages", "write_files", "runcmd"): | |
| 105 | 143 | if key not in doc: | |
| 106 | bad |= fail(f"missing {key}", "deploy/cloud-init/vole.yaml") | ||
| 144 | bad |= fail(f"missing {key}", where) | ||
| 107 | 145 | ||
| 108 | if SAMPLE["hostname"] != doc.get("hostname"): | ||
| 109 | bad |= fail("hostname was not substituted", "deploy/cloud-init/vole.yaml") | ||
| 146 | if doc.get("hostname") not in (SAMPLE["hostname"], SAMPLE["runner_hostname"]): | ||
| 147 | bad |= fail("hostname was not substituted", where) | ||
| 110 | 148 | ||
| 111 | 149 | files = {f.get("path"): f for f in doc.get("write_files") or []} | |
| 112 | for wanted in ("/opt/vole-deploy/vole.acme.toml", "/opt/vole-deploy/vole.proxy.toml", | ||
| 113 | "/opt/vole-deploy/bind.conf", "/opt/vole-deploy/admin.pub", | ||
| 114 | "/usr/local/sbin/vole-first-boot"): | ||
| 150 | for wanted in TEMPLATES[template]["files"]: | ||
| 115 | 151 | if wanted not in files: | |
| 116 | bad |= fail(f"no write_files entry for {wanted}", "deploy/cloud-init/vole.yaml") | ||
| 152 | bad |= fail(f"no write_files entry for {wanted}", where) | ||
| 117 | 153 | ||
| 118 | # A public key is not a secret, and the boot script reads it as the vole user. | ||
| 119 | key_mode = files.get("/opt/vole-deploy/admin.pub", {}).get("permissions") | ||
| 120 | if key_mode not in ("0644", "0444", "0640", "0440", "0600"): | ||
| 121 | bad |= fail(f"admin.pub permissions {key_mode!r} is not a literal mode", "deploy/cloud-init/vole.yaml") | ||
| 154 | for path in TEMPLATES[template]["files"]: | ||
| 155 | mode = files.get(path, {}).get("permissions") | ||
| 156 | if not isinstance(mode, str) or not re.fullmatch(r"0[0-7]{3}", mode): | ||
| 157 | bad |= fail(f"{path} permissions {mode!r} is not a literal mode", where) | ||
| 158 | |||
| 159 | for path in TEMPLATES[template]["scripts"]: | ||
| 160 | script = files.get(path, {}).get("content") | ||
| 161 | if not script: | ||
| 162 | bad |= fail(f"{path} has no content", where) | ||
| 163 | continue | ||
| 164 | bad |= check_sh(script, f"{where} ({os.path.basename(path)})") | ||
| 165 | |||
| 166 | return bad | ||
| 167 | |||
| 168 | |||
| 169 | def check_vole_template(names): | ||
| 170 | """The forge template's two configurations, and which one the script installs.""" | ||
| 171 | bad = 0 | ||
| 172 | where = "deploy/cloud-init/vole.yaml" | ||
| 173 | doc = yaml.safe_load(render("vole.yaml", names)) | ||
| 174 | files = {f.get("path"): f for f in doc.get("write_files") or []} | ||
| 122 | 175 | ||
| 123 | script = files.get("/usr/local/sbin/vole-first-boot", {}).get("content") | ||
| 124 | if not script: | ||
| 125 | return bad | fail("the boot script has no content", "deploy/cloud-init/vole.yaml") | ||
| 176 | # A public key is not a secret, and the boot script reads it as the vole user. | ||
| 177 | if files.get("/opt/vole-deploy/admin.pub", {}).get("permissions") not in ("0644", "0444", "0640", "0440"): | ||
| 178 | bad |= fail("admin.pub must be readable by the account that registers it", where) | ||
| 126 | 179 | ||
| 127 | # The configuration the script installs has to be the one that was written, | ||
| 128 | # chosen by the mode rather than hardcoded to one of them. | ||
| 180 | script = files.get("/usr/local/sbin/vole-first-boot", {}).get("content", "") | ||
| 181 | # The configuration the script installs has to be chosen by the mode, not | ||
| 182 | # hardcoded to one of them. | ||
| 129 | 183 | if 'vole.$TLS_MODE.toml" /etc/vole/vole.toml' not in script: | |
| 130 | bad |= fail("the boot script does not install the configuration the mode names", "deploy/cloud-init/vole.yaml") | ||
| 184 | bad |= fail("the boot script does not install the configuration the mode names", where) | ||
| 131 | 185 | if "TLS_MODE='" not in script: | |
| 132 | bad |= fail("the boot script does not assign TLS_MODE", "deploy/cloud-init/vole.yaml") | ||
| 186 | bad |= fail("the boot script does not assign TLS_MODE", where) | ||
| 187 | return bad | ||
| 188 | |||
| 189 | |||
| 190 | def check_runner_template(names): | ||
| 191 | """The runner template, the packaged unit, and the argument file agree. | ||
| 192 | |||
| 193 | Three files describe one runner by hand: the template writes its arguments, | ||
| 194 | the unit reads them, and the package ships the example the operator copies. | ||
| 195 | A name that drifts between them leaves a machine enabled and idle. | ||
| 196 | """ | ||
| 197 | bad = 0 | ||
| 198 | where = "deploy/cloud-init/runner.yaml" | ||
| 199 | doc = yaml.safe_load(render("runner.yaml", names)) | ||
| 200 | files = {f.get("path"): f for f in doc.get("write_files") or []} | ||
| 133 | 201 | ||
| 134 | bad |= check_sh(script, "deploy/cloud-init/vole.yaml (boot script)") | ||
| 202 | env = files.get("/opt/vole-runner/runner.env", {}).get("content", "") | ||
| 203 | args = [line for line in env.splitlines() if line.startswith("VOLE_RUNNER_ARGS=")] | ||
| 204 | if len(args) != 1: | ||
| 205 | return fail("runner.env must set VOLE_RUNNER_ARGS exactly once", where) | ||
| 206 | if f"--remote ssh://git@{SAMPLE['forge_host']}:2222" not in args[0]: | ||
| 207 | bad |= fail("VOLE_RUNNER_ARGS does not point at the instance", where) | ||
| 208 | if SAMPLE["runner_labels"] not in args[0]: | ||
| 209 | bad |= fail("VOLE_RUNNER_ARGS does not carry the labels", where) | ||
| 210 | |||
| 211 | script = files.get("/usr/local/sbin/vole-runner-first-boot", {}).get("content", "") | ||
| 212 | for needed in ("install -m 0640 /opt/vole-runner/runner.env /etc/vole/runner.env", | ||
| 213 | "systemctl enable vole-runner", | ||
| 214 | "/etc/vole/runner_known_hosts"): | ||
| 215 | if needed not in script: | ||
| 216 | bad |= fail(f"the boot script never mentions {needed!r}", where) | ||
| 217 | if "--known-hosts /etc/vole/runner_known_hosts" not in args[0]: | ||
| 218 | bad |= fail("the runner checks a different known_hosts than the script pins", where) | ||
| 219 | |||
| 220 | unit = read("packaging/systemd/vole-runner.service") | ||
| 221 | if "ExecStart=/usr/bin/vole runner $VOLE_RUNNER_ARGS" not in unit: | ||
| 222 | bad |= fail("the unit does not run the runner with VOLE_RUNNER_ARGS", | ||
| 223 | "packaging/systemd/vole-runner.service") | ||
| 224 | if "EnvironmentFile=-/etc/vole/runner.env" not in unit: | ||
| 225 | bad |= fail("the unit does not read /etc/vole/runner.env", | ||
| 226 | "packaging/systemd/vole-runner.service") | ||
| 227 | |||
| 228 | example = read("packaging/runner.env") | ||
| 229 | if "VOLE_RUNNER_ARGS=--remote ssh://git@" not in example: | ||
| 230 | bad |= fail("the shipped example has no working argument line", "packaging/runner.env") | ||
| 231 | |||
| 232 | package = read("packaging/nfpm.yaml") | ||
| 233 | for needed in ("packaging/systemd/vole-runner.service", "packaging/runner.env"): | ||
| 234 | if needed not in package: | ||
| 235 | bad |= fail(f"{needed} is not in the package", "packaging/nfpm.yaml") | ||
| 135 | 236 | return bad | |
| 136 | 237 | ||
| 137 | 238 | ||
| 239 | def read(rel): | ||
| 240 | with open(os.path.join(ROOT, rel), encoding="utf-8") as f: | ||
| 241 | return f.read() | ||
| 242 | |||
| 243 | |||
| 138 | 244 | def check_sh(text, where): | |
| 139 | 245 | result = subprocess.run(["sh", "-n"], input=text, text=True, capture_output=True) | |
| 140 | 246 | if result.returncode != 0: | |
| @@ -190,7 +296,19 @@ def check_container(): | |||
| 190 | 296 | def main(): | |
| 191 | 297 | bad = 0 | |
| 192 | 298 | bad |= check_yaml_files() | |
| 193 | bad |= check_cloud_init() | ||
| 299 | |||
| 300 | lists = placeholder_lists() | ||
| 301 | for template in TEMPLATES: | ||
| 302 | if template not in lists: | ||
| 303 | bad |= fail(f"render.sh does not render it", f"deploy/cloud-init/{template}") | ||
| 304 | continue | ||
| 305 | bad |= check_cloud_init(template, lists[template]) | ||
| 306 | |||
| 307 | if "vole.yaml" in lists: | ||
| 308 | bad |= check_vole_template(lists["vole.yaml"]) | ||
| 309 | if "runner.yaml" in lists: | ||
| 310 | bad |= check_runner_template(lists["runner.yaml"]) | ||
| 311 | |||
| 194 | 312 | bad |= check_scripts() | |
| 195 | 313 | bad |= check_container() | |
| 196 | 314 | return 1 if bad else 0 | |