vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Check the deployment files before a user's first boot
dbae2e01bcec3167e007123a211c8cdd5f356487 · Keenan Nemetz · 3d ago · parent b8e7a567
Check the deployment files before a user's first boot The files a new instance is built from are the ones nobody runs before a user does. A stray placeholder, a boot script with a syntax error, or a compose file naming a path that is not there all fail minutes after someone paid for a machine. check-deploy.py parses every YAML under deploy/, renders the template with the placeholder list render.sh uses (so the two cannot drift), checks the rendered document has the files it promises, and runs `sh -n` on the boot script and the container entrypoint. Terraform is formatted with `terraform fmt -check`, from a version pinned by checksum for both architectures a runner can be, beside nfpm and govulncheck. The link checker also skips .terraform, which every operator will create in place, and the ignore file keeps Terraform's working directory out of Git while keeping its lock files in.
scripts/check-deploy.py +200
| @@ -0,0 +1,200 @@ | |||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Fail when a file under deploy/ would break someone's first deployment. | ||
| 3 | |||
| 4 | The deployment files are the ones nobody runs before a user does: a cloud-init | ||
| 5 | template with a stray placeholder, a boot script with a shell syntax error, or a | ||
| 6 | compose file pointing at a path that is not there all fail on somebody's first | ||
| 7 | boot, minutes after they spent money on a machine. | ||
| 8 | |||
| 9 | What is checked: | ||
| 10 | |||
| 11 | - every YAML file under deploy/ parses; | ||
| 12 | - deploy/cloud-init/vole.yaml renders with the placeholder list that | ||
| 13 | deploy/cloud-init/render.sh uses, and no placeholder is left behind; | ||
| 14 | - the rendered user data is a cloud-config document with the files it promises; | ||
| 15 | - the boot script, the container entrypoint, and the render script are valid | ||
| 16 | `sh`, and the scripts are executable. | ||
| 17 | |||
| 18 | Terraform is checked with `terraform fmt -check` in the workflow, because it needs | ||
| 19 | the tool; this file only needs python3 and PyYAML. | ||
| 20 | """ | ||
| 21 | |||
| 22 | import os | ||
| 23 | import re | ||
| 24 | import subprocess | ||
| 25 | import sys | ||
| 26 | |||
| 27 | import yaml | ||
| 28 | |||
| 29 | ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | ||
| 30 | DEPLOY = os.path.join(ROOT, "deploy") | ||
| 31 | |||
| 32 | # Values with the right shapes: a bare name, a real looking ed25519 key, an https | ||
| 33 | # URL, and a date-free instance name. They are never used against anything. | ||
| 34 | SAMPLE = { | ||
| 35 | "hostname": "forge.example.com", | ||
| 36 | "admin_account": "admin", | ||
| 37 | "admin_ssh_key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl admin@example.com", | ||
| 38 | "admin_email": "admin@example.com", | ||
| 39 | "instance_name": "Example Forge", | ||
| 40 | "tls_mode": "acme", | ||
| 41 | "vole_package_url": "https://forge.example.com/r/vole/releases/v0.2.0/artifacts/vole_0.2.0-1_amd64.deb", | ||
| 42 | } | ||
| 43 | |||
| 44 | |||
| 45 | def fail(message, where=""): | ||
| 46 | print(f"{where}: {message}" if where else message, file=sys.stderr) | ||
| 47 | return 1 | ||
| 48 | |||
| 49 | |||
| 50 | def check_yaml_files(): | ||
| 51 | bad = 0 | ||
| 52 | for root, dirs, files in os.walk(DEPLOY): | ||
| 53 | dirs[:] = [d for d in dirs if d != ".terraform"] | ||
| 54 | for name in files: | ||
| 55 | if not name.endswith((".yaml", ".yml")): | ||
| 56 | continue | ||
| 57 | path = os.path.join(root, name) | ||
| 58 | with open(path, encoding="utf-8") as f: | ||
| 59 | try: | ||
| 60 | yaml.safe_load(f) | ||
| 61 | except yaml.YAMLError as err: | ||
| 62 | bad |= fail(f"not valid YAML: {err}", os.path.relpath(path, ROOT)) | ||
| 63 | return bad | ||
| 64 | |||
| 65 | |||
| 66 | def placeholders(): | ||
| 67 | """The placeholder names, taken from the one place that lists them.""" | ||
| 68 | path = os.path.join(DEPLOY, "cloud-init", "render.sh") | ||
| 69 | with open(path, encoding="utf-8") as f: | ||
| 70 | text = f.read() | ||
| 71 | match = re.search(r"^placeholder_list='([^']*)'", text, re.MULTILINE) | ||
| 72 | if not match: | ||
| 73 | raise SystemExit(f"{path}: no placeholder_list line") | ||
| 74 | return re.findall(r"\$\{(\w+)\}", match.group(1)) | ||
| 75 | |||
| 76 | |||
| 77 | def render(): | ||
| 78 | names = placeholders() | ||
| 79 | missing = [n for n in names if n not in SAMPLE] | ||
| 80 | if missing: | ||
| 81 | raise SystemExit(f"render.sh names placeholders with no sample value: {missing}") | ||
| 82 | path = os.path.join(DEPLOY, "cloud-init", "vole.yaml") | ||
| 83 | with open(path, encoding="utf-8") as f: | ||
| 84 | text = f.read() | ||
| 85 | for name in names: | ||
| 86 | text = text.replace("${" + name + "}", SAMPLE[name]) | ||
| 87 | return text | ||
| 88 | |||
| 89 | |||
| 90 | def check_cloud_init(): | ||
| 91 | bad = 0 | ||
| 92 | rendered = render() | ||
| 93 | if not rendered.startswith("#cloud-config"): | ||
| 94 | bad |= fail("the first line must be #cloud-config", "deploy/cloud-init/vole.yaml") | ||
| 95 | |||
| 96 | left = re.findall(r"\$\{\w+\}", rendered) | ||
| 97 | if left: | ||
| 98 | bad |= fail(f"placeholders left after rendering: {sorted(set(left))}", "deploy/cloud-init/vole.yaml") | ||
| 99 | |||
| 100 | doc = yaml.safe_load(rendered) | ||
| 101 | if not isinstance(doc, dict): | ||
| 102 | return bad | fail("the document is not a mapping", "deploy/cloud-init/vole.yaml") | ||
| 103 | |||
| 104 | for key in ("hostname", "packages", "write_files", "runcmd"): | ||
| 105 | if key not in doc: | ||
| 106 | bad |= fail(f"missing {key}", "deploy/cloud-init/vole.yaml") | ||
| 107 | |||
| 108 | if SAMPLE["hostname"] != doc.get("hostname"): | ||
| 109 | bad |= fail("hostname was not substituted", "deploy/cloud-init/vole.yaml") | ||
| 110 | |||
| 111 | files = {f.get("path"): f for f in doc.get("write_files") or []} | ||
| 112 | for wanted in ("/opt/vole-deploy/vole.acme.toml", "/opt/vole-deploy/vole.proxy.toml", | ||
| 113 | "/opt/vole-deploy/bind.conf", "/opt/vole-deploy/admin.pub", | ||
| 114 | "/usr/local/sbin/vole-first-boot"): | ||
| 115 | if wanted not in files: | ||
| 116 | bad |= fail(f"no write_files entry for {wanted}", "deploy/cloud-init/vole.yaml") | ||
| 117 | |||
| 118 | # A public key is not a secret, and the boot script reads it as the vole user. | ||
| 119 | key_mode = files.get("/opt/vole-deploy/admin.pub", {}).get("permissions") | ||
| 120 | if key_mode not in ("0644", "0444", "0640", "0440", "0600"): | ||
| 121 | bad |= fail(f"admin.pub permissions {key_mode!r} is not a literal mode", "deploy/cloud-init/vole.yaml") | ||
| 122 | |||
| 123 | script = files.get("/usr/local/sbin/vole-first-boot", {}).get("content") | ||
| 124 | if not script: | ||
| 125 | return bad | fail("the boot script has no content", "deploy/cloud-init/vole.yaml") | ||
| 126 | |||
| 127 | # The configuration the script installs has to be the one that was written, | ||
| 128 | # chosen by the mode rather than hardcoded to one of them. | ||
| 129 | if 'vole.$TLS_MODE.toml" /etc/vole/vole.toml' not in script: | ||
| 130 | bad |= fail("the boot script does not install the configuration the mode names", "deploy/cloud-init/vole.yaml") | ||
| 131 | if "TLS_MODE='" not in script: | ||
| 132 | bad |= fail("the boot script does not assign TLS_MODE", "deploy/cloud-init/vole.yaml") | ||
| 133 | |||
| 134 | bad |= check_sh(script, "deploy/cloud-init/vole.yaml (boot script)") | ||
| 135 | return bad | ||
| 136 | |||
| 137 | |||
| 138 | def check_sh(text, where): | ||
| 139 | result = subprocess.run(["sh", "-n"], input=text, text=True, capture_output=True) | ||
| 140 | if result.returncode != 0: | ||
| 141 | return fail(result.stderr.strip(), where) | ||
| 142 | return 0 | ||
| 143 | |||
| 144 | |||
| 145 | def check_scripts(): | ||
| 146 | bad = 0 | ||
| 147 | for rel in ("cloud-init/render.sh", "docker/entrypoint.sh"): | ||
| 148 | path = os.path.join(DEPLOY, rel) | ||
| 149 | if not os.path.exists(path): | ||
| 150 | bad |= fail("missing", f"deploy/{rel}") | ||
| 151 | continue | ||
| 152 | if not os.access(path, os.X_OK): | ||
| 153 | bad |= fail("is not executable", f"deploy/{rel}") | ||
| 154 | with open(path, encoding="utf-8") as f: | ||
| 155 | bad |= check_sh(f.read(), f"deploy/{rel}") | ||
| 156 | return bad | ||
| 157 | |||
| 158 | |||
| 159 | def check_container(): | ||
| 160 | bad = 0 | ||
| 161 | compose_path = os.path.join(DEPLOY, "docker", "compose.yaml") | ||
| 162 | with open(compose_path, encoding="utf-8") as f: | ||
| 163 | compose = yaml.safe_load(f) | ||
| 164 | service = (compose.get("services") or {}).get("vole") or {} | ||
| 165 | build = service.get("build") or {} | ||
| 166 | if not build: | ||
| 167 | return fail("no vole service with a build", "deploy/docker/compose.yaml") | ||
| 168 | |||
| 169 | context = os.path.normpath(os.path.join(os.path.dirname(compose_path), build.get("context", "."))) | ||
| 170 | dockerfile = os.path.normpath(os.path.join(context, build.get("dockerfile", "Dockerfile"))) | ||
| 171 | if not os.path.exists(dockerfile): | ||
| 172 | bad |= fail(f"the build names {dockerfile}, which does not exist", "deploy/docker/compose.yaml") | ||
| 173 | |||
| 174 | # Every COPY in the Dockerfile has to name something in the build context, | ||
| 175 | # because a missing file fails the build minutes in. | ||
| 176 | with open(dockerfile, encoding="utf-8") as f: | ||
| 177 | for line in f: | ||
| 178 | match = re.match(r"\s*COPY\s+(?!--from)(\S+)\s+\S+", line) | ||
| 179 | if not match: | ||
| 180 | continue | ||
| 181 | source = os.path.join(context, match.group(1)) | ||
| 182 | if "*" not in match.group(1) and not os.path.exists(source): | ||
| 183 | bad |= fail(f"COPY {match.group(1)} is not in the build context", "deploy/docker/Dockerfile") | ||
| 184 | |||
| 185 | if not (service.get("volumes") or []): | ||
| 186 | bad |= fail("no volume; the instance would not survive a restart", "deploy/docker/compose.yaml") | ||
| 187 | return bad | ||
| 188 | |||
| 189 | |||
| 190 | def main(): | ||
| 191 | bad = 0 | ||
| 192 | bad |= check_yaml_files() | ||
| 193 | bad |= check_cloud_init() | ||
| 194 | bad |= check_scripts() | ||
| 195 | bad |= check_container() | ||
| 196 | return 1 if bad else 0 | ||
| 197 | |||
| 198 | |||
| 199 | if __name__ == "__main__": | ||
| 200 | sys.exit(main()) | ||