vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Add packaging, a CI workflow, and containerised local runs
3b39a84c913cabe15bfa6a60dd176ff2dd28d241 · Keenan Nemetz · 14d ago · parent 30a72490
Add packaging, a CI workflow, and containerised local runs A v* tag now builds a .deb and a .rpm for amd64 and arm64 with nfpm and attaches them, with checksums, to the release. The package ships the systemd unit, a config at /etc/vole/vole.toml, and a postinstall that creates the service account and prepares the instance that config names; `vole admin init --config` is what makes that possible, and the binary's version is stamped from the tag instead of the source tree. scripts/ci.sh becomes .vole/workflows/ci.yaml, so `vole workflow run ci` executes exactly what a push runs, and the Markdown link check moves to scripts/check-links.py. Local runs now always execute in the job image and share ci/container with the runner; --no-container is gone, because a run in a different environment proves nothing about CI. The runner mounts a per-repository namespace at /cache and sets no variable for it: a job opts in through its image's env or the new job-level env, which is pinned with the queue and delivered with the lease. Whole namespaces are evicted least-recently-written first so a module cache is never half-deleted, and contributor branches get their own namespace so an untrusted tip cannot seed the cache a protected branch builds into. Running the suite in that container found four bugs worth naming: /tmp was mounted noexec, which broke `go test` and `go run`; the read-only root is gone, because images make toolchain directories world-writable deliberately and the mount set is what protects the host; OpenSSH needs a passwd entry for the job's uid; and a lost metadata push reported as "cannot lock ref" was not classified as a race, so concurrent CLI writes gave up on the first collision. That retry now backs off exponentially with jitter.
ci/cache/cache.go +231
| @@ -0,0 +1,231 @@ | |||
| 1 | // Package cache keeps the directories a job reuses between runs: downloaded | ||
| 2 | // dependencies and compiled output that are expensive to recreate. | ||
| 3 | // | ||
| 4 | // The runner owns the directory, namespaces it, and caps it. It does not decide | ||
| 5 | // what a cache is: it mounts one directory at Mount and sets no variable that | ||
| 6 | // points at it. Which tool uses it — and therefore whether a job shares state with | ||
| 7 | // the job before it — is the repository's decision, expressed in the image's | ||
| 8 | // `env:` or in the step itself. A runner that guessed would be making a claim | ||
| 9 | // about a toolchain it does not test, and a job that wants no shared state would | ||
| 10 | // have to defend against a default it never asked for. | ||
| 11 | // | ||
| 12 | // Each job sees exactly one directory. Namespaces are whole directories so | ||
| 13 | // eviction never leaves half a module cache behind: a namespace is either there or | ||
| 14 | // it is not. | ||
| 15 | package cache | ||
| 16 | |||
| 17 | import ( | ||
| 18 | "crypto/sha256" | ||
| 19 | "encoding/hex" | ||
| 20 | "fmt" | ||
| 21 | "io/fs" | ||
| 22 | "os" | ||
| 23 | "path/filepath" | ||
| 24 | "sort" | ||
| 25 | "strconv" | ||
| 26 | "strings" | ||
| 27 | "time" | ||
| 28 | ) | ||
| 29 | |||
| 30 | // Mount is where the runner mounts a job's namespace inside the container. It is | ||
| 31 | // fixed rather than configurable because the variables below name subdirectories | ||
| 32 | // of it, and those are what the tools actually read. | ||
| 33 | const Mount = "/cache" | ||
| 34 | |||
| 35 | // DefaultBytes is the cap a runner uses when none is configured: five gigabytes | ||
| 36 | // holds a race-enabled Go cache for a project this size several times over. | ||
| 37 | const DefaultBytes int64 = 5 << 30 | ||
| 38 | |||
| 39 | // Namespace is the directory one job's cache lives in. The parts are ordered so | ||
| 40 | // that the two things worth separating are separated first: a repository never | ||
| 41 | // sees another repository's cache, and an untrusted contributor branch never sees | ||
| 42 | // the cache a protected branch builds into. | ||
| 43 | func Namespace(root, repo, trust, image string) string { | ||
| 44 | return filepath.Join(root, segment(repo), segment(trust), segment(image)) | ||
| 45 | } | ||
| 46 | |||
| 47 | // ImageKey names the environment a cache was built for. A definition is keyed by | ||
| 48 | // its digest, so changing it starts a new cache; a registry reference is keyed by | ||
| 49 | // the reference plus a short hash, so two references cannot collide after | ||
| 50 | // sanitising. | ||
| 51 | func ImageKey(imageSHA, imageRef string) string { | ||
| 52 | if imageSHA != "" { | ||
| 53 | sha := imageSHA | ||
| 54 | if len(sha) > 16 { | ||
| 55 | sha = sha[:16] | ||
| 56 | } | ||
| 57 | return "file-" + segment(sha) | ||
| 58 | } | ||
| 59 | sum := sha256.Sum256([]byte(imageRef)) | ||
| 60 | name := imageRef | ||
| 61 | if len(name) > 48 { | ||
| 62 | name = name[:48] | ||
| 63 | } | ||
| 64 | return "ref-" + segment(name) + "-" + hex.EncodeToString(sum[:4]) | ||
| 65 | } | ||
| 66 | |||
| 67 | // Trim deletes whole namespaces, least recently written first, until root holds at | ||
| 68 | // most maxBytes. It returns how many bytes it removed. A namespace that cannot be | ||
| 69 | // removed is skipped rather than leaving the cache in an unknown state. | ||
| 70 | // | ||
| 71 | // Whole namespaces rather than files: a Go module cache with one file missing is | ||
| 72 | // worse than no Go module cache, because the failure is a checksum error deep in a | ||
| 73 | // build rather than a download. | ||
| 74 | func Trim(root string, maxBytes int64) (int64, error) { | ||
| 75 | if maxBytes <= 0 { | ||
| 76 | return 0, nil | ||
| 77 | } | ||
| 78 | type namespace struct { | ||
| 79 | path string | ||
| 80 | size int64 | ||
| 81 | newest time.Time | ||
| 82 | } | ||
| 83 | var all []namespace | ||
| 84 | var total int64 | ||
| 85 | for _, repo := range dirs(root) { | ||
| 86 | for _, trust := range dirs(repo) { | ||
| 87 | for _, image := range dirs(trust) { | ||
| 88 | size, newest := measure(image) | ||
| 89 | all = append(all, namespace{image, size, newest}) | ||
| 90 | total += size | ||
| 91 | } | ||
| 92 | } | ||
| 93 | } | ||
| 94 | if total <= maxBytes { | ||
| 95 | return 0, nil | ||
| 96 | } | ||
| 97 | sort.Slice(all, func(i, j int) bool { return all[i].newest.Before(all[j].newest) }) | ||
| 98 | |||
| 99 | var removed int64 | ||
| 100 | for _, ns := range all { | ||
| 101 | if total-removed <= maxBytes { | ||
| 102 | break | ||
| 103 | } | ||
| 104 | if err := forceRemoveAll(ns.path); err != nil { | ||
| 105 | continue | ||
| 106 | } | ||
| 107 | removed += ns.size | ||
| 108 | } | ||
| 109 | return removed, nil | ||
| 110 | } | ||
| 111 | |||
| 112 | // Clear removes every namespace under root. It is for a runner that does not reuse | ||
| 113 | // the cache: the directory still has to be mounted, because an image may point at | ||
| 114 | // it, but nothing from a previous job should be there. | ||
| 115 | func Clear(root string) error { | ||
| 116 | var first error | ||
| 117 | for _, repo := range dirs(root) { | ||
| 118 | for _, trust := range dirs(repo) { | ||
| 119 | for _, image := range dirs(trust) { | ||
| 120 | if err := forceRemoveAll(image); err != nil && first == nil { | ||
| 121 | first = err | ||
| 122 | } | ||
| 123 | } | ||
| 124 | } | ||
| 125 | } | ||
| 126 | return first | ||
| 127 | } | ||
| 128 | |||
| 129 | // ParseSize reads a size such as "512M", "5GiB", or a plain byte count. Suffixes | ||
| 130 | // are binary, which is what people mean when they type 5G for a cache. | ||
| 131 | func ParseSize(s string) (int64, error) { | ||
| 132 | text := strings.ToUpper(strings.TrimSpace(s)) | ||
| 133 | if text == "" { | ||
| 134 | return 0, fmt.Errorf("a size is required") | ||
| 135 | } | ||
| 136 | mult := int64(1) | ||
| 137 | for _, suffix := range []struct { | ||
| 138 | text string | ||
| 139 | mult int64 | ||
| 140 | }{ | ||
| 141 | {"KIB", 1 << 10}, {"MIB", 1 << 20}, {"GIB", 1 << 30}, {"TIB", 1 << 40}, | ||
| 142 | {"KB", 1 << 10}, {"MB", 1 << 20}, {"GB", 1 << 30}, {"TB", 1 << 40}, | ||
| 143 | {"K", 1 << 10}, {"M", 1 << 20}, {"G", 1 << 30}, {"T", 1 << 40}, | ||
| 144 | {"B", 1}, | ||
| 145 | } { | ||
| 146 | if strings.HasSuffix(text, suffix.text) { | ||
| 147 | mult = suffix.mult | ||
| 148 | text = strings.TrimSpace(strings.TrimSuffix(text, suffix.text)) | ||
| 149 | break | ||
| 150 | } | ||
| 151 | } | ||
| 152 | n, err := strconv.ParseInt(text, 10, 64) | ||
| 153 | if err != nil || n <= 0 { | ||
| 154 | return 0, fmt.Errorf("%q is not a positive size", s) | ||
| 155 | } | ||
| 156 | if n > (1<<62)/mult { | ||
| 157 | return 0, fmt.Errorf("%q is too large", s) | ||
| 158 | } | ||
| 159 | return n * mult, nil | ||
| 160 | } | ||
| 161 | |||
| 162 | // dirs lists the subdirectories of p, ignoring anything that cannot be read. A | ||
| 163 | // cache is best-effort: a permission problem there must not fail a job. | ||
| 164 | func dirs(p string) []string { | ||
| 165 | entries, err := os.ReadDir(p) | ||
| 166 | if err != nil { | ||
| 167 | return nil | ||
| 168 | } | ||
| 169 | out := make([]string, 0, len(entries)) | ||
| 170 | for _, e := range entries { | ||
| 171 | if e.IsDir() { | ||
| 172 | out = append(out, filepath.Join(p, e.Name())) | ||
| 173 | } | ||
| 174 | } | ||
| 175 | return out | ||
| 176 | } | ||
| 177 | |||
| 178 | func measure(dir string) (size int64, newest time.Time) { | ||
| 179 | _ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { | ||
| 180 | if err != nil { | ||
| 181 | return nil | ||
| 182 | } | ||
| 183 | info, err := d.Info() | ||
| 184 | if err != nil { | ||
| 185 | return nil | ||
| 186 | } | ||
| 187 | if info.ModTime().After(newest) { | ||
| 188 | newest = info.ModTime() | ||
| 189 | } | ||
| 190 | if !d.IsDir() { | ||
| 191 | size += info.Size() | ||
| 192 | } | ||
| 193 | return nil | ||
| 194 | }) | ||
| 195 | return size, newest | ||
| 196 | } | ||
| 197 | |||
| 198 | // forceRemoveAll removes a tree that compilers have made read-only: the Go module | ||
| 199 | // cache, for one, chmods its directories to 0555 after extracting them. | ||
| 200 | func forceRemoveAll(dir string) error { | ||
| 201 | _ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { | ||
| 202 | if err == nil && d.IsDir() { | ||
| 203 | _ = os.Chmod(p, 0o700) | ||
| 204 | } | ||
| 205 | return nil | ||
| 206 | }) | ||
| 207 | return os.RemoveAll(dir) | ||
| 208 | } | ||
| 209 | |||
| 210 | // segment makes one path component out of a name that came from a repository, a | ||
| 211 | // ref, or an image reference. | ||
| 212 | func segment(s string) string { | ||
| 213 | clean := strings.Map(func(r rune) rune { | ||
| 214 | switch { | ||
| 215 | case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': | ||
| 216 | return r | ||
| 217 | case r == '.', r == '_', r == '-': | ||
| 218 | return r | ||
| 219 | default: | ||
| 220 | return '_' | ||
| 221 | } | ||
| 222 | }, s) | ||
| 223 | clean = strings.Trim(clean, ".") | ||
| 224 | if clean == "" { | ||
| 225 | clean = "unknown" | ||
| 226 | } | ||
| 227 | if len(clean) > 64 { | ||
| 228 | clean = clean[:64] | ||
| 229 | } | ||
| 230 | return clean | ||
| 231 | } | ||