vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 2, first slice: issues as files on a branch
37c01e5aa9b98bbea8567e741cd3af508ae1ba7d · Keenan Nemetz · 18d ago · parent 197ecaa2
Phase 2, first slice: issues as files on a branch Push-based issues, per the decision to keep the web read-only. The format is the contract; nothing here requires a Vole binary on a contributor's machine. issues/<ulid>/meta.json issues/<ulid>/description.md issues/<ulid>/comments/<ulid>.md on refs/heads/issues Design points worth recording: - IDs are ULIDs generated by the writer, so there is no server-assigned counter, no contention, and no ID allocation in the write path at all. - Issues live on a branch rather than a custom ref, so a plain `git clone` carries them. That is the same lesson that moved the user roster off refs/vole/users: `git clone` only mirrors refs/heads/*. - File-per-comment means two people replying to the same issue never touch the same file, so the only merge conflict is two edits to one comment. - Validation happens in pre-receive, so a malformed tree never reaches the web. Every problem is reported at once with the offending path, and a rejected push leaves the branch unmoved. The index gained issue rows, refreshed when the issues ref moves rather than by a hook, plus coverage in the full rebuild. The list page reads the index; the detail page reads git, so bodies and comments are never stale. Markdown is rendered with goldmark and sanitised with bluemonday. Issue bodies arrive by push, which makes them untrusted input on a public page; a test asserts that <script> and javascript: URLs do not survive. Verified by hand with a user who has an editor and a key: filed an issue, added a comment, saw both rendered, then pushed an issue with no title and got back "issues/01M5.../meta.json: title is required" with the branch unmoved.
internal/issues/issues.go +346
| @@ -0,0 +1,346 @@ | |||
| 1 | // Package issues implements Vole's issue format: a tree of plain files on a | ||
| 2 | // branch in the repository itself. | ||
| 3 | // | ||
| 4 | // issues/<ulid>/meta.json | ||
| 5 | // issues/<ulid>/description.md | ||
| 6 | // issues/<ulid>/comments/<ulid>.md | ||
| 7 | // | ||
| 8 | // The format is the contract, not any particular tool. Someone with an editor and | ||
| 9 | // a push can file a bug without installing Vole; `vole issue` is sugar over the | ||
| 10 | // same tree. Issues live on a branch (not a custom ref) so that a plain | ||
| 11 | // `git clone` carries them, which is what "issues travel with the repo" means in | ||
| 12 | // practice. | ||
| 13 | // | ||
| 14 | // IDs are ULIDs, generated by the writer. Nothing is assigned server-side, so | ||
| 15 | // concurrent authors never contend and there is no counter to lose. | ||
| 16 | package issues | ||
| 17 | |||
| 18 | import ( | ||
| 19 | "crypto/rand" | ||
| 20 | "encoding/json" | ||
| 21 | "fmt" | ||
| 22 | "regexp" | ||
| 23 | "sort" | ||
| 24 | "strings" | ||
| 25 | "time" | ||
| 26 | |||
| 27 | "github.com/rodent-software/vole/internal/gitx" | ||
| 28 | ) | ||
| 29 | |||
| 30 | const ( | ||
| 31 | // Ref is the branch carrying issue data. | ||
| 32 | Ref = "refs/heads/issues" | ||
| 33 | // Dir is the top-level directory inside that branch. | ||
| 34 | Dir = "issues" | ||
| 35 | |||
| 36 | metaName = "meta.json" | ||
| 37 | descName = "description.md" | ||
| 38 | commentsDir = "comments" | ||
| 39 | maxTitleLen = 200 | ||
| 40 | maxIssuesScan = 20000 | ||
| 41 | ) | ||
| 42 | |||
| 43 | var ( | ||
| 44 | ulidRE = regexp.MustCompile(`^[0-9A-HJKMNP-TV-Z]{26}$`) | ||
| 45 | pathRE = regexp.MustCompile(`^issues/([0-9A-HJKMNP-TV-Z]{26})(?:/(meta\.json|description\.md)|/comments/([0-9A-HJKMNP-TV-Z]{26})\.md)$`) | ||
| 46 | labelRE = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,31}$`) | ||
| 47 | ) | ||
| 48 | |||
| 49 | type Issue struct { | ||
| 50 | ID string | ||
| 51 | Title string | ||
| 52 | State string | ||
| 53 | Author string | ||
| 54 | Created time.Time | ||
| 55 | Updated time.Time | ||
| 56 | Labels []string | ||
| 57 | Assignees []string | ||
| 58 | Description string | ||
| 59 | Comments []Comment | ||
| 60 | Path string // relative path of the issue directory | ||
| 61 | } | ||
| 62 | |||
| 63 | type Comment struct { | ||
| 64 | ID string | ||
| 65 | Author string | ||
| 66 | Created time.Time | ||
| 67 | Body string | ||
| 68 | } | ||
| 69 | |||
| 70 | func (i *Issue) Open() bool { return i.State == "open" } | ||
| 71 | |||
| 72 | // meta is the on-disk JSON. Field names are part of the public format. | ||
| 73 | type meta struct { | ||
| 74 | Title string `json:"title"` | ||
| 75 | State string `json:"state"` | ||
| 76 | Author string `json:"author"` | ||
| 77 | Created string `json:"created"` | ||
| 78 | Updated string `json:"updated,omitempty"` | ||
| 79 | Labels []string `json:"labels,omitempty"` | ||
| 80 | Assignees []string `json:"assignees,omitempty"` | ||
| 81 | } | ||
| 82 | |||
| 83 | // Load reads every issue at ref. It validates as it goes, so a malformed tree | ||
| 84 | // produces an error naming the offending path rather than partial data. | ||
| 85 | func Load(g *gitx.Repo, ref string) ([]*Issue, error) { | ||
| 86 | commit, err := g.RevParse(ref) | ||
| 87 | if err != nil { | ||
| 88 | return nil, err | ||
| 89 | } | ||
| 90 | if commit == "" { | ||
| 91 | return nil, nil | ||
| 92 | } | ||
| 93 | blobs, err := g.LsTreeR(commit) | ||
| 94 | if err != nil { | ||
| 95 | return nil, err | ||
| 96 | } | ||
| 97 | if len(blobs) > maxIssuesScan { | ||
| 98 | return nil, fmt.Errorf("issues: refusing to scan %d objects", len(blobs)) | ||
| 99 | } | ||
| 100 | |||
| 101 | contents := map[string]string{} | ||
| 102 | for _, e := range blobs { | ||
| 103 | if !strings.HasPrefix(e.Name, Dir+"/") { | ||
| 104 | continue // unrelated files on the issues branch are ignored | ||
| 105 | } | ||
| 106 | data, err := g.BlobBySHA(e.SHA) | ||
| 107 | if err != nil { | ||
| 108 | return nil, err | ||
| 109 | } | ||
| 110 | contents[e.Name] = string(data) | ||
| 111 | } | ||
| 112 | return Parse(contents) | ||
| 113 | } | ||
| 114 | |||
| 115 | // Parse turns a path -> content map into issues, validating the tree. | ||
| 116 | func Parse(contents map[string]string) ([]*Issue, error) { | ||
| 117 | var problems []string | ||
| 118 | byID := map[string]*Issue{} | ||
| 119 | |||
| 120 | // Deterministic order for stable error output. | ||
| 121 | paths := make([]string, 0, len(contents)) | ||
| 122 | for p := range contents { | ||
| 123 | paths = append(paths, p) | ||
| 124 | } | ||
| 125 | sort.Strings(paths) | ||
| 126 | |||
| 127 | for _, path := range paths { | ||
| 128 | if !strings.HasPrefix(path, Dir+"/") { | ||
| 129 | continue | ||
| 130 | } | ||
| 131 | m := pathRE.FindStringSubmatch(path) | ||
| 132 | if m == nil { | ||
| 133 | problems = append(problems, fmt.Sprintf("%s: not part of the issue format (expected issues/<ulid>/meta.json, description.md or comments/<ulid>.md)", path)) | ||
| 134 | continue | ||
| 135 | } | ||
| 136 | id := m[1] | ||
| 137 | iss := byID[id] | ||
| 138 | if iss == nil { | ||
| 139 | iss = &Issue{ID: id, State: "open", Path: Dir + "/" + id} | ||
| 140 | byID[id] = iss | ||
| 141 | } | ||
| 142 | switch { | ||
| 143 | case m[2] == metaName: | ||
| 144 | if err := applyMeta(iss, contents[path], path, &problems); err != nil { | ||
| 145 | return nil, err | ||
| 146 | } | ||
| 147 | case m[2] == descName: | ||
| 148 | iss.Description = contents[path] | ||
| 149 | case m[3] != "": | ||
| 150 | c, prob := parseComment(m[3], contents[path], path) | ||
| 151 | if prob != "" { | ||
| 152 | problems = append(problems, prob) | ||
| 153 | continue | ||
| 154 | } | ||
| 155 | iss.Comments = append(iss.Comments, c) | ||
| 156 | } | ||
| 157 | } | ||
| 158 | |||
| 159 | out := make([]*Issue, 0, len(byID)) | ||
| 160 | for _, iss := range byID { | ||
| 161 | if iss.Title == "" { | ||
| 162 | if _, ok := contents[iss.Path+"/"+metaName]; !ok { | ||
| 163 | problems = append(problems, fmt.Sprintf("%s: missing %s (an issue needs a title)", iss.Path, metaName)) | ||
| 164 | } | ||
| 165 | } | ||
| 166 | sort.Slice(iss.Comments, func(a, b int) bool { return iss.Comments[a].ID < iss.Comments[b].ID }) | ||
| 167 | out = append(out, iss) | ||
| 168 | } | ||
| 169 | sort.Slice(out, func(a, b int) bool { return out[a].ID < out[b].ID }) | ||
| 170 | |||
| 171 | if len(problems) > 0 { | ||
| 172 | return nil, &ValidationError{Problems: problems} | ||
| 173 | } | ||
| 174 | return out, nil | ||
| 175 | } | ||
| 176 | |||
| 177 | func applyMeta(iss *Issue, raw, path string, problems *[]string) error { | ||
| 178 | var m meta | ||
| 179 | dec := json.NewDecoder(strings.NewReader(raw)) | ||
| 180 | dec.DisallowUnknownFields() | ||
| 181 | if err := dec.Decode(&m); err != nil { | ||
| 182 | *problems = append(*problems, fmt.Sprintf("%s: cannot parse: %v", path, err)) | ||
| 183 | return nil | ||
| 184 | } | ||
| 185 | iss.Title = strings.TrimSpace(m.Title) | ||
| 186 | switch { | ||
| 187 | case iss.Title == "": | ||
| 188 | *problems = append(*problems, fmt.Sprintf("%s: title is required", path)) | ||
| 189 | case len(iss.Title) > maxTitleLen: | ||
| 190 | *problems = append(*problems, fmt.Sprintf("%s: title is %d bytes, limit %d", path, len(iss.Title), maxTitleLen)) | ||
| 191 | } | ||
| 192 | if m.State == "" { | ||
| 193 | m.State = "open" | ||
| 194 | } | ||
| 195 | if m.State != "open" && m.State != "closed" { | ||
| 196 | *problems = append(*problems, fmt.Sprintf("%s: state must be \"open\" or \"closed\", got %q", path, m.State)) | ||
| 197 | } | ||
| 198 | iss.State = m.State | ||
| 199 | iss.Author = strings.TrimSpace(m.Author) | ||
| 200 | if iss.Author == "" { | ||
| 201 | *problems = append(*problems, fmt.Sprintf("%s: author is required", path)) | ||
| 202 | } | ||
| 203 | if m.Created == "" { | ||
| 204 | *problems = append(*problems, fmt.Sprintf("%s: created is required", path)) | ||
| 205 | } else if t, err := time.Parse(time.RFC3339, m.Created); err != nil { | ||
| 206 | *problems = append(*problems, fmt.Sprintf("%s: created must be RFC3339: %v", path, err)) | ||
| 207 | } else { | ||
| 208 | iss.Created = t.UTC() | ||
| 209 | } | ||
| 210 | if m.Updated != "" { | ||
| 211 | if t, err := time.Parse(time.RFC3339, m.Updated); err == nil { | ||
| 212 | iss.Updated = t.UTC() | ||
| 213 | } | ||
| 214 | } | ||
| 215 | for _, l := range m.Labels { | ||
| 216 | if !labelRE.MatchString(l) { | ||
| 217 | *problems = append(*problems, fmt.Sprintf("%s: invalid label %q (lowercase letters, digits, dot, dash, underscore)", path, l)) | ||
| 218 | continue | ||
| 219 | } | ||
| 220 | iss.Labels = append(iss.Labels, l) | ||
| 221 | } | ||
| 222 | iss.Assignees = m.Assignees | ||
| 223 | sort.Strings(iss.Labels) | ||
| 224 | return nil | ||
| 225 | } | ||
| 226 | |||
| 227 | // parseComment reads the small frontmatter block comments carry. | ||
| 228 | func parseComment(id, raw, path string) (Comment, string) { | ||
| 229 | c := Comment{ID: id} | ||
| 230 | body := raw | ||
| 231 | if strings.HasPrefix(raw, "---\n") { | ||
| 232 | rest := raw[4:] | ||
| 233 | end := strings.Index(rest, "\n---\n") | ||
| 234 | if end < 0 { | ||
| 235 | return c, fmt.Sprintf("%s: unterminated frontmatter", path) | ||
| 236 | } | ||
| 237 | head := rest[:end] | ||
| 238 | body = rest[end+5:] | ||
| 239 | for _, line := range strings.Split(head, "\n") { | ||
| 240 | line = strings.TrimSpace(line) | ||
| 241 | if line == "" { | ||
| 242 | continue | ||
| 243 | } | ||
| 244 | k, v, ok := strings.Cut(line, ":") | ||
| 245 | if !ok { | ||
| 246 | return c, fmt.Sprintf("%s: frontmatter line %q is not key: value", path, line) | ||
| 247 | } | ||
| 248 | k, v = strings.TrimSpace(k), strings.TrimSpace(v) | ||
| 249 | switch k { | ||
| 250 | case "author": | ||
| 251 | c.Author = v | ||
| 252 | case "created": | ||
| 253 | t, err := time.Parse(time.RFC3339, v) | ||
| 254 | if err != nil { | ||
| 255 | return c, fmt.Sprintf("%s: created must be RFC3339: %v", path, err) | ||
| 256 | } | ||
| 257 | c.Created = t.UTC() | ||
| 258 | default: | ||
| 259 | return c, fmt.Sprintf("%s: unknown frontmatter field %q", path, k) | ||
| 260 | } | ||
| 261 | } | ||
| 262 | } else { | ||
| 263 | return c, fmt.Sprintf("%s: comment needs frontmatter with author and created", path) | ||
| 264 | } | ||
| 265 | if c.Author == "" { | ||
| 266 | return c, fmt.Sprintf("%s: author is required", path) | ||
| 267 | } | ||
| 268 | if c.Created.IsZero() { | ||
| 269 | return c, fmt.Sprintf("%s: created is required", path) | ||
| 270 | } | ||
| 271 | c.Body = strings.TrimLeft(body, "\n") | ||
| 272 | return c, "" | ||
| 273 | } | ||
| 274 | |||
| 275 | // ValidationError carries every problem found in one tree, so a push is rejected | ||
| 276 | // with the whole list rather than one error per attempt. | ||
| 277 | type ValidationError struct { | ||
| 278 | Problems []string | ||
| 279 | } | ||
| 280 | |||
| 281 | func (e *ValidationError) Error() string { | ||
| 282 | if len(e.Problems) == 1 { | ||
| 283 | return e.Problems[0] | ||
| 284 | } | ||
| 285 | return fmt.Sprintf("%d problems: %s", len(e.Problems), strings.Join(e.Problems, "; ")) | ||
| 286 | } | ||
| 287 | |||
| 288 | // Validate checks the issues branch at commit and reports every problem. | ||
| 289 | func Validate(g *gitx.Repo, commit string) []string { | ||
| 290 | if commit == "" || commit == gitx.ZeroSHA { | ||
| 291 | return nil // deleting the branch is allowed | ||
| 292 | } | ||
| 293 | blobs, err := g.LsTreeR(commit) | ||
| 294 | if err != nil { | ||
| 295 | return []string{fmt.Sprintf("cannot read the issues tree: %v", err)} | ||
| 296 | } | ||
| 297 | contents := map[string]string{} | ||
| 298 | for _, e := range blobs { | ||
| 299 | if !strings.HasPrefix(e.Name, Dir+"/") { | ||
| 300 | continue | ||
| 301 | } | ||
| 302 | data, err := g.BlobBySHA(e.SHA) | ||
| 303 | if err != nil { | ||
| 304 | return []string{fmt.Sprintf("cannot read %s: %v", e.Name, err)} | ||
| 305 | } | ||
| 306 | contents[e.Name] = string(data) | ||
| 307 | } | ||
| 308 | if _, err := Parse(contents); err != nil { | ||
| 309 | if ve, ok := err.(*ValidationError); ok { | ||
| 310 | return ve.Problems | ||
| 311 | } | ||
| 312 | return []string{err.Error()} | ||
| 313 | } | ||
| 314 | return nil | ||
| 315 | } | ||
| 316 | |||
| 317 | // NewULID is exported for the CLI: a sortable id with a millisecond timestamp | ||
| 318 | // prefix, so issue lists order sensibly without a server counter. | ||
| 319 | func NewULID(t time.Time) (string, error) { | ||
| 320 | const alphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" | ||
| 321 | var out [26]byte | ||
| 322 | ms := uint64(t.UnixMilli()) | ||
| 323 | for i := 9; i >= 0; i-- { | ||
| 324 | out[i] = alphabet[ms&31] | ||
| 325 | ms >>= 5 | ||
| 326 | } | ||
| 327 | b := make([]byte, 16) | ||
| 328 | if _, err := rand.Read(b); err != nil { | ||
| 329 | return "", err | ||
| 330 | } | ||
| 331 | var acc uint32 | ||
| 332 | var bits uint | ||
| 333 | pos := 10 | ||
| 334 | for _, x := range b { | ||
| 335 | acc = acc<<8 | uint32(x) | ||
| 336 | bits += 8 | ||
| 337 | for bits >= 5 && pos < 26 { | ||
| 338 | bits -= 5 | ||
| 339 | out[pos] = alphabet[(acc>>bits)&31] | ||
| 340 | pos++ | ||
| 341 | } | ||
| 342 | } | ||
| 343 | return string(out[:]), nil | ||
| 344 | } | ||
| 345 | |||
| 346 | func ValidULID(s string) bool { return ulidRE.MatchString(s) } | ||