Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.

vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Issues and requests are frontmatter, not JSON

307cbc2aa624ca9b76b79849e8ce8926414be435 · Keenan Nemetz · 17d ago · parent 427c4761

Issues and requests are frontmatter, not JSON

issues/<ulid>/issue.md and requests/<ulid>/request.md replace meta.json plus
description.md, and approvals/<account>.md replaces approvals/<account>.json.
Every file on refs/heads/vole is now frontmatter markdown read by one package, so
there is one reader and one writer for issues, requests, comments and approvals
instead of a JSON path and a hand-rolled key/value path.

The diff is the interface. A JSON array rewrites its neighbour to add a comma, so
one label cost two changed lines and removing one cost three; a block sequence
costs one. docs/spikes/issue-frontmatter measured the same six edits at 25 changed
lines as meta.json and 10 as frontmatter, and one file per entity means one path
to diff, blame and log.

Three rules make YAML safe to hand-edit rather than something to police:

- the writer always double-quotes free text, so no indicator, reserved word or
  " # " can change what a title means
- a label is quoted only when the parser would not read it back bare ("null"),
  which leaves ordinary labels as they were
- a reader refuses an unquoted title that the parser read differently: "title:
  fix #12 crash" would otherwise be stored as "fix" and nobody would find out

Control characters are refused on both sides, so a tree the server accepts is a
tree it can always rewrite.

`updated` is gone from issues, requests and the index's issues table. It cost a
changed line on every edit — 11 of the 21 frontmatter lines in the spike's
sequence — and the commit that changed the file already carries the time.

Verified: TestRenderParseRoundTrip puts 13 titles that break naive YAML (hash,
colon, leading dash, null, quotes, backslash, unicode, 190 bytes) through the
writer and the reader; the server tests push a rendered issue over SSH and render
it, and the approval-ownership tests withdraw approvals/<account>.md. Then
scripts/ci.sh green (gofmt, vet, build, markdown links, go test -race ./...).

docs/spikes/issue-frontmatter/main.go +530

@@ -0,0 +1,530 @@
1 // Spike: is frontmatter a better on-disk format for issues on refs/heads/vole
2 // than the current meta.json + description.md pair?
3 //
4 // Two modes:
5 //
6 // go run . -mode probe # which YAML scalar forms need quoting, for titles and labels
7 // go run . -mode diffs # real git diffs of both formats for the same edit sequence
8 //
9 // Throwaway. Nothing here ships.
10 package main
11
12 import (
13 "encoding/json"
14 "flag"
15 "fmt"
16 "os"
17 "os/exec"
18 "path/filepath"
19 "sort"
20 "strings"
21
22 "gopkg.in/yaml.v3"
23 )
24
25 const created = "2026-10-23T12:00:00Z"
26
27 func main() {
28 mode := flag.String("mode", "probe", "probe = YAML edge cases; diffs = real git diffs of both formats")
29 noUpdated := flag.Bool("no-updated", false, "diffs: drop the updated field from both formats")
30 flag.Parse()
31 switch *mode {
32 case "probe":
33 probeTitles()
34 probeLabels()
35 case "diffs":
36 if err := diffs(*noUpdated); err != nil {
37 fmt.Fprintln(os.Stderr, "diffs:", err)
38 os.Exit(1)
39 }
40 default:
41 fmt.Fprintln(os.Stderr, "unknown -mode", *mode)
42 os.Exit(2)
43 }
44 }
45
46 // ---------------------------------------------------------------- the format
47
48 // fm is the frontmatter block. Field names are the public format.
49 type fm struct {
50 Title string `yaml:"title"`
51 State string `yaml:"state"`
52 Author string `yaml:"author"`
53 Created string `yaml:"created"`
54 Updated string `yaml:"updated,omitempty"`
55 Labels []string `yaml:"labels,omitempty"`
56 Assignees []string `yaml:"assignees,omitempty"`
57 }
58
59 // quote renders a YAML double-quoted scalar. Control characters are refused
60 // rather than escaped, so a title can never become two lines.
61 func quote(s string) (string, error) {
62 var b strings.Builder
63 b.WriteByte('"')
64 for _, r := range s {
65 switch {
66 case r == '"':
67 b.WriteString(`\"`)
68 case r == '\\':
69 b.WriteString(`\\`)
70 case r < 0x20 || r == 0x7f:
71 return "", fmt.Errorf("control character %q is not allowed in a title", r)
72 default:
73 b.WriteRune(r)
74 }
75 }
76 b.WriteByte('"')
77 return b.String(), nil
78 }
79
80 // plainSafe reports whether s survives an unquoted (plain) YAML scalar
81 // unchanged. It asks the parser instead of carrying a list of YAML's reserved
82 // words, so the answer cannot drift from the reader.
83 func plainSafe(s string) bool {
84 if strings.TrimSpace(s) != s || s == "" {
85 return false
86 }
87 var probe struct {
88 K string `yaml:"k"`
89 }
90 if err := yaml.Unmarshal([]byte("k: "+s+"\n"), &probe); err != nil {
91 return false
92 }
93 return probe.K == s
94 }
95
96 // renderIssue writes one issue as a single frontmatter markdown file: metadata
97 // first, body below the closing fence.
98 func renderIssue(i issue) (string, error) {
99 title, err := quote(i.Title)
100 if err != nil {
101 return "", err
102 }
103 var b strings.Builder
104 b.WriteString("---\n")
105 fmt.Fprintf(&b, "title: %s\n", title)
106 fmt.Fprintf(&b, "state: %s\n", i.State)
107 fmt.Fprintf(&b, "author: %s\n", i.Author)
108 fmt.Fprintf(&b, "created: %s\n", i.Created)
109 if i.Updated != "" {
110 fmt.Fprintf(&b, "updated: %s\n", i.Updated)
111 }
112 if len(i.Labels) > 0 {
113 b.WriteString("labels:\n")
114 labels := append([]string(nil), i.Labels...)
115 sort.Strings(labels)
116 for _, l := range labels {
117 if plainSafe(l) {
118 fmt.Fprintf(&b, " - %s\n", l)
119 continue
120 }
121 q, err := quote(l)
122 if err != nil {
123 return "", err
124 }
125 fmt.Fprintf(&b, " - %s\n", q)
126 }
127 }
128 b.WriteString("---\n")
129 b.WriteString(i.Body)
130 if !strings.HasSuffix(i.Body, "\n") {
131 b.WriteString("\n")
132 }
133 return b.String(), nil
134 }
135
136 // parseIssue is the reader: find the fence, decode the head, keep the rest as
137 // the body.
138 func parseIssue(raw string) (fm, string, error) {
139 var out fm
140 if !strings.HasPrefix(raw, "---\n") {
141 return out, "", fmt.Errorf("no frontmatter fence")
142 }
143 rest := raw[4:]
144 end := strings.Index(rest, "\n---\n")
145 if end < 0 {
146 return out, "", fmt.Errorf("unterminated frontmatter")
147 }
148 head, body := rest[:end], rest[end+5:]
149 dec := yaml.NewDecoder(strings.NewReader(head))
150 dec.KnownFields(true)
151 if err := dec.Decode(&out); err != nil {
152 return out, "", err
153 }
154 return out, body, nil
155 }
156
157 // ---------------------------------------------------------------- probe mode
158
159 func probeTitles() {
160 tests := []struct{ name, title string }{
161 {"plain", "crash on startup"},
162 {"colon-space", "panic: nil map"},
163 {"colon-nospace", "panic:nil map"},
164 {"hash-space", "fix #12 crash"},
165 {"hash-nospace", "fix#12 crash"},
166 {"leading-dash", "- not a list"},
167 {"leading-star", "*anchor"},
168 {"leading-amp", "&anchor"},
169 {"leading-bang", "!tag"},
170 {"leading-pct", "%dir"},
171 {"leading-at", "@mention"},
172 {"leading-tick", "`tick`"},
173 {"leading-brace", "{a: b}"},
174 {"leading-bracket", "[a, b]"},
175 {"leading-gt", "> folded"},
176 {"leading-pipe", "| literal"},
177 {"leading-question", "? maybe"},
178 {"reserved-true", "true"},
179 {"reserved-yes", "yes"},
180 {"reserved-on", "on"},
181 {"reserved-null", "null"},
182 {"reserved-tilde", "~"},
183 {"number-int", "123"},
184 {"number-float", "1.5"},
185 {"number-hex", "0x10"},
186 {"doc-start", "--- separator"},
187 {"doc-end", "... and so on"},
188 {"double-quote", `say "hi"`},
189 {"single-quote", "it's broken"},
190 {"backslash", `path C:\temp`},
191 {"html", "crash on a<b> & c"},
192 {"unicode", "café ☕ breaks"},
193 {"long-200", strings.Repeat("very long title ", 14)[:200]},
194 {"trailing-space", "trailing space "},
195 {"leading-space", " leading space"},
196 {"empty", ""},
197 }
198
199 fmt.Println("== titles: library emission vs always-quoted vs a hand-edit that forgets the quotes ==")
200 fmt.Printf("%-18s %5s %4s %6s %-28s %s\n", "case", "lines", "rt", "quoted", "hand-edit without quotes", "validator")
201
202 var wrap, altered, silently, rejected, quotedBad int
203 for _, t := range tests {
204 // (a) what yaml.v3 emits for the struct.
205 lib, err := yaml.Marshal(struct {
206 Title string `yaml:"title"`
207 State string `yaml:"state"`
208 }{t.title, "open"})
209 if err != nil {
210 fmt.Printf("%-18s marshal error: %v\n", t.name, err)
211 continue
212 }
213 head := strings.SplitN(string(lib), "\nstate:", 2)[0]
214 lines := strings.Count(head, "\n") + 1
215 var back struct {
216 Title string `yaml:"title"`
217 }
218 if err := yaml.Unmarshal(lib, &back); err != nil || back.Title != t.title {
219 fmt.Printf("%-18s library round-trip FAILED (%v)\n", t.name, err)
220 }
221
222 // (b) our writer: always double-quoted.
223 q, qerr := quote(t.title)
224 quotedOK := false
225 if qerr == nil {
226 var pb struct {
227 Title string `yaml:"title"`
228 }
229 if err := yaml.Unmarshal([]byte("title: "+q+"\n"), &pb); err == nil && pb.Title == t.title {
230 quotedOK = true
231 }
232 }
233 if !quotedOK {
234 quotedBad++
235 }
236
237 // (c) a human editing by hand without quotes.
238 var raw struct {
239 Title string `yaml:"title"`
240 }
241 rerr := yaml.Unmarshal([]byte("title: "+t.title+"\n"), &raw)
242 handEdit := "ok"
243 switch {
244 case rerr != nil:
245 handEdit = "parse error (loud)"
246 case raw.Title != t.title && raw.Title == "":
247 handEdit = "silently empty"
248 case raw.Title != t.title:
249 handEdit = "silently " + truncate(raw.Title, 20)
250 }
251 // The proposed validator rule: an unquoted title line must parse back to
252 // exactly its own text; a quoted one is taken at face value.
253 plainLine := "title: " + t.title
254 accepts := strings.HasPrefix(strings.TrimSpace(plainLine[len("title:"):]), `"`) ||
255 (rerr == nil && raw.Title == strings.TrimSpace(t.title))
256 validator := "accept"
257 if !accepts {
258 validator = "REJECT"
259 }
260 if handEdit != "ok" {
261 altered++
262 if accepts {
263 silently++
264 }
265 }
266 if !accepts {
267 rejected++
268 }
269 if lines > 1 {
270 wrap++
271 }
272 fmt.Printf("%-18s %5d %4s %6s %-28s %s\n", t.name, lines, "ok", okStr(quotedOK), handEdit, validator)
273 }
274 fmt.Printf("\ntitles: %d cases. Library emission folds %d past one line. %d hand-edits without quotes change the value, %d of them silently; the validator rejects %d and accepts the rest. Our writer: %d failures.\n",
275 len(tests), wrap, altered, silently, rejected, quotedBad)
276 }
277
278 func probeLabels() {
279 tests := []string{"bug", "needs-repro", "yes", "on", "no", "off", "null", "true", "~", "1.5", "123", "0x10", "y", "n", "a-b_c.d"}
280
281 fmt.Println("\n== labels: block sequence, unquoted vs quoted-when-needed ==")
282 fmt.Printf("%-12s %-10s %-10s %s\n", "label", "unquoted", "as written", "note")
283
284 var unquotedBad int
285 for _, l := range tests {
286 var back struct {
287 Labels []string `yaml:"labels"`
288 }
289 _ = yaml.Unmarshal([]byte("labels:\n - "+l+"\n"), &back)
290 got := "<parse error>"
291 switch {
292 case len(back.Labels) == 1:
293 got = back.Labels[0]
294 }
295 ok := got == l
296 if !ok {
297 unquotedBad++
298 }
299 asWritten := l
300 if !plainSafe(l) {
301 asWritten, _ = quote(l)
302 }
303 note := ""
304 if !ok {
305 note = "silently changes value"
306 }
307 fmt.Printf("%-12s %-10s %-10s %s\n", l, got, asWritten, note)
308 }
309 fmt.Printf("\nlabels: %d of %d unquoted labels do not survive the parse; quoting-when-needed writes %d of %d unchanged\n",
310 unquotedBad, len(tests), countPlain(tests), len(tests))
311 }
312
313 func countPlain(ls []string) int {
314 n := 0
315 for _, l := range ls {
316 if plainSafe(l) {
317 n++
318 }
319 }
320 return n
321 }
322
323 func okStr(b bool) string {
324 if b {
325 return "ok"
326 }
327 return "FAIL"
328 }
329
330 func truncate(s string, n int) string {
331 if len(s) <= n {
332 return s
333 }
334 return s[:n-1] + "…"
335 }
336
337 // ---------------------------------------------------------------- diff mode
338
339 type issue struct {
340 Title string
341 State string
342 Author string
343 Created string
344 Updated string
345 Labels []string
346 Body string
347 }
348
349 // jsonMeta mirrors internal/issues/meta exactly, including omitempty and the
350 // 200-byte title limit the real writer applies.
351 type jsonMeta struct {
352 Title string `json:"title"`
353 State string `json:"state"`
354 Author string `json:"author"`
355 Created string `json:"created"`
356 Updated string `json:"updated,omitempty"`
357 Labels []string `json:"labels,omitempty"`
358 Assignees []string `json:"assignees,omitempty"`
359 }
360
361 func diffs(noUpdated bool) error {
362 steps := []step{
363 {"create", issue{Title: "crash on startup", State: "open", Author: "keenan", Created: created,
364 Labels: []string{"bug"}, Body: "It broke.\n"}},
365 {"add label needs-repro", issue{Title: "crash on startup", State: "open", Author: "keenan", Created: created,
366 Updated: "2026-10-23T12:10:00Z", Labels: []string{"bug", "needs-repro"}, Body: "It broke.\n"}},
367 {"close", issue{Title: "crash on startup", State: "closed", Author: "keenan", Created: created,
368 Updated: "2026-10-23T12:20:00Z", Labels: []string{"bug", "needs-repro"}, Body: "It broke.\n"}},
369 {"reopen", issue{Title: "crash on startup", State: "open", Author: "keenan", Created: created,
370 Updated: "2026-10-23T12:30:00Z", Labels: []string{"bug", "needs-repro"}, Body: "It broke.\n"}},
371 {"retitle", issue{Title: "crash on a<b> & c", State: "open", Author: "keenan", Created: created,
372 Updated: "2026-10-23T12:40:00Z", Labels: []string{"bug", "needs-repro"}, Body: "It broke.\n"}},
373 {"edit body", issue{Title: "crash on a<b> & c", State: "open", Author: "keenan", Created: created,
374 Updated: "2026-10-23T12:50:00Z", Labels: []string{"bug", "needs-repro"},
375 Body: "It broke.\n\nRepro: run vole.\n"}},
376 {"remove last label", issue{Title: "crash on a<b> & c", State: "open", Author: "keenan", Created: created,
377 Updated: "2026-10-23T13:00:00Z", Labels: []string{"bug"}, Body: "It broke.\n\nRepro: run vole.\n"}},
378 }
379 totals, err := runSequence("one issue, six edits", steps, noUpdated, true)
380 if err != nil {
381 return err
382 }
383 fmt.Printf("\n================ changed lines across %d edits ================\n", len(steps)-1)
384 fmt.Printf("%-14s %d\n%-14s %d\n", "json", totals["json"], "frontmatter", totals["frontmatter"])
385
386 first, err := runSequence("first label on a labelless issue", firstLabelSteps(), noUpdated, true)
387 if err != nil {
388 return err
389 }
390 fmt.Printf("\n================ changed lines, first label ever ================\n")
391 fmt.Printf("%-14s %d\n%-14s %d\n", "json", first["json"], "frontmatter", first["frontmatter"])
392 return nil
393 }
394
395 // firstLabelSteps covers the case the JSON writer wastes the most lines on: the
396 // labels key does not exist until the first label does.
397 func firstLabelSteps() []step {
398 base := issue{Title: "slow query", State: "open", Author: "keenan", Created: created,
399 Body: "Queries take four seconds.\n"}
400 withBug := base
401 withBug.Updated = "2026-10-23T14:10:00Z"
402 withBug.Labels = []string{"bug"}
403 back := withBug
404 back.Updated = "2026-10-23T14:20:00Z"
405 back.Labels = nil
406 return []step{{"create", base}, {"add first label", withBug}, {"remove it again", back}}
407 }
408
409 type step struct {
410 name string
411 iss issue
412 }
413
414 // runSequence commits the same edits in both formats and counts the lines git
415 // reports as changed.
416 func runSequence(header string, steps []step, noUpdated, showPatches bool) (map[string]int, error) {
417 root, err := os.MkdirTemp("", "fm-diff-")
418 if err != nil {
419 return nil, err
420 }
421 defer os.RemoveAll(root)
422
423 totals := map[string]int{}
424 for _, format := range []string{"json", "frontmatter"} {
425 dir := filepath.Join(root, format)
426 if err := os.MkdirAll(dir, 0o755); err != nil {
427 return nil, err
428 }
429 if _, err := git(dir, "init", "-q", "-b", "vole"); err != nil {
430 return nil, err
431 }
432 fmt.Printf("\n================ %s [%s] ================\n", header, format)
433 for _, s := range steps {
434 if noUpdated {
435 s.iss.Updated = ""
436 }
437 if err := writeIssue(dir, format, "01JBISSUE0000000000000000A", s.iss); err != nil {
438 return nil, err
439 }
440 if _, err := git(dir, "add", "-A"); err != nil {
441 return nil, err
442 }
443 if _, err := git(dir, "commit", "-q", "-m", s.name); err != nil {
444 return nil, err
445 }
446 if s.name == "create" {
447 continue // the initial add is not a diff
448 }
449 stat, err := git(dir, "diff", "--numstat", "HEAD~1", "HEAD")
450 if err != nil {
451 return nil, err
452 }
453 added, removed := numstat(stat)
454 totals[format] += added + removed
455 fmt.Printf("\n--- %-22s +%d -%d = %d\n", s.name, added, removed, added+removed)
456 if showPatches {
457 patch, err := git(dir, "diff", "-U1", "HEAD~1", "HEAD")
458 if err != nil {
459 return nil, err
460 }
461 fmt.Print(patch)
462 }
463 }
464 }
465 return totals, nil
466 }
467
468 func writeIssue(dir, format string, id string, iss issue) error {
469 switch format {
470 case "json":
471 base := filepath.Join(dir, "issues", id)
472 if err := os.MkdirAll(base, 0o755); err != nil {
473 return err
474 }
475 m := jsonMeta{
476 Title: iss.Title, State: iss.State, Author: iss.Author, Created: iss.Created,
477 Updated: iss.Updated, Labels: iss.Labels, Assignees: nil,
478 }
479 b, err := json.MarshalIndent(m, "", " ")
480 if err != nil {
481 return err
482 }
483 b = append(b, '\n')
484 if err := os.WriteFile(filepath.Join(base, "meta.json"), b, 0o644); err != nil {
485 return err
486 }
487 return os.WriteFile(filepath.Join(base, "description.md"), []byte(iss.Body), 0o644)
488 case "frontmatter":
489 base := filepath.Join(dir, "issues", id)
490 if err := os.MkdirAll(base, 0o755); err != nil {
491 return err
492 }
493 raw, err := renderIssue(iss)
494 if err != nil {
495 return err
496 }
497 return os.WriteFile(filepath.Join(base, "issue.md"), []byte(raw), 0o644)
498 }
499 return fmt.Errorf("unknown format %q", format)
500 }
501
502 func numstat(out string) (added, removed int) {
503 for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
504 f := strings.Fields(line)
505 if len(f) < 2 {
506 continue
507 }
508 var a, r int
509 fmt.Sscanf(f[0], "%d", &a)
510 fmt.Sscanf(f[1], "%d", &r)
511 added += a
512 removed += r
513 }
514 return added, removed
515 }
516
517 func git(dir string, args ...string) (string, error) {
518 cmd := exec.Command("git", args...)
519 cmd.Dir = dir
520 cmd.Env = append(os.Environ(),
521 "GIT_AUTHOR_NAME=spike", "GIT_AUTHOR_EMAIL=spike@example.com",
522 "GIT_COMMITTER_NAME=spike", "GIT_COMMITTER_EMAIL=spike@example.com",
523 "GIT_AUTHOR_DATE="+created, "GIT_COMMITTER_DATE="+created,
524 )
525 out, err := cmd.CombinedOutput()
526 if err != nil {
527 return string(out), fmt.Errorf("git %s: %v: %s", strings.Join(args, " "), err, out)
528 }
529 return string(out), nil
530 }