vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Metadata writers queue on the branch instead of racing for it
122cbe4f62bf87a941538450584ba1b20444c135 · Keenan Nemetz · 7d ago · parent 4739664b
Metadata writers queue on the branch instead of racing for it Every write to refs/heads/vole read the whole branch, decided, and moved the ref with a compare-and-swap, retrying up to 64 times with no backoff and no coordination. Concurrent writers therefore did not queue: they each paid a full read, lost, and paid it again. Measured on a 2,000-file branch, one write needed one attempt alone and 21 with 32 writers, and throughput fell as writers were added because the retries were the load. At 64 simultaneous writers the loop ran out and the write failed for being unlucky. The read-modify-write now runs with one writer at a time, holding a flock in the repository so the SSH hook and the server queue behind each other. The compare-and-swap stays for writers that go around the lock — a push, a hand-run git command, another host — and losing one now waits with jittered backoff before deciding again. Writers read the branch once. The load and the decision happen inside the commit instead of before it, which also stops an edit from replacing a change that landed while the page was open: the decision is made against the tree it writes. A push only does that work when it has metadata to move. Post-receive asks the index which open requests follow the branch it moved, and a branch nobody is proposing from now writes an audit line and no commit. Measurements, 32 concurrent writers: 21.3 CAS attempts per write before, 1.0 after. Through the web, posting comments at concurrency 32: p99 4.7 s -> 0.95 s, and throughput no longer falls as workers are added. The load harness gains a `comment` scenario so the number is visible. Closes: 01M6D9G8H6E0RCQ3AD5TMAGX34
metadata/issue_write.go +122−114
| @@ -89,84 +89,63 @@ func SetIssueState(g *git.Repo, id, state, actor string) (*Issue, error) { | |||
| 89 | 89 | } | |
| 90 | 90 | ||
| 91 | 91 | func updateIssue(g *git.Repo, id string, mutate func(*Issue) error, actor, message string) (*Issue, error) { | |
| 92 | list, err := LoadIssues(g, Ref) | ||
| 93 | if err != nil { | ||
| 94 | return nil, err | ||
| 95 | } | ||
| 96 | var target *Issue | ||
| 97 | for _, iss := range list { | ||
| 98 | if iss.ID == id { | ||
| 99 | target = iss | ||
| 100 | break | ||
| 92 | var out *Issue | ||
| 93 | base := fmt.Sprintf("%s/%s/", IssuesDir, id) | ||
| 94 | _, err := git.CommitFilesBy(g, Ref, func(files map[string][]byte) (string, error) { | ||
| 95 | iss, err := loadIssueFrom(files, id) | ||
| 96 | if err != nil { | ||
| 97 | return "", err | ||
| 101 | 98 | } | |
| 102 | } | ||
| 103 | if target == nil { | ||
| 104 | return nil, fmt.Errorf("issues: no issue %s", id) | ||
| 105 | } | ||
| 106 | if err := mutate(target); err != nil { | ||
| 107 | if errors.Is(err, ErrNoChange) { | ||
| 108 | return nil, ErrNoChange | ||
| 99 | if err := mutate(iss); err != nil { | ||
| 100 | return "", err | ||
| 101 | } | ||
| 102 | if err := validateIssueForWrite(iss); err != nil { | ||
| 103 | return "", err | ||
| 104 | } | ||
| 105 | raw, err := renderIssue(iss) | ||
| 106 | if err != nil { | ||
| 107 | return "", err | ||
| 109 | 108 | } | |
| 110 | return nil, err | ||
| 111 | } | ||
| 112 | if err := validateIssueForWrite(target); err != nil { | ||
| 113 | return nil, err | ||
| 114 | } | ||
| 115 | raw, err := renderIssue(target) | ||
| 116 | if err != nil { | ||
| 117 | return nil, err | ||
| 118 | } | ||
| 119 | base := fmt.Sprintf("%s/%s/", IssuesDir, target.ID) | ||
| 120 | _, err = git.CommitFiles(g, Ref, func(files map[string][]byte) error { | ||
| 121 | 109 | files[base+issueName] = raw | |
| 122 | return nil | ||
| 123 | }, message, actorOr(actor, target.Author)) | ||
| 110 | out = iss | ||
| 111 | return actorOr(actor, iss.Author), nil | ||
| 112 | }, message) | ||
| 124 | 113 | if err != nil { | |
| 125 | 114 | return nil, err | |
| 126 | 115 | } | |
| 127 | return target, nil | ||
| 116 | return out, nil | ||
| 128 | 117 | } | |
| 129 | 118 | ||
| 130 | 119 | // AddComment appends a comment file to an existing issue. | |
| 131 | 120 | func AddIssueComment(g *git.Repo, id string, c *Comment, actor string) error { | |
| 132 | list, err := LoadIssues(g, Ref) | ||
| 133 | if err != nil { | ||
| 134 | return err | ||
| 135 | } | ||
| 136 | var target *Issue | ||
| 137 | for _, iss := range list { | ||
| 138 | if iss.ID == id { | ||
| 139 | target = iss | ||
| 140 | break | ||
| 121 | path := IssueCommentPath(id, c.ID) | ||
| 122 | _, err := git.CommitFilesBy(g, Ref, func(files map[string][]byte) (string, error) { | ||
| 123 | iss, err := loadIssueFrom(files, id) | ||
| 124 | if err != nil { | ||
| 125 | return "", err | ||
| 126 | } | ||
| 127 | if prob := checkCommentReply(iss.Comments, c); prob != "" { | ||
| 128 | return "", fmt.Errorf("issues: %s", prob) | ||
| 129 | } | ||
| 130 | if strings.TrimSpace(c.Author) == "" { | ||
| 131 | c.Author = actor | ||
| 132 | } | ||
| 133 | if strings.TrimSpace(c.Author) == "" { | ||
| 134 | return "", fmt.Errorf("issues: comment needs an author") | ||
| 135 | } | ||
| 136 | if c.Created.IsZero() { | ||
| 137 | c.Created = time.Now().UTC() | ||
| 138 | } | ||
| 139 | raw, err := RenderComment(c) | ||
| 140 | if err != nil { | ||
| 141 | return "", err | ||
| 141 | 142 | } | |
| 142 | } | ||
| 143 | if target == nil { | ||
| 144 | return fmt.Errorf("issues: no issue %s", id) | ||
| 145 | } | ||
| 146 | if prob := checkCommentReply(target.Comments, c); prob != "" { | ||
| 147 | return fmt.Errorf("issues: %s", prob) | ||
| 148 | } | ||
| 149 | if strings.TrimSpace(c.Author) == "" { | ||
| 150 | c.Author = actor | ||
| 151 | } | ||
| 152 | if strings.TrimSpace(c.Author) == "" { | ||
| 153 | return fmt.Errorf("issues: comment needs an author") | ||
| 154 | } | ||
| 155 | if c.Created.IsZero() { | ||
| 156 | c.Created = time.Now().UTC() | ||
| 157 | } | ||
| 158 | raw, err := RenderComment(c) | ||
| 159 | if err != nil { | ||
| 160 | return err | ||
| 161 | } | ||
| 162 | path := fmt.Sprintf("%s/%s/%s/%s.md", IssuesDir, id, commentsDir, c.ID) | ||
| 163 | _, err = git.CommitFiles(g, Ref, func(files map[string][]byte) error { | ||
| 164 | 143 | if _, exists := files[path]; exists { | |
| 165 | return fmt.Errorf("issues: comment %s already exists", c.ID) | ||
| 144 | return "", fmt.Errorf("issues: comment %s already exists", c.ID) | ||
| 166 | 145 | } | |
| 167 | 146 | files[path] = raw | |
| 168 | return nil | ||
| 169 | }, fmt.Sprintf("issue: comment on %s", id), c.Author) | ||
| 147 | return c.Author, nil | ||
| 148 | }, fmt.Sprintf("issue: comment on %s", id)) | ||
| 170 | 149 | return err | |
| 171 | 150 | } | |
| 172 | 151 | ||
| @@ -180,13 +159,6 @@ func AddIssueComment(g *git.Repo, id string, c *Comment, actor string) error { | |||
| 180 | 159 | // Only the named author (or an admin, audited) may change someone else's comment, | |
| 181 | 160 | // and that rule belongs at the push boundary where the account is known. | |
| 182 | 161 | func EditIssueComment(g *git.Repo, issueID, commentID, body, actor string) (*Comment, error) { | |
| 183 | c, path, err := findIssueComment(g, issueID, commentID) | ||
| 184 | if err != nil { | ||
| 185 | return nil, err | ||
| 186 | } | ||
| 187 | if c.Deleted { | ||
| 188 | return nil, fmt.Errorf("issues: cannot edit a deleted comment") | ||
| 189 | } | ||
| 190 | 162 | body = strings.TrimRight(body, "\n") | |
| 191 | 163 | if strings.TrimSpace(body) == "" { | |
| 192 | 164 | return nil, fmt.Errorf("issues: a comment needs a body") | |
| @@ -194,75 +166,111 @@ func EditIssueComment(g *git.Repo, issueID, commentID, body, actor string) (*Com | |||
| 194 | 166 | if problems := bodyProblems("comment", body); len(problems) > 0 { | |
| 195 | 167 | return nil, fmt.Errorf("issues: %s", problems[0]) | |
| 196 | 168 | } | |
| 197 | c.Body = body | ||
| 198 | raw, err := RenderComment(c) | ||
| 199 | if err != nil { | ||
| 200 | return nil, err | ||
| 201 | } | ||
| 202 | _, err = git.CommitFiles(g, Ref, func(files map[string][]byte) error { | ||
| 203 | if _, ok := files[path]; !ok { | ||
| 204 | return fmt.Errorf("issues: no comment %s", commentID) | ||
| 169 | path := IssueCommentPath(issueID, commentID) | ||
| 170 | var out *Comment | ||
| 171 | _, err := git.CommitFilesBy(g, Ref, func(files map[string][]byte) (string, error) { | ||
| 172 | c, err := loadIssueComment(files, issueID, commentID) | ||
| 173 | if err != nil { | ||
| 174 | return "", err | ||
| 175 | } | ||
| 176 | if c.Deleted { | ||
| 177 | return "", fmt.Errorf("issues: cannot edit a deleted comment") | ||
| 178 | } | ||
| 179 | c.Body = body | ||
| 180 | raw, err := RenderComment(c) | ||
| 181 | if err != nil { | ||
| 182 | return "", err | ||
| 205 | 183 | } | |
| 206 | 184 | files[path] = raw | |
| 207 | return nil | ||
| 208 | }, fmt.Sprintf("issue: edit comment %s on %s", commentID, issueID), actorOr(actor, c.Author)) | ||
| 185 | out = c | ||
| 186 | return actorOr(actor, c.Author), nil | ||
| 187 | }, fmt.Sprintf("issue: edit comment %s on %s", commentID, issueID)) | ||
| 209 | 188 | if err != nil { | |
| 210 | 189 | return nil, err | |
| 211 | 190 | } | |
| 212 | return c, nil | ||
| 191 | return out, nil | ||
| 213 | 192 | } | |
| 214 | 193 | ||
| 215 | 194 | // DeleteComment replaces a comment with a tombstone. The text stays in history — | |
| 216 | 195 | // deleting is not redaction, which is still an admin force-push — and the marker | |
| 217 | 196 | // remains so replies keep the parent they answer. | |
| 218 | 197 | func DeleteIssueComment(g *git.Repo, issueID, commentID, actor string) (*Comment, error) { | |
| 219 | c, path, err := findIssueComment(g, issueID, commentID) | ||
| 198 | path := IssueCommentPath(issueID, commentID) | ||
| 199 | var out *Comment | ||
| 200 | _, err := git.CommitFilesBy(g, Ref, func(files map[string][]byte) (string, error) { | ||
| 201 | c, err := loadIssueComment(files, issueID, commentID) | ||
| 202 | if err != nil { | ||
| 203 | return "", err | ||
| 204 | } | ||
| 205 | if c.Deleted { | ||
| 206 | return "", fmt.Errorf("issues: comment %s is already deleted", commentID) | ||
| 207 | } | ||
| 208 | c.Deleted = true | ||
| 209 | c.Body = "" | ||
| 210 | c.Anchor = nil | ||
| 211 | raw, err := RenderComment(c) | ||
| 212 | if err != nil { | ||
| 213 | return "", err | ||
| 214 | } | ||
| 215 | files[path] = raw | ||
| 216 | out = c | ||
| 217 | return actorOr(actor, c.Author), nil | ||
| 218 | }, fmt.Sprintf("issue: delete comment %s on %s", commentID, issueID)) | ||
| 220 | 219 | if err != nil { | |
| 221 | 220 | return nil, err | |
| 222 | 221 | } | |
| 223 | if c.Deleted { | ||
| 224 | return nil, fmt.Errorf("issues: comment %s is already deleted", commentID) | ||
| 222 | return out, nil | ||
| 223 | } | ||
| 224 | |||
| 225 | // loadIssueFrom reads one issue and its comments out of the tree a commit is being | ||
| 226 | // built from. A write decides against the tip it is about to replace, so a change | ||
| 227 | // that landed since the page was drawn is visible instead of being overwritten, and | ||
| 228 | // the branch is read once rather than once for the decision and once for the commit. | ||
| 229 | func loadIssueFrom(files map[string][]byte, id string) (*Issue, error) { | ||
| 230 | prefix := IssuesDir + "/" + id + "/" | ||
| 231 | contents := make(map[string]string) | ||
| 232 | for path, raw := range files { | ||
| 233 | if strings.HasPrefix(path, prefix) { | ||
| 234 | contents[path] = string(raw) | ||
| 235 | } | ||
| 225 | 236 | } | |
| 226 | c.Deleted = true | ||
| 227 | c.Body = "" | ||
| 228 | c.Anchor = nil | ||
| 229 | raw, err := RenderComment(c) | ||
| 237 | list, err := ParseIssues(contents) | ||
| 230 | 238 | if err != nil { | |
| 239 | if ve, ok := err.(*ValidationError); ok { | ||
| 240 | return nil, fmt.Errorf("issues: the metadata tree is invalid: %s", ve.Problems[0]) | ||
| 241 | } | ||
| 231 | 242 | return nil, err | |
| 232 | 243 | } | |
| 233 | _, err = git.CommitFiles(g, Ref, func(files map[string][]byte) error { | ||
| 234 | if _, ok := files[path]; !ok { | ||
| 235 | return fmt.Errorf("issues: no comment %s", commentID) | ||
| 236 | } | ||
| 237 | files[path] = raw | ||
| 238 | return nil | ||
| 239 | }, fmt.Sprintf("issue: delete comment %s on %s", commentID, issueID), actorOr(actor, c.Author)) | ||
| 244 | if len(list) == 0 { | ||
| 245 | return nil, fmt.Errorf("issues: no issue %s", id) | ||
| 246 | } | ||
| 247 | return list[0], nil | ||
| 248 | } | ||
| 249 | |||
| 250 | // loadIssueComment is loadIssueFrom for a write that names one comment. | ||
| 251 | func loadIssueComment(files map[string][]byte, issueID, commentID string) (*Comment, error) { | ||
| 252 | iss, err := loadIssueFrom(files, issueID) | ||
| 240 | 253 | if err != nil { | |
| 241 | 254 | return nil, err | |
| 242 | 255 | } | |
| 256 | c, ok := commentOf(iss.Comments, commentID) | ||
| 257 | if !ok { | ||
| 258 | return nil, fmt.Errorf("issues: no comment %s on %s", commentID, issueID) | ||
| 259 | } | ||
| 243 | 260 | return c, nil | |
| 244 | 261 | } | |
| 245 | 262 | ||
| 246 | // findComment locates one comment on the branch and returns it with the path it | ||
| 247 | // lives at, which is the file the edit or delete rewrites. | ||
| 248 | func findIssueComment(g *git.Repo, issueID, commentID string) (*Comment, string, error) { | ||
| 249 | list, err := LoadIssues(g, Ref) | ||
| 250 | if err != nil { | ||
| 251 | return nil, "", err | ||
| 252 | } | ||
| 253 | for _, iss := range list { | ||
| 254 | if iss.ID != issueID { | ||
| 255 | continue | ||
| 256 | } | ||
| 257 | for i := range iss.Comments { | ||
| 258 | if iss.Comments[i].ID == commentID { | ||
| 259 | c := iss.Comments[i] | ||
| 260 | return &c, IssueCommentPath(issueID, commentID), nil | ||
| 261 | } | ||
| 263 | // commentOf finds one comment by id and returns a copy, so the write can change it | ||
| 264 | // without touching the tree it was parsed from. Issues and change requests keep | ||
| 265 | // their discussion in the same shape, so both find it the same way. | ||
| 266 | func commentOf(list []Comment, id string) (*Comment, bool) { | ||
| 267 | for i := range list { | ||
| 268 | if list[i].ID == id { | ||
| 269 | c := list[i] | ||
| 270 | return &c, true | ||
| 262 | 271 | } | |
| 263 | return nil, "", fmt.Errorf("issues: no comment %s on %s", commentID, issueID) | ||
| 264 | 272 | } | |
| 265 | return nil, "", fmt.Errorf("issues: no issue %s", issueID) | ||
| 273 | return nil, false | ||
| 266 | 274 | } | |
| 267 | 275 | ||
| 268 | 276 | // RenderComment writes a comment as frontmatter plus body. Exported because a | |