Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Phase 7 slice 3: artifacts attach over SSH and are one pointer deep

85cc89fe9eafd1793702d143ea2eb41db635fbe2 · Keenan Nemetz · 17d ago · parent c56911e2 

Phase 7 slice 3: artifacts attach over SSH and are one pointer deep

A released binary is a pointer blob at releases/<tag>/<name> on the metadata
branch. That is the whole record, and it was chosen because the machinery that
already exists does the work: gc marks LFS objects named by pointers in any
tree reachable from any ref (refs/heads/vole is one), and pre-receive's pointer
hygiene already refuses a pointer whose bytes are missing. A JSON manifest of
sha256 strings would have parsed as no pointer and been swept once the grace
period passed.

- cmd/vole attach speaks the pure-SSH LFS transfer protocol itself (version,
  batch, put-object, verify-object), so bytes move over the key that already
  writes: no token, no second endpoint, no new write surface
- upload happens before the pointer commit, because the reverse order publishes
  a download that 404s and receive-pack cannot see bytes that were never in the
  pack. A test pushes a pointer with no object and asserts the push is refused
- the artifact list is read from the pointer tree (internal/releases), which the
  metadata validator now knows about; a file under releases/ that is not a
  pointer is refused at push instead of silently unservable
- downloads: /r/<repo>/releases/<tag>/artifacts/<name> out of the store, ETag =
  object id, always Content-Disposition: attachment with nosniff, ServeContent
  for Range and 304, clone read rule so private artifacts are 404
- the "not on this server" hook message now names vole release attach as well as
  git lfs push, because it fires for both now

Measured end to end through the binary: two 2.9 MiB files attached over SSH in
390 ms, the identical second file deduplicating to one object; both downloaded
in 13.9 ms with matching sha256, 304 on If-None-Match, 206 on Range; vole gc
--dry-run reports the published artifact reachable and removes nothing.

The planned releases/<tag>.json record and drafts are dropped with it: the
pointer directory is the artifact list and being in the ref is what published
means, so the file would only have duplicated the tag.

internal/web/releases.go +106−19

@@ -8,10 +8,14 @@ package web
8 8 import (
9 9 "fmt"
10 10 "html/template"
11 "mime"
11 12 "net/http"
13 "path"
12 14 "strings"
15 "time"
13 16
14 17 "github.com/rodent-software/vole/internal/gitx"
18 "github.com/rodent-software/vole/internal/releases"
15 19 )
16 20
17 21 // archiveConc bounds concurrent compressed archives. Plain tar streams at
@@ -74,6 +78,14 @@ func (w *Web) ReleaseView(rw http.ResponseWriter, r *http.Request) {
74 78 commits, _ := g.CommitsBetween(from, t.Commit, maxReleaseCommits)
75 79 count, _ := g.CountBetween(from, t.Commit)
76 80
81 // The artifact list is the pointer directory on the metadata branch, not a
82 // stored manifest: the same pointers the sweep marks and the hook validates.
83 artifacts, _ := releases.List(g, t.Name)
84 var artifactBytes int64
85 for _, a := range artifacts {
86 artifactBytes += a.Size
87 }
88
77 89 // git archive copies LFS pointer files verbatim (measured in spike D), so a
78 90 // tarball of an LFS repository holds shopping lists where the reader expects
79 91 // bytes. Saying so is cheaper than resolving pointers, and a silent pointer is
@@ -89,29 +101,104 @@ func (w *Web) ReleaseView(rw http.ResponseWriter, r *http.Request) {
89 101
90 102 w.render(rw, "release", struct {
91 103 Page
92 Name string
93 Tag gitx.Tag
94 Notes template.HTML
95 Prev string
96 Count int
97 Commits []gitx.Commit
98 Capped bool
99 CappedAt int
100 LFSPointers bool
104 Name string
105 Tag gitx.Tag
106 Notes template.HTML
107 Prev string
108 Count int
109 Commits []gitx.Commit
110 Capped bool
111 CappedAt int
112 LFSPointers bool
113 Artifacts []releases.Artifact
114 ArtifactBytes int64
101 115 }{
102 Page: w.base(m.Name + " · " + t.Name),
103 Name: m.Name,
104 Tag: t,
105 Notes: notes,
106 Prev: prevName,
107 Count: count,
108 Commits: commits,
109 Capped: count > len(commits),
110 CappedAt: maxReleaseCommits,
111 LFSPointers: lfsPointers,
116 Page: w.base(m.Name + " · " + t.Name),
117 Name: m.Name,
118 Tag: t,
119 Notes: notes,
120 Prev: prevName,
121 Count: count,
122 Commits: commits,
123 Capped: count > len(commits),
124 CappedAt: maxReleaseCommits,
125 LFSPointers: lfsPointers,
126 Artifacts: artifacts,
127 ArtifactBytes: artifactBytes,
112 128 }, 0)
113 129 }
114 130
131 // ReleaseArtifact serves one attached file out of the object store. The read
132 // rule is the clone rule, so a private repository's artifacts are 404 like
133 // everything else it has, and the bytes come from the same store LFS uses — an
134 // artifact is not copied anywhere to be downloadable.
135 func (w *Web) ReleaseArtifact(rw http.ResponseWriter, r *http.Request) {
136 m, ok := w.open(rw, r)
137 if !ok {
138 return
139 }
140 g := m.Git()
141 t, err := g.Tag(r.PathValue("tag"))
142 if err != nil {
143 http.NotFound(rw, r)
144 return
145 }
146 name := r.PathValue("artifact")
147 artifacts, err := releases.List(g, t.Name)
148 if err != nil {
149 http.Error(rw, "vole: cannot read that release", http.StatusInternalServerError)
150 return
151 }
152 var art *releases.Artifact
153 for i := range artifacts {
154 if artifacts[i].Name == name {
155 art = &artifacts[i]
156 break
157 }
158 }
159 if art == nil || w.LFS == nil {
160 http.NotFound(rw, r)
161 return
162 }
163 f, size, err := w.LFS.Open(art.OID)
164 if err != nil {
165 // A pointer whose object is gone is a store that lost bytes, not a
166 // client mistake; it still cannot be served.
167 w.Logger("artifact %s/%s/%s: %v", m.Name, t.Name, name, err)
168 http.NotFound(rw, r)
169 return
170 }
171 defer f.Close()
172
173 // The object id is the content, so it is the validator: the same bytes can
174 // never arrive under a different ETag, and a re-uploaded name cannot serve
175 // stale content.
176 rw.Header().Set("ETag", `"`+art.OID+`"`)
177 rw.Header().Set("Content-Type", artifactType(name))
178 rw.Header().Set("X-Content-Type-Options", "nosniff")
179 rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q",
180 releaseSafe(name)))
181 rw.Header().Set("Cache-Control", "public, max-age=300")
182 // ServeContent handles Range and the conditional request against the ETag
183 // above, which is what makes a 100 MiB artifact resumable.
184 http.ServeContent(rw, r, releaseSafe(name), time.Time{}, f)
185 _ = size
186 }
187
188 // artifactType names the type from the extension and refuses to guess HTML: an
189 // artifact is downloaded, never rendered on this origin, so a wrong type costs
190 // nothing while a right one saves the reader a step.
191 func artifactType(name string) string {
192 switch strings.ToLower(path.Ext(name)) {
193 case ".html", ".htm", ".svg", ".xml":
194 return "application/octet-stream"
195 }
196 if t := mime.TypeByExtension(path.Ext(name)); t != "" {
197 return t
198 }
199 return "application/octet-stream"
200 }
201
115 202 func (w *Web) ReleaseTar(rw http.ResponseWriter, r *http.Request) {
116 203 w.releaseArchive(rw, r, "tar")
117 204 }