vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
|
5ece70a5
Keenan Nemetz
10d ago
|
1 | // Package logs stores one CI attempt's log as a file on the instance. |
| 2 | // | |
| 3 | // A log is not content-addressed and is not shared: it belongs to one repository, | |
| 4 | // one run and one attempt, and the run page is its only reader. Keeping it in its | |
| 5 | // own directory makes a partial log readable while the job is still running, which | |
| 6 | // is the point of streaming it over the runner's connection. | |
| 7 | package logs | |
| 8 | ||
| 9 | import ( | |
| 10 | "errors" | |
| 11 | "fmt" | |
| 12 | "os" | |
| 13 | "path/filepath" | |
| 14 | "strconv" | |
| 15 | ||
|
d37d301f
Keenan Nemetz
17h ago
|
16 | "rodent.software/vole/core/id" |
| 17 | "rodent.software/vole/forge/repo" | |
|
5ece70a5
Keenan Nemetz
10d ago
|
18 | ) |
| 19 | ||
| 20 | const ( | |
| 21 | // MaxBytes bounds one attempt's log. The runner caps its local spool at the same | |
| 22 | // number; the extra room here is for the truncation note the spool appends after | |
| 23 | // it hits the cap. | |
| 24 | MaxBytes int64 = 32 << 20 | |
| 25 | // reserve is room for the truncation note. | |
| 26 | reserve int64 = 4 << 10 | |
| 27 | ) | |
| 28 | ||
| 29 | var ( | |
| 30 | // ErrTooLarge is a log past MaxBytes. | |
| 31 | ErrTooLarge = errors.New("logs: attempt log is too large") | |
| 32 | // ErrOffset is a chunk that does not start where the file ends. | |
| 33 | ErrOffset = errors.New("logs: chunk does not continue the log") | |
| 34 | ) | |
| 35 | ||
| 36 | // Store keeps logs under one root. The layout mirrors the queue: | |
| 37 | // | |
| 38 | // <root>/<repo>/<run id>/<attempt>.log | |
| 39 | type Store struct { | |
| 40 | root string | |
| 41 | max int64 | |
| 42 | } | |
| 43 | ||
| 44 | func New(root string) *Store { return &Store{root: root, max: MaxBytes + reserve} } | |
| 45 | ||
| 46 | // Path is where one attempt's log lives. | |
| 47 | func (s *Store) Path(repoName, runID string, attempt int) (string, error) { | |
| 48 | if err := repo.ValidateName(repoName); err != nil { | |
| 49 | return "", err | |
| 50 | } | |
| 51 | if !id.Valid(runID) { | |
| 52 | return "", fmt.Errorf("logs: %q is not a run id", runID) | |
| 53 | } | |
| 54 | if attempt < 1 { | |
| 55 | return "", fmt.Errorf("logs: attempt %d is not a number", attempt) | |
| 56 | } | |
| 57 | return filepath.Join(s.root, repoName, runID, strconv.Itoa(attempt)+".log"), nil | |
| 58 | } | |
| 59 | ||
| 60 | // Append writes a chunk at offset, creating the file on the first chunk. The chunk | |
| 61 | // must continue the file: a gap would make the step ranges lie about where their | |
| 62 | // bytes are. | |
| 63 | func (s *Store) Append(repoName, runID string, attempt int, offset int64, data []byte) (int64, error) { | |
| 64 | if offset < 0 { | |
| 65 | return 0, ErrOffset | |
| 66 | } | |
| 67 | if offset+int64(len(data)) > s.max { | |
| 68 | return 0, ErrTooLarge | |
| 69 | } | |
| 70 | path, err := s.Path(repoName, runID, attempt) | |
| 71 | if err != nil { | |
| 72 | return 0, err | |
| 73 | } | |
| 74 | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |
| 75 | return 0, err | |
| 76 | } | |
| 77 | f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0o600) | |
| 78 | if err != nil { | |
| 79 | return 0, err | |
| 80 | } | |
| 81 | defer f.Close() | |
| 82 | st, err := f.Stat() | |
| 83 | if err != nil { | |
| 84 | return 0, err | |
| 85 | } | |
| 86 | if st.Size() != offset { | |
| 87 | return 0, fmt.Errorf("%w: %s is %d bytes, chunk starts at %d", ErrOffset, path, st.Size(), offset) | |
| 88 | } | |
| 89 | if _, err := f.WriteAt(data, offset); err != nil { | |
| 90 | return 0, err | |
| 91 | } | |
| 92 | return offset + int64(len(data)), nil | |
| 93 | } | |
| 94 | ||
| 95 | // Size is the bytes stored for one attempt. A missing log is zero: a run that | |
| 96 | // produced no output has no file, not an empty one. | |
| 97 | func (s *Store) Size(repoName, runID string, attempt int) (int64, error) { | |
| 98 | path, err := s.Path(repoName, runID, attempt) | |
| 99 | if err != nil { | |
| 100 | return 0, err | |
| 101 | } | |
| 102 | st, err := os.Stat(path) | |
| 103 | if errors.Is(err, os.ErrNotExist) { | |
| 104 | return 0, nil | |
| 105 | } | |
| 106 | if err != nil { | |
| 107 | return 0, err | |
| 108 | } | |
| 109 | return st.Size(), nil | |
| 110 | } | |
| 111 | ||
| 112 | // Open returns the log for one attempt, positioned at the start. | |
| 113 | func (s *Store) Open(repoName, runID string, attempt int) (*os.File, os.FileInfo, error) { | |
| 114 | path, err := s.Path(repoName, runID, attempt) | |
| 115 | if err != nil { | |
| 116 | return nil, nil, err | |
| 117 | } | |
| 118 | f, err := os.Open(path) | |
| 119 | if err != nil { | |
| 120 | return nil, nil, err | |
| 121 | } | |
| 122 | st, err := f.Stat() | |
| 123 | if err != nil { | |
| 124 | f.Close() | |
| 125 | return nil, nil, err | |
| 126 | } | |
| 127 | return f, st, nil | |
| 128 | } | |
| 129 | ||
| 130 | // Sync flushes the file before the result that points at it is stored. | |
| 131 | func (s *Store) Sync(repoName, runID string, attempt int) error { | |
| 132 | f, _, err := s.Open(repoName, runID, attempt) | |
| 133 | if errors.Is(err, os.ErrNotExist) { | |
| 134 | return nil | |
| 135 | } | |
| 136 | if err != nil { | |
| 137 | return err | |
| 138 | } | |
| 139 | defer f.Close() | |
| 140 | return f.Sync() | |
| 141 | } | |
| 142 | ||
| 143 | // Remove drops one attempt's log and the run directory when it becomes empty. It | |
| 144 | // is idempotent, so a finish that reports no log can clean up a partial stream. | |
| 145 | func (s *Store) Remove(repoName, runID string, attempt int) error { | |
| 146 | path, err := s.Path(repoName, runID, attempt) | |
| 147 | if err != nil { | |
| 148 | return err | |
| 149 | } | |
| 150 | if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { | |
| 151 | return err | |
| 152 | } | |
| 153 | _ = os.Remove(filepath.Dir(path)) // only succeeds when empty | |
| 154 | return nil | |
| 155 | } |