Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Highlight with Prism in a worker and gzip text responses

306f36fcef89c145abcd0c5331453d0b179f37ce · Keenan Nemetz · 12d ago · parent e5542ac6 

Highlight with Prism in a worker and gzip text responses

The code browser now names a blob's language and lets the browser colour it.
Chroma is gone: server/web/highlight.go is an extension table, and
static/prism-worker.js runs Prism — every language Prism ships — off the main
thread. The server sends escaped lines with their numbers and L# anchors, so a
page is complete and scrollable before any script runs and the worker only swaps
the contents of each line cell. The table covers 186 languages; a file it does
not claim stays plain rather than being coloured by a guess.

The bundle those grammars make is 545 KB, so the instance now gzips text
responses itself instead of relying on a proxy, and static assets carry a
content ETag so a browser revalidates with a 304 rather than re-downloading the
bundle on every code page. The raw-download route opts out with
Cache-Control: no-transform because it promises an exact Content-Length.

This also makes JavaScript a requirement rather than a nicety: the label and
assignee token inputs lose their noscript save buttons and their comma-separated
text fallback, and the docs say so.

Measured: release binary 26.08 MB -> 22.70 MB; /r/{repo}/code 11.1 KB -> 2.6 KB
over the wire; the Prism bundle 545 KB -> 206 KB gzipped, fetched once.

scripts/vendor-prism.py regenerates the bundle from Prism's components.json
require graph, and --check fails if the committed assets drift.

scripts/vendor-prism.py +160

@@ -0,0 +1,160 @@
1 #!/usr/bin/env python3
2 """Regenerate the vendored Prism assets in server/web/static.
3
4 Vole's code browser highlights in the browser (server/web/static/prism-worker.js)
5 and names the language on the server (server/web/highlight.go). The grammars
6 themselves are Prism's, concatenated into two files so an instance is still one
7 binary and there is no npm step in this repository:
8
9 prism.min.js the core build: markup, css, clike, javascript
10 prism-languages.min.js every other language Prism ships, in dependency order
11
12 The order is not alphabetical: it comes from the "require" graph in Prism's own
13 components.json, so markup-templating precedes php, c precedes cpp and zig, java
14 precedes scala, and css precedes scss. Cycles or unsatisfied requirements are
15 errors rather than a subtly broken bundle.
16
17 scripts/vendor-prism.py write the files
18 scripts/vendor-prism.py --check compare, exit 1 if they differ
19
20 Bumping PRISM_VERSION is the whole upgrade: rerun, then run the tests, which hold
21 the server's extension table (highlight.go) to the grammars in the bundle.
22 """
23
24 import argparse
25 import json
26 import os
27 import sys
28 import urllib.request
29
30 PRISM_VERSION = "1.30.0"
31
32 CDN = "https://cdnjs.cloudflare.com/ajax/libs/prism/{version}"
33 RAW = "https://raw.githubusercontent.com/PrismJS/prism/v{version}"
34
35 # Languages that ship inside prism.min.js (Prism's "default" build).
36 CORE_LANGUAGES = ("markup", "css", "clike", "javascript")
37
38 STATIC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir,
39 "server", "web", "static")
40
41 HEADER = """/* Prism {version}, every language it ships, concatenated in dependency order from
42 the components published on cdnjs. Prism's core (prism.min.js) carries markup,
43 css, clike and javascript; these {count} extend one of those, or a dependency listed
44 before them here (markup-templating before php, c before cpp/csharp/zig, java
45 before scala, css before scss).
46
47 Vendored rather than fetched so an instance is still one file, and generated by
48 concatenation rather than built: there is no npm build step in this repository.
49 The list and its order come from Prism's components.json "require" graph; a
50 language added here also needs an extension in the server's table
51 (highlight.go), and TestPrismLanguagesAreVendored holds the two together.
52 Regenerate with scripts/vendor-prism.py. */
53
54 """
55
56
57 def fetch(url):
58 request = urllib.request.Request(url, headers={"User-Agent": "vole-vendor-prism"})
59 with urllib.request.urlopen(request) as response:
60 return response.read()
61
62
63 def component(name):
64 return CDN.format(version=PRISM_VERSION) + "/components/prism-" + name + ".min.js"
65
66
67 def requirements(meta, name):
68 require = meta[name].get("require")
69 if not require:
70 return []
71 return [require] if isinstance(require, str) else list(require)
72
73
74 def order_languages(meta):
75 """Return every non-core language id in dependency order.
76
77 Prism's metadata lists what a grammar needs; a language is emitted after its
78 requirements. A missing or circular requirement is a hard error: the bundle
79 would load but colour nothing, which is worse than failing here.
80 """
81 ids = [name for name in meta if name not in CORE_LANGUAGES]
82 known = set(ids) | set(CORE_LANGUAGES)
83 order, done = [], set()
84
85 def visit(name, stack):
86 if name in done:
87 return
88 if name in stack:
89 raise SystemExit("prism: dependency cycle: " + " -> ".join(stack + (name,)))
90 for dep in requirements(meta, name):
91 if dep not in known:
92 raise SystemExit(f"prism: {name} requires {dep}, which Prism does not define")
93 if dep not in CORE_LANGUAGES:
94 visit(dep, stack + (name,))
95 done.add(name)
96 order.append(name)
97
98 for name in ids:
99 visit(name, ())
100 return order
101
102
103 def build():
104 meta = json.loads(fetch(RAW.format(version=PRISM_VERSION) + "/components.json"))["languages"]
105 meta.pop("meta", None)
106 languages = order_languages(meta)
107
108 core = fetch(CDN.format(version=PRISM_VERSION) + "/prism.min.js")
109 license_text = fetch(RAW.format(version=PRISM_VERSION) + "/LICENSE")
110 bundle = [HEADER.format(version=PRISM_VERSION, count=len(languages))]
111 for name in languages:
112 bundle.append(fetch(component(name)).decode("utf-8"))
113 bundle.append("\n")
114 return core, "".join(bundle).encode("utf-8"), license_text, len(languages)
115
116
117 def main():
118 parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
119 parser.add_argument("--check", action="store_true",
120 help="verify the committed files instead of writing them")
121 args = parser.parse_args()
122
123 core, bundle, license_text, count = build()
124 files = {
125 "prism.min.js": core,
126 "prism-languages.min.js": bundle,
127 "prism.LICENSE": license_text,
128 }
129
130 if args.check:
131 stale = []
132 for name, want in files.items():
133 path = os.path.join(STATIC_DIR, name)
134 try:
135 with open(path, "rb") as handle:
136 have = handle.read()
137 except FileNotFoundError:
138 stale.append(f"{name} is missing")
139 continue
140 if have != want:
141 stale.append(f"{name} differs ({len(have)} bytes vendored, {len(want)} upstream)")
142 if stale:
143 for line in stale:
144 print("stale:", line, file=sys.stderr)
145 print("run scripts/vendor-prism.py to regenerate", file=sys.stderr)
146 return 1
147 print(f"ok: Prism {PRISM_VERSION} assets match ({count} languages)")
148 return 0
149
150 for name, body in files.items():
151 path = os.path.join(STATIC_DIR, name)
152 with open(path, "wb") as handle:
153 handle.write(body)
154 print(f"wrote {os.path.relpath(path)} ({len(body)} bytes)")
155 print(f"Prism {PRISM_VERSION}: {count} languages")
156 return 0
157
158
159 if __name__ == "__main__":
160 sys.exit(main())