Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Code

392 commit(s) on main

Compare History
Merge change request 01M70WES5ZZDJ6RP226WB11BBZ: Let people rename and remove their own passkeys keenan · 25m ago 4b8c9e1d
history blame raw 14.1 KiB
1#!/usr/bin/env python3
2"""Regenerate the vendored Prism assets in server/web/static.
3
4Vole's code browser highlights in the browser (server/web/static/prism-worker.js)
5and names the language on the server (server/web/highlight.go). The grammars
6themselves are Prism's, vendored so an instance is still one binary and there is
7no npm step in this repository:
8
9 prism.min.js the core build: markup, css, clike, javascript
10 prism-manifest.js language name -> the files that define it, in order
11 prism-languages/<id>.js one Prism component per file, unmodified
12
13The worker starts with the core and the manifest. When a blob or a fenced block
14names a language the core does not define, the worker imports exactly that
15language's files — usually one, more when the grammar extends something Prism
16keeps out of the core (php needs markup-templating, scala needs java, arduino
17needs cpp). So a page parses the grammar it shows rather than all of them, and
18the manifest's lists are what make that one import.
19
20Dependency order is Prism's, from the "require" graph in components.json: it is
21not alphabetical, because markup-templating precedes php, c precedes cpp and zig,
22java precedes scala, and css precedes scss. Cycles or unsatisfied requirements
23are errors rather than a subtly broken grammar set.
24
25 scripts/vendor-prism.py write the files
26 scripts/vendor-prism.py --check compare, exit 1 if they differ
27
28Bumping PRISM_VERSION is the whole upgrade: rerun, then run the tests, which hold
29the server's extension table (highlight.go) to what the manifest can load.
30"""
31
32import argparse
33import json
34import os
35import re
36import sys
37import urllib.request
38
39PRISM_VERSION = "1.30.0"
40
41CDN = "https://cdnjs.cloudflare.com/ajax/libs/prism/{version}"
42RAW = "https://raw.githubusercontent.com/PrismJS/prism/v{version}"
43
44# Languages that ship inside prism.min.js (Prism's "default" build). Every page
45# pays for these, so they never appear as a manifest entry of their own.
46CORE_LANGUAGES = ("markup", "css", "clike", "javascript")
47
48# Components that change a grammar another component defines instead of
49# registering a language name of their own, so no language would import them.
50# components.json says what they require, not what they touch, hence the table:
51#
52# id: (grammars it inserts into, languages whose output that changes,
53# grammars it only needs present)
54#
55# The first list decides where the extra sits in a language's entry: right after
56# the grammar it rewrites, because a language that extends that grammar clones it
57# when it loads (`typescript` copies `javascript`, so js-extras has to have run
58# first or the clone has no arrow token). The second list decides which entries
59# include it: a language that requires the rewritten grammar inherits the change,
60# and js-templates additionally rewrites the tokens of its own aliases. The third
61# is the grammars js-templates embeds inside a template string; they are imported
62# with it so what a page gets does not depend on what else the page read first.
63EXTRAS = {
64 "css-extras": (("css",), ("css",), ()),
65 "js-extras": (("javascript",), ("javascript", "js"), ()),
66 "js-templates": (("javascript",), ("javascript", "js", "typescript", "ts", "jsx", "tsx"),
67 ("markdown", "graphql", "sql")),
68 "php-extras": (("php",), ("php",), ()),
69 "xml-doc": (("csharp", "fsharp", "vbnet"), ("csharp", "fsharp", "vbnet"), ()),
70}
71
72STATIC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir,
73 "server", "web", "static")
74LANGUAGES_DIR = "prism-languages"
75
76HEADER = """/* Prism {version}, one file per language, loaded on demand.
77
78 The worker (prism-worker.js) imports prism.min.js — markup, css, clike and
79 javascript, Prism's default build — and then this manifest. A language the
80 core does not define is imported from prism-languages/ the first time a blob
81 or a fenced code block names it, so a page parses the grammar it shows
82 instead of all {components} components Prism ships.
83
84 Each entry is a language name to the component files that define it, in
85 dependency order: usually just itself, more when a grammar extends something
86 Prism keeps out of the core (php needs markup-templating, scala needs java,
87 arduino needs cpp). The files are Prism's own minified components, unmodified.
88
89 Vendored rather than fetched so an instance is still one file, and generated
90 rather than built: there is no npm build step in this repository. A language
91 added here also needs an extension in the server's table (highlight.go), and
92 TestPrismLanguagesAreVendored holds the two together. Regenerate with
93 scripts/vendor-prism.py. */
94
95"""
96
97
98def fetch(url):
99 request = urllib.request.Request(url, headers={"User-Agent": "vole-vendor-prism"})
100 with urllib.request.urlopen(request) as response:
101 return response.read()
102
103
104def component(name):
105 return CDN.format(version=PRISM_VERSION) + "/components/prism-" + name + ".min.js"
106
107
108def requirements(meta, name):
109 require = meta[name].get("require")
110 if not require:
111 return []
112 return [require] if isinstance(require, str) else list(require)
113
114
115def order_languages(meta):
116 """Return every non-core language id in dependency order.
117
118 Prism's metadata lists what a grammar needs; a language is emitted after its
119 requirements. A missing or circular requirement is a hard error: the manifest
120 would load but colour nothing, which is worse than failing here.
121 """
122 ids = [name for name in meta if name not in CORE_LANGUAGES]
123 known = set(ids) | set(CORE_LANGUAGES)
124 order, done = [], set()
125
126 def visit(name, stack):
127 if name in done:
128 return
129 if name in stack:
130 raise SystemExit("prism: dependency cycle: " + " -> ".join(stack + (name,)))
131 for dep in requirements(meta, name):
132 if dep not in known:
133 raise SystemExit(f"prism: {name} requires {dep}, which Prism does not define")
134 if dep not in CORE_LANGUAGES:
135 visit(dep, stack + (name,))
136 done.add(name)
137 order.append(name)
138
139 for name in ids:
140 visit(name, ())
141 return order
142
143
144def language_names(source):
145 """Every language name a component registers.
146
147 Prism components name their language as `Prism.languages.name=` or, for a
148 name that is not an identifier, `Prism.languages["name"]=`. Minified sources
149 write `x.languages.name=` against the Prism object, which is why the scan is
150 on `languages.` and not on `Prism.languages.`. An alias is registered the same
151 way in the component that owns the grammar (`languages.sh=languages.bash`),
152 so the component is still the unit the manifest names.
153 """
154 text = source.decode("utf-8")
155 names = set(re.findall(r"\blanguages\.([A-Za-z0-9_$]+)\s*=", text))
156 names.update(re.findall(r"\blanguages\[\s*[\"']([^\"']+)[\"']\s*\]\s*=", text))
157 return names
158
159
160def manifest_entries(meta, order, sources, core_names):
161 """Map every language name to the component files that define it.
162
163 A name can be registered by more than one component: one aliases it, another
164 extends it. The entry is the union of those components and their
165 requirements, in Prism's dependency order, so importing every file in it
166 always leaves the last definition of the name in place. Components that only
167 modify another grammar (EXTRAS) are placed right after the grammar they
168 rewrite and included in the entries that inherit the change. Languages that
169 are already in the core get an entry only when an extra changes them.
170 """
171 position = {name: i for i, name in enumerate(order)}
172 owners = {}
173 for name in order:
174 for language in language_names(sources[name]):
175 owners.setdefault(language, []).append(name)
176
177 def closure(names):
178 needed = set()
179 stack = list(names)
180 while stack:
181 current = stack.pop()
182 if current in needed or current in core_names:
183 continue
184 needed.add(current)
185 stack.extend(requirements(meta, current))
186 return sorted(needed, key=position.__getitem__)
187
188 def languages_of(components):
189 """Every language name this entry defines, requires, or is made of."""
190 names = set()
191 stack = list(components)
192 seen = set()
193 while stack:
194 current = stack.pop()
195 if current in seen:
196 continue
197 seen.add(current)
198 if current in sources:
199 names.update(language_names(sources[current]))
200 for dep in requirements(meta, current):
201 names.add(dep)
202 stack.append(dep)
203 return names
204
205 extras = {}
206 for extra, (mutates, affects, needs) in EXTRAS.items():
207 if extra not in meta:
208 raise SystemExit(f"prism: the extra {extra} is no longer a Prism component")
209 for name in mutates:
210 if name not in core_names and name not in position:
211 raise SystemExit(f"prism: the extra {extra} rewrites {name}, which nothing defines")
212 extras[extra] = (mutates, set(affects), closure(
213 name for need in needs for name in owners.get(need, ())))
214
215 entries = {}
216 for language in list(owners) + sorted(core_names):
217 components = set(closure(owners.get(language, ())))
218 names = languages_of(components) | {language}
219 placed = {name: position[name] * 100 for name in components}
220 for extra, (mutates, affects, needs) in extras.items():
221 if not affects & names:
222 continue
223 # After every grammar the extra rewrites, and before anything that
224 # loads later and could clone it. Core grammars are all in
225 # prism.min.js, so they sit before every component.
226 anchor = max(position.get(name, -1) * 100 for name in mutates)
227 # js-templates decides which languages it can embed inside a template
228 # string when it loads, so the grammars it needs go just before it.
229 start = anchor + 50 - len(needs)
230 for offset, name in enumerate(needs):
231 components.add(name)
232 placed[name] = start + offset
233 placed[extra] = anchor + 50
234 components.add(extra)
235 if not components:
236 continue
237 entries[language] = sorted(components, key=lambda name: (placed[name], name))
238 return entries
239
240
241def build():
242 meta = json.loads(fetch(RAW.format(version=PRISM_VERSION) + "/components.json"))["languages"]
243 meta.pop("meta", None)
244 order = order_languages(meta)
245
246 core = fetch(CDN.format(version=PRISM_VERSION) + "/prism.min.js")
247 license_text = fetch(RAW.format(version=PRISM_VERSION) + "/LICENSE")
248 sources = {name: fetch(component(name)) for name in order}
249
250 entries = manifest_entries(meta, order, sources, language_names(core))
251 # Only components some entry imports are vendored: one that registers no name
252 # and is required by nothing is dead weight Prism would ship and nobody would
253 # load. A component in that position is either an extra the table above has
254 # not been taught yet or a language that lost its last name, and both are
255 # worth stopping for.
256 used = sorted({name for files in entries.values() for name in files})
257 dangling = [name for name in order if name not in used]
258 if dangling:
259 raise SystemExit(
260 "prism: these components are not imported by any language entry: "
261 + ", ".join(dangling)
262 + "\nadd each to EXTRAS with the languages it modifies")
263
264 manifest = HEADER.format(version=PRISM_VERSION, components=len(used))
265 manifest += "self.PRISM_LANGUAGES = " + json.dumps(entries, sort_keys=True, separators=(",", ":")) + ";\n"
266
267 files = {
268 "prism.min.js": core,
269 "prism.LICENSE": license_text,
270 "prism-manifest.js": manifest.encode("utf-8"),
271 }
272 for name in used:
273 files[LANGUAGES_DIR + "/" + name + ".js"] = sources[name]
274 return files, len(entries), len(used)
275
276
277def stray_files(files):
278 """Files under prism-languages/ that the manifest no longer names."""
279 directory = os.path.join(STATIC_DIR, LANGUAGES_DIR)
280 if not os.path.isdir(directory):
281 return []
282 return [name for name in sorted(os.listdir(directory))
283 if LANGUAGES_DIR + "/" + name not in files]
284
285
286def check(files, languages, components):
287 stale = []
288 for name, want in files.items():
289 path = os.path.join(STATIC_DIR, name)
290 try:
291 with open(path, "rb") as handle:
292 have = handle.read()
293 except FileNotFoundError:
294 stale.append(f"{name} is missing")
295 continue
296 if have != want:
297 stale.append(f"{name} differs ({len(have)} bytes vendored, {len(want)} upstream)")
298 for name in stray_files(files):
299 stale.append(f"{LANGUAGES_DIR}/{name} is not named by the manifest")
300 if stale:
301 for line in stale:
302 print("stale:", line, file=sys.stderr)
303 print("run scripts/vendor-prism.py to regenerate", file=sys.stderr)
304 return 1
305 print(f"ok: Prism {PRISM_VERSION} assets match ({languages} languages, {components} components)")
306 return 0
307
308
309def write(files, languages, components):
310 for name, body in files.items():
311 path = os.path.join(STATIC_DIR, name)
312 os.makedirs(os.path.dirname(path), exist_ok=True)
313 with open(path, "wb") as handle:
314 handle.write(body)
315 print(f"wrote {os.path.relpath(path)} ({len(body)} bytes)")
316 for name in stray_files(files):
317 os.remove(os.path.join(STATIC_DIR, LANGUAGES_DIR, name))
318 print(f"removed {LANGUAGES_DIR}/{name}")
319 print(f"Prism {PRISM_VERSION}: {languages} languages from {components} components")
320 return 0
321
322
323def main():
324 parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
325 parser.add_argument("--check", action="store_true",
326 help="verify the committed files instead of writing them")
327 args = parser.parse_args()
328
329 files, languages, components = build()
330 if args.check:
331 return check(files, languages, components)
332 return write(files, languages, components)
333
334
335if __name__ == "__main__":
336 sys.exit(main())
337