vole

Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Code

395 commit(s) on main

Compare History

update readme Keenan Nemetz · 37m ago 32290114

history blame raw 14.1 KiB

1	#!/usr/bin/env python3
2	"""Regenerate the vendored Prism assets in server/web/static.
3
4	Vole's code browser highlights in the browser (server/web/static/prism-worker.js)
5	and names the language on the server (server/web/highlight.go). The grammars
6	themselves are Prism's, vendored so an instance is still one binary and there is
7	no npm step in this repository:
8
9	prism.min.js the core build: markup, css, clike, javascript
10	prism-manifest.js language name -> the files that define it, in order
11	prism-languages/<id>.js one Prism component per file, unmodified
12
13	The worker starts with the core and the manifest. When a blob or a fenced block
14	names a language the core does not define, the worker imports exactly that
15	language's files — usually one, more when the grammar extends something Prism
16	keeps out of the core (php needs markup-templating, scala needs java, arduino
17	needs cpp). So a page parses the grammar it shows rather than all of them, and
18	the manifest's lists are what make that one import.
19
20	Dependency order is Prism's, from the "require" graph in components.json: it is
21	not alphabetical, because markup-templating precedes php, c precedes cpp and zig,
22	java precedes scala, and css precedes scss. Cycles or unsatisfied requirements
23	are errors rather than a subtly broken grammar set.
24
25	scripts/vendor-prism.py write the files
26	scripts/vendor-prism.py --check compare, exit 1 if they differ
27
28	Bumping PRISM_VERSION is the whole upgrade: rerun, then run the tests, which hold
29	the server's extension table (highlight.go) to what the manifest can load.
30	"""
31
32	import argparse
33	import json
34	import os
35	import re
36	import sys
37	import urllib.request
38
39	PRISM_VERSION = "1.30.0"
40
41	CDN = "https://cdnjs.cloudflare.com/ajax/libs/prism/{version}"
42	RAW = "https://raw.githubusercontent.com/PrismJS/prism/v{version}"
43
44	# Languages that ship inside prism.min.js (Prism's "default" build). Every page
45	# pays for these, so they never appear as a manifest entry of their own.
46	CORE_LANGUAGES = ("markup", "css", "clike", "javascript")
47
48	# Components that change a grammar another component defines instead of
49	# registering a language name of their own, so no language would import them.
50	# components.json says what they require, not what they touch, hence the table:
51	#
52	# id: (grammars it inserts into, languages whose output that changes,
53	# grammars it only needs present)
54	#
55	# The first list decides where the extra sits in a language's entry: right after
56	# the grammar it rewrites, because a language that extends that grammar clones it
57	# when it loads (`typescript` copies `javascript`, so js-extras has to have run
58	# first or the clone has no arrow token). The second list decides which entries
59	# include it: a language that requires the rewritten grammar inherits the change,
60	# and js-templates additionally rewrites the tokens of its own aliases. The third
61	# is the grammars js-templates embeds inside a template string; they are imported
62	# with it so what a page gets does not depend on what else the page read first.
63	EXTRAS = {
64	"css-extras": (("css",), ("css",), ()),
65	"js-extras": (("javascript",), ("javascript", "js"), ()),
66	"js-templates": (("javascript",), ("javascript", "js", "typescript", "ts", "jsx", "tsx"),
67	("markdown", "graphql", "sql")),
68	"php-extras": (("php",), ("php",), ()),
69	"xml-doc": (("csharp", "fsharp", "vbnet"), ("csharp", "fsharp", "vbnet"), ()),
70	}
71
72	STATIC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir,
73	"server", "web", "static")
74	LANGUAGES_DIR = "prism-languages"
75
76	HEADER = """/* Prism {version}, one file per language, loaded on demand.
77
78	The worker (prism-worker.js) imports prism.min.js — markup, css, clike and
79	javascript, Prism's default build — and then this manifest. A language the
80	core does not define is imported from prism-languages/ the first time a blob
81	or a fenced code block names it, so a page parses the grammar it shows
82	instead of all {components} components Prism ships.
83
84	Each entry is a language name to the component files that define it, in
85	dependency order: usually just itself, more when a grammar extends something
86	Prism keeps out of the core (php needs markup-templating, scala needs java,
87	arduino needs cpp). The files are Prism's own minified components, unmodified.
88
89	Vendored rather than fetched so an instance is still one file, and generated
90	rather than built: there is no npm build step in this repository. A language
91	added here also needs an extension in the server's table (highlight.go), and
92	TestPrismLanguagesAreVendored holds the two together. Regenerate with
93	scripts/vendor-prism.py. */
94
95	"""
96
97
98	def fetch(url):
99	request = urllib.request.Request(url, headers={"User-Agent": "vole-vendor-prism"})
100	with urllib.request.urlopen(request) as response:
101	return response.read()
102
103
104	def component(name):
105	return CDN.format(version=PRISM_VERSION) + "/components/prism-" + name + ".min.js"
106
107
108	def requirements(meta, name):
109	require = meta[name].get("require")
110	if not require:
111	return []
112	return [require] if isinstance(require, str) else list(require)
113
114
115	def order_languages(meta):
116	"""Return every non-core language id in dependency order.
117
118	Prism's metadata lists what a grammar needs; a language is emitted after its
119	requirements. A missing or circular requirement is a hard error: the manifest
120	would load but colour nothing, which is worse than failing here.
121	"""
122	ids = [name for name in meta if name not in CORE_LANGUAGES]
123	known = set(ids) \| set(CORE_LANGUAGES)
124	order, done = [], set()
125
126	def visit(name, stack):
127	if name in done:
128	return
129	if name in stack:
130	raise SystemExit("prism: dependency cycle: " + " -> ".join(stack + (name,)))
131	for dep in requirements(meta, name):
132	if dep not in known:
133	raise SystemExit(f"prism: {name} requires {dep}, which Prism does not define")
134	if dep not in CORE_LANGUAGES:
135	visit(dep, stack + (name,))
136	done.add(name)
137	order.append(name)
138
139	for name in ids:
140	visit(name, ())
141	return order
142
143
144	def language_names(source):
145	"""Every language name a component registers.
146
147	Prism components name their language as `Prism.languages.name=` or, for a
148	name that is not an identifier, `Prism.languages["name"]=`. Minified sources
149	write `x.languages.name=` against the Prism object, which is why the scan is
150	on `languages.` and not on `Prism.languages.`. An alias is registered the same
151	way in the component that owns the grammar (`languages.sh=languages.bash`),
152	so the component is still the unit the manifest names.
153	"""
154	text = source.decode("utf-8")
155	names = set(re.findall(r"\blanguages\.([A-Za-z0-9_$]+)\s*=", text))
156	names.update(re.findall(r"\blanguages\[\s[\"']([^\"']+)[\"']\s\]\s*=", text))
157	return names
158
159
160	def manifest_entries(meta, order, sources, core_names):
161	"""Map every language name to the component files that define it.
162
163	A name can be registered by more than one component: one aliases it, another
164	extends it. The entry is the union of those components and their
165	requirements, in Prism's dependency order, so importing every file in it
166	always leaves the last definition of the name in place. Components that only
167	modify another grammar (EXTRAS) are placed right after the grammar they
168	rewrite and included in the entries that inherit the change. Languages that
169	are already in the core get an entry only when an extra changes them.
170	"""
171	position = {name: i for i, name in enumerate(order)}
172	owners = {}
173	for name in order:
174	for language in language_names(sources[name]):
175	owners.setdefault(language, []).append(name)
176
177	def closure(names):
178	needed = set()
179	stack = list(names)
180	while stack:
181	current = stack.pop()
182	if current in needed or current in core_names:
183	continue
184	needed.add(current)
185	stack.extend(requirements(meta, current))
186	return sorted(needed, key=position.__getitem__)
187
188	def languages_of(components):
189	"""Every language name this entry defines, requires, or is made of."""
190	names = set()
191	stack = list(components)
192	seen = set()
193	while stack:
194	current = stack.pop()
195	if current in seen:
196	continue
197	seen.add(current)
198	if current in sources:
199	names.update(language_names(sources[current]))
200	for dep in requirements(meta, current):
201	names.add(dep)
202	stack.append(dep)
203	return names
204
205	extras = {}
206	for extra, (mutates, affects, needs) in EXTRAS.items():
207	if extra not in meta:
208	raise SystemExit(f"prism: the extra {extra} is no longer a Prism component")
209	for name in mutates:
210	if name not in core_names and name not in position:
211	raise SystemExit(f"prism: the extra {extra} rewrites {name}, which nothing defines")
212	extras[extra] = (mutates, set(affects), closure(
213	name for need in needs for name in owners.get(need, ())))
214
215	entries = {}
216	for language in list(owners) + sorted(core_names):
217	components = set(closure(owners.get(language, ())))
218	names = languages_of(components) \| {language}
219	placed = {name: position[name] * 100 for name in components}
220	for extra, (mutates, affects, needs) in extras.items():
221	if not affects & names:
222	continue
223	# After every grammar the extra rewrites, and before anything that
224	# loads later and could clone it. Core grammars are all in
225	# prism.min.js, so they sit before every component.
226	anchor = max(position.get(name, -1) * 100 for name in mutates)
227	# js-templates decides which languages it can embed inside a template
228	# string when it loads, so the grammars it needs go just before it.
229	start = anchor + 50 - len(needs)
230	for offset, name in enumerate(needs):
231	components.add(name)
232	placed[name] = start + offset
233	placed[extra] = anchor + 50
234	components.add(extra)
235	if not components:
236	continue
237	entries[language] = sorted(components, key=lambda name: (placed[name], name))
238	return entries
239
240
241	def build():
242	meta = json.loads(fetch(RAW.format(version=PRISM_VERSION) + "/components.json"))["languages"]
243	meta.pop("meta", None)
244	order = order_languages(meta)
245
246	core = fetch(CDN.format(version=PRISM_VERSION) + "/prism.min.js")
247	license_text = fetch(RAW.format(version=PRISM_VERSION) + "/LICENSE")
248	sources = {name: fetch(component(name)) for name in order}
249
250	entries = manifest_entries(meta, order, sources, language_names(core))
251	# Only components some entry imports are vendored: one that registers no name
252	# and is required by nothing is dead weight Prism would ship and nobody would
253	# load. A component in that position is either an extra the table above has
254	# not been taught yet or a language that lost its last name, and both are
255	# worth stopping for.
256	used = sorted({name for files in entries.values() for name in files})
257	dangling = [name for name in order if name not in used]
258	if dangling:
259	raise SystemExit(
260	"prism: these components are not imported by any language entry: "
261	+ ", ".join(dangling)
262	+ "\nadd each to EXTRAS with the languages it modifies")
263
264	manifest = HEADER.format(version=PRISM_VERSION, components=len(used))
265	manifest += "self.PRISM_LANGUAGES = " + json.dumps(entries, sort_keys=True, separators=(",", ":")) + ";\n"
266
267	files = {
268	"prism.min.js": core,
269	"prism.LICENSE": license_text,
270	"prism-manifest.js": manifest.encode("utf-8"),
271	}
272	for name in used:
273	files[LANGUAGES_DIR + "/" + name + ".js"] = sources[name]
274	return files, len(entries), len(used)
275
276
277	def stray_files(files):
278	"""Files under prism-languages/ that the manifest no longer names."""
279	directory = os.path.join(STATIC_DIR, LANGUAGES_DIR)
280	if not os.path.isdir(directory):
281	return []
282	return [name for name in sorted(os.listdir(directory))
283	if LANGUAGES_DIR + "/" + name not in files]
284
285
286	def check(files, languages, components):
287	stale = []
288	for name, want in files.items():
289	path = os.path.join(STATIC_DIR, name)
290	try:
291	with open(path, "rb") as handle:
292	have = handle.read()
293	except FileNotFoundError:
294	stale.append(f"{name} is missing")
295	continue
296	if have != want:
297	stale.append(f"{name} differs ({len(have)} bytes vendored, {len(want)} upstream)")
298	for name in stray_files(files):
299	stale.append(f"{LANGUAGES_DIR}/{name} is not named by the manifest")
300	if stale:
301	for line in stale:
302	print("stale:", line, file=sys.stderr)
303	print("run scripts/vendor-prism.py to regenerate", file=sys.stderr)
304	return 1
305	print(f"ok: Prism {PRISM_VERSION} assets match ({languages} languages, {components} components)")
306	return 0
307
308
309	def write(files, languages, components):
310	for name, body in files.items():
311	path = os.path.join(STATIC_DIR, name)
312	os.makedirs(os.path.dirname(path), exist_ok=True)
313	with open(path, "wb") as handle:
314	handle.write(body)
315	print(f"wrote {os.path.relpath(path)} ({len(body)} bytes)")
316	for name in stray_files(files):
317	os.remove(os.path.join(STATIC_DIR, LANGUAGES_DIR, name))
318	print(f"removed {LANGUAGES_DIR}/{name}")
319	print(f"Prism {PRISM_VERSION}: {languages} languages from {components} components")
320	return 0
321
322
323	def main():
324	parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
325	parser.add_argument("--check", action="store_true",
326	help="verify the committed files instead of writing them")
327	args = parser.parse_args()
328
329	files, languages, components = build()
330	if args.check:
331	return check(files, languages, components)
332	return write(files, languages, components)
333
334
335	if __name__ == "__main__":
336	sys.exit(main())
337