vole

Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Blame · scripts/vendor-prism.py

Line attribution on main · 336 line(s)

View file History

306f36fc Keenan Nemetz 12d ago	1	#!/usr/bin/env python3
	2	"""Regenerate the vendored Prism assets in server/web/static.
	3
	4	Vole's code browser highlights in the browser (server/web/static/prism-worker.js)
	5	and names the language on the server (server/web/highlight.go). The grammars
1b8675b1 Keenan Nemetz 8d ago	6	themselves are Prism's, vendored so an instance is still one binary and there is
1b8675b1 Keenan Nemetz 8d ago	7	no npm step in this repository:
306f36fc Keenan Nemetz 12d ago	8
1b8675b1 Keenan Nemetz 8d ago	9	prism.min.js the core build: markup, css, clike, javascript
	10	prism-manifest.js language name -> the files that define it, in order
	11	prism-languages/<id>.js one Prism component per file, unmodified
306f36fc Keenan Nemetz 12d ago	12
1b8675b1 Keenan Nemetz 8d ago	13	The worker starts with the core and the manifest. When a blob or a fenced block
	14	names a language the core does not define, the worker imports exactly that
	15	language's files — usually one, more when the grammar extends something Prism
	16	keeps out of the core (php needs markup-templating, scala needs java, arduino
	17	needs cpp). So a page parses the grammar it shows rather than all of them, and
	18	the manifest's lists are what make that one import.
	19
	20	Dependency order is Prism's, from the "require" graph in components.json: it is
	21	not alphabetical, because markup-templating precedes php, c precedes cpp and zig,
	22	java precedes scala, and css precedes scss. Cycles or unsatisfied requirements
	23	are errors rather than a subtly broken grammar set.
306f36fc Keenan Nemetz 12d ago	24
	25	scripts/vendor-prism.py write the files
	26	scripts/vendor-prism.py --check compare, exit 1 if they differ
	27
	28	Bumping PRISM_VERSION is the whole upgrade: rerun, then run the tests, which hold
1b8675b1 Keenan Nemetz 8d ago	29	the server's extension table (highlight.go) to what the manifest can load.
306f36fc Keenan Nemetz 12d ago	30	"""
	31
	32	import argparse
	33	import json
	34	import os
1b8675b1 Keenan Nemetz 8d ago	35	import re
306f36fc Keenan Nemetz 12d ago	36	import sys
	37	import urllib.request
	38
	39	PRISM_VERSION = "1.30.0"
	40
	41	CDN = "https://cdnjs.cloudflare.com/ajax/libs/prism/{version}"
	42	RAW = "https://raw.githubusercontent.com/PrismJS/prism/v{version}"
	43
1b8675b1 Keenan Nemetz 8d ago	44	# Languages that ship inside prism.min.js (Prism's "default" build). Every page
1b8675b1 Keenan Nemetz 8d ago	45	# pays for these, so they never appear as a manifest entry of their own.
306f36fc Keenan Nemetz 12d ago	46	CORE_LANGUAGES = ("markup", "css", "clike", "javascript")
306f36fc Keenan Nemetz 12d ago	47
1b8675b1 Keenan Nemetz 8d ago	48	# Components that change a grammar another component defines instead of
	49	# registering a language name of their own, so no language would import them.
	50	# components.json says what they require, not what they touch, hence the table:
	51	#
	52	# id: (grammars it inserts into, languages whose output that changes,
	53	# grammars it only needs present)
	54	#
	55	# The first list decides where the extra sits in a language's entry: right after
	56	# the grammar it rewrites, because a language that extends that grammar clones it
	57	# when it loads (`typescript` copies `javascript`, so js-extras has to have run
	58	# first or the clone has no arrow token). The second list decides which entries
	59	# include it: a language that requires the rewritten grammar inherits the change,
	60	# and js-templates additionally rewrites the tokens of its own aliases. The third
	61	# is the grammars js-templates embeds inside a template string; they are imported
	62	# with it so what a page gets does not depend on what else the page read first.
	63	EXTRAS = {
	64	"css-extras": (("css",), ("css",), ()),
	65	"js-extras": (("javascript",), ("javascript", "js"), ()),
	66	"js-templates": (("javascript",), ("javascript", "js", "typescript", "ts", "jsx", "tsx"),
	67	("markdown", "graphql", "sql")),
	68	"php-extras": (("php",), ("php",), ()),
	69	"xml-doc": (("csharp", "fsharp", "vbnet"), ("csharp", "fsharp", "vbnet"), ()),
	70	}
	71
306f36fc Keenan Nemetz 12d ago	72	STATIC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir,
306f36fc Keenan Nemetz 12d ago	73	"server", "web", "static")
1b8675b1 Keenan Nemetz 8d ago	74	LANGUAGES_DIR = "prism-languages"
	75
	76	HEADER = """/* Prism {version}, one file per language, loaded on demand.
306f36fc Keenan Nemetz 12d ago	77
1b8675b1 Keenan Nemetz 8d ago	78	The worker (prism-worker.js) imports prism.min.js — markup, css, clike and
	79	javascript, Prism's default build — and then this manifest. A language the
	80	core does not define is imported from prism-languages/ the first time a blob
	81	or a fenced code block names it, so a page parses the grammar it shows
	82	instead of all {components} components Prism ships.
306f36fc Keenan Nemetz 12d ago	83
1b8675b1 Keenan Nemetz 8d ago	84	Each entry is a language name to the component files that define it, in
	85	dependency order: usually just itself, more when a grammar extends something
	86	Prism keeps out of the core (php needs markup-templating, scala needs java,
	87	arduino needs cpp). The files are Prism's own minified components, unmodified.
	88
	89	Vendored rather than fetched so an instance is still one file, and generated
	90	rather than built: there is no npm build step in this repository. A language
	91	added here also needs an extension in the server's table (highlight.go), and
	92	TestPrismLanguagesAreVendored holds the two together. Regenerate with
	93	scripts/vendor-prism.py. */
306f36fc Keenan Nemetz 12d ago	94
	95	"""
	96
	97
	98	def fetch(url):
	99	request = urllib.request.Request(url, headers={"User-Agent": "vole-vendor-prism"})
	100	with urllib.request.urlopen(request) as response:
	101	return response.read()
	102
	103
	104	def component(name):
	105	return CDN.format(version=PRISM_VERSION) + "/components/prism-" + name + ".min.js"
	106
	107
	108	def requirements(meta, name):
	109	require = meta[name].get("require")
	110	if not require:
	111	return []
	112	return [require] if isinstance(require, str) else list(require)
	113
	114
	115	def order_languages(meta):
	116	"""Return every non-core language id in dependency order.
	117
	118	Prism's metadata lists what a grammar needs; a language is emitted after its
1b8675b1 Keenan Nemetz 8d ago	119	requirements. A missing or circular requirement is a hard error: the manifest
306f36fc Keenan Nemetz 12d ago	120	would load but colour nothing, which is worse than failing here.
	121	"""
	122	ids = [name for name in meta if name not in CORE_LANGUAGES]
	123	known = set(ids) \| set(CORE_LANGUAGES)
	124	order, done = [], set()
	125
	126	def visit(name, stack):
	127	if name in done:
	128	return
	129	if name in stack:
	130	raise SystemExit("prism: dependency cycle: " + " -> ".join(stack + (name,)))
	131	for dep in requirements(meta, name):
	132	if dep not in known:
	133	raise SystemExit(f"prism: {name} requires {dep}, which Prism does not define")
	134	if dep not in CORE_LANGUAGES:
	135	visit(dep, stack + (name,))
	136	done.add(name)
	137	order.append(name)
	138
	139	for name in ids:
	140	visit(name, ())
	141	return order
	142
	143
1b8675b1 Keenan Nemetz 8d ago	144	def language_names(source):
	145	"""Every language name a component registers.
	146
	147	Prism components name their language as `Prism.languages.name=` or, for a
	148	name that is not an identifier, `Prism.languages["name"]=`. Minified sources
	149	write `x.languages.name=` against the Prism object, which is why the scan is
	150	on `languages.` and not on `Prism.languages.`. An alias is registered the same
	151	way in the component that owns the grammar (`languages.sh=languages.bash`),
	152	so the component is still the unit the manifest names.
	153	"""
	154	text = source.decode("utf-8")
	155	names = set(re.findall(r"\blanguages\.([A-Za-z0-9_$]+)\s*=", text))
	156	names.update(re.findall(r"\blanguages\[\s[\"']([^\"']+)[\"']\s\]\s*=", text))
	157	return names
	158
	159
	160	def manifest_entries(meta, order, sources, core_names):
	161	"""Map every language name to the component files that define it.
	162
	163	A name can be registered by more than one component: one aliases it, another
	164	extends it. The entry is the union of those components and their
	165	requirements, in Prism's dependency order, so importing every file in it
	166	always leaves the last definition of the name in place. Components that only
	167	modify another grammar (EXTRAS) are placed right after the grammar they
	168	rewrite and included in the entries that inherit the change. Languages that
	169	are already in the core get an entry only when an extra changes them.
	170	"""
	171	position = {name: i for i, name in enumerate(order)}
	172	owners = {}
	173	for name in order:
	174	for language in language_names(sources[name]):
	175	owners.setdefault(language, []).append(name)
	176
	177	def closure(names):
	178	needed = set()
	179	stack = list(names)
	180	while stack:
	181	current = stack.pop()
	182	if current in needed or current in core_names:
	183	continue
	184	needed.add(current)
	185	stack.extend(requirements(meta, current))
	186	return sorted(needed, key=position.__getitem__)
	187
	188	def languages_of(components):
	189	"""Every language name this entry defines, requires, or is made of."""
	190	names = set()
	191	stack = list(components)
	192	seen = set()
	193	while stack:
	194	current = stack.pop()
	195	if current in seen:
	196	continue
	197	seen.add(current)
	198	if current in sources:
	199	names.update(language_names(sources[current]))
	200	for dep in requirements(meta, current):
	201	names.add(dep)
	202	stack.append(dep)
	203	return names
	204
	205	extras = {}
	206	for extra, (mutates, affects, needs) in EXTRAS.items():
	207	if extra not in meta:
208	raise SystemExit(f"prism: the extra {extra} is no longer a Prism component")
209	for name in mutates:
210	if name not in core_names and name not in position:
211	raise SystemExit(f"prism: the extra {extra} rewrites {name}, which nothing defines")
212	extras[extra] = (mutates, set(affects), closure(
213	name for need in needs for name in owners.get(need, ())))
214
215	entries = {}
216	for language in list(owners) + sorted(core_names):
217	components = set(closure(owners.get(language, ())))
218	names = languages_of(components) \| {language}
219	placed = {name: position[name] * 100 for name in components}
220	for extra, (mutates, affects, needs) in extras.items():
221	if not affects & names:
222	continue
223	# After every grammar the extra rewrites, and before anything that
224	# loads later and could clone it. Core grammars are all in
225	# prism.min.js, so they sit before every component.
226	anchor = max(position.get(name, -1) * 100 for name in mutates)
227	# js-templates decides which languages it can embed inside a template
228	# string when it loads, so the grammars it needs go just before it.
229	start = anchor + 50 - len(needs)
230	for offset, name in enumerate(needs):
231	components.add(name)
232	placed[name] = start + offset
233	placed[extra] = anchor + 50
234	components.add(extra)
235	if not components:
236	continue
237	entries[language] = sorted(components, key=lambda name: (placed[name], name))
238	return entries
239
240
306f36fc Keenan Nemetz 12d ago	241	def build():
	242	meta = json.loads(fetch(RAW.format(version=PRISM_VERSION) + "/components.json"))["languages"]
	243	meta.pop("meta", None)
1b8675b1 Keenan Nemetz 8d ago	244	order = order_languages(meta)
306f36fc Keenan Nemetz 12d ago	245
	246	core = fetch(CDN.format(version=PRISM_VERSION) + "/prism.min.js")
	247	license_text = fetch(RAW.format(version=PRISM_VERSION) + "/LICENSE")
1b8675b1 Keenan Nemetz 8d ago	248	sources = {name: fetch(component(name)) for name in order}
	249
	250	entries = manifest_entries(meta, order, sources, language_names(core))
	251	# Only components some entry imports are vendored: one that registers no name
	252	# and is required by nothing is dead weight Prism would ship and nobody would
	253	# load. A component in that position is either an extra the table above has
	254	# not been taught yet or a language that lost its last name, and both are
	255	# worth stopping for.
	256	used = sorted({name for files in entries.values() for name in files})
	257	dangling = [name for name in order if name not in used]
	258	if dangling:
	259	raise SystemExit(
	260	"prism: these components are not imported by any language entry: "
	261	+ ", ".join(dangling)
	262	+ "\nadd each to EXTRAS with the languages it modifies")
	263
	264	manifest = HEADER.format(version=PRISM_VERSION, components=len(used))
	265	manifest += "self.PRISM_LANGUAGES = " + json.dumps(entries, sort_keys=True, separators=(",", ":")) + ";\n"
306f36fc Keenan Nemetz 12d ago	266
	267	files = {
	268	"prism.min.js": core,
	269	"prism.LICENSE": license_text,
1b8675b1 Keenan Nemetz 8d ago	270	"prism-manifest.js": manifest.encode("utf-8"),
306f36fc Keenan Nemetz 12d ago	271	}
1b8675b1 Keenan Nemetz 8d ago	272	for name in used:
	273	files[LANGUAGES_DIR + "/" + name + ".js"] = sources[name]
	274	return files, len(entries), len(used)
	275
	276
	277	def stray_files(files):
	278	"""Files under prism-languages/ that the manifest no longer names."""
	279	directory = os.path.join(STATIC_DIR, LANGUAGES_DIR)
	280	if not os.path.isdir(directory):
	281	return []
	282	return [name for name in sorted(os.listdir(directory))
	283	if LANGUAGES_DIR + "/" + name not in files]
	284
	285
	286	def check(files, languages, components):
	287	stale = []
	288	for name, want in files.items():
	289	path = os.path.join(STATIC_DIR, name)
	290	try:
	291	with open(path, "rb") as handle:
	292	have = handle.read()
	293	except FileNotFoundError:
	294	stale.append(f"{name} is missing")
	295	continue
	296	if have != want:
	297	stale.append(f"{name} differs ({len(have)} bytes vendored, {len(want)} upstream)")
	298	for name in stray_files(files):
	299	stale.append(f"{LANGUAGES_DIR}/{name} is not named by the manifest")
	300	if stale:
	301	for line in stale:
	302	print("stale:", line, file=sys.stderr)
	303	print("run scripts/vendor-prism.py to regenerate", file=sys.stderr)
	304	return 1
	305	print(f"ok: Prism {PRISM_VERSION} assets match ({languages} languages, {components} components)")
	306	return 0
306f36fc Keenan Nemetz 12d ago	307
306f36fc Keenan Nemetz 12d ago	308
1b8675b1 Keenan Nemetz 8d ago	309	def write(files, languages, components):
306f36fc Keenan Nemetz 12d ago	310	for name, body in files.items():
306f36fc Keenan Nemetz 12d ago	311	path = os.path.join(STATIC_DIR, name)
1b8675b1 Keenan Nemetz 8d ago	312	os.makedirs(os.path.dirname(path), exist_ok=True)
306f36fc Keenan Nemetz 12d ago	313	with open(path, "wb") as handle:
	314	handle.write(body)
	315	print(f"wrote {os.path.relpath(path)} ({len(body)} bytes)")
1b8675b1 Keenan Nemetz 8d ago	316	for name in stray_files(files):
	317	os.remove(os.path.join(STATIC_DIR, LANGUAGES_DIR, name))
	318	print(f"removed {LANGUAGES_DIR}/{name}")
	319	print(f"Prism {PRISM_VERSION}: {languages} languages from {components} components")
306f36fc Keenan Nemetz 12d ago	320	return 0
	321
	322
1b8675b1 Keenan Nemetz 8d ago	323	def main():
	324	parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
	325	parser.add_argument("--check", action="store_true",
	326	help="verify the committed files instead of writing them")
	327	args = parser.parse_args()
	328
	329	files, languages, components = build()
	330	if args.check:
	331	return check(files, languages, components)
	332	return write(files, languages, components)
	333
	334
306f36fc Keenan Nemetz 12d ago	335	if __name__ == "__main__":
306f36fc Keenan Nemetz 12d ago	336	sys.exit(main())