vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Load a Prism grammar only when a blob names it
1b8675b19e70346d6d02d5fc0c10538f50bf94b2 · Keenan Nemetz · 9d ago · parent 11720250
Load a Prism grammar only when a blob names it The worker imported prism-languages.min.js — every grammar Prism ships — on every code page. That is 546 KB to evaluate before the first token, and a page showing one Go file paid it for all 293 components. Each component is now vendored as its own file, and prism-manifest.js maps a language name to the files that define it in dependency order. The worker starts with Prism's core and the manifest and imports a language's files the first time a job names it, remembering what it has run so a file is never imported twice. A Go blob loads 30 KB instead of 564 KB; a .tsx blob, the widest common entry at 8 files, loads 49 KB. The manifest also carries the components that register no language of their own but change one: css-extras, js-extras, js-templates, php-extras, xml-doc. Each is placed after the grammar it rewrites and included in the entries that inherit it, so a language that extends javascript now gets js-extras' arrow and maybe-class-name tokens too. The old bundle's order handed those to some inheriting languages and not others; nothing loses a token. scripts/vendor-prism.py writes the grammars and the manifest, and --check compares them byte for byte with cdnjs. TestPrismLanguagesAreVendored, TestPrismGrammarsMatchTheManifest and TestPrismWorkerLoadsGrammarsOnDemand hold the server's extension table, the vendored set and the worker's lazy import together.
server/web/highlight_test.go +98−20
| @@ -1,7 +1,9 @@ | |||
| 1 | 1 | package web | |
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | "encoding/json" | ||
| 4 | 5 | "os" | |
| 6 | "path/filepath" | ||
| 5 | 7 | "strings" | |
| 6 | 8 | "testing" | |
| 7 | 9 | ) | |
| @@ -59,33 +61,109 @@ func TestPrismWorkerDisablesPrismOwnMessageHandler(t *testing.T) { | |||
| 59 | 61 | } | |
| 60 | 62 | } | |
| 61 | 63 | ||
| 62 | // TestPrismLanguagesAreVendored holds the server's table to the worker's bundle. | ||
| 63 | // A name here with no grammar there colours nothing, and nothing fails at build | ||
| 64 | // time; the blob just quietly stays plain. | ||
| 64 | // A worker that goes back to importing every grammar up front still colours | ||
| 65 | // correctly, so nothing else fails: the page just pays 565 KB and a few hundred | ||
| 66 | // milliseconds again. This is the assertion that notices. | ||
| 67 | func TestPrismWorkerLoadsGrammarsOnDemand(t *testing.T) { | ||
| 68 | t.Parallel() | ||
| 69 | source, err := os.ReadFile("static/prism-worker.js") | ||
| 70 | if err != nil { | ||
| 71 | t.Fatal(err) | ||
| 72 | } | ||
| 73 | for _, want := range []string{ | ||
| 74 | "importScripts('prism.min.js', 'prism-manifest.js')", | ||
| 75 | "'prism-languages/' + name + '.js'", | ||
| 76 | } { | ||
| 77 | if !strings.Contains(string(source), want) { | ||
| 78 | t.Errorf("prism-worker.js no longer contains %q", want) | ||
| 79 | } | ||
| 80 | } | ||
| 81 | } | ||
| 82 | |||
| 83 | // readPrismManifest parses static/prism-manifest.js. The generator writes one | ||
| 84 | // assignment whose value is a JSON object, so decoding stops at the end of that | ||
| 85 | // object and the trailing semicolon is never read. | ||
| 86 | func readPrismManifest(t *testing.T) map[string][]string { | ||
| 87 | t.Helper() | ||
| 88 | source, err := os.ReadFile("static/prism-manifest.js") | ||
| 89 | if err != nil { | ||
| 90 | t.Fatalf("the Prism manifest is missing: %v", err) | ||
| 91 | } | ||
| 92 | const assignment = "self.PRISM_LANGUAGES = " | ||
| 93 | at := strings.Index(string(source), assignment) | ||
| 94 | if at < 0 { | ||
| 95 | t.Fatal("static/prism-manifest.js does not assign self.PRISM_LANGUAGES") | ||
| 96 | } | ||
| 97 | var languages map[string][]string | ||
| 98 | rest := strings.NewReader(string(source[at+len(assignment):])) | ||
| 99 | if err := json.NewDecoder(rest).Decode(&languages); err != nil { | ||
| 100 | t.Fatalf("static/prism-manifest.js is not a JSON object: %v", err) | ||
| 101 | } | ||
| 102 | return languages | ||
| 103 | } | ||
| 104 | |||
| 105 | // TestPrismLanguagesAreVendored holds the server's table to what the worker can | ||
| 106 | // load. A name here that neither the core nor the manifest defines colours | ||
| 107 | // nothing, and nothing fails at build time; the blob just quietly stays plain. | ||
| 108 | // | ||
| 109 | // The core covers the languages every Prism page starts with — markup, css, | ||
| 110 | // clike, javascript — and their aliases, so those have no files to import unless | ||
| 111 | // a component modifies them. | ||
| 65 | 112 | func TestPrismLanguagesAreVendored(t *testing.T) { | |
| 66 | 113 | t.Parallel() | |
| 67 | var sources []string | ||
| 68 | for _, path := range []string{"static/prism.min.js", "static/prism-languages.min.js"} { | ||
| 69 | data, err := os.ReadFile(path) | ||
| 70 | if err != nil { | ||
| 71 | t.Fatalf("vendored Prism is missing: %v", err) | ||
| 114 | core, err := os.ReadFile("static/prism.min.js") | ||
| 115 | if err != nil { | ||
| 116 | t.Fatalf("vendored Prism is missing: %v", err) | ||
| 117 | } | ||
| 118 | inCore := func(lang string) bool { | ||
| 119 | // Prism registers a language as `languages.name=` or, for a name with | ||
| 120 | // punctuation in it, `languages["name"]=`. | ||
| 121 | return strings.Contains(string(core), "languages."+lang+"=") || | ||
| 122 | strings.Contains(string(core), `languages["`+lang+`"]=`) | ||
| 123 | } | ||
| 124 | languages := readPrismManifest(t) | ||
| 125 | for name, lang := range prismLanguages { | ||
| 126 | if inCore(lang) { | ||
| 127 | continue | ||
| 128 | } | ||
| 129 | if _, ok := languages[lang]; !ok { | ||
| 130 | t.Errorf("%q is mapped to %q, which neither static/prism.min.js nor static/prism-manifest.js defines", name, lang) | ||
| 72 | 131 | } | |
| 73 | sources = append(sources, string(data)) | ||
| 74 | 132 | } | |
| 75 | defined := func(lang string) bool { | ||
| 76 | for _, src := range sources { | ||
| 77 | // Prism registers a language as `languages.name=` or, for a name with | ||
| 78 | // punctuation in it, `languages["name"]=`. | ||
| 79 | if strings.Contains(src, "languages."+lang+"=") || | ||
| 80 | strings.Contains(src, `languages["`+lang+`"]=`) { | ||
| 81 | return true | ||
| 133 | } | ||
| 134 | |||
| 135 | // TestPrismGrammarsMatchTheManifest keeps the manifest and the vendored files in | ||
| 136 | // step in both directions: a file it names but nobody wrote only fails in a | ||
| 137 | // browser, as an unhighlighted blob, and a file nobody names is left behind when | ||
| 138 | // Prism drops a language. scripts/vendor-prism.py --check says the same thing | ||
| 139 | // after a network round trip; this is the answer without one. | ||
| 140 | func TestPrismGrammarsMatchTheManifest(t *testing.T) { | ||
| 141 | t.Parallel() | ||
| 142 | languages := readPrismManifest(t) | ||
| 143 | named := map[string]bool{} | ||
| 144 | for lang, files := range languages { | ||
| 145 | if len(files) == 0 { | ||
| 146 | t.Errorf("%q is in the manifest with no files to import", lang) | ||
| 147 | } | ||
| 148 | for _, file := range files { | ||
| 149 | named[file] = true | ||
| 150 | if _, err := os.Stat(filepath.Join("static", "prism-languages", file+".js")); err != nil { | ||
| 151 | t.Errorf("%q imports %s, which is not vendored: %v", lang, file, err) | ||
| 82 | 152 | } | |
| 83 | 153 | } | |
| 84 | return false | ||
| 85 | 154 | } | |
| 86 | for name, lang := range prismLanguages { | ||
| 87 | if !defined(lang) { | ||
| 88 | t.Errorf("%q is mapped to %q, which static/prism-languages.min.js does not define", name, lang) | ||
| 155 | entries, err := os.ReadDir(filepath.Join("static", "prism-languages")) | ||
| 156 | if err != nil { | ||
| 157 | t.Fatalf("the Prism grammars are missing: %v", err) | ||
| 158 | } | ||
| 159 | for _, entry := range entries { | ||
| 160 | name, ok := strings.CutSuffix(entry.Name(), ".js") | ||
| 161 | if !ok || entry.IsDir() { | ||
| 162 | t.Errorf("static/prism-languages/%s is not a vendored grammar", entry.Name()) | ||
| 163 | continue | ||
| 164 | } | ||
| 165 | if !named[name] { | ||
| 166 | t.Errorf("static/prism-languages/%s is in no manifest entry", entry.Name()) | ||
| 89 | 167 | } | |
| 90 | 168 | } | |
| 91 | 169 | } | |