Skip to content
This repository was archived by the owner on Feb 7, 2024. It is now read-only.

Commit 1c0d415

Browse files
authoredDec 16, 2023
feat(twoslash): expose core module, independent from twoslash (#52)
1 parent 02e386a commit 1c0d415

File tree

10 files changed

+221
-183
lines changed

10 files changed

+221
-183
lines changed
 

‎docs/.vitepress/config.ts

+6
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,12 @@ export default defineConfig({
102102
options.transformers?.splice(options.transformers.indexOf(cleanup), 1)
103103
},
104104
},
105+
{
106+
name: 'shikiji:remove-escape',
107+
postprocess(code) {
108+
return code.replace(/\[\\\!code/g, '[!code')
109+
},
110+
},
105111
],
106112
},
107113

‎docs/guide/index.md

+3-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ outline: deep
1212

1313
No custom RegEx to maintain, no custom CSS to maintain, no custom HTML to maintain. And as your favorite languages and themes in VS Code evolve - your syntax highlighting will evolve too.
1414

15-
Shikiji is a ESM-rewrite of [Shiki](https://github.com/shikijs/shiki) with quite many improvements. We aim to [merge this project back to Shiki as a milestone update](https://github.com/shikijs/shiki/issues/510).
15+
Shikiji is a ESM-rewrite of [Shiki](https://github.com/shikijs/shiki) with quite many improvements. We aim to [merge this project back to Shiki as a milestone update](https://github.com/shikijs/shiki/issues/510). [Breaking changes from Shiki and the compatibility build](/guide/compat) if you are migrating.
1616

1717
About the name, <ruby text-lg text-brand-yellow>式<rt>shiki</rt></ruby><ruby text-lg text-brand-red>辞<rt>ji</rt></ruby> is a Japanese word meaing ["Ceremonial Speech"](https://jisho.org/word/%E5%BC%8F%E8%BE%9E). <ruby text-brand-yellow text-lg>式<rt>shiki</rt></ruby> is inherited from [shiki](https://github.com/shikijs/shiki) means ["Style"](https://jisho.org/word/%E5%BC%8F) and <ruby text-brand-red text-lg>辞<rt>ji</rt></ruby> means ["Word"](https://jisho.org/word/%E8%BE%9E).
1818

@@ -36,6 +36,8 @@ Here is a little playground for you to try out how Shikiji highlights your code.
3636

3737
<ShikijiMiniPlayground />
3838

39+
[Install Shikiji](/guide/install) to use it in your project.
40+
3941
## Who is using?
4042

4143
Projects that depend on Shikiji (sort alphabetically):

‎docs/packages/transformers.md

+11-2
Original file line numberDiff line numberDiff line change
@@ -36,15 +36,24 @@ Use `[!code ++]` and `[!code --]` to mark added and removed lines.
3636

3737
For example, the following code
3838

39+
````md
3940
```ts
4041
export function foo() {
41-
console.log('hewwo') // [!code --]
42-
console.log('hello') // [!code ++]
42+
console.log('hewwo') // [\!code --]
43+
console.log('hello') // [\!code ++]
4344
}
4445
```
46+
````
4547

4648
will be transformed to
4749

50+
```ts
51+
export function foo() {
52+
console.log('hewwo') // [!code --]
53+
console.log('hello') // [!code ++]
54+
}
55+
```
56+
4857
```html
4958
<!-- Output (stripped of `style` attributes for clarity) -->
5059
<pre class="shiki has-diff"> <!-- Notice `has-diff` -->

‎packages/shikiji-twoslash/build.config.ts

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { defineBuildConfig } from 'unbuild'
33
export default defineBuildConfig({
44
entries: [
55
'src/index.ts',
6+
'src/core.ts',
67
],
78
declaration: true,
89
rollup: {

‎packages/shikiji-twoslash/package.json

+8-1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@
2222
"types": "./dist/index.d.mts",
2323
"default": "./dist/index.mjs"
2424
},
25+
"./core": {
26+
"types": "./dist/core.d.mts",
27+
"default": "./dist/core.mjs"
28+
},
2529
"./style-rich.css": "./style-rich.css",
2630
"./style-classic.css": "./style-classic.css",
2731
"./*": "./dist/*"
@@ -31,6 +35,9 @@
3135
"types": "./dist/index.d.mts",
3236
"typesVersions": {
3337
"*": {
38+
"./core": [
39+
"./dist/core.d.mts"
40+
],
3441
"*": [
3542
"./dist/*",
3643
"./*"
@@ -49,7 +56,7 @@
4956
},
5057
"dependencies": {
5158
"@typescript/twoslash": "^3.2.4",
52-
"shikiji": "workspace:*"
59+
"shikiji-core": "workspace:*"
5360
},
5461
"devDependencies": {
5562
"@iconify-json/carbon": "^1.1.26",

‎packages/shikiji-twoslash/src/core.ts

+184
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
/**
2+
* This file is the core of the shikiji-twoslash package,
3+
* Decoupled from twoslash's implementation and allowing to introduce custom implementation or cache system.
4+
*/
5+
import type { twoslasher } from '@typescript/twoslash'
6+
import type { ShikijiTransformer } from 'shikiji-core'
7+
import type { Element, ElementContent, Text } from 'hast'
8+
import type { ModuleKind, ScriptTarget } from 'typescript'
9+
10+
import { addClassToHast } from 'shikiji-core'
11+
import { rendererClassic } from './renderer-classic'
12+
import type { TransformerTwoSlashOptions } from './types'
13+
14+
export * from './types'
15+
export * from './renderer-classic'
16+
export * from './renderer-rich'
17+
export * from './icons'
18+
19+
export function defaultTwoSlashOptions() {
20+
return {
21+
customTags: ['annotate', 'log', 'warn', 'error'],
22+
defaultCompilerOptions: {
23+
module: 99 satisfies ModuleKind.ESNext,
24+
target: 99 satisfies ScriptTarget.ESNext,
25+
},
26+
}
27+
}
28+
29+
export function createTransformer(runTwoslasher: typeof twoslasher) {
30+
return function transformerTwoSlash(options: TransformerTwoSlashOptions = {}): ShikijiTransformer {
31+
const {
32+
langs = ['ts', 'tsx'],
33+
twoslashOptions = defaultTwoSlashOptions(),
34+
langAlias = {
35+
typescript: 'ts',
36+
json5: 'json',
37+
yml: 'yaml',
38+
},
39+
explicitTrigger = false,
40+
renderer = rendererClassic(),
41+
throws = true,
42+
} = options
43+
const filter = options.filter || ((lang, _, options) => langs.includes(lang) && (!explicitTrigger || /\btwoslash\b/.test(options.meta?.__raw || '')))
44+
return {
45+
preprocess(code, shikijiOptions) {
46+
let lang = shikijiOptions.lang
47+
if (lang in langAlias)
48+
lang = langAlias[shikijiOptions.lang]
49+
50+
if (filter(lang, code, shikijiOptions)) {
51+
shikijiOptions.mergeWhitespaces = false
52+
const twoslash = runTwoslasher(code, lang, twoslashOptions)
53+
this.meta.twoslash = twoslash
54+
return twoslash.code
55+
}
56+
},
57+
pre(pre) {
58+
if (this.meta.twoslash)
59+
addClassToHast(pre, 'twoslash lsp')
60+
},
61+
code(codeEl) {
62+
const twoslash = this.meta.twoslash
63+
if (!twoslash)
64+
return
65+
66+
const insertAfterLine = (line: number, nodes: ElementContent[]) => {
67+
if (!nodes.length)
68+
return
69+
let index: number
70+
if (line >= this.lines.length) {
71+
index = codeEl.children.length
72+
}
73+
else {
74+
const lineEl = this.lines[line]
75+
index = codeEl.children.indexOf(lineEl)
76+
if (index === -1) {
77+
if (throws)
78+
throw new Error(`[shikiji-twoslash] Cannot find line ${line} in code element`)
79+
return
80+
}
81+
}
82+
83+
// If there is a newline after this line, remove it because we have the error element take place.
84+
const nodeAfter = codeEl.children[index + 1]
85+
if (nodeAfter && nodeAfter.type === 'text' && nodeAfter.value === '\n')
86+
codeEl.children.splice(index + 1, 1)
87+
codeEl.children.splice(index + 1, 0, ...nodes)
88+
}
89+
90+
const locateTextToken = (
91+
line: number,
92+
character: number,
93+
) => {
94+
const lineEl = this.lines[line]
95+
if (!lineEl) {
96+
if (throws)
97+
throw new Error(`[shikiji-twoslash] Cannot find line ${line} in code element`)
98+
}
99+
const textNodes = lineEl.children.flatMap(i => i.type === 'element' ? i.children || [] : []) as (Text | Element)[]
100+
let index = 0
101+
for (const token of textNodes) {
102+
if ('value' in token && typeof token.value === 'string')
103+
index += token.value.length
104+
105+
if (index > character)
106+
return token
107+
}
108+
if (throws)
109+
throw new Error(`[shikiji-twoslash] Cannot find token at L${line}:${character}`)
110+
}
111+
112+
const skipTokens = new Set<Element | Text>()
113+
114+
for (const error of twoslash.errors) {
115+
if (error.line == null || error.character == null)
116+
return
117+
const token = locateTextToken(error.line, error.character)
118+
if (!token)
119+
continue
120+
121+
skipTokens.add(token)
122+
123+
if (renderer.nodeError) {
124+
const clone = { ...token }
125+
Object.assign(token, renderer.nodeError.call(this, error, clone))
126+
}
127+
128+
if (renderer.lineError)
129+
insertAfterLine(error.line, renderer.lineError.call(this, error))
130+
}
131+
132+
for (const query of twoslash.queries) {
133+
if (query.kind === 'completions') {
134+
const token = locateTextToken(query.line - 1, query.offset)
135+
if (!token)
136+
throw new Error(`[shikiji-twoslash] Cannot find token at L${query.line}:${query.offset}`)
137+
skipTokens.add(token)
138+
139+
if (renderer.nodeCompletions) {
140+
const clone = { ...token }
141+
Object.assign(token, renderer.nodeCompletions.call(this, query, clone))
142+
}
143+
144+
if (renderer.lineCompletions)
145+
insertAfterLine(query.line, renderer.lineCompletions.call(this, query))
146+
}
147+
else if (query.kind === 'query') {
148+
const token = locateTextToken(query.line - 1, query.offset)
149+
if (!token)
150+
throw new Error(`[shikiji-twoslash] Cannot find token at L${query.line}:${query.offset}`)
151+
152+
skipTokens.add(token)
153+
154+
if (renderer.nodeQuery) {
155+
const clone = { ...token }
156+
Object.assign(token, renderer.nodeQuery.call(this, query, clone))
157+
}
158+
159+
if (renderer.lineQuery)
160+
insertAfterLine(query.line, renderer.lineQuery.call(this, query, token))
161+
}
162+
}
163+
164+
for (const info of twoslash.staticQuickInfos) {
165+
const token = locateTextToken(info.line, info.character)
166+
if (!token || token.type !== 'text')
167+
continue
168+
169+
// If it's already rendered as popup or error, skip it
170+
if (skipTokens.has(token))
171+
continue
172+
173+
const clone = { ...token }
174+
Object.assign(token, renderer.nodeStaticInfo.call(this, info, clone))
175+
}
176+
177+
if (renderer.lineCustomTag) {
178+
for (const tag of twoslash.tags)
179+
insertAfterLine(tag.line, renderer.lineCustomTag.call(this, tag))
180+
}
181+
},
182+
}
183+
}
184+
}

‎packages/shikiji-twoslash/src/icons.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,6 @@ import completionIcons from './icons-completions.json'
44
import tagIcons from './icons-tags.json'
55

66
export type CompletionItem = NonNullable<TwoSlashReturn['queries'][0]['completions']>[0]
7-
export const defaultCompletionIcons: Record<CompletionItem['kind'], Element | undefined> = completionIcons as any
87

8+
export const defaultCompletionIcons: Record<CompletionItem['kind'], Element | undefined> = completionIcons as any
99
export const defaultCustomTagIcons: Record<string, Element | undefined> = tagIcons as any
+3-174
Original file line numberDiff line numberDiff line change
@@ -1,177 +1,6 @@
11
import { twoslasher } from '@typescript/twoslash'
2-
import type { ShikijiTransformer } from 'shikiji'
3-
import { addClassToHast } from 'shikiji/core'
4-
import type { Element, ElementContent, Text } from 'hast'
5-
import type { ModuleKind, ScriptTarget } from 'typescript'
6-
import { rendererClassic } from './renderer-classic'
7-
import type { TransformerTwoSlashOptions } from './types'
2+
import { createTransformer } from './core'
83

9-
export * from './types'
10-
export * from './renderer-classic'
11-
export * from './renderer-rich'
12-
export * from './icons'
4+
export * from './core'
135

14-
export function defaultTwoSlashOptions() {
15-
return {
16-
customTags: ['annotate', 'log', 'warn', 'error'],
17-
defaultCompilerOptions: {
18-
module: 99 satisfies ModuleKind.ESNext,
19-
target: 99 satisfies ScriptTarget.ESNext,
20-
},
21-
}
22-
}
23-
24-
export function transformerTwoSlash(options: TransformerTwoSlashOptions = {}): ShikijiTransformer {
25-
const {
26-
langs = ['ts', 'tsx'],
27-
twoslashOptions = defaultTwoSlashOptions(),
28-
langAlias = {
29-
typescript: 'ts',
30-
json5: 'json',
31-
yml: 'yaml',
32-
},
33-
explicitTrigger = false,
34-
renderer = rendererClassic(),
35-
throws = true,
36-
} = options
37-
const filter = options.filter || ((lang, _, options) => langs.includes(lang) && (!explicitTrigger || /\btwoslash\b/.test(options.meta?.__raw || '')))
38-
return {
39-
preprocess(code, shikijiOptions) {
40-
let lang = shikijiOptions.lang
41-
if (lang in langAlias)
42-
lang = langAlias[shikijiOptions.lang]
43-
44-
if (filter(lang, code, shikijiOptions)) {
45-
shikijiOptions.mergeWhitespaces = false
46-
const twoslash = twoslasher(code, lang, twoslashOptions)
47-
this.meta.twoslash = twoslash
48-
return twoslash.code
49-
}
50-
},
51-
pre(pre) {
52-
if (this.meta.twoslash)
53-
addClassToHast(pre, 'twoslash lsp')
54-
},
55-
code(codeEl) {
56-
const twoslash = this.meta.twoslash
57-
if (!twoslash)
58-
return
59-
60-
const insertAfterLine = (line: number, nodes: ElementContent[]) => {
61-
if (!nodes.length)
62-
return
63-
let index: number
64-
if (line >= this.lines.length) {
65-
index = codeEl.children.length
66-
}
67-
else {
68-
const lineEl = this.lines[line]
69-
index = codeEl.children.indexOf(lineEl)
70-
if (index === -1) {
71-
if (throws)
72-
throw new Error(`[shikiji-twoslash] Cannot find line ${line} in code element`)
73-
return
74-
}
75-
}
76-
77-
// If there is a newline after this line, remove it because we have the error element take place.
78-
const nodeAfter = codeEl.children[index + 1]
79-
if (nodeAfter && nodeAfter.type === 'text' && nodeAfter.value === '\n')
80-
codeEl.children.splice(index + 1, 1)
81-
codeEl.children.splice(index + 1, 0, ...nodes)
82-
}
83-
84-
const locateTextToken = (
85-
line: number,
86-
character: number,
87-
) => {
88-
const lineEl = this.lines[line]
89-
if (!lineEl) {
90-
if (throws)
91-
throw new Error(`[shikiji-twoslash] Cannot find line ${line} in code element`)
92-
}
93-
const textNodes = lineEl.children.flatMap(i => i.type === 'element' ? i.children || [] : []) as (Text | Element)[]
94-
let index = 0
95-
for (const token of textNodes) {
96-
if ('value' in token && typeof token.value === 'string')
97-
index += token.value.length
98-
99-
if (index > character)
100-
return token
101-
}
102-
if (throws)
103-
throw new Error(`[shikiji-twoslash] Cannot find token at L${line}:${character}`)
104-
}
105-
106-
const skipTokens = new Set<Element | Text>()
107-
108-
for (const error of twoslash.errors) {
109-
if (error.line == null || error.character == null)
110-
return
111-
const token = locateTextToken(error.line, error.character)
112-
if (!token)
113-
continue
114-
115-
skipTokens.add(token)
116-
117-
if (renderer.nodeError) {
118-
const clone = { ...token }
119-
Object.assign(token, renderer.nodeError.call(this, error, clone))
120-
}
121-
122-
if (renderer.lineError)
123-
insertAfterLine(error.line, renderer.lineError.call(this, error))
124-
}
125-
126-
for (const query of twoslash.queries) {
127-
if (query.kind === 'completions') {
128-
const token = locateTextToken(query.line - 1, query.offset)
129-
if (!token)
130-
throw new Error(`[shikiji-twoslash] Cannot find token at L${query.line}:${query.offset}`)
131-
skipTokens.add(token)
132-
133-
if (renderer.nodeCompletions) {
134-
const clone = { ...token }
135-
Object.assign(token, renderer.nodeCompletions.call(this, query, clone))
136-
}
137-
138-
if (renderer.lineCompletions)
139-
insertAfterLine(query.line, renderer.lineCompletions.call(this, query))
140-
}
141-
else if (query.kind === 'query') {
142-
const token = locateTextToken(query.line - 1, query.offset)
143-
if (!token)
144-
throw new Error(`[shikiji-twoslash] Cannot find token at L${query.line}:${query.offset}`)
145-
146-
skipTokens.add(token)
147-
148-
if (renderer.nodeQuery) {
149-
const clone = { ...token }
150-
Object.assign(token, renderer.nodeQuery.call(this, query, clone))
151-
}
152-
153-
if (renderer.lineQuery)
154-
insertAfterLine(query.line, renderer.lineQuery.call(this, query, token))
155-
}
156-
}
157-
158-
for (const info of twoslash.staticQuickInfos) {
159-
const token = locateTextToken(info.line, info.character)
160-
if (!token || token.type !== 'text')
161-
continue
162-
163-
// If it's already rendered as popup or error, skip it
164-
if (skipTokens.has(token))
165-
continue
166-
167-
const clone = { ...token }
168-
Object.assign(token, renderer.nodeStaticInfo.call(this, info, clone))
169-
}
170-
171-
if (renderer.lineCustomTag) {
172-
for (const tag of twoslash.tags)
173-
insertAfterLine(tag.line, renderer.lineCustomTag.call(this, tag))
174-
}
175-
},
176-
}
177-
}
6+
export const transformerTwoSlash = createTransformer(twoslasher)

‎packages/shikiji-twoslash/src/types.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import type { TwoSlashOptions, TwoSlashReturn } from '@typescript/twoslash'
2-
import type { CodeToHastOptions, ShikijiTransformerContext } from 'shikiji'
2+
import type { CodeToHastOptions, ShikijiTransformerContext } from 'shikiji-core'
33
import type { Element, ElementContent, Text } from 'hast'
44

5-
declare module 'shikiji' {
5+
declare module 'shikiji-core' {
66
interface ShikijiTransformerContextMeta {
77
twoslash?: TwoSlashReturn
88
}

‎pnpm-lock.yaml

+2-2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)
This repository has been archived.