Skip to content

Commit baa95ca

Browse files
RunDevelopmentRunDevelopment
authored
Keyman: Improved tokenization (#3224)
1 parent d04d166 commit baa95ca

15 files changed

+291
-173
lines changed

‎components/prism-keyman.js

Lines changed: 42 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,44 @@
11
Prism.languages.keyman = {
2-
'comment': /\bc\s.*/i,
3-
'function': /\[\s*(?:(?:ALT|CAPS|CTRL|LALT|LCTRL|NCAPS|RALT|RCTRL|SHIFT)\s+)*(?:[TKU]_[\w?]+|".+?"|'.+?')\s*\]/i, // virtual key
4-
'string': /("|').*?\1/,
5-
'bold': [ // header statements, system stores and variable system stores
6-
/&(?:baselayout|bitmap|capsalwaysoff|capsononly|copyright|ethnologuecode|hotkey|includecodes|keyboardversion|kmw_embedcss|kmw_embedjs|kmw_helpfile|kmw_helptext|kmw_rtl|language|layer|layoutfile|message|mnemoniclayout|name|oldcharposmatching|platform|shiftfreescaps|targets|version|visualkeyboard|windowslanguages)\b/i,
7-
/\b(?:bitmap|bitmaps|caps always off|caps on only|copyright|hotkey|language|layout|message|name|shift frees caps|version)\b/i
8-
],
9-
'keyword': /\b(?:any|baselayout|beep|call|context|deadkey|dk|if|index|layer|notany|nul|outs|platform|reset|return|save|set|store|use)\b/i, // rule keywords
10-
'atrule': /\b(?:ansi|begin|group|match|nomatch|unicode|using keys)\b/i, // structural keywords
11-
'number': /\b(?:U\+[\dA-F]+|d\d+|x[\da-f]+|\d+)\b/i, // U+####, x###, d### characters and numbers
12-
'operator': /[+>\\,()]/,
13-
'tag': /\$(?:keyman|keymanonly|keymanweb|kmfl|weaver):/i // prefixes
2+
'comment': {
3+
pattern: /\bc .*/i,
4+
greedy: true
5+
},
6+
'string': {
7+
pattern: /"[^"\r\n]*"|'[^'\r\n]*'/,
8+
greedy: true
9+
},
10+
'virtual-key': {
11+
pattern: /\[\s*(?:(?:ALT|CAPS|CTRL|LALT|LCTRL|NCAPS|RALT|RCTRL|SHIFT)\s+)*(?:[TKU]_[\w?]+|[A-E]\d\d?|"[^"\r\n]*"|'[^'\r\n]*')\s*\]/i,
12+
greedy: true,
13+
alias: 'function' // alias for styles
14+
},
15+
16+
// https://help.keyman.com/developer/language/guide/headers
17+
'header-keyword': {
18+
pattern: /&\w+/,
19+
alias: 'bold' // alias for styles
20+
},
21+
'header-statement': {
22+
pattern: /\b(?:bitmap|bitmaps|caps always off|caps on only|copyright|hotkey|language|layout|message|name|shift frees caps|version)\b/i,
23+
alias: 'bold' // alias for styles
24+
},
25+
26+
'rule-keyword': {
27+
pattern: /\b(?:any|baselayout|beep|call|context|deadkey|dk|if|index|layer|notany|nul|outs|platform|reset|return|save|set|store|use)\b/i,
28+
alias: 'keyword'
29+
},
30+
'structural-keyword': {
31+
pattern: /\b(?:ansi|begin|group|match|nomatch|unicode|using keys)\b/i,
32+
alias: 'keyword'
33+
},
34+
35+
'compile-target': {
36+
pattern: /\$(?:keyman|keymanonly|keymanweb|kmfl|weaver):/i,
37+
alias: 'property'
38+
},
39+
40+
// U+####, x###, d### characters and numbers
41+
'number': /\b(?:U\+[\dA-F]+|d\d+|x[\da-f]+|\d+)\b/i,
42+
'operator': /[+>\\$]|\.\./,
43+
'punctuation': /[()=,]/
1444
};

‎components/prism-keyman.min.js

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/languages/keyman/atrule_feature.test

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎tests/languages/keyman/bold_feature.test

Lines changed: 0 additions & 59 deletions
This file was deleted.

‎tests/languages/keyman/compile-target_feature.test

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
$keyman:
2+
$kmfl:
3+
$weaver:
4+
$keymanweb:
5+
$keymanonly:
6+
7+
c Note: return statement is only supported in .kmx in Keyman 13
8+
$keymanonly: + 'a' > 'a' return
9+
10+
----------------------------------------------------
11+
12+
[
13+
["compile-target", "$keyman:"],
14+
["compile-target", "$kmfl:"],
15+
["compile-target", "$weaver:"],
16+
["compile-target", "$keymanweb:"],
17+
["compile-target", "$keymanonly:"],
18+
19+
["comment", "c Note: return statement is only supported in .kmx in Keyman 13"],
20+
21+
["compile-target", "$keymanonly:"],
22+
["operator", "+"],
23+
["string", "'a'"],
24+
["operator", ">"],
25+
["string", "'a'"],
26+
["rule-keyword", "return"]
27+
]

‎tests/languages/keyman/function_feature.test

Lines changed: 0 additions & 39 deletions
This file was deleted.

‎tests/languages/keyman/header-keyword.test

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
&baselayout
2+
&bitmap
3+
&capsononly
4+
&capsalwaysoff
5+
&shiftfreescaps
6+
&copyright
7+
&ethnologuecode
8+
&hotkey
9+
&includecodes
10+
&keyboardversion
11+
&kmw_embedcss
12+
&kmw_embedjs
13+
&kmw_helpfile
14+
&kmw_helptext
15+
&kmw_rtl
16+
&language
17+
&layer
18+
&layoutfile
19+
&message
20+
&mnemoniclayout
21+
&name
22+
&oldcharposmatching
23+
&platform
24+
&targets
25+
&version
26+
&visualkeyboard
27+
&windowslanguages
28+
29+
----------------------------------------------------
30+
31+
[
32+
["header-keyword", "&baselayout"],
33+
["header-keyword", "&bitmap"],
34+
["header-keyword", "&capsononly"],
35+
["header-keyword", "&capsalwaysoff"],
36+
["header-keyword", "&shiftfreescaps"],
37+
["header-keyword", "&copyright"],
38+
["header-keyword", "&ethnologuecode"],
39+
["header-keyword", "&hotkey"],
40+
["header-keyword", "&includecodes"],
41+
["header-keyword", "&keyboardversion"],
42+
["header-keyword", "&kmw_embedcss"],
43+
["header-keyword", "&kmw_embedjs"],
44+
["header-keyword", "&kmw_helpfile"],
45+
["header-keyword", "&kmw_helptext"],
46+
["header-keyword", "&kmw_rtl"],
47+
["header-keyword", "&language"],
48+
["header-keyword", "&layer"],
49+
["header-keyword", "&layoutfile"],
50+
["header-keyword", "&message"],
51+
["header-keyword", "&mnemoniclayout"],
52+
["header-keyword", "&name"],
53+
["header-keyword", "&oldcharposmatching"],
54+
["header-keyword", "&platform"],
55+
["header-keyword", "&targets"],
56+
["header-keyword", "&version"],
57+
["header-keyword", "&visualkeyboard"],
58+
["header-keyword", "&windowslanguages"]
59+
]

‎tests/languages/keyman/header-statement.test

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
bitmap
2+
bitmaps
3+
caps on only
4+
caps always off
5+
shift frees caps
6+
copyright
7+
hotkey
8+
language
9+
layout
10+
message
11+
name
12+
version
13+
14+
----------------------------------------------------
15+
16+
[
17+
["header-statement", "bitmap"],
18+
["header-statement", "bitmaps"],
19+
["header-statement", "caps on only"],
20+
["header-statement", "caps always off"],
21+
["header-statement", "shift frees caps"],
22+
["header-statement", "copyright"],
23+
["header-statement", "hotkey"],
24+
["header-statement", "language"],
25+
["header-statement", "layout"],
26+
["header-statement", "message"],
27+
["header-statement", "name"],
28+
["header-statement", "version"]
29+
]

‎tests/languages/keyman/keyword_feature.test

Lines changed: 0 additions & 21 deletions
This file was deleted.

‎tests/languages/keyman/operator_feature.test

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
1-
+ >
2-
\ ,
3-
( )
1+
+ > \
2+
3+
$vowel_a
4+
'a' .. 'z'
45

56
----------------------------------------------------
67

78
[
8-
["operator", "+"], ["operator", ">"],
9-
["operator", "\\"], ["operator", ","],
10-
["operator", "("], ["operator", ")"]
9+
["operator", "+"], ["operator", ">"], ["operator", "\\"],
10+
11+
["operator", "$"], "vowel_a\r\n",
12+
["string", "'a'"], ["operator", ".."], ["string", "'z'"]
1113
]
1214

1315
----------------------------------------------------
1416

15-
Checks for all operators.
17+
Checks for all operators.

‎tests/languages/keyman/punctuation_feature.test

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
( )
2+
, =
3+
4+
----------------------------------------------------
5+
6+
[
7+
["punctuation", "("], ["punctuation", ")"],
8+
["punctuation", ","], ["punctuation", "="]
9+
]

0 commit comments

Comments
 (0)