Skip to content

Commit 4028520

Browse files
RunDevelopmentRunDevelopment
authoredJul 13, 2020
Core: Fixed greedy matching bug (#2032)
·
v1.30.0v1.21.0
1 parent ed8fff9 commit 4028520

File tree

7 files changed

+140
-148
lines changed

7 files changed

+140
-148
lines changed
 

‎components/prism-core.js

+42-31
Original file line numberDiff line numberDiff line change
@@ -715,12 +715,11 @@ _self.Prism = _;
715715
* @param {string | TokenStream} content See {@link Token#content content}
716716
* @param {string|string[]} [alias] The alias(es) of the token.
717717
* @param {string} [matchedStr=""] A copy of the full string this token was created from.
718-
* @param {boolean} [greedy=false] Whether the pattern that created this token is greedy or not. Will be removed soon.
719718
* @class
720719
* @global
721720
* @public
722721
*/
723-
function Token(type, content, alias, matchedStr, greedy) {
722+
function Token(type, content, alias, matchedStr) {
724723
/**
725724
* The type of the token.
726725
*
@@ -748,8 +747,8 @@ function Token(type, content, alias, matchedStr, greedy) {
748747
* @public
749748
*/
750749
this.alias = alias;
751-
this.length = (matchedStr || "").length|0;
752-
this.greedy = !!greedy;
750+
// Copy of the full string this token was created from
751+
this.length = (matchedStr || '').length | 0;
753752
}
754753

755754
/**
@@ -826,11 +825,15 @@ Token.stringify = function stringify(o, language) {
826825
* @param {any} grammar
827826
* @param {LinkedListNode<string | Token>} startNode
828827
* @param {number} startPos
829-
* @param {boolean} [oneshot=false]
830-
* @param {string} [target]
828+
* @param {RematchOptions} [rematch]
829+
* @returns {void}
831830
* @private
831+
*
832+
* @typedef RematchOptions
833+
* @property {string} cause
834+
* @property {number} reach
832835
*/
833-
function matchGrammar(text, tokenList, grammar, startNode, startPos, oneshot, target) {
836+
function matchGrammar(text, tokenList, grammar, startNode, startPos, rematch) {
834837
for (var token in grammar) {
835838
if (!grammar.hasOwnProperty(token) || !grammar[token]) {
836839
continue;
@@ -840,31 +843,36 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, oneshot, ta
840843
patterns = Array.isArray(patterns) ? patterns : [patterns];
841844

842845
for (var j = 0; j < patterns.length; ++j) {
843-
if (target && target == token + ',' + j) {
846+
if (rematch && rematch.cause == token + ',' + j) {
844847
return;
845848
}
846849

847-
var pattern = patterns[j],
848-
inside = pattern.inside,
849-
lookbehind = !!pattern.lookbehind,
850-
greedy = !!pattern.greedy,
850+
var patternObj = patterns[j],
851+
inside = patternObj.inside,
852+
lookbehind = !!patternObj.lookbehind,
853+
greedy = !!patternObj.greedy,
851854
lookbehindLength = 0,
852-
alias = pattern.alias;
855+
alias = patternObj.alias;
853856

854-
if (greedy && !pattern.pattern.global) {
857+
if (greedy && !patternObj.pattern.global) {
855858
// Without the global flag, lastIndex won't work
856-
var flags = pattern.pattern.toString().match(/[imsuy]*$/)[0];
857-
pattern.pattern = RegExp(pattern.pattern.source, flags + 'g');
859+
var flags = patternObj.pattern.toString().match(/[imsuy]*$/)[0];
860+
patternObj.pattern = RegExp(patternObj.pattern.source, flags + 'g');
858861
}
859862

860-
pattern = pattern.pattern || pattern;
863+
/** @type {RegExp} */
864+
var pattern = patternObj.pattern || patternObj;
861865

862866
for ( // iterate the token list and keep track of the current token/string position
863867
var currentNode = startNode.next, pos = startPos;
864868
currentNode !== tokenList.tail;
865869
pos += currentNode.value.length, currentNode = currentNode.next
866870
) {
867871

872+
if (rematch && pos >= rematch.reach) {
873+
break;
874+
}
875+
868876
var str = currentNode.value;
869877

870878
if (tokenList.length > text.length) {
@@ -907,7 +915,7 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, oneshot, ta
907915
// find the last node which is affected by this match
908916
for (
909917
var k = currentNode;
910-
k !== tokenList.tail && (p < to || (typeof k.value === 'string' && !k.prev.value.greedy));
918+
k !== tokenList.tail && (p < to || typeof k.value === 'string');
911919
k = k.next
912920
) {
913921
removeCount++;
@@ -925,10 +933,6 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, oneshot, ta
925933
}
926934

927935
if (!match) {
928-
if (oneshot) {
929-
break;
930-
}
931-
932936
continue;
933937
}
934938

@@ -937,11 +941,16 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, oneshot, ta
937941
}
938942

939943
var from = match.index + lookbehindLength,
940-
match = match[0].slice(lookbehindLength),
941-
to = from + match.length,
944+
matchStr = match[0].slice(lookbehindLength),
945+
to = from + matchStr.length,
942946
before = str.slice(0, from),
943947
after = str.slice(to);
944948

949+
var reach = pos + str.length;
950+
if (rematch && reach > rematch.reach) {
951+
rematch.reach = reach;
952+
}
953+
945954
var removeFrom = currentNode.prev;
946955

947956
if (before) {
@@ -951,19 +960,21 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, oneshot, ta
951960

952961
removeRange(tokenList, removeFrom, removeCount);
953962

954-
var wrapped = new Token(token, inside ? _.tokenize(match, inside) : match, alias, match, greedy);
963+
var wrapped = new Token(token, inside ? _.tokenize(matchStr, inside) : matchStr, alias, matchStr);
955964
currentNode = addAfter(tokenList, removeFrom, wrapped);
956965

957966
if (after) {
958967
addAfter(tokenList, currentNode, after);
959968
}
960969

961-
962-
if (removeCount > 1)
963-
matchGrammar(text, tokenList, grammar, currentNode.prev, pos, true, token + ',' + j);
964-
965-
if (oneshot)
966-
break;
970+
if (removeCount > 1) {
971+
// at least one Token object was removed, so we have to do some rematching
972+
// this can only happen if the current pattern is greedy
973+
matchGrammar(text, tokenList, grammar, currentNode.prev, pos, {
974+
cause: token + ',' + j,
975+
reach: reach
976+
});
977+
}
967978
}
968979
}
969980
}

‎components/prism-core.min.js

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/Token.html

+5-44
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ <h2>
6868

6969

7070

71-
<h4 class="name" id="Token"><span class="type-signature"></span>new Token<span class="signature">(type, content, alias<span class="signature-attributes">opt</span>, matchedStr<span class="signature-attributes">opt</span>, greedy<span class="signature-attributes">opt</span>)</span><span class="type-signature"></span></h4>
71+
<h4 class="name" id="Token"><span class="type-signature"></span>new Token<span class="signature">(type, content, alias<span class="signature-attributes">opt</span>, matchedStr<span class="signature-attributes">opt</span>)</span><span class="type-signature"></span></h4>
7272

7373

7474

@@ -80,7 +80,7 @@ <h4 class="name" id="Token"><span class="type-signature"></span>new Token<span c
8080

8181
<dt class="tag-source">Source:</dt>
8282
<dd class="tag-source"><ul class="dummy"><li>
83-
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line723">line 723</a>
83+
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line722">line 722</a>
8484
</li></ul></dd>
8585

8686

@@ -311,45 +311,6 @@ <h5>Parameters:</h5>
311311
</tr>
312312

313313

314-
315-
<tr>
316-
317-
<td class="name"><code>greedy</code></td>
318-
319-
320-
<td class="type">
321-
322-
323-
<span class="param-type">boolean</span>
324-
325-
326-
327-
</td>
328-
329-
330-
<td class="attributes">
331-
332-
&lt;optional><br>
333-
334-
335-
336-
337-
338-
</td>
339-
340-
341-
342-
<td class="default">
343-
344-
<code>false</code>
345-
346-
</td>
347-
348-
349-
<td class="description last"><p>Whether the pattern that created this token is greedy or not. Will be removed soon.</p></td>
350-
</tr>
351-
352-
353314
</tbody>
354315
</table>
355316

@@ -403,7 +364,7 @@ <h4 class="name" id="alias"><span class="type-signature"></span>alias<span class
403364

404365
<dt class="tag-source">Source:</dt>
405366
<dd class="tag-source"><ul class="dummy"><li>
406-
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line750">line 750</a>
367+
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line749">line 749</a>
407368
</li></ul></dd>
408369

409370

@@ -486,7 +447,7 @@ <h4 class="name" id="content"><span class="type-signature"></span>content<span c
486447

487448
<dt class="tag-source">Source:</dt>
488449
<dd class="tag-source"><ul class="dummy"><li>
489-
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line742">line 742</a>
450+
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line741">line 741</a>
490451
</li></ul></dd>
491452

492453

@@ -563,7 +524,7 @@ <h4 class="name" id="type"><span class="type-signature"></span>type<span class="
563524

564525
<dt class="tag-source">Source:</dt>
565526
<dd class="tag-source"><ul class="dummy"><li>
566-
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line733">line 733</a>
527+
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line732">line 732</a>
567528
</li></ul></dd>
568529

569530

‎docs/global.html

+5-5
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ <h4 class="name" id="Grammar">Grammar</h4>
143143

144144
<dt class="tag-source">Source:</dt>
145145
<dd class="tag-source"><ul class="dummy"><li>
146-
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1146">line 1146</a>
146+
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1157">line 1157</a>
147147
</li></ul></dd>
148148

149149

@@ -274,7 +274,7 @@ <h4 class="name" id="GrammarToken">GrammarToken</h4>
274274

275275
<dt class="tag-source">Source:</dt>
276276
<dd class="tag-source"><ul class="dummy"><li>
277-
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1125">line 1125</a>
277+
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1136">line 1136</a>
278278
</li></ul></dd>
279279

280280

@@ -559,7 +559,7 @@ <h4 class="name" id="HighlightCallback"><span class="type-signature"></span>High
559559

560560
<dt class="tag-source">Source:</dt>
561561
<dd class="tag-source"><ul class="dummy"><li>
562-
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1154">line 1154</a>
562+
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1165">line 1165</a>
563563
</li></ul></dd>
564564

565565

@@ -713,7 +713,7 @@ <h4 class="name" id="HookCallback"><span class="type-signature"></span>HookCallb
713713

714714
<dt class="tag-source">Source:</dt>
715715
<dd class="tag-source"><ul class="dummy"><li>
716-
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1164">line 1164</a>
716+
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1175">line 1175</a>
717717
</li></ul></dd>
718718

719719

@@ -859,7 +859,7 @@ <h4 class="name" id="TokenStream">TokenStream</h4>
859859

860860
<dt class="tag-source">Source:</dt>
861861
<dd class="tag-source"><ul class="dummy"><li>
862-
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line755">line 755</a>
862+
<a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line754">line 754</a>
863863
</li></ul></dd>
864864

865865

‎docs/prism-core.js.html

+42-31
Original file line numberDiff line numberDiff line change
@@ -768,12 +768,11 @@ <h1 class="page-title">prism-core.js</h1>
768768
* @param {string | TokenStream} content See {@link Token#content content}
769769
* @param {string|string[]} [alias] The alias(es) of the token.
770770
* @param {string} [matchedStr=""] A copy of the full string this token was created from.
771-
* @param {boolean} [greedy=false] Whether the pattern that created this token is greedy or not. Will be removed soon.
772771
* @class
773772
* @global
774773
* @public
775774
*/
776-
function Token(type, content, alias, matchedStr, greedy) {
775+
function Token(type, content, alias, matchedStr) {
777776
/**
778777
* The type of the token.
779778
*
@@ -801,8 +800,8 @@ <h1 class="page-title">prism-core.js</h1>
801800
* @public
802801
*/
803802
this.alias = alias;
804-
this.length = (matchedStr || "").length|0;
805-
this.greedy = !!greedy;
803+
// Copy of the full string this token was created from
804+
this.length = (matchedStr || '').length | 0;
806805
}
807806

808807
/**
@@ -879,11 +878,15 @@ <h1 class="page-title">prism-core.js</h1>
879878
* @param {any} grammar
880879
* @param {LinkedListNode&lt;string | Token>} startNode
881880
* @param {number} startPos
882-
* @param {boolean} [oneshot=false]
883-
* @param {string} [target]
881+
* @param {RematchOptions} [rematch]
882+
* @returns {void}
884883
* @private
884+
*
885+
* @typedef RematchOptions
886+
* @property {string} cause
887+
* @property {number} reach
885888
*/
886-
function matchGrammar(text, tokenList, grammar, startNode, startPos, oneshot, target) {
889+
function matchGrammar(text, tokenList, grammar, startNode, startPos, rematch) {
887890
for (var token in grammar) {
888891
if (!grammar.hasOwnProperty(token) || !grammar[token]) {
889892
continue;
@@ -893,31 +896,36 @@ <h1 class="page-title">prism-core.js</h1>
893896
patterns = Array.isArray(patterns) ? patterns : [patterns];
894897

895898
for (var j = 0; j &lt; patterns.length; ++j) {
896-
if (target &amp;&amp; target == token + ',' + j) {
899+
if (rematch &amp;&amp; rematch.cause == token + ',' + j) {
897900
return;
898901
}
899902

900-
var pattern = patterns[j],
901-
inside = pattern.inside,
902-
lookbehind = !!pattern.lookbehind,
903-
greedy = !!pattern.greedy,
903+
var patternObj = patterns[j],
904+
inside = patternObj.inside,
905+
lookbehind = !!patternObj.lookbehind,
906+
greedy = !!patternObj.greedy,
904907
lookbehindLength = 0,
905-
alias = pattern.alias;
908+
alias = patternObj.alias;
906909

907-
if (greedy &amp;&amp; !pattern.pattern.global) {
910+
if (greedy &amp;&amp; !patternObj.pattern.global) {
908911
// Without the global flag, lastIndex won't work
909-
var flags = pattern.pattern.toString().match(/[imsuy]*$/)[0];
910-
pattern.pattern = RegExp(pattern.pattern.source, flags + 'g');
912+
var flags = patternObj.pattern.toString().match(/[imsuy]*$/)[0];
913+
patternObj.pattern = RegExp(patternObj.pattern.source, flags + 'g');
911914
}
912915

913-
pattern = pattern.pattern || pattern;
916+
/** @type {RegExp} */
917+
var pattern = patternObj.pattern || patternObj;
914918

915919
for ( // iterate the token list and keep track of the current token/string position
916920
var currentNode = startNode.next, pos = startPos;
917921
currentNode !== tokenList.tail;
918922
pos += currentNode.value.length, currentNode = currentNode.next
919923
) {
920924

925+
if (rematch &amp;&amp; pos >= rematch.reach) {
926+
break;
927+
}
928+
921929
var str = currentNode.value;
922930

923931
if (tokenList.length > text.length) {
@@ -960,7 +968,7 @@ <h1 class="page-title">prism-core.js</h1>
960968
// find the last node which is affected by this match
961969
for (
962970
var k = currentNode;
963-
k !== tokenList.tail &amp;&amp; (p &lt; to || (typeof k.value === 'string' &amp;&amp; !k.prev.value.greedy));
971+
k !== tokenList.tail &amp;&amp; (p &lt; to || typeof k.value === 'string');
964972
k = k.next
965973
) {
966974
removeCount++;
@@ -978,10 +986,6 @@ <h1 class="page-title">prism-core.js</h1>
978986
}
979987

980988
if (!match) {
981-
if (oneshot) {
982-
break;
983-
}
984-
985989
continue;
986990
}
987991

@@ -990,11 +994,16 @@ <h1 class="page-title">prism-core.js</h1>
990994
}
991995

992996
var from = match.index + lookbehindLength,
993-
match = match[0].slice(lookbehindLength),
994-
to = from + match.length,
997+
matchStr = match[0].slice(lookbehindLength),
998+
to = from + matchStr.length,
995999
before = str.slice(0, from),
9961000
after = str.slice(to);
9971001

1002+
var reach = pos + str.length;
1003+
if (rematch &amp;&amp; reach > rematch.reach) {
1004+
rematch.reach = reach;
1005+
}
1006+
9981007
var removeFrom = currentNode.prev;
9991008

10001009
if (before) {
@@ -1004,19 +1013,21 @@ <h1 class="page-title">prism-core.js</h1>
10041013

10051014
removeRange(tokenList, removeFrom, removeCount);
10061015

1007-
var wrapped = new Token(token, inside ? _.tokenize(match, inside) : match, alias, match, greedy);
1016+
var wrapped = new Token(token, inside ? _.tokenize(matchStr, inside) : matchStr, alias, matchStr);
10081017
currentNode = addAfter(tokenList, removeFrom, wrapped);
10091018

10101019
if (after) {
10111020
addAfter(tokenList, currentNode, after);
10121021
}
10131022

1014-
1015-
if (removeCount > 1)
1016-
matchGrammar(text, tokenList, grammar, currentNode.prev, pos, true, token + ',' + j);
1017-
1018-
if (oneshot)
1019-
break;
1023+
if (removeCount > 1) {
1024+
// at least one Token object was removed, so we have to do some rematching
1025+
// this can only happen if the current pattern is greedy
1026+
matchGrammar(text, tokenList, grammar, currentNode.prev, pos, {
1027+
cause: token + ',' + j,
1028+
reach: reach
1029+
});
1030+
}
10201031
}
10211032
}
10221033
}

‎prism.js

+42-31
Original file line numberDiff line numberDiff line change
@@ -720,12 +720,11 @@ _self.Prism = _;
720720
* @param {string | TokenStream} content See {@link Token#content content}
721721
* @param {string|string[]} [alias] The alias(es) of the token.
722722
* @param {string} [matchedStr=""] A copy of the full string this token was created from.
723-
* @param {boolean} [greedy=false] Whether the pattern that created this token is greedy or not. Will be removed soon.
724723
* @class
725724
* @global
726725
* @public
727726
*/
728-
function Token(type, content, alias, matchedStr, greedy) {
727+
function Token(type, content, alias, matchedStr) {
729728
/**
730729
* The type of the token.
731730
*
@@ -753,8 +752,8 @@ function Token(type, content, alias, matchedStr, greedy) {
753752
* @public
754753
*/
755754
this.alias = alias;
756-
this.length = (matchedStr || "").length|0;
757-
this.greedy = !!greedy;
755+
// Copy of the full string this token was created from
756+
this.length = (matchedStr || '').length | 0;
758757
}
759758

760759
/**
@@ -831,11 +830,15 @@ Token.stringify = function stringify(o, language) {
831830
* @param {any} grammar
832831
* @param {LinkedListNode<string | Token>} startNode
833832
* @param {number} startPos
834-
* @param {boolean} [oneshot=false]
835-
* @param {string} [target]
833+
* @param {RematchOptions} [rematch]
834+
* @returns {void}
836835
* @private
836+
*
837+
* @typedef RematchOptions
838+
* @property {string} cause
839+
* @property {number} reach
837840
*/
838-
function matchGrammar(text, tokenList, grammar, startNode, startPos, oneshot, target) {
841+
function matchGrammar(text, tokenList, grammar, startNode, startPos, rematch) {
839842
for (var token in grammar) {
840843
if (!grammar.hasOwnProperty(token) || !grammar[token]) {
841844
continue;
@@ -845,31 +848,36 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, oneshot, ta
845848
patterns = Array.isArray(patterns) ? patterns : [patterns];
846849

847850
for (var j = 0; j < patterns.length; ++j) {
848-
if (target && target == token + ',' + j) {
851+
if (rematch && rematch.cause == token + ',' + j) {
849852
return;
850853
}
851854

852-
var pattern = patterns[j],
853-
inside = pattern.inside,
854-
lookbehind = !!pattern.lookbehind,
855-
greedy = !!pattern.greedy,
855+
var patternObj = patterns[j],
856+
inside = patternObj.inside,
857+
lookbehind = !!patternObj.lookbehind,
858+
greedy = !!patternObj.greedy,
856859
lookbehindLength = 0,
857-
alias = pattern.alias;
860+
alias = patternObj.alias;
858861

859-
if (greedy && !pattern.pattern.global) {
862+
if (greedy && !patternObj.pattern.global) {
860863
// Without the global flag, lastIndex won't work
861-
var flags = pattern.pattern.toString().match(/[imsuy]*$/)[0];
862-
pattern.pattern = RegExp(pattern.pattern.source, flags + 'g');
864+
var flags = patternObj.pattern.toString().match(/[imsuy]*$/)[0];
865+
patternObj.pattern = RegExp(patternObj.pattern.source, flags + 'g');
863866
}
864867

865-
pattern = pattern.pattern || pattern;
868+
/** @type {RegExp} */
869+
var pattern = patternObj.pattern || patternObj;
866870

867871
for ( // iterate the token list and keep track of the current token/string position
868872
var currentNode = startNode.next, pos = startPos;
869873
currentNode !== tokenList.tail;
870874
pos += currentNode.value.length, currentNode = currentNode.next
871875
) {
872876

877+
if (rematch && pos >= rematch.reach) {
878+
break;
879+
}
880+
873881
var str = currentNode.value;
874882

875883
if (tokenList.length > text.length) {
@@ -912,7 +920,7 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, oneshot, ta
912920
// find the last node which is affected by this match
913921
for (
914922
var k = currentNode;
915-
k !== tokenList.tail && (p < to || (typeof k.value === 'string' && !k.prev.value.greedy));
923+
k !== tokenList.tail && (p < to || typeof k.value === 'string');
916924
k = k.next
917925
) {
918926
removeCount++;
@@ -930,10 +938,6 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, oneshot, ta
930938
}
931939

932940
if (!match) {
933-
if (oneshot) {
934-
break;
935-
}
936-
937941
continue;
938942
}
939943

@@ -942,11 +946,16 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, oneshot, ta
942946
}
943947

944948
var from = match.index + lookbehindLength,
945-
match = match[0].slice(lookbehindLength),
946-
to = from + match.length,
949+
matchStr = match[0].slice(lookbehindLength),
950+
to = from + matchStr.length,
947951
before = str.slice(0, from),
948952
after = str.slice(to);
949953

954+
var reach = pos + str.length;
955+
if (rematch && reach > rematch.reach) {
956+
rematch.reach = reach;
957+
}
958+
950959
var removeFrom = currentNode.prev;
951960

952961
if (before) {
@@ -956,19 +965,21 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, oneshot, ta
956965

957966
removeRange(tokenList, removeFrom, removeCount);
958967

959-
var wrapped = new Token(token, inside ? _.tokenize(match, inside) : match, alias, match, greedy);
968+
var wrapped = new Token(token, inside ? _.tokenize(matchStr, inside) : matchStr, alias, matchStr);
960969
currentNode = addAfter(tokenList, removeFrom, wrapped);
961970

962971
if (after) {
963972
addAfter(tokenList, currentNode, after);
964973
}
965974

966-
967-
if (removeCount > 1)
968-
matchGrammar(text, tokenList, grammar, currentNode.prev, pos, true, token + ',' + j);
969-
970-
if (oneshot)
971-
break;
975+
if (removeCount > 1) {
976+
// at least one Token object was removed, so we have to do some rematching
977+
// this can only happen if the current pattern is greedy
978+
matchGrammar(text, tokenList, grammar, currentNode.prev, pos, {
979+
cause: token + ',' + j,
980+
reach: reach
981+
});
982+
}
972983
}
973984
}
974985
}

‎tests/core/greedy.js

+3-5
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,6 @@ describe('Greedy matching', function () {
5555
});
5656
});
5757

58-
// https://github.com/PrismJS/prism/issues/1492
59-
/*
6058
it('should correctly rematch tokens', function () {
6159
testTokens({
6260
grammar: {
@@ -75,14 +73,14 @@ describe('Greedy matching', function () {
7573
code: `<'> '' ''\n<"> "" ""`,
7674
expected: [
7775
["c", "<'>"],
78-
["a", "''"],
79-
["a", "''"],
76+
" '",
77+
["a", "' '"],
78+
"'\n",
8079

8180
["c", "<\">"],
8281
["b", "\"\""],
8382
["b", "\"\""],
8483
]
8584
});
8685
});
87-
*/
8886
});

0 commit comments

Comments
 (0)
Please sign in to comment.