Skip to content

Commit fbddcd5

Browse files
dylanscottfdintino
dylanscott
authored and
fdintino
committedMar 4, 2019
lexer more accurately tracks token line and column information
·
v3.2.4v3.2.0
1 parent 889ef80 commit fbddcd5

File tree

3 files changed

+297
-2
lines changed

3 files changed

+297
-2
lines changed
 

‎nunjucks/src/lexer.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -375,7 +375,7 @@ class Tokenizer {
375375

376376
_extractString(str) {
377377
if (this._matches(str)) {
378-
this.index += str.length;
378+
this.forwardN(str.length);
379379
return str;
380380
}
381381
return null;

‎tests/compiler.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -816,7 +816,7 @@
816816
tmpl.render({}, function(err, res) {
817817
expect(res).to.be(undefined);
818818
expect(err.toString()).to.be([
819-
'Template render error: (parse-error.njk) [Line 1, Column 24]',
819+
'Template render error: (parse-error.njk) [Line 1, Column 26]',
820820
' unexpected token: ,',
821821
].join('\n'));
822822
done();

‎tests/lexer.js

+295
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,17 @@
3232
if (lib.isArray(type)) {
3333
expect(tok.type).to.be(type[0]);
3434
expect(tok.value).to.be(type[1]);
35+
} else if (lib.isObject(type)) {
36+
expect(tok.type).to.be(type.type);
37+
if (type.value != null) {
38+
expect(tok.value).to.be(type.value);
39+
}
40+
if (type.lineno != null) {
41+
expect(tok.lineno).to.be(type.lineno);
42+
}
43+
if (type.colno != null) {
44+
expect(tok.colno).to.be(type.colno);
45+
}
3546
} else {
3647
expect(tok.type).to.be(type);
3748
}
@@ -436,5 +447,289 @@
436447
lexer.TOKEN_SYMBOL,
437448
lexer.TOKEN_VARIABLE_END);
438449
});
450+
451+
it('should keep track of token positions', function() {
452+
hasTokens(lexer.lex('{{ 3 != 4 == 5 <= 6 >= 7 < 8 > 9 }}'),
453+
{
454+
type: lexer.TOKEN_VARIABLE_START,
455+
lineno: 0,
456+
colno: 0,
457+
},
458+
{
459+
type: lexer.TOKEN_INT,
460+
value: '3',
461+
lineno: 0,
462+
colno: 3,
463+
},
464+
{
465+
type: lexer.TOKEN_OPERATOR,
466+
value: '!=',
467+
lineno: 0,
468+
colno: 5,
469+
},
470+
{
471+
type: lexer.TOKEN_INT,
472+
value: '4',
473+
lineno: 0,
474+
colno: 8,
475+
},
476+
{
477+
type: lexer.TOKEN_OPERATOR,
478+
value: '==',
479+
lineno: 0,
480+
colno: 10,
481+
},
482+
{
483+
type: lexer.TOKEN_INT,
484+
value: '5',
485+
lineno: 0,
486+
colno: 13,
487+
},
488+
{
489+
type: lexer.TOKEN_OPERATOR,
490+
value: '<=',
491+
lineno: 0,
492+
colno: 15,
493+
},
494+
{
495+
type: lexer.TOKEN_INT,
496+
value: '6',
497+
lineno: 0,
498+
colno: 18,
499+
},
500+
{
501+
type: lexer.TOKEN_OPERATOR,
502+
lineno: 0,
503+
colno: 20,
504+
value: '>=',
505+
},
506+
{
507+
type: lexer.TOKEN_INT,
508+
lineno: 0,
509+
colno: 23,
510+
value: '7',
511+
},
512+
{
513+
type: lexer.TOKEN_OPERATOR,
514+
value: '<',
515+
lineno: 0,
516+
colno: 25,
517+
},
518+
{
519+
type: lexer.TOKEN_INT,
520+
value: '8',
521+
lineno: 0,
522+
colno: 27,
523+
},
524+
{
525+
type: lexer.TOKEN_OPERATOR,
526+
value: '>',
527+
lineno: 0,
528+
colno: 29,
529+
},
530+
{
531+
type: lexer.TOKEN_INT,
532+
value: '9',
533+
lineno: 0,
534+
colno: 31,
535+
},
536+
{
537+
type: lexer.TOKEN_VARIABLE_END,
538+
lineno: 0,
539+
colno: 33,
540+
});
541+
542+
hasTokens(lexer.lex('{% if something %}{{ value }}{% else %}{{ otherValue }}{% endif %}'),
543+
{
544+
type: lexer.TOKEN_BLOCK_START,
545+
lineno: 0,
546+
colno: 0,
547+
},
548+
{
549+
type: lexer.TOKEN_SYMBOL,
550+
value: 'if',
551+
lineno: 0,
552+
colno: 3,
553+
},
554+
{
555+
type: lexer.TOKEN_SYMBOL,
556+
value: 'something',
557+
lineno: 0,
558+
colno: 6,
559+
},
560+
{
561+
type: lexer.TOKEN_BLOCK_END,
562+
lineno: 0,
563+
colno: 16,
564+
},
565+
{
566+
type: lexer.TOKEN_VARIABLE_START,
567+
lineno: 0,
568+
colno: 18,
569+
},
570+
{
571+
type: lexer.TOKEN_SYMBOL,
572+
value: 'value',
573+
lineno: 0,
574+
colno: 21,
575+
},
576+
{
577+
type: lexer.TOKEN_VARIABLE_END,
578+
lineno: 0,
579+
colno: 27,
580+
},
581+
{
582+
type: lexer.TOKEN_BLOCK_START,
583+
lineno: 0,
584+
colno: 29,
585+
},
586+
{
587+
type: lexer.TOKEN_SYMBOL,
588+
value: 'else',
589+
lineno: 0,
590+
colno: 32,
591+
},
592+
{
593+
type: lexer.TOKEN_BLOCK_END,
594+
lineno: 0,
595+
colno: 37,
596+
},
597+
{
598+
type: lexer.TOKEN_VARIABLE_START,
599+
lineno: 0,
600+
colno: 39,
601+
},
602+
{
603+
type: lexer.TOKEN_SYMBOL,
604+
value: 'otherValue',
605+
lineno: 0,
606+
colno: 42,
607+
},
608+
{
609+
type: lexer.TOKEN_VARIABLE_END,
610+
lineno: 0,
611+
colno: 53,
612+
},
613+
{
614+
type: lexer.TOKEN_BLOCK_START,
615+
lineno: 0,
616+
colno: 55,
617+
},
618+
{
619+
type: lexer.TOKEN_SYMBOL,
620+
value: 'endif',
621+
lineno: 0,
622+
colno: 58,
623+
},
624+
{
625+
type: lexer.TOKEN_BLOCK_END,
626+
lineno: 0,
627+
colno: 64,
628+
});
629+
630+
hasTokens(lexer.lex('{% if something %}\n{{ value }}\n{% else %}\n{{ otherValue }}\n{% endif %}'),
631+
{
632+
type: lexer.TOKEN_BLOCK_START,
633+
lineno: 0,
634+
colno: 0,
635+
},
636+
{
637+
type: lexer.TOKEN_SYMBOL,
638+
value: 'if',
639+
lineno: 0,
640+
colno: 3,
641+
},
642+
{
643+
type: lexer.TOKEN_SYMBOL,
644+
value: 'something',
645+
lineno: 0,
646+
colno: 6,
647+
},
648+
{
649+
type: lexer.TOKEN_BLOCK_END,
650+
lineno: 0,
651+
colno: 16,
652+
},
653+
{
654+
type: lexer.TOKEN_DATA,
655+
value: '\n',
656+
},
657+
{
658+
type: lexer.TOKEN_VARIABLE_START,
659+
lineno: 1,
660+
colno: 0,
661+
},
662+
{
663+
type: lexer.TOKEN_SYMBOL,
664+
value: 'value',
665+
lineno: 1,
666+
colno: 3,
667+
},
668+
{
669+
type: lexer.TOKEN_VARIABLE_END,
670+
lineno: 1,
671+
colno: 9,
672+
},
673+
{
674+
type: lexer.TOKEN_DATA,
675+
value: '\n',
676+
},
677+
{
678+
type: lexer.TOKEN_BLOCK_START,
679+
lineno: 2,
680+
colno: 0,
681+
},
682+
{
683+
type: lexer.TOKEN_SYMBOL,
684+
value: 'else',
685+
lineno: 2,
686+
colno: 3,
687+
},
688+
{
689+
type: lexer.TOKEN_BLOCK_END,
690+
lineno: 2,
691+
colno: 8,
692+
},
693+
{
694+
type: lexer.TOKEN_DATA,
695+
value: '\n',
696+
},
697+
{
698+
type: lexer.TOKEN_VARIABLE_START,
699+
lineno: 3,
700+
colno: 0,
701+
},
702+
{
703+
type: lexer.TOKEN_SYMBOL,
704+
value: 'otherValue',
705+
lineno: 3,
706+
colno: 3,
707+
},
708+
{
709+
type: lexer.TOKEN_VARIABLE_END,
710+
lineno: 3,
711+
colno: 14,
712+
},
713+
{
714+
type: lexer.TOKEN_DATA,
715+
value: '\n',
716+
},
717+
{
718+
type: lexer.TOKEN_BLOCK_START,
719+
lineno: 4,
720+
colno: 0,
721+
},
722+
{
723+
type: lexer.TOKEN_SYMBOL,
724+
value: 'endif',
725+
lineno: 4,
726+
colno: 3,
727+
},
728+
{
729+
type: lexer.TOKEN_BLOCK_END,
730+
lineno: 4,
731+
colno: 9,
732+
});
733+
});
439734
});
440735
}());

0 commit comments

Comments
 (0)
Please sign in to comment.