Adds the (string) position of tokens to the token tree.

Author: synga-nl

src/twig.core.js

@@ -325,9 +325,8 @@ module.exports = function (Twig) {
      */
     Twig.tokenize = function (template) {
         const tokens = [];
-        // An offset for reporting errors locations in the template.
-        let errorOffset = 0;
-
+        // An offset for reporting errors locations and the position of the nodes in the template.
+        let currentPosition = 0;
         // The start and type of the first token found in the template.
         let foundToken = null;
         // The end position of the matched token.
@@ -343,29 +342,41 @@ module.exports = function (Twig) {
                 // No more tokens -> add the rest of the template as a raw-type token
                 tokens.push({
                     type: Twig.token.type.raw,
-                    value: template
+                    value: template,
+                    position: {
+                        start: currentPosition,
+                        end: currentPosition + foundToken.position
+                    }
                 });
                 template = '';
             } else {
                 // Add a raw type token for anything before the start of the token
                 if (foundToken.position > 0) {
                     tokens.push({
                         type: Twig.token.type.raw,
-                        value: template.slice(0, Math.max(0, foundToken.position))
+                        value: template.slice(0, Math.max(0, foundToken.position)),
+                        position: {
+                            start: currentPosition,
+                            end: currentPosition + Math.max(0, foundToken.position)
+                        }
                     });
                 }
 
                 template = template.slice(foundToken.position + foundToken.def.open.length);
-                errorOffset += foundToken.position + foundToken.def.open.length;
+                currentPosition += foundToken.position + foundToken.def.open.length;
 
                 // Find the end of the token
-                end = Twig.token.findEnd(template, foundToken.def, errorOffset);
+                end = Twig.token.findEnd(template, foundToken.def, currentPosition);
 
                 Twig.log.trace('Twig.tokenize: ', 'Token ends at ', end);
 
                 tokens.push({
                     type: foundToken.def.type,
-                    value: template.slice(0, Math.max(0, end)).trim()
+                    value: template.slice(0, Math.max(0, end)).trim(),
+                    position: {
+                        start: currentPosition - foundToken.def.open.length,
+                        end: currentPosition + end + foundToken.def.close.length
+                    }
                 });
 
                 if (template.slice(end + foundToken.def.close.length, end + foundToken.def.close.length + 1) === '\n') {
@@ -385,7 +396,7 @@ module.exports = function (Twig) {
                 template = template.slice(end + foundToken.def.close.length);
 
                 // Increment the position in the template
-                errorOffset += end + foundToken.def.close.length;
+                currentPosition += end + foundToken.def.close.length;
             }
         }
 
@@ -434,6 +445,7 @@ module.exports = function (Twig) {
             const compileLogic = function (token) {
                 // Compile the logic token
                 logicToken = Twig.logic.compile.call(self, token);
+                logicToken.position = token.position;
 
                 type = logicToken.type;
                 open = Twig.logic.handler[type].open;
@@ -458,8 +470,13 @@ module.exports = function (Twig) {
 
                     tokOutput = {
                         type: Twig.token.type.logic,
-                        token: prevToken
+                        token: prevToken,
+                        position: {
+                            open: prevToken.position,
+                            close: token.position
+                        }
                     };
+
                     if (stack.length > 0) {
                         intermediateOutput.push(tokOutput);
                     } else {
@@ -486,7 +503,8 @@ module.exports = function (Twig) {
                 } else if (open !== undefined && open) {
                     tokOutput = {
                         type: Twig.token.type.logic,
-                        token: logicToken
+                        token: logicToken,
+                        position: logicToken.position
                     };
                     // Standalone token (like {% set ... %}
                     if (stack.length > 0) {

src/twig.expression.js

@@ -1118,11 +1118,12 @@ module.exports = function (Twig) {
     /**
      * Break an expression into tokens defined in Twig.expression.definitions.
      *
-     * @param {string} expression The string to tokenize.
+     * @param {Object} rawToken The string to tokenize.
      *
      * @return {Array} An array of tokens.
      */
-    Twig.expression.tokenize = function (expression) {
+    Twig.expression.tokenize = function (rawToken) {
+        let expression = rawToken.value;
         const tokens = [];
         // Keep an offset of the location in the expression for error messages.
         let expOffset = 0;
@@ -1170,11 +1171,17 @@ module.exports = function (Twig) {
 
             invalidMatches = [];
 
-            tokens.push({
+            const token = {
                 type,
                 value: match[0],
                 match
-            });
+            };
+
+            if (rawToken.position) {
+                token.position = rawToken.position;
+            }
+
+            tokens.push(token);
 
             matchFound = true;
             next = tokenNext;
@@ -1240,15 +1247,14 @@ module.exports = function (Twig) {
      * @return {Object} The compiled token.
      */
     Twig.expression.compile = function (rawToken) {
-        const expression = rawToken.value;
         // Tokenize expression
-        const tokens = Twig.expression.tokenize(expression);
+        const tokens = Twig.expression.tokenize(rawToken);
         let token = null;
         const output = [];
         const stack = [];
         let tokenTemplate = null;
 
-        Twig.log.trace('Twig.expression.compile: ', 'Compiling ', expression);
+        Twig.log.trace('Twig.expression.compile: ', 'Compiling ', rawToken.value);
 
         // Push tokens into RPN stack using the Shunting-yard algorithm
         // See http://en.wikipedia.org/wiki/Shunting_yard_algorithm

test/test.core.js

@@ -508,4 +508,22 @@ describe('Twig.js Core ->', function () {
             }).should.throw(/Unable to find template file/);
         });
     });
+
+    describe('tokens should have the correct positions in template', () => {
+        it('should show the correct token positions in a simple template', () => {
+            const tokens = twig({data: '{{ unit }}'}).tokens;
+            tokens[0].position.should.eql({start: 0, end: 10});
+        });
+
+        it('should show the correct token positions in a advanced template', () => {
+            const tokens = twig({data:'I want to {{ try }} a more {% if advanced | length > 3 %}{{ variable }}{% endif %} template {% set unit = 2 %}{# This is a comment #}{{ variable_after_comment }}'}).tokens;
+            tokens[0].position.should.eql({start: 0, end: 10});
+            tokens[1].position.should.eql({start: 10, end: 19});
+            tokens[2].position.should.eql({start: 19, end: 27});
+            tokens[3].position.should.eql({open: {start: 27, end: 57}, close: {start: 71, end: 82}});
+            tokens[3].token.output[0].position.should.eql({start: 57, end: 71});
+            tokens[5].position.should.eql({start: 92, end: 110});
+            tokens[6].position.should.eql({start: 133, end: 161});
+        });
+    });
 });