How to use the moo.states function in moo

To help you get started, we’ve selected a few moo examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github accordproject / cicero / packages / cicero-core / lib / tdl.js View on Github external
},
      []
    );
  };

const moo = require("moo");

const escapeNearley = (x) => {
    return x.replace(/\t/g, '\\t') // Replace tab due to Nearley bug #nearley/issues/413
        .replace(/\f/g, '\\f')
        .replace(/\r/g, '\\r');
}

// we use lexer states to distinguish between the tokens
// in the text and the tokens inside the variables
const lexer = moo.states({
    main: {
        // a chunk is everything up until '[{', even across newlines. We then trim off the '[{'
        // we also push the lexer into the 'var' state
        Chunk: {
            match: /[^]*?\[{/,
            lineBreaks: true,
            push: 'var',
            value: x => escapeNearley(x.slice(0, -2))
        },
        // we now need to consume everything up until the end of the buffer.
        // note that the order of these two rules is important!
        LastChunk : {
            match: /[^]+/,
            lineBreaks: true,
            value: x => escapeNearley(x)
        }
github kach / nearley / lib / nearley-language-bootstrapped.js View on Github external
value: x => JSON.parse(x),
        next: 'main',
    },
    btstring: {
        match: /`[^`]*`/,
        value: x => x.slice(1, -1),
        next: 'main',
    },
}, literals([
    ",", "|", "$", "%", "(", ")",
    ":?", ":*", ":+",
    "@include", "@builtin", "@",
    "]",
]))

var lexer = moo.states({
    main: Object.assign({}, rules, {
        charclass: {
            match: /\.|\[(?:\\.|[^\\\n])+?\]/,
            value: x => new RegExp(x),
        },
    }),
    // Both macro arguments and charclasses are both enclosed in [ ].
    // We disambiguate based on whether the previous token was a `word`.
    afterWord: Object.assign({}, rules, {
        "[": {match: "[", next: 'main'},
    }),
})

function insensitive(sl) {
    var s = sl.literal;
    var result = [];
github DefinitelyTyped / DefinitelyTyped / types / moo / moo-tests.ts View on Github external
import * as moo from 'moo';

let lexer = moo.compile({
    lparen: '(',
    word:  /[a-z]+/,
    rparen: ')',
    keyword: ['while', 'if', 'else', 'moo', 'cows']
});

lexer = moo.states({
    main: {
        strstart: {match: '`', push: 'lit'},
        ident:    /\w+/,
        lbrace:   {match: '{', push: 'main'},
        rbrace:   {match: '}', pop: 1},
        colon:    ':',
        space:    {match: /\s+/, lineBreaks: true},
    },
    lit: {
        interp:   {match: '${', push: 'main'},
        escape:   /\\./,
        strend:   {match: '`', pop: 1},
        const:    {match: /(?:[^$`]|\$(?!\{))+/, lineBreaks: true},
    },
});
github dataform-co / dataform / sqlx / lexer.ts View on Github external
};

const INNER_SQL_BLOCK_LEXER_TOKEN_NAMES = {
  STATEMENT_SEPERATOR: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_statementSeparator",
  SINGLE_LINE_COMMENT: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_singleLineComment",
  MULTI_LINE_COMMENT: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_multiLineComment",
  SINGLE_QUOTE_STRING: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_singleQuoteString",
  DOUBLE_QUOTE_STRING: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_doubleQuoteString",
  START_JS_PLACEHOLDER: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_startJsPlaceholder",
  CLOSE_BLOCK: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_closeBlock",
  ESCAPED_BACKTICK: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_escapedBacktick",
  BACKTICK: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_backtick",
  CAPTURE_EVERYTHING_ELSE: LEXER_STATE_NAMES.INNER_SQL_BLOCK + "_captureEverythingElse"
};

const lexer = moo.states(buildSqlxLexer());

export interface ISyntaxTreeNode {
  contentType: "sql" | "js" | "jsPlaceholder" | "sqlStatementSeparator" | "sqlComment";
  contents: Array;
}

function appendToNode(node: ISyntaxTreeNode, tokenValue: string) {
  if (node.contents.length > 0 && typeof node.contents[node.contents.length - 1] === "string") {
    node.contents[node.contents.length - 1] = node.contents[node.contents.length - 1] + tokenValue;
    return;
  }
  node.contents.push(tokenValue);
}

export function constructSyntaxTree(code: string): ISyntaxTreeNode {
  const parentNode: ISyntaxTreeNode = { contentType: "sql", contents: [] };
github tategakibunko / TypeNovel / dist / lexer.js View on Github external
return {
    argsStart: '(',
    argsEnd: argsEnd,
    rangeCommentStart: { match: '/*', push: 'rangeComment' },
    comma: ',',
    objStart: { match: '{', push: 'object' },
    arrayStart: { match: '[', push: 'array' },
    literalDq: rexLiteralDq,
    literalSq: rexLiteralSq,
    float: rexFloat,
    integer: rexInt,
    ws: { match: /\s+/, lineBreaks: true },
  };
};

const lexer = moo.states({
  plain: plainTextRule,
  lineComment: {
    ws: { match: '\n', lineBreaks: true, pop: true },
    lineCommentText: /[^\n]+/,
  },
  rangeComment: {
    ws: { match: '\n', lineBreaks: true },
    rangeCommentEnd: { match: '*/', pop: true },
    rangeCommentText1: /[^\n]+(?=\*\/)/,
    rangeCommentText2: /[^\n]+/,
  },
  annot: {
    annotName: { match: rexTagName, next: 'annotArgs' },
    ws: { match: /\s+/, lineBreaks: true },
  },
  block: {
github tategakibunko / TypeNovel / src / lexer.js View on Github external
return {
    argsStart: '(',
    argsEnd: argsEnd,
    rangeCommentStart: { match: '/*', push: 'rangeComment' },
    comma: ',',
    objStart: { match: '{', push: 'object' },
    arrayStart: { match: '[', push: 'array' },
    literalDq: rexLiteralDq,
    literalSq: rexLiteralSq,
    float: rexFloat,
    integer: rexInt,
    ws: { match: /\s+/, lineBreaks: true },
  };
};

const lexer = moo.states({
  plain: plainTextRule,
  lineComment: {
    ws: { match: '\n', lineBreaks: true, pop: true },
    lineCommentText: /[^\n]+/,
  },
  rangeComment: {
    ws: { match: '\n', lineBreaks: true },
    rangeCommentEnd: { match: '*/', pop: true },
    rangeCommentText1: /[^\n]+(?=\*\/)/,
    rangeCommentText2: /[^\n]+/,
  },
  annot: {
    annotName: { match: rexTagName, next: 'annotArgs' },
    ws: { match: /\s+/, lineBreaks: true },
  },
  block: {
github byteball / ocore / formula / grammars / ojson.js View on Github external
(function () {
function id(x) { return x[0]; }


const moo = require('moo')

let lexer = moo.states({
	main: {
		space: {match: /\s+/, lineBreaks: true},
		comment: /\/\/.*$/,
		blockComment: { match: /\/\*[^]*?\*\//, lineBreaks: true },
		formulaDoubleStart: { match: '"{', push: 'formulaDouble' },
		formulaSingleStart: { match: "'{", push: 'formulaSingle' },
		formulaBackStart: { match: '`{', push: 'formulaBack' },
		'{': '{',
		'}': '}',
		'[': '[',
		']': ']',
		':': ':',
		',': ',',
		base64: [
			/'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{4})'/,
			/"(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{4})"/,

moo

Optimised tokenizer/lexer generator! 🐄 Much performance. Moo!

BSD-3-Clause
Latest version published 1 year ago

Package Health Score

74 / 100
Full package analysis