Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_float_without_lexer(self):
expected_error = UnexpectedCharacters if LEXER.startswith('dynamic') else UnexpectedToken
if PARSER == 'cyk':
expected_error = ParseError
g = _Lark("""start: ["+"|"-"] float
float: digit* "." digit+ exp?
| digit+ exp
exp: ("e"|"E") ["+"|"-"] digit+
digit: "0"|"1"|"2"|"3"|"4"|"5"|"6"|"7"|"8"|"9"
""")
g.parse("1.2")
g.parse("-.2e9")
g.parse("+2e-9")
self.assertRaises( expected_error, g.parse, "+2e-9e")
print " >>", str(e)
except VisitError as e:
# e.orig_exc
print "[Transform] FAIL (expected)"
print " >>", str(e)
for test_str in good:
print "\n #", repr(test_str)
try:
tree = parser.parse(test_str)
print "\n[ Parsing ] OK"
tree = transformer.transform(tree)
print "[Transform] OK"
#print "[Full Type]", "OK" if tree.is_fully_typed() else "FAIL"
print ""
print repr(tree)
except (UnexpectedToken, UnexpectedCharacters, SyntaxError) as e:
print "[ Parsing ] FAIL"
print " >>", str(e)
return 1
except VisitError as e:
# e.orig_exc
print "[Transform] FAIL"
print " >>", str(e)
print ""
print repr(tree)
return 1
print "\nAll", str(len(bad) + len(good)), "tests passed."
return 0
def add_wrapper(module):
if getattr(module, "wrapper", False):
return module
rules = find_merge_rule_classes(module)
registry = utils.Registry()
for rule in rules:
for spec, action in rule.mapping.items():
try:
converted_spec = convert_spec(spec)
except (lark.exceptions.UnexpectedCharacters, NotImplementedError):
# XXX
converted_spec = '"aklsdjfkldjfk"'
# TODO Make this easier to read.
registry.register(converted_spec)(
lambda data, action=action: _run(action, data)
)
for extra in rule.extras:
registry.define(extra.make_definitions())
# TODO use module.context
wrapper = utils.Wrapper(registry, context=context.AlwaysContext())
module.wrapper = wrapper
# XXX Avoid mutate-and-return.
return module
new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
new_item.node.add_family(new_item.s, item.rule, new_item.start, item.node, token)
else:
new_item = item
if new_item.expect in self.TERMINALS:
# add (B ::= Aai+1.B, h, y) to Q'
next_to_scan.add(new_item)
else:
# add (B ::= Aa+1.B, h, y) to Ei+1
next_set.add(new_item)
del delayed_matches[i+1] # No longer needed, so unburden memory
if not next_set and not delayed_matches and not next_to_scan:
raise UnexpectedCharacters(stream, i, text_line, text_column, {item.expect.name for item in to_scan}, set(to_scan))
return next_to_scan
message = "No terminal defined for '%s' at line %d col %d" % (seq[lex_pos], line, column)
self.line = line
self.column = column
self.allowed = allowed
self.considered_tokens = considered_tokens
self.pos_in_stream = lex_pos
self.state = state
message += '\n\n' + self.get_context(seq)
if allowed:
message += '\nExpecting: %s\n' % allowed
if token_history:
message += '\nPrevious tokens: %s\n' % ', '.join(repr(t) for t in token_history)
super(UnexpectedCharacters, self).__init__(message)
logging.basicConfig(level=logging.DEBUG)
parser = Lark(PROPERTY_GRAMMAR, parser="lalr", start="property", debug=True)
type_checker = HplTypeChecker(TEST_TOPICS, TEST_FIELDS, TEST_CONSTANTS)
transformer = PropertyTransformer(type_checker=type_checker)
for test_str in FAILING_TESTS:
try:
tree = parser.parse(test_str)
tree = transformer.transform(tree)
print ""
print test_str
assert False, "expected failure"
except UnexpectedToken as e:
pass
except UnexpectedCharacters as e:
pass
except TypeError as e:
pass
except SyntaxError as e:
pass
for test_str in PASSING_TESTS:
print ""
print test_str
tree = parser.parse(test_str)
tree = transformer.transform(tree)
print tree
print "All", str(len(FAILING_TESTS) + len(PASSING_TESTS)), "tests passed."
def lex(self, stream, get_parser_state):
parser_state = get_parser_state()
l = _Lex(self.lexers[parser_state], parser_state)
try:
for x in l.lex(stream, self.root_lexer.newline_types, self.root_lexer.ignore_types):
yield x
parser_state = get_parser_state()
l.lexer = self.lexers[parser_state]
l.state = parser_state # For debug only, no need to worry about multithreading
except UnexpectedCharacters as e:
# In the contextual lexer, UnexpectedCharacters can mean that the terminal is defined,
# but not in the current context.
# This tests the input against the global context, to provide a nicer error.
root_match = self.root_lexer.match(stream, e.pos_in_stream)
if not root_match:
raise
value, type_ = root_match
t = Token(type_, value, e.pos_in_stream, e.line, e.column)
raise UnexpectedToken(t, e.allowed, state=e.state)
''' Deserialize `text` (a `str` or `unicode` instance containing a JSON
document with Python or JavaScript like comments) to a Python object.
:param text: serialized JSON string with or without comments.
:param kwargs: all the arguments that `json.loads `_ accepts.
:returns: dict or list.
'''
if isinstance(text, (bytes, bytearray)):
text = text.decode(detect_encoding(text), 'surrogatepass')
try:
parsed = parser.parse(text)
final_text = serializer.reconstruct(parsed)
except lark.exceptions.UnexpectedCharacters:
raise ValueError('Unable to parse text', text)
return json.loads(final_text, *args, **kwargs)