How to use the asttokens.CodeText function in asttokens

def test_mark_tokens_simple(self):
    source = read_fixture('astroid', 'module.py')
    root = ast.parse(source)
    code = asttokens.CodeText(source)
    code.mark_tokens(root)
    all_nodes = list(asttokens.walk(root))

    def get_node_text(line, col, type_name):
      token = code.get_token(line, col)
      for n in all_nodes:
        if n.first_token == token and n.__class__.__name__ == type_name:
          return code.get_text(n)

    # Line 14 is: [indent 4] MY_DICT[key] = val
    self.assertEqual(get_node_text(14, 4, 'Name'), 'MY_DICT')
    self.assertEqual(get_node_text(14, 4, 'Subscript'), 'MY_DICT[key]')
    self.assertEqual(get_node_text(14, 4, 'Assign'), 'MY_DICT[key] = val')

    # Line 35 is: [indent 12] raise XXXError()
    self.assertEqual(get_node_text(35, 12, 'Raise'), 'raise XXXError()')

def test_mark_tokens_multiline(self):
    source = (
"""(    # line1
a,      # line2
b +     # line3
  c +   # line4
  d     # line5
)""")
    root = ast.parse(source)
    code = asttokens.CodeText(source)
    code.mark_tokens(root)

    all_nodes = {code.get_text(node) for node in ast.walk(root)}
    self.assertEqual(all_nodes, {
      None,           # nodes we don't care about
      source,
      'a', 'b', 'c', 'd',
      # All other expressions preserve newlines and comments but are parenthesized.
      '(b +     # line3\n  c)',
      '(b +     # line3\n  c +   # line4\n  d)',
      '(a,      # line2\nb +     # line3\n  c +   # line4\n  d)',
    })

def test_codetext_simple(self):
    source = "import re  # comment\n\nfoo = 'bar'\n"
    ctext = asttokens.CodeText(source)
    self.assertEqual(ctext.text, source)
    self.assertEqual([str(t) for t in ctext.tokens], [
      "NAME:'import'",
      "NAME:'re'",
      "COMMENT:'# comment'",
      "NEWLINE:'\\n'",
      "NL:'\\n'",
      "NAME:'foo'",
      "OP:'='",
      'STRING:"\'bar\'"',
      "NEWLINE:'\\n'",
      "ENDMARKER:''"
    ])

    self.assertEqual(ctext.tokens[5].type, token.NAME)
    self.assertEqual(ctext.tokens[5].string, 'foo')

def test_foo(self):
    source = "(a,\na + b\n + c + d)"
    root = ast.parse(source)
    code = asttokens.CodeText(source)
    for t in  code.tokens:
      print t
    code.mark_tokens(root)
    for n in ast.walk(root):
      print repr(n), ast.dump(n, include_attributes=True)
      print code.get_text(n)

def test_codetext_methods(self):
    source = "import re  # comment\n\nfoo = 'bar'\n"
    ctext = asttokens.CodeText(source)
    self.assertEqual(str(ctext.tokens[3]), "NEWLINE:'\\n'")
    self.assertEqual(str(ctext.tokens[4]), "NL:'\\n'")
    self.assertEqual(str(ctext.tokens[5]), "NAME:'foo'")
    self.assertEqual(str(ctext.tokens[6]), "OP:'='")
    self.assertEqual(ctext.prev_token(ctext.tokens[5]), ctext.tokens[3])
    self.assertEqual(ctext.prev_token(ctext.tokens[5], include_extra=True), ctext.tokens[4])
    self.assertEqual(ctext.next_token(ctext.tokens[5]), ctext.tokens[6])
    self.assertEqual(ctext.next_token(ctext.tokens[1]), ctext.tokens[3])
    self.assertEqual(ctext.next_token(ctext.tokens[1], include_extra=True), ctext.tokens[2])

    self.assertEqual(ctext.get_token_from_offset(21), ctext.tokens[4])
    self.assertEqual(ctext.get_token_from_offset(22), ctext.tokens[5])
    self.assertEqual(ctext.get_token_from_offset(23), ctext.tokens[5])
    self.assertEqual(ctext.get_token_from_offset(24), ctext.tokens[5])
    self.assertEqual(ctext.get_token_from_offset(25), ctext.tokens[5])
    self.assertEqual(ctext.get_token_from_offset(26), ctext.tokens[6])

def test_assign_first_tokens(self):
    source = read_fixture('astroid', 'module.py')
    root = ast.parse(source)
    code = asttokens.CodeText(source)
    code.mark_tokens(root)
    all_nodes = list(asttokens.walk(root))

    def get_node_types_at(line, col):
      token = code.get_token(line, col)
      return {n.__class__.__name__ for n in all_nodes if n.first_token == token}

    # Line 14 is: [indent 4] MY_DICT[key] = val
    self.assertEqual(get_node_types_at(14, 4), {'Name', 'Subscript', 'Assign'})

    # Line 35 is: [indent 12] raise XXXError()
    self.assertEqual(get_node_types_at(35, 12), {'Raise'})
    self.assertEqual(get_node_types_at(35, 18), {'Call', 'Name'})

    # Line 53 is: [indent 12] autre = [a for (a, b) in MY_DICT if b]
    self.assertEqual(get_node_types_at(53, 20), {'ListComp'})

def test_mark_tokens(self):
    # There is a generic way to test. We can take an arbitrary piece of code, parse it, and for
    # each AST node, extract the piece of code text from first to last token. For many node types,
    # that piece should itself be compilable, and the resulting AST node should be equivalent.
    dir_path = get_fixture_path("astroid")
    for module in os.listdir(dir_path):
      if not module.endswith('.py'):
        continue
      print "PROCESSING", module
      source = read_fixture("astroid", module)
      root = ast.parse(source)
      code = asttokens.CodeText(source)
      code.mark_tokens(root)

      for node in ast.walk(root):
        if not isinstance(node, (ast.stmt, ast.expr)):
          continue
        # TODO Bleh. dedent gets borken when there are unindented comments.
        text = code.get_text(node)
        indented = re.match(r'^[ \t]+\S', text)
        if indented:
          text = "def dummy():\n" + text
        #print "TEXT", text
        try:
          rebuilt_node = ast.parse(text, 'exec').body[0]
        except Exception:
          print "CAN'T PARSE", text, repr(text)
          raise