How to use the hy.lex.tokenize.tokenize function in hy

def test_fn_split():
    """Test if we can get a statement something right."""
    one, two = tokenize(code)
    assert one.get_invocation() == {
        "function": "+",
        "args": [
            1, 1
        ]
    }
    assert two.get_invocation() == {
        "function": "fn",
        "args": [
            "foo", "bar"
        ]

def test_string_in_list():
    """ String in list """
    fn = tokenize('(fn [1 2 "three four" 5])')[0]
    assert fn == [
        "fn", [1, 2, "three four", 5]
    ]

'el',
            ['+',
                1,
                2,
                ['==',
                    1,
                    20
                ],
                ['-',
                    1,
                    1
                ],
            ]
        ],
        ['fn1', 'foo', 'bar']
    ] == tokenize("(fn el (+ 1 2 (== 1 20) (- 1 1)))(fn1 foo bar)")

def test_mid_recurse_comment():
    """ Test some crazy recursion with a comment """

    assert [
        ['fn',
            'one',
            ['fn', 'two'],
            ['fn', 'three'],
        ]
    ] == tokenize("""
(fn one ; this is a test

def test_list_recurse():
    """ test we can recurse lists """
    fn = tokenize("(fn [1 2 3 4 [5 6 7]])")[0]
    assert fn == [
        "fn", [1, 2, 3, 4, [5, 6, 7]]
    ]

def test_list_lex():
    """test basic lexing of lists"""
    fn = tokenize("(fn [1 2 3 4])")[0]
    assert fn == [
        "fn", [1, 2, 3, 4]
    ]

def test_list_recurse_with_comment():
    """ test we can recurse lists """
    fn = tokenize("""
(fn [1 ; this is a test
     2 3 4 [5 6 7]])
""")[0]
    assert fn == [
        "fn", [1, 2, 3, 4, [5, 6, 7]]
    ]

def _lex(*args):
    ret = []
    for thing in args:
        ret.append(_hy_tok(thing))
    return ret

def _hy_import_file(fd, name):
    m = forge_module(
        name,
        fd,
        tokenize(open(fd, 'r').read())
    )
    return m