from mako.lexer import Lexer
from mako import exceptions, util, compat
from test.util import flatten_result
from mako.template import Template
import re
from test import TemplateTest, eq_, assert_raises_message

# create fake parsetree classes which are constructed
# exactly as the repr() of a real parsetree object.
# this allows us to use a Python construct as the source
# of a comparable repr(), which is also hit by the 2to3 tool.

def repr_arg(x):
    if isinstance(x, dict):
        return util.sorted_dict_repr(x)
    else:
        return repr(x)

def _as_unicode(arg):
    if isinstance(arg, compat.string_types):
        return compat.text_type(arg)
    elif isinstance(arg, dict):
        return dict(
            (_as_unicode(k), _as_unicode(v))
            for k, v in arg.items()
        )
    else:
        return arg
from mako import parsetree
for cls in list(parsetree.__dict__.values()):
    if isinstance(cls, type) and \
        issubclass(cls, parsetree.Node):
        clsname = cls.__name__
        exec(("""
class %s(object):
    def __init__(self, *args):
        self.args = [_as_unicode(arg) for arg in args]
    def __repr__(self):
        return "%%s(%%s)" %% (
            self.__class__.__name__,
            ", ".join(repr_arg(x) for x in self.args)
            )
""" % clsname), locals())

# NOTE: most assertion expressions were generated, then formatted
# by PyTidy, hence the dense formatting.

class LexerTest(TemplateTest):

    def _compare(self, node, expected):
        eq_(repr(node), repr(expected))

    def test_text_and_tag(self):
        template = """
<b>Hello world</b>
        <%def name="foo()">
                this is a def.
        </%def>

        and some more text.
"""
        node = Lexer(template).parse()
        self._compare(node, TemplateNode({},
                      [Text('''\n<b>Hello world</b>\n        ''', (1,
                      1)), DefTag('def', {'name': 'foo()'}, (3, 9),
                      [Text('''\n                this is a def.\n        ''',
                      (3, 28))]),
                      Text('''\n\n        and some more text.\n''',
                      (5, 16))]))

    def test_unclosed_tag(self):
        template = """

            <%def name="foo()">
             other text
        """
        try:
            nodes = Lexer(template).parse()
            assert False
        except exceptions.SyntaxException:
            eq_(
                str(compat.exception_as()),
                "Unclosed tag: <%def> at line: 5 char: 9"
            )

    def test_onlyclosed_tag(self):
        template = \
            """
            <%def name="foo()">
                foo
            </%def>

            </%namespace>

            hi.
        """
        self.assertRaises(exceptions.SyntaxException,
                          Lexer(template).parse)

    def test_noexpr_allowed(self):
        template = \
            """
            <%namespace name="${foo}"/>
        """
        self.assertRaises(exceptions.CompileException,
                          Lexer(template).parse)

    def test_unmatched_tag(self):
        template = \
            """
        <%namespace name="bar">
        <%def name="foo()">
            foo
            </%namespace>
        </%def>


        hi.
"""
        self.assertRaises(exceptions.SyntaxException,
                          Lexer(template).parse)

    def test_nonexistent_tag(self):
        template = """
            <%lala x="5"/>
        """
        self.assertRaises(exceptions.CompileException,
                          Lexer(template).parse)

    def test_wrongcase_tag(self):
        template = \
            """
            <%DEF name="foo()">
            </%def>

        """
        self.assertRaises(exceptions.CompileException,
                          Lexer(template).parse)

    def test_percent_escape(self):
        template = \
            """

%% some whatever.

    %% more some whatever
    % if foo:
    % endif
        """
        node = Lexer(template).parse()
        self._compare(node, TemplateNode({}, [Text('''\n\n''',
                      (1, 1)), Text('''% some whatever.\n\n''', (3, 2)),
                      Text('   %% more some whatever\n', (5, 2)),
                      ControlLine('if', 'if foo:', False, (6, 1)),
                      ControlLine('if', 'endif', True, (7, 1)),
                      Text('        ', (8, 1))]))

    def test_old_multiline_comment(self):
        template = """#*"""
        node = Lexer(template).parse()
        self._compare(node, TemplateNode({}, [Text('''#*''', (1, 1))]))


    def test_text_tag(self):
        template = \
            """
        ## comment
        % if foo:
            hi
        % endif
        <%text>
            # more code

            % more code
            <%illegal compionent>/></>
            <%def name="laal()">def</%def>


        </%text>

        <%def name="foo()">this is foo</%def>

        % if bar:
            code
        % endif
        """
        node = Lexer(template).parse()
        self._compare(node,
            TemplateNode({}, [Text('\n', (1, 1)),
                    Comment('comment', (2, 1)),
                    ControlLine('if', 'if foo:', False, (3, 1)),
                    Text('            hi\n', (4, 1)),
                    ControlLine('if', 'endif', True, (5, 1)),
                    Text('        ', (6, 1)),
                    TextTag('text', {}, (6, 9),
                    [Text('\n            # more code\n\n           '
                        ' % more code\n            <%illegal compionent>/></>\n'
                        '            <%def name="laal()">def</%def>\n\n\n        ',
                        (6, 16))]), Text('\n\n        ', (14, 17)),
                    DefTag('def', {'name': 'foo()'}, (16, 9),
                    [Text('this is foo', (16, 28))]), Text('\n\n', (16, 46)),
                    ControlLine('if', 'if bar:', False, (18, 1)),
                    Text('            code\n', (19, 1)),
                    ControlLine('if', 'endif', True, (20, 1)),
                    Text('        ', (21, 1))])
        )

    def test_def_syntax(self):
        template = \
            """
        <%def lala>
            hi
        </%def>
"""
        self.assertRaises(exceptions.CompileException,
                          Lexer(template).parse)

    def test_def_syntax_2(self):
        template = \
            """
        <%def name="lala">
            hi
        </%def>
    """
        self.assertRaises(exceptions.CompileException,
                          Lexer(template).parse)

    def test_whitespace_equals(self):
        template = \
            """
            <%def name = "adef()" >
              adef
            </%def>
        """
        node = Lexer(template).parse()
        self._compare(node, TemplateNode({}, [Text('\n            ',
                      (1, 1)), DefTag('def', {'name': 'adef()'}, (2,
                      13),
                      [Text('''\n              adef\n            ''',
                      (2, 36))]), Text('\n        ', (4, 20))]))

    def test_ns_tag_closed(self):
        template = \
            """

            <%self:go x="1" y="2" z="${'hi' + ' ' + 'there'}"/>
        """
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({},
                      [Text('''

            ''', (1, 1)),
                      CallNamespaceTag('self:go', {'x': '1', 'y'
                      : '2', 'z': "${'hi' + ' ' + 'there'}"}, (3,
                      13), []), Text('\n        ', (3, 64))]))

    def test_ns_tag_empty(self):
        template = \
            """
            <%form:option value=""></%form:option>
        """
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({}, [Text('\n            ',
                      (1, 1)), CallNamespaceTag('form:option',
                      {'value': ''}, (2, 13), []), Text('\n        '
                      , (2, 51))]))

    def test_ns_tag_open(self):
        template = \
            """

            <%self:go x="1" y="${process()}">
                this is the body
            </%self:go>
        """
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({},
                      [Text('''

            ''', (1, 1)),
                      CallNamespaceTag('self:go', {'x': '1', 'y'
                      : '${process()}'}, (3, 13),
                      [Text('''
                this is the body
            ''',
                      (3, 46))]), Text('\n        ', (5, 24))]))

    def test_expr_in_attribute(self):
        """test some slightly trickier expressions.

        you can still trip up the expression parsing, though, unless we
        integrated really deeply somehow with AST."""

        template = \
            """
            <%call expr="foo>bar and 'lala' or 'hoho'"/>
            <%call expr='foo<bar and hoho>lala and "x" + "y"'/>
        """
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({}, [Text('\n            ',
                      (1, 1)), CallTag('call', {'expr'
                      : "foo>bar and 'lala' or 'hoho'"}, (2, 13), []),
                      Text('\n            ', (2, 57)), CallTag('call'
                      , {'expr': 'foo<bar and hoho>lala and "x" + "y"'
                      }, (3, 13), []), Text('\n        ', (3, 64))]))

    def test_pagetag(self):
        template = \
            """
            <%page cached="True", args="a, b"/>

            some template
        """
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({}, [Text('\n            ',
                      (1, 1)), PageTag('page', {'args': 'a, b',
                      'cached': 'True'}, (2, 13), []),
                      Text('''

            some template
        ''',
                      (2, 48))]))

    def test_nesting(self):
        template = \
            """

        <%namespace name="ns">
            <%def name="lala(hi, there)">
                <%call expr="something()"/>
            </%def>
        </%namespace>

        """
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({},
                      [Text('''

        ''', (1, 1)),
                      NamespaceTag('namespace', {'name': 'ns'}, (3,
                      9), [Text('\n            ', (3, 31)),
                      DefTag('def', {'name': 'lala(hi, there)'}, (4,
                      13), [Text('\n                ', (4, 42)),
                      CallTag('call', {'expr': 'something()'}, (5,
                      17), []), Text('\n            ', (5, 44))]),
                      Text('\n        ', (6, 20))]),
                      Text('''

        ''', (7, 22))]))

    if compat.py3k:
        def test_code(self):
            template = \
"""text
    <%
        print("hi")
        for x in range(1,5):
            print(x)
    %>
more text
    <%!
        import foo
    %>
"""
            nodes = Lexer(template).parse()
            self._compare(nodes,
            TemplateNode({}, [
                Text('text\n    ', (1, 1)),
                Code('\nprint("hi")\nfor x in range(1,5):\n    '
                            'print(x)\n    \n', False, (2, 5)),
                Text('\nmore text\n    ', (6, 7)),
                Code('\nimport foo\n    \n', True, (8, 5)),
                Text('\n', (10, 7))])
            )


    else:

        def test_code(self):
            template = \
"""text
    <%
        print "hi"
        for x in range(1,5):
            print x
    %>
more text
    <%!
        import foo
    %>
"""
            nodes = Lexer(template).parse()
            self._compare(nodes,
            TemplateNode({}, [
                Text('text\n    ', (1, 1)),
                Code('\nprint "hi"\nfor x in range(1,5):\n    '
                            'print x\n    \n', False, (2, 5)),
                Text('\nmore text\n    ', (6, 7)),
                Code('\nimport foo\n    \n', True, (8, 5)),
                Text('\n', (10, 7))])
            )

    def test_code_and_tags(self):
        template = \
            """
<%namespace name="foo">
    <%def name="x()">
        this is x
    </%def>
    <%def name="y()">
        this is y
    </%def>
</%namespace>

<%
    result = []
    data = get_data()
    for x in data:
        result.append(x+7)
%>

    result: <%call expr="foo.x(result)"/>
"""
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({}, [Text('\n', (1, 1)),
                      NamespaceTag('namespace', {'name': 'foo'}, (2,
                      1), [Text('\n    ', (2, 24)), DefTag('def',
                      {'name': 'x()'}, (3, 5),
                      [Text('''\n        this is x\n    ''', (3, 22))]),
                      Text('\n    ', (5, 12)), DefTag('def', {'name'
                      : 'y()'}, (6, 5),
                      [Text('''\n        this is y\n    ''', (6, 22))]),
                      Text('\n', (8, 12))]), Text('''\n\n''', (9, 14)),
                      Code('''\nresult = []\ndata = get_data()\n'''
                      '''for x in data:\n    result.append(x+7)\n\n''',
                      False, (11, 1)), Text('''\n\n    result: ''', (16,
                      3)), CallTag('call', {'expr': 'foo.x(result)'
                      }, (18, 13), []), Text('\n', (18, 42))]))

    def test_expression(self):
        template = \
            """
        this is some ${text} and this is ${textwith | escapes, moreescapes}
        <%def name="hi()">
            give me ${foo()} and ${bar()}
        </%def>
        ${hi()}
"""
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({},
                      [Text('\n        this is some ', (1, 1)),
                      Expression('text', [], (2, 22)),
                      Text(' and this is ', (2, 29)),
                      Expression('textwith ', ['escapes', 'moreescapes'
                      ], (2, 42)), Text('\n        ', (2, 76)),
                      DefTag('def', {'name': 'hi()'}, (3, 9),
                      [Text('\n            give me ', (3, 27)),
                      Expression('foo()', [], (4, 21)), Text(' and ',
                      (4, 29)), Expression('bar()', [], (4, 34)),
                      Text('\n        ', (4, 42))]), Text('\n        '
                      , (5, 16)), Expression('hi()', [], (6, 9)),
                      Text('\n', (6, 16))]))


    def test_tricky_expression(self):
        template = """

            ${x and "|" or "hi"}
        """
        nodes = Lexer(template).parse()
        self._compare(
            nodes,
            TemplateNode({}, [
                Text('\n\n            ', (1, 1)),
                Expression('x and "|" or "hi"', [], (3, 13)),
                Text('\n        ', (3, 33))
            ])
        )

        template = r"""

            ${hello + '''heres '{|}' text | | }''' | escape1}
            ${'Tricky string: ' + '\\\"\\\'|\\'}
        """
        nodes = Lexer(template).parse()
        self._compare(
            nodes,
            TemplateNode({}, [
                Text('\n\n            ', (1, 1)),
                Expression("hello + '''heres '{|}' text | | }''' ",
                                ['escape1'], (3, 13)),
                Text('\n            ', (3, 62)),
                Expression(r"""'Tricky string: ' + '\\\"\\\'|\\'""",
                                [], (4, 13)),
                Text('\n        ', (4, 49))
            ])
        )

    def test_tricky_code(self):
        if compat.py3k:
            template = """<% print('hi %>') %>"""
            nodes = Lexer(template).parse()
            self._compare(nodes, TemplateNode({},
                          [Code("print('hi %>') \n", False, (1, 1))]))
        else:
            template = """<% print 'hi %>' %>"""
            nodes = Lexer(template).parse()
            self._compare(nodes, TemplateNode({},
                          [Code("print 'hi %>' \n", False, (1, 1))]))

    def test_tricky_code_2(self):
        template = \
            """<%
        # someone's comment
%>
        """
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({},
                      [Code("""
        # someone's comment

""",
                      False, (1, 1)), Text('\n        ', (3, 3))]))

    if compat.py3k:
        def test_tricky_code_3(self):
            template = \
                """<%
            print('hi')
            # this is a comment
            # another comment
            x = 7 # someone's '''comment
            print('''
        there
        ''')
            # someone else's comment
%> '''and now some text '''"""
            nodes = Lexer(template).parse()
            self._compare(nodes, TemplateNode({},
                          [Code("""
print('hi')
# this is a comment
# another comment
x = 7 # someone's '''comment
print('''
        there
        ''')
# someone else's comment

""",
                          False, (1, 1)),
                          Text(" '''and now some text '''", (10,
                          3))]))
    else:
        def test_tricky_code_3(self):
            template = \
                """<%
            print 'hi'
            # this is a comment
            # another comment
            x = 7 # someone's '''comment
            print '''
        there
        '''
            # someone else's comment
%> '''and now some text '''"""
            nodes = Lexer(template).parse()
            self._compare(nodes, TemplateNode({},
                      [Code("""\nprint 'hi'\n# this is a comment\n"""
                      """# another comment\nx = 7 """
                      """# someone's '''comment\nprint '''\n        """
                      """there\n        '''\n# someone else's """
                      """comment\n\n""",
                      False, (1, 1)),
                      Text(" '''and now some text '''", (10, 3))]))

    def test_tricky_code_4(self):
        template = \
            """<% foo = "\\"\\\\" %>"""
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({},
                      [Code("""foo = "\\"\\\\" \n""",
                      False, (1, 1))]))

    def test_tricky_code_5(self):
        template = \
            """before ${ {'key': 'value'} } after"""
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({},
                      [Text('before ', (1, 1)),
                      Expression(" {'key': 'value'} ", [], (1, 8)),
                      Text(' after', (1, 29))]))

    def test_tricky_code_6(self):
        template = \
            """before ${ (0x5302 | 0x0400) } after"""
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({},
                      [Text('before ', (1, 1)),
                      Expression(" (0x5302 | 0x0400) ", [], (1, 8)),
                      Text(' after', (1, 30))]))

    def test_control_lines(self):
        template = \
            """
text text la la
% if foo():
 mroe text la la blah blah
% endif

        and osme more stuff
        % for l in range(1,5):
    tex tesl asdl l is ${l} kfmas d
      % endfor
    tetx text

"""
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({},
                      [Text('''\ntext text la la\n''', (1, 1)),
                      ControlLine('if', 'if foo():', False, (3, 1)),
                      Text(' mroe text la la blah blah\n', (4, 1)),
                      ControlLine('if', 'endif', True, (5, 1)),
                      Text('''\n        and osme more stuff\n''', (6,
                      1)), ControlLine('for', 'for l in range(1,5):',
                      False, (8, 1)), Text('    tex tesl asdl l is ',
                      (9, 1)), Expression('l', [], (9, 24)),
                      Text(' kfmas d\n', (9, 28)), ControlLine('for',
                      'endfor', True, (10, 1)),
                      Text('''    tetx text\n\n''', (11, 1))]))

    def test_control_lines_2(self):
        template = \
"""% for file in requestattr['toc'].filenames:
    x
% endfor
"""
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({}, [ControlLine('for',
                      "for file in requestattr['toc'].filenames:",
                      False, (1, 1)), Text('    x\n', (2, 1)),
                      ControlLine('for', 'endfor', True, (3, 1))]))

    def test_long_control_lines(self):
        template = \
        """
    % for file in \\
        requestattr['toc'].filenames:
        x
    % endfor
        """
        nodes = Lexer(template).parse()
        self._compare(
            nodes,
            TemplateNode({}, [
                Text('\n', (1, 1)),
                ControlLine('for', "for file in \\\n        "
                                "requestattr['toc'].filenames:",
                                False, (2, 1)),
                Text('        x\n', (4, 1)),
                ControlLine('for', 'endfor', True, (5, 1)),
                Text('        ', (6, 1))
            ])
        )

    def test_unmatched_control(self):
        template = """

        % if foo:
            % for x in range(1,5):
        % endif
"""
        assert_raises_message(
            exceptions.SyntaxException,
            "Keyword 'endif' doesn't match keyword 'for' at line: 5 char: 1",
            Lexer(template).parse
        )

    def test_unmatched_control_2(self):
        template = """

        % if foo:
            % for x in range(1,5):
            % endfor
"""

        assert_raises_message(
            exceptions.SyntaxException,
            "Unterminated control keyword: 'if' at line: 3 char: 1",
            Lexer(template).parse
        )

    def test_unmatched_control_3(self):
        template = """

        % if foo:
            % for x in range(1,5):
            % endlala
        % endif
"""
        assert_raises_message(
            exceptions.SyntaxException,
            "Keyword 'endlala' doesn't match keyword 'for' at line: 5 char: 1",
            Lexer(template).parse
        )

    def test_ternary_control(self):
        template = \
            """
        % if x:
            hi
        % elif y+7==10:
            there
        % elif lala:
            lala
        % else:
            hi
        % endif
"""
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({}, [Text('\n', (1, 1)),
                      ControlLine('if', 'if x:', False, (2, 1)),
                      Text('            hi\n', (3, 1)),
                      ControlLine('elif', 'elif y+7==10:', False, (4,
                      1)), Text('            there\n', (5, 1)),
                      ControlLine('elif', 'elif lala:', False, (6,
                      1)), Text('            lala\n', (7, 1)),
                      ControlLine('else', 'else:', False, (8, 1)),
                      Text('            hi\n', (9, 1)),
                      ControlLine('if', 'endif', True, (10, 1))]))

    def test_integration(self):
        template = \
            """<%namespace name="foo" file="somefile.html"/>
 ## inherit from foobar.html
<%inherit file="foobar.html"/>

<%def name="header()">
     <div>header</div>
</%def>
<%def name="footer()">
    <div> footer</div>
</%def>

<table>
    % for j in data():
    <tr>
        % for x in j:
            <td>Hello ${x| h}</td>
        % endfor
    </tr>
    % endfor
</table>
"""
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({}, [NamespaceTag('namespace'
                      , {'file': 'somefile.html', 'name': 'foo'},
                      (1, 1), []), Text('\n', (1, 46)),
                      Comment('inherit from foobar.html', (2, 1)),
                      InheritTag('inherit', {'file': 'foobar.html'},
                      (3, 1), []), Text('''\n\n''', (3, 31)),
                      DefTag('def', {'name': 'header()'}, (5, 1),
                      [Text('''\n     <div>header</div>\n''', (5,
                      23))]), Text('\n', (7, 8)), DefTag('def',
                      {'name': 'footer()'}, (8, 1),
                      [Text('''\n    <div> footer</div>\n''', (8,
                      23))]), Text('''\n\n<table>\n''', (10, 8)),
                      ControlLine('for', 'for j in data():', False,
                      (13, 1)), Text('    <tr>\n', (14, 1)),
                      ControlLine('for', 'for x in j:', False, (15,
                      1)), Text('            <td>Hello ', (16, 1)),
                      Expression('x', ['h'], (16, 23)), Text('</td>\n'
                      , (16, 30)), ControlLine('for', 'endfor', True,
                      (17, 1)), Text('    </tr>\n', (18, 1)),
                      ControlLine('for', 'endfor', True, (19, 1)),
                      Text('</table>\n', (20, 1))]))

    def test_comment_after_statement(self):
        template = \
            """
        % if x: #comment
            hi
        % else: #next
            hi
        % endif #end
"""
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({}, [Text('\n', (1, 1)),
                      ControlLine('if', 'if x: #comment', False, (2,
                      1)), Text('            hi\n', (3, 1)),
                      ControlLine('else', 'else: #next', False, (4,
                      1)), Text('            hi\n', (5, 1)),
                      ControlLine('if', 'endif #end', True, (6, 1))]))

    def test_crlf(self):
        template = util.read_file(self._file_path("crlf.html"))
        nodes = Lexer(template).parse()
        self._compare(
            nodes,
            TemplateNode({}, [
                Text('<html>\r\n\r\n', (1, 1)),
                PageTag('page', {
                            'args': "a=['foo',\n                'bar']"
                        }, (3, 1), []),
                Text('\r\n\r\nlike the name says.\r\n\r\n', (4, 26)),
                ControlLine('for', 'for x in [1,2,3]:', False, (8, 1)),
                Text('        ', (9, 1)),
                Expression('x', [], (9, 9)),
                ControlLine('for', 'endfor', True, (10, 1)),
                Text('\r\n', (11, 1)),
                Expression("trumpeter == 'Miles' and "
                                "trumpeter or \\\n      'Dizzy'",
                                [], (12, 1)),
                Text('\r\n\r\n', (13, 15)),
                DefTag('def', {'name': 'hi()'}, (15, 1), [
                    Text('\r\n    hi!\r\n', (15, 19))]),
                    Text('\r\n\r\n</html>\r\n', (17, 8))
                ])
        )
        assert flatten_result(Template(template).render()) \
            == """<html> like the name says. 1 2 3 Dizzy </html>"""

    def test_comments(self):
        template = \
            """
<style>
 #someselector
 # other non comment stuff
</style>
## a comment

# also not a comment

   ## this is a comment

this is ## not a comment

<%doc> multiline
comment
</%doc>

hi
"""
        nodes = Lexer(template).parse()
        self._compare(nodes, TemplateNode({},
                      [Text('''\n<style>\n #someselector\n # '''
                        '''other non comment stuff\n</style>\n''',
                      (1, 1)), Comment('a comment', (6, 1)),
                      Text('''\n# also not a comment\n\n''', (7, 1)),
                      Comment('this is a comment', (10, 1)),
                      Text('''\nthis is ## not a comment\n\n''', (11,
                      1)), Comment(''' multiline\ncomment\n''', (14,
                      1)), Text('''

hi
''', (16, 8))]))

    def test_docs(self):
        template = \
            """
        <%doc>
            this is a comment
        </%doc>
        <%def name="foo()">
            <%doc>
                this is the foo func
            </%doc>
        </%def>
        """
        nodes = Lexer(template).parse()
        self._compare(nodes,
            TemplateNode({}, [Text('\n        ', (1,
              1)),
              Comment('''\n            this is a comment\n        ''',
              (2, 9)), Text('\n        ', (4, 16)),
              DefTag('def', {'name': 'foo()'}, (5, 9),
              [Text('\n            ', (5, 28)),
              Comment('''\n                this is the foo func\n'''
                '''            ''',
              (6, 13)), Text('\n        ', (8, 20))]),
              Text('\n        ', (9, 16))]))

    def test_preprocess(self):

        def preproc(text):
            return re.sub(r'(?<=\n)\s*#[^#]', '##', text)

        template = \
            """
    hi
    # old style comment
# another comment
"""
        nodes = Lexer(template, preprocessor=preproc).parse()
        self._compare(nodes, TemplateNode({}, [Text('''\n    hi\n''',
                      (1, 1)), Comment('old style comment', (3, 1)),
                      Comment('another comment', (4, 1))]))