Yaptoo (Yaptu outrageously obfuscated) - or yet yet another templating utility

'''
Yaptoo (Yaptu Outrageously Obfuscated) by Michael Palmer
based on: Yaptu (http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52305) by Alex Martelli

Changes from Yaptu:
- separated template compilation from merging
- added some error reporting
- changed default template syntax (It remains easy to make your own by defining a bunch of regexes)
- added syntax for comments
- added Cheetah-style variable substitution
- limited flow control to 'for' and 'if'
- added 'include' function

Limitations:
- Statements, expressions, or comments cannot span multiple lines
- In 'for' loops, there is neither 'break' nor 'continue'
- No 'controller' behaviour of any kind, i.e. you cannot run templates in a 'standalone' fashion.

Yaptoo is intended solely for use in an auxiliary role.
It is lightweight and fast.
'''

import re, os.path
from cStringIO import StringIO

try:
    from traceback import format_exc
except: # lacking before 2.4
    def format_exc():
        s = cStringIO.StringIO()
        traceback.print_exc(file=s)
        return s.getvalue()


class YaptooError(Exception):
    pass


class YaptooErrorHandler(object):
    '''
    used per inheritance by both Template and _Merger classes
    '''
    verboseErrors = False

    def error(self, offense, comment=None):
        '''
        wrap Exception so that user better sees what happened where.
        '''

        packagedInfo=['\n---------------------']
        if comment:
            packagedInfo.append(comment)
        packagedInfo.append("Offensive statement or expression: %s" % offense)
        packagedInfo.append('Underlying exception:')
        packagedInfo.append(format_exc().splitlines()[-1])

        # annotate with the proper source line
        stmt, src, num = self.lines[self.currentLine]
        packagedInfo.append("\nSource line: %s \nLine number: %d\nSource file: %s" % (stmt.strip(), num+1, src) )

        if self.verboseErrors and hasattr(self, 'globals'):
            packagedInfo.append('Variables available when exception occured:')

            globs = self.globals.copy()
            # remove confusing things from the namespace...
            globs.pop('__builtins__', None)
            globs.pop('_mergeBlock__', None)
            globs.pop('_resolveSimple__', None)

            items = globs.items()
            items.sort()

            def printNice(item, indent=20):
                from pprint import pformat
                out = []
                fill = ' ' * indent

                k,v = item
                dataList = pformat(v).split('\n')
                out=[]
                out.append(k.ljust(indent) + dataList.pop(0))
                for l in dataList:
                    out.append(fill + l)
                return '\n'.join(out)

            for it in items:
                packagedInfo.append('' + printNice(it))

        packagedInfo.append('---------------------')

        raise YaptooError, '\n'.join(packagedInfo)


class Template(YaptooErrorHandler):
    '''
    Base class for compiling template definition files or strings
    '''
    ## Begin template syntax definition

    # 'expressionRegex': this regex is used to identify Python expressions
    # I like this style, it stands out well in HTML
    # however, you need to avoid  ']]' in python expressions such as lists of lists,
    # e.g. by inserting a space:  [[a,b], [c,d] ]
    # alternatively you could use
    # re.compile("\<\<\s*(.*?)\s*\>\>"),                    # example: << i**33 >>
    #  - safe, but does not stand out well in html
    expressionRegex = re.compile("\[\[\s*(.*?)\s*\]\]")     # example: [[ i**33 ]]

    # this regex identifies comments (comments will be stripped from the output)
    commentRegex = re.compile('\s*##.*')                    # like so: this will print ## but this won't

    # include directives
    includeRegex = re.compile(r'\s*#include\s+([\/\\]?\w+(\.\w+)*([\/\\]\w+(\.\w+)*)*)')

    # this regex is for flow control statements. It must capture the statement itself but nothing else.
    # here, statements are prefixed with #:  #for x in y:   #end for, #if, #else:  , #end if
    statementRegex = re.compile(r'\s*#(for .+?:|if .+?:|elif .+?:|else:|end (?:if|for))\s*')

    # these two regexes are solely for catching a specific unworkable syntax in for-in statements
    forInDottedRe = re.compile('for\s.*?\$[^\,]*?\..*?\sin')  #
    # the regex below depends on the one above
    forInRe = re.compile('(for\s.*?\sin)(.+)', re.DOTALL)

    # a regex that will catch other Miss Happen-Statements. Note that on any line it will be tried
    # only after statementRegex, so it doesn't hurt if the regex by itself would also match valid statements
    faultyStatement = re.compile(r'\s*#.*')            # will capture anything like '#howdi, rowdy!',

    # this marker at the end of a line will consume the linebreak and all subsequent continuous whitespace
    joinLines = re.compile('\:\>\s+', re.DOTALL)            # :>

    # good ol $varname substitution
    simpleSubstitutionRegex = re.compile("\$((?:[a-zA-Z_]\w*)(?:\.\$?[a-zA-Z_]\w*)*)")

    ## End template syntax definition

    def __init__( self,
                  sourceString=None,
                  sourceFile=None,
                  templateDir='',
                  stripEmptyLines = False,
                  renderMissingNames = False,
                  renderMissingFormat = '<span style="color:red">%s</span>'
                ):

        self.templateDir = templateDir
        self.stripEmptyLines = stripEmptyLines
        self.renderMissingNames = renderMissingNames
        self.renderMissingFormat = renderMissingFormat

        assert(sourceString and not sourceFile) or (sourceFile and not sourceString), 'Must pass either file or string, not both'

        # compile the template
        source = sourceString or sourceFile

        # figure out whether we have a string or a file
        if sourceFile:
            if not hasattr(sourceFile, 'readlines'):        # assume it's a file name
                try:
                    sourceFile = open(os.path.join(self.templateDir, sourceFile))
                    sourceName = sourceFile
                except:
                    self.error(sourceFile, 'could not open file')
            else:
                if hasattr(sourceFile, 'name'):
                    sourceName = sourceFile.name
                else:   # it could be a StringIO or somthin
                    sourceName = '(main template string)'
        elif sourceString:
            sourceFile = StringIO(sourceString)
            sourceName = '(main template string)'

        # load file by lines and recursively expand includes, strip comments
        self.lines = self._preprocessSource(sourceFile, sourceName)

        # compile the loaded template
        self.length = len(self.lines)
        self._compiledLines = [0] * self.length # dummy list because elements won't be assigned in order
        self._compile(0, self.length)


    def merge(self, *data):
        '''
        wrapper around _Merger class, needed for thread safety
        (_Merger instances aren't threadsafe, so we just throw them away after single use)
        '''
        return _Merger(self, *data)._merge()


    def renderValue(self, val):
        '''
        this is a hook in which you can implement all kinds of fancy custom rendering
        for your own objects. The default is just to apply built-in 'str'.
        '''
        return str(val)


    def _preprocessSource(self, sourceFile, sourceName):
        '''
        load file by lines, strip comments, recursively expand include instructions
        keep track of the origin of each line
        '''
        rawLines = sourceFile.readlines()
        sourceFile.close()

        processed = []

        for lineNumber,line in enumerate(rawLines):
            commentStripped = self.commentRegex.sub('', line)
            if not self.stripEmptyLines or commentStripped.strip():
                 processed.append((commentStripped, sourceName, lineNumber))

        # now, check whether we have any include files
        for x in range(len(processed)-1, -1, -1):    # go backwards b/c we will insert more lines
            line = processed[x][0]
            matched = self.includeRegex.match(line)
            if matched:
                includeFileName = matched.group(1)
                fullName = os.path.join(self.templateDir, includeFileName)
                includeFile = open(fullName)
                processed[x:x+1] = self._preprocessSource(includeFile, includeFileName)
        return processed


    def _preprocessPython(self, python):
        '''
        preprocess python statements or expressions to deal with
        interspersed simplified syntax. Helper for _compile.
        '''
        def subst(mo):
            expr = mo.group(1)
            return "_resolveSimple__('%s')" % expr

        isFor = self.forInRe.match(python)
        if isFor:
            forClause, restClause = isFor.group(1), isFor.group(2)
            if self.forInDottedRe.match(forClause):
                self.error(python,
                       "Sorry, Yaptoo cannot handle $-style with dots in loop control variables." + \
                       "Please use explicit Python syntax (e.g use #for x['y'] instead of #for $x.y )"
                          )
            # if we got here, there are no dotted $-style expressions in 'for .. in'
            return forClause.replace('$','') + self.simpleSubstitutionRegex.sub(subst, restClause)
        else:
            return self.simpleSubstitutionRegex.sub(subst, python)

    def _compile(self, i, last):
        '''
        recursively compile the template definition
        '''
        while i < last:
            self.currentLine = i  # needed for error reporting and for caching compiled flow control statements

            line = self.lines[i][0]

            stmt = self.statementRegex.match(line)
            if stmt:
                statement = stmt.group(1)
                firstWord = statement.split()[0]

                if not firstWord in ['if', 'for']:
                    self.error(statement, '%s cannot start a block' % firstWord)

                statementLines = [i]        # record all statements at this level, use as boundaries for recursive compiling

                j = i+1                     # j is the first line contained in this block
                nest = 1                    # count nesting levels of statements

                while j<last:               # look for continuation or end of the block
                    line = self.lines[j][0]
                    stmt = self.statementRegex.match(line)

                    if stmt:
                        followingStatement = stmt.group(1)
                        words = followingStatement.split()

                        if words[0] == 'end':       # found a statement-end
                            nest -= 1

                            if nest == 0:           # this clause ends the current block
                                endWhich = words[1]
                                if endWhich != firstWord:
                                    self.error("Block delimiter mismatch: '%s' / 'end %s'" % (firstWord, endWhich))
                                statementLines.append(j)
                                break

                        elif words[0] in ['if', 'for']:    # begin of a nested statement
                            nest += 1

                        elif nest == 1 and words[0] in ['else:', 'elif']: # look for continuation only at this nesting level
                            if words[0] == 'elif' and firstWord != 'if':
                                self.error("Block delimiter mismatch: '%s' / '%s'" %  (firstWord, words[0]))

                            statementLines.append(j)
                            # create a compound statement ('if elif else', 'for else')
                            statement += '_mergeBlock__(%s,%s)\n%s' % (i+1, j, followingStatement)
                            i = j
                    j += 1

                if nest > 0:
                    self.error(self.lines[self.currentLine], "Missing statement terminator somewhere inside of '%s' block" % firstWord)
                statement += "_mergeBlock__(%s,%s)" % (i+1, j)
                nextLineNo = j+1

                expanded = self._preprocessPython(statement)
                try:
                    compiled = compile(expanded,'<template>','exec')
                    if expanded == statement:
                        self._compiledLines[self.currentLine] = ('exec', (expanded, compiled), nextLineNo)
                    else:
                        self._compiledLines[self.currentLine] = ('exec',
                                    ('%s \nexpanded to:\n%s)' % (statement, expanded),
                                    compiled),
                                nextLineNo)

                except SyntaxError:
                    if expanded != statement:
                        self.error('\n' + expanded, 'Syntax error in flow control statement (expanded from %s)' % statement)
                    else:
                        self.error('\n' + statement, 'Syntax error in flow control statement')

                # now, compile the bits and pieces between the flow control statements of the current block
                for n in range(len(statementLines)-1):
                    startNested, endNested = statementLines[n] + 1, statementLines[n+1]
                    self._compile(startNested, endNested)

                i = nextLineNo


            elif self.faultyStatement.match(line):
                self.error(self.faultyStatement.match(line).group().strip(), 'wrong statement syntax')

            else:       # normal line, copy with substitution. lines can contain
                # - arbitrary python expressions that will be 'evaled'
                # - "$varname.attribute.item.$x" style identifiers
                # -  plain strings

                lineTuples = []     # collect all the constituents of a line as (dispatchKey, funcArg) tuples

                # first, break the line up into marked-up python expressions and intervening strings.
                # this will yield a list that alternatingly contains strings and expressions
                firstFrags = self.expressionRegex.split(line)

                for ff in range(len(firstFrags)):
                    firstItem = firstFrags[ff]

                    if not ff % 2:
                        # even numbers will contain text frags, break them up again to extract the simple-subst. identifiers
                        secondFrags = self.simpleSubstitutionRegex.split(firstItem)

                        for sf in range(len(secondFrags)):
                            item = secondFrags[sf]

                            if not sf % 2 and item: # again, the even numbers will contain plain text
                                lineTuples.append(('text', item))
                            elif sf % 2:   # this is an identifier for simplified syntax substitution.
                                lineTuples.append(('simple', item))

                    else:   # this is a python expression for eval'ing
                        expanded = self._preprocessPython(firstItem)
                        try:
                            compiled = compile(expanded, '<template>', 'eval')
                        except SyntaxError:
                            if expanded != firstItem:
                                self.error(expanded, 'Error compiling Python expression (expanded from %s)' % firstItem)
                            else:
                                self.error(expanded, 'Error compiling Python expression')
                        if expanded == firstItem:
                            lineTuples.append(('eval', (firstItem, compiled)))
                        else:
                            lineTuples.append(('eval',
                                                 ('%s\nexpanded from:\n%s' % (expanded, firstItem),
                                               compiled)))

                self._compiledLines[i] = ('content', lineTuples, i+1)
                i += 1

class _Merger(YaptooErrorHandler):
    '''
    merge a new data set into a compiled template. This class is NOT threadsafe and not intended to be used directly.
    Instead, use Template.merge (which is threadsafe b/c it makes and throws away a fresh _Merger instance on every call).
    '''

    def __init__(self, template, *data):
        '''
        fill the template with a data set and return the string. you may pass one or more dictionaries
        which will be searched on order for the variables in the template.
        '''
        self._t = template
        self._compiledLines = template._compiledLines
        self.renderMissingNames = template.renderMissingNames   # needed by ErrorHandler
        self.renderMissingFormat = template.renderMissingFormat
        self.lines = template.lines

        self.globals = {}
        for x in range(len(data)-1, -1, -1):  # update in reverse order to achieve 'search list' behaviour
            self.globals.update(data[x])

        # add some magic to the global namespace, needed by compiled exec-statements
        self.globals['_mergeBlock__'] = self._mergeBlock
        self.globals['_resolveSimple__'] = self._resolveSimple

        # append output here
        self.out = []

        # method switchers
        self.lineDispatch = {                            # deal with the items that make up a line
                            'text'   : self.out.append,
                            'eval'   : self._evalSubst,
                            'simple' : self._simpleSubst,
                        }

        self.blockDispatch = {                           # deal with the various lines in a block
                            'content': self._mergeLine,
                            'exec'   : self._execute,
                        }

    def _merge(self):
        '''
        Don't use directly - use Template.merge instead
        '''
        self._mergeBlock(0, self._t.length)
        return self.postProcess(''.join(self.out))

    def _mergeBlock(self, first, last):
        '''
        merge a block of lines. Helper for merge.
        '''
        lineNumber = first
        while lineNumber < last:
            self.currentLine = lineNumber
            instruction, args, lineNumber = self._compiledLines[lineNumber]
            self.blockDispatch[instruction](args)   # this will call '_execute' or '_mergeLine'

    def _execute(self, execStuff):
        '''
        execute a precompiled flow control statement. Helper for merge.
        '''
        execStatement, compiled = execStuff
        try:
            exec compiled in self.globals
        except YaptooError:
            raise       # let errors from nested code propagate, as they have already been annotated
        except:         # error was caused here
            self.error('\n'+execStatement, 'Error while executing compiled flow control statement')

    def _mergeLine(self, lineTuples):
        '''
        merge a previously compiled content (as opposed to flow control) line.
        Called via blockDispatch['content']
        '''
        for key, funcArg in lineTuples:
            self.lineDispatch[key](funcArg) # this will call out.append, _evalSubst, or _simpleSubst

    def _evalSubst(self, tup):
        '''
        eval an arbitrary Python exp and insert the value
        '''
        try:
            val = eval(tup[1], self.globals)
        except: # tell user what occurred and where
            if self.renderMissingNames:
                val = self.renderMissingFormat % tup[0]
            else:
                self.error(tup[0], "Error while eval'ing compiled Python expression")
        if val == None:
            val = ''
        self._renderValue(val)


    def _simpleSubst(self, expr):
        '''
        resolve an expression like '$hans.wurst.senf' and append its value to the output
        this is used from outside statements and expressions
        '''
        try:
            val = self._resolveSimple(expr)
        except: # tell user what occurred and where
            if self.renderMissingNames:
                val = self.renderMissingFormat % ('$' + expr)
            else:
                raise
        self._renderValue(val)


    def _resolveSimple(self, expr):
        '''
        resolve $-style expressions
        '''
        frags = expr.split('.')

        try:
            resolved = self.globals[frags[0]]
        except KeyError:
            self.error(frags[0], 'missing name %s' % frags[0])

        for i in range(1, len(frags)):
            frag = frags[i]

            if frag.startswith('$'):    # refers to a global variable
                frag = frag[1:]         # cut it off...
                isGlobal = True
                try:
                    term = self.globals[frag]
                except KeyError:
                    self.error(frag, 'missing name %s' % frag)
            else:
                isGlobal = False
                term = frag

            try:
                kid = resolved[term]
                # o.k., it's there...
            except TypeError:
                self.error(expr, "\nTried to look up key '%s' in wrong type of container '%s'" % (frag, frags[i-1]))

            except (KeyError, AttributeError):
                if not isGlobal:
                    try:    # second, try to get it as an attribute
                        kid = getattr(resolved, frag)
                        # no, it's here...
                    except AttributeError:
                        self.error(expr, "\nCan't find item or attribute '%s' in '%s'" % (frag, '.'.join(frags[:i])))
                else:
                    self.error(expr, "\nCan't find item '%s' in '%s'" % (frag, '.'.join(frags[:i])))

            resolved = kid    # repeat for next round

        return resolved

    def _renderValue(self, val):
        '''
        wrapper for Template.renderValue
        '''
        self.out.append(self._t.renderValue(val))

    def postProcess(self, bigString):
        '''
        apply last-ditch changes to the merged output string.
        for now, just throw out the line joiners with their newlines
        '''
        return self._t.joinLines.sub('', bigString)


#########################################################
if __name__ == '__main__': # simple usage demonstration

    tmpl='''
    <html>
    <body>
    #include includeTest.tmpl
    <table>
    #for $x, $y in $rows:
        <tr><td>$x</td><td>$y</td></tr>
    #end for
    #for $key in $theDict:
        <tr><td>$key</td><td>$theDict.$key</td></tr>
    #end for
    #for $row in $listOfDicts:
      #if 'Nickname' in $row:
        <tr><td>$row.FirstName</td><td>$row.Nickname</td></tr>
      #end if
    #end for
    </table>
    </body>
    </html>
    '''
    def test():
        rows = [('spam', 'eggs'), ('more spam', 'more eggs')]
        theDict = {'some':'spam and eggs', 'more':'more spam and eggs'}
        listOfDicts = [
            {'FirstName':'Uwe', 'LastName':'Seeler', 'Nickname':'Uns Uwe'},
            {'FirstName':'Gerd', 'LastName':'Mueller', 'Nickname':'Der Bomber'}
        ]

        t = Template(sourceString=tmpl)
        return t.merge(locals())

    print test()