import sre_parse, sre_compile, sys

class Scanner:
    def __init__(self, lexicon, flags=0):
        from sre_constants import BRANCH, SUBPATTERN
        self.lexicon = lexicon
        # combine phrases into a compound pattern
        p = []
        s = sre_parse.Pattern()
        s.flags = flags
        for phrase, action in lexicon:
            p.append(sre_parse.SubPattern(s, [
                (SUBPATTERN, (len(p)+1, sre_parse.parse(phrase, flags))),
                ]))
        p = sre_parse.SubPattern(s, [(BRANCH, (None, p))])
        s.groups = len(p)
        self.scanner = sre_compile.compile(p)
    def scan(self, string):
        result = []
        append = result.append
        match = self.scanner.scanner(string).match
        i = 0
        while 1:
            m = match()
            if not m:
                break
            j = m.end()
            if i == j:
                break
            action = self.lexicon[m.lastindex-1][1]
            if callable(action):
                self.match = m
                action = action(self, m.group())
            if action is not None:
                append(action)
            i = j
        return result, string[i:]

class token:
	tokens = []
	def __init__(self, scanner, value):
		self.val = value
		self.line = scanner.line
		token.tokens.append(self)
	def __str__(self):
		return '<Token type:%s val:%s>' % (self.__class__, self.val)

### identifier class
class t_ident(token):
	def __init__(self, scanner, value):
		token.__init__(self, scanner, value)

### number classes
class t_num(token):
	def __init__(self, scanner, value):
		token.__init__(self, scanner, value)
		
class t_int(token):
	def __init__(self, scanner, value):
		token.__init__(self, scanner, value)
		self.val = int(value)
		
class t_float(token):
	def __init__(self, scanner, value):
		token.__init__(self, scanner, value)
		self.val = float(value)

### operator class
class t_op(token):
	def __init__(self, scanner, value):
		token.__init__(self, scanner, value)

### disposables
def t_newline(scanner, value):
	scanner.line += 1
	return None

def t_whitespace(scanner, value):
	return None
	
### other classes

class t_lbrac(token):
	def __init__(self, scanner, value):
		token.__init__(self, scanner, value)

class t_rbrac(token):
	def __init__(self, scanner, value):
		token.__init__(self, scanner, value)

ident = r'[a-zA-Z_][a-zA-Z0-9_]*'
num = r'[0-9]+'
integer_num = r'[\+-]?[0-9]+'
float_num = r'[\+-]?[0-9]+\.[0-9]*'
operator = '[\+\-*/\^<>=%]'
newline = r'\n+'
whitespace = r'[ \t]+'
lbrac = r'\('
rbrac = r'\)'

patterns  = [
	(ident, t_ident),
	(float_num, t_float),
	(integer_num, t_int),
	(operator, t_op),
	(newline, t_newline),
	(whitespace, t_whitespace),
	(lbrac, t_lbrac),
	(rbrac, t_rbrac)
	]

class lexer:

	def __init__(self, src, patterns):
		self.scanner = Scanner(patterns)
		self.scanner.line = 1
		self.tokens = self.scanner.scan(src)[0]
		self.src = src
		self.lines = src.split('\n')
		self.idx = 0
	
	def next(self):
		self.token = self.tokens[self.idx]
		self.idx += 1
		return self.token
		
test = lexer('print range(30)', patterns)
print test.next()