#!/usr/bin/env python
# -*- coding: utf-8 -*-

#error, debug and warning
from functions import *

#The lexer/tokeniser object
class Lexer(object):
	u"""
	The lexer/tokeniser class. use the tokenise(string) method to return a list of tokens
	Theoretically bug-free, it's hard to produce a hard to find bug in so few lines of code and it's been extensively tested
	"""
	def __init__(self):
		u"""
		Just sets up a couple of constants
		"""
		self.__forbidden_in_identifiers = ['"', "'", "[", "]", "(", ")", "{", "}"]

	def tokenise(self, string):
		u"""
		Returns a list of all tokens in a string.
		Token format is [a, b] where:
		a is a character representing the token type
		b (when present) is the token itself
		"""
		ret = []
		tokens = []
		ret = self.__next_token(string)
		while ret != []:
			tokens.append(ret[1:])
			string = ret[0]
			ret = self.__next_token(string)
		return tokens

	def __next_token(self, string):
		u"""
		Returns the next token in the string and
		the string with the token trimmed off the front
		"""
		if len(string) == 0: return []
		#Skip whitespace
		while ord(string[0]) < 33:
			string = string[1:]
		ch = string[0]
		#What is it?
		if ch == "(":
			return [string[1:], "("]
		elif ch == ")":
			return [string[1:], ")"]
		elif ch == "[":
			return [string[1:], "["]
		elif ch == "]":
			return [string[1:], "]"]
		elif ch == "{":
			return [string[1:], "{"]
		elif ch == "}":
			return [string[1:], "}"]
		elif ch == '"':
			[collectedstring, remnants] = self.__collect_string(string[1:], '"')
			return [remnants, '"', collectedstring]
		elif ch == "'":
			[collectedstring, remnants] = self.__collect_string(string[1:], "'")
			return [remnants, "'", collectedstring]
		elif ch == ":":
			return [string[1:], "'"]
		else:
			[collectedidentifier, remnants] = self.__collect_identifier(string) 
			return [remnants, "i", collectedidentifier]
			
	def __collect_string(self, string, quote):
		u"""
		Returns the contents of the string we started collecting
		Also returns the tail
		"""
		escape = False
		chars = []
		count = 1
		for ch in string:
			count += 1
			if escape:
				if ch == quote:
					chars.append(quote)
				else:
					warning("Unrecognised string escape sequence: \\%s" % (ch))
				escape = False
			elif ch == quote:
				return ["".join(chars), string[count:]]
			elif ch == '\\':
				escape = True
			else:
				chars.append(ch)
		#Crap, we ran out of string
		error("Ran out of string to collect, incomplete statement")
			

	def __collect_identifier(self, string):
		u"""
		Returns the rest of an identifier name and the trailing string
		Identifiers can be anything but:
		-whitespace
		-brackets (of any sort)
		-quote symbols (all sorts)
		"""
		chars = []
		count = 0
		for ch in string:
			count += 1
			if self.__forbidden_in_identifiers.count(ch) > 0:
				return ["".join(chars), string[count-1:]]
			elif ord(ch) > 32:
				chars.append(ch)
			else:
				return ["".join(chars), string[count:]]
		return ["".join(chars), string[count:]]