Class InputTokenizer Object text pos
Methods Tokenizer
new
	super new.
	pos <- 0.
	text <- ''
|
input: inputString
	pos <- 1.
	text <- inputString.
|
next: recognizer
	(pos = ((text size) + 1))
	ifTrue:
	[
		^nil.
	].
	recognizer setNextToken: (text at: pos).
	whileFalse (recognizer hasValidValue)
	[
		pos <- (pos + 1).
		(pos = ((text size) + 1))
		ifTrue:
		[
			^nil.
		].
		ifFalse: [recognizer setNextToken: (text at: pos)].
	]
	^(recognizer value).
|
nextToken
	(pos = (text size))
	ifTrue:
	[
		^nil.
	]
	ifFalse:
	[
		pos <- (pos + 1).
		^(text at: pos).
	].
]