(define call/cc call-with-current-continuation)
(define 1+ (lambda (n) (+ 1 n)))
(define last-cons (lambda (l) (list-tail l (- (length l) 1))))
(define last (lambda (l) (list-ref l (- (length l) 1))))

(define string-lexer
  (lambda (s tokens)
    (letrec
        ((get-next
          (lambda (i previous-match result)
            (let
                ((match
                  (call/cc
                   (lambda (return)
                     (map (lambda (token)
                            (if (member (string-ref s i) (string->list token))
                                (return token)))
                          tokens)
                     (return "")))))
              (if (< (1+ i) (string-length s))
                  (get-next (1+ i)
                            match
                            (if (string=? match previous-match)
                                (begin
                                  (set-car! (last-cons result)
                                            (string-append (last result)
                                                           (string (string-ref s i))))
                                  result)
                                (append result (cons (string (string-ref s i)) '()))))
                  result)))))
      (get-next 0 "" '()))))

; this function leaves the last letter off, like so:
;
; > (string-lexer "huk asdf    t huk" '("huk" "fdsa"))
; ("huk" " " "asdf" "    t " "hu")
;
; i cannot figure out why, if i make the conditional
; (< i (string-length s)) it makes an index out of range
; error. also tell me the stupid things i am doing in
; this code :|