-- 
-- This file Copyright 2001 Jeffrey B Putnam
-- Please Contact me - jefu@eou.edu - for information 
-- 
data Exp = Num String | Var | Op String Exp Exp | Fun String Exp | Empty 

tokenize [] [] le =  le   

tokenize l lr le =   rl 
                  where 
                     (t,l1) = getToken l 
                     rl = tokenize l1 (t:lr) le 
	       
getToken []  = ([], []) 
getToken l@(x:xs) 
  | isOpToken x         = ([x], xs) 
  | isAlpha           x =  span isAlpha   l 
  | isNumChar         x =  span isNumChar l 
  | x == ' '            =  ([], dropWhile (== ' ') xs )
  | otherwise           =  ([], xs) 

isOpChar  x = elem x "+-*/" 
isNumChar x = elem x "-0123456789."