(in-package :lexer) (defparameter *eof* (code-char 0)) (defun eof? (char) (char= char *eof*)) (defun digit? (char) (char<= #\0 char #\9)) (defun letter? (char) (or (char<= #\a char #\z) (char<= #\A char #\Z))) (defun whitespace? (char) (or (char= char #\Space) (char= char #\Tab) (char= char #\Newline) (char= char #\Linefeed) (char= char #\_))) (defparameter *lexer-start-pos* -1) (defclass lexer () ((input :reader input :initarg :input) (len :reader len :initarg :length) (pos :accessor pos :initform *lexer-start-pos*) (ch :accessor ch) (peek-ch :accessor peek-ch))) (defun make (input-string &optional len) (let ((l (make-instance 'lexer :input input-string :length (if (not (null len)) len (length input-string))))) (lexer-peek* l) (lexer-read l) l)) (defmethod lexer-reset ((l lexer)) (setf (pos l) *lexer-start-pos*) (lexer-peek* l) (lexer-read l)) (defmethod lexer-read ((l lexer)) (prog1 (setf (ch l) (peek-ch l)) (incf (pos l)) (lexer-peek* l))) (defmethod lexer-peek* ((l lexer)) (setf (peek-ch l) (if (< (+ 1 (pos l)) (len l)) (char (input l) (+ 1 (pos l))) *eof*))) (defmethod lexer-peek ((l lexer)) (peek-ch l)) (defmethod skip-ws ((l lexer)) (loop :while (whitespace? (ch l)) :do (lexer-read l))) (defmethod next-token ((l lexer)) (skip-ws l) (block nil (prog1 (case (ch l) (#\= (cond ((char= #\= (lexer-peek l)) (lexer-read l) (token:as-token :t/==)) (t ;;(whitespace? (lexer-peek l)) (token:as-token :t/=)))) (#\+ (token:as-token :t/+)) (#\- (token:as-token :t/-)) (#\! (cond ((char= #\= (lexer-peek l)) (lexer-read l) (token:as-token :t/!=)) (t ;; (whitespace? (lexer-peek l)) (token:as-token :t/!)))) (#\/ (token:as-token :t//)) (#\* (token:as-token :t/*)) (#\< (token:as-token :t/<)) (#\> (token:as-token :t/>)) (#\; (token:as-token :t/semicolon)) (#\, (token:as-token :t/comma)) (#\( (token:as-token :t/lparen)) (#\) (token:as-token :t/rparen)) (#\{ (token:as-token :t/lbrace)) (#\} (token:as-token :t/rbrace)) (otherwise (cond ((eof? (ch l)) (token:as-token :t/eof)) ((letter? (ch l)) (let ((literal (read-identifier l))) (return (token:make (token:lookup-identifier literal) literal)))) ((digit? (ch l)) (return (token:make :t/int (read-number l)))) (t (token:make :t/illegal "ILLEGAL"))))) (lexer-read l)))) (defmethod read-number ((l lexer)) (let ((start (pos l))) (lexer-read l) (loop :while (digit? (ch l)) :do (lexer-read l)) (make-string-view (input l) start (pos l)))) (defmethod read-identifier ((l lexer)) (let ((start (pos l))) (lexer-read l) (loop :while (letter? (ch l)) :do (lexer-read l)) (make-string-view (input l) start (pos l)))) (defmethod lex ((l lexer)) (loop :for token := (next-token l) :collect token :until (token:type-is token :t/eof)))