fisl

fisl is scheme lox
git clone git://git.alexkarle.com.com/fisl
Log | Files | Refs | README

commit ae08188772f7d552a0608c4d81b9af87e33c8eab (patch)
parent 73f7950f8ee0b150634774031ed46fc7305f3e6d
Author: Alex Karle <alex@alexkarle.com>
Date:   Sun,  2 Oct 2022 15:04:16 -0400

scanner: Add support for literals and keywords!

Diffstat:
Acomment.lox | 2++
Mscanner.scm | 57++++++++++++++++++++++++++++++++++++++++++++++++---------
Mtest.lox | 3+++
Atest2.lox | 4++++
4 files changed, 57 insertions(+), 9 deletions(-)

diff --git a/comment.lox b/comment.lox @@ -0,0 +1,2 @@ +// foo +!= diff --git a/scanner.scm b/scanner.scm @@ -14,7 +14,38 @@ (line ,line))) (define (is-digit c) - (and (char<=? #\0 c) (char>=? #\9 c))) + (and c (char<=? #\0 c) (char>=? #\9 c))) + +(define (is-alpha c) + (and c + (or + (eq? c #\_) + (and (char<=? #\a c) (char>=? #\z c)) + (and (char<=? #\A c) (char>=? #\Z c))))) + +(define (get-keyword k) + (let ((kpair (assoc k '(("and" AND) + ("class" CLASS) + ("else" ELSE) + ("false" FALSE) + ("for" FOR) + ("fun" FUN) + ("if" IF) + ("nil" NIL) + ("or" OR) + ("print" PRINT) + ("return" RETURN) + ("super" SUPER) + ("this" THIS) + ("true" TRUE) + ("var" VAR) + ("while" WHILE))))) + (if kpair (cadr kpair) #f))) + + + +(define (is-alnum c) + (and c (or (is-alpha c) (is-digit c)))) (define (scan src fname) (define (peek i) @@ -27,13 +58,17 @@ ; Gets all tokens after 'start', tracks state in i (current char), line, in (define (tok type s2 i2) ; Helper to make a token, cons it to our list, and recurse with fresh state - (let ((tok (cond - ((eq? type 'STRING) (make-token type (substring src (add1 s2) i2) #f line)) - ((eq? type 'NUMBER) (make-token type - (string->number (substring src s2 (add1 i2))) - #f line)) - (else (make-token type (substring src s2 (add1 i2)) #f line))))) - (cons tok (get-tokens (add1 i2) (add1 i2) line #f)))) + (let ((text (substring src s2 (add1 i2)))) + (let ((tok (cond + ((eq? type 'STRING) (make-token type text (substring src (add1 s2) i2) line)) + ((eq? type 'NUMBER) (make-token type text (string->number text) line)) + ((eq? type 'IDENTIFIER) + (let ((k (get-keyword text))) + (if k + (make-token k text #f line) + (make-token 'IDENTIFIER text #f line)))) + (else (make-token type text #f line))))) + (cons tok (get-tokens (add1 i2) (add1 i2) line #f))))) (define (next l2) ; Helper to iterate while keeping state @@ -61,7 +96,10 @@ (cond ((is-digit c) (next line)) (else (tok 'NUMBER s (sub1 i))))) - ((eq? in 'identifier) #f) + ((eq? in 'alpha) + (cond + ((is-alnum c) (next line)) + (else (tok 'IDENTIFIER s (sub1 i))))) ((eq? in '=) (if (eq? #\= c) (tok 'EQUAL_EQUAL s i) (tok 'EQUAL s s))) ((eq? in '>) (if (eq? #\> c) (tok 'GREATER_EQUAL s i) (tok 'GREATER s s))) ((eq? in '<) (if (eq? #\< c) (tok 'LESS_EQUAL s i) (tok 'LESS s s))) @@ -85,6 +123,7 @@ ((eq? #\/ c) (get-tokens s (add1 i) line '/)) ((eq? #\" c) (get-tokens s (add1 i) line 'string)) ((is-digit c) (get-tokens s (add1 i) line 'number)) + ((is-alpha c) (get-tokens s (add1 i) line 'alpha)) ((eq? #\space c) (get-tokens (add1 i) (add1 i) line #f)) ((eq? #\tab c) (get-tokens (add1 i) (add1 i) line #f)) ((eq? #\newline c) (get-tokens (add1 i) (add1 i) (add1 line) #f)) diff --git a/test.lox b/test.lox @@ -7,3 +7,6 @@ {...} 456.123! 123 + +woah +woah_there2 != 5 diff --git a/test2.lox b/test2.lox @@ -0,0 +1,4 @@ +foo = 5; +while (foo != 0) { + print foo; +}