umx_compiler

commit 74f2a1ec1374f47939afe92253630cacb116eec5
parent 8285fef11f1e4d6d8a7aaa15dc362bfd021d9d01
Author: bsandro <email@bsandro.tech>
Date:   Sun, 26 Jun 2022 22:54:23 +0300

tokenizing strings

Diffstat:
M
lexer/lexer.go
 | 
14
++++++++++++++
M
lexer/lexer_test.go
 | 
8
++++++++
M
token/token.go
 | 
1
+

3 files changed, 23 insertions(+), 0 deletions(-)
diff --git a/lexer/lexer.go b/lexer/lexer.go
@@ -75,6 +75,9 @@ func (l *Lexer) NextToken() token.Token {
 		tok = newToken(token.GT, l.ch)
 	case ',':
 		tok = newToken(token.COMMA, l.ch)
+	case '"':
+		tok.Type = token.STRING
+		tok.Literal = l.readString()
 	case 0:
 		tok.Literal = ""
 		tok.Type = token.EOF
@@ -129,3 +132,14 @@ func (l *Lexer) skipWhitespace() {
 		l.readChar()
 	}
 }
+
+func (l *Lexer) readString() string {
+	position := l.position + 1
+	for {
+		l.readChar()
+		if l.ch == '"' || l.ch == 0 {
+			break
+		}
+	}
+	return l.input[position:l.position]
+}
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
@@ -22,6 +22,9 @@ if (5 < 10) {
 
 10 == 10;
 10 != 8;
+
+"foo123";
+"foo bar";
 `
 
 	tests := []struct {
@@ -105,6 +108,11 @@ if (5 < 10) {
 		{token.INT, "8"},
 		{token.SEMICOLON, ";"},
 
+		{token.STRING, "foo123"},
+		{token.SEMICOLON, ";"},
+		{token.STRING, "foo bar"},
+		{token.SEMICOLON, ";"},
+
 		{token.EOF, ""},
 	}
 
diff --git a/token/token.go b/token/token.go
@@ -35,6 +35,7 @@ const (
 	RETURN    = "RETURN"
 	EQUAL     = "=="
 	NOT_EQUAL = "!="
+	STRING    = "STRING"
 )
 
 var keywords = map[string]TokenType{