11// Source text -> Tokens
2- use crate :: compiler:: { ast:: { Pos , Span } , CompilerError , ToCompileResult } ;
2+ use crate :: compiler:: { ast:: { Pos , Span } , CompilerError } ;
33
44const CASE_SENSITIVITY : bool = true ;
55
@@ -8,18 +8,17 @@ pub struct LexError {
88 pub span : Span ,
99 pub message : String ,
1010}
11-
12- impl < T > ToCompileResult < T > for Result < T , Vec < LexError > > {
13- fn into_cresult ( self ) -> Result < T , super :: CompilerError > {
14- self . map_err ( |err| CompilerError :: LexError ( err) )
11+ impl From < Vec < LexError > > for CompilerError {
12+ fn from ( value : Vec < LexError > ) -> Self {
13+ CompilerError :: LexError ( value)
1514 }
1615}
1716
1817#[ derive( Debug , Clone , PartialEq ) ]
1918pub enum TokenKind {
2019 // Identifiers and Literals
2120 Identifier ( String ) ,
22- Int ( i32 ) ,
21+ Int ( u32 ) ,
2322 Float ( f64 ) ,
2423 String ( String ) ,
2524
@@ -90,7 +89,11 @@ pub struct Token {
9089
9190pub fn lex_all ( src : & str ) -> Result < Vec < Token > , Vec < LexError > > {
9291 let mut lexer = Lexer :: new ( & src) ;
93- lexer. lex_all ( )
92+ if lexer. lex_all ( ) . is_ok ( ) {
93+ Ok ( lexer. tokens )
94+ } else {
95+ Err ( lexer. errors )
96+ }
9497}
9598
9699pub struct Lexer {
@@ -99,6 +102,7 @@ pub struct Lexer {
99102 errors : Vec < LexError > ,
100103 line : usize ,
101104 col : usize ,
105+ pub tokens : Vec < Token >
102106}
103107impl Lexer {
104108 pub fn new ( input : & str ) -> Self {
@@ -108,27 +112,25 @@ impl Lexer {
108112 errors : Vec :: new ( ) ,
109113 line : 1 ,
110114 col : 1 ,
115+ tokens : Vec :: new ( ) ,
111116 }
112117 }
113118
114- pub fn lex_all ( & mut self ) -> Result < Vec < Token > , Vec < LexError > > {
115- let mut tokens = Vec :: new ( ) ;
119+ pub fn lex_all ( & mut self ) -> Result < ( ) , & [ LexError ] > {
116120 loop {
117- let tok = self . next_token ( ) ;
118- if matches ! ( tok. kind, TokenKind :: EOF ) {
119- tokens. push ( tok) ; break ;
120- } else {
121- tokens. push ( tok) ;
121+ self . next_token ( ) ;
122+ if matches ! ( self . tokens. last( ) , Some ( Token { kind: TokenKind :: EOF , .. } ) ) {
123+ break ;
122124 }
123125 }
124126 if self . errors . is_empty ( ) {
125- Ok ( tokens )
127+ Ok ( ( ) )
126128 } else {
127- Err ( self . errors . clone ( ) )
129+ Err ( & self . errors )
128130 }
129131 }
130132
131- pub fn next_token ( & mut self ) -> Token {
133+ fn next_token ( & mut self ) {
132134 self . skip_whitespace_and_comment ( ) ;
133135
134136 let start_line = self . line ;
@@ -139,22 +141,20 @@ impl Lexer {
139141 } ;
140142
141143 if c. is_alphabetic ( ) {
142- self . lex_identifier_or_keyword ( start_line, start_col)
144+ self . lex_identifier_or_keyword ( start_line, start_col) ;
143145 } else if c. is_ascii_digit ( ) {
144- self . lex_number ( start_line, start_col)
146+ self . lex_number ( start_line, start_col) ;
145147 } else if c == '.' && self . peek_ahead ( 1 ) . map_or ( false , |n| n. is_ascii_digit ( ) ) {
146- self . lex_number ( start_line, start_col)
147- } else if c == '-' && self . peek_ahead ( 1 ) . map_or ( false , |n| n. is_ascii_digit ( ) ) {
148- self . lex_number ( start_line, start_col)
148+ self . lex_number ( start_line, start_col) ;
149149 } else if c == '"' {
150- self . lex_string ( start_line, start_col)
150+ self . lex_string ( start_line, start_col) ;
151151 } else {
152- self . lex_symbol ( start_line, start_col)
152+ self . lex_symbol ( start_line, start_col) ;
153153 }
154154 }
155155
156- fn make_token ( & self , kind : TokenKind , line : usize , col : usize ) -> Token {
157- Token { kind, span : self . make_span ( line, col) }
156+ fn make_token ( & mut self , kind : TokenKind , line : usize , col : usize ) {
157+ self . tokens . push ( Token { kind, span : self . make_span ( line, col) } ) ;
158158 }
159159
160160 fn make_span ( & self , line : usize , col : usize ) -> Span {
@@ -232,7 +232,7 @@ impl Lexer {
232232
233233 // --------- Identifiers & Keywords ---------
234234
235- fn lex_identifier_or_keyword ( & mut self , line : usize , col : usize ) -> Token {
235+ fn lex_identifier_or_keyword ( & mut self , line : usize , col : usize ) {
236236 let mut s = String :: new ( ) ;
237237 while let Some ( c) = self . peek ( ) {
238238 if c. is_alphanumeric ( ) || c == '_' {
@@ -271,7 +271,7 @@ impl Lexer {
271271
272272 // --------- Numbers ---------
273273
274- fn lex_number ( & mut self , line : usize , col : usize ) -> Token {
274+ fn lex_number ( & mut self , line : usize , col : usize ) {
275275 let mut num_str = String :: new ( ) ;
276276 let mut has_dot = false ;
277277 let mut has_exp = false ;
@@ -338,7 +338,7 @@ impl Lexer {
338338 self . make_token ( kind, line, col)
339339 }
340340
341- fn lex_based_number ( & mut self , base : u32 , line : usize , col : usize ) -> Token {
341+ fn lex_based_number ( & mut self , base : u32 , line : usize , col : usize ) {
342342 let mut s = String :: new ( ) ;
343343 while let Some ( c) = self . peek ( ) {
344344 match c {
@@ -363,13 +363,13 @@ impl Lexer {
363363 }
364364
365365 let cleaned = s. replace ( "_" , "" ) ;
366- let value = i32 :: from_str_radix ( & cleaned, base) . unwrap_or ( 0 ) ;
366+ let value = u32 :: from_str_radix ( & cleaned, base) . unwrap_or ( 0 ) ;
367367 self . make_token ( TokenKind :: Int ( value) , line, col)
368368 }
369369
370370 // --------- Strings ---------
371371
372- fn lex_string ( & mut self , line : usize , col : usize ) -> Token {
372+ fn lex_string ( & mut self , line : usize , col : usize ) {
373373 self . advance ( ) ; // consume '"'
374374 let mut s = String :: new ( ) ;
375375
@@ -403,7 +403,7 @@ impl Lexer {
403403
404404 // --------- Symbols & Operators ---------
405405
406- fn lex_symbol ( & mut self , line : usize , col : usize ) -> Token {
406+ fn lex_symbol ( & mut self , line : usize , col : usize ) {
407407 use TokenKind :: * ;
408408 let c = self . advance ( ) . unwrap ( ) ;
409409
@@ -478,7 +478,8 @@ mod tests {
478478 let mut lexer = Lexer :: new ( src) ;
479479 let mut i = 0 ;
480480 loop {
481- let tok = lexer. next_token ( ) ;
481+ lexer. next_token ( ) ;
482+ let tok = lexer. tokens . last ( ) . unwrap ( ) ;
482483 println ! ( "{:?}" , tok) ;
483484 assert_eq ! ( tok. kind, expected[ i] ) ;
484485 if matches ! ( tok. kind, TokenKind :: EOF ) { break ; }
@@ -500,7 +501,8 @@ mod tests {
500501 let mut lexer = Lexer :: new ( src) ;
501502 let mut i = 0 ;
502503 loop {
503- let tok = lexer. next_token ( ) ;
504+ lexer. next_token ( ) ;
505+ let tok = lexer. tokens . last ( ) . unwrap ( ) ;
504506 println ! ( "{:?}" , tok) ;
505507 assert_eq ! ( tok. kind, expected[ i] ) ;
506508 if matches ! ( tok. kind, TokenKind :: EOF ) { break ; }
@@ -521,7 +523,8 @@ mod tests {
521523 let mut lexer = Lexer :: new ( src) ;
522524 let mut i = 0 ;
523525 loop {
524- let tok = lexer. next_token ( ) ;
526+ lexer. next_token ( ) ;
527+ let tok = lexer. tokens . last ( ) . unwrap ( ) ;
525528 println ! ( "{:?}" , tok) ;
526529 assert_eq ! ( tok. kind, expected[ i] ) ;
527530 if matches ! ( tok. kind, TokenKind :: EOF ) { break ; }
0 commit comments