issue 11: fix number parser (#16)

Author: bzick

parser.go

@@ -59,13 +59,6 @@ func newInfParser(t *Tokenizer, reader io.Reader, bufferSize uint) *parsing {
 	}
 }
 
-func (p *parsing) prev() {
-	if p.pos > 0 {
-		p.pos--
-		p.curr = p.str[p.pos]
-	}
-}
-
 func (p *parsing) ensureBytes(n int) bool {
 	if p.pos+n >= len(p.str) {
 		if p.reader != nil {
@@ -273,66 +266,68 @@ func (p *parsing) parseKeyword() bool {
 	return false
 }
 
-const (
-	stageCoefficient = iota + 1
-	stageMantissa
-	stagePower
-)
-
 func (p *parsing) parseNumber() bool {
 	var start = -1
-	var needNumber = true
+	var end = -1
+	var floatTraitPos = -1
+	var hasPoint = false
+	var hasNumber = false
+	var hasExp = false
 
-	var stage uint8 = 0
 	for p.curr != 0 {
 		if isNumberByte(p.curr) {
-			needNumber = false
 			if start == -1 {
-				if stage == 0 {
-					stage = stageCoefficient
-					start = p.pos
-				}
-			}
-		} else if p.t.allowNumberUnderscore && p.curr == '_' {
-			if stage != stageCoefficient {
-				break
-			}
-			// todo checks double underscore
-		} else if !needNumber && p.curr == '.' {
-			if stage != stageCoefficient {
-				break
-			}
-			stage = stageMantissa
-			needNumber = true
-		} else if !needNumber && (p.curr == 'e' || p.curr == 'E') {
-			if stage != stageMantissa && stage != stageCoefficient {
-				break
-			}
-			ePowSign := false
-			switch p.nextByte() {
-			case '-', '+':
-				ePowSign = true
-				p.next()
+				start = p.pos
 			}
-			needNumber = true
-			if isNumberByte(p.nextByte()) {
-				stage = stagePower
-			} else {
-				if ePowSign { // rollback sign position
-					p.prev()
+			end = p.pos
+			hasNumber = true
+		} else {
+			nextByte := p.nextByte()
+			if p.curr == '_' {
+				if !hasNumber || (!p.t.allowNumberUnderscore || !isNumberByte(nextByte)) {
+					break
+				}
+			} else if p.curr == '.' {
+				if hasPoint {
+					break
+				} else if isNumberByte(nextByte) {
+					if start == -1 { // floats can be started from a pointer
+						start = p.pos
+					}
+				} else if !(nextByte == 'e' || nextByte == 'E' || nextByte == 0) {
+					break
+				}
+				floatTraitPos = p.pos
+				end = p.pos
+				hasPoint = true
+			} else if p.curr == 'e' || p.curr == 'E' {
+				if !hasNumber && !(isNumberByte(nextByte) || nextByte == '-') || hasExp {
+					break
 				}
+				floatTraitPos = p.pos
+				hasExp = true
+				hasPoint = true
+			} else if hasExp && (p.curr == '-' || p.curr == '+') {
+				if isNumberByte(nextByte) {
+					if start == -1 { // numbers can be started from a sign
+						start = p.pos
+					}
+				} else {
+					break
+				}
+			} else {
 				break
 			}
-		} else {
-			break
 		}
 		p.next()
 	}
-	if stage == 0 {
+	if start == -1 {
 		return false
 	}
-	p.token.value = p.str[start:p.pos]
-	if stage == stageCoefficient {
+	end = end + 1
+	p.pos = end
+	p.token.value = p.str[start:end]
+	if floatTraitPos == -1 || floatTraitPos > end-1 {
 		p.token.key = TokenInteger
 		p.token.offset = p.offset + start
 	} else {
@@ -343,7 +338,7 @@ func (p *parsing) parseNumber() bool {
 	return true
 }
 
-// match compare next bytes from data with `r`
+// match compares next bytes from data with `r`
 func (p *parsing) match(r []byte, seek bool) bool {
 	if r[0] == p.curr {
 		if len(r) > 1 {

stream_test.go

@@ -243,6 +243,19 @@ func TestIssues13_SequenceLongerThenStream(t *testing.T) {
 	require.False(t, ok)
 }
 
+func TestIssue11(t *testing.T) {
+	parser := New()
+	parser.AllowKeywordSymbols(nil, Numbers)
+	parser.DefineTokens(1, []string{".."})
+
+	stream := parser.ParseString("1..2")
+	require.Equal(t, "1", string(stream.CurrentToken().Value()))
+	stream.GoNext()
+	require.Equal(t, "..", string(stream.CurrentToken().Value()))
+	stream.GoNext()
+	require.Equal(t, "2", string(stream.CurrentToken().Value()))
+}
+
 var pattern = []byte(`<item count=10 valid id="n9762"> Носки <![CDATA[ socks ]]></item>`)
 
 type dataGenerator struct {

tokenizer.go

@@ -155,8 +155,8 @@ func (t *Tokenizer) SetWhiteSpaces(ws []byte) *Tokenizer {
 }
 
 // AllowKeywordSymbols sets major and minor symbols for keywords.
-// Major symbols (any quantity) might be in begin, in middle and in the end of keyword.
-// Minor symbols (any quantity) might be in middle and in the end of the keyword.
+// Major symbols (any quantity) might be in the beginning, at the middle and at the end of keyword.
+// Minor symbols (any quantity) might be at the middle and at the end of the keyword.
 //
 //	parser.AllowKeywordSymbols(tokenizer.Underscore, tokenizer.Numbers)
 //	// allows: "_one23", "__one2__two3"
@@ -200,7 +200,7 @@ func (t *Tokenizer) AllowNumberUnderscore() *Tokenizer {
 
 // DefineTokens add custom token.
 // There `key` unique is identifier of `tokens`, `tokens` — slice of string of tokens.
-// If key already exists tokens will be rewritten.
+// If a key already exists, tokens will be rewritten.
 func (t *Tokenizer) DefineTokens(key TokenKey, tokens []string) *Tokenizer {
 	var tks []*tokenRef
 	if key < 1 {