Fix reusing a *Tokenizer after an empty stream has been closed

When parsing an empty reader, Stream.head does not get allocated by the pool
at any time and holds a pointer to undefToken.

Previously, on Stream.Close() it was released to the pool, which added the
pointer to undefToken to the pool. As the next call to ParseStream() has to
allocate a new token for the first keyword, it got it from the pool and
received the last released token, which was the pointer to undefToken.
Finally when Stream.IsValid() checked if Stream.head == undefToken, this
evaluated to true, and returned false.

Now, to make sure that we never release a pointer to undefToken into the pool,
we add a check for this in Stream.Close().

Closes: #30

Author: n-peugnet

stream.go

@@ -67,7 +67,7 @@ func (s *Stream) SetHistorySize(size int) *Stream {
 
 // Close releases all token objects to pool
 func (s *Stream) Close() {
-	for ptr := s.head; ptr != nil; {
+	for ptr := s.head; ptr != nil && ptr != undefToken; {
 		p := ptr.next
 		s.t.freeToken(ptr)
 		ptr = p

stream_test.go

@@ -286,6 +286,29 @@ func TestIssue26(t *testing.T) {
 	}
 }
 
+// TestIssue30 second use of ParseStream() after a first closed empty stream,
+// produces an incorrectly invalid stream.
+func TestIssue30(t *testing.T) {
+	parser := New()
+
+	// first stream is empty
+	buf1 := bytes.NewBufferString("")
+	stream1 := parser.ParseStream(buf1, 4096)
+	for stream1.IsValid() {
+		t.Error("stream1 should have been invalid from the beginning")
+	}
+	// closing the first stream before opening the new one
+	stream1.Close()
+
+	// second stream created from the same parser
+	buf2 := bytes.NewBufferString("hello world")
+	stream2 := parser.ParseStream(buf2, 4096)
+	defer stream2.Close()
+	if !stream2.IsValid() {
+		t.Fatal("stream2 should be valid")
+	}
+}
+
 func TestStreamOverflow(t *testing.T) {
 	parser := New()
 	buf := bytes.NewBuffer([]byte("a b c"))