|
1 | 1 | package tokenizer |
2 | 2 |
|
3 | 3 | import ( |
4 | | - "bytes" |
5 | 4 | "github.com/stretchr/testify/require" |
6 | 5 | "testing" |
7 | 6 | ) |
@@ -341,50 +340,3 @@ func TestTokenizeInject(t *testing.T) { |
341 | 340 | }, |
342 | 341 | }, stream.GetSnippet(10, 10), "parsed %s as %s", str, stream) |
343 | 342 | } |
344 | | - |
345 | | -func FuzzStream(f *testing.F) { |
346 | | - testcases := []string{ |
347 | | - `{id: 1, key: "object number 1"}`, |
348 | | - "hello\n \n\tworld", |
349 | | - } |
350 | | - |
351 | | - for _, tc := range testcases { |
352 | | - f.Add(tc) // Use f.Add to provide a seed corpus |
353 | | - } |
354 | | - f.Fuzz(func(t *testing.T, orig string) { |
355 | | - origBytes := []byte(orig) |
356 | | - buffer := bytes.NewBuffer(origBytes) |
357 | | - tokenizer := New() |
358 | | - commaKey := TokenKey(10) |
359 | | - colonKey := TokenKey(11) |
360 | | - openKey := TokenKey(12) |
361 | | - closeKey := TokenKey(13) |
362 | | - dquoteKey := TokenKey(14) |
363 | | - tokenizer.DefineTokens(commaKey, []string{","}) |
364 | | - tokenizer.DefineTokens(colonKey, []string{":"}) |
365 | | - tokenizer.DefineTokens(openKey, []string{"{"}) |
366 | | - tokenizer.DefineTokens(closeKey, []string{"}"}) |
367 | | - tokenizer.DefineStringToken(dquoteKey, `"`, `"`).SetEscapeSymbol('\\') |
368 | | - |
369 | | - stream := tokenizer.ParseStream(buffer, 100) |
370 | | - var actual []byte |
371 | | - for stream.IsValid() { |
372 | | - current := stream.CurrentToken() |
373 | | - // t.Logf("%#v", current) |
374 | | - actual = append(actual, current.Indent()...) |
375 | | - actual = append(actual, current.Value()...) |
376 | | - stream.GoNext() |
377 | | - } |
378 | | - // t.Logf("%#v", stream.CurrentToken()) |
379 | | - |
380 | | - // As we only concatenate the indents of each token, the trailing |
381 | | - // whitespaces and token separators are lost, so we trim these |
382 | | - // characters on the right of both actual and expected slices. |
383 | | - trimset := ". \t\r\n\x00" |
384 | | - expected := bytes.TrimRight(origBytes, trimset) |
385 | | - actual = bytes.TrimRight(actual, trimset) |
386 | | - if !bytes.Equal(expected, actual) { |
387 | | - t.Errorf("input:\n%q\nexpected:\n%q\nactual:\n%q", orig, expected, actual) |
388 | | - } |
389 | | - }) |
390 | | -} |
0 commit comments