package tokenize_test import ( "fmt" "io" "strings" "testing" "unicode/utf8" tokenize "git.makaay.nl/mauricem/go-parsekit/tokenize" ) // TODO For error handling, it would be really cool if for example the // 10.0.300.1/24 case would return an actual error stating that // 300 is not a valid octet for an IPv4 address. // Biggest thing to take care of here, is that errors should not stop // a Parser flow (since we might be trying to match different cases in // sequence), but a Parser flow should optionally be able to make use // of the actual error. // The same goes for a Tokenizer, since those can also make use of // optional matching using tokenize.C.Any(...) for example. If matching // for Any(IPv4, Digits), the example case should simply end up with 10 // after the IPv4 mismatch. func ExampleNew() { // Build the tokenizer for ip/mask. var c, a, t = tokenize.C, tokenize.A, tokenize.T ip := t.Str("ip", a.IPv4) mask := t.Int8("mask", a.IPv4CIDRMask) cidr := c.Seq(ip, a.Slash, mask) tokenizer := tokenize.New(cidr) for _, input := range []string{ "000.000.000.000/000", "192.168.0.1/24", "255.255.255.255/32", "10.0.300.1/24", "not an IPv4 CIDR", } { // Execute returns a Result and an error, which is nil on success. result, err := tokenizer(input) if err == nil { tokens := result.Tokens() fmt.Printf("Result: %s\n", tokens) } else { fmt.Printf("Error: %s\n", err) } } // Output: // Result: [ip("0.0.0.0") mask((int8)0)] // Result: [ip("192.168.0.1") mask((int8)24)] // Result: [ip("255.255.255.255") mask((int8)32)] // Error: mismatch at start of file // Error: mismatch at start of file } func TestCallingPeekRune_PeeksRuneOnInput(t *testing.T) { tokenizeAPI := makeTokenizeAPI() r, _, _ := tokenizeAPI.Rune.Peek(0) AssertEqual(t, 'T', r, "first rune") } func TestInputCanAcceptRunesFromReader(t *testing.T) { tokenAPI := makeTokenizeAPI() r0, _, _ := tokenAPI.Rune.Peek(0) tokenAPI.Rune.Accept(r0) r1, _, _ := tokenAPI.Rune.Peek(0) // 0, because read offset resets to 0 after Accept* calls. r2, _, _ := tokenAPI.Rune.Peek(1) tokenAPI.Rune.AcceptMulti(r1, r2) AssertEqual(t, "Tes", tokenAPI.Output.String(), "i.String()") } // func TestCallingMergeOnTopLevelAPI_Panics(t *testing.T) { // AssertPanic(t, PanicT{ // Function: func() { // tokenAPI := makeTokenizeAPI() // tokenAPI.Merge(0) // }, // Regexp: true, // Expect: `tokenize\.API\.Merge\(\): Merge\(\) called at /.*_test.go:\d+ on the top-level API`}) // } // func TestCallingMergeOnForkParentAPI_Panics(t *testing.T) { // AssertPanic(t, PanicT{ // Function: func() { // tokenAPI := makeTokenizeAPI() // child := tokenAPI.Fork() // tokenAPI.Fork() // tokenAPI.Merge(child) // }, // Regexp: true, // Expect: `tokenize\.API\.Merge\(\): Merge\(\) called at /.*_test.go:\d+ ` + // `on API stack level 1, but the current stack level is 2 \(forgot to Dispose\(\) a forked child\?\)`}) // } // func TestCallingDisposeOnTopLevelAPI_Panics(t *testing.T) { // AssertPanic(t, PanicT{ // Function: func() { // tokenAPI := makeTokenizeAPI() // tokenAPI.Dispose(0) // }, // Regexp: true, // Expect: `tokenize\.API\.Dispose\(\): Dispose\(\) called at /.*_test.go:\d+ on the top-level API`}) // } // func TestCallingDisposeOnForkParentAPI_Panics(t *testing.T) { // AssertPanic(t, PanicT{ // Function: func() { // tokenAPI := makeTokenizeAPI() // child := tokenAPI.Fork() // tokenAPI.Fork() // tokenAPI.Dispose(child) // }, // Regexp: true, // Expect: `tokenize\.API\.Dispose\(\): Dispose\(\) called at /.*_test.go:\d+ ` + // `on API stack level 1, but the current stack level is 2 \(forgot to Dispose\(\) a forked child\?\)`}) // } // func TestCallingForkOnForkedParentAPI_Panics(t *testing.T) { // AssertPanic(t, PanicT{ // Function: func() { // tokenAPI := makeTokenizeAPI() // tokenAPI.Fork() // g := tokenAPI.Fork() // tokenAPI.Fork() // tokenAPI.Merge(g) // }, // Regexp: true, // Expect: `tokenize\.API\.Merge\(\): Merge\(\) called at /.*_test.go:\d+ ` + // `on API stack level 2, but the current stack level is 3 \(forgot to Dispose\(\) a forked child\?\)`}) // } func TestAccept_UpdatesCursor(t *testing.T) { tokenAPI := tokenize.NewAPI(strings.NewReader("input\r\nwith\r\nnewlines")) AssertEqual(t, "start of file", tokenAPI.Input.Cursor(), "cursor 1") for j := 0; j < 6; j++ { // read "input\r", cursor end up at "\n" r, _, _ := tokenAPI.Rune.Peek(0) tokenAPI.Rune.Accept(r) } AssertEqual(t, "line 1, column 7", tokenAPI.Input.Cursor(), "cursor 2") r, _, _ := tokenAPI.Rune.Peek(0) // read "\n", cursor ends up at start of new line tokenAPI.Rune.Accept(r) AssertEqual(t, "line 2, column 1", tokenAPI.Input.Cursor(), "cursor 3") for j := 0; j < 10; j++ { // read "with\r\nnewl", cursor end up at "i" b, _ := tokenAPI.Byte.Peek(0) tokenAPI.Byte.Accept(b) } AssertEqual(t, "line 3, column 5", tokenAPI.Input.Cursor(), "cursor 4") } func TestWhenCallingPeekruneAtEndOfFile_EOFIsReturned(t *testing.T) { tokenAPI := tokenize.NewAPI(strings.NewReader("X")) r, _, _ := tokenAPI.Rune.Peek(0) tokenAPI.Rune.Accept(r) r, _, err := tokenAPI.Rune.Peek(0) AssertEqual(t, true, r == utf8.RuneError, "returned rune from NextRune()") AssertEqual(t, true, err == io.EOF, "returned error from NextRune()") } // func TestAfterReadingruneAtEndOfFile_EarlierRunesCanStillBeAccessed(t *testing.T) { // i := tokenize.NewAPI(strings.NewReader("X")) // child := i.Fork() // // To to the EOF. // r, _, _ := i.Rune.Peek(0) // i.Rune.Accept(r) // r, _, err := i.Rune.Peek(0) // AssertEqual(t, true, r == utf8.RuneError, "returned rune from 2nd NextRune()") // AssertEqual(t, true, err == io.EOF, "returned error from 2nd NextRune()") // // Brings the read offset back to the start. // i.Dispose(child) // // So here we should see the same input data as before. // r, _, err = i.Rune.Peek(0) // AssertEqual(t, 'X', r, "returned rune from 2nd NextRune()") // AssertEqual(t, true, err == nil, "returned error from 2nd NextRune()") // } func makeTokenizeAPI() *tokenize.API { return tokenize.NewAPI("Testing") }