package tokenize_test import ( "fmt" "testing" "git.makaay.nl/mauricem/go-parsekit/tokenize" ) func ExampleNewAPI() { tokenize.NewAPI("The input that the API will handle") // Output: } func ExampleAPI_NextRune() { api := tokenize.NewAPI("The input that the API will handle") r, err := api.NextRune() fmt.Printf("Rune read from input; %c\n", r) fmt.Printf("The error: %v\n", err) fmt.Printf("API results: %q\n", api.Result().String()) // Output: // Rune read from input; T // The error: // API results: "" } func ExampleAPI_Accept() { api := tokenize.NewAPI("The input that the API will handle") api.NextRune() // reads 'T' api.Accept() // adds 'T' to the API results api.NextRune() // reads 'h' api.Accept() // adds 'h' to the API results api.NextRune() // reads 'e', but it is not added to the API results fmt.Printf("API results: %q\n", api.Result().String()) // Output: // API results: "Th" } func ExampleAPI_Result() { api := tokenize.NewAPI("") result := api.Result() result.AddRunes("Some runes") result.AddRunes([]rune{' ', 'a', 'd', 'd', 'e', 'd'}) result.AddRunes(' ', 'i', 'n', ' ', "various ways") fmt.Printf("API result first 10 runes: %q\n", api.Result().Runes()[0:10]) fmt.Printf("API result runes as string: %q\n", api.Result().String()) result.SetRunes("new ", "set ", "of ", 'r', 'u', 'n', 'e', 's') fmt.Printf("API result runes as string: %q\n", api.Result().String()) fmt.Printf("API result runes: %q\n", api.Result().Runes()) fmt.Printf("API third rune: %q\n", api.Result().Rune(2)) result.AddTokens(tokenize.Token{ Runes: []rune("demo 1"), Type: 42, Value: "towel"}) result.AddTokens(tokenize.Token{ Runes: []rune("demo 2"), Type: 73, Value: "Zaphod"}) fmt.Printf("API result tokens: %v\n", api.Result().Tokens()) fmt.Printf("API second result token: %v\n", api.Result().Token(1)) // Output: // API result first 10 runes: ['S' 'o' 'm' 'e' ' ' 'r' 'u' 'n' 'e' 's'] // API result runes as string: "Some runes added in various ways" // API result runes as string: "new set of runes" // API result runes: ['n' 'e' 'w' ' ' 's' 'e' 't' ' ' 'o' 'f' ' ' 'r' 'u' 'n' 'e' 's'] // API third rune: 'w' // API result tokens: [42("towel") 73("Zaphod")] // API second result token: 73("Zaphod") } func ExampleAPI_Reset() { api := tokenize.NewAPI("Very important input!") api.NextRune() api.Accept() api.NextRune() api.Accept() fmt.Printf("API results: %q at %s\n", api.Result().String(), api.Result().Cursor()) // Reset clears the results, but keeps the cursor position. api.Reset() fmt.Printf("API results: %q at %s\n", api.Result().String(), api.Result().Cursor()) api.NextRune() api.Accept() api.NextRune() api.Accept() fmt.Printf("API results: %q at %s\n", api.Result().String(), api.Result().Cursor()) // Output: // API results: "Ve" at line 1, column 3 // API results: "" at line 1, column 3 // API results: "ry" at line 1, column 5 } func ExampleAPI_Fork() { // This custom Handler checks for input 'a', 'b' or 'c'. abcHandler := func(t tokenize.API) bool { a := tokenize.A for _, r := range []rune{'a', 'b', 'c'} { child := t.Fork() // fork, so we won't change parent t if a.Rune(r)(child) { child.Merge() // accept results into parent t return true // and report a successful match } } // If we get here, then no match was found. Return false to communicate // this to the caller. return false } // Note: a custom Handler is normally not what you need. // You can make use of the parser/combinator tooling to make the // implementation a lot simpler and to take care of forking at // the appropriate places. The handler from above can be replaced with: simpler := tokenize.A.RuneRange('a', 'c') result, err := tokenize.New(abcHandler)("another test") fmt.Println(result, err) result, err = tokenize.New(simpler)("curious") fmt.Println(result, err) result, err = tokenize.New(abcHandler)("bang on!") fmt.Println(result, err) result, err = tokenize.New(abcHandler)("not a match") fmt.Println(result, err) // Output: // a // c // b // mismatch at start of file } func ExampleAPI_Merge() { tokenHandler := func(t tokenize.API) bool { child1 := t.Fork() child1.NextRune() // reads 'H' child1.Accept() child1.NextRune() // reads 'i' child1.Accept() child2 := child1.Fork() child2.NextRune() // reads ' ' child2.Accept() child2.NextRune() // reads 'd' child2.Accept() child1.Merge() // We merge child1, which has read 'H' and 'i' only. return true } result, _ := tokenize.New(tokenHandler)("Hi mister X!") fmt.Println(result) // Output: // Hi } func TestMultipleLevelsOfForksAndMerges(t *testing.T) { api := tokenize.NewAPI("abcdefghijklmnopqrstuvwxyz") // Fork a few levels. child1 := api.Fork() child2 := child1.Fork() child3 := child2.Fork() child4 := child3.Fork() // Read some data from child4. r, _ := child4.NextRune() child4.Accept() AssertEqual(t, 'a', r, "child4 rune 1") r, _ = child4.NextRune() child4.Accept() AssertEqual(t, 'b', r, "child4 rune 2") // Merge it to child3. child4.Merge() // Read some more from child4. r, _ = child4.NextRune() child4.Accept() AssertEqual(t, 'c', r, "child4 rune 3") AssertEqual(t, "line 1, column 4", child4.Result().Cursor().String(), "cursor child4 rune 3") AssertEqual(t, "line 1, column 3", child3.Result().Cursor().String(), "cursor child3 rune 3, before merge of child 4") // Again, merge it to child3. child4.Merge() AssertEqual(t, "line 1, column 4", child3.Result().Cursor().String(), "cursor child3 rune 3, after merge of child 4") // Now read some data from child3. r, _ = child3.NextRune() child3.Accept() r, _ = child3.NextRune() child3.Accept() r, _ = child3.NextRune() child3.Accept() AssertEqual(t, 'f', r, "child3 rune 5") AssertEqual(t, "abcdef", child3.Result().String(), "child3 total result after rune 6") // Temporarily go some new forks from here, but don't use their outcome. child3sub1 := child3.Fork() child3sub1.NextRune() child3sub1.Accept() child3sub1.NextRune() child3sub1.Accept() child3sub2 := child3sub1.Fork() child3sub2.NextRune() child3sub2.Accept() child3sub2.Merge() // Instead merge the pre-forking results from child3 to child2. child3.Merge() AssertEqual(t, "abcdef", child2.Result().String(), "child2 total result after merge of child3") AssertEqual(t, "line 1, column 7", child2.Result().Cursor().String(), "cursor child2 after merge child3") // Merge child2 to child1. child2.Merge() // Merge child1 a few times to the top level api. child1.Merge() child1.Merge() child1.Merge() child1.Merge() // Read some data from the top level api. r, _ = api.NextRune() api.Accept() AssertEqual(t, "abcdefg", api.Result().String(), "api string end result") AssertEqual(t, "line 1, column 8", api.Result().Cursor().String(), "api cursor end result") }