package tokenize_test import ( "fmt" "git.makaay.nl/mauricem/go-parsekit/tokenize" ) func ExampleNewAPI() { tokenize.NewAPI("The input that the API will handle") // Output: } func ExampleAPI_NextRune() { api := tokenize.NewAPI("The input that the API will handle") r, err := api.NextRune() fmt.Printf("Rune read from input; %c\n", r) fmt.Printf("The error: %v\n", err) fmt.Printf("API results: %q\n", api.Result().String()) // Output: // Rune read from input; T // The error: // API results: "" } func ExampleAPI_Accept() { api := tokenize.NewAPI("The input that the API will handle") api.NextRune() // reads 'T' api.Accept() // adds 'T' to the API results api.NextRune() // reads 'h' api.Accept() // adds 'h' to the API results api.NextRune() // reads 'e', but it is not added to the API results fmt.Printf("API results: %q\n", api.Result().String()) // Output: // API results: "Th" } func ExampleAPI_Result() { api := tokenize.NewAPI("") result := api.Result() result.AddRunes("Some runes") result.AddRunes([]rune{' ', 'a', 'd', 'd', 'e', 'd'}) result.AddRunes(' ', 'i', 'n', ' ', "various ways") fmt.Printf("API result first 10 runes: %q\n", api.Result().Runes()[0:10]) fmt.Printf("API result runes as string: %q\n", api.Result().String()) result.SetRunes("new ", "set ", "of ", 'r', 'u', 'n', 'e', 's') fmt.Printf("API result runes as string: %q\n", api.Result().String()) fmt.Printf("API result runes: %q\n", api.Result().Runes()) fmt.Printf("API third rune: %q\n", api.Result().Rune(2)) result.AddTokens(&tokenize.Token{ Runes: []rune("demo 1"), Type: 42, Value: "towel"}) result.AddTokens(&tokenize.Token{ Runes: []rune("demo 2"), Type: 73, Value: "Zaphod"}) fmt.Printf("API result tokens: %v\n", api.Result().Tokens()) fmt.Printf("API second result token: %v\n", api.Result().Token(1)) // Output: // API result first 10 runes: ['S' 'o' 'm' 'e' ' ' 'r' 'u' 'n' 'e' 's'] // API result runes as string: "Some runes added in various ways" // API result runes as string: "new set of runes" // API result runes: ['n' 'e' 'w' ' ' 's' 'e' 't' ' ' 'o' 'f' ' ' 'r' 'u' 'n' 'e' 's'] // API third rune: 'w' // API result tokens: [42("demo 1", value = (string)towel) 73("demo 2", value = (string)Zaphod)] // API second result token: 73("demo 2", value = (string)Zaphod) } func ExampleAPI_Reset() { api := tokenize.NewAPI("Very important input!") api.NextRune() api.Accept() api.NextRune() api.Accept() fmt.Printf("API results: %q at %s\n", api.Result().String(), api.Result().Cursor()) // Reset clears the results, but keeps the cursor position. api.Reset() fmt.Printf("API results: %q at %s\n", api.Result().String(), api.Result().Cursor()) api.NextRune() api.Accept() api.NextRune() api.Accept() fmt.Printf("API results: %q at %s\n", api.Result().String(), api.Result().Cursor()) // Output: // API results: "Ve" at line 1, column 3 // API results: "" at line 1, column 3 // API results: "ry" at line 1, column 5 } func ExampleAPI_Fork() { // This custom Handler checks for input 'a', 'b' or 'c'. abcHandler := func(t *tokenize.API) bool { a := tokenize.A for _, r := range []rune{'a', 'b', 'c'} { child := t.Fork() // fork, so we won't change parent t if a.Rune(r)(child) { child.Merge() // accept results into parent t return true // and report a successful match } } // If we get here, then no match was found. Return false to communicate // this to the caller. return false } // Note: a custom Handler is normally not what you need. // You can make use of the parser/combinator tooling to make the // implementation a lot simpler and to take care of forking at // the appropriate places. The handler from above can be replaced with: simpler := tokenize.A.RuneRange('a', 'c') result, err := tokenize.New(abcHandler)("another test") fmt.Println(result, err) result, err = tokenize.New(simpler)("curious") fmt.Println(result, err) result, err = tokenize.New(abcHandler)("bang on!") fmt.Println(result, err) result, err = tokenize.New(abcHandler)("not a match") fmt.Println(result, err) // Output: // a // c // b // mismatch at start of file } func ExampleAPI_Dispose() { api := tokenize.NewAPI("My uninspired piece of input") child := api.Fork() // ... do stuff with child ... child.NextRune() child.Accept() child.NextRune() child.Accept() // ... dispose of the child results ... child.Dispose() // The parent still reads from the start of the input. r, _ := api.NextRune() fmt.Printf("Rune read from parent: %c\n", r) // Output: // Rune read from parent: M } func ExampleAPI_Merge() { tokenHandler := func(t *tokenize.API) bool { child1 := t.Fork() child1.NextRune() // reads 'H' child1.Accept() child1.NextRune() // reads 'i' child1.Accept() child2 := child1.Fork() child2.NextRune() // reads ' ' child2.Accept() child2.NextRune() // reads 'd' child2.Accept() child1.Merge() // We merge child1, which has read 'H' and 'i' only. return true } result, _ := tokenize.New(tokenHandler)("Hi mister X!") fmt.Println(result) // Output: // Hi }