245 lines
6.8 KiB
Go
245 lines
6.8 KiB
Go
package tokenize_test
|
|
|
|
import (
|
|
"fmt"
|
|
"testing"
|
|
|
|
"git.makaay.nl/mauricem/go-parsekit/tokenize"
|
|
)
|
|
|
|
func ExampleNewAPI() {
|
|
tokenize.NewAPI("The input that the API will handle")
|
|
|
|
// Output:
|
|
}
|
|
|
|
func ExampleAPI_NextRune() {
|
|
api := tokenize.NewAPI("The input that the API will handle")
|
|
r, err := api.NextRune()
|
|
fmt.Printf("Rune read from input; %c\n", r)
|
|
fmt.Printf("The error: %v\n", err)
|
|
fmt.Printf("API results: %q\n", api.Result().String())
|
|
|
|
// Output:
|
|
// Rune read from input; T
|
|
// The error: <nil>
|
|
// API results: ""
|
|
}
|
|
|
|
func ExampleAPI_Accept() {
|
|
api := tokenize.NewAPI("The input that the API will handle")
|
|
api.NextRune() // reads 'T'
|
|
api.Accept() // adds 'T' to the API results
|
|
api.NextRune() // reads 'h'
|
|
api.Accept() // adds 'h' to the API results
|
|
api.NextRune() // reads 'e', but it is not added to the API results
|
|
|
|
fmt.Printf("API results: %q\n", api.Result().String())
|
|
|
|
// Output:
|
|
// API results: "Th"
|
|
}
|
|
|
|
func ExampleAPI_Result() {
|
|
api := tokenize.NewAPI("")
|
|
|
|
result := api.Result()
|
|
|
|
result.AddRunes("Some runes")
|
|
result.AddRunes([]rune{' ', 'a', 'd', 'd', 'e', 'd'})
|
|
result.AddRunes(' ', 'i', 'n', ' ', "various ways")
|
|
fmt.Printf("API result first 10 runes: %q\n", api.Result().Runes()[0:10])
|
|
fmt.Printf("API result runes as string: %q\n", api.Result().String())
|
|
|
|
result.SetRunes("new ", "set ", "of ", 'r', 'u', 'n', 'e', 's')
|
|
fmt.Printf("API result runes as string: %q\n", api.Result().String())
|
|
fmt.Printf("API result runes: %q\n", api.Result().Runes())
|
|
fmt.Printf("API third rune: %q\n", api.Result().Rune(2))
|
|
|
|
result.AddTokens(tokenize.Token{
|
|
Runes: []rune("demo 1"),
|
|
Type: 42,
|
|
Value: "towel"})
|
|
result.AddTokens(tokenize.Token{
|
|
Runes: []rune("demo 2"),
|
|
Type: 73,
|
|
Value: "Zaphod"})
|
|
fmt.Printf("API result tokens: %v\n", api.Result().Tokens())
|
|
fmt.Printf("API second result token: %v\n", api.Result().Token(1))
|
|
|
|
// Output:
|
|
// API result first 10 runes: ['S' 'o' 'm' 'e' ' ' 'r' 'u' 'n' 'e' 's']
|
|
// API result runes as string: "Some runes added in various ways"
|
|
// API result runes as string: "new set of runes"
|
|
// API result runes: ['n' 'e' 'w' ' ' 's' 'e' 't' ' ' 'o' 'f' ' ' 'r' 'u' 'n' 'e' 's']
|
|
// API third rune: 'w'
|
|
// API result tokens: [42("towel") 73("Zaphod")]
|
|
// API second result token: 73("Zaphod")
|
|
}
|
|
|
|
func ExampleAPI_Reset() {
|
|
api := tokenize.NewAPI("Very important input!")
|
|
|
|
api.NextRune()
|
|
api.Accept()
|
|
api.NextRune()
|
|
api.Accept()
|
|
fmt.Printf("API results: %q at %s\n", api.Result().String(), api.Result().Cursor())
|
|
|
|
// Reset clears the results, but keeps the cursor position.
|
|
api.Reset()
|
|
fmt.Printf("API results: %q at %s\n", api.Result().String(), api.Result().Cursor())
|
|
|
|
api.NextRune()
|
|
api.Accept()
|
|
api.NextRune()
|
|
api.Accept()
|
|
fmt.Printf("API results: %q at %s\n", api.Result().String(), api.Result().Cursor())
|
|
|
|
// Output:
|
|
// API results: "Ve" at line 1, column 3
|
|
// API results: "" at line 1, column 3
|
|
// API results: "ry" at line 1, column 5
|
|
}
|
|
|
|
func ExampleAPI_Fork() {
|
|
// This custom Handler checks for input 'a', 'b' or 'c'.
|
|
abcHandler := func(t tokenize.API) bool {
|
|
a := tokenize.A
|
|
for _, r := range []rune{'a', 'b', 'c'} {
|
|
child := t.Fork() // fork, so we won't change parent t
|
|
if a.Rune(r)(child) {
|
|
child.Merge() // accept results into parent t
|
|
return true // and report a successful match
|
|
}
|
|
}
|
|
// If we get here, then no match was found. Return false to communicate
|
|
// this to the caller.
|
|
return false
|
|
}
|
|
|
|
// Note: a custom Handler is normally not what you need.
|
|
// You can make use of the parser/combinator tooling to make the
|
|
// implementation a lot simpler and to take care of forking at
|
|
// the appropriate places. The handler from above can be replaced with:
|
|
simpler := tokenize.A.RuneRange('a', 'c')
|
|
|
|
result, err := tokenize.New(abcHandler)("another test")
|
|
fmt.Println(result, err)
|
|
result, err = tokenize.New(simpler)("curious")
|
|
fmt.Println(result, err)
|
|
result, err = tokenize.New(abcHandler)("bang on!")
|
|
fmt.Println(result, err)
|
|
result, err = tokenize.New(abcHandler)("not a match")
|
|
fmt.Println(result, err)
|
|
|
|
// Output:
|
|
// a <nil>
|
|
// c <nil>
|
|
// b <nil>
|
|
// <nil> mismatch at start of file
|
|
}
|
|
|
|
func ExampleAPI_Merge() {
|
|
tokenHandler := func(t tokenize.API) bool {
|
|
child1 := t.Fork()
|
|
child1.NextRune() // reads 'H'
|
|
child1.Accept()
|
|
child1.NextRune() // reads 'i'
|
|
child1.Accept()
|
|
|
|
child2 := child1.Fork()
|
|
child2.NextRune() // reads ' '
|
|
child2.Accept()
|
|
child2.NextRune() // reads 'd'
|
|
child2.Accept()
|
|
|
|
child1.Merge() // We merge child1, which has read 'H' and 'i' only.
|
|
return true
|
|
}
|
|
|
|
result, _ := tokenize.New(tokenHandler)("Hi mister X!")
|
|
fmt.Println(result)
|
|
|
|
// Output:
|
|
// Hi
|
|
}
|
|
|
|
func TestMultipleLevelsOfForksAndMerges(t *testing.T) {
|
|
api := tokenize.NewAPI("abcdefghijklmnopqrstuvwxyz")
|
|
|
|
// Fork a few levels.
|
|
child1 := api.Fork()
|
|
child2 := child1.Fork()
|
|
child3 := child2.Fork()
|
|
child4 := child3.Fork()
|
|
|
|
// Read some data from child4.
|
|
r, _ := child4.NextRune()
|
|
child4.Accept()
|
|
AssertEqual(t, 'a', r, "child4 rune 1")
|
|
|
|
r, _ = child4.NextRune()
|
|
child4.Accept()
|
|
AssertEqual(t, 'b', r, "child4 rune 2")
|
|
|
|
// Merge it to child3.
|
|
child4.Merge()
|
|
|
|
// Read some more from child4.
|
|
r, _ = child4.NextRune()
|
|
child4.Accept()
|
|
AssertEqual(t, 'c', r, "child4 rune 3")
|
|
AssertEqual(t, "line 1, column 4", child4.Result().Cursor().String(), "cursor child4 rune 3")
|
|
|
|
AssertEqual(t, "line 1, column 3", child3.Result().Cursor().String(), "cursor child3 rune 3, before merge of child 4")
|
|
|
|
// Again, merge it to child3.
|
|
child4.Merge()
|
|
AssertEqual(t, "line 1, column 4", child3.Result().Cursor().String(), "cursor child3 rune 3, after merge of child 4")
|
|
|
|
// Now read some data from child3.
|
|
r, _ = child3.NextRune()
|
|
child3.Accept()
|
|
r, _ = child3.NextRune()
|
|
child3.Accept()
|
|
r, _ = child3.NextRune()
|
|
child3.Accept()
|
|
AssertEqual(t, 'f', r, "child3 rune 5")
|
|
|
|
AssertEqual(t, "abcdef", child3.Result().String(), "child3 total result after rune 6")
|
|
|
|
// Temporarily go some new forks from here, but don't use their outcome.
|
|
child3sub1 := child3.Fork()
|
|
child3sub1.NextRune()
|
|
child3sub1.Accept()
|
|
child3sub1.NextRune()
|
|
child3sub1.Accept()
|
|
child3sub2 := child3sub1.Fork()
|
|
child3sub2.NextRune()
|
|
child3sub2.Accept()
|
|
child3sub2.Merge()
|
|
|
|
// Instead merge the pre-forking results from child3 to child2.
|
|
child3.Merge()
|
|
|
|
AssertEqual(t, "abcdef", child2.Result().String(), "child2 total result after merge of child3")
|
|
AssertEqual(t, "line 1, column 7", child2.Result().Cursor().String(), "cursor child2 after merge child3")
|
|
|
|
// Merge child2 to child1.
|
|
child2.Merge()
|
|
|
|
// Merge child1 a few times to the top level api.
|
|
child1.Merge()
|
|
child1.Merge()
|
|
child1.Merge()
|
|
child1.Merge()
|
|
|
|
// Read some data from the top level api.
|
|
r, _ = api.NextRune()
|
|
api.Accept()
|
|
|
|
AssertEqual(t, "abcdefg", api.Result().String(), "api string end result")
|
|
AssertEqual(t, "line 1, column 8", api.Result().Cursor().String(), "api cursor end result")
|
|
}
|