From 9d98c9dff708a63b04412649a8860b772a8511f9 Mon Sep 17 00:00:00 2001 From: Maurice Makaay Date: Fri, 19 Jul 2019 22:57:06 +0000 Subject: [PATCH] Moving output functions to its own substruct of the API. --- parse/api.go | 8 +- tokenize/api.go | 113 ++++++++++++++-------------- tokenize/api_test.go | 92 +++++++++++----------- tokenize/handlers_builtin.go | 50 ++++++------ tokenize/tokenize.go | 4 +- tokenize/tokenizer_test.go | 2 +- tokenize/tokenizer_whitebox_test.go | 24 +++--- 7 files changed, 148 insertions(+), 145 deletions(-) diff --git a/parse/api.go b/parse/api.go index 90dd2b4..0a74835 100644 --- a/parse/api.go +++ b/parse/api.go @@ -29,8 +29,8 @@ type API struct { func (p *API) Peek(tokenHandler tokenize.Handler) bool { forkedAPI, ok := p.invokeHandler("Peek", tokenHandler) if ok { - p.Result.Tokens = p.tokenAPI.Tokens() - p.Result.Runes = p.tokenAPI.Runes() + p.Result.Tokens = p.tokenAPI.Output.Tokens() + p.Result.Runes = p.tokenAPI.Output.Runes() } p.tokenAPI.Dispose(forkedAPI) return ok @@ -46,8 +46,8 @@ func (p *API) Accept(tokenHandler tokenize.Handler) bool { forkedAPI, ok := p.invokeHandler("Accept", tokenHandler) if ok { // Keep track of the results. - p.Result.Tokens = p.tokenAPI.Tokens() - p.Result.Runes = p.tokenAPI.Runes() + p.Result.Tokens = p.tokenAPI.Output.Tokens() + p.Result.Runes = p.tokenAPI.Output.Runes() // Merge to the parent level. p.tokenAPI.Merge(forkedAPI) diff --git a/tokenize/api.go b/tokenize/api.go index ecf0985..fd44b72 100644 --- a/tokenize/api.go +++ b/tokenize/api.go @@ -78,6 +78,7 @@ type API struct { stackFrames []stackFrame // the stack frames, containing stack level-specific data stackLevel int // the current stack level stackFrame *stackFrame // the current stack frame + Output Output // provides output-related functionality } type stackFrame struct { @@ -93,6 +94,11 @@ type stackFrame struct { err error // can be used by a Handler to report a specific issue with the input } +// Output provides output-related functionality for the tokenize API. +type Output struct { + api *API +} + const initialStackDepth = 64 const initialTokenStoreLength = 64 const initialByteStoreLength = 1024 @@ -107,6 +113,7 @@ func NewAPI(input interface{}) *API { tokens: make([]Token, initialTokenStoreLength), stackFrames: make([]stackFrame, initialStackDepth), } + api.Output = Output{api: api} api.stackFrame = &api.stackFrames[0] return api @@ -460,73 +467,69 @@ func (i *API) FlushInput() bool { return false } -func (i *API) String() string { - bytes := i.bytes[i.stackFrame.bytesStart:i.stackFrame.bytesEnd] +func (o Output) String() string { + bytes := o.api.bytes[o.api.stackFrame.bytesStart:o.api.stackFrame.bytesEnd] return string(bytes) } -func (i *API) Runes() []rune { - bytes := i.bytes[i.stackFrame.bytesStart:i.stackFrame.bytesEnd] +func (o Output) Runes() []rune { + bytes := o.api.bytes[o.api.stackFrame.bytesStart:o.api.stackFrame.bytesEnd] return []rune(string(bytes)) } -func (i *API) Rune(offset int) rune { - r, _ := utf8.DecodeRune(i.bytes[i.stackFrame.bytesStart+offset:]) +func (o Output) Rune(offset int) rune { + r, _ := utf8.DecodeRune(o.api.bytes[o.api.stackFrame.bytesStart+offset:]) return r } -func (i *API) ClearBytes() { - i.stackFrame.bytesEnd = i.stackFrame.bytesStart +func (o Output) ClearData() { + o.api.stackFrame.bytesEnd = o.api.stackFrame.bytesStart } -func (i *API) SetBytes(bytes ...byte) { - i.ClearBytes() - i.AddBytes(bytes...) +func (o Output) SetBytes(bytes ...byte) { + o.ClearData() + o.AddBytes(bytes...) } -func (i *API) AddBytes(bytes ...byte) { +func (o Output) AddBytes(bytes ...byte) { // Grow the runes capacity when needed. - newBytesEnd := i.stackFrame.bytesEnd + len(bytes) - if cap(i.bytes) < newBytesEnd { + newBytesEnd := o.api.stackFrame.bytesEnd + len(bytes) + if cap(o.api.bytes) < newBytesEnd { newBytes := make([]byte, newBytesEnd*2) - copy(newBytes, i.bytes) - i.bytes = newBytes + copy(newBytes, o.api.bytes) + o.api.bytes = newBytes } - copy(i.bytes[i.stackFrame.bytesEnd:], bytes) - i.stackFrame.bytesEnd = newBytesEnd + copy(o.api.bytes[o.api.stackFrame.bytesEnd:], bytes) + o.api.stackFrame.bytesEnd = newBytesEnd } -func (i *API) ClearRunes() { - i.stackFrame.bytesEnd = i.stackFrame.bytesStart +func (o Output) SetRunes(runes ...rune) { + o.ClearData() + o.AddRunes(runes...) } -func (i *API) SetRunes(runes ...rune) { - i.ClearRunes() - i.AddRunes(runes...) -} - -func (i *API) AddRunes(runes ...rune) { +func (o Output) AddRunes(runes ...rune) { // Grow the runes capacity when needed. runesAsString := string(runes) - newBytesEnd := i.stackFrame.bytesEnd + len(runesAsString) - if cap(i.bytes) < newBytesEnd { + newBytesEnd := o.api.stackFrame.bytesEnd + len(runesAsString) + if cap(o.api.bytes) < newBytesEnd { newBytes := make([]byte, newBytesEnd*2) - copy(newBytes, i.bytes) - i.bytes = newBytes + copy(newBytes, o.api.bytes) + o.api.bytes = newBytes } - copy(i.bytes[i.stackFrame.bytesEnd:], runesAsString) - i.stackFrame.bytesEnd = newBytesEnd + copy(o.api.bytes[o.api.stackFrame.bytesEnd:], runesAsString) + o.api.stackFrame.bytesEnd = newBytesEnd } -func (i *API) AddString(s string) { - i.AddBytes([]byte(s)...) +func (o Output) AddString(s string) { + o.AddBytes([]byte(s)...) } -func (i *API) SetString(s string) { - i.ClearBytes() - i.SetBytes([]byte(s)...) +func (o Output) SetString(s string) { + o.ClearData() + o.SetBytes([]byte(s)...) } func (i *API) Cursor() string { @@ -536,38 +539,38 @@ func (i *API) Cursor() string { return fmt.Sprintf("line %d, column %d", i.stackFrame.line+1, i.stackFrame.column+1) } -func (i *API) Tokens() []Token { - return i.tokens[i.stackFrame.tokenStart:i.stackFrame.tokenEnd] +func (o Output) Tokens() []Token { + return o.api.tokens[o.api.stackFrame.tokenStart:o.api.stackFrame.tokenEnd] } -func (i *API) Token(offset int) Token { - return i.tokens[i.stackFrame.tokenStart+offset] +func (o Output) Token(offset int) Token { + return o.api.tokens[o.api.stackFrame.tokenStart+offset] } -func (i *API) TokenValue(offset int) interface{} { - return i.tokens[i.stackFrame.tokenStart+offset].Value +func (o Output) TokenValue(offset int) interface{} { + return o.api.tokens[o.api.stackFrame.tokenStart+offset].Value } -func (i *API) ClearTokens() { - i.stackFrame.tokenEnd = i.stackFrame.tokenStart +func (o Output) ClearTokens() { + o.api.stackFrame.tokenEnd = o.api.stackFrame.tokenStart } -func (i *API) SetTokens(tokens ...Token) { - i.ClearTokens() - i.AddTokens(tokens...) +func (o Output) SetTokens(tokens ...Token) { + o.ClearTokens() + o.AddTokens(tokens...) } -func (i *API) AddTokens(tokens ...Token) { +func (o Output) AddTokens(tokens ...Token) { // Grow the tokens capacity when needed. - newTokenEnd := i.stackFrame.tokenEnd + len(tokens) - if cap(i.tokens) < newTokenEnd { + newTokenEnd := o.api.stackFrame.tokenEnd + len(tokens) + if cap(o.api.tokens) < newTokenEnd { newTokens := make([]Token, newTokenEnd*2) - copy(newTokens, i.tokens) - i.tokens = newTokens + copy(newTokens, o.api.tokens) + o.api.tokens = newTokens } for offset, t := range tokens { - i.tokens[i.stackFrame.tokenEnd+offset] = t + o.api.tokens[o.api.stackFrame.tokenEnd+offset] = t } - i.stackFrame.tokenEnd = newTokenEnd + o.api.stackFrame.tokenEnd = newTokenEnd } diff --git a/tokenize/api_test.go b/tokenize/api_test.go index 7c917dc..0c2a9fe 100644 --- a/tokenize/api_test.go +++ b/tokenize/api_test.go @@ -19,7 +19,7 @@ func ExampleNewAPI() { // r, err := api.NextRune() // fmt.Printf("Rune read from input; %c\n", r) // fmt.Printf("The error: %v\n", err) -// fmt.Printf("API results: %q\n", api.String()) +// fmt.Printf("API results: %q\n", api.Output.String()) // // Output: // // Rune read from input; T @@ -55,7 +55,7 @@ func ExampleAPI_AcceptRune() { // Reads 'e', but does not accept it to the API results. r, _, _ = api.PeekRune(0) - fmt.Printf("API results: %q\n", api.String()) + fmt.Printf("API results: %q\n", api.Output.String()) // Output: // API results: "Th" @@ -74,7 +74,7 @@ func ExampleAPI_AcceptRunes() { // Accepts only 'T' and 'h' into the API results. api.AcceptRunes(r0, r1) - fmt.Printf("API results: %q\n", api.String()) + fmt.Printf("API results: %q\n", api.Output.String()) // Output: // API results: "Th" @@ -99,7 +99,7 @@ func ExampleAPI_SkipRune() { } } - fmt.Printf("API results: %q\n", api.String()) + fmt.Printf("API results: %q\n", api.Output.String()) // Output: // API results: "eiuaeAIiae" @@ -108,29 +108,29 @@ func ExampleAPI_SkipRune() { func ExampleAPI_modifyingResults() { api := tokenize.NewAPI("") - api.AddString("Some runes") - api.AddRunes(' ', 'a', 'd', 'd', 'e', 'd') - api.AddRunes(' ', 'i', 'n', ' ') - api.AddString("various ways") - fmt.Printf("API result first 10 runes: %q\n", api.Runes()[0:10]) - fmt.Printf("API result runes as string: %q\n", api.String()) + api.Output.AddString("Some runes") + api.Output.AddRunes(' ', 'a', 'd', 'd', 'e', 'd') + api.Output.AddRunes(' ', 'i', 'n', ' ') + api.Output.AddString("various ways") + fmt.Printf("API result first 10 runes: %q\n", api.Output.Runes()[0:10]) + fmt.Printf("API result runes as string: %q\n", api.Output.String()) - api.SetString("new ") - api.AddString("set ") - api.AddString("of ") - api.AddRunes('r', 'u', 'n', 'e', 's') - fmt.Printf("API result runes as string: %q\n", api.String()) - fmt.Printf("API result runes: %q\n", api.Runes()) - fmt.Printf("API third rune: %q\n", api.Rune(2)) + api.Output.SetString("new ") + api.Output.AddString("set ") + api.Output.AddString("of ") + api.Output.AddRunes('r', 'u', 'n', 'e', 's') + fmt.Printf("API result runes as string: %q\n", api.Output.String()) + fmt.Printf("API result runes: %q\n", api.Output.Runes()) + fmt.Printf("API third rune: %q\n", api.Output.Rune(2)) - api.AddTokens(tokenize.Token{ + api.Output.AddTokens(tokenize.Token{ Type: 42, Value: "towel"}) - api.AddTokens(tokenize.Token{ + api.Output.AddTokens(tokenize.Token{ Type: 73, Value: "Zaphod"}) - fmt.Printf("API result tokens: %v\n", api.Tokens()) - fmt.Printf("API second result token: %v\n", api.Token(1)) + fmt.Printf("API result tokens: %v\n", api.Output.Tokens()) + fmt.Printf("API second result token: %v\n", api.Output.Token(1)) // Output: // API result first 10 runes: ['S' 'o' 'm' 'e' ' ' 'r' 'u' 'n' 'e' 's'] @@ -149,18 +149,18 @@ func ExampleAPI_Reset() { api.AcceptRune(r) r, _, _ = api.PeekRune(0) // read 'e' api.AcceptRune(r) - fmt.Printf("API results: %q at %s\n", api.String(), api.Cursor()) + fmt.Printf("API results: %q at %s\n", api.Output.String(), api.Cursor()) // Reset clears the results. api.Reset() - fmt.Printf("API results: %q at %s\n", api.String(), api.Cursor()) + fmt.Printf("API results: %q at %s\n", api.Output.String(), api.Cursor()) // So then doing the same read operations, the same data are read. r, _, _ = api.PeekRune(0) // read 'V' api.AcceptRune(r) r, _, _ = api.PeekRune(0) // read 'e' api.AcceptRune(r) - fmt.Printf("API results: %q at %s\n", api.String(), api.Cursor()) + fmt.Printf("API results: %q at %s\n", api.Output.String(), api.Cursor()) // Output: // API results: "Ve" at line 1, column 3 @@ -246,23 +246,23 @@ func TestMultipleLevelsOfForksAndMerges(t *testing.T) { r, _, _ := api.PeekRune(0) AssertEqual(t, 'a', r, "child4 rune 1") api.AcceptRune(r) - AssertEqual(t, "a", api.String(), "child4 runes after rune 1") + AssertEqual(t, "a", api.Output.String(), "child4 runes after rune 1") // Read another rune 'b' from child4. r, _, _ = api.PeekRune(0) AssertEqual(t, 'b', r, "child4 rune 2") api.AcceptRune(r) - AssertEqual(t, "ab", api.String(), "child4 runes after rune 2") + AssertEqual(t, "ab", api.Output.String(), "child4 runes after rune 2") // Merge "ab" from child4 to child3. api.Merge(child4) - AssertEqual(t, "", api.String(), "child4 runes after first merge") + AssertEqual(t, "", api.Output.String(), "child4 runes after first merge") // Read some more from child4. r, _, _ = api.PeekRune(0) AssertEqual(t, 'c', r, "child4 rune 3") api.AcceptRune(r) - AssertEqual(t, "c", api.String(), "child4 runes after rune 1") + AssertEqual(t, "c", api.Output.String(), "child4 runes after rune 1") AssertEqual(t, "line 1, column 4", api.Cursor(), "cursor child4 rune 3") // Merge "c" from child4 to child3. @@ -272,7 +272,7 @@ func TestMultipleLevelsOfForksAndMerges(t *testing.T) { api.Dispose(child4) // Child3 should now have the compbined results "abc" from child4's work. - AssertEqual(t, "abc", api.String(), "child3 after merge of child4") + AssertEqual(t, "abc", api.Output.String(), "child3 after merge of child4") AssertEqual(t, "line 1, column 4", api.Cursor(), "cursor child3 rune 3, after merge of child4") // Now read some data from child3. @@ -288,7 +288,7 @@ func TestMultipleLevelsOfForksAndMerges(t *testing.T) { AssertEqual(t, 'f', r, "child3 rune 5") api.AcceptRune(r) - AssertEqual(t, "abcdef", api.String(), "child3 total result after rune 6") + AssertEqual(t, "abcdef", api.Output.String(), "child3 total result after rune 6") // Temporarily go some new forks from here, but don't use their outcome. child3sub1 := api.Fork() @@ -308,7 +308,7 @@ func TestMultipleLevelsOfForksAndMerges(t *testing.T) { api.Merge(child3) api.Dispose(child3) - AssertEqual(t, "abcdef", api.String(), "child2 total result after merge of child3") + AssertEqual(t, "abcdef", api.Output.String(), "child2 total result after merge of child3") AssertEqual(t, "line 1, column 7", api.Cursor(), "cursor child2 after merge child3") // Merge child2 to child1 and dispose of it. @@ -328,23 +328,23 @@ func TestMultipleLevelsOfForksAndMerges(t *testing.T) { r, _, _ = api.PeekRune(0) api.AcceptRune(r) - AssertEqual(t, "abcdefg", api.String(), "api string end result") + AssertEqual(t, "abcdefg", api.Output.String(), "api string end result") AssertEqual(t, "line 1, column 8", api.Cursor(), "api cursor end result") } -func TestClearRunes(t *testing.T) { +func TestClearData(t *testing.T) { api := tokenize.NewAPI("Laphroaig") r, _, _ := api.PeekRune(0) // Read 'L' api.AcceptRune(r) // Add to runes r, _, _ = api.PeekRune(0) // Read 'a' api.AcceptRune(r) // Add to runes - api.ClearRunes() // Clear the runes, giving us a fresh start. + api.Output.ClearData() // Clear the runes, giving us a fresh start. r, _, _ = api.PeekRune(0) // Read 'p' api.AcceptRune(r) // Add to runes r, _, _ = api.PeekRune(0) // Read 'r' api.AcceptRune(r) // Add to runes - AssertEqual(t, "ph", api.String(), "api string end result") + AssertEqual(t, "ph", api.Output.String(), "api string end result") } func TestMergeScenariosForTokens(t *testing.T) { @@ -355,34 +355,34 @@ func TestMergeScenariosForTokens(t *testing.T) { token3 := tokenize.Token{Value: 3} token4 := tokenize.Token{Value: 4} - api.SetTokens(token1) - tokens := api.Tokens() + api.Output.SetTokens(token1) + tokens := api.Output.Tokens() AssertEqual(t, 1, len(tokens), "Tokens 1") child := api.Fork() - tokens = api.Tokens() + tokens = api.Output.Tokens() AssertEqual(t, 0, len(tokens), "Tokens 2") - api.AddTokens(token2) + api.Output.AddTokens(token2) api.Merge(child) api.Dispose(child) - tokens = api.Tokens() + tokens = api.Output.Tokens() AssertEqual(t, 2, len(tokens), "Tokens 3") child = api.Fork() - api.AddTokens(token3) + api.Output.AddTokens(token3) api.Reset() - api.AddTokens(token4) + api.Output.AddTokens(token4) api.Merge(child) api.Dispose(child) - tokens = api.Tokens() + tokens = api.Output.Tokens() AssertEqual(t, 3, len(tokens), "Tokens 4") - AssertEqual(t, 1, api.TokenValue(0).(int), "Tokens 4, value 0") - AssertEqual(t, 2, api.TokenValue(1).(int), "Tokens 4, value 1") - AssertEqual(t, 4, api.TokenValue(2).(int), "Tokens 4, value 2") + AssertEqual(t, 1, api.Output.TokenValue(0).(int), "Tokens 4, value 0") + AssertEqual(t, 2, api.Output.TokenValue(1).(int), "Tokens 4, value 1") + AssertEqual(t, 4, api.Output.TokenValue(2).(int), "Tokens 4, value 2") } diff --git a/tokenize/handlers_builtin.go b/tokenize/handlers_builtin.go index e848268..0dd3490 100644 --- a/tokenize/handlers_builtin.go +++ b/tokenize/handlers_builtin.go @@ -987,7 +987,7 @@ func MatchIntegerBetween(min int64, max int64) Handler { if !digits(t) { return false } - value, _ := strconv.ParseInt(t.String(), 10, 64) + value, _ := strconv.ParseInt(t.Output.String(), 10, 64) if value < min || value > max { return false } @@ -1346,16 +1346,16 @@ func MatchOctet(normalize bool) Handler { if !max3Digits(t) { return false } - value, _ := strconv.ParseInt(t.String(), 10, 16) + value, _ := strconv.ParseInt(t.Output.String(), 10, 16) if value > 255 { return false } if normalize { - runes := t.Runes() + runes := t.Output.Runes() for len(runes) > 1 && runes[0] == '0' { runes = runes[1:] } - t.SetRunes(runes...) + t.Output.SetRunes(runes...) } return true } @@ -1397,13 +1397,13 @@ func MatchIPv4Netmask(normalize bool) Handler { } // Check if the mask is provided in canonical form (at the binary level, ones followed by zeroes). - mask := net.IPv4Mask(t.TokenValue(0).(byte), t.TokenValue(1).(byte), t.TokenValue(2).(byte), t.TokenValue(3).(byte)) + mask := net.IPv4Mask(t.Output.TokenValue(0).(byte), t.Output.TokenValue(1).(byte), t.Output.TokenValue(2).(byte), t.Output.TokenValue(3).(byte)) ones, bits := mask.Size() if ones == 0 && bits == 0 { return false } - t.ClearTokens() + t.Output.ClearTokens() return true } } @@ -1432,18 +1432,18 @@ func MatchIPv4Net(normalize bool) Handler { return true } - maskToken := t.Token(1) + maskToken := t.Output.Token(1) if maskToken.Type == "cidr" { - t.SetString(fmt.Sprintf("%s/%d", t.TokenValue(0), t.TokenValue(1).(uint8))) + t.Output.SetString(fmt.Sprintf("%s/%d", t.Output.TokenValue(0), t.Output.TokenValue(1).(uint8))) } else { - o := strings.Split(t.TokenValue(1).(string), ".") + o := strings.Split(t.Output.TokenValue(1).(string), ".") b := func(idx int) byte { i, _ := strconv.Atoi(o[idx]); return byte(i) } mask := net.IPv4Mask(b(0), b(1), b(2), b(3)) bits, _ := mask.Size() - t.SetString(fmt.Sprintf("%s/%d", t.TokenValue(0), bits)) + t.Output.SetString(fmt.Sprintf("%s/%d", t.Output.TokenValue(0), bits)) } - t.ClearTokens() + t.Output.ClearTokens() return true } } @@ -1472,13 +1472,13 @@ func MatchIPv6(normalize bool) Handler { } // Invalid IPv6, when net.ParseIP() cannot handle it. - parsed := net.ParseIP(t.String()) + parsed := net.ParseIP(t.Output.String()) if parsed == nil { return false } if normalize { - t.SetString(parsed.String()) + t.Output.SetString(parsed.String()) } return true } @@ -1501,8 +1501,8 @@ func matchCIDRMask(bits int64, normalize bool) Handler { if !mask(t) { return false } - bits, _ := strconv.Atoi(t.String()) - t.SetString(fmt.Sprintf("%d", bits)) + bits, _ := strconv.Atoi(t.Output.String()) + t.Output.SetString(fmt.Sprintf("%d", bits)) return true } } @@ -1643,8 +1643,8 @@ func ModifyByCallback(handler Handler, modfunc func(string) string) Handler { return func(t *API) bool { child := t.Fork() if handler(t) { - s := modfunc(t.String()) - t.SetString(s) + s := modfunc(t.Output.String()) + t.Output.SetString(s) t.Merge(child) t.Dispose(child) return true @@ -1661,7 +1661,7 @@ func ModifyByCallback(handler Handler, modfunc func(string) string) Handler { // an 'n'-character). func MakeStrLiteralToken(toktype interface{}, handler Handler) Handler { return MakeTokenByCallback(toktype, handler, func(t *API) interface{} { - literal := t.String() + literal := t.Output.String() return literal }) } @@ -1673,7 +1673,7 @@ func MakeStrLiteralToken(toktype interface{}, handler Handler) Handler { func MakeStrInterpretedToken(toktype interface{}, handler Handler) Handler { return MakeTokenByCallback(toktype, handler, func(t *API) interface{} { // TODO ERROR HANDLING - interpreted, _ := interpretString(t.String()) + interpreted, _ := interpretString(t.Output.String()) return interpreted }) } @@ -1697,7 +1697,7 @@ func interpretString(str string) (string, error) { func MakeRuneToken(toktype interface{}, handler Handler) Handler { return MakeTokenByCallback(toktype, handler, func(t *API) interface{} { // TODO ERROR HANDLING --- not a 1 rune input - return t.Rune(0) + return t.Output.Rune(0) }) } @@ -1707,7 +1707,7 @@ func MakeRuneToken(toktype interface{}, handler Handler) Handler { func MakeByteToken(toktype interface{}, handler Handler) Handler { return MakeTokenByCallback(toktype, handler, func(t *API) interface{} { // TODO ERROR HANDLING --- not a 1 byte input - return byte(t.Rune(0)) + return byte(t.Output.Rune(0)) }) } @@ -1912,7 +1912,7 @@ func MakeBooleanToken(toktype interface{}, handler Handler) Handler { func makeStrconvToken(name string, toktype interface{}, handler Handler, convert func(s string) (interface{}, error)) Handler { return MakeTokenByCallback(toktype, handler, func(t *API) interface{} { - value, err := convert(t.String()) + value, err := convert(t.Output.String()) if err != nil { // TODO meh, panic feels so bad here. Maybe just turn this case into "no match"? panic(fmt.Sprintf("%s token invalid (%s)", name, err)) @@ -1943,7 +1943,7 @@ func MakeTokenByCallback(toktype interface{}, handler Handler, makeValue func(t // tokens will end up in the order "date", "year", "month", "day". When we'd add the // token to the child here, the order would have been "year", "month", "day", "date". token := Token{Type: toktype, Value: makeValue(t)} - t.AddTokens(token) + t.Output.AddTokens(token) t.Merge(child) t.Dispose(child) @@ -1960,10 +1960,10 @@ func MakeTokenGroup(toktype interface{}, handler Handler) Handler { return func(t *API) bool { child := t.Fork() if handler(t) { - tokens := t.Tokens() + tokens := t.Output.Tokens() tokensCopy := make([]Token, len(tokens)) copy(tokensCopy, tokens) - t.SetTokens(Token{Type: toktype, Value: tokensCopy}) + t.Output.SetTokens(Token{Type: toktype, Value: tokensCopy}) t.Merge(child) t.Dispose(child) return true diff --git a/tokenize/tokenize.go b/tokenize/tokenize.go index d7d0d42..35c9064 100644 --- a/tokenize/tokenize.go +++ b/tokenize/tokenize.go @@ -47,8 +47,8 @@ func New(tokenHandler Handler) Func { return nil, err } result := &Result{ - Runes: api.Runes(), - Tokens: api.Tokens(), + Runes: api.Output.Runes(), + Tokens: api.Output.Tokens(), } return result, nil } diff --git a/tokenize/tokenizer_test.go b/tokenize/tokenizer_test.go index 0706a0f..efa07ad 100644 --- a/tokenize/tokenizer_test.go +++ b/tokenize/tokenizer_test.go @@ -69,7 +69,7 @@ func TestInputCanAcceptRunesFromReader(t *testing.T) { r2, _, _ := i.PeekRune(1) i.AcceptRunes(r1, r2) - AssertEqual(t, "Tes", i.String(), "i.String()") + AssertEqual(t, "Tes", i.Output.String(), "i.String()") } func TestCallingMergeOnTopLevelAPI_Panics(t *testing.T) { diff --git a/tokenize/tokenizer_whitebox_test.go b/tokenize/tokenizer_whitebox_test.go index 9c9e715..139b2cb 100644 --- a/tokenize/tokenizer_whitebox_test.go +++ b/tokenize/tokenizer_whitebox_test.go @@ -9,7 +9,7 @@ func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) { i := NewAPI("Testing") r, _, _ := i.PeekRune(0) i.AcceptRune(r) // T - AssertEqual(t, "T", i.String(), "accepted rune in input") + AssertEqual(t, "T", i.Output.String(), "accepted rune in input") // Fork child := i.Fork() @@ -21,14 +21,14 @@ func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) { i.AcceptRune(r) // e r, _, _ = i.PeekRune(0) i.AcceptRune(r) // s - AssertEqual(t, "es", i.String(), "result runes in fork") + AssertEqual(t, "es", i.Output.String(), "result runes in fork") AssertEqual(t, 1, i.stackFrames[i.stackLevel-1].offset, "parent offset") AssertEqual(t, 3, i.stackFrame.offset, "child offset") // Merge fork back into parent i.Merge(child) i.Dispose(child) - AssertEqual(t, "Tes", i.String(), "result runes in parent Input after Merge()") + AssertEqual(t, "Tes", i.Output.String(), "result runes in parent Input after Merge()") AssertEqual(t, 3, i.stackFrame.offset, "parent offset") } @@ -44,17 +44,17 @@ func TestGivenForkedChildWhichAcceptedRune_AfterMerging_RuneEndsUpInParentResult f2 := i.Fork() r, _, _ = i.PeekRune(0) i.AcceptRune(r) // s - AssertEqual(t, "s", i.String(), "f2 String()") + AssertEqual(t, "s", i.Output.String(), "f2 String()") AssertEqual(t, 3, i.stackFrame.offset, "f2.offset A") i.Merge(f2) i.Dispose(f2) - AssertEqual(t, "es", i.String(), "f1 String()") + AssertEqual(t, "es", i.Output.String(), "f1 String()") AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A") i.Merge(f1) i.Dispose(f1) - AssertEqual(t, "Tes", i.String(), "top-level API String()") + AssertEqual(t, "Tes", i.Output.String(), "top-level API String()") AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A") } @@ -83,7 +83,7 @@ func TestFlushInput(t *testing.T) { r, _, _ = i.PeekRune(0) i.AcceptRune(r) // o - AssertEqual(t, "cool", i.String(), "end result") + AssertEqual(t, "cool", i.Output.String(), "end result") } func TestInputFlusherWrapper(t *testing.T) { @@ -92,19 +92,19 @@ func TestInputFlusherWrapper(t *testing.T) { api := NewAPI("abaab") runeA(api) AssertEqual(t, 1, api.stackFrame.offset, "offset after 1 read") - AssertEqual(t, "a", api.String(), "runes after 1 read") + AssertEqual(t, "a", api.Output.String(), "runes after 1 read") flushB(api) AssertEqual(t, 0, api.stackFrame.offset, "offset after 2 reads + input flush") - AssertEqual(t, "ab", api.String(), "runes after 2 reads") + AssertEqual(t, "ab", api.Output.String(), "runes after 2 reads") runeA(api) AssertEqual(t, 1, api.stackFrame.offset, "offset after 3 reads") - AssertEqual(t, "aba", api.String(), "runes after 3 reads") + AssertEqual(t, "aba", api.Output.String(), "runes after 3 reads") runeA(api) AssertEqual(t, 2, api.stackFrame.offset, "offset after 4 reads") - AssertEqual(t, "abaa", api.String(), "runes after 4 reads") + AssertEqual(t, "abaa", api.Output.String(), "runes after 4 reads") flushB(api) AssertEqual(t, 0, api.stackFrame.offset, "offset after 5 reads + input flush") - AssertEqual(t, "abaab", api.String(), "runes after 5 reads") + AssertEqual(t, "abaab", api.Output.String(), "runes after 5 reads") } func AssertEqual(t *testing.T, expected interface{}, actual interface{}, forWhat string) {