diff --git a/tokenize/api.go b/tokenize/api.go index e9ac3c1..a2a3a0d 100644 --- a/tokenize/api.go +++ b/tokenize/api.go @@ -69,11 +69,14 @@ import ( // can lead to hard to track bugs. I much prefer this forking method, since // no bookkeeping has to be implemented when implementing a parser. type API struct { - stackFrames []stackFrame // the stack frames, containing stack level-specific data - stackLevel int // the current stack level - stackFrame *stackFrame // the current stack frame - Input *Input // provides input-related functionality - Output *Output // provides output-related functionality + stackFrames []stackFrame // the stack frames, containing stack level-specific data + stackLevel int // the current stack level + stackFrame *stackFrame // the current stack frame + Input *Input // provides input-related functionality + reader *read.Buffer // the input data reader + Output *Output // provides output-related functionality + outputTokens []Token // accepted tokens + outputData []byte // accepted data } type stackFrame struct { @@ -98,17 +101,11 @@ const initialByteStoreLength = 1024 // for parsekit.read.New(). func NewAPI(input interface{}) *API { api := &API{ + reader: read.New(input), stackFrames: make([]stackFrame, initialStackDepth), } - api.Input = &Input{ - api: api, - reader: read.New(input), - } - api.Output = &Output{ - api: api, - data: make([]byte, initialByteStoreLength), - tokens: make([]Token, initialTokenStoreLength), - } + api.Input = &Input{api: api} + api.Output = &Output{api: api} api.stackFrame = &api.stackFrames[0] return api @@ -207,6 +204,25 @@ func (tokenAPI *API) Merge(stackLevel int) { tokenAPI.stackFrame.err = nil } +// Reset moves the input cursor back to the beginning for the currently active API child. +// Aditionally, any output (bytes and tokens) that was emitted from the API child are +// cleared as well. +func (api *API) Reset() { + if api.stackLevel == 0 { + api.stackFrame.column = 0 + api.stackFrame.line = 0 + api.stackFrame.offset = 0 + } else { + parent := api.stackFrames[api.stackLevel-1] + api.stackFrame.column = parent.column + api.stackFrame.line = parent.line + api.stackFrame.offset = parent.offset + } + api.stackFrame.bytesEnd = api.stackFrame.bytesStart + api.stackFrame.tokenEnd = api.stackFrame.tokenStart + api.stackFrame.err = nil +} + func (tokenAPI *API) Dispose(stackLevel int) { if stackLevel == 0 { callerPanic("Dispose", "tokenize.API.{name}(): {name}() called at {caller} "+ diff --git a/tokenize/api_input.go b/tokenize/api_input.go index aff6cee..9cbe21d 100644 --- a/tokenize/api_input.go +++ b/tokenize/api_input.go @@ -3,40 +3,11 @@ package tokenize import ( "fmt" "unicode/utf8" - - "git.makaay.nl/mauricem/go-parsekit/read" ) // Input provides input-related functionality for the tokenize API. type Input struct { - api *API - reader *read.Buffer // the input data reader -} - -// Reset moves the input cursor back to the beginning for the currently active API child. -// Aditionally, any output (bytes and tokens) that was emitted from the API child are -// cleared as well. -func (i *Input) Reset() { - if i.api.stackLevel == 0 { - i.api.stackFrame.column = 0 - i.api.stackFrame.line = 0 - i.api.stackFrame.offset = 0 - } else { - parent := i.api.stackFrames[i.api.stackLevel-1] - i.api.stackFrame.column = parent.column - i.api.stackFrame.line = parent.line - i.api.stackFrame.offset = parent.offset - } - i.api.stackFrame.bytesEnd = i.api.stackFrame.bytesStart - i.api.stackFrame.tokenEnd = i.api.stackFrame.tokenStart - i.api.stackFrame.err = nil -} - -func (i *Input) Cursor() string { - if i.api.stackFrame.line == 0 && i.api.stackFrame.column == 0 { - return fmt.Sprintf("start of file") - } - return fmt.Sprintf("line %d, column %d", i.api.stackFrame.line+1, i.api.stackFrame.column+1) + api *API } // PeekByte returns the byte at the provided byte offset. @@ -45,7 +16,11 @@ func (i *Input) Cursor() string { // When an offset is requested that is beyond the length of the available input // data, then the error will be io.EOF. func (i *Input) PeekByte(offset int) (byte, error) { - return i.reader.ByteAt(i.api.stackFrame.offset + offset) + return i.api.peekByte(offset) +} + +func (api *API) peekByte(offset int) (byte, error) { + return api.reader.ByteAt(api.stackFrame.offset + offset) } // SkipByte is used to skip over a single bytes that was read from the input. @@ -59,8 +34,12 @@ func (i *Input) PeekByte(offset int) (byte, error) { // After the call, byte offset 0 for PeekByte() and PeekRune() will point at // the first byte after the skipped byte. func (i *Input) SkipByte(b byte) { - i.api.stackFrame.moveCursorByByte(b) - i.api.stackFrame.offset++ + i.api.skipByte(b) +} + +func (api *API) skipByte(b byte) { + api.stackFrame.moveCursorByByte(b) + api.stackFrame.offset++ } // SkipBytes is used to skip over one or more bytes that were read from the input. @@ -74,9 +53,13 @@ func (i *Input) SkipByte(b byte) { // After the call, byte offset 0 for PeekByte() and PeekRune() will point at // the first byte after the skipped bytes. func (i *Input) SkipBytes(bytes ...byte) { + i.api.skipBytes(bytes...) +} + +func (api *API) skipBytes(bytes ...byte) { for _, b := range bytes { - i.api.stackFrame.moveCursorByByte(b) - i.api.stackFrame.offset++ + api.stackFrame.moveCursorByByte(b) + api.stackFrame.offset++ } } @@ -92,20 +75,24 @@ func (i *Input) SkipBytes(bytes ...byte) { // After the call, byte offset 0 for PeekByte() and PeekRune() will point at // the first byte after the accepted byte. func (i *Input) AcceptByte(b byte) { - curBytesEnd := i.api.stackFrame.bytesEnd + i.api.acceptByte(b) +} + +func (api *API) acceptByte(b byte) { + curBytesEnd := api.stackFrame.bytesEnd maxRequiredBytes := curBytesEnd + 1 // Grow the bytes capacity when needed. - if cap(i.api.Output.data) < maxRequiredBytes { + if cap(api.outputData) < maxRequiredBytes { newBytes := make([]byte, maxRequiredBytes*2) - copy(newBytes, i.api.Output.data) - i.api.Output.data = newBytes + copy(newBytes, api.outputData) + api.outputData = newBytes } - i.api.Output.data[curBytesEnd] = b - i.api.stackFrame.moveCursorByByte(b) - i.api.stackFrame.bytesEnd++ - i.api.stackFrame.offset++ + api.outputData[curBytesEnd] = b + api.stackFrame.moveCursorByByte(b) + api.stackFrame.bytesEnd++ + api.stackFrame.offset++ } // AcceptBytes is used to accept one or more bytes that were read from the input. @@ -120,22 +107,26 @@ func (i *Input) AcceptByte(b byte) { // After the call, byte offset 0 for PeekByte() and PeekRune() will point at // the first byte after the accepted bytes. func (i *Input) AcceptBytes(bytes ...byte) { - curBytesEnd := i.api.stackFrame.bytesEnd + i.api.acceptBytes(bytes...) +} + +func (api *API) acceptBytes(bytes ...byte) { + curBytesEnd := api.stackFrame.bytesEnd newBytesEnd := curBytesEnd + len(bytes) // Grow the bytes capacity when needed. - if cap(i.api.Output.data) < newBytesEnd { + if cap(api.outputData) < newBytesEnd { newBytes := make([]byte, newBytesEnd*2) - copy(newBytes, i.api.Output.data) - i.api.Output.data = newBytes + copy(newBytes, api.outputData) + api.outputData = newBytes } - copy(i.api.Output.data[curBytesEnd:], bytes) + copy(api.outputData[curBytesEnd:], bytes) for _, b := range bytes { - i.api.stackFrame.moveCursorByByte(b) - i.api.stackFrame.offset++ + api.stackFrame.moveCursorByByte(b) + api.stackFrame.offset++ } - i.api.stackFrame.bytesEnd = newBytesEnd + api.stackFrame.bytesEnd = newBytesEnd } // PeekRune returns the UTF8 rune at the provided byte offset, including its byte width. @@ -152,7 +143,11 @@ func (i *Input) AcceptBytes(bytes ...byte) { // When an offset is requested that is beyond the length of the available input // data, then the error will be io.EOF. func (i *Input) PeekRune(offset int) (rune, int, error) { - return i.reader.RuneAt(i.api.stackFrame.offset + offset) + return i.api.peekRune(offset) +} + +func (api *API) peekRune(offset int) (rune, int, error) { + return api.reader.RuneAt(api.stackFrame.offset + offset) } // SkipRune is used to skip over a single rune that was read from the input. @@ -166,8 +161,12 @@ func (i *Input) PeekRune(offset int) (rune, int, error) { // After the call, byte offset 0 for PeekByte() and PeekRune() will point at // the first byte after the skipped rune. func (i *Input) SkipRune(r rune) { - i.api.stackFrame.moveCursorByRune(r) - i.api.stackFrame.offset += utf8.RuneLen(r) + i.api.skipRune(r) +} + +func (api *API) skipRune(r rune) { + api.stackFrame.moveCursorByRune(r) + api.stackFrame.offset += utf8.RuneLen(r) } // SkipRunes is used to skip over one or more runes that were read from the input. @@ -181,9 +180,13 @@ func (i *Input) SkipRune(r rune) { // After the call, byte offset 0 for PeekByte() and PeekRune() will point at // the first byte after the skipped runes. func (i *Input) SkipRunes(runes ...rune) { + i.api.skipRunes(runes...) +} + +func (api *API) skipRunes(runes ...rune) { for _, r := range runes { - i.api.stackFrame.moveCursorByRune(r) - i.api.stackFrame.offset += utf8.RuneLen(r) + api.stackFrame.moveCursorByRune(r) + api.stackFrame.offset += utf8.RuneLen(r) } } @@ -199,20 +202,24 @@ func (i *Input) SkipRunes(runes ...rune) { // After the call, byte offset 0 for PeekByte() and PeekRune() will point at // the first byte after the accepted rune. func (i *Input) AcceptRune(r rune) { - curBytesEnd := i.api.stackFrame.bytesEnd + i.api.acceptRune(r) +} + +func (api *API) acceptRune(r rune) { + curBytesEnd := api.stackFrame.bytesEnd maxRequiredBytes := curBytesEnd + utf8.UTFMax // Grow the runes capacity when needed. - if cap(i.api.Output.data) < maxRequiredBytes { + if cap(api.outputData) < maxRequiredBytes { newBytes := make([]byte, maxRequiredBytes*2) - copy(newBytes, i.api.Output.data) - i.api.Output.data = newBytes + copy(newBytes, api.outputData) + api.outputData = newBytes } - i.api.stackFrame.moveCursorByRune(r) - w := utf8.EncodeRune(i.api.Output.data[curBytesEnd:], r) - i.api.stackFrame.bytesEnd += w - i.api.stackFrame.offset += w + api.stackFrame.moveCursorByRune(r) + w := utf8.EncodeRune(api.outputData[curBytesEnd:], r) + api.stackFrame.bytesEnd += w + api.stackFrame.offset += w } // AcceptRunes is used to accept one or more runes that were read from the input. @@ -227,25 +234,29 @@ func (i *Input) AcceptRune(r rune) { // After the call, byte offset 0 for PeekByte() and PeekRune() will point at // the first byte after the accepted runes. func (i *Input) AcceptRunes(runes ...rune) { + i.api.acceptRunes(runes...) +} + +func (api *API) acceptRunes(runes ...rune) { runesAsString := string(runes) byteLen := len(runesAsString) - curBytesEnd := i.api.stackFrame.bytesEnd + curBytesEnd := api.stackFrame.bytesEnd newBytesEnd := curBytesEnd + byteLen // Grow the runes capacity when needed. - if cap(i.api.Output.data) < newBytesEnd { + if cap(api.outputData) < newBytesEnd { newBytes := make([]byte, newBytesEnd*2) - copy(newBytes, i.api.Output.data) - i.api.Output.data = newBytes + copy(newBytes, api.outputData) + api.outputData = newBytes } for _, r := range runes { - i.api.stackFrame.moveCursorByRune(r) + api.stackFrame.moveCursorByRune(r) } - copy(i.api.Output.data[curBytesEnd:], runesAsString) + copy(api.outputData[curBytesEnd:], runesAsString) - i.api.stackFrame.bytesEnd = newBytesEnd - i.api.stackFrame.offset += byteLen + api.stackFrame.bytesEnd = newBytesEnd + api.stackFrame.offset += byteLen } // Flush flushes input data from the read.Buffer up to the current @@ -255,10 +266,25 @@ func (i *Input) AcceptRunes(runes ...rune) { // When writing your own TokenHandler, you normally won't have to call this // method yourself. It is automatically called by parsekit when possible. func (i *Input) Flush() bool { - if i.api.stackFrame.offset > 0 { - i.reader.Flush(i.api.stackFrame.offset) - i.api.stackFrame.offset = 0 + return i.api.flushInput() +} + +func (api *API) flushInput() bool { + if api.stackFrame.offset > 0 { + api.reader.Flush(api.stackFrame.offset) + api.stackFrame.offset = 0 return true } return false } + +func (i *Input) Cursor() string { + return i.api.cursor() +} + +func (api *API) cursor() string { + if api.stackFrame.line == 0 && api.stackFrame.column == 0 { + return fmt.Sprintf("start of file") + } + return fmt.Sprintf("line %d, column %d", api.stackFrame.line+1, api.stackFrame.column+1) +} diff --git a/tokenize/api_output.go b/tokenize/api_output.go index 33e751e..39752d1 100644 --- a/tokenize/api_output.go +++ b/tokenize/api_output.go @@ -6,107 +6,168 @@ import ( // Output provides output-related functionality for the tokenize API. type Output struct { - api *API - tokens []Token // accepted tokens - data []byte // accepted data + api *API } func (o *Output) String() string { - bytes := o.data[o.api.stackFrame.bytesStart:o.api.stackFrame.bytesEnd] + return o.api.dataAsString() +} +func (api *API) dataAsString() string { + bytes := api.outputData[api.stackFrame.bytesStart:api.stackFrame.bytesEnd] return string(bytes) } func (o *Output) Runes() []rune { - bytes := o.data[o.api.stackFrame.bytesStart:o.api.stackFrame.bytesEnd] + return o.api.dataAsRunes() +} + +func (api *API) dataAsRunes() []rune { + bytes := api.outputData[api.stackFrame.bytesStart:api.stackFrame.bytesEnd] return []rune(string(bytes)) } func (o *Output) Rune(offset int) rune { - r, _ := utf8.DecodeRune(o.data[o.api.stackFrame.bytesStart+offset:]) + return o.api.dataRune(offset) +} + +func (api *API) dataRune(offset int) rune { + r, _ := utf8.DecodeRune(api.outputData[api.stackFrame.bytesStart+offset:]) return r } func (o *Output) ClearData() { - o.api.stackFrame.bytesEnd = o.api.stackFrame.bytesStart + o.api.dataClear() +} + +func (api *API) dataClear() { + api.stackFrame.bytesEnd = api.stackFrame.bytesStart } func (o *Output) SetBytes(bytes ...byte) { - o.ClearData() - o.AddBytes(bytes...) + o.api.dataSetBytes(bytes...) +} + +func (api *API) dataSetBytes(bytes ...byte) { + api.dataClear() + api.dataAddBytes(bytes...) } func (o *Output) AddBytes(bytes ...byte) { - // Grow the runes capacity when needed. - newBytesEnd := o.api.stackFrame.bytesEnd + len(bytes) - if cap(o.data) < newBytesEnd { - newBytes := make([]byte, newBytesEnd*2) - copy(newBytes, o.data) - o.data = newBytes - } - - copy(o.data[o.api.stackFrame.bytesEnd:], bytes) - o.api.stackFrame.bytesEnd = newBytesEnd + o.api.dataAddBytes(bytes...) } -func (o Output) SetRunes(runes ...rune) { - o.ClearData() - o.AddRunes(runes...) +func (api *API) dataAddBytes(bytes ...byte) { + // Grow the runes capacity when needed. + newBytesEnd := api.stackFrame.bytesEnd + len(bytes) + if cap(api.outputData) < newBytesEnd { + newBytes := make([]byte, newBytesEnd*2) + copy(newBytes, api.outputData) + api.outputData = newBytes + } + + copy(api.outputData[api.stackFrame.bytesEnd:], bytes) + api.stackFrame.bytesEnd = newBytesEnd +} + +func (o *Output) SetRunes(runes ...rune) { + o.api.dataSetRunes(runes...) +} + +func (api *API) dataSetRunes(runes ...rune) { + api.dataClear() + api.dataAddRunes(runes...) } func (o *Output) AddRunes(runes ...rune) { + o.api.dataAddRunes(runes...) +} + +func (api *API) dataAddRunes(runes ...rune) { // Grow the runes capacity when needed. runesAsString := string(runes) - newBytesEnd := o.api.stackFrame.bytesEnd + len(runesAsString) - if cap(o.data) < newBytesEnd { + newBytesEnd := api.stackFrame.bytesEnd + len(runesAsString) + if cap(api.outputData) < newBytesEnd { newBytes := make([]byte, newBytesEnd*2) - copy(newBytes, o.data) - o.data = newBytes + copy(newBytes, api.outputData) + api.outputData = newBytes } - copy(o.data[o.api.stackFrame.bytesEnd:], runesAsString) - o.api.stackFrame.bytesEnd = newBytesEnd + copy(api.outputData[api.stackFrame.bytesEnd:], runesAsString) + api.stackFrame.bytesEnd = newBytesEnd } func (o *Output) AddString(s string) { - o.AddBytes([]byte(s)...) + o.api.dataAddString(s) +} + +func (api *API) dataAddString(s string) { + api.dataAddBytes([]byte(s)...) } func (o *Output) SetString(s string) { - o.ClearData() - o.SetBytes([]byte(s)...) + o.api.dataSetString(s) +} + +func (api *API) dataSetString(s string) { + api.dataClear() + api.dataSetBytes([]byte(s)...) } func (o *Output) Tokens() []Token { - return o.tokens[o.api.stackFrame.tokenStart:o.api.stackFrame.tokenEnd] + return o.api.tokens() +} + +func (api *API) tokens() []Token { + return api.outputTokens[api.stackFrame.tokenStart:api.stackFrame.tokenEnd] } func (o *Output) Token(offset int) Token { - return o.tokens[o.api.stackFrame.tokenStart+offset] + return o.api.token(offset) +} + +func (api *API) token(offset int) Token { + return api.outputTokens[api.stackFrame.tokenStart+offset] } func (o *Output) TokenValue(offset int) interface{} { - return o.tokens[o.api.stackFrame.tokenStart+offset].Value + return o.api.tokenValue(offset) +} + +func (api *API) tokenValue(offset int) interface{} { + return api.outputTokens[api.stackFrame.tokenStart+offset].Value } func (o *Output) ClearTokens() { - o.api.stackFrame.tokenEnd = o.api.stackFrame.tokenStart + o.api.tokensClear() +} + +func (api *API) tokensClear() { + api.stackFrame.tokenEnd = api.stackFrame.tokenStart } func (o *Output) SetTokens(tokens ...Token) { - o.ClearTokens() - o.AddTokens(tokens...) + o.api.tokensSet(tokens...) +} + +func (api *API) tokensSet(tokens ...Token) { + api.tokensClear() + api.tokensAdd(tokens...) } func (o *Output) AddTokens(tokens ...Token) { + o.api.tokensAdd(tokens...) +} + +func (api *API) tokensAdd(tokens ...Token) { // Grow the tokens capacity when needed. - newTokenEnd := o.api.stackFrame.tokenEnd + len(tokens) - if cap(o.tokens) < newTokenEnd { + newTokenEnd := api.stackFrame.tokenEnd + len(tokens) + if cap(api.outputTokens) < newTokenEnd { newTokens := make([]Token, newTokenEnd*2) - copy(newTokens, o.tokens) - o.tokens = newTokens + copy(newTokens, api.outputTokens) + api.outputTokens = newTokens } for offset, t := range tokens { - o.tokens[o.api.stackFrame.tokenEnd+offset] = t + api.outputTokens[api.stackFrame.tokenEnd+offset] = t } - o.api.stackFrame.tokenEnd = newTokenEnd + api.stackFrame.tokenEnd = newTokenEnd } diff --git a/tokenize/api_test.go b/tokenize/api_test.go index 1db8398..fec74f3 100644 --- a/tokenize/api_test.go +++ b/tokenize/api_test.go @@ -19,7 +19,7 @@ func ExampleNewAPI() { // r, err := api.NextRune() // fmt.Printf("Rune read from input; %c\n", r) // fmt.Printf("The error: %v\n", err) -// fmt.Printf("API results: %q\n", api.Output.String()) +// fmt.Printf("API results: %q\n", api.dataAsString()) // // Output: // // Rune read from input; T @@ -152,7 +152,7 @@ func ExampleAPI_Reset() { fmt.Printf("API results: %q at %s\n", api.Output.String(), api.Input.Cursor()) // Reset clears the results. - api.Input.Reset() + api.Reset() fmt.Printf("API results: %q at %s\n", api.Output.String(), api.Input.Cursor()) // So then doing the same read operations, the same data are read. @@ -374,7 +374,7 @@ func TestMergeScenariosForTokens(t *testing.T) { child = api.Fork() api.Output.AddTokens(token3) - api.Input.Reset() + api.Reset() api.Output.AddTokens(token4) api.Merge(child) diff --git a/tokenize/handlers_builtin.go b/tokenize/handlers_builtin.go index a8c3ebf..0c29923 100644 --- a/tokenize/handlers_builtin.go +++ b/tokenize/handlers_builtin.go @@ -350,9 +350,9 @@ var T = struct { // MatchByte creates a Handler function that matches against the provided byte. func MatchByte(expected byte) Handler { return func(t *API) bool { - b, err := t.Input.PeekByte(0) + b, err := t.peekByte(0) if err == nil && b == expected { - t.Input.AcceptByte(b) + t.acceptByte(b) return true } return false @@ -365,9 +365,9 @@ func MatchRune(expected rune) Handler { return MatchByte(byte(expected)) } return func(t *API) bool { - r, _, err := t.Input.PeekRune(0) + r, _, err := t.peekRune(0) if err == nil && r == expected { - t.Input.AcceptRune(r) + t.acceptRune(r) return true } return false @@ -378,13 +378,13 @@ func MatchRune(expected rune) Handler { // one of the provided bytes. The first match counts. func MatchBytes(expected ...byte) Handler { return func(t *API) bool { - b, err := t.Input.PeekByte(0) + b, err := t.peekByte(0) if err != nil { return false } for _, e := range expected { if b == e { - t.Input.AcceptByte(b) + t.acceptByte(b) return true } } @@ -408,13 +408,13 @@ func MatchRunes(expected ...rune) Handler { return MatchBytes(expectedBytes...) } return func(t *API) bool { - r, _, err := t.Input.PeekRune(0) + r, _, err := t.peekRune(0) if err != nil { return false } for _, e := range expected { if r == e { - t.Input.AcceptRune(r) + t.acceptRune(r) return true } } @@ -434,9 +434,9 @@ func MatchByteRange(start byte, end byte) Handler { callerPanic("MatchByteRange", "Handler: {name} definition error at {caller}: start %q must not be < end %q", start, end) } return func(t *API) bool { - r, err := t.Input.PeekByte(0) + r, err := t.peekByte(0) if err == nil && r >= start && r <= end { - t.Input.AcceptByte(r) + t.acceptByte(r) return true } return false @@ -458,9 +458,9 @@ func MatchRuneRange(start rune, end rune) Handler { return MatchByteRange(byte(start), byte(end)) } return func(t *API) bool { - r, _, err := t.Input.PeekRune(0) + r, _, err := t.peekRune(0) if err == nil && r >= start && r <= end { - t.Input.AcceptRune(r) + t.acceptRune(r) return true } return false @@ -471,18 +471,18 @@ func MatchRuneRange(start rune, end rune) Handler { // a DOS-style newline (CRLF, \r\n) or a UNIX-style newline (just a LF, \n). func MatchNewline() Handler { return func(t *API) bool { - b1, err := t.Input.PeekByte(0) + b1, err := t.peekByte(0) if err != nil { return false } if b1 == '\n' { - t.Input.AcceptBytes(b1) + t.acceptBytes(b1) return true } if b1 == '\r' { - b2, err := t.Input.PeekByte(1) + b2, err := t.peekByte(1) if err == nil && b2 == '\n' { - t.Input.AcceptBytes(b1, b2) + t.acceptBytes(b1, b2) return true } } @@ -497,9 +497,9 @@ func MatchNewline() Handler { // newlines, then take a look at MatchWhitespace(). func MatchBlank() Handler { return func(t *API) bool { - b, err := t.Input.PeekByte(0) + b, err := t.peekByte(0) if err == nil && (b == ' ' || b == '\t') { - t.Input.AcceptByte(b) + t.acceptByte(b) return true } return false @@ -516,20 +516,20 @@ func MatchBlank() Handler { func MatchBlanks() Handler { return func(t *API) bool { // Match the first blank. - b, err := t.Input.PeekByte(0) + b, err := t.peekByte(0) if err != nil || (b != ' ' && b != '\t') { return false } - t.Input.AcceptByte(b) + t.acceptByte(b) // Now match any number of followup blanks. We've already got // a successful match at this point, so we'll always return true at the end. for { - b, err := t.Input.PeekByte(0) + b, err := t.peekByte(0) if err != nil || (b != ' ' && b != '\t') { return true } - t.Input.AcceptByte(b) + t.acceptByte(b) } } } @@ -540,35 +540,35 @@ func MatchBlanks() Handler { func MatchWhitespace() Handler { return func(t *API) bool { // Match the first whitespace. - b1, err := t.Input.PeekByte(0) + b1, err := t.peekByte(0) if err != nil || (b1 != ' ' && b1 != '\t' && b1 != '\n' && b1 != '\r') { return false } if b1 == '\r' { - b2, err := t.Input.PeekByte(1) + b2, err := t.peekByte(1) if err != nil || b2 != '\n' { return false } - t.Input.AcceptBytes(b1, b2) + t.acceptBytes(b1, b2) } else { - t.Input.AcceptByte(b1) + t.acceptByte(b1) } // Now match any number of followup whitespace. We've already got // a successful match at this point, so we'll always return true at the end. for { - b1, err := t.Input.PeekByte(0) + b1, err := t.peekByte(0) if err != nil || (b1 != ' ' && b1 != '\t' && b1 != '\n' && b1 != '\r') { return true } if b1 == '\r' { - b2, err := t.Input.PeekByte(1) + b2, err := t.peekByte(1) if err != nil || b2 != '\n' { return true } - t.Input.AcceptBytes(b1, b2) + t.acceptBytes(b1, b2) } else { - t.Input.AcceptByte(b1) + t.acceptByte(b1) } } } @@ -588,9 +588,9 @@ func MatchUnicodeSpace() Handler { // so those can be used. E.g. MatchRuneByCallback(unicode.IsLower). func MatchByteByCallback(callback func(byte) bool) Handler { return func(t *API) bool { - b, err := t.Input.PeekByte(0) + b, err := t.peekByte(0) if err == nil && callback(b) { - t.Input.AcceptByte(b) + t.acceptByte(b) return true } return false @@ -605,9 +605,9 @@ func MatchByteByCallback(callback func(byte) bool) Handler { // so those can be used. E.g. MatchRuneByCallback(unicode.IsLower). func MatchRuneByCallback(callback func(rune) bool) Handler { return func(t *API) bool { - r, _, err := t.Input.PeekRune(0) + r, _, err := t.peekRune(0) if err == nil && callback(r) { - t.Input.AcceptRune(r) + t.acceptRune(r) return true } return false @@ -617,18 +617,18 @@ func MatchRuneByCallback(callback func(rune) bool) Handler { // MatchEndOfLine creates a Handler that matches a newline ("\r\n" or "\n") or EOF. func MatchEndOfLine() Handler { return func(t *API) bool { - b1, err := t.Input.PeekByte(0) + b1, err := t.peekByte(0) if err != nil { return err == io.EOF } if b1 == '\n' { - t.Input.AcceptByte(b1) + t.acceptByte(b1) return true } if b1 == '\r' { - b2, _ := t.Input.PeekByte(1) + b2, _ := t.peekByte(1) if b2 == '\n' { - t.Input.AcceptBytes(b1, b2) + t.acceptBytes(b1, b2) return true } } @@ -644,20 +644,20 @@ func MatchStr(expected string) Handler { offset := 0 for _, e := range expectedRunes { if e <= '\x7F' { - b, err := t.Input.PeekByte(offset) + b, err := t.peekByte(offset) if err != nil || b != byte(e) { return false } offset++ } else { - r, w, err := t.Input.PeekRune(offset) + r, w, err := t.peekRune(offset) if err != nil || e != r { return false } offset += w } } - t.Input.AcceptRunes(expectedRunes...) + t.acceptRunes(expectedRunes...) return true } } @@ -673,14 +673,14 @@ func MatchStrNoCase(expected string) Handler { i := 0 for _, e := range expected { if e <= '\x7F' { - b, err := t.Input.PeekByte(width) + b, err := t.peekByte(width) if err != nil || (b != byte(e) && unicode.ToUpper(rune(b)) != unicode.ToUpper(e)) { return false } matches[i] = rune(b) width++ } else { - r, w, err := t.Input.PeekRune(width) + r, w, err := t.peekRune(width) if err != nil || (r != e && unicode.ToUpper(r) != unicode.ToUpper(e)) { return false } @@ -689,7 +689,7 @@ func MatchStrNoCase(expected string) Handler { } i++ } - t.Input.AcceptRunes(matches...) + t.acceptRunes(matches...) return true } } @@ -743,7 +743,7 @@ func MatchAny(handlers ...Handler) Handler { t.Dispose(child) return true } - t.Input.Reset() + t.Reset() } t.Dispose(child) @@ -762,9 +762,9 @@ func MatchNot(handler Handler) Handler { return false } t.Dispose(child) - r, _, err := t.Input.PeekRune(0) + r, _, err := t.peekRune(0) if err == nil { - t.Input.AcceptRune(r) + t.acceptRune(r) return true } return false @@ -941,7 +941,7 @@ func MatchNotFollowedBy(lookAhead Handler, handler Handler) Handler { func MakeInputFlusher(handler Handler) Handler { return func(t *API) bool { if handler(t) { - t.Input.Flush() + t.flushInput() return true } return false @@ -956,13 +956,13 @@ func MakeInputFlusher(handler Handler) Handler { func MatchSigned(handler Handler) Handler { return func(t *API) bool { child := t.Fork() - b, err := t.Input.PeekByte(0) + b, err := t.peekByte(0) if err != nil { t.Dispose(child) return false } if b == '-' || b == '+' { - t.Input.AcceptByte(b) + t.acceptByte(b) } if handler(t) { t.Merge(child) @@ -988,7 +988,7 @@ func MatchIntegerBetween(min int64, max int64) Handler { if !digits(t) { return false } - value, _ := strconv.ParseInt(t.Output.String(), 10, 64) + value, _ := strconv.ParseInt(t.dataAsString(), 10, 64) if value < min || value > max { return false } @@ -1002,7 +1002,7 @@ func MatchIntegerBetween(min int64, max int64) Handler { func MatchEndOfFile() Handler { return func(t *API) bool { child := t.Fork() - _, err := t.Input.PeekByte(0) + _, err := t.peekByte(0) t.Dispose(child) return err == io.EOF } @@ -1018,9 +1018,9 @@ func MatchUntilEndOfLine() Handler { // MatchAnyByte creates a Handler function that accepts any byte from the input. func MatchAnyByte() Handler { return func(t *API) bool { - b, err := t.Input.PeekByte(0) + b, err := t.peekByte(0) if err == nil { - t.Input.AcceptByte(b) + t.acceptByte(b) return true } return false @@ -1032,9 +1032,9 @@ func MatchAnyByte() Handler { // replacement rune \uFFFD (i.e. utf8.RuneError), which displays as �. func MatchAnyRune() Handler { return func(t *API) bool { - r, _, err := t.Input.PeekRune(0) + r, _, err := t.peekRune(0) if err == nil { - t.Input.AcceptRune(r) + t.acceptRune(r) return true } return false @@ -1045,9 +1045,9 @@ func MatchAnyRune() Handler { // UTF8 rune can be read from the input. func MatchValidRune() Handler { return func(t *API) bool { - r, _, err := t.Input.PeekRune(0) + r, _, err := t.peekRune(0) if err == nil && r != utf8.RuneError { - t.Input.AcceptRune(r) + t.acceptRune(r) return true } return false @@ -1058,9 +1058,9 @@ func MatchValidRune() Handler { // UTF8 rune can be read from the input. func MatchInvalidRune() Handler { return func(t *API) bool { - r, _, err := t.Input.PeekRune(0) + r, _, err := t.peekRune(0) if err == nil && r == utf8.RuneError { - t.Input.AcceptRune(r) + t.acceptRune(r) return true } return false @@ -1078,19 +1078,19 @@ func MatchDigit() Handler { func MatchDigits() Handler { return func(t *API) bool { // Check if the first character is a digit. - b, err := t.Input.PeekByte(0) + b, err := t.peekByte(0) if err != nil || b < '0' || b > '9' { return false } - t.Input.AcceptByte(b) + t.acceptByte(b) // Continue accepting bytes as long as they are digits. for { - b, err := t.Input.PeekByte(0) + b, err := t.peekByte(0) if err != nil || b < '0' || b > '9' { return true } - t.Input.AcceptByte(b) + t.acceptByte(b) } } } @@ -1109,7 +1109,7 @@ func MatchDigitNotZero() Handler { func MatchInteger(normalize bool) Handler { return func(t *API) bool { // Check if the first character is a digit. - b, err := t.Input.PeekByte(0) + b, err := t.peekByte(0) if err != nil || b < '0' || b > '9' { return false } @@ -1117,33 +1117,33 @@ func MatchInteger(normalize bool) Handler { // When normalization is requested, drop leading zeroes. if normalize && b == '0' { for { - b2, err := t.Input.PeekByte(1) + b2, err := t.peekByte(1) // The next character is a zero, skip the leading zero and check again. if err == nil && b2 == b { - t.Input.SkipByte('0') + t.skipByte('0') continue } // The next character is not a zero, nor a digit at all. // We're looking at a zero on its own here. if err != nil || b2 < '1' || b2 > '9' { - t.Input.AcceptByte('0') + t.acceptByte('0') return true } // The next character is a digit. SKip the leading zero and go with the digit. - t.Input.SkipByte('0') - t.Input.AcceptByte(b2) + t.skipByte('0') + t.acceptByte(b2) break } } // Continue accepting bytes as long as they are digits. for { - b, err := t.Input.PeekByte(0) + b, err := t.peekByte(0) if err != nil || b < '0' || b > '9' { return true } - t.Input.AcceptByte(b) + t.acceptByte(b) } } } @@ -1158,7 +1158,7 @@ func MatchInteger(normalize bool) Handler { func MatchDecimal(normalize bool) Handler { return func(t *API) bool { // Check if the first character is a digit. - b, err := t.Input.PeekByte(0) + b, err := t.peekByte(0) if err != nil || b < '0' || b > '9' { return false } @@ -1166,58 +1166,58 @@ func MatchDecimal(normalize bool) Handler { // When normalization is requested, drop leading zeroes. if normalize && b == '0' { for { - b2, err := t.Input.PeekByte(1) + b2, err := t.peekByte(1) // The next character is a zero, skip the leading zero and check again. if err == nil && b2 == b { - t.Input.SkipByte('0') + t.skipByte('0') continue } // The next character is a dot, go with the zero before the dot and // let the upcoming code handle the dot. if err == nil && b2 == '.' { - t.Input.AcceptByte('0') + t.acceptByte('0') break } // The next character is not a zero, nor a digit at all. // We're looking at a zero on its own here. if err != nil || b2 < '1' || b2 > '9' { - t.Input.AcceptByte('0') + t.acceptByte('0') return true } // The next character is a digit. SKip the leading zero and go with the digit. - t.Input.SkipByte('0') - t.Input.AcceptByte(b2) + t.skipByte('0') + t.acceptByte(b2) break } } // Continue accepting bytes as long as they are digits. for { - b, err = t.Input.PeekByte(0) + b, err = t.peekByte(0) if err != nil || b < '0' || b > '9' { break } - t.Input.AcceptBytes(b) + t.acceptBytes(b) } // No dot or no digit after a dot? Then we're done. if b != '.' { return true } - b, err = t.Input.PeekByte(1) + b, err = t.peekByte(1) if err != nil || b < '0' || b > '9' { return true } // Continue accepting bytes as long as they are digits. - t.Input.AcceptBytes('.', b) + t.acceptBytes('.', b) for { - b, err = t.Input.PeekByte(0) + b, err = t.peekByte(0) if err != nil || b < '0' || b > '9' { break } - t.Input.AcceptByte(b) + t.acceptByte(b) } return true } @@ -1232,52 +1232,52 @@ func MatchDecimal(normalize bool) Handler { // False falues: false, FALSE, False, 0, f, F func MatchBoolean() Handler { return func(t *API) bool { - b1, err := t.Input.PeekByte(0) + b1, err := t.peekByte(0) if err != nil { return false } if b1 == '1' || b1 == '0' { - t.Input.AcceptByte(b1) + t.acceptByte(b1) return true } if b1 == 't' || b1 == 'T' { - b2, err := t.Input.PeekByte(1) + b2, err := t.peekByte(1) if err != nil || (b2 != 'R' && b2 != 'r') { - t.Input.AcceptByte(b1) + t.acceptByte(b1) return true } - b3, _ := t.Input.PeekByte(2) - b4, err := t.Input.PeekByte(3) + b3, _ := t.peekByte(2) + b4, err := t.peekByte(3) if err == nil && b2 == 'r' && b3 == 'u' && b4 == 'e' { - t.Input.AcceptBytes(b1, b2, b3, b4) + t.acceptBytes(b1, b2, b3, b4) return true } if err == nil && b1 == 'T' && b2 == 'R' && b3 == 'U' && b4 == 'E' { - t.Input.AcceptBytes(b1, b2, b3, b4) + t.acceptBytes(b1, b2, b3, b4) return true } - t.Input.AcceptByte(b1) + t.acceptByte(b1) return true } if b1 == 'f' || b1 == 'F' { - b2, err := t.Input.PeekByte(1) + b2, err := t.peekByte(1) if err != nil || (b2 != 'A' && b2 != 'a') { - t.Input.AcceptByte(b1) + t.acceptByte(b1) return true } - b3, _ := t.Input.PeekByte(2) - b4, _ := t.Input.PeekByte(3) - b5, err := t.Input.PeekByte(4) + b3, _ := t.peekByte(2) + b4, _ := t.peekByte(3) + b5, err := t.peekByte(4) if err == nil && b2 == 'a' && b3 == 'l' && b4 == 's' && b5 == 'e' { - t.Input.AcceptBytes(b1, b2, b3, b4, b5) + t.acceptBytes(b1, b2, b3, b4, b5) return true } if err == nil && b1 == 'F' && b2 == 'A' && b3 == 'L' && b4 == 'S' && b5 == 'E' { - t.Input.AcceptBytes(b1, b2, b3, b4, b5) + t.acceptBytes(b1, b2, b3, b4, b5) return true } - t.Input.AcceptByte(b1) + t.acceptByte(b1) return true } return false @@ -1324,9 +1324,9 @@ func MatchUnicodeLower() Handler { // digit can be read from the input. func MatchHexDigit() Handler { return func(t *API) bool { - b, err := t.Input.PeekByte(0) + b, err := t.peekByte(0) if err == nil && ((b >= '0' && b <= '9') || (b >= 'a' && b <= 'f') || (b >= 'A' && b <= 'F')) { - t.Input.AcceptByte(b) + t.acceptByte(b) return true } return false @@ -1344,28 +1344,28 @@ func MatchHexDigit() Handler { func MatchOctet(normalize bool) Handler { return func(t *API) bool { // Digit 1 - b0, err := t.Input.PeekByte(0) + b0, err := t.peekByte(0) if err != nil || b0 < '0' || b0 > '9' { return false } // Digit 2 - b1, err := t.Input.PeekByte(1) + b1, err := t.peekByte(1) if err != nil || b1 < '0' || b1 > '9' { // Output 1-digit octet. - t.Input.AcceptByte(b0) + t.acceptByte(b0) return true } // Digit 3 - b2, err := t.Input.PeekByte(2) + b2, err := t.peekByte(2) if err != nil || b2 < '0' || b2 > '9' { // Output 2-digit octet. if normalize && b0 == '0' { - t.Input.SkipByte(b0) - t.Input.AcceptByte(b1) + t.skipByte(b0) + t.acceptByte(b1) } else { - t.Input.AcceptBytes(b0, b1) + t.acceptBytes(b0, b1) } return true } @@ -1377,15 +1377,15 @@ func MatchOctet(normalize bool) Handler { // Output 3-digit octet. if normalize && b0 == '0' { - t.Input.SkipByte(b0) + t.skipByte(b0) if b1 == '0' { - t.Input.SkipByte(b1) + t.skipByte(b1) } else { - t.Input.AcceptByte(b1) + t.acceptByte(b1) } - t.Input.AcceptByte(b2) + t.acceptByte(b2) } else { - t.Input.AcceptBytes(b0, b1, b2) + t.acceptBytes(b0, b1, b2) } return true } @@ -1427,13 +1427,13 @@ func MatchIPv4Netmask(normalize bool) Handler { } // Check if the mask is provided in canonical form (at the binary level, ones followed by zeroes). - mask := net.IPv4Mask(t.Output.TokenValue(0).(byte), t.Output.TokenValue(1).(byte), t.Output.TokenValue(2).(byte), t.Output.TokenValue(3).(byte)) + mask := net.IPv4Mask(t.tokenValue(0).(byte), t.tokenValue(1).(byte), t.tokenValue(2).(byte), t.tokenValue(3).(byte)) ones, bits := mask.Size() if ones == 0 && bits == 0 { return false } - t.Output.ClearTokens() + t.tokensClear() return true } } @@ -1462,18 +1462,18 @@ func MatchIPv4Net(normalize bool) Handler { return true } - maskToken := t.Output.Token(1) + maskToken := t.token(1) if maskToken.Type == "cidr" { - t.Output.SetString(fmt.Sprintf("%s/%d", t.Output.TokenValue(0), t.Output.TokenValue(1).(uint8))) + t.dataSetString(fmt.Sprintf("%s/%d", t.tokenValue(0), t.tokenValue(1).(uint8))) } else { - o := strings.Split(t.Output.TokenValue(1).(string), ".") + o := strings.Split(t.tokenValue(1).(string), ".") b := func(idx int) byte { i, _ := strconv.Atoi(o[idx]); return byte(i) } mask := net.IPv4Mask(b(0), b(1), b(2), b(3)) bits, _ := mask.Size() - t.Output.SetString(fmt.Sprintf("%s/%d", t.Output.TokenValue(0), bits)) + t.dataSetString(fmt.Sprintf("%s/%d", t.tokenValue(0), bits)) } - t.Output.ClearTokens() + t.tokensClear() return true } } @@ -1502,13 +1502,13 @@ func MatchIPv6(normalize bool) Handler { } // Invalid IPv6, when net.ParseIP() cannot handle it. - parsed := net.ParseIP(t.Output.String()) + parsed := net.ParseIP(t.dataAsString()) if parsed == nil { return false } if normalize { - t.Output.SetString(parsed.String()) + t.dataSetString(parsed.String()) } return true } @@ -1531,8 +1531,8 @@ func matchCIDRMask(bits int64, normalize bool) Handler { if !mask(t) { return false } - bits, _ := strconv.Atoi(t.Output.String()) - t.Output.SetString(fmt.Sprintf("%d", bits)) + bits, _ := strconv.Atoi(t.dataAsString()) + t.dataSetString(fmt.Sprintf("%d", bits)) return true } } @@ -1586,7 +1586,7 @@ func ModifyDrop(handler Handler) Handler { func ModifyDropUntilEndOfLine() Handler { return func(t *API) bool { for { - b, err := t.Input.PeekByte(0) + b, err := t.peekByte(0) if err != nil { if err == io.EOF { return true @@ -1596,7 +1596,7 @@ func ModifyDropUntilEndOfLine() Handler { if b == '\n' { return true } - t.Input.SkipByte(b) + t.skipByte(b) } } } @@ -1673,8 +1673,8 @@ func ModifyByCallback(handler Handler, modfunc func(string) string) Handler { return func(t *API) bool { child := t.Fork() if handler(t) { - s := modfunc(t.Output.String()) - t.Output.SetString(s) + s := modfunc(t.dataAsString()) + t.dataSetString(s) t.Merge(child) t.Dispose(child) return true @@ -1691,7 +1691,7 @@ func ModifyByCallback(handler Handler, modfunc func(string) string) Handler { // an 'n'-character). func MakeStrLiteralToken(toktype interface{}, handler Handler) Handler { return MakeTokenByCallback(toktype, handler, func(t *API) interface{} { - literal := t.Output.String() + literal := t.dataAsString() return literal }) } @@ -1703,7 +1703,7 @@ func MakeStrLiteralToken(toktype interface{}, handler Handler) Handler { func MakeStrInterpretedToken(toktype interface{}, handler Handler) Handler { return MakeTokenByCallback(toktype, handler, func(t *API) interface{} { // TODO ERROR HANDLING - interpreted, _ := interpretString(t.Output.String()) + interpreted, _ := interpretString(t.dataAsString()) return interpreted }) } @@ -1727,7 +1727,7 @@ func interpretString(str string) (string, error) { func MakeRuneToken(toktype interface{}, handler Handler) Handler { return MakeTokenByCallback(toktype, handler, func(t *API) interface{} { // TODO ERROR HANDLING --- not a 1 rune input - return t.Output.Rune(0) + return t.dataRune(0) }) } @@ -1737,7 +1737,7 @@ func MakeRuneToken(toktype interface{}, handler Handler) Handler { func MakeByteToken(toktype interface{}, handler Handler) Handler { return MakeTokenByCallback(toktype, handler, func(t *API) interface{} { // TODO ERROR HANDLING --- not a 1 byte input - return byte(t.Output.Rune(0)) + return byte(t.dataRune(0)) }) } @@ -1942,7 +1942,7 @@ func MakeBooleanToken(toktype interface{}, handler Handler) Handler { func makeStrconvToken(name string, toktype interface{}, handler Handler, convert func(s string) (interface{}, error)) Handler { return MakeTokenByCallback(toktype, handler, func(t *API) interface{} { - value, err := convert(t.Output.String()) + value, err := convert(t.dataAsString()) if err != nil { // TODO meh, panic feels so bad here. Maybe just turn this case into "no match"? panic(fmt.Sprintf("%s token invalid (%s)", name, err)) @@ -1973,7 +1973,7 @@ func MakeTokenByCallback(toktype interface{}, handler Handler, makeValue func(t // tokens will end up in the order "date", "year", "month", "day". When we'd add the // token to the child here, the order would have been "year", "month", "day", "date". token := Token{Type: toktype, Value: makeValue(t)} - t.Output.AddTokens(token) + t.tokensAdd(token) t.Merge(child) t.Dispose(child) @@ -1990,10 +1990,10 @@ func MakeTokenGroup(toktype interface{}, handler Handler) Handler { return func(t *API) bool { child := t.Fork() if handler(t) { - tokens := t.Output.Tokens() + tokens := t.tokens() tokensCopy := make([]Token, len(tokens)) copy(tokensCopy, tokens) - t.Output.SetTokens(Token{Type: toktype, Value: tokensCopy}) + t.tokensSet(Token{Type: toktype, Value: tokensCopy}) t.Merge(child) t.Dispose(child) return true diff --git a/tokenize/tokenize.go b/tokenize/tokenize.go index 63c75b9..18fe65f 100644 --- a/tokenize/tokenize.go +++ b/tokenize/tokenize.go @@ -43,12 +43,12 @@ func New(tokenHandler Handler) Func { ok := tokenHandler(api) if !ok { - err := fmt.Errorf("mismatch at %s", api.Input.Cursor()) + err := fmt.Errorf("mismatch at %s", api.cursor()) return nil, err } result := &Result{ - Runes: api.Output.Runes(), - Tokens: api.Output.Tokens(), + Runes: api.dataAsRunes(), + Tokens: api.tokens(), } return result, nil } diff --git a/tokenize/tokenizer_whitebox_test.go b/tokenize/tokenizer_whitebox_test.go index e26feb6..0adb12e 100644 --- a/tokenize/tokenizer_whitebox_test.go +++ b/tokenize/tokenizer_whitebox_test.go @@ -9,7 +9,7 @@ func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) { i := NewAPI("Testing") r, _, _ := i.Input.PeekRune(0) i.Input.AcceptRune(r) // T - AssertEqual(t, "T", i.Output.String(), "accepted rune in input") + AssertEqual(t, "T", i.dataAsString(), "accepted rune in input") // Fork child := i.Fork() @@ -21,14 +21,14 @@ func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) { i.Input.AcceptRune(r) // e r, _, _ = i.Input.PeekRune(0) i.Input.AcceptRune(r) // s - AssertEqual(t, "es", i.Output.String(), "result runes in fork") + AssertEqual(t, "es", i.dataAsString(), "result runes in fork") AssertEqual(t, 1, i.stackFrames[i.stackLevel-1].offset, "parent offset") AssertEqual(t, 3, i.stackFrame.offset, "child offset") // Merge fork back into parent i.Merge(child) i.Dispose(child) - AssertEqual(t, "Tes", i.Output.String(), "result runes in parent Input after Merge()") + AssertEqual(t, "Tes", i.dataAsString(), "result runes in parent Input after Merge()") AssertEqual(t, 3, i.stackFrame.offset, "parent offset") } @@ -44,17 +44,17 @@ func TestGivenForkedChildWhichAcceptedRune_AfterMerging_RuneEndsUpInParentResult f2 := i.Fork() r, _, _ = i.Input.PeekRune(0) i.Input.AcceptRune(r) // s - AssertEqual(t, "s", i.Output.String(), "f2 String()") + AssertEqual(t, "s", i.dataAsString(), "f2 String()") AssertEqual(t, 3, i.stackFrame.offset, "f2.offset A") i.Merge(f2) i.Dispose(f2) - AssertEqual(t, "es", i.Output.String(), "f1 String()") + AssertEqual(t, "es", i.dataAsString(), "f1 String()") AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A") i.Merge(f1) i.Dispose(f1) - AssertEqual(t, "Tes", i.Output.String(), "top-level API String()") + AssertEqual(t, "Tes", i.dataAsString(), "top-level API String()") AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A") } @@ -83,7 +83,7 @@ func TestFlushInput(t *testing.T) { r, _, _ = i.Input.PeekRune(0) i.Input.AcceptRune(r) // o - AssertEqual(t, "cool", i.Output.String(), "end result") + AssertEqual(t, "cool", i.dataAsString(), "end result") } func TestInputFlusherWrapper(t *testing.T) { @@ -92,19 +92,19 @@ func TestInputFlusherWrapper(t *testing.T) { api := NewAPI("abaab") runeA(api) AssertEqual(t, 1, api.stackFrame.offset, "offset after 1 read") - AssertEqual(t, "a", api.Output.String(), "runes after 1 read") + AssertEqual(t, "a", api.dataAsString(), "runes after 1 read") flushB(api) AssertEqual(t, 0, api.stackFrame.offset, "offset after 2 reads + input flush") - AssertEqual(t, "ab", api.Output.String(), "runes after 2 reads") + AssertEqual(t, "ab", api.dataAsString(), "runes after 2 reads") runeA(api) AssertEqual(t, 1, api.stackFrame.offset, "offset after 3 reads") - AssertEqual(t, "aba", api.Output.String(), "runes after 3 reads") + AssertEqual(t, "aba", api.dataAsString(), "runes after 3 reads") runeA(api) AssertEqual(t, 2, api.stackFrame.offset, "offset after 4 reads") - AssertEqual(t, "abaa", api.Output.String(), "runes after 4 reads") + AssertEqual(t, "abaa", api.dataAsString(), "runes after 4 reads") flushB(api) AssertEqual(t, 0, api.stackFrame.offset, "offset after 5 reads + input flush") - AssertEqual(t, "abaab", api.Output.String(), "runes after 5 reads") + AssertEqual(t, "abaab", api.dataAsString(), "runes after 5 reads") } func AssertEqual(t *testing.T, expected interface{}, actual interface{}, forWhat string) {