Brought back some lost performance. Doing everything via api.Input/Output causes an extra level of indirection and it does not cost that much, but we do loose performance through that route. So added private methods for the API struct, which are used internally to squeeze out a bit of extra performance.

tokenize/api.go

@@ -73,7 +73,10 @@ type API struct {
 	stackLevel   int          // the current stack level
 	stackFrame   *stackFrame  // the current stack frame
 	Input        *Input       // provides input-related functionality
+	reader       *read.Buffer // the input data reader
 	Output       *Output      // provides output-related functionality
+	outputTokens []Token      // accepted tokens
+	outputData   []byte       // accepted data
 }
 
 type stackFrame struct {
@@ -98,17 +101,11 @@ const initialByteStoreLength = 1024
 // for parsekit.read.New().
 func NewAPI(input interface{}) *API {
 	api := &API{
+		reader:      read.New(input),
 		stackFrames: make([]stackFrame, initialStackDepth),
 	}
-	api.Input = &Input{
+	api.Input = &Input{api: api}
-		api:    api,
+	api.Output = &Output{api: api}
-		reader: read.New(input),
-	}
-	api.Output = &Output{
-		api:    api,
-		data:   make([]byte, initialByteStoreLength),
-		tokens: make([]Token, initialTokenStoreLength),
-	}
 	api.stackFrame = &api.stackFrames[0]
 
 	return api
@@ -207,6 +204,25 @@ func (tokenAPI *API) Merge(stackLevel int) {
 	tokenAPI.stackFrame.err = nil
 }
 
+// Reset moves the input cursor back to the beginning for the currently active API child.
+// Aditionally, any output (bytes and tokens) that was emitted from the API child are
+// cleared as well.
+func (api *API) Reset() {
+	if api.stackLevel == 0 {
+		api.stackFrame.column = 0
+		api.stackFrame.line = 0
+		api.stackFrame.offset = 0
+	} else {
+		parent := api.stackFrames[api.stackLevel-1]
+		api.stackFrame.column = parent.column
+		api.stackFrame.line = parent.line
+		api.stackFrame.offset = parent.offset
+	}
+	api.stackFrame.bytesEnd = api.stackFrame.bytesStart
+	api.stackFrame.tokenEnd = api.stackFrame.tokenStart
+	api.stackFrame.err = nil
+}
+
 func (tokenAPI *API) Dispose(stackLevel int) {
 	if stackLevel == 0 {
 		callerPanic("Dispose", "tokenize.API.{name}(): {name}() called at {caller} "+

tokenize/api_input.go

@@ -3,40 +3,11 @@ package tokenize
 import (
 	"fmt"
 	"unicode/utf8"
-
-	"git.makaay.nl/mauricem/go-parsekit/read"
 )
 
 // Input provides input-related functionality for the tokenize API.
 type Input struct {
 	api *API
-	reader *read.Buffer // the input data reader
-}
-
-// Reset moves the input cursor back to the beginning for the currently active API child.
-// Aditionally, any output (bytes and tokens) that was emitted from the API child are
-// cleared as well.
-func (i *Input) Reset() {
-	if i.api.stackLevel == 0 {
-		i.api.stackFrame.column = 0
-		i.api.stackFrame.line = 0
-		i.api.stackFrame.offset = 0
-	} else {
-		parent := i.api.stackFrames[i.api.stackLevel-1]
-		i.api.stackFrame.column = parent.column
-		i.api.stackFrame.line = parent.line
-		i.api.stackFrame.offset = parent.offset
-	}
-	i.api.stackFrame.bytesEnd = i.api.stackFrame.bytesStart
-	i.api.stackFrame.tokenEnd = i.api.stackFrame.tokenStart
-	i.api.stackFrame.err = nil
-}
-
-func (i *Input) Cursor() string {
-	if i.api.stackFrame.line == 0 && i.api.stackFrame.column == 0 {
-		return fmt.Sprintf("start of file")
-	}
-	return fmt.Sprintf("line %d, column %d", i.api.stackFrame.line+1, i.api.stackFrame.column+1)
 }
 
 // PeekByte returns the byte at the provided byte offset.
@@ -45,7 +16,11 @@ func (i *Input) Cursor() string {
 // When an offset is requested that is beyond the length of the available input
 // data, then the error will be io.EOF.
 func (i *Input) PeekByte(offset int) (byte, error) {
-	return i.reader.ByteAt(i.api.stackFrame.offset + offset)
+	return i.api.peekByte(offset)
+}
+
+func (api *API) peekByte(offset int) (byte, error) {
+	return api.reader.ByteAt(api.stackFrame.offset + offset)
 }
 
 // SkipByte is used to skip over a single bytes that was read from the input.
@@ -59,8 +34,12 @@ func (i *Input) PeekByte(offset int) (byte, error) {
 // After the call, byte offset 0 for PeekByte() and PeekRune() will point at
 // the first byte after the skipped byte.
 func (i *Input) SkipByte(b byte) {
-	i.api.stackFrame.moveCursorByByte(b)
+	i.api.skipByte(b)
-	i.api.stackFrame.offset++
+}
+
+func (api *API) skipByte(b byte) {
+	api.stackFrame.moveCursorByByte(b)
+	api.stackFrame.offset++
 }
 
 // SkipBytes is used to skip over one or more bytes that were read from the input.
@@ -74,9 +53,13 @@ func (i *Input) SkipByte(b byte) {
 // After the call, byte offset 0 for PeekByte() and PeekRune() will point at
 // the first byte after the skipped bytes.
 func (i *Input) SkipBytes(bytes ...byte) {
+	i.api.skipBytes(bytes...)
+}
+
+func (api *API) skipBytes(bytes ...byte) {
 	for _, b := range bytes {
-		i.api.stackFrame.moveCursorByByte(b)
+		api.stackFrame.moveCursorByByte(b)
-		i.api.stackFrame.offset++
+		api.stackFrame.offset++
 	}
 }
 
@@ -92,20 +75,24 @@ func (i *Input) SkipBytes(bytes ...byte) {
 // After the call, byte offset 0 for PeekByte() and PeekRune() will point at
 // the first byte after the accepted byte.
 func (i *Input) AcceptByte(b byte) {
-	curBytesEnd := i.api.stackFrame.bytesEnd
+	i.api.acceptByte(b)
+}
+
+func (api *API) acceptByte(b byte) {
+	curBytesEnd := api.stackFrame.bytesEnd
 	maxRequiredBytes := curBytesEnd + 1
 
 	// Grow the bytes capacity when needed.
-	if cap(i.api.Output.data) < maxRequiredBytes {
+	if cap(api.outputData) < maxRequiredBytes {
 		newBytes := make([]byte, maxRequiredBytes*2)
-		copy(newBytes, i.api.Output.data)
+		copy(newBytes, api.outputData)
-		i.api.Output.data = newBytes
+		api.outputData = newBytes
 	}
 
-	i.api.Output.data[curBytesEnd] = b
+	api.outputData[curBytesEnd] = b
-	i.api.stackFrame.moveCursorByByte(b)
+	api.stackFrame.moveCursorByByte(b)
-	i.api.stackFrame.bytesEnd++
+	api.stackFrame.bytesEnd++
-	i.api.stackFrame.offset++
+	api.stackFrame.offset++
 }
 
 // AcceptBytes is used to accept one or more bytes that were read from the input.
@@ -120,22 +107,26 @@ func (i *Input) AcceptByte(b byte) {
 // After the call, byte offset 0 for PeekByte() and PeekRune() will point at
 // the first byte after the accepted bytes.
 func (i *Input) AcceptBytes(bytes ...byte) {
-	curBytesEnd := i.api.stackFrame.bytesEnd
+	i.api.acceptBytes(bytes...)
+}
+
+func (api *API) acceptBytes(bytes ...byte) {
+	curBytesEnd := api.stackFrame.bytesEnd
 	newBytesEnd := curBytesEnd + len(bytes)
 
 	// Grow the bytes capacity when needed.
-	if cap(i.api.Output.data) < newBytesEnd {
+	if cap(api.outputData) < newBytesEnd {
 		newBytes := make([]byte, newBytesEnd*2)
-		copy(newBytes, i.api.Output.data)
+		copy(newBytes, api.outputData)
-		i.api.Output.data = newBytes
+		api.outputData = newBytes
 	}
 
-	copy(i.api.Output.data[curBytesEnd:], bytes)
+	copy(api.outputData[curBytesEnd:], bytes)
 	for _, b := range bytes {
-		i.api.stackFrame.moveCursorByByte(b)
+		api.stackFrame.moveCursorByByte(b)
-		i.api.stackFrame.offset++
+		api.stackFrame.offset++
 	}
-	i.api.stackFrame.bytesEnd = newBytesEnd
+	api.stackFrame.bytesEnd = newBytesEnd
 }
 
 // PeekRune returns the UTF8 rune at the provided byte offset, including its byte width.
@@ -152,7 +143,11 @@ func (i *Input) AcceptBytes(bytes ...byte) {
 // When an offset is requested that is beyond the length of the available input
 // data, then the error will be io.EOF.
 func (i *Input) PeekRune(offset int) (rune, int, error) {
-	return i.reader.RuneAt(i.api.stackFrame.offset + offset)
+	return i.api.peekRune(offset)
+}
+
+func (api *API) peekRune(offset int) (rune, int, error) {
+	return api.reader.RuneAt(api.stackFrame.offset + offset)
 }
 
 // SkipRune is used to skip over a single rune that was read from the input.
@@ -166,8 +161,12 @@ func (i *Input) PeekRune(offset int) (rune, int, error) {
 // After the call, byte offset 0 for PeekByte() and PeekRune() will point at
 // the first byte after the skipped rune.
 func (i *Input) SkipRune(r rune) {
-	i.api.stackFrame.moveCursorByRune(r)
+	i.api.skipRune(r)
-	i.api.stackFrame.offset += utf8.RuneLen(r)
+}
+
+func (api *API) skipRune(r rune) {
+	api.stackFrame.moveCursorByRune(r)
+	api.stackFrame.offset += utf8.RuneLen(r)
 }
 
 // SkipRunes is used to skip over one or more runes that were read from the input.
@@ -181,9 +180,13 @@ func (i *Input) SkipRune(r rune) {
 // After the call, byte offset 0 for PeekByte() and PeekRune() will point at
 // the first byte after the skipped runes.
 func (i *Input) SkipRunes(runes ...rune) {
+	i.api.skipRunes(runes...)
+}
+
+func (api *API) skipRunes(runes ...rune) {
 	for _, r := range runes {
-		i.api.stackFrame.moveCursorByRune(r)
+		api.stackFrame.moveCursorByRune(r)
-		i.api.stackFrame.offset += utf8.RuneLen(r)
+		api.stackFrame.offset += utf8.RuneLen(r)
 	}
 }
 
@@ -199,20 +202,24 @@ func (i *Input) SkipRunes(runes ...rune) {
 // After the call, byte offset 0 for PeekByte() and PeekRune() will point at
 // the first byte after the accepted rune.
 func (i *Input) AcceptRune(r rune) {
-	curBytesEnd := i.api.stackFrame.bytesEnd
+	i.api.acceptRune(r)
+}
+
+func (api *API) acceptRune(r rune) {
+	curBytesEnd := api.stackFrame.bytesEnd
 	maxRequiredBytes := curBytesEnd + utf8.UTFMax
 
 	// Grow the runes capacity when needed.
-	if cap(i.api.Output.data) < maxRequiredBytes {
+	if cap(api.outputData) < maxRequiredBytes {
 		newBytes := make([]byte, maxRequiredBytes*2)
-		copy(newBytes, i.api.Output.data)
+		copy(newBytes, api.outputData)
-		i.api.Output.data = newBytes
+		api.outputData = newBytes
 	}
 
-	i.api.stackFrame.moveCursorByRune(r)
+	api.stackFrame.moveCursorByRune(r)
-	w := utf8.EncodeRune(i.api.Output.data[curBytesEnd:], r)
+	w := utf8.EncodeRune(api.outputData[curBytesEnd:], r)
-	i.api.stackFrame.bytesEnd += w
+	api.stackFrame.bytesEnd += w
-	i.api.stackFrame.offset += w
+	api.stackFrame.offset += w
 }
 
 // AcceptRunes is used to accept one or more runes that were read from the input.
@@ -227,25 +234,29 @@ func (i *Input) AcceptRune(r rune) {
 // After the call, byte offset 0 for PeekByte() and PeekRune() will point at
 // the first byte after the accepted runes.
 func (i *Input) AcceptRunes(runes ...rune) {
+	i.api.acceptRunes(runes...)
+}
+
+func (api *API) acceptRunes(runes ...rune) {
 	runesAsString := string(runes)
 	byteLen := len(runesAsString)
-	curBytesEnd := i.api.stackFrame.bytesEnd
+	curBytesEnd := api.stackFrame.bytesEnd
 	newBytesEnd := curBytesEnd + byteLen
 
 	// Grow the runes capacity when needed.
-	if cap(i.api.Output.data) < newBytesEnd {
+	if cap(api.outputData) < newBytesEnd {
 		newBytes := make([]byte, newBytesEnd*2)
-		copy(newBytes, i.api.Output.data)
+		copy(newBytes, api.outputData)
-		i.api.Output.data = newBytes
+		api.outputData = newBytes
 	}
 
 	for _, r := range runes {
-		i.api.stackFrame.moveCursorByRune(r)
+		api.stackFrame.moveCursorByRune(r)
 	}
-	copy(i.api.Output.data[curBytesEnd:], runesAsString)
+	copy(api.outputData[curBytesEnd:], runesAsString)
 
-	i.api.stackFrame.bytesEnd = newBytesEnd
+	api.stackFrame.bytesEnd = newBytesEnd
-	i.api.stackFrame.offset += byteLen
+	api.stackFrame.offset += byteLen
 }
 
 // Flush flushes input data from the read.Buffer up to the current
@@ -255,10 +266,25 @@ func (i *Input) AcceptRunes(runes ...rune) {
 // When writing your own TokenHandler, you normally won't have to call this
 // method yourself. It is automatically called by parsekit when possible.
 func (i *Input) Flush() bool {
-	if i.api.stackFrame.offset > 0 {
+	return i.api.flushInput()
-		i.reader.Flush(i.api.stackFrame.offset)
+}
-		i.api.stackFrame.offset = 0
+
+func (api *API) flushInput() bool {
+	if api.stackFrame.offset > 0 {
+		api.reader.Flush(api.stackFrame.offset)
+		api.stackFrame.offset = 0
 		return true
 	}
 	return false
 }
+
+func (i *Input) Cursor() string {
+	return i.api.cursor()
+}
+
+func (api *API) cursor() string {
+	if api.stackFrame.line == 0 && api.stackFrame.column == 0 {
+		return fmt.Sprintf("start of file")
+	}
+	return fmt.Sprintf("line %d, column %d", api.stackFrame.line+1, api.stackFrame.column+1)
+}

tokenize/api_output.go

@@ -7,106 +7,167 @@ import (
 // Output provides output-related functionality for the tokenize API.
 type Output struct {
 	api *API
-	tokens []Token // accepted tokens
-	data   []byte  // accepted data
 }
 
 func (o *Output) String() string {
-	bytes := o.data[o.api.stackFrame.bytesStart:o.api.stackFrame.bytesEnd]
+	return o.api.dataAsString()
+}
+func (api *API) dataAsString() string {
+	bytes := api.outputData[api.stackFrame.bytesStart:api.stackFrame.bytesEnd]
 	return string(bytes)
 }
 
 func (o *Output) Runes() []rune {
-	bytes := o.data[o.api.stackFrame.bytesStart:o.api.stackFrame.bytesEnd]
+	return o.api.dataAsRunes()
+}
+
+func (api *API) dataAsRunes() []rune {
+	bytes := api.outputData[api.stackFrame.bytesStart:api.stackFrame.bytesEnd]
 	return []rune(string(bytes))
 }
 
 func (o *Output) Rune(offset int) rune {
-	r, _ := utf8.DecodeRune(o.data[o.api.stackFrame.bytesStart+offset:])
+	return o.api.dataRune(offset)
+}
+
+func (api *API) dataRune(offset int) rune {
+	r, _ := utf8.DecodeRune(api.outputData[api.stackFrame.bytesStart+offset:])
 	return r
 }
 
 func (o *Output) ClearData() {
-	o.api.stackFrame.bytesEnd = o.api.stackFrame.bytesStart
+	o.api.dataClear()
+}
+
+func (api *API) dataClear() {
+	api.stackFrame.bytesEnd = api.stackFrame.bytesStart
 }
 
 func (o *Output) SetBytes(bytes ...byte) {
-	o.ClearData()
+	o.api.dataSetBytes(bytes...)
-	o.AddBytes(bytes...)
+}
+
+func (api *API) dataSetBytes(bytes ...byte) {
+	api.dataClear()
+	api.dataAddBytes(bytes...)
 }
 
 func (o *Output) AddBytes(bytes ...byte) {
+	o.api.dataAddBytes(bytes...)
+}
+
+func (api *API) dataAddBytes(bytes ...byte) {
 	// Grow the runes capacity when needed.
-	newBytesEnd := o.api.stackFrame.bytesEnd + len(bytes)
+	newBytesEnd := api.stackFrame.bytesEnd + len(bytes)
-	if cap(o.data) < newBytesEnd {
+	if cap(api.outputData) < newBytesEnd {
 		newBytes := make([]byte, newBytesEnd*2)
-		copy(newBytes, o.data)
+		copy(newBytes, api.outputData)
-		o.data = newBytes
+		api.outputData = newBytes
 	}
 
-	copy(o.data[o.api.stackFrame.bytesEnd:], bytes)
+	copy(api.outputData[api.stackFrame.bytesEnd:], bytes)
-	o.api.stackFrame.bytesEnd = newBytesEnd
+	api.stackFrame.bytesEnd = newBytesEnd
 }
 
-func (o Output) SetRunes(runes ...rune) {
+func (o *Output) SetRunes(runes ...rune) {
-	o.ClearData()
+	o.api.dataSetRunes(runes...)
-	o.AddRunes(runes...)
+}
+
+func (api *API) dataSetRunes(runes ...rune) {
+	api.dataClear()
+	api.dataAddRunes(runes...)
 }
 
 func (o *Output) AddRunes(runes ...rune) {
-	// Grow the runes capacity when needed.
+	o.api.dataAddRunes(runes...)
-	runesAsString := string(runes)
-	newBytesEnd := o.api.stackFrame.bytesEnd + len(runesAsString)
-	if cap(o.data) < newBytesEnd {
-		newBytes := make([]byte, newBytesEnd*2)
-		copy(newBytes, o.data)
-		o.data = newBytes
 }
 
-	copy(o.data[o.api.stackFrame.bytesEnd:], runesAsString)
+func (api *API) dataAddRunes(runes ...rune) {
-	o.api.stackFrame.bytesEnd = newBytesEnd
+	// Grow the runes capacity when needed.
+	runesAsString := string(runes)
+	newBytesEnd := api.stackFrame.bytesEnd + len(runesAsString)
+	if cap(api.outputData) < newBytesEnd {
+		newBytes := make([]byte, newBytesEnd*2)
+		copy(newBytes, api.outputData)
+		api.outputData = newBytes
+	}
+
+	copy(api.outputData[api.stackFrame.bytesEnd:], runesAsString)
+	api.stackFrame.bytesEnd = newBytesEnd
 }
 
 func (o *Output) AddString(s string) {
-	o.AddBytes([]byte(s)...)
+	o.api.dataAddString(s)
+}
+
+func (api *API) dataAddString(s string) {
+	api.dataAddBytes([]byte(s)...)
 }
 
 func (o *Output) SetString(s string) {
-	o.ClearData()
+	o.api.dataSetString(s)
-	o.SetBytes([]byte(s)...)
+}
+
+func (api *API) dataSetString(s string) {
+	api.dataClear()
+	api.dataSetBytes([]byte(s)...)
 }
 
 func (o *Output) Tokens() []Token {
-	return o.tokens[o.api.stackFrame.tokenStart:o.api.stackFrame.tokenEnd]
+	return o.api.tokens()
+}
+
+func (api *API) tokens() []Token {
+	return api.outputTokens[api.stackFrame.tokenStart:api.stackFrame.tokenEnd]
 }
 
 func (o *Output) Token(offset int) Token {
-	return o.tokens[o.api.stackFrame.tokenStart+offset]
+	return o.api.token(offset)
+}
+
+func (api *API) token(offset int) Token {
+	return api.outputTokens[api.stackFrame.tokenStart+offset]
 }
 
 func (o *Output) TokenValue(offset int) interface{} {
-	return o.tokens[o.api.stackFrame.tokenStart+offset].Value
+	return o.api.tokenValue(offset)
+}
+
+func (api *API) tokenValue(offset int) interface{} {
+	return api.outputTokens[api.stackFrame.tokenStart+offset].Value
 }
 
 func (o *Output) ClearTokens() {
-	o.api.stackFrame.tokenEnd = o.api.stackFrame.tokenStart
+	o.api.tokensClear()
+}
+
+func (api *API) tokensClear() {
+	api.stackFrame.tokenEnd = api.stackFrame.tokenStart
 }
 
 func (o *Output) SetTokens(tokens ...Token) {
-	o.ClearTokens()
+	o.api.tokensSet(tokens...)
-	o.AddTokens(tokens...)
+}
+
+func (api *API) tokensSet(tokens ...Token) {
+	api.tokensClear()
+	api.tokensAdd(tokens...)
 }
 
 func (o *Output) AddTokens(tokens ...Token) {
+	o.api.tokensAdd(tokens...)
+}
+
+func (api *API) tokensAdd(tokens ...Token) {
 	// Grow the tokens capacity when needed.
-	newTokenEnd := o.api.stackFrame.tokenEnd + len(tokens)
+	newTokenEnd := api.stackFrame.tokenEnd + len(tokens)
-	if cap(o.tokens) < newTokenEnd {
+	if cap(api.outputTokens) < newTokenEnd {
 		newTokens := make([]Token, newTokenEnd*2)
-		copy(newTokens, o.tokens)
+		copy(newTokens, api.outputTokens)
-		o.tokens = newTokens
+		api.outputTokens = newTokens
 	}
 	for offset, t := range tokens {
-		o.tokens[o.api.stackFrame.tokenEnd+offset] = t
+		api.outputTokens[api.stackFrame.tokenEnd+offset] = t
 	}
-	o.api.stackFrame.tokenEnd = newTokenEnd
+	api.stackFrame.tokenEnd = newTokenEnd
 }

tokenize/api_test.go

@@ -19,7 +19,7 @@ func ExampleNewAPI() {
 // 	r, err := api.NextRune()
 // 	fmt.Printf("Rune read from input; %c\n", r)
 // 	fmt.Printf("The error: %v\n", err)
-// 	fmt.Printf("API results: %q\n", api.Output.String())
+// 	fmt.Printf("API results: %q\n", api.dataAsString())
 
 // 	// Output:
 // 	// Rune read from input; T
@@ -152,7 +152,7 @@ func ExampleAPI_Reset() {
 	fmt.Printf("API results: %q at %s\n", api.Output.String(), api.Input.Cursor())
 
 	// Reset clears the results.
-	api.Input.Reset()
+	api.Reset()
 	fmt.Printf("API results: %q at %s\n", api.Output.String(), api.Input.Cursor())
 
 	// So then doing the same read operations, the same data are read.
@@ -374,7 +374,7 @@ func TestMergeScenariosForTokens(t *testing.T) {
 
 	child = api.Fork()
 	api.Output.AddTokens(token3)
-	api.Input.Reset()
+	api.Reset()
 	api.Output.AddTokens(token4)
 
 	api.Merge(child)

tokenize/handlers_builtin.go

@@ -350,9 +350,9 @@ var T = struct {
 // MatchByte creates a Handler function that matches against the provided byte.
 func MatchByte(expected byte) Handler {
 	return func(t *API) bool {
-		b, err := t.Input.PeekByte(0)
+		b, err := t.peekByte(0)
 		if err == nil && b == expected {
-			t.Input.AcceptByte(b)
+			t.acceptByte(b)
 			return true
 		}
 		return false
@@ -365,9 +365,9 @@ func MatchRune(expected rune) Handler {
 		return MatchByte(byte(expected))
 	}
 	return func(t *API) bool {
-		r, _, err := t.Input.PeekRune(0)
+		r, _, err := t.peekRune(0)
 		if err == nil && r == expected {
-			t.Input.AcceptRune(r)
+			t.acceptRune(r)
 			return true
 		}
 		return false
@@ -378,13 +378,13 @@ func MatchRune(expected rune) Handler {
 // one of the provided bytes. The first match counts.
 func MatchBytes(expected ...byte) Handler {
 	return func(t *API) bool {
-		b, err := t.Input.PeekByte(0)
+		b, err := t.peekByte(0)
 		if err != nil {
 			return false
 		}
 		for _, e := range expected {
 			if b == e {
-				t.Input.AcceptByte(b)
+				t.acceptByte(b)
 				return true
 			}
 		}
@@ -408,13 +408,13 @@ func MatchRunes(expected ...rune) Handler {
 		return MatchBytes(expectedBytes...)
 	}
 	return func(t *API) bool {
-		r, _, err := t.Input.PeekRune(0)
+		r, _, err := t.peekRune(0)
 		if err != nil {
 			return false
 		}
 		for _, e := range expected {
 			if r == e {
-				t.Input.AcceptRune(r)
+				t.acceptRune(r)
 				return true
 			}
 		}
@@ -434,9 +434,9 @@ func MatchByteRange(start byte, end byte) Handler {
 		callerPanic("MatchByteRange", "Handler: {name} definition error at {caller}: start %q must not be < end %q", start, end)
 	}
 	return func(t *API) bool {
-		r, err := t.Input.PeekByte(0)
+		r, err := t.peekByte(0)
 		if err == nil && r >= start && r <= end {
-			t.Input.AcceptByte(r)
+			t.acceptByte(r)
 			return true
 		}
 		return false
@@ -458,9 +458,9 @@ func MatchRuneRange(start rune, end rune) Handler {
 		return MatchByteRange(byte(start), byte(end))
 	}
 	return func(t *API) bool {
-		r, _, err := t.Input.PeekRune(0)
+		r, _, err := t.peekRune(0)
 		if err == nil && r >= start && r <= end {
-			t.Input.AcceptRune(r)
+			t.acceptRune(r)
 			return true
 		}
 		return false
@@ -471,18 +471,18 @@ func MatchRuneRange(start rune, end rune) Handler {
 // a DOS-style newline (CRLF, \r\n) or a UNIX-style newline (just a LF, \n).
 func MatchNewline() Handler {
 	return func(t *API) bool {
-		b1, err := t.Input.PeekByte(0)
+		b1, err := t.peekByte(0)
 		if err != nil {
 			return false
 		}
 		if b1 == '\n' {
-			t.Input.AcceptBytes(b1)
+			t.acceptBytes(b1)
 			return true
 		}
 		if b1 == '\r' {
-			b2, err := t.Input.PeekByte(1)
+			b2, err := t.peekByte(1)
 			if err == nil && b2 == '\n' {
-				t.Input.AcceptBytes(b1, b2)
+				t.acceptBytes(b1, b2)
 				return true
 			}
 		}
@@ -497,9 +497,9 @@ func MatchNewline() Handler {
 // newlines, then take a look at MatchWhitespace().
 func MatchBlank() Handler {
 	return func(t *API) bool {
-		b, err := t.Input.PeekByte(0)
+		b, err := t.peekByte(0)
 		if err == nil && (b == ' ' || b == '\t') {
-			t.Input.AcceptByte(b)
+			t.acceptByte(b)
 			return true
 		}
 		return false
@@ -516,20 +516,20 @@ func MatchBlank() Handler {
 func MatchBlanks() Handler {
 	return func(t *API) bool {
 		// Match the first blank.
-		b, err := t.Input.PeekByte(0)
+		b, err := t.peekByte(0)
 		if err != nil || (b != ' ' && b != '\t') {
 			return false
 		}
-		t.Input.AcceptByte(b)
+		t.acceptByte(b)
 
 		// Now match any number of followup blanks. We've already got
 		// a successful match at this point, so we'll always return true at the end.
 		for {
-			b, err := t.Input.PeekByte(0)
+			b, err := t.peekByte(0)
 			if err != nil || (b != ' ' && b != '\t') {
 				return true
 			}
-			t.Input.AcceptByte(b)
+			t.acceptByte(b)
 		}
 	}
 }
@@ -540,35 +540,35 @@ func MatchBlanks() Handler {
 func MatchWhitespace() Handler {
 	return func(t *API) bool {
 		// Match the first whitespace.
-		b1, err := t.Input.PeekByte(0)
+		b1, err := t.peekByte(0)
 		if err != nil || (b1 != ' ' && b1 != '\t' && b1 != '\n' && b1 != '\r') {
 			return false
 		}
 		if b1 == '\r' {
-			b2, err := t.Input.PeekByte(1)
+			b2, err := t.peekByte(1)
 			if err != nil || b2 != '\n' {
 				return false
 			}
-			t.Input.AcceptBytes(b1, b2)
+			t.acceptBytes(b1, b2)
 		} else {
-			t.Input.AcceptByte(b1)
+			t.acceptByte(b1)
 		}
 
 		// Now match any number of followup whitespace. We've already got
 		// a successful match at this point, so we'll always return true at the end.
 		for {
-			b1, err := t.Input.PeekByte(0)
+			b1, err := t.peekByte(0)
 			if err != nil || (b1 != ' ' && b1 != '\t' && b1 != '\n' && b1 != '\r') {
 				return true
 			}
 			if b1 == '\r' {
-				b2, err := t.Input.PeekByte(1)
+				b2, err := t.peekByte(1)
 				if err != nil || b2 != '\n' {
 					return true
 				}
-				t.Input.AcceptBytes(b1, b2)
+				t.acceptBytes(b1, b2)
 			} else {
-				t.Input.AcceptByte(b1)
+				t.acceptByte(b1)
 			}
 		}
 	}
@@ -588,9 +588,9 @@ func MatchUnicodeSpace() Handler {
 // so those can be used. E.g. MatchRuneByCallback(unicode.IsLower).
 func MatchByteByCallback(callback func(byte) bool) Handler {
 	return func(t *API) bool {
-		b, err := t.Input.PeekByte(0)
+		b, err := t.peekByte(0)
 		if err == nil && callback(b) {
-			t.Input.AcceptByte(b)
+			t.acceptByte(b)
 			return true
 		}
 		return false
@@ -605,9 +605,9 @@ func MatchByteByCallback(callback func(byte) bool) Handler {
 // so those can be used. E.g. MatchRuneByCallback(unicode.IsLower).
 func MatchRuneByCallback(callback func(rune) bool) Handler {
 	return func(t *API) bool {
-		r, _, err := t.Input.PeekRune(0)
+		r, _, err := t.peekRune(0)
 		if err == nil && callback(r) {
-			t.Input.AcceptRune(r)
+			t.acceptRune(r)
 			return true
 		}
 		return false
@@ -617,18 +617,18 @@ func MatchRuneByCallback(callback func(rune) bool) Handler {
 // MatchEndOfLine creates a Handler that matches a newline ("\r\n" or "\n") or EOF.
 func MatchEndOfLine() Handler {
 	return func(t *API) bool {
-		b1, err := t.Input.PeekByte(0)
+		b1, err := t.peekByte(0)
 		if err != nil {
 			return err == io.EOF
 		}
 		if b1 == '\n' {
-			t.Input.AcceptByte(b1)
+			t.acceptByte(b1)
 			return true
 		}
 		if b1 == '\r' {
-			b2, _ := t.Input.PeekByte(1)
+			b2, _ := t.peekByte(1)
 			if b2 == '\n' {
-				t.Input.AcceptBytes(b1, b2)
+				t.acceptBytes(b1, b2)
 				return true
 			}
 		}
@@ -644,20 +644,20 @@ func MatchStr(expected string) Handler {
 		offset := 0
 		for _, e := range expectedRunes {
 			if e <= '\x7F' {
-				b, err := t.Input.PeekByte(offset)
+				b, err := t.peekByte(offset)
 				if err != nil || b != byte(e) {
 					return false
 				}
 				offset++
 			} else {
-				r, w, err := t.Input.PeekRune(offset)
+				r, w, err := t.peekRune(offset)
 				if err != nil || e != r {
 					return false
 				}
 				offset += w
 			}
 		}
-		t.Input.AcceptRunes(expectedRunes...)
+		t.acceptRunes(expectedRunes...)
 		return true
 	}
 }
@@ -673,14 +673,14 @@ func MatchStrNoCase(expected string) Handler {
 		i := 0
 		for _, e := range expected {
 			if e <= '\x7F' {
-				b, err := t.Input.PeekByte(width)
+				b, err := t.peekByte(width)
 				if err != nil || (b != byte(e) && unicode.ToUpper(rune(b)) != unicode.ToUpper(e)) {
 					return false
 				}
 				matches[i] = rune(b)
 				width++
 			} else {
-				r, w, err := t.Input.PeekRune(width)
+				r, w, err := t.peekRune(width)
 				if err != nil || (r != e && unicode.ToUpper(r) != unicode.ToUpper(e)) {
 					return false
 				}
@@ -689,7 +689,7 @@ func MatchStrNoCase(expected string) Handler {
 			}
 			i++
 		}
-		t.Input.AcceptRunes(matches...)
+		t.acceptRunes(matches...)
 		return true
 	}
 }
@@ -743,7 +743,7 @@ func MatchAny(handlers ...Handler) Handler {
 				t.Dispose(child)
 				return true
 			}
-			t.Input.Reset()
+			t.Reset()
 		}
 		t.Dispose(child)
 
@@ -762,9 +762,9 @@ func MatchNot(handler Handler) Handler {
 			return false
 		}
 		t.Dispose(child)
-		r, _, err := t.Input.PeekRune(0)
+		r, _, err := t.peekRune(0)
 		if err == nil {
-			t.Input.AcceptRune(r)
+			t.acceptRune(r)
 			return true
 		}
 		return false
@@ -941,7 +941,7 @@ func MatchNotFollowedBy(lookAhead Handler, handler Handler) Handler {
 func MakeInputFlusher(handler Handler) Handler {
 	return func(t *API) bool {
 		if handler(t) {
-			t.Input.Flush()
+			t.flushInput()
 			return true
 		}
 		return false
@@ -956,13 +956,13 @@ func MakeInputFlusher(handler Handler) Handler {
 func MatchSigned(handler Handler) Handler {
 	return func(t *API) bool {
 		child := t.Fork()
-		b, err := t.Input.PeekByte(0)
+		b, err := t.peekByte(0)
 		if err != nil {
 			t.Dispose(child)
 			return false
 		}
 		if b == '-' || b == '+' {
-			t.Input.AcceptByte(b)
+			t.acceptByte(b)
 		}
 		if handler(t) {
 			t.Merge(child)
@@ -988,7 +988,7 @@ func MatchIntegerBetween(min int64, max int64) Handler {
 		if !digits(t) {
 			return false
 		}
-		value, _ := strconv.ParseInt(t.Output.String(), 10, 64)
+		value, _ := strconv.ParseInt(t.dataAsString(), 10, 64)
 		if value < min || value > max {
 			return false
 		}
@@ -1002,7 +1002,7 @@ func MatchIntegerBetween(min int64, max int64) Handler {
 func MatchEndOfFile() Handler {
 	return func(t *API) bool {
 		child := t.Fork()
-		_, err := t.Input.PeekByte(0)
+		_, err := t.peekByte(0)
 		t.Dispose(child)
 		return err == io.EOF
 	}
@@ -1018,9 +1018,9 @@ func MatchUntilEndOfLine() Handler {
 // MatchAnyByte creates a Handler function that accepts any byte from the input.
 func MatchAnyByte() Handler {
 	return func(t *API) bool {
-		b, err := t.Input.PeekByte(0)
+		b, err := t.peekByte(0)
 		if err == nil {
-			t.Input.AcceptByte(b)
+			t.acceptByte(b)
 			return true
 		}
 		return false
@@ -1032,9 +1032,9 @@ func MatchAnyByte() Handler {
 // replacement rune \uFFFD (i.e. utf8.RuneError), which displays as <20>.
 func MatchAnyRune() Handler {
 	return func(t *API) bool {
-		r, _, err := t.Input.PeekRune(0)
+		r, _, err := t.peekRune(0)
 		if err == nil {
-			t.Input.AcceptRune(r)
+			t.acceptRune(r)
 			return true
 		}
 		return false
@@ -1045,9 +1045,9 @@ func MatchAnyRune() Handler {
 // UTF8 rune can be read from the input.
 func MatchValidRune() Handler {
 	return func(t *API) bool {
-		r, _, err := t.Input.PeekRune(0)
+		r, _, err := t.peekRune(0)
 		if err == nil && r != utf8.RuneError {
-			t.Input.AcceptRune(r)
+			t.acceptRune(r)
 			return true
 		}
 		return false
@@ -1058,9 +1058,9 @@ func MatchValidRune() Handler {
 // UTF8 rune can be read from the input.
 func MatchInvalidRune() Handler {
 	return func(t *API) bool {
-		r, _, err := t.Input.PeekRune(0)
+		r, _, err := t.peekRune(0)
 		if err == nil && r == utf8.RuneError {
-			t.Input.AcceptRune(r)
+			t.acceptRune(r)
 			return true
 		}
 		return false
@@ -1078,19 +1078,19 @@ func MatchDigit() Handler {
 func MatchDigits() Handler {
 	return func(t *API) bool {
 		// Check if the first character is a digit.
-		b, err := t.Input.PeekByte(0)
+		b, err := t.peekByte(0)
 		if err != nil || b < '0' || b > '9' {
 			return false
 		}
-		t.Input.AcceptByte(b)
+		t.acceptByte(b)
 
 		// Continue accepting bytes as long as they are digits.
 		for {
-			b, err := t.Input.PeekByte(0)
+			b, err := t.peekByte(0)
 			if err != nil || b < '0' || b > '9' {
 				return true
 			}
-			t.Input.AcceptByte(b)
+			t.acceptByte(b)
 		}
 	}
 }
@@ -1109,7 +1109,7 @@ func MatchDigitNotZero() Handler {
 func MatchInteger(normalize bool) Handler {
 	return func(t *API) bool {
 		// Check if the first character is a digit.
-		b, err := t.Input.PeekByte(0)
+		b, err := t.peekByte(0)
 		if err != nil || b < '0' || b > '9' {
 			return false
 		}
@@ -1117,33 +1117,33 @@ func MatchInteger(normalize bool) Handler {
 		// When normalization is requested, drop leading zeroes.
 		if normalize && b == '0' {
 			for {
-				b2, err := t.Input.PeekByte(1)
+				b2, err := t.peekByte(1)
 
 				// The next character is a zero, skip the leading zero and check again.
 				if err == nil && b2 == b {
-					t.Input.SkipByte('0')
+					t.skipByte('0')
 					continue
 				}
 				// The next character is not a zero, nor a digit at all.
 				// We're looking at a zero on its own here.
 				if err != nil || b2 < '1' || b2 > '9' {
-					t.Input.AcceptByte('0')
+					t.acceptByte('0')
 					return true
 				}
 				// The next character is a digit. SKip the leading zero and go with the digit.
-				t.Input.SkipByte('0')
+				t.skipByte('0')
-				t.Input.AcceptByte(b2)
+				t.acceptByte(b2)
 				break
 			}
 		}
 
 		// Continue accepting bytes as long as they are digits.
 		for {
-			b, err := t.Input.PeekByte(0)
+			b, err := t.peekByte(0)
 			if err != nil || b < '0' || b > '9' {
 				return true
 			}
-			t.Input.AcceptByte(b)
+			t.acceptByte(b)
 		}
 	}
 }
@@ -1158,7 +1158,7 @@ func MatchInteger(normalize bool) Handler {
 func MatchDecimal(normalize bool) Handler {
 	return func(t *API) bool {
 		// Check if the first character is a digit.
-		b, err := t.Input.PeekByte(0)
+		b, err := t.peekByte(0)
 		if err != nil || b < '0' || b > '9' {
 			return false
 		}
@@ -1166,58 +1166,58 @@ func MatchDecimal(normalize bool) Handler {
 		// When normalization is requested, drop leading zeroes.
 		if normalize && b == '0' {
 			for {
-				b2, err := t.Input.PeekByte(1)
+				b2, err := t.peekByte(1)
 
 				// The next character is a zero, skip the leading zero and check again.
 				if err == nil && b2 == b {
-					t.Input.SkipByte('0')
+					t.skipByte('0')
 					continue
 				}
 				// The next character is a dot, go with the zero before the dot and
 				// let the upcoming code handle the dot.
 				if err == nil && b2 == '.' {
-					t.Input.AcceptByte('0')
+					t.acceptByte('0')
 					break
 				}
 				// The next character is not a zero, nor a digit at all.
 				// We're looking at a zero on its own here.
 				if err != nil || b2 < '1' || b2 > '9' {
-					t.Input.AcceptByte('0')
+					t.acceptByte('0')
 					return true
 				}
 				// The next character is a digit. SKip the leading zero and go with the digit.
-				t.Input.SkipByte('0')
+				t.skipByte('0')
-				t.Input.AcceptByte(b2)
+				t.acceptByte(b2)
 				break
 			}
 		}
 
 		// Continue accepting bytes as long as they are digits.
 		for {
-			b, err = t.Input.PeekByte(0)
+			b, err = t.peekByte(0)
 			if err != nil || b < '0' || b > '9' {
 				break
 			}
-			t.Input.AcceptBytes(b)
+			t.acceptBytes(b)
 		}
 
 		// No dot or no digit after a dot? Then we're done.
 		if b != '.' {
 			return true
 		}
-		b, err = t.Input.PeekByte(1)
+		b, err = t.peekByte(1)
 		if err != nil || b < '0' || b > '9' {
 			return true
 		}
 
 		// Continue accepting bytes as long as they are digits.
-		t.Input.AcceptBytes('.', b)
+		t.acceptBytes('.', b)
 		for {
-			b, err = t.Input.PeekByte(0)
+			b, err = t.peekByte(0)
 			if err != nil || b < '0' || b > '9' {
 				break
 			}
-			t.Input.AcceptByte(b)
+			t.acceptByte(b)
 		}
 		return true
 	}
@@ -1232,52 +1232,52 @@ func MatchDecimal(normalize bool) Handler {
 // False falues: false, FALSE, False, 0, f, F
 func MatchBoolean() Handler {
 	return func(t *API) bool {
-		b1, err := t.Input.PeekByte(0)
+		b1, err := t.peekByte(0)
 		if err != nil {
 			return false
 		}
 		if b1 == '1' || b1 == '0' {
-			t.Input.AcceptByte(b1)
+			t.acceptByte(b1)
 			return true
 		}
 		if b1 == 't' || b1 == 'T' {
-			b2, err := t.Input.PeekByte(1)
+			b2, err := t.peekByte(1)
 			if err != nil || (b2 != 'R' && b2 != 'r') {
-				t.Input.AcceptByte(b1)
+				t.acceptByte(b1)
 				return true
 			}
-			b3, _ := t.Input.PeekByte(2)
+			b3, _ := t.peekByte(2)
-			b4, err := t.Input.PeekByte(3)
+			b4, err := t.peekByte(3)
 			if err == nil && b2 == 'r' && b3 == 'u' && b4 == 'e' {
-				t.Input.AcceptBytes(b1, b2, b3, b4)
+				t.acceptBytes(b1, b2, b3, b4)
 				return true
 			}
 			if err == nil && b1 == 'T' && b2 == 'R' && b3 == 'U' && b4 == 'E' {
-				t.Input.AcceptBytes(b1, b2, b3, b4)
+				t.acceptBytes(b1, b2, b3, b4)
 				return true
 			}
-			t.Input.AcceptByte(b1)
+			t.acceptByte(b1)
 			return true
 		}
 
 		if b1 == 'f' || b1 == 'F' {
-			b2, err := t.Input.PeekByte(1)
+			b2, err := t.peekByte(1)
 			if err != nil || (b2 != 'A' && b2 != 'a') {
-				t.Input.AcceptByte(b1)
+				t.acceptByte(b1)
 				return true
 			}
-			b3, _ := t.Input.PeekByte(2)
+			b3, _ := t.peekByte(2)
-			b4, _ := t.Input.PeekByte(3)
+			b4, _ := t.peekByte(3)
-			b5, err := t.Input.PeekByte(4)
+			b5, err := t.peekByte(4)
 			if err == nil && b2 == 'a' && b3 == 'l' && b4 == 's' && b5 == 'e' {
-				t.Input.AcceptBytes(b1, b2, b3, b4, b5)
+				t.acceptBytes(b1, b2, b3, b4, b5)
 				return true
 			}
 			if err == nil && b1 == 'F' && b2 == 'A' && b3 == 'L' && b4 == 'S' && b5 == 'E' {
-				t.Input.AcceptBytes(b1, b2, b3, b4, b5)
+				t.acceptBytes(b1, b2, b3, b4, b5)
 				return true
 			}
-			t.Input.AcceptByte(b1)
+			t.acceptByte(b1)
 			return true
 		}
 		return false
@@ -1324,9 +1324,9 @@ func MatchUnicodeLower() Handler {
 // digit can be read from the input.
 func MatchHexDigit() Handler {
 	return func(t *API) bool {
-		b, err := t.Input.PeekByte(0)
+		b, err := t.peekByte(0)
 		if err == nil && ((b >= '0' && b <= '9') || (b >= 'a' && b <= 'f') || (b >= 'A' && b <= 'F')) {
-			t.Input.AcceptByte(b)
+			t.acceptByte(b)
 			return true
 		}
 		return false
@@ -1344,28 +1344,28 @@ func MatchHexDigit() Handler {
 func MatchOctet(normalize bool) Handler {
 	return func(t *API) bool {
 		// Digit 1
-		b0, err := t.Input.PeekByte(0)
+		b0, err := t.peekByte(0)
 		if err != nil || b0 < '0' || b0 > '9' {
 			return false
 		}
 
 		// Digit 2
-		b1, err := t.Input.PeekByte(1)
+		b1, err := t.peekByte(1)
 		if err != nil || b1 < '0' || b1 > '9' {
 			// Output 1-digit octet.
-			t.Input.AcceptByte(b0)
+			t.acceptByte(b0)
 			return true
 		}
 
 		// Digit 3
-		b2, err := t.Input.PeekByte(2)
+		b2, err := t.peekByte(2)
 		if err != nil || b2 < '0' || b2 > '9' {
 			// Output 2-digit octet.
 			if normalize && b0 == '0' {
-				t.Input.SkipByte(b0)
+				t.skipByte(b0)
-				t.Input.AcceptByte(b1)
+				t.acceptByte(b1)
 			} else {
-				t.Input.AcceptBytes(b0, b1)
+				t.acceptBytes(b0, b1)
 			}
 			return true
 		}
@@ -1377,15 +1377,15 @@ func MatchOctet(normalize bool) Handler {
 
 		// Output 3-digit octet.
 		if normalize && b0 == '0' {
-			t.Input.SkipByte(b0)
+			t.skipByte(b0)
 			if b1 == '0' {
-				t.Input.SkipByte(b1)
+				t.skipByte(b1)
 			} else {
-				t.Input.AcceptByte(b1)
+				t.acceptByte(b1)
 			}
-			t.Input.AcceptByte(b2)
+			t.acceptByte(b2)
 		} else {
-			t.Input.AcceptBytes(b0, b1, b2)
+			t.acceptBytes(b0, b1, b2)
 		}
 		return true
 	}
@@ -1427,13 +1427,13 @@ func MatchIPv4Netmask(normalize bool) Handler {
 		}
 
 		// Check if the mask is provided in canonical form (at the binary level, ones followed by zeroes).
-		mask := net.IPv4Mask(t.Output.TokenValue(0).(byte), t.Output.TokenValue(1).(byte), t.Output.TokenValue(2).(byte), t.Output.TokenValue(3).(byte))
+		mask := net.IPv4Mask(t.tokenValue(0).(byte), t.tokenValue(1).(byte), t.tokenValue(2).(byte), t.tokenValue(3).(byte))
 		ones, bits := mask.Size()
 		if ones == 0 && bits == 0 {
 			return false
 		}
 
-		t.Output.ClearTokens()
+		t.tokensClear()
 		return true
 	}
 }
@@ -1462,18 +1462,18 @@ func MatchIPv4Net(normalize bool) Handler {
 			return true
 		}
 
-		maskToken := t.Output.Token(1)
+		maskToken := t.token(1)
 		if maskToken.Type == "cidr" {
-			t.Output.SetString(fmt.Sprintf("%s/%d", t.Output.TokenValue(0), t.Output.TokenValue(1).(uint8)))
+			t.dataSetString(fmt.Sprintf("%s/%d", t.tokenValue(0), t.tokenValue(1).(uint8)))
 		} else {
-			o := strings.Split(t.Output.TokenValue(1).(string), ".")
+			o := strings.Split(t.tokenValue(1).(string), ".")
 			b := func(idx int) byte { i, _ := strconv.Atoi(o[idx]); return byte(i) }
 			mask := net.IPv4Mask(b(0), b(1), b(2), b(3))
 			bits, _ := mask.Size()
-			t.Output.SetString(fmt.Sprintf("%s/%d", t.Output.TokenValue(0), bits))
+			t.dataSetString(fmt.Sprintf("%s/%d", t.tokenValue(0), bits))
 		}
 
-		t.Output.ClearTokens()
+		t.tokensClear()
 		return true
 	}
 }
@@ -1502,13 +1502,13 @@ func MatchIPv6(normalize bool) Handler {
 		}
 
 		// Invalid IPv6, when net.ParseIP() cannot handle it.
-		parsed := net.ParseIP(t.Output.String())
+		parsed := net.ParseIP(t.dataAsString())
 		if parsed == nil {
 			return false
 		}
 
 		if normalize {
-			t.Output.SetString(parsed.String())
+			t.dataSetString(parsed.String())
 		}
 		return true
 	}
@@ -1531,8 +1531,8 @@ func matchCIDRMask(bits int64, normalize bool) Handler {
 		if !mask(t) {
 			return false
 		}
-		bits, _ := strconv.Atoi(t.Output.String())
+		bits, _ := strconv.Atoi(t.dataAsString())
-		t.Output.SetString(fmt.Sprintf("%d", bits))
+		t.dataSetString(fmt.Sprintf("%d", bits))
 		return true
 	}
 }
@@ -1586,7 +1586,7 @@ func ModifyDrop(handler Handler) Handler {
 func ModifyDropUntilEndOfLine() Handler {
 	return func(t *API) bool {
 		for {
-			b, err := t.Input.PeekByte(0)
+			b, err := t.peekByte(0)
 			if err != nil {
 				if err == io.EOF {
 					return true
@@ -1596,7 +1596,7 @@ func ModifyDropUntilEndOfLine() Handler {
 			if b == '\n' {
 				return true
 			}
-			t.Input.SkipByte(b)
+			t.skipByte(b)
 		}
 	}
 }
@@ -1673,8 +1673,8 @@ func ModifyByCallback(handler Handler, modfunc func(string) string) Handler {
 	return func(t *API) bool {
 		child := t.Fork()
 		if handler(t) {
-			s := modfunc(t.Output.String())
+			s := modfunc(t.dataAsString())
-			t.Output.SetString(s)
+			t.dataSetString(s)
 			t.Merge(child)
 			t.Dispose(child)
 			return true
@@ -1691,7 +1691,7 @@ func ModifyByCallback(handler Handler, modfunc func(string) string) Handler {
 // an 'n'-character).
 func MakeStrLiteralToken(toktype interface{}, handler Handler) Handler {
 	return MakeTokenByCallback(toktype, handler, func(t *API) interface{} {
-		literal := t.Output.String()
+		literal := t.dataAsString()
 		return literal
 	})
 }
@@ -1703,7 +1703,7 @@ func MakeStrLiteralToken(toktype interface{}, handler Handler) Handler {
 func MakeStrInterpretedToken(toktype interface{}, handler Handler) Handler {
 	return MakeTokenByCallback(toktype, handler, func(t *API) interface{} {
 		// TODO ERROR HANDLING
-		interpreted, _ := interpretString(t.Output.String())
+		interpreted, _ := interpretString(t.dataAsString())
 		return interpreted
 	})
 }
@@ -1727,7 +1727,7 @@ func interpretString(str string) (string, error) {
 func MakeRuneToken(toktype interface{}, handler Handler) Handler {
 	return MakeTokenByCallback(toktype, handler, func(t *API) interface{} {
 		// TODO ERROR HANDLING --- not a 1 rune input
-		return t.Output.Rune(0)
+		return t.dataRune(0)
 	})
 }
 
@@ -1737,7 +1737,7 @@ func MakeRuneToken(toktype interface{}, handler Handler) Handler {
 func MakeByteToken(toktype interface{}, handler Handler) Handler {
 	return MakeTokenByCallback(toktype, handler, func(t *API) interface{} {
 		// TODO ERROR HANDLING --- not a 1 byte input
-		return byte(t.Output.Rune(0))
+		return byte(t.dataRune(0))
 	})
 }
 
@@ -1942,7 +1942,7 @@ func MakeBooleanToken(toktype interface{}, handler Handler) Handler {
 
 func makeStrconvToken(name string, toktype interface{}, handler Handler, convert func(s string) (interface{}, error)) Handler {
 	return MakeTokenByCallback(toktype, handler, func(t *API) interface{} {
-		value, err := convert(t.Output.String())
+		value, err := convert(t.dataAsString())
 		if err != nil {
 			// TODO meh, panic feels so bad here. Maybe just turn this case into "no match"?
 			panic(fmt.Sprintf("%s token invalid (%s)", name, err))
@@ -1973,7 +1973,7 @@ func MakeTokenByCallback(toktype interface{}, handler Handler, makeValue func(t
 			// tokens will end up in the order "date", "year", "month", "day". When we'd add the
 			// token to the child here, the order would have been "year", "month", "day", "date".
 			token := Token{Type: toktype, Value: makeValue(t)}
-			t.Output.AddTokens(token)
+			t.tokensAdd(token)
 			t.Merge(child)
 			t.Dispose(child)
 
@@ -1990,10 +1990,10 @@ func MakeTokenGroup(toktype interface{}, handler Handler) Handler {
 	return func(t *API) bool {
 		child := t.Fork()
 		if handler(t) {
-			tokens := t.Output.Tokens()
+			tokens := t.tokens()
 			tokensCopy := make([]Token, len(tokens))
 			copy(tokensCopy, tokens)
-			t.Output.SetTokens(Token{Type: toktype, Value: tokensCopy})
+			t.tokensSet(Token{Type: toktype, Value: tokensCopy})
 			t.Merge(child)
 			t.Dispose(child)
 			return true

tokenize/tokenize.go

@@ -43,12 +43,12 @@ func New(tokenHandler Handler) Func {
 		ok := tokenHandler(api)
 
 		if !ok {
-			err := fmt.Errorf("mismatch at %s", api.Input.Cursor())
+			err := fmt.Errorf("mismatch at %s", api.cursor())
 			return nil, err
 		}
 		result := &Result{
-			Runes:  api.Output.Runes(),
+			Runes:  api.dataAsRunes(),
-			Tokens: api.Output.Tokens(),
+			Tokens: api.tokens(),
 		}
 		return result, nil
 	}

tokenize/tokenizer_whitebox_test.go

@@ -9,7 +9,7 @@ func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) {
 	i := NewAPI("Testing")
 	r, _, _ := i.Input.PeekRune(0)
 	i.Input.AcceptRune(r) // T
-	AssertEqual(t, "T", i.Output.String(), "accepted rune in input")
+	AssertEqual(t, "T", i.dataAsString(), "accepted rune in input")
 
 	// Fork
 	child := i.Fork()
@@ -21,14 +21,14 @@ func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) {
 	i.Input.AcceptRune(r) // e
 	r, _, _ = i.Input.PeekRune(0)
 	i.Input.AcceptRune(r) // s
-	AssertEqual(t, "es", i.Output.String(), "result runes in fork")
+	AssertEqual(t, "es", i.dataAsString(), "result runes in fork")
 	AssertEqual(t, 1, i.stackFrames[i.stackLevel-1].offset, "parent offset")
 	AssertEqual(t, 3, i.stackFrame.offset, "child offset")
 
 	// Merge fork back into parent
 	i.Merge(child)
 	i.Dispose(child)
-	AssertEqual(t, "Tes", i.Output.String(), "result runes in parent Input after Merge()")
+	AssertEqual(t, "Tes", i.dataAsString(), "result runes in parent Input after Merge()")
 	AssertEqual(t, 3, i.stackFrame.offset, "parent offset")
 }
 
@@ -44,17 +44,17 @@ func TestGivenForkedChildWhichAcceptedRune_AfterMerging_RuneEndsUpInParentResult
 	f2 := i.Fork()
 	r, _, _ = i.Input.PeekRune(0)
 	i.Input.AcceptRune(r) // s
-	AssertEqual(t, "s", i.Output.String(), "f2 String()")
+	AssertEqual(t, "s", i.dataAsString(), "f2 String()")
 	AssertEqual(t, 3, i.stackFrame.offset, "f2.offset A")
 
 	i.Merge(f2)
 	i.Dispose(f2)
-	AssertEqual(t, "es", i.Output.String(), "f1 String()")
+	AssertEqual(t, "es", i.dataAsString(), "f1 String()")
 	AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A")
 
 	i.Merge(f1)
 	i.Dispose(f1)
-	AssertEqual(t, "Tes", i.Output.String(), "top-level API String()")
+	AssertEqual(t, "Tes", i.dataAsString(), "top-level API String()")
 	AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A")
 }
 
@@ -83,7 +83,7 @@ func TestFlushInput(t *testing.T) {
 	r, _, _ = i.Input.PeekRune(0)
 	i.Input.AcceptRune(r) // o
 
-	AssertEqual(t, "cool", i.Output.String(), "end result")
+	AssertEqual(t, "cool", i.dataAsString(), "end result")
 }
 
 func TestInputFlusherWrapper(t *testing.T) {
@@ -92,19 +92,19 @@ func TestInputFlusherWrapper(t *testing.T) {
 	api := NewAPI("abaab")
 	runeA(api)
 	AssertEqual(t, 1, api.stackFrame.offset, "offset after 1 read")
-	AssertEqual(t, "a", api.Output.String(), "runes after 1 read")
+	AssertEqual(t, "a", api.dataAsString(), "runes after 1 read")
 	flushB(api)
 	AssertEqual(t, 0, api.stackFrame.offset, "offset after 2 reads + input flush")
-	AssertEqual(t, "ab", api.Output.String(), "runes after 2 reads")
+	AssertEqual(t, "ab", api.dataAsString(), "runes after 2 reads")
 	runeA(api)
 	AssertEqual(t, 1, api.stackFrame.offset, "offset after 3 reads")
-	AssertEqual(t, "aba", api.Output.String(), "runes after 3 reads")
+	AssertEqual(t, "aba", api.dataAsString(), "runes after 3 reads")
 	runeA(api)
 	AssertEqual(t, 2, api.stackFrame.offset, "offset after 4 reads")
-	AssertEqual(t, "abaa", api.Output.String(), "runes after 4 reads")
+	AssertEqual(t, "abaa", api.dataAsString(), "runes after 4 reads")
 	flushB(api)
 	AssertEqual(t, 0, api.stackFrame.offset, "offset after 5 reads + input flush")
-	AssertEqual(t, "abaab", api.Output.String(), "runes after 5 reads")
+	AssertEqual(t, "abaab", api.dataAsString(), "runes after 5 reads")
 }
 
 func AssertEqual(t *testing.T, expected interface{}, actual interface{}, forWhat string) {