go-parsekit/tokenize/api_result.go

package tokenize

// Result holds the bytes and tokens as produced by the tokenizer.
type Result struct {
	Tokens []Token
	Bytes  []byte
}

func (result *Result) String() string {
	return string(result.Bytes)
}