package tokenize import ( "unicode/utf8" ) // Output provides output-related functionality for the tokenize API. type Output struct { api *API } func (o *Output) String() string { return o.api.dataAsString() } func (api *API) dataAsString() string { bytes := api.outputData[api.stackFrame.bytesStart:api.stackFrame.bytesEnd] return string(bytes) } func (o *Output) Runes() []rune { return o.api.dataAsRunes() } func (api *API) dataAsRunes() []rune { bytes := api.outputData[api.stackFrame.bytesStart:api.stackFrame.bytesEnd] return []rune(string(bytes)) } func (o *Output) Rune(offset int) rune { return o.api.dataRune(offset) } func (api *API) dataRune(offset int) rune { r, _ := utf8.DecodeRune(api.outputData[api.stackFrame.bytesStart+offset:]) return r } func (o *Output) ClearData() { o.api.dataClear() } func (api *API) dataClear() { api.stackFrame.bytesEnd = api.stackFrame.bytesStart } func (o *Output) SetBytes(bytes ...byte) { o.api.dataSetBytes(bytes...) } func (o *Output) AddByte(b byte) { o.api.dataAddByte(b) } func (o *Output) AddBytes(bytes ...byte) { o.api.Byte.AppendMulti(bytes...) } func (o *Output) SetRunes(runes ...rune) { o.api.dataSetRunes(runes...) } func (api *API) dataSetRunes(runes ...rune) { api.dataClear() api.dataAddRunes(runes...) } func (o *Output) AddRunes(runes ...rune) { o.api.dataAddRunes(runes...) } func (api *API) dataAddRunes(runes ...rune) { runesAsString := string(runes) newBytesEnd := api.stackFrame.bytesEnd + len(runesAsString) api.growOutputData(newBytesEnd) copy(api.outputData[api.stackFrame.bytesEnd:], runesAsString) api.stackFrame.bytesEnd = newBytesEnd } func (o *Output) AddString(s string) { o.api.dataAddString(s) } func (api *API) dataAddString(s string) { api.Byte.AppendMulti([]byte(s)...) } func (o *Output) SetString(s string) { o.api.dataSetString(s) } func (api *API) dataSetString(s string) { api.dataClear() api.dataSetBytes([]byte(s)...) } func (o *Output) Tokens() []Token { return o.api.tokens() } func (api *API) tokens() []Token { return api.outputTokens[api.stackFrame.tokenStart:api.stackFrame.tokenEnd] } func (o *Output) Token(offset int) Token { return o.api.token(offset) } func (api *API) token(offset int) Token { return api.outputTokens[api.stackFrame.tokenStart+offset] } func (o *Output) TokenValue(offset int) interface{} { return o.api.tokenValue(offset) } func (api *API) tokenValue(offset int) interface{} { return api.outputTokens[api.stackFrame.tokenStart+offset].Value } func (o *Output) ClearTokens() { o.api.tokensClear() } func (api *API) tokensClear() { api.stackFrame.tokenEnd = api.stackFrame.tokenStart } func (o *Output) SetTokens(tokens ...Token) { o.api.tokensSet(tokens...) } func (api *API) tokensSet(tokens ...Token) { api.tokensClear() api.tokensAdd(tokens...) type Func func(input interface{}) (*Result, error) // Result holds the runes and tokens as produced by the tokenizer. type Result struct { Tokens []Token Runes []rune } } func (o *Output) AddTokens(tokens ...Token) { o.api.tokensAdd(tokens...) } func (api *API) tokensAdd(tokens ...Token) { // Grow the tokens capacity when needed. newTokenEnd := api.stackFrame.tokenEnd + len(tokens) api.growOutputTokens(newTokenEnd) for offset, t := range tokens { api.outputTokens[api.stackFrame.tokenEnd+offset] = t } api.stackFrame.tokenEnd = newTokenEnd } func (api *API) growOutputTokens(requiredTokens int) { if cap(api.outputTokens) < requiredTokens { newTokens := make([]Token, requiredTokens*2) copy(newTokens, api.outputTokens) api.outputTokens = newTokens } } func (api *API) growOutputData(requiredBytes int) { if cap(api.outputData) < requiredBytes { newBytes := make([]byte, requiredBytes*2) copy(newBytes, api.outputData) api.outputData = newBytes } }