package tokenize import ( "unicode/utf8" ) // Output provides output-related functionality for the tokenize API. type Output struct { api *API } func (o Output) String() string { a := o.api bytes := a.outputBytes[a.pointers.bytesStart:a.pointers.bytesEnd] return string(bytes) } func (o Output) Runes() []rune { return []rune(o.String()) } func (o Output) Rune(offset int) rune { a := o.api r, _ := utf8.DecodeRune(a.outputBytes[a.pointers.bytesStart+offset:]) return r } type Split [2]int func (o Output) Split() Split { a := o.api split := Split{a.pointers.bytesStart, a.pointers.tokenStart} a.pointers.bytesStart = a.pointers.bytesEnd a.pointers.tokenStart = a.pointers.tokenEnd return split } func (o Output) MergeSplit(split Split) { a := o.api a.pointers.bytesStart = split[0] a.pointers.tokenStart = split[1] } func (o Output) Reset() { a := o.api a.pointers.bytesEnd = a.pointers.bytesStart a.pointers.tokenEnd = a.pointers.tokenStart } func (o Output) ClearData() { a := o.api a.pointers.bytesEnd = a.pointers.bytesStart } func (o Output) SetBytes(bytes ...byte) { o.ClearData() o.AddBytes(bytes...) } func (o Output) AddByte(b byte) { a := o.api curBytesEnd := a.pointers.bytesEnd a.growOutputData(curBytesEnd + 1) a.outputBytes[curBytesEnd] = b a.pointers.bytesEnd++ } func (o Output) SetRunes(runes ...rune) { o.ClearData() o.AddRunes(runes...) } func (o Output) AddBytes(bytes ...byte) { a := o.api curBytesEnd := a.pointers.bytesEnd newBytesEnd := curBytesEnd + len(bytes) a.growOutputData(newBytesEnd) copy(a.outputBytes[curBytesEnd:], bytes) a.pointers.bytesEnd = newBytesEnd } func (o Output) AddRunes(runes ...rune) { a := o.api runesAsString := string(runes) newBytesEnd := a.pointers.bytesEnd + len(runesAsString) a.growOutputData(newBytesEnd) copy(a.outputBytes[a.pointers.bytesEnd:], runesAsString) a.pointers.bytesEnd = newBytesEnd } func (o Output) AddString(s string) { o.AddBytes([]byte(s)...) } func (o Output) SetString(s string) { o.ClearData() o.AddBytes([]byte(s)...) } func (o Output) Tokens() []Token { a := o.api return a.outputTokens[a.pointers.tokenStart:a.pointers.tokenEnd] } func (o Output) Token(offset int) Token { a := o.api return a.outputTokens[a.pointers.tokenStart+offset] } func (o Output) TokenValue(offset int) interface{} { a := o.api return a.outputTokens[a.pointers.tokenStart+offset].Value } func (o Output) ClearTokens() { a := o.api a.pointers.tokenEnd = a.pointers.tokenStart } func (o Output) SetTokens(tokens ...Token) { o.ClearTokens() o.AddTokens(tokens...) } func (o Output) AddToken(token Token) { a := o.api tokenEnd := a.pointers.tokenEnd a.growOutputTokens(tokenEnd + 1) a.outputTokens[tokenEnd] = token a.pointers.tokenEnd++ } func (o Output) InsertTokenAtStart(token Token) { a := o.api tokenEnd := a.pointers.tokenEnd tokenStart := a.pointers.tokenStart a.growOutputTokens(tokenEnd + 1) if tokenStart == tokenEnd { a.outputTokens[tokenEnd] = token } else { copy(a.outputTokens[tokenStart+1:], a.outputTokens[tokenStart:tokenEnd]) a.outputTokens[tokenStart] = token } a.pointers.tokenEnd++ } func (o Output) AddTokens(tokens ...Token) { a := o.api a.growOutputTokens(a.pointers.tokenEnd + len(tokens)) for _, t := range tokens { a.outputTokens[a.pointers.tokenEnd] = t a.pointers.tokenEnd++ } } func (api *API) growOutputTokens(requiredTokens int) { if cap(api.outputTokens) < requiredTokens { newTokens := make([]Token, requiredTokens*2) copy(newTokens, api.outputTokens) api.outputTokens = newTokens } } func (api *API) growOutputData(requiredBytes int) { if cap(api.outputBytes) < requiredBytes { newBytes := make([]byte, requiredBytes*2) copy(newBytes, api.outputBytes) api.outputBytes = newBytes } }