package tokenize import ( "unicode/utf8" ) // Output provides output-related functionality for the tokenize API. type Output struct { api *API suspended int // whether or not the output has been suspended (i.e. value > 0) } func (o *Output) Suspend() { o.suspended++ } func (o *Output) Resume() { o.suspended-- } func (o Output) Bytes() []byte { a := o.api return a.outputBytes[a.pointers.bytesStart:a.pointers.bytesEnd] } func (o Output) String() string { return string(o.Bytes()) } func (o Output) Runes() []rune { return []rune(o.String()) } func (o Output) Rune(offset int) rune { a := o.api r, _ := utf8.DecodeRune(a.outputBytes[a.pointers.bytesStart+offset:]) return r } func (o Output) Flush() { a := o.api a.pointers.bytesStart = a.pointers.bytesEnd a.pointers.tokenStart = a.pointers.tokenEnd } func (o Output) ClearData() { if o.suspended > 0 { return } a := o.api a.pointers.bytesEnd = a.pointers.bytesStart } func (o Output) SetBytes(bytes ...byte) { if o.suspended > 0 { return } o.ClearData() o.AddBytes(bytes...) } func (o Output) AddByte(b byte) { if o.suspended > 0 { return } a := o.api curBytesEnd := a.pointers.bytesEnd a.growOutputData(curBytesEnd + 1) a.outputBytes[curBytesEnd] = b a.pointers.bytesEnd++ } func (o Output) SetRunes(runes ...rune) { if o.suspended > 0 { return } o.ClearData() o.AddRunes(runes...) } func (o Output) AddBytes(bytes ...byte) { if o.suspended > 0 { return } a := o.api curBytesEnd := a.pointers.bytesEnd newBytesEnd := curBytesEnd + len(bytes) a.growOutputData(newBytesEnd) copy(a.outputBytes[curBytesEnd:], bytes) a.pointers.bytesEnd = newBytesEnd } func (o Output) AddRunes(runes ...rune) { if o.suspended > 0 { return } a := o.api runesAsString := string(runes) newBytesEnd := a.pointers.bytesEnd + len(runesAsString) a.growOutputData(newBytesEnd) copy(a.outputBytes[a.pointers.bytesEnd:], runesAsString) a.pointers.bytesEnd = newBytesEnd } func (o Output) AddString(s string) { if o.suspended > 0 { return } o.AddBytes([]byte(s)...) } func (o Output) SetString(s string) { if o.suspended > 0 { return } o.ClearData() o.AddBytes([]byte(s)...) } func (o Output) Tokens() []Token { a := o.api return a.outputTokens[a.pointers.tokenStart:a.pointers.tokenEnd] } func (o Output) Token(offset int) Token { a := o.api return a.outputTokens[a.pointers.tokenStart+offset] } func (o Output) TokenValue(offset int) interface{} { a := o.api return a.outputTokens[a.pointers.tokenStart+offset].Value } func (o Output) ClearTokens() { if o.suspended > 0 { return } a := o.api a.pointers.tokenEnd = a.pointers.tokenStart } func (o Output) SetTokens(tokens ...Token) { if o.suspended > 0 { return } o.ClearTokens() o.AddTokens(tokens...) } func (o Output) AddToken(token Token) { if o.suspended > 0 { return } a := o.api tokenEnd := a.pointers.tokenEnd a.growOutputTokens(tokenEnd + 1) a.outputTokens[tokenEnd] = token a.pointers.tokenEnd++ } func (o Output) InsertTokenAtStart(token Token) { if o.suspended > 0 { return } a := o.api tokenEnd := a.pointers.tokenEnd tokenStart := a.pointers.tokenStart a.growOutputTokens(tokenEnd + 1) if tokenStart == tokenEnd { a.outputTokens[tokenEnd] = token } else { copy(a.outputTokens[tokenStart+1:], a.outputTokens[tokenStart:tokenEnd]) a.outputTokens[tokenStart] = token } a.pointers.tokenEnd++ } func (o Output) AddTokens(tokens ...Token) { if o.suspended > 0 { return } a := o.api a.growOutputTokens(a.pointers.tokenEnd + len(tokens)) for _, t := range tokens { a.outputTokens[a.pointers.tokenEnd] = t a.pointers.tokenEnd++ } } func (api *API) growOutputTokens(requiredTokens int) { if cap(api.outputTokens) < requiredTokens { newTokens := make([]Token, requiredTokens*2) copy(newTokens, api.outputTokens) api.outputTokens = newTokens } } func (api *API) growOutputData(requiredBytes int) { if cap(api.outputBytes) < requiredBytes { newBytes := make([]byte, requiredBytes*2) copy(newBytes, api.outputBytes) api.outputBytes = newBytes } }