160 lines
3.4 KiB
Go
160 lines
3.4 KiB
Go
package tokenize
|
|
|
|
import (
|
|
"unicode/utf8"
|
|
)
|
|
|
|
// Output provides output-related functionality for the tokenize API.
|
|
type Output struct {
|
|
api *API
|
|
}
|
|
|
|
func (o Output) Bytes() []byte {
|
|
a := o.api
|
|
return a.outputBytes[a.pointers.bytesStart:a.pointers.bytesEnd]
|
|
}
|
|
|
|
func (o Output) String() string {
|
|
return string(o.Bytes())
|
|
}
|
|
|
|
func (o Output) Runes() []rune {
|
|
return []rune(o.String())
|
|
}
|
|
|
|
func (o Output) Rune(offset int) rune {
|
|
a := o.api
|
|
r, _ := utf8.DecodeRune(a.outputBytes[a.pointers.bytesStart+offset:])
|
|
return r
|
|
}
|
|
|
|
func (o Output) Flush() {
|
|
a := o.api
|
|
a.pointers.bytesStart = 0
|
|
a.pointers.bytesEnd = 0
|
|
a.pointers.tokenStart = 0
|
|
a.pointers.tokenEnd = 0
|
|
}
|
|
|
|
func (o Output) ClearData() {
|
|
a := o.api
|
|
a.pointers.bytesEnd = a.pointers.bytesStart
|
|
}
|
|
|
|
func (o Output) SetBytes(bytes ...byte) {
|
|
o.ClearData()
|
|
o.AddBytes(bytes...)
|
|
}
|
|
|
|
func (o Output) AddByte(b byte) {
|
|
a := o.api
|
|
curBytesEnd := a.pointers.bytesEnd
|
|
a.growOutputData(curBytesEnd + 1)
|
|
a.outputBytes[curBytesEnd] = b
|
|
a.pointers.bytesEnd++
|
|
}
|
|
|
|
func (o Output) SetRunes(runes ...rune) {
|
|
o.ClearData()
|
|
o.AddRunes(runes...)
|
|
}
|
|
|
|
func (o Output) AddBytes(bytes ...byte) {
|
|
a := o.api
|
|
curBytesEnd := a.pointers.bytesEnd
|
|
newBytesEnd := curBytesEnd + len(bytes)
|
|
a.growOutputData(newBytesEnd)
|
|
copy(a.outputBytes[curBytesEnd:], bytes)
|
|
a.pointers.bytesEnd = newBytesEnd
|
|
}
|
|
|
|
func (o Output) AddRunes(runes ...rune) {
|
|
a := o.api
|
|
runesAsString := string(runes)
|
|
newBytesEnd := a.pointers.bytesEnd + len(runesAsString)
|
|
a.growOutputData(newBytesEnd)
|
|
copy(a.outputBytes[a.pointers.bytesEnd:], runesAsString)
|
|
a.pointers.bytesEnd = newBytesEnd
|
|
}
|
|
|
|
func (o Output) AddString(s string) {
|
|
o.AddBytes([]byte(s)...)
|
|
}
|
|
|
|
func (o Output) SetString(s string) {
|
|
o.ClearData()
|
|
o.AddBytes([]byte(s)...)
|
|
}
|
|
|
|
func (o Output) Tokens() []Token {
|
|
a := o.api
|
|
return a.outputTokens[a.pointers.tokenStart:a.pointers.tokenEnd]
|
|
}
|
|
|
|
func (o Output) Token(offset int) Token {
|
|
a := o.api
|
|
return a.outputTokens[a.pointers.tokenStart+offset]
|
|
}
|
|
|
|
func (o Output) TokenValue(offset int) interface{} {
|
|
a := o.api
|
|
return a.outputTokens[a.pointers.tokenStart+offset].Value
|
|
}
|
|
|
|
func (o Output) ClearTokens() {
|
|
a := o.api
|
|
a.pointers.tokenEnd = a.pointers.tokenStart
|
|
}
|
|
|
|
func (o Output) SetTokens(tokens ...Token) {
|
|
o.ClearTokens()
|
|
o.AddTokens(tokens...)
|
|
}
|
|
|
|
func (o Output) AddToken(token Token) {
|
|
a := o.api
|
|
tokenEnd := a.pointers.tokenEnd
|
|
a.growOutputTokens(tokenEnd + 1)
|
|
a.outputTokens[tokenEnd] = token
|
|
a.pointers.tokenEnd++
|
|
}
|
|
|
|
func (o Output) InsertTokenAtStart(token Token) {
|
|
a := o.api
|
|
tokenEnd := a.pointers.tokenEnd
|
|
tokenStart := a.pointers.tokenStart
|
|
a.growOutputTokens(tokenEnd + 1)
|
|
if tokenStart == tokenEnd {
|
|
a.outputTokens[tokenEnd] = token
|
|
} else {
|
|
copy(a.outputTokens[tokenStart+1:], a.outputTokens[tokenStart:tokenEnd])
|
|
a.outputTokens[tokenStart] = token
|
|
}
|
|
a.pointers.tokenEnd++
|
|
}
|
|
|
|
func (o Output) AddTokens(tokens ...Token) {
|
|
a := o.api
|
|
a.growOutputTokens(a.pointers.tokenEnd + len(tokens))
|
|
for _, t := range tokens {
|
|
a.outputTokens[a.pointers.tokenEnd] = t
|
|
a.pointers.tokenEnd++
|
|
}
|
|
}
|
|
|
|
func (api *API) growOutputTokens(requiredTokens int) {
|
|
if cap(api.outputTokens) < requiredTokens {
|
|
newTokens := make([]Token, requiredTokens*2)
|
|
copy(newTokens, api.outputTokens)
|
|
api.outputTokens = newTokens
|
|
}
|
|
}
|
|
|
|
func (api *API) growOutputData(requiredBytes int) {
|
|
if cap(api.outputBytes) < requiredBytes {
|
|
newBytes := make([]byte, requiredBytes*2)
|
|
copy(newBytes, api.outputBytes)
|
|
api.outputBytes = newBytes
|
|
}
|
|
}
|