180 lines
3.6 KiB
Go
180 lines
3.6 KiB
Go
package tokenize
|
|
|
|
import (
|
|
"unicode/utf8"
|
|
)
|
|
|
|
// Output provides output-related functionality for the tokenize API.
|
|
type Output struct {
|
|
api *API
|
|
}
|
|
|
|
func (o Output) String() string {
|
|
a := o.api
|
|
f := a.stackFrame
|
|
bytes := a.outputBytes[f.bytesStart:f.bytesEnd]
|
|
return string(bytes)
|
|
}
|
|
|
|
func (o Output) Runes() []rune {
|
|
return []rune(o.String())
|
|
}
|
|
|
|
func (o Output) Rune(offset int) rune {
|
|
a := o.api
|
|
r, _ := utf8.DecodeRune(a.outputBytes[a.stackFrame.bytesStart+offset:])
|
|
return r
|
|
}
|
|
|
|
type Split [2]int
|
|
|
|
func (o Output) Split() Split {
|
|
f := o.api.stackFrame
|
|
split := Split{f.bytesStart, f.tokenStart}
|
|
f.bytesStart = f.bytesEnd
|
|
f.tokenStart = f.tokenEnd
|
|
return split
|
|
}
|
|
|
|
func (o Output) MergeSplit(split Split) {
|
|
f := o.api.stackFrame
|
|
f.bytesStart = split[0]
|
|
f.tokenStart = split[1]
|
|
}
|
|
|
|
func (o Output) Reset() {
|
|
f := o.api.stackFrame
|
|
f.bytesEnd = f.bytesStart
|
|
f.tokenEnd = f.tokenStart
|
|
f.err = nil
|
|
}
|
|
|
|
func (o Output) ClearData() {
|
|
f := o.api.stackFrame
|
|
f.bytesEnd = f.bytesStart
|
|
}
|
|
|
|
func (o Output) SetBytes(bytes ...byte) {
|
|
o.ClearData()
|
|
o.AddBytes(bytes...)
|
|
}
|
|
|
|
func (o Output) AddByte(b byte) {
|
|
a := o.api
|
|
f := a.stackFrame
|
|
curBytesEnd := f.bytesEnd
|
|
a.growOutputData(curBytesEnd + 1)
|
|
a.outputBytes[curBytesEnd] = b
|
|
f.bytesEnd++
|
|
}
|
|
|
|
func (o Output) SetRunes(runes ...rune) {
|
|
o.ClearData()
|
|
o.AddRunes(runes...)
|
|
}
|
|
|
|
func (o Output) AddBytes(bytes ...byte) {
|
|
a := o.api
|
|
f := a.stackFrame
|
|
curBytesEnd := f.bytesEnd
|
|
newBytesEnd := curBytesEnd + len(bytes)
|
|
a.growOutputData(newBytesEnd)
|
|
copy(a.outputBytes[curBytesEnd:], bytes)
|
|
f.bytesEnd = newBytesEnd
|
|
}
|
|
|
|
func (o Output) AddRunes(runes ...rune) {
|
|
a := o.api
|
|
f := a.stackFrame
|
|
runesAsString := string(runes)
|
|
newBytesEnd := f.bytesEnd + len(runesAsString)
|
|
a.growOutputData(newBytesEnd)
|
|
copy(a.outputBytes[f.bytesEnd:], runesAsString)
|
|
f.bytesEnd = newBytesEnd
|
|
}
|
|
|
|
func (o Output) AddString(s string) {
|
|
o.AddBytes([]byte(s)...)
|
|
}
|
|
|
|
func (o Output) SetString(s string) {
|
|
o.ClearData()
|
|
o.AddBytes([]byte(s)...)
|
|
}
|
|
|
|
func (o Output) Tokens() []Token {
|
|
a := o.api
|
|
f := a.stackFrame
|
|
return a.outputTokens[f.tokenStart:f.tokenEnd]
|
|
}
|
|
|
|
func (o Output) Token(offset int) Token {
|
|
a := o.api
|
|
return a.outputTokens[a.stackFrame.tokenStart+offset]
|
|
}
|
|
|
|
func (o Output) TokenValue(offset int) interface{} {
|
|
a := o.api
|
|
return a.outputTokens[a.stackFrame.tokenStart+offset].Value
|
|
}
|
|
|
|
func (o Output) ClearTokens() {
|
|
f := o.api.stackFrame
|
|
f.tokenEnd = f.tokenStart
|
|
}
|
|
|
|
func (o Output) SetTokens(tokens ...Token) {
|
|
o.ClearTokens()
|
|
o.AddTokens(tokens...)
|
|
}
|
|
|
|
func (o Output) AddToken(token Token) {
|
|
a := o.api
|
|
f := a.stackFrame
|
|
tokenEnd := f.tokenEnd
|
|
a.growOutputTokens(tokenEnd + 1)
|
|
a.outputTokens[tokenEnd] = token
|
|
f.tokenEnd++
|
|
}
|
|
|
|
func (o Output) InsertTokenAtStart(token Token) {
|
|
a := o.api
|
|
f := a.stackFrame
|
|
tokenEnd := f.tokenEnd
|
|
tokenStart := f.tokenStart
|
|
a.growOutputTokens(tokenEnd + 1)
|
|
if tokenStart == tokenEnd {
|
|
a.outputTokens[tokenEnd] = token
|
|
} else {
|
|
copy(a.outputTokens[tokenStart+1:], a.outputTokens[tokenStart:tokenEnd])
|
|
a.outputTokens[tokenStart] = token
|
|
}
|
|
f.tokenEnd++
|
|
}
|
|
|
|
func (o Output) AddTokens(tokens ...Token) {
|
|
a := o.api
|
|
f := a.stackFrame
|
|
a.growOutputTokens(f.tokenEnd + len(tokens))
|
|
for _, t := range tokens {
|
|
a.outputTokens[f.tokenEnd] = t
|
|
f.tokenEnd++
|
|
}
|
|
}
|
|
|
|
func (api *API) growOutputTokens(requiredTokens int) {
|
|
if cap(api.outputTokens) < requiredTokens {
|
|
newTokens := make([]Token, requiredTokens*2)
|
|
copy(newTokens, api.outputTokens)
|
|
api.outputTokens = newTokens
|
|
}
|
|
}
|
|
|
|
func (api *API) growOutputData(requiredBytes int) {
|
|
if cap(api.outputBytes) < requiredBytes {
|
|
newBytes := make([]byte, requiredBytes*2)
|
|
copy(newBytes, api.outputBytes)
|
|
api.outputBytes = newBytes
|
|
}
|
|
}
|