209 lines
4.6 KiB
Go
209 lines
4.6 KiB
Go
package tokenize
|
|
|
|
import (
|
|
"unicode/utf8"
|
|
)
|
|
|
|
// Output provides output-related functionality for the tokenize API.
|
|
type Output struct {
|
|
api *API
|
|
}
|
|
|
|
func (o *Output) String() string {
|
|
return o.api.dataAsString()
|
|
}
|
|
func (api *API) dataAsString() string {
|
|
bytes := api.outputData[api.stackFrame.bytesStart:api.stackFrame.bytesEnd]
|
|
return string(bytes)
|
|
}
|
|
|
|
func (o *Output) Runes() []rune {
|
|
return o.api.dataAsRunes()
|
|
}
|
|
|
|
func (api *API) dataAsRunes() []rune {
|
|
bytes := api.outputData[api.stackFrame.bytesStart:api.stackFrame.bytesEnd]
|
|
return []rune(string(bytes))
|
|
}
|
|
|
|
func (o *Output) Rune(offset int) rune {
|
|
return o.api.dataRune(offset)
|
|
}
|
|
|
|
func (api *API) dataRune(offset int) rune {
|
|
r, _ := utf8.DecodeRune(api.outputData[api.stackFrame.bytesStart+offset:])
|
|
return r
|
|
}
|
|
|
|
func (o *Output) ClearData() {
|
|
o.api.dataClear()
|
|
}
|
|
|
|
func (api *API) dataClear() {
|
|
api.stackFrame.bytesEnd = api.stackFrame.bytesStart
|
|
}
|
|
|
|
func (o *Output) SetBytes(bytes ...byte) {
|
|
o.api.dataSetBytes(bytes...)
|
|
}
|
|
|
|
func (api *API) dataSetBytes(bytes ...byte) {
|
|
api.dataClear()
|
|
api.dataAddBytes(bytes...)
|
|
}
|
|
|
|
func (o *Output) AddByte(b byte) {
|
|
o.api.dataAddByte(b)
|
|
}
|
|
|
|
func (api *API) dataAddByte(b byte) {
|
|
curBytesEnd := api.stackFrame.bytesEnd
|
|
newBytesEnd := curBytesEnd + 1
|
|
|
|
// Grow the bytes capacity when needed.
|
|
if cap(api.outputData) < newBytesEnd {
|
|
newBytes := make([]byte, newBytesEnd*2)
|
|
copy(newBytes, api.outputData)
|
|
api.outputData = newBytes
|
|
}
|
|
|
|
api.stackFrame.bytesEnd++
|
|
api.outputData[curBytesEnd] = b
|
|
}
|
|
|
|
func (o *Output) AddBytes(bytes ...byte) {
|
|
o.api.dataAddBytes(bytes...)
|
|
}
|
|
|
|
func (api *API) dataAddBytes(bytes ...byte) {
|
|
curBytesEnd := api.stackFrame.bytesEnd
|
|
newBytesEnd := curBytesEnd + len(bytes)
|
|
|
|
// Grow the runes capacity when needed.
|
|
if cap(api.outputData) < newBytesEnd {
|
|
newBytes := make([]byte, newBytesEnd*2)
|
|
copy(newBytes, api.outputData)
|
|
api.outputData = newBytes
|
|
}
|
|
|
|
copy(api.outputData[curBytesEnd:], bytes)
|
|
api.stackFrame.bytesEnd = newBytesEnd
|
|
}
|
|
|
|
func (o *Output) SetRunes(runes ...rune) {
|
|
o.api.dataSetRunes(runes...)
|
|
}
|
|
|
|
func (api *API) dataSetRunes(runes ...rune) {
|
|
api.dataClear()
|
|
api.dataAddRunes(runes...)
|
|
}
|
|
|
|
func (o *Output) AddRunes(runes ...rune) {
|
|
o.api.dataAddRunes(runes...)
|
|
}
|
|
|
|
func (api *API) dataAddRunes(runes ...rune) {
|
|
// Grow the runes capacity when needed.
|
|
runesAsString := string(runes)
|
|
newBytesEnd := api.stackFrame.bytesEnd + len(runesAsString)
|
|
if cap(api.outputData) < newBytesEnd {
|
|
newBytes := make([]byte, newBytesEnd*2)
|
|
copy(newBytes, api.outputData)
|
|
api.outputData = newBytes
|
|
}
|
|
|
|
copy(api.outputData[api.stackFrame.bytesEnd:], runesAsString)
|
|
api.stackFrame.bytesEnd = newBytesEnd
|
|
}
|
|
|
|
func (o *Output) AddString(s string) {
|
|
o.api.dataAddString(s)
|
|
}
|
|
|
|
func (api *API) dataAddString(s string) {
|
|
api.dataAddBytes([]byte(s)...)
|
|
}
|
|
|
|
func (o *Output) SetString(s string) {
|
|
o.api.dataSetString(s)
|
|
}
|
|
|
|
func (api *API) dataSetString(s string) {
|
|
api.dataClear()
|
|
api.dataSetBytes([]byte(s)...)
|
|
}
|
|
|
|
func (o *Output) Tokens() []Token {
|
|
return o.api.tokens()
|
|
}
|
|
|
|
func (api *API) tokens() []Token {
|
|
return api.outputTokens[api.stackFrame.tokenStart:api.stackFrame.tokenEnd]
|
|
}
|
|
|
|
func (o *Output) Token(offset int) Token {
|
|
return o.api.token(offset)
|
|
}
|
|
|
|
func (api *API) token(offset int) Token {
|
|
return api.outputTokens[api.stackFrame.tokenStart+offset]
|
|
}
|
|
|
|
func (o *Output) TokenValue(offset int) interface{} {
|
|
return o.api.tokenValue(offset)
|
|
}
|
|
|
|
func (api *API) tokenValue(offset int) interface{} {
|
|
return api.outputTokens[api.stackFrame.tokenStart+offset].Value
|
|
}
|
|
|
|
func (o *Output) ClearTokens() {
|
|
o.api.tokensClear()
|
|
}
|
|
|
|
func (api *API) tokensClear() {
|
|
api.stackFrame.tokenEnd = api.stackFrame.tokenStart
|
|
}
|
|
|
|
func (o *Output) SetTokens(tokens ...Token) {
|
|
o.api.tokensSet(tokens...)
|
|
}
|
|
|
|
func (api *API) tokensSet(tokens ...Token) {
|
|
api.tokensClear()
|
|
api.tokensAdd(tokens...)
|
|
type Func func(input interface{}) (*Result, error)
|
|
|
|
// Result holds the runes and tokens as produced by the tokenizer.
|
|
type Result struct {
|
|
Tokens []Token
|
|
Runes []rune
|
|
}
|
|
}
|
|
|
|
func (o *Output) AddTokens(tokens ...Token) {
|
|
o.api.tokensAdd(tokens...)
|
|
}
|
|
|
|
func (api *API) tokensAdd(tokens ...Token) {
|
|
// Grow the tokens capacity when needed.
|
|
newTokenEnd := api.stackFrame.tokenEnd + len(tokens)
|
|
if cap(api.outputTokens) < newTokenEnd {
|
|
type Func func(input interface{}) (*Result, error)
|
|
|
|
// Result holds the runes and tokens as produced by the tokenizer.
|
|
type Result struct {
|
|
Tokens []Token
|
|
Runes []rune
|
|
}
|
|
newTokens := make([]Token, newTokenEnd*2)
|
|
copy(newTokens, api.outputTokens)
|
|
api.outputTokens = newTokens
|
|
}
|
|
for offset, t := range tokens {
|
|
api.outputTokens[api.stackFrame.tokenEnd+offset] = t
|
|
}
|
|
api.stackFrame.tokenEnd = newTokenEnd
|
|
}
|