go-parsekit/tokenhandlerresult.go

package parsekit

import (
	"fmt"
	"strings"
)

// TokenHandlerResult is a struct that is used for holding and managing tokenizing results as
// produced by a TokenHandler.
type TokenHandlerResult struct {
	lastRune *runeInfo // Information about the last rune read using NextRune()
	runes    []rune
	tokens   []*Token
	cursor   *Cursor // current read cursor position, relative to the start of the file
	offset   int     // current rune offset relative to the Reader's sliding window
	err      *Error  // can be used by a TokenHandler to report a specific issue with the input
}

type runeInfo struct {
	r   rune
	err error
}

// Token defines a lexical token as produced by TokenHandlers.
//
// The only mandatory data in a Token are the Runes. The Type and Value fields
// are optional fields that can be filled with data at will.
//
// The use of the Type field is to let a tokenizer communicate to
// the parser what type of token it's handling.
//
// The use of the Value field is to store any kind af data along with the token.
// One use of this can be found in the built-in token maker functions like
// MakeInt8Token(), which store an interpreted version of the input string
// in the Value field.
type Token struct {
	Runes []rune      // the runes that make up the token
	Type  interface{} // optional token type, can be any type that a parser author sees fit
	Value interface{} // optional token value, of any type as well
}

func (t Token) String() string {
	tokenType := ""
	if t.Type != nil {
		tokenType = fmt.Sprintf("%v", t.Type)
	}

	value := ""
	if t.Value != nil {
		value = fmt.Sprintf(", value = (%T)%v", t.Value, t.Value)
	}

	return fmt.Sprintf("%v(%q%s)", tokenType, string(t.Runes), value)
}

// newTokenHandlerResult initializes an empty TokenHandlerResult struct.
func newTokenHandlerResult() *TokenHandlerResult {
	return &TokenHandlerResult{
		runes:  []rune{},
		tokens: []*Token{},
		cursor: &Cursor{},
	}
}

// ClearRunes clears the runes in the TokenHandlerResult.
func (r *TokenHandlerResult) ClearRunes() {
	r.runes = []rune{}
}

// SetRunes replaces the Runes from the TokenHandlerResult with the provided input.
func (r *TokenHandlerResult) SetRunes(s interface{}) {
	r.ClearRunes()
	r.addRunes(s)
}

// AddRunes is used to add runes to the TokenHandlerResult.
func (r *TokenHandlerResult) AddRunes(set ...interface{}) {
	r.addRunes(set...)
}

// AddRunes is used to add runes to the TokenHandlerResult.
func (r *TokenHandlerResult) addRunes(set ...interface{}) {
	for _, s := range set {
		switch s := s.(type) {
		case string:
			r.runes = append(r.runes, []rune(s)...)
		case []rune:
			r.runes = append(r.runes, s...)
		case rune:
			r.runes = append(r.runes, s)
		default:
			callerPanic(2, "parsekit.TokenHandlerResult.AddRunes(): unsupported type '%T' used at {caller}", s)
		}
	}
}

// Runes retrieves the Runes from the TokenHandlerResult.
func (r *TokenHandlerResult) Runes() []rune {
	return r.runes
}

// Rune retrieve a single rune from the TokenHandlerResult at the specified index.
func (r *TokenHandlerResult) Rune(idx int) rune {
	return r.runes[idx]
}

// String returns the Runes from the TokenHandlerResult as a string.
func (r *TokenHandlerResult) String() string {
	return string(r.runes)
}

// ClearTokens clears the tokens in the TokenHandlerResult.
func (r *TokenHandlerResult) ClearTokens() {
	r.tokens = []*Token{}
}

// SetTokens replaces the Tokens from the TokenHandlerResult with the provided input.
func (r *TokenHandlerResult) SetTokens(tokens []*Token) {
	r.ClearTokens()
	for _, t := range tokens {
		r.AddToken(t)
	}
}

// AddToken is used to add a Token to the TokenHandlerResult.
func (r *TokenHandlerResult) AddToken(t *Token) {
	r.tokens = append(r.tokens, t)
}

// SliceOfTokens is an alias for []*Token type. The method Tokens() returns
// this type. A String() method is defined for it, to make it easy to
// format the tokens as a string for testing / debugging purposes.
type SliceOfTokens []*Token

func (ts SliceOfTokens) String() string {
	parts := make([]string, len(ts))
	for i, t := range ts {
		parts[i] = t.String()
	}
	return strings.Join(parts, " ")
}

// Tokens retrieves the Tokens from the TokenHandlerResult.
func (r *TokenHandlerResult) Tokens() SliceOfTokens {
	return r.tokens
}

// Token retrieves a single Token from the TokenHandlerResult at the specified index.
func (r *TokenHandlerResult) Token(idx int) *Token {
	return r.tokens[idx]
}

// Values retrieves a slice containing only the Values for the TokenHandlerResult Tokens.
func (r *TokenHandlerResult) Values() []interface{} {
	values := make([]interface{}, len(r.tokens))
	for i, tok := range r.tokens {
		values[i] = tok.Value
	}
	return values
}

// Value retrieves a single Value from the TokenHandlerResult Token at the specified index.
func (r *TokenHandlerResult) Value(idx int) interface{} {
	return r.tokens[idx].Value
}

func (r *TokenHandlerResult) Cursor() *Cursor {
	return r.cursor
}