go-parsekit/tokenize/result.go

package tokenize

import (
	"fmt"
)

// Result is a struct that is used for holding tokenizer results as produced
// by a tokenize.Handler. It also provides the API that Handlers and Parsers
// can use to store and retrieve the results.
type Result struct {
	lastRune *runeInfo // Information about the last rune read using NextRune()
	runes    []rune    // runes as added to the result by tokenize.Handler functions
	tokens   []*Token  // Tokens as added to the result by tokenize.Handler functions
	cursor   *Cursor   // current read cursor position, relative to the start of the file
	offset   int       // current rune offset relative to the Reader's sliding window
	err      error     // can be used by a Handler to report a specific issue with the input
}

type runeInfo struct {
	r   rune
	err error
}

// Token defines a lexical token as produced by tokenize.Handlers.
//
// The only mandatory data in a Token are the Runes. The Type and Value fields
// are optional fields that can be filled with data at will.
//
// The use of the Type field is to let a tokenizer communicate to
// the parser what type of token it's handling.
//
// The use of the Value field is to store any kind af data along with the token.
// One use of this can be found in the built-in token maker functions like
// MakeInt8Token(), which store an interpreted version of the input string
// in the Value field.
type Token struct {
	Runes []rune      // the runes that make up the token
	Type  interface{} // optional token type, can be any type that a parser author sees fit
	Value interface{} // optional token value, of any type as well
}

func (t Token) String() string {
	tokenType := ""
	if t.Type != nil {
		tokenType = fmt.Sprintf("%v", t.Type)
	}

	value := ""
	if t.Value != nil {
		value = fmt.Sprintf(", value = (%T)%v", t.Value, t.Value)
	}

	return fmt.Sprintf("%v(%q%s)", tokenType, string(t.Runes), value)
}

// newResult initializes an empty Result struct.
func newResult() *Result {
	return &Result{
		runes:  []rune{},
		tokens: []*Token{},
		cursor: &Cursor{},
	}
}

// ClearRunes clears the runes in the Result.
func (r *Result) ClearRunes() {
	r.runes = []rune{}
}

// SetRunes replaces the Runes from the Result with the provided input.
func (r *Result) SetRunes(s ...interface{}) {
	r.ClearRunes()
	r.addRunes(s...)
}

// AddRunes is used to add runes to the Result.
func (r *Result) AddRunes(set ...interface{}) {
	r.addRunes(set...)
}

func (r *Result) addRunes(set ...interface{}) {
	for _, s := range set {
		switch s := s.(type) {
		case string:
			r.runes = append(r.runes, []rune(s)...)
		case []rune:
			r.runes = append(r.runes, s...)
		case rune:
			r.runes = append(r.runes, s)
		default:
			callerPanic(2, "tokenize.Result.AddRunes(): unsupported type '%T' used at {caller}", s)
		}
	}
}

// Runes retrieves the Runes from the Result.
func (r *Result) Runes() []rune {
	return r.runes
}

// Rune retrieve a single rune from the Result at the specified index.
func (r *Result) Rune(idx int) rune {
	return r.runes[idx]
}

// String returns the Runes from the Result as a string.
func (r *Result) String() string {
	return string(r.runes)
}

// ClearTokens clears the tokens in the Result.
func (r *Result) ClearTokens() {
	r.tokens = []*Token{}
}

// SetTokens replaces the Tokens from the Result with the provided tokens.
func (r *Result) SetTokens(tokens ...*Token) {
	r.ClearTokens()
	for _, t := range tokens {
		r.AddTokens(t)
	}
}

// AddTokens is used to add Tokens to the Result.
func (r *Result) AddTokens(tokens ...*Token) {
	r.tokens = append(r.tokens, tokens...)
}

// Tokens retrieves the Tokens from the Result.
func (r *Result) Tokens() []*Token {
	return r.tokens
}

// Token retrieves a single Token from the Result at the specified index.
func (r *Result) Token(idx int) *Token {
	return r.tokens[idx]
}

// Values retrieves a slice containing only the Values for the Result Tokens.
func (r *Result) Values() []interface{} {
	values := make([]interface{}, len(r.tokens))
	for i, tok := range r.tokens {
		values[i] = tok.Value
	}
	return values
}

// Value retrieves a single Value from the Result Token at the specified index.
func (r *Result) Value(idx int) interface{} {
	return r.tokens[idx].Value
}

// Cursor retrieves the read cursor from the Result. This is the first
// cursor position after the runes that were read and accepted by the Handler.
func (r *Result) Cursor() *Cursor {
	return r.cursor
}