package parsekit import ( "fmt" "strings" ) // Result holds results as produced by a TokenHandler. type TokenResult struct { lastRune *runeInfo // Information about the last rune read using NextRune() runes []rune tokens []*Token } type runeInfo struct { r rune err error } // Token defines a lexical token as produced by TokenHandlers. type Token struct { Type interface{} // token type, can be any type that a parser author sees fit Runes []rune // the runes that make up the token Value interface{} // an optional value of any type } // newTokenResult initializes an empty result struct. func newTokenResult() *TokenResult { return &TokenResult{ runes: []rune{}, tokens: []*Token{}, } } // ClearRunes clears the runes in the TokenResult. func (r *TokenResult) ClearRunes() { r.runes = []rune{} } // SetRunes replaces the Runes from the TokenResult with the provided input. func (r *TokenResult) SetRunes(s interface{}) { r.ClearRunes() r.addRunes(s) } // AddRunes is used to add runes to the TokenResult. func (r *TokenResult) AddRunes(set ...interface{}) { r.addRunes(set...) } // AddRunes is used to add runes to the TokenResult. func (r *TokenResult) addRunes(set ...interface{}) { for _, s := range set { switch s := s.(type) { case string: r.runes = append(r.runes, []rune(s)...) case []rune: r.runes = append(r.runes, s...) case rune: r.runes = append(r.runes, s) default: callerPanic(2, "parsekit.TokenResult.AddRunes(): unsupported type '%T' used at {caller}", s) } } } // Runes retrieves the Runes from the TokenResult. func (r *TokenResult) Runes() []rune { return r.runes } // Rune retrieve a single rune from the TokenResult at the specified index. func (r *TokenResult) Rune(idx int) rune { return r.runes[idx] } // String returns the Runes from the TokenResult as a string. func (r *TokenResult) String() string { return string(r.runes) } // ClearTokens clears the tokens in the TokenResult. func (r *TokenResult) ClearTokens() { r.tokens = []*Token{} } // SetTokens replaces the Tokens from the TokenResult with the provided input. func (r *TokenResult) SetTokens(tokens []*Token) { r.ClearTokens() for _, t := range tokens { r.AddToken(t) } } // AddToken is used to add a Token to the TokenResult. func (r *TokenResult) AddToken(t *Token) { r.tokens = append(r.tokens, t) } // SliceOfTokens is an alias for []*Token type. The method Tokens() returns // this type. A String() method is defined for it, to make it easy to // format the tokens as a string for testing / debugging purposes. type SliceOfTokens []*Token func (ts SliceOfTokens) String() string { parts := make([]string, len(ts)) for i, t := range ts { str := fmt.Sprintf("%v(%T:%v)", t.Type, t.Value, t.Value) parts[i] = str } return strings.Join(parts, " ") } // Tokens retrieves the Tokens from the TokenResult. func (r *TokenResult) Tokens() SliceOfTokens { return r.tokens } // Token retrieves a single Token from the TokenResult at the specified index. func (r *TokenResult) Token(idx int) *Token { return r.tokens[idx] } // Values retrieves a slice containing only the Values for the TokenResult Tokens. func (r *TokenResult) Values() []interface{} { values := make([]interface{}, len(r.tokens)) for i, tok := range r.tokens { values[i] = tok.Value } return values } // Value retrieves a single Value from the TokenResult Token at the specified index. func (r *TokenResult) Value(idx int) interface{} { return r.tokens[idx].Value }