package parsekit import ( "fmt" "strings" ) // TokenHandlerResult is a struct that is used for holding and managing tokenizing results as // produced by a TokenHandler. type TokenHandlerResult struct { lastRune *runeInfo // Information about the last rune read using NextRune() runes []rune tokens []*Token cursor *Cursor // current read cursor position, relative to the start of the file offset int // current rune offset relative to the Reader's sliding window err *Error // can be used by a TokenHandler to report a specific issue with the input } type runeInfo struct { r rune err error } // Token defines a lexical token as produced by TokenHandlers. // // The only mandatory data in a Token are the Runes. The Type and Value fields // are optional fields that can be filled with data at will. // // The use of the Type field is to let a tokenizer communicate to // the parser what type of token it's handling. // // The use of the Value field is to store any kind af data along with the token. // One use of this can be found in the built-in token maker functions like // MakeInt8Token(), which store an interpreted version of the input string // in the Value field. type Token struct { Runes []rune // the runes that make up the token Type interface{} // optional token type, can be any type that a parser author sees fit Value interface{} // optional token value, of any type as well } func (t Token) String() string { tokenType := "" if t.Type != nil { tokenType = fmt.Sprintf("%v", t.Type) } value := "" if t.Value != nil { value = fmt.Sprintf(", value = (%T)%v", t.Value, t.Value) } return fmt.Sprintf("%v(%q%s)", tokenType, string(t.Runes), value) } // newTokenHandlerResult initializes an empty TokenHandlerResult struct. func newTokenHandlerResult() *TokenHandlerResult { return &TokenHandlerResult{ runes: []rune{}, tokens: []*Token{}, cursor: &Cursor{}, } } // ClearRunes clears the runes in the TokenHandlerResult. func (r *TokenHandlerResult) ClearRunes() { r.runes = []rune{} } // SetRunes replaces the Runes from the TokenHandlerResult with the provided input. func (r *TokenHandlerResult) SetRunes(s interface{}) { r.ClearRunes() r.addRunes(s) } // AddRunes is used to add runes to the TokenHandlerResult. func (r *TokenHandlerResult) AddRunes(set ...interface{}) { r.addRunes(set...) } // AddRunes is used to add runes to the TokenHandlerResult. func (r *TokenHandlerResult) addRunes(set ...interface{}) { for _, s := range set { switch s := s.(type) { case string: r.runes = append(r.runes, []rune(s)...) case []rune: r.runes = append(r.runes, s...) case rune: r.runes = append(r.runes, s) default: callerPanic(2, "parsekit.TokenHandlerResult.AddRunes(): unsupported type '%T' used at {caller}", s) } } } // Runes retrieves the Runes from the TokenHandlerResult. func (r *TokenHandlerResult) Runes() []rune { return r.runes } // Rune retrieve a single rune from the TokenHandlerResult at the specified index. func (r *TokenHandlerResult) Rune(idx int) rune { return r.runes[idx] } // String returns the Runes from the TokenHandlerResult as a string. func (r *TokenHandlerResult) String() string { return string(r.runes) } // ClearTokens clears the tokens in the TokenHandlerResult. func (r *TokenHandlerResult) ClearTokens() { r.tokens = []*Token{} } // SetTokens replaces the Tokens from the TokenHandlerResult with the provided input. func (r *TokenHandlerResult) SetTokens(tokens []*Token) { r.ClearTokens() for _, t := range tokens { r.AddToken(t) } } // AddToken is used to add a Token to the TokenHandlerResult. func (r *TokenHandlerResult) AddToken(t *Token) { r.tokens = append(r.tokens, t) } // SliceOfTokens is an alias for []*Token type. The method Tokens() returns // this type. A String() method is defined for it, to make it easy to // format the tokens as a string for testing / debugging purposes. type SliceOfTokens []*Token func (ts SliceOfTokens) String() string { parts := make([]string, len(ts)) for i, t := range ts { parts[i] = t.String() } return strings.Join(parts, " ") } // Tokens retrieves the Tokens from the TokenHandlerResult. func (r *TokenHandlerResult) Tokens() SliceOfTokens { return r.tokens } // Token retrieves a single Token from the TokenHandlerResult at the specified index. func (r *TokenHandlerResult) Token(idx int) *Token { return r.tokens[idx] } // Values retrieves a slice containing only the Values for the TokenHandlerResult Tokens. func (r *TokenHandlerResult) Values() []interface{} { values := make([]interface{}, len(r.tokens)) for i, tok := range r.tokens { values[i] = tok.Value } return values } // Value retrieves a single Value from the TokenHandlerResult Token at the specified index. func (r *TokenHandlerResult) Value(idx int) interface{} { return r.tokens[idx].Value } // Cursor retrieves the read cursor from the TokenHandlerResult. This is the // first cursor position after the runes that were read by the TokenHandler. func (r *TokenHandlerResult) Cursor() *Cursor { return r.cursor }