package tokenize import ( "fmt" ) // Result is a struct that is used for holding tokenizer results as produced // by a tokenize.Handler. It also provides the API that Handlers and Parsers // can use to store and retrieve the results. type Result struct { lastRune *runeInfo // Information about the last rune read using NextRune() runes []rune // runes as added to the result by tokenize.Handler functions tokens []*Token // Tokens as added to the result by tokenize.Handler functions cursor *Cursor // current read cursor position, relative to the start of the file offset int // current rune offset relative to the Reader's sliding window err error // can be used by a Handler to report a specific issue with the input } type runeInfo struct { r rune width int8 err error } // Token defines a lexical token as produced by tokenize.Handlers. // // The only mandatory data in a Token are the Runes. The Type and Value fields // are optional fields that can be filled with data at will. // // The use of the Type field is to let a tokenizer communicate to // the parser what type of token it's handling. // // The use of the Value field is to store any kind af data along with the token. // One use of this can be found in the built-in token maker functions like // MakeInt8Token(), which store an interpreted version of the input string // in the Value field. type Token struct { Runes []rune // the runes that make up the token Type interface{} // optional token type, can be any type that a parser author sees fit Value interface{} // optional token value, of any type as well } func (t Token) String() string { tokenType := "" if t.Type != nil { tokenType = fmt.Sprintf("%v", t.Type) } value := "" if t.Value != nil { value = fmt.Sprintf(", value = (%T)%v", t.Value, t.Value) } return fmt.Sprintf("%v(%q%s)", tokenType, string(t.Runes), value) } // newResult initializes an empty Result struct. func newResult() *Result { return &Result{ runes: []rune{}, tokens: []*Token{}, cursor: &Cursor{}, } } // ClearRunes clears the runes in the Result. func (r *Result) ClearRunes() { r.runes = []rune{} } // SetRunes replaces the Runes from the Result with the provided input. func (r *Result) SetRunes(s ...interface{}) { r.ClearRunes() r.addRunes("SetRunes", s...) } // AddRunes is used to add runes to the Result. func (r *Result) AddRunes(set ...interface{}) { r.addRunes("AddRunes", set...) } func (r *Result) addRunes(name string, set ...interface{}) { for _, s := range set { switch s := s.(type) { case string: r.runes = append(r.runes, []rune(s)...) case []rune: r.runes = append(r.runes, s...) case rune: r.runes = append(r.runes, s) default: callerPanic(name, "tokenize.Result.{name}(): unsupported type '%T' used at {caller}", s) } } } // Runes retrieves the Runes from the Result. func (r *Result) Runes() []rune { return r.runes } // Rune retrieve a single rune from the Result at the specified index. func (r *Result) Rune(idx int) rune { return r.runes[idx] } // String returns the Runes from the Result as a string. func (r *Result) String() string { return string(r.runes) } // ClearTokens clears the tokens in the Result. func (r *Result) ClearTokens() { r.tokens = []*Token{} } // SetTokens replaces the Tokens from the Result with the provided tokens. func (r *Result) SetTokens(tokens ...*Token) { r.ClearTokens() for _, t := range tokens { r.AddTokens(t) } } // AddTokens is used to add Tokens to the Result. func (r *Result) AddTokens(tokens ...*Token) { r.tokens = append(r.tokens, tokens...) } // Tokens retrieves the Tokens from the Result. func (r *Result) Tokens() []*Token { return r.tokens } // Token retrieves a single Token from the Result at the specified index. func (r *Result) Token(idx int) *Token { return r.tokens[idx] } // Values retrieves a slice containing only the Values for the Result Tokens. func (r *Result) Values() []interface{} { values := make([]interface{}, len(r.tokens)) for i, tok := range r.tokens { values[i] = tok.Value } return values } // Value retrieves a single Value from the Result Token at the specified index. func (r *Result) Value(idx int) interface{} { return r.tokens[idx].Value } // Cursor retrieves the read cursor from the Result. This is the first // cursor position after the runes that were read and accepted by the Handler. func (r *Result) Cursor() *Cursor { return r.cursor }