package lexer import ( "fmt" "strings" "unicode/utf8" ) // Lexer holds the state of the lexer. type Lexer struct { input string // the scanned input string state stateFn // a function that handles the current state stack []stateFn // state function stack, for nested parsing pos int // current byte scanning position in the input newline bool // keep track of when we have scanned a newline linenr int // current line number in the input linepos int // current position in the input line width int // width of the last rune read, for supporting backup() buffer StringBuffer // an efficient buffer, used to build string values items chan Item // channel of resulting lexer items nextItem Item // the current item as reached by Next() and retrieved by Get() err *Error // an error when lexing failed, retrieved by Error() } // Error is used as the error type when lexing errors occur. // The error includes some extra meta information to allow for useful // error messages to the user. type Error struct { Message string LineNr int LinePos int } func (err *Error) Error() string { return err.Message } // Lex takes an input string and initializes the TOML lexer for it. // Usage: // // l := lexer.Lex("...inputstring...") // for l.Next() { // item := l.Get() // ... handle item ... // } // if e := l.Error(); e != nil { // ... handle error message ... // } func Lex(input string) *Lexer { return &Lexer{ input: input, state: stateKeyValuePair, items: make(chan Item, 2), } } // Next advances to the next lexer item in the input string. // When a next item was found, then true is returned. // On error or reaching the end of the input, false is returned. func (l *Lexer) Next() bool { if l.state == nil { panic("This should not happen: nil state reached, but entering Next()") } for { select { case i := <-l.items: if i.Type == ItemEOF { return false } if i.Type == ItemError { l.err = &Error{i.Value, l.linenr, l.linepos} return false } l.nextItem = i return true default: l.state = l.state(l) } } } func (l *Lexer) Error() *Error { return l.err } // Get returns the next lexer item, as reached by Next() func (l *Lexer) Get() Item { return l.nextItem } // ToArray returns lexer items as an array. // When an error occurs during scanning, a partial result will be // returned, accompanied by the error that occurred. func (l *Lexer) ToArray() ([]Item, *Error) { var items []Item for l.Next() { items = append(items, l.Get()) } return items, l.Error() } // pushState adds the state function to its stack. // This is used for implementing nested parsing. func (l *Lexer) pushState(state stateFn) { l.stack = append(l.stack, state) } // popState pops the last pushed state from its stack. func (l *Lexer) popState() stateFn { last := len(l.stack) - 1 head, tail := l.stack[:last], l.stack[last] l.stack = head return tail } // atEndOfFile returns true when there is no more data available in the input. func (l *Lexer) atEndOfFile() bool { return l.pos >= len(l.input) } // emit passes a lexer item back to the client, including the provided string. func (l *Lexer) emit(t itemType, s string) { l.items <- Item{t, s} l.buffer.Reset() } // emitLiteral passes a lexer item back to the client, including the accumulated // string buffer data as a literal string. func (l *Lexer) emitLiteral(t itemType) { l.emit(t, l.buffer.AsLiteralString()) } // emitTrimmedLiteral passes a lexer item back to the client, including the // accumulated string buffer data as a literal string with whitespace // trimmed from it. func (l *Lexer) emitTrimmedLiteral(t itemType) { l.emit(t, strings.TrimSpace(l.buffer.AsLiteralString())) } // emitInterpreted passes a lexer item back to the client, including the // accumulated string buffer data an interpreted string (handling escape // codes like \n, \t, \uXXXX, etc.) // This method might return an error, in case there is data in the // string buffer that is not valid for string interpretation. func (l *Lexer) emitInterpreted(t itemType) error { s, err := l.buffer.AsInterpretedString() if err != nil { return err } l.emit(t, s) return nil } // emitError emits a lexer error item back to the client. func (l *Lexer) emitError(message string) { l.emit(ItemError, message) } // backup steps back one rune // Can be called only once per call of next. func (l *Lexer) backup() { l.pos -= l.width l.linepos-- } // peek returns but does not advance to the next rune(s) in the input. // Returns the rune, its width and a boolean. The boolean will be false in case // no upcoming rune can be peeked (end of data or invalid UTF8 character). func (l *Lexer) peek() (rune, int, bool) { r, w := utf8.DecodeRuneInString(l.input[l.pos:]) switch { case r == utf8.RuneError: return utf8.RuneError, w, false default: return r, w, true } } // peekMulti takes a peek at multiple upcoming runes in the input. // Returns a slice of runes and a boolean. The boolean will be false in case // less upcoming runes can be peeked than the requested amount // (end of data or invalid UTF8 character). func (l *Lexer) peekMulti(amount int) ([]rune, int, bool) { width := 0 var peeked []rune for i := 0; i < amount; i++ { r, w := utf8.DecodeRuneInString(l.input[l.pos+width:]) switch { case r == utf8.RuneError: return peeked, 0, false default: width += w peeked = append(peeked, r) } } return peeked, width, true } // acceptNext adds the specified amount of runes from the input to the string buffer. // If not enough runes could be read (end of file or invalid UTF8 data), then false is returned. func (l *Lexer) acceptNext(count int) bool { for i := 0; i < count; i++ { if r, ok := l.next(); ok { l.buffer.WriteRune(r) } else { return false } } return true } // acceptFrom adds the next rune from the input to the string buffer // when it matches in the provided runes. If the next rune does // not match, false is returned. // func (l *Lexer) acceptFrom(runes string) bool { // r, ok := l.next() // if strings.IndexRune(runes, r) >= 0 { // l.buffer.WriteRune(r) // return true // } // l.backup() // return false // } // acceptRun adds consecutive runes from the input to the string // buffer when they match the provided runes. If no runes were added // at all, false it returned. func (l *Lexer) acceptRun(match string) bool { accepted := false for l.accept(match) { accepted = true } return accepted } // TODO meh... ugly rune. var endOfFile rune = -1 // next returns the next rune from the input and a boolean indicating if // reading the input was successful. // When the end of input is reached, or an invalid UTF8 character is // read, then false is returned. func (l *Lexer) next() (rune, bool) { if l.newline { l.linepos = 0 l.linenr++ } else { l.linepos++ } l.width = 0 r, w := utf8.DecodeRuneInString(l.input[l.pos:]) switch { case r == utf8.RuneError && w == 0: l.emitError("unexpected end of file") return utf8.RuneError, false case r == utf8.RuneError: l.emitError("invalid UTF8 character") return utf8.RuneError, false default: l.width = w l.pos += w l.newline = r == '\n' return r, true } } // skip skips runes when all provided matches are satisfied. // Returns true when one or more runes were skipped. func (l *Lexer) skipMatching(matches ...string) bool { if _, w, ok := l.match(matches...); ok { l.pos += w return true } return false } // skipConsecutive skips consecutive runes from the provided match. // Returns true when one or more runes were skipped. func (l *Lexer) skipConsecutive(match string) bool { didSkip := false for l.skipMatching(match) { didSkip = true } return didSkip } // accept adds the next rune to the string buffer and returns true if it's // from the valid set of runes. Otherwise false is returned. func (l *Lexer) accept(match string) bool { if r, ok := l.next(); ok { if strings.IndexRune(match, r) >= 0 { l.buffer.WriteRune(r) return true } } l.backup() return false } func (l *Lexer) upcoming(runes ...string) bool { _, _, ok := l.match(runes...) return ok } func (l *Lexer) match(matches ...string) ([]rune, int, bool) { peeked, width, ok := l.peekMulti(len(matches)) if ok { for i, r := range matches { if strings.IndexRune(r, peeked[i]) < 0 { return peeked, width, false } } return peeked, width, true } return peeked, width, false } // error returns an error token and terminates the scan // by returning nil to l.run. func (l *Lexer) errorf(format string, args ...interface{}) stateFn { l.items <- Item{ ItemError, fmt.Sprintf(format, args...), } return nil } func (l *Lexer) unexpectedInputError(expected string) stateFn { // next() takes care of error messages for ok == false. if r, ok := l.next(); ok { l.emitError(fmt.Sprintf("unexpected character %q (expected %s)", r, expected)) } return nil } func (l *Lexer) unexpectedEndOfFile(expected string) stateFn { return l.errorf("Unexpected end of file (expected %s)", expected) }