go-toml/lexer/lexer.go

package lexer

import (
	"fmt"
	"strings"
	"unicode/utf8"
)

// Lexer holds the state of the lexer.
type Lexer struct {
	input    string       // the scanned input string
	state    stateFn      // a function that handles the current state
	stack    []stateFn    // state function stack, for nested parsing
	pos      int          // current byte scanning position in the input
	newline  bool         // keep track of when we have scanned a newline
	linenr   int          // current line number in the input
	linepos  int          // current position in the input line
	width    int          // width of the last rune read, for supporting backup()
	buffer   StringBuffer // an efficient buffer, used to build string values
	items    chan Item    // channel of resulting lexer items
	nextItem Item         // the current item as reached by Next() and retrieved by Get()
	err      *Error       // an error when lexing failed, retrieved by Error()
}

// Error is used as the error type when lexing errors occur.
// The error includes some extra meta information to allow for useful
// error messages to the user.
type Error struct {
	Message string
	LineNr  int
	LinePos int
}

func (err *Error) Error() string {
	return err.Message
}

// Lex takes an input string and initializes the TOML lexer for it.
// Usage:
//
//     l := lexer.Lex("...inputstring...")
//     for l.Next() {
//         item := l.Get()
//         ... handle item ...
//     }
//     if e := l.Error(); e != nil {
//         ... handle error message ...
//     }
func Lex(input string) *Lexer {
	return &Lexer{
		input: input,
		state: stateKeyValuePair,
		items: make(chan Item, 2),
	}
}

// Next advances to the next lexer item in the input string.
// When a next item was found, then true is returned.
// On error or reaching the end of the input, false is returned.
func (l *Lexer) Next() bool {
	if l.state == nil {
		panic("This should not happen: nil state reached, but entering Next()")
	}
	for {
		select {
		case i := <-l.items:
			if i.Type == ItemEOF {
				return false
			}
			if i.Type == ItemError {
				l.err = &Error{i.Value, l.linenr, l.linepos}
				return false
			}
			l.nextItem = i
			return true
		default:
			l.state = l.state(l)
		}
	}
}

func (l *Lexer) Error() *Error {
	return l.err
}

// Get returns the next lexer item, as reached by Next()
func (l *Lexer) Get() Item {
	return l.nextItem
}

// ToArray returns lexer items as an array.
// When an error occurs during scanning, a partial result will be
// returned, accompanied by the error that occurred.
func (l *Lexer) ToArray() ([]Item, *Error) {
	var items []Item
	for l.Next() {
		items = append(items, l.Get())
	}
	return items, l.Error()
}

// pushState adds the state function to its stack.
// This is used for implementing nested parsing.
func (l *Lexer) pushState(state stateFn) {
	l.stack = append(l.stack, state)
}

// popState pops the last pushed state from its stack.
func (l *Lexer) popState() stateFn {
	last := len(l.stack) - 1
	head, tail := l.stack[:last], l.stack[last]
	l.stack = head
	return tail
}

// atEndOfFile returns true when there is no more data available in the input.
func (l *Lexer) atEndOfFile() bool {
	return l.pos >= len(l.input)
}

// emit passes a lexer item back to the client, including the provided string.
func (l *Lexer) emit(t itemType, s string) {
	l.items <- Item{t, s}
	l.buffer.Reset()
}

// emitLiteral passes a lexer item back to the client, including the accumulated
// string buffer data as a literal string.
func (l *Lexer) emitLiteral(t itemType) {
	l.emit(t, l.buffer.AsLiteralString())
}

// emitTrimmedLiteral passes a lexer item back to the client, including the
// accumulated string buffer data as a literal string with whitespace
// trimmed from it.
func (l *Lexer) emitTrimmedLiteral(t itemType) {
	l.emit(t, strings.TrimSpace(l.buffer.AsLiteralString()))
}

// emitInterpreted passes a lexer item back to the client, including the
// accumulated string buffer data an interpreted string (handling escape
// codes like \n, \t, \uXXXX, etc.)
// This method might return an error, in case there is data in the
// string buffer that is not valid for string interpretation.
func (l *Lexer) emitInterpreted(t itemType) error {
	s, err := l.buffer.AsInterpretedString()
	if err != nil {
		return err
	}
	l.emit(t, s)
	return nil
}

// emitError emits a lexer error item back to the client.
func (l *Lexer) emitError(message string) {
	l.emit(ItemError, message)
}

// backup steps back one rune
// Can be called only once per call of next.
func (l *Lexer) backup() {
	l.pos -= l.width
	l.linepos--
}

// peek returns but does not advance to the next rune(s) in the input.
// Returns the rune, its width and a boolean. The boolean will be false in case
// no upcoming rune can be peeked (end of data or invalid UTF8 character).
func (l *Lexer) peek() (rune, int, bool) {
	r, w := utf8.DecodeRuneInString(l.input[l.pos:])
	return r, w, r != utf8.RuneError
}

// peekMulti takes a peek at multiple upcoming runes in the input.
// Returns a slice of runes and a boolean. The boolean will be false in case
// less upcoming runes can be peeked than the requested amount
// (end of data or invalid UTF8 character).
func (l *Lexer) peekMulti(amount int) ([]rune, int, bool) {
	width := 0
	var peeked []rune
	for i := 0; i < amount; i++ {
		r, w := utf8.DecodeRuneInString(l.input[l.pos+width:])
		switch {
		case r == utf8.RuneError:
			return peeked, width, false
		default:
			width += w
			peeked = append(peeked, r)
		}
	}
	return peeked, width, true
}

// acceptNext adds the specified amount of runes from the input to the string buffer.
// If not enough runes could be read (end of file or invalid UTF8 data), then false is returned.
func (l *Lexer) acceptNext(count int) bool {
	for i := 0; i < count; i++ {
		if r, ok := l.next(); ok {
			l.buffer.WriteRune(r)
		} else {
			return false
		}
	}
	return true
}

// acceptConsecutive adds consecutive runes from the input to the string
// buffer when they match the rune match.
// If any runes were added then true is returned, false otherwise.
func (l *Lexer) acceptConsecutive(match string) bool {
	accepted := false
	for l.accept(match) {
		accepted = true
	}
	return accepted
}

// next returns the next rune from the input and a boolean indicating if
// reading the input was successful.
// When the end of input is reached, or an invalid UTF8 character is
// read, then false is returned.
func (l *Lexer) next() (rune, bool) {
	r, w, ok := l.peek()
	if ok {
		l.width = w
		l.pos += w
		l.advanceCursor(r)
		return r, true
	}
	l.width = 0
	if r == utf8.RuneError && w == 0 {
		l.emitError("unexpected end of file")
	} else {
		l.emitError("invalid UTF8 character")
	}
	return r, false
}

func (l *Lexer) advanceCursor(r rune) {
	if l.newline {
		l.linepos = 0
		l.linenr++
	} else {
		l.linepos++
	}
	l.newline = r == '\n'
}

// skip skips runes, but only when all provided matches are satisfied.
// Returns true when one or more runes were skipped.
func (l *Lexer) skipMatching(matches ...string) bool {
	if runes, w, ok := l.match(matches...); ok {
		l.pos += w
		for _, r := range runes {
			l.advanceCursor(r)
		}
		return true
	}
	return false
}

// skipConsecutive skips consecutive runes from the provided match.
// Returns true when one or more runes were skipped.
func (l *Lexer) skipConsecutive(match string) bool {
	didSkip := false
	for l.skipMatching(match) {
		didSkip = true
	}
	return didSkip
}

// accept adds the next rune to the string buffer and returns true if it's
// from the valid set of runes. Otherwise false is returned.
func (l *Lexer) accept(match string) bool {
	if r, ok := l.next(); ok {
		if strings.IndexRune(match, r) >= 0 {
			l.buffer.WriteRune(r)
			return true
		}
	}
	l.backup()
	return false
}

// upcoming checks if the upcoming runes satisfy the provided rune matches.
// This is a lot like the match method, with the difference that
// this one only returns the boolean value.
func (l *Lexer) upcoming(matches ...string) bool {
	_, _, ok := l.match(matches...)
	return ok
}

// match checks if the upcoming runes satisfy the provided rune matches.
// It returns a slice of runes that were found, their total byte width
// and a boolean indicating whether or not all provided matches matched
// the input data.
func (l *Lexer) match(matches ...string) ([]rune, int, bool) {
	peeked, width, ok := l.peekMulti(len(matches))
	if ok {
		for i, r := range matches {
			if strings.IndexRune(r, peeked[i]) < 0 {
				return peeked, width, false
			}
		}
		return peeked, width, true
	}
	return peeked, width, false
}

// error returns an error token and terminates the scan
// by returning nil to l.run.
func (l *Lexer) errorf(format string, args ...interface{}) stateFn {
	l.items <- Item{
		ItemError,
		fmt.Sprintf(format, args...),
	}
	return nil
}

func (l *Lexer) unexpectedInputError(expected string) stateFn {
	// next() takes care of error messages for ok == false.
	if r, ok := l.next(); ok {
		l.emitError(fmt.Sprintf("unexpected character %q (expected %s)", r, expected))
	}
	return nil
}

func (l *Lexer) unexpectedEndOfFile(expected string) stateFn {
	return l.errorf("Unexpected end of file (expected %s)", expected)
}