A little round of code cleanup.

lexer/items.go

@@ -25,8 +25,6 @@ type Item struct {
 // String returns a string representation of the lexer item.
 func (i Item) String() string {
 	switch i.Type {
-	case ItemEOF:
-		return "EOF"
 	case ItemKey:
 		return fmt.Sprintf("[%s]", i.Value)
 	case ItemKeyDot:
@@ -40,8 +38,6 @@ func (i Item) String() string {
 // String returns a string representation of the lexer item type.
 func (i itemType) String() string {
 	switch i {
-	case ItemError:
-		return "ERR"
 	case ItemComment:
 		return "#"
 	case ItemString:

lexer/lexer.go

@@ -8,18 +8,18 @@ import (
 
 // Lexer holds the state of the lexer.
 type Lexer struct {
-	input    string       // the scanned input string
+	input        string       // the scanned input string
-	state    stateFn      // a function that handles the current state
+	state        stateFn      // a function that handles the current state
-	stack    []stateFn    // state function stack, for nested parsing
+	stack        []stateFn    // state function stack, for nested parsing
-	pos      int          // current byte scanning position in the input
+	pos          int          // current byte scanning position in the input
-	newline  bool         // keep track of when we have scanned a newline
+	newline      bool         // keep track of when we have scanned a newline
-	linenr   int          // current line number in the input
+	cursorRow    int          // current row number in the input
-	linepos  int          // current position in the input line
+	cursorColumn int          // current column position in the input
-	width    int          // width of the last rune read, for supporting backup()
+	width        int          // width of the last rune read, for supporting backup()
-	buffer   StringBuffer // an efficient buffer, used to build string values
+	buffer       StringBuffer // an efficient buffer, used to build string values
-	items    chan Item    // channel of resulting lexer items
+	items        chan Item    // channel of resulting lexer items
-	nextItem Item         // the current item as reached by Next() and retrieved by Get()
+	item         Item         // the current item as reached by Next() and retrieved by Get()
-	err      *Error       // an error when lexing failed, retrieved by Error()
+	err          *Error       // an error when lexing failed, retrieved by Error()
 }
 
 // Error is used as the error type when lexing errors occur.
@@ -27,8 +27,8 @@ type Lexer struct {
 // error messages to the user.
 type Error struct {
 	Message string
-	LineNr  int
+	Row     int
-	LinePos int
+	Column  int
 }
 
 func (err *Error) Error() string {
@@ -36,16 +36,6 @@ func (err *Error) Error() string {
 }
 
 // Lex takes an input string and initializes the TOML lexer for it.
-// Usage:
-//
-//     l := lexer.Lex("...inputstring...")
-//     for l.Next() {
-//         item := l.Get()
-//         ... handle item ...
-//     }
-//     if e := l.Error(); e != nil {
-//         ... handle error message ...
-//     }
 func Lex(input string) *Lexer {
 	return &Lexer{
 		input: input,
@@ -55,9 +45,10 @@ func Lex(input string) *Lexer {
 }
 
 // Next advances to the next lexer item in the input string.
-// When a next item was found, then true is returned.
+// When a valid item was found, then the boolean return parameter is returned.
-// On error or reaching the end of the input, false is returned.
+// On error or when reaching the end of the input, false is returned.
-func (l *Lexer) Next() bool {
+// When an error occurred, it will be set in the error return value.
+func (l *Lexer) Next() (Item, *Error, bool) {
 	if l.state == nil {
 		panic("This should not happen: nil state reached, but entering Next()")
 	}
@@ -65,38 +56,32 @@ func (l *Lexer) Next() bool {
 		select {
 		case i := <-l.items:
 			if i.Type == ItemEOF {
-				return false
+				return i, nil, false
 			}
 			if i.Type == ItemError {
-				l.err = &Error{i.Value, l.linenr, l.linepos}
+				l.err = &Error{i.Value, l.cursorRow, l.cursorColumn}
-				return false
+				return i, l.err, false
 			}
-			l.nextItem = i
+			l.item = i
-			return true
+			return i, nil, true
 		default:
 			l.state = l.state(l)
 		}
 	}
 }
 
-func (l *Lexer) Error() *Error {
-	return l.err
-}
-
-// Get returns the next lexer item, as reached by Next()
-func (l *Lexer) Get() Item {
-	return l.nextItem
-}
-
 // ToArray returns lexer items as an array.
 // When an error occurs during scanning, a partial result will be
 // returned, accompanied by the error that occurred.
 func (l *Lexer) ToArray() ([]Item, *Error) {
 	var items []Item
-	for l.Next() {
+	for {
-		items = append(items, l.Get())
+		item, err, more := l.Next()
+		if !more {
+			return items, err
+		}
+		items = append(items, item)
 	}
-	return items, l.Error()
 }
 
 // pushState adds the state function to its stack.
@@ -160,7 +145,7 @@ func (l *Lexer) emitError(message string) {
 // Can be called only once per call of next.
 func (l *Lexer) backup() {
 	l.pos -= l.width
-	l.linepos--
+	l.cursorColumn--
 }
 
 // peek returns but does not advance to the next rune(s) in the input.
@@ -238,10 +223,10 @@ func (l *Lexer) next() (rune, bool) {
 
 func (l *Lexer) advanceCursor(r rune) {
 	if l.newline {
-		l.linepos = 0
+		l.cursorColumn = 0
-		l.linenr++
+		l.cursorRow++
 	} else {
-		l.linepos++
+		l.cursorColumn++
 	}
 	l.newline = r == '\n'
 }

lexer/states_test.go

@@ -11,11 +11,11 @@ import (
 func TestErrorsIncludeLineAndRowPosition(t *testing.T) {
 	_, err := lexer.Lex("# 12345 abcde\t\n\n\n# 67890\r\n# 12345\xbc").ToArray()
 	t.Logf("Got error: %s", err.Error())
-	if err.LineNr != 4 {
+	if err.Row != 4 {
-		t.Errorf("Unexpected line number: %d (expected %d)", err.LineNr, 4)
+		t.Errorf("Unexpected line number: %d (expected %d)", err.Row, 4)
 	}
-	if err.LinePos != 6 {
+	if err.Column != 6 {
-		t.Errorf("Unexpected line position: %d (expected %d)", err.LinePos, 6)
+		t.Errorf("Unexpected line position: %d (expected %d)", err.Column, 6)
 	}
 }