From aeb48edc441d7582f3ec77e8afd56ff1e375edff Mon Sep 17 00:00:00 2001 From: Maurice Makaay Date: Fri, 17 May 2019 00:34:00 +0000 Subject: [PATCH] A little round of code cleanup. --- lexer/items.go | 4 --- lexer/lexer.go | 81 ++++++++++++++++++-------------------------- lexer/states_test.go | 8 ++--- 3 files changed, 37 insertions(+), 56 deletions(-) diff --git a/lexer/items.go b/lexer/items.go index a6a86a4..e64515f 100644 --- a/lexer/items.go +++ b/lexer/items.go @@ -25,8 +25,6 @@ type Item struct { // String returns a string representation of the lexer item. func (i Item) String() string { switch i.Type { - case ItemEOF: - return "EOF" case ItemKey: return fmt.Sprintf("[%s]", i.Value) case ItemKeyDot: @@ -40,8 +38,6 @@ func (i Item) String() string { // String returns a string representation of the lexer item type. func (i itemType) String() string { switch i { - case ItemError: - return "ERR" case ItemComment: return "#" case ItemString: diff --git a/lexer/lexer.go b/lexer/lexer.go index fc54969..1ca4b06 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -8,18 +8,18 @@ import ( // Lexer holds the state of the lexer. type Lexer struct { - input string // the scanned input string - state stateFn // a function that handles the current state - stack []stateFn // state function stack, for nested parsing - pos int // current byte scanning position in the input - newline bool // keep track of when we have scanned a newline - linenr int // current line number in the input - linepos int // current position in the input line - width int // width of the last rune read, for supporting backup() - buffer StringBuffer // an efficient buffer, used to build string values - items chan Item // channel of resulting lexer items - nextItem Item // the current item as reached by Next() and retrieved by Get() - err *Error // an error when lexing failed, retrieved by Error() + input string // the scanned input string + state stateFn // a function that handles the current state + stack []stateFn // state function stack, for nested parsing + pos int // current byte scanning position in the input + newline bool // keep track of when we have scanned a newline + cursorRow int // current row number in the input + cursorColumn int // current column position in the input + width int // width of the last rune read, for supporting backup() + buffer StringBuffer // an efficient buffer, used to build string values + items chan Item // channel of resulting lexer items + item Item // the current item as reached by Next() and retrieved by Get() + err *Error // an error when lexing failed, retrieved by Error() } // Error is used as the error type when lexing errors occur. @@ -27,8 +27,8 @@ type Lexer struct { // error messages to the user. type Error struct { Message string - LineNr int - LinePos int + Row int + Column int } func (err *Error) Error() string { @@ -36,16 +36,6 @@ func (err *Error) Error() string { } // Lex takes an input string and initializes the TOML lexer for it. -// Usage: -// -// l := lexer.Lex("...inputstring...") -// for l.Next() { -// item := l.Get() -// ... handle item ... -// } -// if e := l.Error(); e != nil { -// ... handle error message ... -// } func Lex(input string) *Lexer { return &Lexer{ input: input, @@ -55,9 +45,10 @@ func Lex(input string) *Lexer { } // Next advances to the next lexer item in the input string. -// When a next item was found, then true is returned. -// On error or reaching the end of the input, false is returned. -func (l *Lexer) Next() bool { +// When a valid item was found, then the boolean return parameter is returned. +// On error or when reaching the end of the input, false is returned. +// When an error occurred, it will be set in the error return value. +func (l *Lexer) Next() (Item, *Error, bool) { if l.state == nil { panic("This should not happen: nil state reached, but entering Next()") } @@ -65,38 +56,32 @@ func (l *Lexer) Next() bool { select { case i := <-l.items: if i.Type == ItemEOF { - return false + return i, nil, false } if i.Type == ItemError { - l.err = &Error{i.Value, l.linenr, l.linepos} - return false + l.err = &Error{i.Value, l.cursorRow, l.cursorColumn} + return i, l.err, false } - l.nextItem = i - return true + l.item = i + return i, nil, true default: l.state = l.state(l) } } } -func (l *Lexer) Error() *Error { - return l.err -} - -// Get returns the next lexer item, as reached by Next() -func (l *Lexer) Get() Item { - return l.nextItem -} - // ToArray returns lexer items as an array. // When an error occurs during scanning, a partial result will be // returned, accompanied by the error that occurred. func (l *Lexer) ToArray() ([]Item, *Error) { var items []Item - for l.Next() { - items = append(items, l.Get()) + for { + item, err, more := l.Next() + if !more { + return items, err + } + items = append(items, item) } - return items, l.Error() } // pushState adds the state function to its stack. @@ -160,7 +145,7 @@ func (l *Lexer) emitError(message string) { // Can be called only once per call of next. func (l *Lexer) backup() { l.pos -= l.width - l.linepos-- + l.cursorColumn-- } // peek returns but does not advance to the next rune(s) in the input. @@ -238,10 +223,10 @@ func (l *Lexer) next() (rune, bool) { func (l *Lexer) advanceCursor(r rune) { if l.newline { - l.linepos = 0 - l.linenr++ + l.cursorColumn = 0 + l.cursorRow++ } else { - l.linepos++ + l.cursorColumn++ } l.newline = r == '\n' } diff --git a/lexer/states_test.go b/lexer/states_test.go index 1050a06..b72b9a8 100644 --- a/lexer/states_test.go +++ b/lexer/states_test.go @@ -11,11 +11,11 @@ import ( func TestErrorsIncludeLineAndRowPosition(t *testing.T) { _, err := lexer.Lex("# 12345 abcde\t\n\n\n# 67890\r\n# 12345\xbc").ToArray() t.Logf("Got error: %s", err.Error()) - if err.LineNr != 4 { - t.Errorf("Unexpected line number: %d (expected %d)", err.LineNr, 4) + if err.Row != 4 { + t.Errorf("Unexpected line number: %d (expected %d)", err.Row, 4) } - if err.LinePos != 6 { - t.Errorf("Unexpected line position: %d (expected %d)", err.LinePos, 6) + if err.Column != 6 { + t.Errorf("Unexpected line position: %d (expected %d)", err.Column, 6) } }