Removed some source files that are not used anymore, after the refactoring steps.
This commit is contained in:
parent
f86ef2b918
commit
db4a8f7942
305
lexer/lexer.no
305
lexer/lexer.no
|
|
@ -1,305 +0,0 @@
|
|||
package lexer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/mmakaay/toml/parser"
|
||||
)
|
||||
|
||||
// Lexer holds the state of the lexer.
|
||||
type Lexer struct {
|
||||
input string // the scanned input
|
||||
state parser.StateFn // a function that handles the current state
|
||||
stack []parser.StateFn // state function stack, for nested parsing
|
||||
len int // the total length of the input in bytes
|
||||
pos int // current byte scanning position in the input
|
||||
newline bool // keep track of when we have scanned a newline
|
||||
cursorRow int // current row number in the input
|
||||
cursorColumn int // current column position in the input
|
||||
buffer StringBuffer // an efficient buffer, used to build string values
|
||||
items chan parser.Item // channel of resulting lexer items
|
||||
item parser.Item // the current item as reached by Next() and retrieved by Get()
|
||||
err *Error // an error when lexing failed, retrieved by Error()
|
||||
}
|
||||
|
||||
// Error is used as the error type when lexing errors occur.
|
||||
// The error includes some extra meta information to allow for useful
|
||||
// error messages to the user.
|
||||
type Error struct {
|
||||
Message string
|
||||
Row int
|
||||
Column int
|
||||
}
|
||||
|
||||
func (err *Error) Error() string {
|
||||
return err.Message
|
||||
}
|
||||
|
||||
// New takes an input string and initializes the lexer for it.
|
||||
func New(input string) *Lexer {
|
||||
return &Lexer{
|
||||
input: input,
|
||||
len: len(input),
|
||||
state: stateKeyValuePair,
|
||||
items: make(chan parser.Item, 2),
|
||||
}
|
||||
}
|
||||
|
||||
// Next advances to the next lexer item in the input string.
|
||||
// When a valid item was found, then the boolean return parameter will be true.
|
||||
// On error or when reaching the end of the input, false is returned.
|
||||
// When an error occurred, it will be set in the error return value, nil otherwise.
|
||||
func (l *Lexer) Next() (parser.Item, *Error, bool) {
|
||||
for {
|
||||
select {
|
||||
case i := <-l.items:
|
||||
switch {
|
||||
case i.Type == ItemEOF:
|
||||
return i, nil, false
|
||||
case i.Type == ItemError:
|
||||
l.err = &Error{i.Value, l.cursorRow, l.cursorColumn}
|
||||
return i, l.err, false
|
||||
default:
|
||||
l.item = i
|
||||
return i, nil, true
|
||||
}
|
||||
default:
|
||||
l.state = l.state(l)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ToArray returns lexer items as an array (mainly intended for testing purposes)
|
||||
// When an error occurs during scanning, a partial result will be
|
||||
// returned, accompanied by the error that occurred.
|
||||
func (l *Lexer) ToArray() ([]parser.Item, *Error) {
|
||||
var items []parser.Item
|
||||
for {
|
||||
item, err, more := l.Next()
|
||||
if !more {
|
||||
return items, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
|
||||
// pushState adds the state function to its stack.
|
||||
// This is used for implementing nested parsing.
|
||||
func (l *Lexer) pushState(state stateFn) {
|
||||
l.stack = append(l.stack, state)
|
||||
}
|
||||
|
||||
// popState pops the last pushed state from its stack.
|
||||
func (l *Lexer) popState() stateFn {
|
||||
last := len(l.stack) - 1
|
||||
head, tail := l.stack[:last], l.stack[last]
|
||||
l.stack = head
|
||||
return tail
|
||||
}
|
||||
|
||||
// atEndOfFile returns true when there is no more data available in the input.
|
||||
func (l *Lexer) atEndOfFile() bool {
|
||||
return l.pos >= l.len
|
||||
}
|
||||
|
||||
// emit passes a lexer item back to the client, including the provided string.
|
||||
func (l *Lexer) emit(t parser.ItemType, s string) {
|
||||
l.items <- parser.Item{Type: t, Value: s}
|
||||
l.buffer.Reset()
|
||||
}
|
||||
|
||||
// emitLiteral passes a lexer item back to the client, including the accumulated
|
||||
// string buffer data as a literal string.
|
||||
func (l *Lexer) emitLiteral(t parser.ItemType) {
|
||||
l.emit(t, l.buffer.AsLiteralString())
|
||||
}
|
||||
|
||||
// emitTrimmedLiteral passes a lexer item back to the client, including the
|
||||
// accumulated string buffer data as a literal string with whitespace
|
||||
// trimmed from it.
|
||||
func (l *Lexer) emitTrimmedLiteral(t parser.ItemType) {
|
||||
l.emit(t, strings.TrimSpace(l.buffer.AsLiteralString()))
|
||||
}
|
||||
|
||||
// emitInterpreted passes a lexer item back to the client, including the
|
||||
// accumulated string buffer data an interpreted string (handling escape
|
||||
// codes like \n, \t, \uXXXX, etc.)
|
||||
// This method might return an error, in case there is data in the
|
||||
// string buffer that is not valid for string interpretation.
|
||||
func (l *Lexer) emitInterpreted(t parser.ItemType) error {
|
||||
s, err := l.buffer.AsInterpretedString()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
l.emit(t, s)
|
||||
return nil
|
||||
}
|
||||
|
||||
// emitError emits a lexer error item back to the client.
|
||||
func (l *Lexer) emitError(format string, args ...interface{}) stateFn {
|
||||
message := fmt.Sprintf(format, args...)
|
||||
l.emit(ItemError, message)
|
||||
return nil
|
||||
}
|
||||
|
||||
// peek returns but does not advance to the next rune(s) in the input.
|
||||
// Returns the rune, its width and a boolean. The boolean will be false in case
|
||||
// no upcoming rune can be peeked (end of data or invalid UTF8 character).
|
||||
func (l *Lexer) peek() (rune, int, bool) {
|
||||
r, w := utf8.DecodeRuneInString(l.input[l.pos:])
|
||||
return r, w, r != utf8.RuneError
|
||||
}
|
||||
|
||||
// peekMulti takes a peek at multiple upcoming runes in the input.
|
||||
// Returns a slice of runes and a boolean. The boolean will be false in case
|
||||
// less upcoming runes can be peeked than the requested amount
|
||||
// (end of data or invalid UTF8 character).
|
||||
func (l *Lexer) peekMulti(amount int) ([]rune, int, bool) {
|
||||
width := 0
|
||||
var peeked []rune
|
||||
for i := 0; i < amount; i++ {
|
||||
r, w := utf8.DecodeRuneInString(l.input[l.pos+width:])
|
||||
switch {
|
||||
case r == utf8.RuneError:
|
||||
return peeked, width, false
|
||||
default:
|
||||
width += w
|
||||
peeked = append(peeked, r)
|
||||
}
|
||||
}
|
||||
return peeked, width, true
|
||||
}
|
||||
|
||||
// acceptAny adds the next rune from the input to the string buffer.
|
||||
// If no rune could be read (end of file or invalid UTF8 data), then
|
||||
// false is returned.
|
||||
func (l *Lexer) acceptAny() bool {
|
||||
if r, ok := l.next(); ok {
|
||||
l.buffer.WriteRune(r)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// accept adds the next rune to the string buffer and returns true if it's
|
||||
// from the valid set of runes. Otherwise false is returned.
|
||||
func (l *Lexer) accept(matches ...string) bool {
|
||||
return l.acceptPattern(matches...)
|
||||
}
|
||||
|
||||
// AcceptMatching adds the next runes to the string buffer, but only
|
||||
// if the upcoming runes satisfy the provided pattern.
|
||||
// When runes were added then true is returned, false otherwise.
|
||||
func (l *Lexer) acceptPattern(pattern ...string) bool {
|
||||
return l.progress(func(r rune) { l.buffer.WriteRune(r) }, pattern...)
|
||||
}
|
||||
|
||||
func (l *Lexer) progress(callback func(rune), matches ...string) bool {
|
||||
if runes, w, ok := l.match(matches...); ok {
|
||||
l.pos += w
|
||||
for _, r := range runes {
|
||||
callback(r)
|
||||
l.advanceCursor(r)
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// acceptConsecutive adds consecutive runes from the input to the string
|
||||
// buffer when they match the rune match.
|
||||
// If any runes were added then true is returned, false otherwise.
|
||||
func (l *Lexer) acceptConsecutive(match string) bool {
|
||||
accepted := false
|
||||
for l.accept(match) {
|
||||
accepted = true
|
||||
}
|
||||
return accepted
|
||||
}
|
||||
|
||||
// advanceCursor advances the rune cursor one position in the
|
||||
// input data. While doing so, it keeps tracks of newlines,
|
||||
// so we can report on row + column positions on error.
|
||||
func (l *Lexer) advanceCursor(r rune) {
|
||||
if l.newline {
|
||||
l.cursorColumn = 0
|
||||
l.cursorRow++
|
||||
} else {
|
||||
l.cursorColumn++
|
||||
}
|
||||
l.newline = r == '\n'
|
||||
}
|
||||
|
||||
// skip skips runes, but only when all provided matches are satisfied.
|
||||
// Returns true when one or more runes were skipped.
|
||||
func (l *Lexer) skipMatching(pattern ...string) bool {
|
||||
return l.progress(func(r rune) {}, pattern...)
|
||||
}
|
||||
|
||||
// skipConsecutive skips consecutive runes from the provided match.
|
||||
// Returns true when one or more runes were skipped.
|
||||
func (l *Lexer) skipConsecutive(pattern string) bool {
|
||||
didSkip := false
|
||||
for l.skipMatching(pattern) {
|
||||
didSkip = true
|
||||
}
|
||||
return didSkip
|
||||
}
|
||||
|
||||
// upcoming checks if the upcoming runes satisfy the provided rune matches.
|
||||
// This is a lot like the match method, with the difference that
|
||||
// this one only returns the boolean value.
|
||||
func (l *Lexer) upcoming(matches ...string) bool {
|
||||
_, _, ok := l.match(matches...)
|
||||
return ok
|
||||
}
|
||||
|
||||
// next returns the next rune from the input and a boolean indicating if
|
||||
// reading the input was successful.
|
||||
// When the end of input is reached, or an invalid UTF8 character is
|
||||
// read, then false is returned.
|
||||
func (l *Lexer) next() (rune, bool) {
|
||||
r, w, ok := l.peek()
|
||||
if ok {
|
||||
l.pos += w
|
||||
l.advanceCursor(r)
|
||||
return r, true
|
||||
}
|
||||
if r == utf8.RuneError && w == 0 {
|
||||
l.emitError("unexpected end of file")
|
||||
} else {
|
||||
l.emitError("invalid UTF8 character")
|
||||
}
|
||||
return r, false
|
||||
}
|
||||
|
||||
// match checks if the upcoming runes satisfy the provided rune matches.
|
||||
// It returns a slice of runes that were found, their total byte width
|
||||
// and a boolean indicating whether or not all provided matches matched
|
||||
// the input data.
|
||||
func (l *Lexer) match(matches ...string) ([]rune, int, bool) {
|
||||
peeked, width, ok := l.peekMulti(len(matches))
|
||||
if ok {
|
||||
for i, r := range matches {
|
||||
if strings.IndexRune(r, peeked[i]) < 0 {
|
||||
return peeked, width, false
|
||||
}
|
||||
}
|
||||
return peeked, width, true
|
||||
}
|
||||
return peeked, width, false
|
||||
}
|
||||
|
||||
func (l *Lexer) unexpectedInputError(expected string) stateFn {
|
||||
// next() takes care of emitting errors for ok == false.
|
||||
if r, ok := l.next(); ok {
|
||||
return l.emitError(fmt.Sprintf("unexpected character %q (expected %s)", r, expected))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *Lexer) unexpectedEndOfFile(expected string) stateFn {
|
||||
return l.emitError("Unexpected end of file (expected %s)", expected)
|
||||
}
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
package lexer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// StringBuffer is a string buffer implementation, which is used by the lexer
|
||||
// to efficiently accumulate runes from the input and eventually turn these
|
||||
// into a string, either literal or interpreted.
|
||||
type StringBuffer struct {
|
||||
buffer bytes.Buffer
|
||||
}
|
||||
|
||||
// Reset resets the string buffer, in order to build a new string.
|
||||
func (b *StringBuffer) Reset() *StringBuffer {
|
||||
b.buffer.Reset()
|
||||
return b
|
||||
}
|
||||
|
||||
// WriteString adds the runes of the input string to the string buffer.
|
||||
func (b *StringBuffer) WriteString(s string) *StringBuffer {
|
||||
for _, r := range s {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// WriteRune adds a single rune to the string buffer.
|
||||
func (b *StringBuffer) WriteRune(r rune) *StringBuffer {
|
||||
b.buffer.WriteRune(r)
|
||||
return b
|
||||
}
|
||||
|
||||
// AsLiteralString returns the string buffer as a literal string.
|
||||
// Literal means that no escape sequences are processed.
|
||||
func (b *StringBuffer) AsLiteralString() string {
|
||||
return b.buffer.String()
|
||||
}
|
||||
|
||||
// AsInterpretedString returns the string in its interpreted form.
|
||||
// Interpreted means that escape sequences are handled in the way that Go would
|
||||
// have, had it been inside double quotes. It translates for example escape
|
||||
// sequences like "\n", "\t", \uXXXX" and "\UXXXXXXXX" into their string
|
||||
// representations.
|
||||
// Since the input might contain invalid escape sequences, this method
|
||||
// also returns an error. When an error is returned, the returned string will
|
||||
// contain the string as far as it could be interpreted.
|
||||
func (b *StringBuffer) AsInterpretedString() (string, error) {
|
||||
var sb strings.Builder
|
||||
tail := b.buffer.String()
|
||||
for len(tail) > 0 {
|
||||
r, _, newtail, err := strconv.UnquoteChar(tail, '"')
|
||||
if err != nil {
|
||||
return sb.String(), err
|
||||
}
|
||||
tail = newtail
|
||||
sb.WriteRune(r)
|
||||
}
|
||||
return sb.String(), nil
|
||||
}
|
||||
|
|
@ -1,87 +0,0 @@
|
|||
package lexer_test
|
||||
|
||||
import "testing"
|
||||
import "github.com/mmakaay/toml/lexer"
|
||||
|
||||
func TestGeneratingStringDoesNotResetBuffer(t *testing.T) {
|
||||
var b lexer.StringBuffer
|
||||
s1, _ := b.WriteString(`hi\nthere`).AsInterpretedString()
|
||||
s2 := b.AsLiteralString()
|
||||
if s1 != "hi\nthere" {
|
||||
t.Fatalf("Did not get expected string\"X\" for try 1, but %q", s1)
|
||||
}
|
||||
if s2 != "hi\\nthere" {
|
||||
t.Fatalf("Did not get expected string\"X\" for try 2, but %q", s2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetResetsBuffer(t *testing.T) {
|
||||
var b lexer.StringBuffer
|
||||
s := b.WriteRune('X').Reset().AsLiteralString()
|
||||
if s != "" {
|
||||
t.Fatalf("Did not get expected empty string, but %q", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAsLiteralString(t *testing.T) {
|
||||
b := lexer.StringBuffer{}
|
||||
for _, c := range []stringbufT{
|
||||
{"empty string", ``, ``, OK},
|
||||
{"simple string", `Simple string!`, `Simple string!`, OK},
|
||||
{"single quote", `'`, `'`, OK},
|
||||
{"double quote", `"`, `"`, OK},
|
||||
{"escaped single quote", `\'`, `\'`, OK},
|
||||
{"escaped double quote", `\"`, `\"`, OK},
|
||||
{"escape anything", `\x\t\f\n\r\'\"\\`, `\x\t\f\n\r\'\"\\`, OK},
|
||||
{"UTF8 escapes", `\uceb2\U00e0b8bf`, `\uceb2\U00e0b8bf`, OK},
|
||||
{"actual newline", "on\nmultiple\nlines", "on\nmultiple\nlines", OK},
|
||||
} {
|
||||
s := b.Reset().WriteString(c.in).AsLiteralString()
|
||||
if s != c.out {
|
||||
t.Fatalf("[%s] %q -> %q failed: actual result = %q", c.name, c.in, c.out, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAsInterpretedString(t *testing.T) {
|
||||
b := lexer.StringBuffer{}
|
||||
for _, c := range []stringbufT{
|
||||
{"empty string", "", "", OK},
|
||||
{"one character", "Simple string!", "Simple string!", OK},
|
||||
{"escaped single quote", `\'`, "", FAIL},
|
||||
{"escaped double quote", `\"`, `"`, OK},
|
||||
{"bare single quote", `'`, "'", OK},
|
||||
{"string in single quotes", `'Hello'`, `'Hello'`, OK},
|
||||
{"string in escaped double quotes", `\"Hello\"`, `"Hello"`, OK},
|
||||
{"escape something", `\t\f\n\r\"\\`, "\t\f\n\r\"\\", OK},
|
||||
{"short UTF8 escapes", `\u2318Wh\u00e9\u00e9!`, `⌘Whéé!`, OK},
|
||||
{"long UTF8 escapes", `\U0001014D \u2318 Wh\u00e9\u00e9!`, `𐅍 ⌘ Whéé!`, OK},
|
||||
{"UTF8 characters", "Ѝюج wut Ж ?", "Ѝюج wut Ж ?", OK},
|
||||
{"example from spec",
|
||||
`I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF.`,
|
||||
"I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF.", OK},
|
||||
} {
|
||||
s, err := b.Reset().WriteString(c.in).AsInterpretedString()
|
||||
if c.isSuccessCase && err != nil {
|
||||
t.Fatalf("[%s] unexpected error for input %q: %s", c.name, c.in, err)
|
||||
}
|
||||
if !c.isSuccessCase && err == nil {
|
||||
t.Fatalf("[%s] expected a failure, but no failure occurred", c.name)
|
||||
}
|
||||
if s != c.out && c.isSuccessCase {
|
||||
t.Fatalf("[%s] %q -> %q failed: actual result = %q", c.name, c.in, c.out, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type stringbufT struct {
|
||||
name string
|
||||
in string
|
||||
out string
|
||||
isSuccessCase bool
|
||||
}
|
||||
|
||||
const (
|
||||
OK bool = true
|
||||
FAIL bool = false
|
||||
)
|
||||
Loading…
Reference in New Issue