Initial import, work in progress.
This commit is contained in:
Ubuntu
commit
f6efd34b31
|
|
@ -0,0 +1,54 @@
|
||||||
|
package lexer
|
||||||
|
|
||||||
|
import "fmt"
|
||||||
|
|
||||||
|
// itemType represents the type of lexer items.
|
||||||
|
type itemType int
|
||||||
|
|
||||||
|
// Definition of all the lexer item types for the TOML lexer.
|
||||||
|
const (
|
||||||
|
ItemError itemType = iota // An error occurred
|
||||||
|
ItemEOF // End of input reached
|
||||||
|
ItemComment // Comment string, starts with # till en of line
|
||||||
|
ItemKey // Key of a key/value pair
|
||||||
|
ItemKeyDot // Dot for a dotted key
|
||||||
|
ItemKeyValueAssignment // Equal sign for a key/value pair assignment
|
||||||
|
ItemStringValue // A value of type string
|
||||||
|
)
|
||||||
|
|
||||||
|
// Item represents a lexer item returned from the scanner.
|
||||||
|
type Item struct {
|
||||||
|
Type itemType //Type, e.g. itemNumber, itemSquareBracket
|
||||||
|
Value string // Value, e.g. "10.42", "["
|
||||||
|
}
|
||||||
|
|
||||||
|
// String returns a string representation of the lexer item.
|
||||||
|
func (i Item) String() string {
|
||||||
|
switch i.Type {
|
||||||
|
case ItemEOF:
|
||||||
|
return "EOF"
|
||||||
|
case ItemError:
|
||||||
|
return "Error: " + i.Value
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s(%q)", i.Type, i.Value)
|
||||||
|
}
|
||||||
|
|
||||||
|
// String returns a string representation of the lexer item type.
|
||||||
|
func (i itemType) String() string {
|
||||||
|
switch i {
|
||||||
|
case ItemError:
|
||||||
|
return "Error"
|
||||||
|
case ItemComment:
|
||||||
|
return "Comment"
|
||||||
|
case ItemKey:
|
||||||
|
return "Key"
|
||||||
|
case ItemKeyDot:
|
||||||
|
return "KeyDot"
|
||||||
|
case ItemKeyValueAssignment:
|
||||||
|
return "Assignment"
|
||||||
|
case ItemStringValue:
|
||||||
|
return "StringValue"
|
||||||
|
default:
|
||||||
|
return fmt.Sprintf("<type id %d>", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,244 @@
|
||||||
|
package lexer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Lexer holds the state of the scanner.
|
||||||
|
type Lexer struct {
|
||||||
|
input string // the scanned input string
|
||||||
|
state stateFn // the current state
|
||||||
|
stack []stateFn // state stack, for nested parsing
|
||||||
|
start int // start position of the currently scanned item
|
||||||
|
pos int // current scanning position in the input
|
||||||
|
width int // width of the last rune read
|
||||||
|
strValue strings.Builder // used to build string values
|
||||||
|
items chan Item // channel of scanned items
|
||||||
|
nextItem Item // the current item as reached by Next() and retrieved by Get()
|
||||||
|
err error // an error message when lexing failed, retrieved by Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Lex takes an input string and initializes the TOML lexer for it.
|
||||||
|
// Usage:
|
||||||
|
//
|
||||||
|
// l := lexer.Lex("...inputstring...")
|
||||||
|
// for l.Next() {
|
||||||
|
// item := l.Get()
|
||||||
|
// ... handle item ...
|
||||||
|
// }
|
||||||
|
// if e := l.Error(); e != nil {
|
||||||
|
// ... handle error message ...
|
||||||
|
// }
|
||||||
|
func Lex(input string) *Lexer {
|
||||||
|
return &Lexer{
|
||||||
|
input: input,
|
||||||
|
state: stateKeyValuePair,
|
||||||
|
items: make(chan Item, 2),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Next advances to the next lexer item in the input string.
|
||||||
|
// When a next item was found, then true is returned.
|
||||||
|
// On error or reaching the end of the input, false is returned.
|
||||||
|
func (l *Lexer) Next() bool {
|
||||||
|
if l.state == nil {
|
||||||
|
panic("This should not happen: nil state reached, but entering Next()")
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case i := <-l.items:
|
||||||
|
if i.Type == ItemEOF {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if i.Type == ItemError {
|
||||||
|
l.err = errors.New(i.Value)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
l.nextItem = i
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
l.state = l.state(l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Lexer) Error() error {
|
||||||
|
return l.err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ToArray returns lexer items as an array.
|
||||||
|
// When an error occurs during scanning, a partial result will be
|
||||||
|
// returned, accompanied by the error that occurred.
|
||||||
|
func (l *Lexer) ToArray() ([]Item, error) {
|
||||||
|
var items []Item
|
||||||
|
for l.Next() {
|
||||||
|
items = append(items, l.Get())
|
||||||
|
}
|
||||||
|
return items, l.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get returns the next lexer item, as reached by Next()
|
||||||
|
func (l *Lexer) Get() Item {
|
||||||
|
return l.nextItem
|
||||||
|
}
|
||||||
|
|
||||||
|
// pushState adds the state function to its stack.
|
||||||
|
// This is used for implementing nested parsing.
|
||||||
|
func (l *Lexer) pushState(state stateFn) {
|
||||||
|
l.stack = append(l.stack, state)
|
||||||
|
}
|
||||||
|
|
||||||
|
// popState pops the last pushed state from its stack.
|
||||||
|
func (l *Lexer) popState() stateFn {
|
||||||
|
last := len(l.stack) - 1
|
||||||
|
head, tail := l.stack[:last], l.stack[last]
|
||||||
|
l.stack = head
|
||||||
|
return tail
|
||||||
|
}
|
||||||
|
|
||||||
|
// getAcceptedString returns the string as accepted by the
|
||||||
|
// accept* methods so far.
|
||||||
|
func (l *Lexer) getAcceptedString() string {
|
||||||
|
return l.input[l.start:l.pos]
|
||||||
|
}
|
||||||
|
|
||||||
|
// emit passes a scanned item back to the client.
|
||||||
|
func (l *Lexer) emit(t itemType, v string) {
|
||||||
|
l.items <- Item{t, v}
|
||||||
|
l.start = l.pos
|
||||||
|
}
|
||||||
|
|
||||||
|
// ignore skips over the pending input before the current position.
|
||||||
|
func (l *Lexer) ignore() {
|
||||||
|
l.start = l.pos
|
||||||
|
}
|
||||||
|
|
||||||
|
// backup steps back one rune
|
||||||
|
// Can be called only once per call of next.
|
||||||
|
func (l *Lexer) backup() {
|
||||||
|
l.pos -= l.width
|
||||||
|
}
|
||||||
|
|
||||||
|
// peek returns but does not advance to the next rune(s) in the input.
|
||||||
|
func (l *Lexer) peek() rune {
|
||||||
|
r := l.next()
|
||||||
|
l.backup()
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// accept consumes the next rune if it's from the valid set of runes.
|
||||||
|
func (l *Lexer) accept(runes string) bool {
|
||||||
|
if strings.IndexRune(runes, l.next()) >= 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
l.backup()
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Lexer) upcoming(runes string) bool {
|
||||||
|
if l.accept(runes) {
|
||||||
|
l.backup()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// acceptNot consumes the next rune if it's not from the set of runes.
|
||||||
|
func (l *Lexer) acceptNot(runes string) bool {
|
||||||
|
r := l.next()
|
||||||
|
if r == endOfFile {
|
||||||
|
l.backup()
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if strings.IndexRune(runes, r) < 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
l.backup()
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// acceptUntil consumes a run of runes until ones from the
|
||||||
|
// valid set is encountered.
|
||||||
|
func (l *Lexer) acceptUntil(runes string) bool {
|
||||||
|
accepted := false
|
||||||
|
for l.acceptNot(runes) {
|
||||||
|
accepted = true
|
||||||
|
}
|
||||||
|
return accepted
|
||||||
|
}
|
||||||
|
|
||||||
|
// acceptRun consumes a run of runes from the set of accepted runes.
|
||||||
|
func (l *Lexer) acceptWhile(runes string) bool {
|
||||||
|
accepted := false
|
||||||
|
for l.accept(runes) {
|
||||||
|
accepted = true
|
||||||
|
}
|
||||||
|
return accepted
|
||||||
|
}
|
||||||
|
|
||||||
|
// skip skips a run of runes from the set of accepted runs.
|
||||||
|
func (l *Lexer) skip(runes string) {
|
||||||
|
if l.acceptWhile(runes) {
|
||||||
|
l.ignore()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// skipUntil skips a run of runes, until a rune from the set of
|
||||||
|
// runes of EOF is reached.
|
||||||
|
func (l *Lexer) skipUntil(runes string) {
|
||||||
|
if l.acceptUntil(runes) {
|
||||||
|
l.ignore()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Lexer) newString() {
|
||||||
|
l.strValue.Reset()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Lexer) addToString(r rune) {
|
||||||
|
l.strValue.WriteRune(r)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Lexer) getString() string {
|
||||||
|
return l.strValue.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
var endOfFile rune = -1
|
||||||
|
|
||||||
|
// next returns the next rune in the input.
|
||||||
|
func (l *Lexer) next() rune {
|
||||||
|
if l.pos >= len(l.input) {
|
||||||
|
l.width = 0
|
||||||
|
return endOfFile
|
||||||
|
}
|
||||||
|
r, w := utf8.DecodeRuneInString(l.input[l.pos:])
|
||||||
|
l.width = w
|
||||||
|
l.pos += w
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// error returns an error token and terminates the scan
|
||||||
|
// by returning nil to l.run.
|
||||||
|
func (l *Lexer) errorf(format string, args ...interface{}) stateFn {
|
||||||
|
l.items <- Item{
|
||||||
|
ItemError,
|
||||||
|
fmt.Sprintf(format, args...),
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Lexer) unexpectedTokenError(expected string) stateFn {
|
||||||
|
var actual string
|
||||||
|
switch {
|
||||||
|
case l.peek() == endOfFile:
|
||||||
|
actual = "end of file"
|
||||||
|
case !utf8.ValidString(l.input[l.start:]):
|
||||||
|
actual = "non-UTF8 data"
|
||||||
|
default:
|
||||||
|
actual = fmt.Sprintf("token '%c'", l.peek())
|
||||||
|
}
|
||||||
|
return l.errorf("Unexpected %s (expected %s)", actual, expected)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,164 @@
|
||||||
|
package lexer_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/mmakaay/toml/lexer"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestInvalidUtf8Data(t *testing.T) {
|
||||||
|
assertFailureAndCheck(t, "\xbc", []string{}, "Unexpected non-UTF8 data (expected end of file)")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmptyInput(t *testing.T) {
|
||||||
|
assertSuccessAndCheck(t, "", []string{})
|
||||||
|
}
|
||||||
|
func TestWhiteSpace(t *testing.T) {
|
||||||
|
assertSuccessAndCheck(t, " ", []string{})
|
||||||
|
assertSuccessAndCheck(t, "\t", []string{})
|
||||||
|
assertSuccessAndCheck(t, " \t \t ", []string{})
|
||||||
|
}
|
||||||
|
func TestWhiteSpaceAndNewlines(t *testing.T) {
|
||||||
|
assertSuccessAndCheck(t, "\n", []string{})
|
||||||
|
assertSuccessAndCheck(t, "\n \t\r\n", []string{})
|
||||||
|
}
|
||||||
|
func TestWhitespacePlusComment(t *testing.T) {
|
||||||
|
assertSuccessAndCheck(t, "#", []string{`Comment("#")`})
|
||||||
|
assertSuccessAndCheck(t, " \t \t #", []string{`Comment("#")`})
|
||||||
|
assertSuccessAndCheck(t, " \t \t # not empty", []string{`Comment("# not empty")`})
|
||||||
|
assertSuccessAndCheck(t, " \t \t # not empty\r\r\r\n", []string{`Comment("# not empty")`})
|
||||||
|
assertSuccessAndCheck(t, "\n \t\r\n# AAP\r\n", []string{`Comment("# AAP")`})
|
||||||
|
assertSuccessAndCheck(t, "# two lines\n# of comments\n",
|
||||||
|
[]string{`Comment("# two lines")`, `Comment("# of comments")`})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBareKeyWithoutValue(t *testing.T) {
|
||||||
|
err := "Unexpected end of file (expected an '=' value assignment)"
|
||||||
|
assertFailureAndCheck(t, "=", []string{`Key("a")`}, err)
|
||||||
|
assertFailureAndCheck(t, " a", []string{`Key("a")`}, err)
|
||||||
|
assertFailureAndCheck(t, " a ", []string{`Key("a")`}, err)
|
||||||
|
assertFailureAndCheck(t, "ab", []string{`Key("ab")`}, err)
|
||||||
|
assertFailureAndCheck(t, "Ab", []string{`Key("Ab")`}, err)
|
||||||
|
assertFailureAndCheck(t, "Ab1", []string{`Key("Ab1")`}, err)
|
||||||
|
assertFailureAndCheck(t, "_Ab1", []string{`Key("_Ab1")`}, err)
|
||||||
|
assertFailureAndCheck(t, "_-Ab1", []string{`Key("_-Ab1")`}, err)
|
||||||
|
assertFailureAndCheck(t, "_-Ab1_this-is_GOOD987", []string{`Key("_-Ab1_this-is_GOOD987")`}, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDottedKey(t *testing.T) {
|
||||||
|
err := "Unexpected end of file (expected an '=' value assignment)"
|
||||||
|
assertFailureAndCheck(t, "a.b", []string{`Key("a")`, `KeyDot(".")`, `Key("b")`}, err)
|
||||||
|
assertFailureAndCheck(t, " a .\t\t b\t ", []string{`Key("a")`, `KeyDot(".")`, `Key("b")`}, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKeyWithAssignmentButNoValue(t *testing.T) {
|
||||||
|
err := "Unexpected end of file (expected a value)"
|
||||||
|
assertFailureAndCheck(t, " some_cool_key = ", []string{`Key("some_cool_key")`, `Assignment("=")`}, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmptyBasicStringValue(t *testing.T) {
|
||||||
|
assertSuccessAndCheck(t, `a=""`, []string{`Key("a")`, `Assignment("=")`, `StringValue("")`})
|
||||||
|
assertSuccessAndCheck(t, `a=""#hi`, []string{`Key("a")`, `Assignment("=")`, `StringValue("")`, `Comment("#hi")`})
|
||||||
|
assertSuccessAndCheck(t, `a = ""`, []string{`Key("a")`, `Assignment("=")`, `StringValue("")`})
|
||||||
|
assertSuccessAndCheck(t, `a.b = ""`, []string{`Key("a")`, `KeyDot(".")`, `Key("b")`, `Assignment("=")`, `StringValue("")`})
|
||||||
|
}
|
||||||
|
func TestBasicStringValue(t *testing.T) {
|
||||||
|
assertSuccessAndCheck(t, `_ = "b"`,
|
||||||
|
[]string{
|
||||||
|
`Key("_")`,
|
||||||
|
`Assignment("=")`,
|
||||||
|
`StringValue("b")`})
|
||||||
|
assertSuccessAndCheck(t, `thing = "A cool ʎǝʞ" # huh, it's up-side down!!`,
|
||||||
|
[]string{
|
||||||
|
`Key("thing")`,
|
||||||
|
`Assignment("=")`,
|
||||||
|
`StringValue("A cool ʎǝʞ")`,
|
||||||
|
`Comment("# huh, it's up-side down!!")`})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInvalidEscapeSequence(t *testing.T) {
|
||||||
|
assertFailure(t, `a="\x"`, `Invalid escape sequence \x in string value`)
|
||||||
|
}
|
||||||
|
func TestBasicStringValueEscapes(t *testing.T) {
|
||||||
|
for in, out := range map[string]string{
|
||||||
|
`\b`: "\b",
|
||||||
|
`\t`: "\t",
|
||||||
|
`\n`: "\n",
|
||||||
|
`\f`: "\f",
|
||||||
|
`\r`: "\r",
|
||||||
|
`\"`: "\"",
|
||||||
|
`\b\t\n\f\r\"`: "\b\t\n\f\r\"",
|
||||||
|
} {
|
||||||
|
l := assertSuccess(t, fmt.Sprintf(`x="%s"`, in))
|
||||||
|
s := l[2]
|
||||||
|
if out != s.Value {
|
||||||
|
t.Fatalf("Unexpected result when parsing '%s'", in)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// func TestBasicStringUnicodeEscapes(t *testing.T) {
|
||||||
|
// for in, out := range map[string]string{
|
||||||
|
// `\u`: "\b",
|
||||||
|
// } {
|
||||||
|
// l := assertSuccess(t, fmt.Sprintf(`x="%s"`, in))
|
||||||
|
// s := l[2]
|
||||||
|
// if out != s.Value {
|
||||||
|
// t.Fatalf("Unexpected result when parsing '%s'", in)
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
func TestTwoKeyValuePairs(t *testing.T) {
|
||||||
|
assertSuccessAndCheck(t, "a=\"Hello\" #comment1\nb=\"World!\"#comment2\r\n",
|
||||||
|
[]string{
|
||||||
|
`Key("a")`,
|
||||||
|
`Assignment("=")`,
|
||||||
|
`StringValue("Hello")`,
|
||||||
|
`Comment("#comment1")`,
|
||||||
|
`Key("b")`,
|
||||||
|
`Assignment("=")`,
|
||||||
|
`StringValue("World!")`,
|
||||||
|
`Comment("#comment2")`})
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertSuccessAndCheck(t *testing.T, input string, expected []string) {
|
||||||
|
l := assertSuccess(t, input)
|
||||||
|
assertItems(t, l, expected)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertFailureAndCheck(t *testing.T, input string, expected []string, expectedErr string) {
|
||||||
|
l := assertFailure(t, input, expectedErr)
|
||||||
|
assertItems(t, l, expected)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertFailure(t *testing.T, input string, expectedErr string) []lexer.Item {
|
||||||
|
l, err := lexer.Lex(input).ToArray()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Expected lexer error '%s', but no error occurred", expectedErr)
|
||||||
|
}
|
||||||
|
if err.Error() != expectedErr {
|
||||||
|
t.Fatalf("Mismatch between expected and actual error:\nExpected: %s\nActual: %s\n", expectedErr, err)
|
||||||
|
}
|
||||||
|
return l
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertSuccess(t *testing.T, input string) []lexer.Item {
|
||||||
|
l, err := lexer.Lex(input).ToArray()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Unexpected lexer error: %s", err)
|
||||||
|
}
|
||||||
|
return l
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertItems(t *testing.T, l []lexer.Item, expected []string) {
|
||||||
|
if len(expected) != len(l) {
|
||||||
|
t.Fatalf("Unexpected number of lexer items: %d (expected: %d)", len(l), len(expected))
|
||||||
|
}
|
||||||
|
for i, e := range expected {
|
||||||
|
if l[i].String() != e {
|
||||||
|
t.Fatalf("Unexpected lexer item at index %d: %s (expected: %s)", i, l[i], e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,219 @@
|
||||||
|
package lexer
|
||||||
|
|
||||||
|
// stateFn represents the state of the scanner as a function
|
||||||
|
// that returns the next state.
|
||||||
|
type stateFn func(*Lexer) stateFn
|
||||||
|
|
||||||
|
const (
|
||||||
|
whitespace string = " \t"
|
||||||
|
newline string = "\r\n"
|
||||||
|
startOfComment string = "#"
|
||||||
|
equal string = "="
|
||||||
|
lower string = "abcdefghijklmnopqrstuvwxyz"
|
||||||
|
upper string = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||||
|
digits string = "0123456789"
|
||||||
|
dot string = "."
|
||||||
|
underscore string = "_"
|
||||||
|
dash string = "-"
|
||||||
|
singleQuote string = "'"
|
||||||
|
doubleQuote string = "\""
|
||||||
|
backslash string = "\\"
|
||||||
|
someQuote string = singleQuote + doubleQuote
|
||||||
|
singleQuote3 string = singleQuote + singleQuote + singleQuote
|
||||||
|
doubleQuote3 string = doubleQuote + doubleQuote + doubleQuote
|
||||||
|
bareKey string = lower + upper + digits + underscore + dash
|
||||||
|
startOfKey string = bareKey + someQuote
|
||||||
|
)
|
||||||
|
|
||||||
|
func stateKeyValuePair(l *Lexer) stateFn {
|
||||||
|
l.skip(whitespace + newline)
|
||||||
|
if l.upcoming(startOfComment) {
|
||||||
|
return stateComment
|
||||||
|
}
|
||||||
|
if l.upcoming(startOfKey) {
|
||||||
|
return stateKey
|
||||||
|
}
|
||||||
|
return stateEndOfFile
|
||||||
|
}
|
||||||
|
|
||||||
|
// A hash symbol marks the rest of the line as a comment.
|
||||||
|
func stateComment(l *Lexer) stateFn {
|
||||||
|
l.acceptUntil(newline)
|
||||||
|
l.emit(ItemComment, l.getAcceptedString())
|
||||||
|
l.skip(newline)
|
||||||
|
return stateKeyValuePair
|
||||||
|
}
|
||||||
|
|
||||||
|
// A key may be either bare, quoted or dotted.
|
||||||
|
func stateKey(l *Lexer) stateFn {
|
||||||
|
if l.upcoming(bareKey) {
|
||||||
|
return stateBareKey
|
||||||
|
}
|
||||||
|
return l.unexpectedTokenError("a valid key name")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bare keys may only contain ASCII letters, ASCII digits,
|
||||||
|
// underscores, and dashes (A-Za-z0-9_-). Note that bare
|
||||||
|
// keys are allowed to be composed of only ASCII digits,
|
||||||
|
// e.g. 1234, but are always interpreted as strings.
|
||||||
|
func stateBareKey(l *Lexer) stateFn {
|
||||||
|
l.acceptWhile(bareKey)
|
||||||
|
l.emit(ItemKey, l.getAcceptedString())
|
||||||
|
return stateEndOfKeyOrKeyDot
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dotted keys are a sequence of bare or quoted keys joined with a dot.
|
||||||
|
// This allows for grouping similar properties together:
|
||||||
|
func stateEndOfKeyOrKeyDot(l *Lexer) stateFn {
|
||||||
|
// Whitespace around dot-separated parts is ignored, however,
|
||||||
|
// best practice is to not use any extraneous whitespace.
|
||||||
|
l.skip(whitespace)
|
||||||
|
if l.accept(dot) {
|
||||||
|
l.emit(ItemKeyDot, ".")
|
||||||
|
l.skip(whitespace)
|
||||||
|
return stateKey
|
||||||
|
}
|
||||||
|
return stateKeyAssignment
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keys are on the left of the equals sign and values are on the right.
|
||||||
|
// Whitespace is ignored around key names and values. The key, equals
|
||||||
|
// sign, and value must be on the same line (though some values can
|
||||||
|
// be broken over multiple lines).
|
||||||
|
func stateKeyAssignment(l *Lexer) stateFn {
|
||||||
|
l.skip(whitespace)
|
||||||
|
if l.accept(equal) {
|
||||||
|
l.emit(ItemKeyValueAssignment, "=")
|
||||||
|
l.skip(whitespace)
|
||||||
|
return stateValue
|
||||||
|
}
|
||||||
|
return l.unexpectedTokenError("an '=' value assignment")
|
||||||
|
}
|
||||||
|
|
||||||
|
func stateValue(l *Lexer) stateFn {
|
||||||
|
l.skip(whitespace)
|
||||||
|
if l.upcoming(someQuote) {
|
||||||
|
return stateStringValue
|
||||||
|
}
|
||||||
|
return l.unexpectedTokenError("a value")
|
||||||
|
}
|
||||||
|
|
||||||
|
// There are four ways to express strings: basic, multi-line basic, literal,
|
||||||
|
// and multi-line literal. All strings must contain only valid UTF-8 characters.
|
||||||
|
func stateStringValue(l *Lexer) stateFn {
|
||||||
|
if l.accept(doubleQuote) {
|
||||||
|
return stateBasicStringValue
|
||||||
|
}
|
||||||
|
return l.unexpectedTokenError("a string value")
|
||||||
|
}
|
||||||
|
|
||||||
|
func stateBasicStringValue(l *Lexer) stateFn {
|
||||||
|
// Possibly a """ multi-line string start,
|
||||||
|
// possibly the end of an "" empty string.
|
||||||
|
if l.accept(doubleQuote) {
|
||||||
|
// A """ multi-line string.
|
||||||
|
if l.accept(doubleQuote) {
|
||||||
|
l.ignore()
|
||||||
|
return stateMultiLineBasicString
|
||||||
|
}
|
||||||
|
// An "" empty string.
|
||||||
|
l.ignore()
|
||||||
|
l.emit(ItemStringValue, "")
|
||||||
|
return stateKeyValuePair
|
||||||
|
}
|
||||||
|
l.ignore()
|
||||||
|
return stateBasicString
|
||||||
|
}
|
||||||
|
|
||||||
|
// Basic strings are surrounded by quotation marks. Any Unicode character
|
||||||
|
// may be used except those that must be escaped: quotation mark, backslash,
|
||||||
|
// and the control characters (U+0000 to U+001F, U+007F).
|
||||||
|
//
|
||||||
|
// For convenience, some popular characters have a compact escape sequence.
|
||||||
|
//
|
||||||
|
// \b - backspace (U+0008)
|
||||||
|
// \t - tab (U+0009)
|
||||||
|
// \n - linefeed (U+000A)
|
||||||
|
// \f - form feed (U+000C)
|
||||||
|
// \r - carriage return (U+000D)
|
||||||
|
// \" - quote (U+0022)
|
||||||
|
// \\ - backslash (U+005C)
|
||||||
|
// \uXXXX - unicode (U+XXXX)
|
||||||
|
// \UXXXXXXXX - unicode (U+XXXXXXXX)
|
||||||
|
//
|
||||||
|
// Any Unicode character may be escaped with the \uXXXX or \UXXXXXXXX forms.
|
||||||
|
// The escape codes must be valid Unicode scalar values.
|
||||||
|
//
|
||||||
|
// All other escape sequences not listed above are reserved and,
|
||||||
|
// if used, TOML should produce an error.
|
||||||
|
|
||||||
|
var basicEscapes = map[rune]rune{
|
||||||
|
'b': rune(8),
|
||||||
|
't': rune(9),
|
||||||
|
'n': rune(10),
|
||||||
|
'f': rune(12),
|
||||||
|
'r': rune(13),
|
||||||
|
'"': rune(34),
|
||||||
|
'\\': rune(92),
|
||||||
|
}
|
||||||
|
|
||||||
|
func stateParseBasicString(l *Lexer) stateFn {
|
||||||
|
for {
|
||||||
|
switch {
|
||||||
|
case l.upcoming(endOfFile):
|
||||||
|
l.unexpectedTokenError("basic string token")
|
||||||
|
case l.upcoming(doubleQuote):
|
||||||
|
return l.popState()
|
||||||
|
case l.accept(backslash):
|
||||||
|
r := l.next()
|
||||||
|
if escaped, ok := basicEscapes[r]; ok {
|
||||||
|
l.addToString(escaped)
|
||||||
|
} else {
|
||||||
|
return l.errorf("Invalid escape sequence \\%c in string value", r)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
l.addToString(l.next())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func stateBasicString(l *Lexer) stateFn {
|
||||||
|
l.newString()
|
||||||
|
l.pushState(stateBasicStringEnd)
|
||||||
|
return stateParseBasicString
|
||||||
|
|
||||||
|
parsing:
|
||||||
|
for {
|
||||||
|
r := l.next()
|
||||||
|
if r == endOfFile {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if r == '"' {
|
||||||
|
l.emit(ItemStringValue, l.getString())
|
||||||
|
return stateKeyValuePair
|
||||||
|
}
|
||||||
|
if r == '\\' {
|
||||||
|
r = l.next()
|
||||||
|
if escaped, ok := basicEscapes[r]; ok {
|
||||||
|
l.addToString(escaped)
|
||||||
|
continue parsing
|
||||||
|
}
|
||||||
|
return l.errorf("Invalid escape sequence \\%c in string value", r)
|
||||||
|
}
|
||||||
|
l.addToString(r)
|
||||||
|
}
|
||||||
|
return l.unexpectedTokenError("valid basic string rune")
|
||||||
|
}
|
||||||
|
|
||||||
|
func stateMultiLineBasicString(l *Lexer) stateFn {
|
||||||
|
return l.errorf("Not yet implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
func stateEndOfFile(l *Lexer) stateFn {
|
||||||
|
i := l.peek()
|
||||||
|
if i == endOfFile {
|
||||||
|
l.emit(ItemEOF, "EOF")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return l.unexpectedTokenError("end of file")
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue