OpenSourceProjects
/
go-toml
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Moved all parser code into subpackage 'parse'.

This commit is contained in:
Maurice Makaay 2019-06-26 21:51:42 +00:00
parent 54cb75955d
commit b95f255ae7
31 changed files with 264 additions and 646 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@ -32,9 +32,5 @@ tags
*.out
# ---> VisualStudioCode
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
.vscode

245
ast.go
View File

@ -1,245 +0,0 @@
package toml
import (
"fmt"
"regexp"
"sort"
"strings"
"time"
)
// item represents a TOML item.
type item struct {
Type valueType
Data []interface{}
}
// table represents a TOML table.
type table map[string]*item
// valueType identifies the type of a TOML value.
type valueType string
const (
tString valueType = "string" // "various", 'types', """of""", '''strings'''
tInteger valueType = "integer" // 12345, 0xffee12a0, 0o0755, 0b101101011
tFloat valueType = "float" // 10.1234, 143E-12, 43.28377e+4, +inf, -inf, nan
tBoolean valueType = "boolean" // true or false
tOffsetDateTime valueType = "offset datetime" // 2019-06-18 10:32:15.173645362+0200
tLocalDateTime valueType = "datetime" // 2018-12-25 12:12:18.876772533
tLocalDate valueType = "date" // 2017-05-17
tLocalTime valueType = "time" // 23:01:22
tArrayOfTables valueType = "array" // defined using an [[array.of.tables]]
tArray valueType = "static array" // defined using ["an", "inline", "array"]
tTable valueType = "table" // defined using { "inline" = "table" } or [standard.table]
)
// newItem instantiates a new item struct.
func newItem(valueType valueType, data ...interface{}) *item {
return &item{Type: valueType, Data: data}
}
// newKey instantiates a new key.
func newKey(key ...string) []string {
return key
}
// parser holds the state for the TOML parser. All parsing functions are
// methods of this struct.
type parser struct {
Root table // the root-level TOML table (each TOML doc is implicitly a table)
Current table // the currently active TOML table
CurrentKey []string // the key for the currently active TOML table
}
func newParser() *parser {
p := &parser{Root: make(table)}
p.Current = p.Root
return p
}
func (t *parser) setKeyValuePair(key []string, value *item) error {
// First make sure the table structure for storing the value exists.
node, lastKeyPart, err := t.makeTablePath(key)
if err != nil {
return fmt.Errorf("invalid key/value pair: %s", err)
}
// Check if the key is still free for use.
if existing, ok := node[lastKeyPart]; ok {
path := t.formatKeyPath(key, len(key)-1)
return fmt.Errorf("invalid key/value pair: %s item already exists at key %s", existing.Type, path)
}
// It is, store the value in the table.
node[lastKeyPart] = value
return nil
}
func (t *parser) openTable(key []string) error {
t.CurrentKey = nil
t.Current = t.Root
// Go over all requested levels of the key. For all levels, except the last
// one, it is okay if a table already exists. For at least the last level,
// no table or value must exist, because that would mean we are overwriting
// an existing key/value pair, which is not allowed.
node, lastKeyPart, err := t.makeTablePath(key)
if err != nil {
return fmt.Errorf("invalid table: %s", err)
}
// Check if the key is still free for use.
if existing, ok := node[lastKeyPart]; ok {
path := t.formatKeyPath(key, len(key)-1)
return fmt.Errorf("invalid table: %s item already exists at key %s", existing.Type, path)
}
// The subtable does not exist yet. Create the subtable.
subTable := make(table)
node[lastKeyPart] = newItem(tTable, subTable)
node = subTable
// From here on, key/value pairs are added to the newly defined table.
t.Current = node
t.CurrentKey = key
return nil
}
func (t *parser) openArrayOfTables(key []string) error {
t.CurrentKey = nil
t.Current = t.Root
// Go over all requested levels of the key. For all levels, except the last
// one, it is okay if a table already exists. For the last level, either
// no item must exist (in which case a table array will be created), or a
// table array must exist.
// Other cases would mean we are overwriting an existing key/value pair,
// which is not allowed.
node, lastKeyPart, err := t.makeTablePath(key)
if err != nil {
return fmt.Errorf("invalid table array: %s", err)
}
// At the last key position, there must be either no value yet, or the
// existing value must be a table array. Other values are invalid.
if existing, ok := node[lastKeyPart]; ok {
if existing.Type != tArrayOfTables {
path := t.formatKeyPath(key, len(key)-1)
return fmt.Errorf("invalid table array: %s item already exists at key %s", existing.Type, path)
}
// A table array exists. Add a new table to this array.
array := node[lastKeyPart]
subTable := make(table)
tables := array.Data
tables = append(tables, newItem(tTable, subTable))
array.Data = tables
node = subTable
} else {
// No value exists at the defined key path. Create a new table array.
subTable := make(table)
node[lastKeyPart] = newItem(tArrayOfTables, newItem(tTable, subTable))
node = subTable
}
// From here on, key/value pairs are added to the newly defined table.
t.Current = node
t.CurrentKey = key
return nil
}
func (t *parser) makeTablePath(key []string) (table, string, error) {
node := t.Current
for i, keyPart := range key {
// Arrived at the last key part? Then the path towards that key is
// setup correctly. Return the last part, so the caller can use it.
isLast := i == len(key)-1
if isLast {
return node, keyPart, nil
}
if subItem, ok := node[keyPart]; ok {
// You cannot overwrite an already defined key, regardless its value.
// When a value already exists at the current key, this can only be a table.
if subItem.Type != tTable {
path := t.formatKeyPath(key, i)
return nil, "", fmt.Errorf("%s item already exists at key %s", subItem.Type, path)
}
// All is okay, traverse to the subtable.
node = subItem.Data[0].(table)
} else {
// The subtable does not exist yet. Create the subtable.
subTable := make(table)
node[keyPart] = newItem(tTable, subTable)
node = subTable
}
}
panic("makeTablePath(): empty key provided; a key must have at least one key part")
}
func (t *parser) formatKeyPath(key []string, end int) string {
var sb strings.Builder
sb.WriteRune('[')
if t.CurrentKey != nil {
for i, keyPart := range t.CurrentKey {
if i > 0 {
sb.WriteString("->")
}
sb.WriteString(formatKeyName(keyPart))
}
}
for i, keyPart := range key {
if t.CurrentKey != nil || i > 0 {
sb.WriteString("->")
}
sb.WriteString(formatKeyName(keyPart))
if i == end {
break
}
}
sb.WriteRune(']')
return sb.String()
}
func formatKeyName(key string) string {
if ok, _ := regexp.Match(`^\w+$`, []byte(key)); ok {
return key
}
return fmt.Sprintf("%q", key)
}
func (t table) String() string {
return newItem(tTable, t).String()
}
// String() produces a JSON-like (but not JSON) string representation of the value.
func (parseItem item) String() string {
switch parseItem.Type {
case tString:
return fmt.Sprintf("%q", parseItem.Data[0])
case tOffsetDateTime:
return parseItem.Data[0].(time.Time).Format(time.RFC3339Nano)
case tLocalDateTime:
return parseItem.Data[0].(time.Time).Format("2006-01-02 15:04:05.999999999")
case tLocalDate:
return parseItem.Data[0].(time.Time).Format("2006-01-02")
case tLocalTime:
return parseItem.Data[0].(time.Time).Format("15:04:05.999999999")
case tArrayOfTables:
fallthrough
case tArray:
items := make([]string, len(parseItem.Data))
for i, value := range parseItem.Data {
items[i] = value.(*item).String()
}
return fmt.Sprintf("[%s]", strings.Join(items, ", "))
case tTable:
pairs := parseItem.Data[0].(table)
keys := make([]string, len(pairs))
i := 0
for k := range pairs {
keys[i] = k
i++
}
sort.Strings(keys)
items := make([]string, len(pairs))
for i, k := range keys {
items[i] = fmt.Sprintf("%q: %s", k, pairs[k].String())
}
return fmt.Sprintf("{%s}", strings.Join(items, ", "))
default:
return fmt.Sprintf("%v", parseItem.Data[0])
}
}

8
ast/ast.go
View File

@ -191,15 +191,15 @@ func (doc *Document) makeTablePath(key Key) (Table, string, error) {
if isLast {
return node, keyPart, nil
}
if subItem, ok := node[keyPart]; ok {
if subValue, ok := node[keyPart]; ok {
// You cannot overwrite an already defined key, regardless its value.
// When a value already exists at the current key, this can only be a table.
if subItem.Type != TypeTable {
if subValue.Type != TypeTable {
path := doc.formatKeyPath(key, i)
return nil, "", fmt.Errorf("%s value already exists at key %s", subItem.Type, path)
return nil, "", fmt.Errorf("%s value already exists at key %s", subValue.Type, path)
}
// All is okay, traverse to the subtable.
node = subItem.Data[0].(Table)
node = subValue.Data[0].(Table)
} else {
// The subtable does not exist yet. Create the subtable.
subTable := make(Table)

12
ast/ast_test.go
View File

@ -103,7 +103,7 @@ func Test_GivenExistingTableAtKey_CreatingTableAtSameKey_ReturnsError(t *testing
`{"key1": {"key2": {}}}`)
}
func Test_GivenExistingItemAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
func Test_GivenExistingValueAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *ast.Document) {
p := ast.NewDocument()
p.SetKeyValuePair(ast.NewKey("key"), ast.NewValue(ast.TypeString, "value"))
@ -113,7 +113,7 @@ func Test_GivenExistingItemAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.
`{"key": "value"}`)
}
func Test_GivenExistingItemInKeyPath_CreatingTable_ReturnsError(t *testing.T) {
func Test_GivenExistingValueInKeyPath_CreatingTable_ReturnsError(t *testing.T) {
testAST(t, func() (error, *ast.Document) {
p := ast.NewDocument()
p.SetKeyValuePair(ast.NewKey("key"), ast.NewValue(ast.TypeString, "value"))
@ -123,7 +123,7 @@ func Test_GivenExistingItemInKeyPath_CreatingTable_ReturnsError(t *testing.T) {
`{"key": "value"}`)
}
func Test_GivenExistingItemAtDeepKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
func Test_GivenExistingValueAtDeepKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *ast.Document) {
p := ast.NewDocument()
p.OpenTable(ast.NewKey("deep", "table"))
@ -134,7 +134,7 @@ func Test_GivenExistingItemAtDeepKey_CreatingTableAtSameKey_ReturnsError(t *test
`{"deep": {"table": {"key": 0}}}`)
}
func Test_GivenExistingItemAtDeepKeyFromSubTable_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
func Test_GivenExistingValueAtDeepKeyFromSubTable_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *ast.Document) {
p := ast.NewDocument()
p.OpenTable(ast.NewKey("deep", "table"))
@ -158,7 +158,7 @@ func Test_FormattingOfQuotedPathPartInError(t *testing.T) {
`{"must be quoted": {"this one too": 0}}`)
}
func Test_GivenExistingItemAtKey_CreatingArrayOfTablesAtSameKey_ReturnsError(t *testing.T) {
func Test_GivenExistingValueAtKey_CreatingArrayOfTablesAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *ast.Document) {
p := ast.NewDocument()
p.Root["key"] = ast.NewValue(ast.TypeString, "value")
@ -168,7 +168,7 @@ func Test_GivenExistingItemAtKey_CreatingArrayOfTablesAtSameKey_ReturnsError(t *
`{"key": "value"}`)
}
func Test_GivenExistingItemInKeyPath_CreatingArrayOfTables_ReturnsError(t *testing.T) {
func Test_GivenExistingValueInKeyPath_CreatingArrayOfTables_ReturnsError(t *testing.T) {
testAST(t, func() (error, *ast.Document) {
p := ast.NewDocument()
p.Root["key"] = ast.NewValue(ast.TypeString, "value")

22
ast/string.go
View File

@ -15,28 +15,28 @@ func (t Table) String() string {
// String() produces a JSON-like (but not JSON) string representation of the value.
// This string version is mainly useful for testing and debugging purposes.
func (parseItem Value) String() string {
switch parseItem.Type {
func (value Value) String() string {
switch value.Type {
case TypeString:
return fmt.Sprintf("%q", parseItem.Data[0])
return fmt.Sprintf("%q", value.Data[0])
case TypeOffsetDateTime:
return parseItem.Data[0].(time.Time).Format(time.RFC3339Nano)
return value.Data[0].(time.Time).Format(time.RFC3339Nano)
case TypeLocalDateTime:
return parseItem.Data[0].(time.Time).Format("2006-01-02 15:04:05.999999999")
return value.Data[0].(time.Time).Format("2006-01-02 15:04:05.999999999")
case TypeLocalDate:
return parseItem.Data[0].(time.Time).Format("2006-01-02")
return value.Data[0].(time.Time).Format("2006-01-02")
case TypeLocalTime:
return parseItem.Data[0].(time.Time).Format("15:04:05.999999999")
return value.Data[0].(time.Time).Format("15:04:05.999999999")
case TypeArrayOfTables:
fallthrough
case TypeArray:
values := make(Key, len(parseItem.Data))
for i, value := range parseItem.Data {
values := make(Key, len(value.Data))
for i, value := range value.Data {
values[i] = value.(*Value).String()
}
return fmt.Sprintf("[%s]", strings.Join(values, ", "))
case TypeTable:
pairs := parseItem.Data[0].(Table)
pairs := value.Data[0].(Table)
keys := make([]string, len(pairs))
i := 0
for k := range pairs {
@ -50,6 +50,6 @@ func (parseItem Value) String() string {
}
return fmt.Sprintf("{%s}", strings.Join(values, ", "))
default:
return fmt.Sprintf("%v", parseItem.Data[0])
return fmt.Sprintf("%v", value.Data[0])
}
}

177
ast_test.go
View File

@ -1,177 +0,0 @@
package toml
import (
"testing"
)
func TestAST_ConstructStructure(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.setKeyValuePair(newKey("ding"), newItem(tInteger, 10))
p.setKeyValuePair(newKey("dong"), newItem(tString, "not a song"))
p.openTable(newKey("key1", "key2 a"))
p.setKeyValuePair(newKey("dooh"), newItem(tBoolean, true))
p.setKeyValuePair(newKey("dah"), newItem(tBoolean, false))
p.openTable(newKey("key1", "key2 b"))
p.setKeyValuePair(newKey("dieh"), newItem(tFloat, 1.111))
p.setKeyValuePair(newKey("duh"), newItem(tFloat, 1.18e-12))
p.setKeyValuePair(newKey("foo", "bar"), newItem(tArrayOfTables, newItem(tInteger, 1), newItem(tInteger, 2)))
p.openArrayOfTables(newKey("aaah", "table array"))
p.setKeyValuePair(newKey("a"), newItem(tFloat, 1.234))
p.openArrayOfTables(newKey("aaah", "table array"))
p.setKeyValuePair(newKey("b"), newItem(tFloat, 2.345))
p.setKeyValuePair(newKey("c"), newItem(tString, "bingo!"))
p.openArrayOfTables(newKey("aaah", "table array"))
return nil, p
},
"",
`{"aaah": {"table array": [{"a": 1.234}, {"b": 2.345, "c": "bingo!"}, {}]}, `+
`"ding": 10, "dong": "not a song", `+
`"key1": {"key2 a": {"dah": false, "dooh": true}, "key2 b": {"dieh": 1.111, "duh": 1.18e-12, "foo": {"bar": [1, 2]}}}}`)
}
func TestAST_EmptyKeyForCreatingTablePath_Panics(t *testing.T) {
defer func() {
r := recover()
if r.(string) != "makeTablePath(): empty key provided; a key must have at least one key part" {
t.Fatalf("Did not get the expected panic message")
}
}()
p := newParser()
p.openTable(newKey())
}
func TestAST_StoreValueInRootTable(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.setKeyValuePair(newKey("key1"), newItem(tString, "value1"))
return p.setKeyValuePair(newKey("key2"), newItem(tString, "value2")), p
},
"",
`{"key1": "value1", "key2": "value2"}`)
}
func TestAST_StoreValueWithMultipartKey_CreatesTableHierarchy(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
return p.setKeyValuePair(newKey("key1", "key2", "key3"), newItem(tString, "value")), p
},
"",
`{"key1": {"key2": {"key3": "value"}}}`)
}
func TestAST_StoreDuplicateKeyInRootTable_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.setKeyValuePair(newKey("key"), newItem(tString, "value"))
return p.setKeyValuePair(newKey("key"), newItem(tInteger, 321)), p
},
`invalid key/value pair: string item already exists at key [key]`,
`{"key": "value"}`)
}
func TestAST_StoreValueWithMultipartKey_UnderSubtable_CreatesTableHierarchy(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("tablekey1", "tablekey2"))
return p.setKeyValuePair(newKey("valuekey1", "valuekey2", "valuekey3"), newItem(tString, "value")), p
},
"",
`{"tablekey1": {"tablekey2": {"valuekey1": {"valuekey2": {"valuekey3": "value"}}}}}`)
}
func TestAST_StoreKeyPathWherePathContainsNonTableAlready_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("key1"))
p.setKeyValuePair(newKey("key2"), newItem(tInteger, 0))
return p.setKeyValuePair(newKey("key2", "key3"), newItem(tString, "value")), p
},
`invalid key/value pair: integer item already exists at key [key1->key2]`,
`{"key1": {"key2": 0}}`)
}
func TestAST_GivenExistingTableAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("key1", "key2"))
return p.openTable(newKey("key1", "key2")), p
},
`invalid table: table item already exists at key [key1->key2]`,
`{"key1": {"key2": {}}}`)
}
func TestAST_GivenExistingItemAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.setKeyValuePair(newKey("key"), newItem(tString, "value"))
return p.openTable(newKey("key")), p
},
`invalid table: string item already exists at key [key]`,
`{"key": "value"}`)
}
func TestAST_GivenExistingItemInKeyPath_CreatingTable_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.setKeyValuePair(newKey("key"), newItem(tString, "value"))
return p.openTable(newKey("key", "subkey")), p
},
`invalid table: string item already exists at key [key]`,
`{"key": "value"}`)
}
func TestAST_GivenExistingItemAtDeepKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("deep", "table"))
p.setKeyValuePair(newKey("key"), newItem(tInteger, 0))
return p.openTable(newKey("deep", "table", "key")), p
},
`invalid table: integer item already exists at key [deep->table->key]`,
`{"deep": {"table": {"key": 0}}}`)
}
func TestAST_GivenExistingItemAtDeepKeyFromSubTable_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("deep", "table"))
p.setKeyValuePair(newKey("key1", "key2"), newItem(tInteger, 0))
return p.setKeyValuePair(newKey("key1", "key2"), newItem(tBoolean, true)), p
},
// This test mainly tests the formatting of [deep->table->key1->key2], being a concatenation
// of the currently active table plus the multipart key for setKeyValuePair().
`invalid key/value pair: integer item already exists at key [deep->table->key1->key2]`,
`{"deep": {"table": {"key1": {"key2": 0}}}}`)
}
func TestAST_FormattingOfQuotedPathPartInError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("must be quoted"))
p.setKeyValuePair(newKey("this one too"), newItem(tInteger, 0))
return p.setKeyValuePair(newKey("this one too"), newItem(tInteger, 0)), p
},
`invalid key/value pair: integer item already exists at key ["must be quoted"->"this one too"]`,
`{"must be quoted": {"this one too": 0}}`)
}
func TestAST_GivenExistingItemAtKey_CreatingArrayOfTablesAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.Root["key"] = newItem(tString, "value")
return p.openArrayOfTables(newKey("key")), p
},
`invalid table array: string item already exists at key [key]`,
`{"key": "value"}`)
}
func TestAST_GivenExistingItemInKeyPath_CreatingArrayOfTables_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.Root["key"] = newItem(tString, "value")
return p.openArrayOfTables(newKey("key", "subkey")), p
},
`invalid table array: string item already exists at key [key]`,
`{"key": "value"}`)
}

15
comment.go
View File

@ -1,15 +0,0 @@
package toml
import (
"git.makaay.nl/mauricem/go-parsekit/parse"
)
// A '#' hash symbol marks the rest of the line as a comment.
// All characters up to the end of the line are included in the comment.
var comment = c.Seq(a.Hash, c.ZeroOrMore(c.Not(a.EndOfLine)), m.Drop(a.EndOfLine))
func (t *parser) startComment(p *parse.API) {
if !p.Accept(comment) {
p.Expected("comment")
}
}

2
go.sum
View File

@ -1,2 +1,4 @@
git.makaay.nl/mauricem/go-parsekit v0.0.0-20190521150537-747456517939 h1:cMBHhfSJR2BZgVN7NmP+c2agNlXDef4Iz6+XQp5AqdU=
git.makaay.nl/mauricem/go-parsekit v0.0.0-20190521150537-747456517939/go.mod h1:/mo+aM5Im5rkBqBvXTAsVR0//OfsAAiFyvuxxcxGGlU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml-test v0.1.0/go.mod h1:xhCXZ5sE5Y4uZrhYxoOUTfGFH+wVyuKlcOC371elHo8=

8
comment_test.go → parse/comment_test.go
View File

@ -1,10 +1,10 @@
package toml
package parse
import (
"testing"
)
func TestComment2(t *testing.T) {
func TestComment(t *testing.T) {
for _, test := range []parseTest{
{``, `{}`, `unexpected end of file (expected comment) at start of file`},
{`#`, `{}`, ``},
@ -13,9 +13,9 @@ func TestComment2(t *testing.T) {
{"# ending in EOL & EOF\r\n", `{}`, ``},
{`# \xxx/ \u can't escape/`, `{}`, ``},
{"# \tlexe\r accepts embedded ca\r\riage \returns\r\n", `{}`, ``},
{"# with data and newline\ncode continues here", `{}`, `unexpected input (expected end of file) at line 2, column 1`},
{"# with data and newline\ncode continues here", `{}`, `unexpected input (expected a value assignment) at line 2, column 5`},
} {
p := newParser()
testParse(t, p, p.startComment, test)
testParse(t, p, p.startDocument, test)
}
}

32
parse/document.go Normal file
View File

@ -0,0 +1,32 @@
package parse
import (
"git.makaay.nl/mauricem/go-parsekit/parse"
)
var (
// Keys may be either bare or quoted.
detectKey = c.Any(bareKeyRune, a.SingleQuote, a.DoubleQuote)
// Both [tables] and [[arrays of tables]] start with a square open bracket.
detectTableOrArrayOfTables = a.SquareOpen
)
func (t *parser) startDocument(p *parse.API) {
for {
p.Accept(dropWhitespace)
p.Accept(dropComment)
switch {
case p.Peek(detectTableOrArrayOfTables):
p.Handle(t.startTable)
case p.Peek(detectKey):
p.Handle(t.startKeyValuePair)
default:
p.ExpectEndOfFile()
return
}
if p.IsStoppedOrInError() {
return
}
}
}

23
helpers_test.go → parse/helpers_test.go
View File

@ -1,4 +1,4 @@
package toml
package parse
import (
"fmt"
@ -7,27 +7,6 @@ import (
"git.makaay.nl/mauricem/go-parsekit/parse"
)
func testAST(t *testing.T, code func() (error, *parser), expectedError string, expectedData string) {
err, p := code()
if expectedError == "" {
if err != nil {
t.Fatalf("Unexpected error: %s", err)
}
} else {
if err == nil {
t.Fatalf("An error was expected, but no error was returned")
} else if err.Error() != expectedError {
t.Fatalf("Unexpected error:\nexpected: %s\nactual: %s\n", expectedError, err.Error())
}
}
if expectedData == "" {
return
}
if expectedData != p.Root.String() {
t.Fatalf("Unexpected data after parsing:\nexpected: %s\nactual: %s\n", expectedData, p.Root.String())
}
}
type parseTest struct {
input interface{}
expected string

46
keyvaluepair.go → parse/keyvaluepair.go
View File

@ -1,10 +1,11 @@
package toml
package parse
import (
"git.makaay.nl/mauricem/go-parsekit/parse"
)
// The primary building block of a TOML document is the key/value pair.
// The primary building block of a TOML document is the table.
// Tables are filled with key/value pairs.
var (
// Keys are on the left of the equals sign and values are on the right.
@ -13,55 +14,34 @@ var (
// broken over multiple lines).
keyAssignment = c.Seq(dropBlanks, a.Equal, dropBlanks)
// A key may be either bare, quoted or dotted. Bare keys may only
// A key may be either bare, quoted or dotted. Bare keys may only
// contain ASCII letters, ASCII digits, underscores, and dashes
// (A-Za-z0-9_-). Note that bare keys are allowed to be composed of only
// ASCII digits, e.g. 1234, but are always interpreted as strings.
bareKeyRune = c.Any(a.ASCIILower, a.ASCIIUpper, a.Digit, a.Underscore, a.Minus)
bareKey = c.OneOrMore(bareKeyRune)
//
// Quoted keys follow the exact same rules as either basic strings or
// literal strings and allow you to use a much broader set of key names.
// Best practice is to use bare keys except when absolutely necessary.
// A bare key must be non-empty, but an empty quoted key is allowed
// (though discouraged).
startOfKey = c.Any(bareKeyRune, a.SingleQuote, a.DoubleQuote)
bareKeyRune = c.Any(a.ASCIILower, a.ASCIIUpper, a.Digit, a.Underscore, a.Minus)
bareKey = c.OneOrMore(bareKeyRune)
// Dotted keys are a sequence of bare or quoted keys joined with a dot.
// This allows for grouping similar properties together. Blanks
// around dot-separated parts are ignored, however, best practice is to
// not use any extraneous blanks.
keySeparatorDot = c.Seq(dropBlanks, a.Dot, dropBlanks)
// Both [tables] and [[arrays of tables]] start with a square open bracket.
startOfTableOrArrayOfTables = a.SquareOpen
)
func (t *parser) startKeyValuePair(p *parse.API) {
for {
p.Accept(dropWhitespace)
switch {
case p.Peek(a.Hash):
p.Handle(t.startComment)
case p.Peek(startOfTableOrArrayOfTables):
p.Handle(t.startTable)
case p.Peek(startOfKey):
key, ok := t.parseKey(p, []string{})
if ok && p.Handle(t.startAssignment) {
if value, ok := t.parseValue(p); ok {
err := t.setKeyValuePair(key, value)
if err != nil {
p.Error("%s", err)
return
}
}
key, ok := t.parseKey(p, []string{})
if ok && p.Handle(t.startAssignment) {
if value, ok := t.parseValue(p); ok {
err := t.SetKeyValuePair(key, value)
if err != nil {
p.Error("%s", err)
}
default:
p.ExpectEndOfFile()
return
}
if p.IsStoppedOrInError() {
return
}
}
}

16
keyvaluepair_test.go → parse/keyvaluepair_test.go
View File

@ -1,4 +1,4 @@
package toml
package parse
import "testing"
@ -14,6 +14,7 @@ func TestKey(t *testing.T) {
{"key1.key2=0", `{"key1": {"key2": 0}}`, ``},
{"key . with . spaces=0", `{"key": {"with": {"spaces": 0}}}`, ``},
{"key \t . \twithtabs\t . \tandspaces=0", `{"key": {"withtabs": {"andspaces": 0}}}`, ``},
{"key1='value1' key2='value2' # on same line", `{"key1": "value1", "key2": "value2"}`, ``},
// Single quoted key tests
{"''=0", `{"": 0}`, ``},
{"'single quoted'=0", `{"single quoted": 0}`, ``},
@ -29,7 +30,7 @@ func TestKey(t *testing.T) {
{`this.'i\s'."madness\t".''=0`, `{"this": {"i\\s": {"madness\t": {"": 0}}}}`, ``},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}
@ -48,10 +49,10 @@ func TestKeyValuePair(t *testing.T) {
{`another = "# This is not a comment"`, `{"another": "# This is not a comment"}`, ``},
{"key1=\"value1\"key2=\"value2\"\r\nkey3a.key3b=\"value3\"", `{"key1": "value1", "key2": "value2", "key3a": {"key3b": "value3"}}`, ``},
{"with=\"comments\"# boring \nanother.cool =\"one\" \t # to the end\r\n", `{"another": {"cool": "one"}, "with": "comments"}`, ``},
{"key='value'\nkey='another value'", `{"key": "value"}`, `invalid key/value pair: string item already exists at key [key] at line 2, column 20`},
{"key='value'\nkey='another value'", `{"key": "value"}`, `invalid key/value pair: string value already exists at key [key] at line 2, column 20`},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}
@ -75,10 +76,11 @@ func TestKeyValuePair_ForAllTypes(t *testing.T) {
{"time=15:03:11", `{"time": 15:03:11}`, ``},
{"datetime=2021-02-01 15:03:11.123", `{"datetime": 2021-02-01 15:03:11.123}`, ``},
{"offset_datetime=1111-11-11 11:11:11.111111111+11:11", `{"offset_datetime": 1111-11-11T11:11:11.111111111+11:11}`, ``},
{"static_array=['a', 'static', 'array']", `{"static_array": ["a", "static", "array"]}`, ``},
{"inline_array=['a', 'inline', 'array']", `{"inline_array": ["a", "inline", "array"]}`, ``},
{`inline_table={"inline"="table"}`, `{"inline_table": {"inline": "table"}}`, ``},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}
@ -140,6 +142,6 @@ func TestKeyValuePair_ExamplesFromSpecification(t *testing.T) {
`{"points": [{"x": 1, "y": 2, "z": 3}, {"x": 7, "y": 8, "z": 9}, {"x": 2, "y": 4, "z": 8}]}`, ``},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}

56
parse/parse.go Normal file
View File

@ -0,0 +1,56 @@
package parse
import (
"git.makaay.nl/mauricem/go-parsekit/parse"
"git.makaay.nl/mauricem/go-parsekit/tokenize"
"git.makaay.nl/mauricem/go-toml/ast"
)
// Some globally useful tokenizer definitions.
var (
c, a, m, tok = tokenize.C, tokenize.A, tokenize.M, tokenize.T
// From the specs: "Whitespace means tab (0x09) or space (0x20)."
// In this package, we name this a blank, to be in line with the
// terminology as used in parsekit.
blank = a.Runes('\t', ' ')
// Newline means LF (0x0A) or CRLF (0x0D0A).
// This matches the default newline as defined by parsekit.
newline = a.Newline
dropBlanks = m.Drop(c.ZeroOrMore(blank))
dropWhitespace = m.Drop(c.ZeroOrMore(blank.Or(newline)))
// A '#' hash symbol marks the rest of the line as a comment.
// All characters up to the end of the line are included in the comment.
dropComment = m.Drop(c.Seq(a.Hash, c.ZeroOrMore(c.Not(a.EndOfLine)), m.Drop(a.EndOfLine)))
)
// parser embeds the TOML ast.Document, so it can be extended with methods
// that implement the parsing logic. This makes the ast.Document part of
// the state of the parser, making it possible to let parsing code call
// ast.Document methods directly to build the abstract syntax tree for the
// parsed TOML input.
type parser struct {
*ast.Document
}
func newParser() *parser {
doc := ast.NewDocument()
return &parser{doc}
}
// Run the TOML parser against the provided input data.
//
// For an overview of allowed inputs, take a look at the documentation for
// parsekit.read.New().
//
// This function returns a TOML ast structure and an error (or nil when
// no error occurred). When an error occurred, the TOML ast struct will
// contain the data that could be parsed up to the error.
func Run(input interface{}) (ast.Table, error) {
p := newParser()
err := parse.New(p.startDocument)(input)
return p.Root, err
}

23
parse/parse_test.go Normal file
View File

@ -0,0 +1,23 @@
package parse_test
import (
"fmt"
"git.makaay.nl/mauricem/go-toml/parse"
)
func ExampleRun() {
doc, err := parse.Run("key = 'value' key2 = 'another value'")
fmt.Println(doc, err)
doc, err = parse.Run("key = 'value'\n[table]\nanother_key = 'another one'")
fmt.Println(doc, err)
doc, err = parse.Run("key1 = 'valid' key2 = invalid")
fmt.Println(doc, err)
// Output:
// {"key": "value", "key2": "another value"} <nil>
// {"key": "value", "table": {"another_key": "another one"}} <nil>
// {"key1": "valid"} unexpected input (expected a value) at line 1, column 23
}

5
value.go → parse/value.go
View File

@ -1,7 +1,8 @@
package toml
package parse
import (
"git.makaay.nl/mauricem/go-parsekit/parse"
"git.makaay.nl/mauricem/go-toml/ast"
)
var (
@ -16,7 +17,7 @@ var (
// Values must be of the following types: String, Integer, Float, Boolean,
// Datetime, Array, or Inline Table. Unspecified values are invalid.
func (t *parser) parseValue(p *parse.API) (*item, bool) {
func (t *parser) parseValue(p *parse.API) (*ast.Value, bool) {
switch {
case p.Peek(detectString):
return t.parseString(p)

21
value_array.go → parse/value_array.go
View File

@ -1,7 +1,8 @@
package toml
package parse
import (
"git.makaay.nl/mauricem/go-parsekit/parse"
"git.makaay.nl/mauricem/go-toml/ast"
)
// Arrays are square brackets with values inside. Whitespace is ignored.
@ -30,13 +31,13 @@ import (
// 2, # this is ok
// ]
var (
arraySpace = c.ZeroOrMore(c.Any(blank, newline, comment))
arraySpace = c.ZeroOrMore(c.Any(blank, newline, dropComment))
arrayOpen = a.SquareOpen.Then(arraySpace)
arraySeparator = c.Seq(arraySpace, a.Comma, arraySpace)
arrayClose = c.Seq(c.Optional(arraySpace.Then(a.Comma)), arraySpace, a.SquareClose)
)
func (t *parser) parseArray(p *parse.API) (*item, bool) {
func (t *parser) parseArray(p *parse.API) (*ast.Value, bool) {
// Check for the start of the array.
if !p.Accept(arrayOpen) {
p.Expected("an array")
@ -45,11 +46,11 @@ func (t *parser) parseArray(p *parse.API) (*item, bool) {
// Check for an empty array.
if p.Accept(arrayClose) {
return newItem(tArray), true
return ast.NewValue(ast.TypeArray), true
}
// Not an empty array, parse the array items.
items := []interface{}{}
// Not an empty array, parse the array values.
values := []interface{}{}
for {
// Check for a value item.
value, ok := t.parseValue(p)
@ -59,16 +60,16 @@ func (t *parser) parseArray(p *parse.API) (*item, bool) {
// Data types may not be mixed (different ways to define strings should be
// considered the same type, and so should arrays with different element types).
if len(items) > 0 && value.Type != items[0].(*item).Type {
p.Error("type mismatch in array of %ss: found an item of type %s", items[0].(*item).Type, value.Type)
if len(values) > 0 && value.Type != values[0].(*ast.Value).Type {
p.Error("type mismatch in array of %ss: found an item of type %s", values[0].(*ast.Value).Type, value.Type)
return nil, false
}
items = append(items, value)
values = append(values, value)
// Check for the end of the array.
if p.Accept(arrayClose) {
return newItem(tArray, items...), true
return ast.NewValue(ast.TypeArray, values...), true
}
// Not the end of the array? Then we should find an array separator.

4
value_array_test.go → parse/value_array_test.go
View File

@ -1,4 +1,4 @@
package toml
package parse
import (
"testing"
@ -42,6 +42,6 @@ func TestArray(t *testing.T) {
{`x=[[1],'a']`, `{}`, `type mismatch in array of static arrays: found an item of type string at line 1, column 11`},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}

22
parse/value_boolean.go Normal file
View File

@ -0,0 +1,22 @@
package parse
import (
"git.makaay.nl/mauricem/go-parsekit/parse"
"git.makaay.nl/mauricem/go-toml/ast"
)
var falseValue = ast.NewValue(ast.TypeBoolean, false)
var trueValue = ast.NewValue(ast.TypeBoolean, true)
// Booleans are just the tokens you're used to. Always lowercase.
func (t *parser) parseBoolean(p *parse.API) (*ast.Value, bool) {
switch {
case p.Accept(a.Str("true")):
return trueValue, true
case p.Accept(a.Str("false")):
return falseValue, true
default:
p.Expected("true or false")
return nil, false
}
}

4
value_boolean_test.go → parse/value_boolean_test.go
View File

@ -1,4 +1,4 @@
package toml
package parse
import (
"testing"
@ -22,6 +22,6 @@ func TestBoolean(t *testing.T) {
{`x=0`, `{"x": 0}`, ``},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}

19
value_datetime.go → parse/value_datetime.go
View File

@ -1,10 +1,11 @@
package toml
package parse
import (
"time"
"git.makaay.nl/mauricem/go-parsekit/parse"
"git.makaay.nl/mauricem/go-parsekit/tokenize"
"git.makaay.nl/mauricem/go-toml/ast"
)
var (
@ -66,14 +67,14 @@ var (
// The full date/time parse format, based on the above definitions.
// The first token denotes the type of date/time value.
// The rest of the tokens contain layout fragments for time.Parse().
offsetDateTime = tok.Str(tOffsetDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok, tzTok))
localDateTime = tok.Str(tLocalDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok))
localDate = tok.Str(tLocalDate, dateTok)
localTime = tok.Str(tLocalTime, c.Seq(timeTok, microTok))
offsetDateTime = tok.Str(ast.TypeOffsetDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok, tzTok))
localDateTime = tok.Str(ast.TypeLocalDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok))
localDate = tok.Str(ast.TypeLocalDate, dateTok)
localTime = tok.Str(ast.TypeLocalTime, c.Seq(timeTok, microTok))
datetime = c.Any(offsetDateTime, localDateTime, localDate, localTime)
)
func (t *parser) parseDateTime(p *parse.API) (*item, bool) {
func (t *parser) parseDateTime(p *parse.API) (*ast.Value, bool) {
if !p.Accept(datetime) {
p.Expected("a date and/or time")
return nil, false
@ -82,7 +83,7 @@ func (t *parser) parseDateTime(p *parse.API) (*item, bool) {
valueType := getDateTimeValueType(&tokens)
input, value, err := getDateTimeValue(&tokens)
if err == nil {
return newItem(valueType, value), true
return ast.NewValue(valueType, value), true
}
p.Error("invalid date/time value %s: %s", input, err)
return nil, false
@ -90,8 +91,8 @@ func (t *parser) parseDateTime(p *parse.API) (*item, bool) {
// The first token is a token that wraps the complete date/time input.
// Its type denotes the type of date/time value that it wraps.
func getDateTimeValueType(tokens *[]*tokenize.Token) valueType {
return (*tokens)[0].Type.(valueType)
func getDateTimeValueType(tokens *[]*tokenize.Token) ast.ValueType {
return (*tokens)[0].Type.(ast.ValueType)
}
// The rest of the tokens contain fragments that can be used with

4
value_datetime_test.go → parse/value_datetime_test.go
View File

@ -1,4 +1,4 @@
package toml
package parse
import (
"testing"
@ -35,6 +35,6 @@ func TestDateTime(t *testing.T) {
{`x=25:01:01`, `{}`, `invalid date/time value 25:01:01: parsing time "25:01:01": hour out of range at line 1, column 11`},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}

21
value_number.go → parse/value_number.go
View File

@ -1,10 +1,11 @@
package toml
package parse
import (
"math"
"strconv"
"git.makaay.nl/mauricem/go-parsekit/parse"
"git.makaay.nl/mauricem/go-toml/ast"
)
var (
@ -65,28 +66,28 @@ var (
nan = a.Signed(a.Str("nan"))
)
func (t *parser) parseNumber(p *parse.API) (*item, bool) {
func (t *parser) parseNumber(p *parse.API) (*ast.Value, bool) {
switch {
case p.Accept(tok.Float64(nil, float)):
return newItem(tFloat, p.Result().Value(0).(float64)), true
return ast.NewValue(ast.TypeFloat, p.Result().Value(0).(float64)), true
case p.Accept(nan):
return newItem(tFloat, math.NaN()), true
return ast.NewValue(ast.TypeFloat, math.NaN()), true
case p.Accept(inf):
if p.Result().Rune(0) == '-' {
return newItem(tFloat, math.Inf(-1)), true
return ast.NewValue(ast.TypeFloat, math.Inf(-1)), true
}
return newItem(tFloat, math.Inf(+1)), true
return ast.NewValue(ast.TypeFloat, math.Inf(+1)), true
case p.Accept(a.Zero):
return t.parseIntegerStartingWithZero(p)
case p.Accept(tok.Int64(nil, integer)):
return newItem(tInteger, p.Result().Value(0).(int64)), true
return ast.NewValue(ast.TypeInteger, p.Result().Value(0).(int64)), true
default:
p.Expected("a number")
return nil, false
}
}
func (t *parser) parseIntegerStartingWithZero(p *parse.API) (*item, bool) {
func (t *parser) parseIntegerStartingWithZero(p *parse.API) (*ast.Value, bool) {
var value int64
var err error
switch {
@ -97,10 +98,10 @@ func (t *parser) parseIntegerStartingWithZero(p *parse.API) (*item, bool) {
case p.Accept(binary):
value, err = strconv.ParseInt(p.Result().Value(0).(string), 2, 64)
default:
return newItem(tInteger, int64(0)), true
return ast.NewValue(ast.TypeInteger, int64(0)), true
}
if err == nil {
return newItem(tInteger, value), true
return ast.NewValue(ast.TypeInteger, value), true
}
p.Error("invalid integer value 0%s: %s", p.Result().String(), err)
return nil, false

6
value_number_test.go → parse/value_number_test.go
View File

@ -1,4 +1,4 @@
package toml
package parse
import (
"testing"
@ -75,7 +75,7 @@ func TestInteger(t *testing.T) {
{`x=0b10000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000`, `{}`, `invalid integer value 0b1000000000000000000000000000000000000000000000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000000000000000000000000000000000000000000000": value out of range at line 1, column 76`},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}
@ -101,6 +101,6 @@ func TestFloat(t *testing.T) {
{`x=-inf`, `{"x": -Inf}`, ``},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}

7
value_string.go → parse/value_string.go
View File

@ -1,9 +1,10 @@
package toml
package parse
import (
"strings"
"git.makaay.nl/mauricem/go-parsekit/parse"
"git.makaay.nl/mauricem/go-toml/ast"
)
var (
@ -43,7 +44,7 @@ var (
// There are four ways to express strings: basic, multi-line basic, literal and
// multi-line literal. All strings must contain only valid UTF-8 characters.
func (t *parser) parseString(p *parse.API) (*item, bool) {
func (t *parser) parseString(p *parse.API) (*ast.Value, bool) {
var value string
var ok bool
switch {
@ -59,7 +60,7 @@ func (t *parser) parseString(p *parse.API) (*item, bool) {
p.Expected("a string value")
}
if ok {
return newItem(tString, value), ok
return ast.NewValue(ast.TypeString, value), ok
}
return nil, false
}

14
value_string_test.go → parse/value_string_test.go
View File

@ -1,4 +1,4 @@
package toml
package parse
import (
"fmt"
@ -46,7 +46,7 @@ func TestString(t *testing.T) {
{"x='''\n literal multi-line\n string value\n'''", `{"x": " literal multi-line\n string value\n"}`, ``},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}
@ -65,7 +65,7 @@ func TestBasipString(t *testing.T) {
{"x=\"Character that must be escaped \x7f\"", `{}`, `invalid character in string value: '\u007f' (must be escaped) at line 1, column 35`},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}
@ -83,7 +83,7 @@ func TestMultiLineBasipString(t *testing.T) {
{"x=\"\"\"Invalid rune \xcd\"\"\"", `{}`, `invalid UTF8 rune at line 1, column 19`},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}
@ -101,7 +101,7 @@ func TestLiteralString(t *testing.T) {
{"x='Invalid rune \xcd'", `{}`, `invalid UTF8 rune at line 1, column 17`},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}
@ -116,7 +116,7 @@ func TestMultiLineLiteralString(t *testing.T) {
{"x='''No invalid runes allowed \xcd'''", `{}`, `invalid UTF8 rune at line 1, column 31`},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}
@ -127,6 +127,6 @@ func TestBasipStringWithUnescapedControlCharacters(t *testing.T) {
p := newParser()
input := fmt.Sprintf(`x="%c"`, rune(i))
expected := fmt.Sprintf(`invalid character in string value: %q (must be escaped) at line 1, column 4`, rune(i))
testParse(t, p, p.startKeyValuePair, parseTest{input, "{}", expected})
testParse(t, p, p.startDocument, parseTest{input, "{}", expected})
}
}

31
value_table.go → parse/value_table.go
View File

@ -1,17 +1,18 @@
package toml
package parse
import (
"git.makaay.nl/mauricem/go-parsekit/parse"
"git.makaay.nl/mauricem/go-toml/ast"
)
var (
// Opener and closer for [table].
tableOpen = c.Seq(dropBlanks, a.SquareOpen, dropBlanks)
tableClose = c.Seq(dropBlanks, a.SquareClose, dropBlanks, a.EndOfLine.Or(comment))
tableClose = c.Seq(dropBlanks, a.SquareClose, dropBlanks, a.EndOfLine.Or(dropComment))
// Opener and closer for [[array.of.tables]].
tableArrayOpen = c.Seq(dropBlanks, a.SquareOpen, a.SquareOpen, dropBlanks)
tableArrayClose = c.Seq(dropBlanks, a.SquareClose, a.SquareClose, dropBlanks, a.EndOfLine.Or(comment))
tableArrayClose = c.Seq(dropBlanks, a.SquareClose, a.SquareClose, dropBlanks, a.EndOfLine.Or(dropComment))
// Opener, separator and closer for { inline: "tables" }.
inlineTableOpen = c.Seq(dropBlanks, a.CurlyOpen, dropBlanks)
@ -74,11 +75,11 @@ func (t *parser) startArrayOfTables(p *parse.API) {
p.Expected("closing ']]' for array of tables name")
return
}
if err := t.openArrayOfTables(key); err != nil {
if err := t.OpenArrayOfTables(key); err != nil {
p.Error("%s", err)
return
}
p.Handle(t.startKeyValuePair)
p.Handle(t.startDocument)
}
}
@ -126,11 +127,11 @@ func (t *parser) startPlainTable(p *parse.API) {
p.Expected("closing ']' for table name")
return
}
if err := t.openTable(key); err != nil {
if err := t.OpenTable(key); err != nil {
p.Error("%s", err)
return
}
p.Handle(t.startKeyValuePair)
p.Handle(t.startDocument)
}
}
@ -150,34 +151,34 @@ func (t *parser) startPlainTable(p *parse.API) {
// name = { first = "Tom", last = "Preston-Werner" }
// point = { x = 1, y = 2 }
// animal = { type.name = "pug" }
func (t *parser) parseInlineTable(p *parse.API) (*item, bool) {
func (t *parser) parseInlineTable(p *parse.API) (*ast.Value, bool) {
// Check for the start of the array.
if !p.Accept(inlineTableOpen) {
p.Expected("an inline table")
return nil, false
}
subt := newParser()
subdoc := newParser()
// Check for an empty inline table.
if p.Accept(inlineTableClose) {
return newItem(tTable, subt.Root), true
return ast.NewValue(ast.TypeTable, subdoc.Root), true
}
// Not an empty table, parse the table data.
for {
key, ok := subt.parseKey(p, []string{})
key, ok := subdoc.parseKey(p, []string{})
if !ok {
return nil, false
}
if !p.Handle(subt.startAssignment) {
if !p.Handle(subdoc.startAssignment) {
return nil, false
}
value, ok := subt.parseValue(p)
value, ok := subdoc.parseValue(p)
if !ok {
return nil, false
}
err := subt.setKeyValuePair(key, value)
err := subdoc.SetKeyValuePair(key, value)
if err != nil {
p.Error("%s", err)
return nil, false
@ -185,7 +186,7 @@ func (t *parser) parseInlineTable(p *parse.API) (*item, bool) {
// Check for the end of the inline table.
if p.Accept(inlineTableClose) {
return newItem(tTable, subt.Root), true
return ast.NewValue(ast.TypeTable, subdoc.Root), true
}
// Not the end of the inline table? Then we should find a key/value pair separator.

12
value_table_test.go → parse/value_table_test.go
View File

@ -1,4 +1,4 @@
package toml
package parse
import (
"testing"
@ -43,7 +43,7 @@ func TestTable(t *testing.T) {
{"[a]\n[b] #another table \na=1\n", `{"a": {}, "b": {"a": 1}}`, ``},
{"[a]\nx=1\ny=2\n[b] #another table \nx=1\ny=2021-01-01", `{"a": {"x": 1, "y": 2}, "b": {"x": 1, "y": 2021-01-01}}`, ``},
{"[a]\nx=1\ny=2\n[a.b] #subtable \nx=1\ny=2021-01-01", `{"a": {"b": {"x": 1, "y": 2021-01-01}, "x": 1, "y": 2}}`, ``},
{"[a]\n[a]", `{"a": {}}`, `invalid table: table item already exists at key [a] at line 2, column 4`},
{"[a]\n[a]", `{"a": {}}`, `invalid table: table value already exists at key [a] at line 2, column 4`},
} {
p := newParser()
testParse(t, p, p.startTable, test)
@ -75,7 +75,7 @@ func TestArrayOfTables(t *testing.T) {
{"[[a]]\n[['a']]\n[b]\n[[\"a\"]]", `{"a": [{}, {}, {}], "b": {}}`, ``},
{"[[a]]\nx=1\n[['a']]\nx=2\ny=3\n[[\"a\"]]", `{"a": [{"x": 1}, {"x": 2, "y": 3}, {}]}`, ``},
{"[a]\n[[a.b]]\nx=1\n[[a.b]]\nx=2\n[a.c]\ny=1234", `{"a": {"b": [{"x": 1}, {"x": 2}], "c": {"y": 1234}}}`, ``},
{"[a]\n[[a]]", `{"a": {}}`, `invalid table array: table item already exists at key [a] at line 2, column 6`},
{"[a]\n[[a]]", `{"a": {}}`, `invalid table array: table value already exists at key [a] at line 2, column 6`},
} {
p := newParser()
testParse(t, p, p.startTable, test)
@ -106,10 +106,10 @@ func TestInlineTable(t *testing.T) {
{"x={a=1", `{}`, `unexpected end of file (expected an array separator) at line 1, column 7`},
{"x={a=1,", `{}`, `unexpected end of file (expected a key name) at line 1, column 8`},
{"x={a=1,}", `{}`, `unexpected input (expected a key name) at line 1, column 8`},
{"x={a=1,a=2}", `{}`, `invalid key/value pair: integer item already exists at key [a] at line 1, column 11`},
{"x={a={b=1,b=2}}", `{}`, `invalid key/value pair: integer item already exists at key [b] at line 1, column 14`},
{"x={a=1,a=2}", `{}`, `invalid key/value pair: integer value already exists at key [a] at line 1, column 11`},
{"x={a={b=1,b=2}}", `{}`, `invalid key/value pair: integer value already exists at key [b] at line 1, column 14`},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}

4
value_test.go → parse/value_test.go
View File

@ -1,4 +1,4 @@
package toml
package parse
import "testing"
@ -40,6 +40,6 @@ func TestValue(t *testing.T) {
{"x={1=1,2=2,3=3}", `{"x": {"1": 1, "2": 2, "3": 3}}`, ``},
} {
p := newParser()
testParse(t, p, p.startKeyValuePair, test)
testParse(t, p, p.startDocument, test)
}
}

22
toml.go
View File

@ -1,22 +0,0 @@
package toml
import (
"git.makaay.nl/mauricem/go-parsekit/tokenize"
)
// Some globally useful tokenizer definitions.
var (
c, a, m, tok = tokenize.C, tokenize.A, tokenize.M, tokenize.T
// From the specs: "Whitespace means tab (0x09) or space (0x20)."
// In this package, we name this a blank, to be in line with the
// terminology as used in parsekit.
blank = a.Runes('\t', ' ')
// Newline means LF (0x0A) or CRLF (0x0D0A).
// This matches the default newline as defined by parsekit.
newline = a.Newline
dropBlanks = m.Drop(c.ZeroOrMore(blank))
dropWhitespace = m.Drop(c.ZeroOrMore(blank.Or(newline)))
)

21
value_boolean.go
View File

@ -1,21 +0,0 @@
package toml
import (
"git.makaay.nl/mauricem/go-parsekit/parse"
)
var falseItem = newItem(tBoolean, false)
var trueItem = newItem(tBoolean, true)
// Booleans are just the tokens you're used to. Always lowercase.
func (t *parser) parseBoolean(p *parse.API) (*item, bool) {
switch {
case p.Accept(a.Str("true")):
return trueItem, true
case p.Accept(a.Str("false")):
return falseItem, true
default:
p.Expected("true or false")
return nil, false
}
}