OpenSourceProjects
/
go-toml
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Huge overhaul in AST handling. The AST is now fully integrated with the parser, which has been simplified quite a bit because of this.

This commit is contained in:
Maurice Makaay 2019-06-25 21:29:05 +00:00
parent 15560b29b0
commit c536dd1243
21 changed files with 928 additions and 690 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

308
ast.go
View File

@ -2,6 +2,7 @@ package toml
import ( import (
"fmt" "fmt"
"regexp"
"sort" "sort"
"strings" "strings"
"time" "time"
@ -14,18 +15,12 @@ type item struct {
} }
// table represents a TOML table. // table represents a TOML table.
type table map[string]item type table map[string]*item
// itemType identifies the semantic role of a TOML item. // itemType identifies the semantic role of a TOML item.
type itemType string type itemType string
const ( const (
// TODO Would be nice to not need these in the end.
pComment itemType = "comment"
pKey itemType = "key"
pAssign itemType = "assign"
// TODO and just use these data types.
pString itemType = "string" // "various", 'types', """of""", '''strings''' pString itemType = "string" // "various", 'types', """of""", '''strings'''
pInteger itemType = "integer" // 12345, 0xffee12a0, 0o0755, 0b101101011 pInteger itemType = "integer" // 12345, 0xffee12a0, 0o0755, 0b101101011
pFloat itemType = "float" // 10.1234, 143E-12, 43.28377e+4, +inf, -inf, nan pFloat itemType = "float" // 10.1234, 143E-12, 43.28377e+4, +inf, -inf, nan
@ -34,14 +29,176 @@ const (
pLocalDateTime itemType = "datetime" // 2018-12-25 12:12:18.876772533 pLocalDateTime itemType = "datetime" // 2018-12-25 12:12:18.876772533
pLocalDate itemType = "date" // 2017-05-17 pLocalDate itemType = "date" // 2017-05-17
pLocalTime itemType = "time" // 23:01:22 pLocalTime itemType = "time" // 23:01:22
pArray itemType = "array" // defined using an [[array.of.tables]] pArrayOfTables itemType = "array" // defined using an [[array.of.tables]]
pStaticArray itemType = "static array" // defined using ["an", "inline", "array"] pArray itemType = "static array" // defined using ["an", "inline", "array"]
pTable itemType = "table" // defined using { "inline" = "table" } or [standard.table] pTable itemType = "table" // defined using { "inline" = "table" } or [standard.table]
) )
// newItem instantiates a new item struct. // newItem instantiates a new item struct.
func newItem(itemType itemType, data ...interface{}) item { func newItem(itemType itemType, data ...interface{}) *item {
return item{Type: itemType, Data: data} return &item{Type: itemType, Data: data}
}
// newKey instantiates a new key.
func newKey(key ...string) []string {
return key
}
// parser holds the state for the TOML parser. All parsing functions are
// methods of this struct.
type parser struct {
Items []*item // a buffer for holding parsed items
Root table // the root-level TOML table (each TOML doc is implicitly a table)
Current table // the currently active TOML table
CurrentKey []string // the key for the currently active TOML table
}
func newParser() *parser {
p := &parser{Root: make(table)}
p.Current = p.Root
return p
}
func (t *parser) setKeyValuePair(key []string, value *item) error {
// First make sure the table structure for storing the value exists.
node, lastKeyPart, err := t.makeTablePath(key)
if err != nil {
return fmt.Errorf("invalid key/value pair: %s", err)
}
// Check if the key is still free for use.
if existing, ok := node[lastKeyPart]; ok {
path := t.formatKeyPath(key, len(key)-1)
return fmt.Errorf("invalid key/value pair: %s item already exists at key %s", existing.Type, path)
}
// It is, store the value in the table.
node[lastKeyPart] = value
return nil
}
func (t *parser) openTable(key []string) error {
t.CurrentKey = nil
t.Current = t.Root
// Go over all requested levels of the key. For all levels, except the last
// one, it is okay if a table already exists. For at least the last level,
// no table or value must exist, because that would mean we are overwriting
// an existing key/value pair, which is not allowed.
node, lastKeyPart, err := t.makeTablePath(key)
if err != nil {
return fmt.Errorf("invalid table: %s", err)
}
// Check if the key is still free for use.
if existing, ok := node[lastKeyPart]; ok {
path := t.formatKeyPath(key, len(key)-1)
return fmt.Errorf("invalid table: %s item already exists at key %s", existing.Type, path)
}
// The subtable does not exist yet. Create the subtable.
subTable := make(table)
node[lastKeyPart] = newItem(pTable, subTable)
node = subTable
// From here on, key/value pairs are added to the newly defined table.
t.Current = node
t.CurrentKey = key
return nil
}
func (t *parser) openArrayOfTables(key []string) error {
t.CurrentKey = nil
t.Current = t.Root
// Go over all requested levels of the key. For all levels, except the last
// one, it is okay if a table already exists. For the last level, either
// no item must exist (in which case a table array will be created), or a
// table array must exist.
// Other cases would mean we are overwriting an existing key/value pair,
// which is not allowed.
node, lastKeyPart, err := t.makeTablePath(key)
if err != nil {
return fmt.Errorf("invalid table array: %s", err)
}
// At the last key position, there must be either no value yet, or the
// existing value must be a table array. Other values are invalid.
if existing, ok := node[lastKeyPart]; ok {
if existing.Type != pArrayOfTables {
path := t.formatKeyPath(key, len(key)-1)
return fmt.Errorf("invalid table array: %s item already exists at key %s", existing.Type, path)
}
// A table array exists. Add a new table to this array.
array := node[lastKeyPart]
subTable := make(table)
tables := array.Data
tables = append(tables, newItem(pTable, subTable))
array.Data = tables
node = subTable
} else {
// No value exists at the defined key path. Create a new table array.
subTable := make(table)
node[lastKeyPart] = newItem(pArrayOfTables, newItem(pTable, subTable))
node = subTable
}
// From here on, key/value pairs are added to the newly defined table.
t.Current = node
t.CurrentKey = key
return nil
}
func (t *parser) makeTablePath(key []string) (table, string, error) {
node := t.Current
for i, keyPart := range key {
// Arrived at the last key part? Then the path towards that key is
// setup correctly. Return the last part, so the caller can use it.
isLast := i == len(key)-1
if isLast {
return node, keyPart, nil
}
if subItem, ok := node[keyPart]; ok {
// You cannot overwrite an already defined key, regardless its value.
// When a value already exists at the current key, this can only be a table.
if subItem.Type != pTable {
path := t.formatKeyPath(key, i)
return nil, "", fmt.Errorf("%s item already exists at key %s", subItem.Type, path)
}
// All is okay, traverse to the subtable.
node = subItem.Data[0].(table)
} else {
// The subtable does not exist yet. Create the subtable.
subTable := make(table)
node[keyPart] = newItem(pTable, subTable)
node = subTable
}
}
panic("makeTablePath(): empty key provided; a key must have at least one key part")
}
func (t *parser) formatKeyPath(key []string, end int) string {
var sb strings.Builder
sb.WriteRune('[')
if t.CurrentKey != nil {
for i, keyPart := range t.CurrentKey {
if i > 0 {
sb.WriteString("->")
}
sb.WriteString(formatKeyName(keyPart))
}
}
for i, keyPart := range key {
if t.CurrentKey != nil || i > 0 {
sb.WriteString("->")
}
sb.WriteString(formatKeyName(keyPart))
if i == end {
break
}
}
sb.WriteRune(']')
return sb.String()
}
func formatKeyName(key string) string {
if ok, _ := regexp.Match(`^\w+$`, []byte(key)); ok {
return key
}
return fmt.Sprintf("%q", key)
} }
func (t table) String() string { func (t table) String() string {
@ -52,12 +209,6 @@ func (parseItem item) String() string {
switch parseItem.Type { switch parseItem.Type {
case pString: case pString:
return fmt.Sprintf("%q", parseItem.Data[0]) return fmt.Sprintf("%q", parseItem.Data[0])
case pInteger:
return fmt.Sprintf("%v", parseItem.Data[0])
case pFloat:
return fmt.Sprintf("%v", parseItem.Data[0])
case pBoolean:
return fmt.Sprintf("%v", parseItem.Data[0])
case pOffsetDateTime: case pOffsetDateTime:
return parseItem.Data[0].(time.Time).Format(time.RFC3339Nano) return parseItem.Data[0].(time.Time).Format(time.RFC3339Nano)
case pLocalDateTime: case pLocalDateTime:
@ -66,12 +217,12 @@ func (parseItem item) String() string {
return parseItem.Data[0].(time.Time).Format("2006-01-02") return parseItem.Data[0].(time.Time).Format("2006-01-02")
case pLocalTime: case pLocalTime:
return parseItem.Data[0].(time.Time).Format("15:04:05.999999999") return parseItem.Data[0].(time.Time).Format("15:04:05.999999999")
case pArray: case pArrayOfTables:
fallthrough fallthrough
case pStaticArray: case pArray:
items := make([]string, len(parseItem.Data[0].([]item))) items := make([]string, len(parseItem.Data))
for i, d := range parseItem.Data[0].([]item) { for i, value := range parseItem.Data {
items[i] = d.String() items[i] = value.(*item).String()
} }
return fmt.Sprintf("[%s]", strings.Join(items, ", ")) return fmt.Sprintf("[%s]", strings.Join(items, ", "))
case pTable: case pTable:
@ -88,118 +239,7 @@ func (parseItem item) String() string {
items[i] = fmt.Sprintf("%q: %s", k, pairs[k].String()) items[i] = fmt.Sprintf("%q: %s", k, pairs[k].String())
} }
return fmt.Sprintf("{%s}", strings.Join(items, ", ")) return fmt.Sprintf("{%s}", strings.Join(items, ", "))
case pComment:
return fmt.Sprintf("comment(%q)", parseItem.Data[0])
case pKey:
items := make([]string, len(parseItem.Data))
for i, e := range parseItem.Data {
items[i] = fmt.Sprintf("%q", e)
}
return fmt.Sprintf("key(%s)", strings.Join(items, ", "))
case pAssign:
return "="
default: default:
panic(fmt.Sprintf("Missing String() formatting for item type '%s'", parseItem.Type)) return fmt.Sprintf("%v", parseItem.Data[0])
} }
} }
// parser holds the state for the TOML parser. All parsing functions are
// methods of this struct.
type parser struct {
Items []item // a buffer for holding parsed items
Root table // the root-level TOML table (each TOML doc is implicitly a table)
Current table // the currently active TOML table
}
func newParser() *parser {
p := &parser{Root: make(table)}
p.Current = p.Root
return p
}
func (t *parser) addParsedItem(itemType itemType, data ...interface{}) {
t.Items = append(t.Items, newItem(itemType, data...))
}
func (t *parser) setValue(key item, value item) error {
// When the key has multiple elements, then first make sure the table structure
// for storing the value exists.
var valueKey string
node := t.Current
l := len(key.Data)
if l > 1 {
pathKeys := key.Data[0 : l-1]
valueKey = key.Data[l-1].(string)
for i, name := range pathKeys {
name := name.(string)
if subItem, ok := node[name]; ok {
// An item was found at the current key. It is expected to be a table.
if subItem.Type != pTable {
path := formatKeyPath(key, i)
return fmt.Errorf("invalid key used: %s item already exists at key %s", subItem.Type, path)
}
node = subItem.Data[0].(table)
} else {
// No item was found at the current key. Create a new subtable.
subTable := make(table)
node[name] = newItem(pTable, subTable)
node = subTable
}
}
} else {
valueKey = key.Data[0].(string)
node = t.Current
}
if existing, ok := node[valueKey]; ok {
path := "moetnog"
return fmt.Errorf("Cannot store value: %s item already exists at key %s", existing.Type, path)
}
node[valueKey] = value
return nil
}
func (t *parser) openTable(key item) (table, error) {
node := t.Root
// Go over all requested levels of the key. For all levels, except the last
// one, it is okay if a table already exists. For at least the last level,
// no table or value must exist, because that would mean we are overwriting
// an existing key/value pair, which is not allowed.
for i, value := range key.Data {
keyName := value.(string)
if subItem, ok := node[keyName]; ok {
// You cannot overwrite an already defined key, regardless its value.
if subItem.Type != pTable {
path := formatKeyPath(key, i)
return nil, fmt.Errorf("Cannot create table: %s item already exists at key %s", subItem.Type, path)
}
// Like keys, you cannot define any table more than once. Doing so is invalid.
isLast := i == len(key.Data)-1
if isLast {
path := formatKeyPath(key, i)
return nil, fmt.Errorf("Cannot create table: table for key %s already exists", path)
}
node = subItem.Data[0].(table)
} else {
// Create the subtable.
subTable := make(table)
node[keyName] = newItem(pTable, subTable)
node = subTable
}
}
// From here on, key/value pairs are added to the newly defined table.
t.Current = node
return node, nil
}
func formatKeyPath(key item, end int) string {
var sb strings.Builder
for i := 0; i <= end; i++ {
if i > 0 {
sb.WriteRune('.')
}
sb.WriteString(fmt.Sprintf("%q", key.Data[i].(string)))
}
return sb.String()
}

174
ast_test.go
View File

@ -7,63 +7,171 @@ import (
func TestAST_ConstructStructure(t *testing.T) { func TestAST_ConstructStructure(t *testing.T) {
testAST(t, func() (error, *parser) { testAST(t, func() (error, *parser) {
p := newParser() p := newParser()
p.setValue(newItem(pKey, "ding"), newItem(pInteger, 10)) p.setKeyValuePair(newKey("ding"), newItem(pInteger, 10))
p.setValue(newItem(pKey, "dong"), newItem(pString, "not a song")) p.setKeyValuePair(newKey("dong"), newItem(pString, "not a song"))
p.openTable(newItem(pKey, "key1", "key2 a")) p.openTable(newKey("key1", "key2 a"))
p.setValue(newItem(pKey, "dooh"), newItem(pBoolean, true)) p.setKeyValuePair(newKey("dooh"), newItem(pBoolean, true))
p.setValue(newItem(pKey, "dah"), newItem(pBoolean, false)) p.setKeyValuePair(newKey("dah"), newItem(pBoolean, false))
p.openTable(newItem(pKey, "key1", "key2 b")) p.openTable(newKey("key1", "key2 b"))
p.setValue(newItem(pKey, "dieh"), newItem(pFloat, 1.111)) p.setKeyValuePair(newKey("dieh"), newItem(pFloat, 1.111))
p.setValue(newItem(pKey, "duh"), newItem(pFloat, 1.18e-12)) p.setKeyValuePair(newKey("duh"), newItem(pFloat, 1.18e-12))
p.setKeyValuePair(newKey("foo", "bar"), newItem(pArrayOfTables, newItem(pInteger, 1), newItem(pInteger, 2)))
p.openArrayOfTables(newKey("aaah", "table array"))
p.setKeyValuePair(newKey("a"), newItem(pFloat, 1.234))
p.openArrayOfTables(newKey("aaah", "table array"))
p.setKeyValuePair(newKey("b"), newItem(pFloat, 2.345))
p.setKeyValuePair(newKey("c"), newItem(pString, "bingo!"))
p.openArrayOfTables(newKey("aaah", "table array"))
return nil, p return nil, p
}, "", `{"ding": 10, "dong": "not a song", "key1": {"key2 a": {"dah": false, "dooh": true}, "key2 b": {"dieh": 1.111, "duh": 1.18e-12}}}`) },
"",
`{"aaah": {"table array": [{"a": 1.234}, {"b": 2.345, "c": "bingo!"}, {}]}, `+
`"ding": 10, "dong": "not a song", `+
`"key1": {"key2 a": {"dah": false, "dooh": true}, "key2 b": {"dieh": 1.111, "duh": 1.18e-12, "foo": {"bar": [1, 2]}}}}`)
}
func TestAST_EmptyKeyForCreatingTablePath_Panics(t *testing.T) {
defer func() {
r := recover()
if r.(string) != "makeTablePath(): empty key provided; a key must have at least one key part" {
t.Fatalf("Did not get the expected panic message")
}
}()
p := newParser()
p.openTable(newKey())
} }
func TestAST_StoreValueInRootTable(t *testing.T) { func TestAST_StoreValueInRootTable(t *testing.T) {
testAST(t, func() (error, *parser) { testAST(t, func() (error, *parser) {
p := newParser() p := newParser()
p.setValue(newItem(pKey, "key1"), newItem(pString, "value1")) p.setKeyValuePair(newKey("key1"), newItem(pString, "value1"))
return p.setValue(newItem(pKey, "key2"), newItem(pString, "value2")), p return p.setKeyValuePair(newKey("key2"), newItem(pString, "value2")), p
}, "", `{"key1": "value1", "key2": "value2"}`) },
"",
`{"key1": "value1", "key2": "value2"}`)
} }
func TestAST_StoreValueWithMultipartKey_CreatesTableHierarchy(t *testing.T) { func TestAST_StoreValueWithMultipartKey_CreatesTableHierarchy(t *testing.T) {
testAST(t, func() (error, *parser) { testAST(t, func() (error, *parser) {
p := newParser() p := newParser()
return p.setValue(newItem(pKey, "key1", "key2", "key3"), newItem(pString, "value")), p return p.setKeyValuePair(newKey("key1", "key2", "key3"), newItem(pString, "value")), p
}, "", `{"key1": {"key2": {"key3": "value"}}}`) },
} "",
`{"key1": {"key2": {"key3": "value"}}}`)
func TestAST_StoreValueWithMultipartKey_UnderSubtable_CreatesTableHierarchy(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newItem(pKey, "tablekey1", "tablekey2"))
return p.setValue(newItem(pKey, "valuekey1", "valuekey2", "valuekey3"), newItem(pString, "value")), p
}, "", `{"tablekey1": {"tablekey2": {"valuekey1": {"valuekey2": {"valuekey3": "value"}}}}}`)
} }
func TestAST_StoreDuplicateKeyInRootTable_ReturnsError(t *testing.T) { func TestAST_StoreDuplicateKeyInRootTable_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) { testAST(t, func() (error, *parser) {
p := newParser() p := newParser()
p.setValue(newItem(pKey, "key"), newItem(pString, "value")) p.setKeyValuePair(newKey("key"), newItem(pString, "value"))
return p.setValue(newItem(pKey, "key"), newItem(pInteger, 321)), p return p.setKeyValuePair(newKey("key"), newItem(pInteger, 321)), p
}, `Cannot store value: string item already exists at key "key"`, "") },
`invalid key/value pair: string item already exists at key [key]`,
`{"key": "value"}`)
}
func TestAST_StoreValueWithMultipartKey_UnderSubtable_CreatesTableHierarchy(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("tablekey1", "tablekey2"))
return p.setKeyValuePair(newKey("valuekey1", "valuekey2", "valuekey3"), newItem(pString, "value")), p
},
"",
`{"tablekey1": {"tablekey2": {"valuekey1": {"valuekey2": {"valuekey3": "value"}}}}}`)
}
func TestAST_StoreKeyPathWherePathContainsNonTableAlready_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("key1"))
p.setKeyValuePair(newKey("key2"), newItem(pInteger, 0))
return p.setKeyValuePair(newKey("key2", "key3"), newItem(pString, "value")), p
},
`invalid key/value pair: integer item already exists at key [key1->key2]`,
`{"key1": {"key2": 0}}`)
} }
func TestAST_GivenExistingTableAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { func TestAST_GivenExistingTableAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) { testAST(t, func() (error, *parser) {
p := newParser() p := newParser()
p.openTable(newItem(pKey, "key1", "key2")) p.openTable(newKey("key1", "key2"))
_, err := p.openTable(newItem(pKey, "key1", "key2")) return p.openTable(newKey("key1", "key2")), p
return err, p },
}, `Cannot create table: table for key "key1"."key2" already exists`, "") `invalid table: table item already exists at key [key1->key2]`,
`{"key1": {"key2": {}}}`)
} }
func TestAST_GivenExistingItemAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { func TestAST_GivenExistingItemAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) { testAST(t, func() (error, *parser) {
p := newParser() p := newParser()
p.Root["key"] = newItem(pString, "value") p.setKeyValuePair(newKey("key"), newItem(pString, "value"))
_, err := p.openTable(newItem(pKey, "key")) return p.openTable(newKey("key")), p
return err, p },
}, `Cannot create table: string item already exists at key "key"`, "") `invalid table: string item already exists at key [key]`,
`{"key": "value"}`)
}
func TestAST_GivenExistingItemInKeyPath_CreatingTable_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.setKeyValuePair(newKey("key"), newItem(pString, "value"))
return p.openTable(newKey("key", "subkey")), p
},
`invalid table: string item already exists at key [key]`,
`{"key": "value"}`)
}
func TestAST_GivenExistingItemAtDeepKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("deep", "table"))
p.setKeyValuePair(newKey("key"), newItem(pInteger, 0))
return p.openTable(newKey("deep", "table", "key")), p
},
`invalid table: integer item already exists at key [deep->table->key]`,
`{"deep": {"table": {"key": 0}}}`)
}
func TestAST_GivenExistingItemAtDeepKeyFromSubTable_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("deep", "table"))
p.setKeyValuePair(newKey("key1", "key2"), newItem(pInteger, 0))
return p.setKeyValuePair(newKey("key1", "key2"), newItem(pBoolean, true)), p
},
// This test mainly tests the formatting of [deep->table->key1->key2], being a concatenation
// of the currently active table plus the multipart key for setKeyValuePair().
`invalid key/value pair: integer item already exists at key [deep->table->key1->key2]`,
`{"deep": {"table": {"key1": {"key2": 0}}}}`)
}
func TestAST_FormattingOfQuotedPathPartInError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("must be quoted"))
p.setKeyValuePair(newKey("this one too"), newItem(pInteger, 0))
return p.setKeyValuePair(newKey("this one too"), newItem(pInteger, 0)), p
},
`invalid key/value pair: integer item already exists at key ["must be quoted"->"this one too"]`,
`{"must be quoted": {"this one too": 0}}`)
}
func TestAST_GivenExistingItemAtKey_CreatingArrayOfTablesAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.Root["key"] = newItem(pString, "value")
return p.openArrayOfTables(newKey("key")), p
},
`invalid table array: string item already exists at key [key]`,
`{"key": "value"}`)
}
func TestAST_GivenExistingItemInKeyPath_CreatingArrayOfTables_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.Root["key"] = newItem(pString, "value")
return p.openArrayOfTables(newKey("key", "subkey")), p
},
`invalid table array: string item already exists at key [key]`,
`{"key": "value"}`)
} }

4
comment.go
View File

@ -9,9 +9,7 @@ import (
var comment = c.Seq(a.Hash, c.ZeroOrMore(c.Not(a.EndOfLine)), m.Drop(a.EndOfLine)) var comment = c.Seq(a.Hash, c.ZeroOrMore(c.Not(a.EndOfLine)), m.Drop(a.EndOfLine))
func (t *parser) startComment(p *parse.API) { func (t *parser) startComment(p *parse.API) {
if p.Accept(comment) { if !p.Accept(comment) {
t.addParsedItem(pComment, p.Result().String())
} else {
p.Expected("comment") p.Expected("comment")
} }
} }

23
comment_test.go
View File

@ -5,20 +5,17 @@ import (
) )
func TestComment2(t *testing.T) { func TestComment2(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseToASTTest{
{``, []string{`Error: unexpected end of file (expected comment) at start of file`}}, {``, `{}`, `unexpected end of file (expected comment) at start of file`},
{`#`, []string{`comment("#")`}}, {`#`, `{}`, ``},
{`# `, []string{`comment("# ")`}}, {`# `, `{}`, ``},
{`# with data`, []string{`comment("# with data")`}}, {`# with data`, `{}`, ``},
{"# ending in EOL & EOF\r\n", []string{`comment("# ending in EOL & EOF")`}}, {"# ending in EOL & EOF\r\n", `{}`, ``},
{`# \xxx/ \u can't escape/`, []string{`comment("# \\xxx/ \\u can't escape/")`}}, {`# \xxx/ \u can't escape/`, `{}`, ``},
{"# \tlexe\r accepts embedded ca\r\riage \returns\r\n", []string{ {"# \tlexe\r accepts embedded ca\r\riage \returns\r\n", `{}`, ``},
`comment("# \tlexe\r accepts embedded ca\r\riage \returns")`}}, {"# with data and newline\ncode continues here", `{}`, `unexpected input (expected end of file) at line 2, column 1`},
{"# with data and newline\ncode continues here", []string{
`comment("# with data and newline")`,
`Error: unexpected input (expected end of file) at line 2, column 1`}},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startComment, test) testParseToAST(t, p, p.startComment, test)
} }
} }

27
helpers_test.go
View File

@ -74,3 +74,30 @@ func testAST(t *testing.T, code func() (error, *parser), expectedError string, e
t.Fatalf("Unexpected data after parsing:\nexpected: %s\nactual: %s\n", expectedData, p.Root.String()) t.Fatalf("Unexpected data after parsing:\nexpected: %s\nactual: %s\n", expectedData, p.Root.String())
} }
} }
type parseToASTTest struct {
input interface{}
expected string
expectedError string
}
func testParseToAST(t *testing.T, p *parser, handler parse.Handler, test parseToASTTest) {
var err error
defer func() {
recovered := recover()
if recovered != nil {
err = fmt.Errorf("Panic: %s", recovered.(string))
}
if err != nil && test.expectedError == "" {
t.Errorf("Unexpected error for input %q: %s", test.input, err)
} else if err != nil && test.expectedError != err.Error() {
t.Errorf("Unexpected error for input %q:\nexpected: %s\nactual: %s\n", test.input, test.expectedError, err.Error())
} else {
result := p.Root.String()
if test.expected != result {
t.Errorf("Unexpected result for input %q:\nexpected: %s\nactual: %s\n", test.input, test.expected, result)
}
}
}()
err = parse.New(handler)(test.input)
}

58
keyvaluepair.go
View File

@ -32,6 +32,9 @@ var (
// around dot-separated parts are ignored, however, best practice is to // around dot-separated parts are ignored, however, best practice is to
// not use any extraneous blanks. // not use any extraneous blanks.
keySeparatorDot = c.Seq(dropBlanks, a.Dot, dropBlanks) keySeparatorDot = c.Seq(dropBlanks, a.Dot, dropBlanks)
// Both [tables] and [[arrays of tables]] start with a square open bracket.
startOfTableOrArrayOfTables = a.SquareOpen
) )
func (t *parser) startKeyValuePair(p *parse.API) { func (t *parser) startKeyValuePair(p *parse.API) {
@ -40,8 +43,15 @@ func (t *parser) startKeyValuePair(p *parse.API) {
switch { switch {
case p.Peek(a.Hash): case p.Peek(a.Hash):
p.Handle(t.startComment) p.Handle(t.startComment)
case p.Peek(startOfTableOrArrayOfTables):
p.Handle(t.startTable)
case p.Peek(startOfKey): case p.Peek(startOfKey):
p.Handle(t.startKey, t.startAssignment, t.startValue) key, ok := t.parseKey(p, []string{})
if ok && p.Handle(t.startAssignment) {
if value, ok := t.parseValue(p); ok {
t.setKeyValuePair(key, value)
}
}
default: default:
p.ExpectEndOfFile() p.ExpectEndOfFile()
return return
@ -61,58 +71,40 @@ func (t *parser) startKeyValuePair(p *parse.API) {
// is to use bare keys except when absolutely necessary. // is to use bare keys except when absolutely necessary.
// A bare key must be non-empty, but an empty quoted key is allowed (though // A bare key must be non-empty, but an empty quoted key is allowed (though
// discouraged). // discouraged).
func (t *parser) startKey(p *parse.API) { func (t *parser) parseKey(p *parse.API, key []string) ([]string, bool) {
var key string var keyPart string
var ok bool var ok bool
switch { switch {
case p.Accept(bareKey): case p.Accept(bareKey):
key, ok = p.Result().String(), true keyPart, ok = p.Result().String(), true
case p.Peek(a.SingleQuote): case p.Peek(a.SingleQuote):
key, ok = t.parseLiteralString("key", p) keyPart, ok = t.parseLiteralString("key", p)
case p.Peek(a.DoubleQuote): case p.Peek(a.DoubleQuote):
key, ok = t.parseBasipString("key", p) keyPart, ok = t.parseBasicString("key", p)
default: default:
p.Expected("a key name") p.Expected("a key name")
return return nil, false
} }
if ok { if !ok {
t.addParsedItem(pKey, key) return nil, false
p.Handle(t.endOfKeyOrDot)
} }
key = append(key, keyPart)
return t.parseEndOfKeyOrDot(p, key)
} }
// Dotted keys are a sequence of bare or quoted keys joined with a dot. // Dotted keys are a sequence of bare or quoted keys joined with a dot.
// This allows for grouping similar properties together. // This allows for grouping similar properties together.
// Whitespace around dot-separated parts is ignored, however, best // Whitespace around dot-separated parts is ignored, however, best
// practice is to not use any extraneous whitespace. // practice is to not use any extraneous whitespace.
func (t *parser) endOfKeyOrDot(p *parse.API) { func (t *parser) parseEndOfKeyOrDot(p *parse.API, key []string) ([]string, bool) {
if p.Accept(keySeparatorDot) { if p.Accept(keySeparatorDot) {
p.Handle(t.startKey) return t.parseKey(p, key)
return
} }
return key, true
// TODO not sure if we really need this index.
// Can't we alway simply use the full item list, given that we'll feed all
// results to the parser's table state?
// Do we even need to emit a key here? Shouldn't we just create the
// table structure in the parser, ready for followup calls to fill the data?
keyStart := len(t.Items) - 1
for keyStart > 0 && t.Items[keyStart-1].Type == pKey {
keyStart--
}
keyLen := len(t.Items) - keyStart
key := make([]interface{}, keyLen)
for i := 0; i < keyLen; i++ {
key[i] = t.Items[keyStart+i].Data[0].(string)
}
t.Items = t.Items[0:keyStart]
t.addParsedItem(pKey, key...)
} }
func (t *parser) startAssignment(p *parse.API) { func (t *parser) startAssignment(p *parse.API) {
if p.Accept(keyAssignment) { if !p.Accept(keyAssignment) {
t.addParsedItem(pAssign)
} else {
p.Expected("a value assignment") p.Expected("a value assignment")
} }
} }

226
keyvaluepair_test.go
View File

@ -3,151 +3,133 @@ package toml
import "testing" import "testing"
func TestKey(t *testing.T) { func TestKey(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseToASTTest{
{"", []string{`Error: unexpected end of file (expected a key name) at start of file`}},
// Bare key tests // Bare key tests
{"barekey", []string{`key("barekey")`}}, {"barekey=0", `{"barekey": 0}`, ``},
{"1234567", []string{`key("1234567")`}}, {"1234567=0", `{"1234567": 0}`, ``},
{"mix-12_34", []string{`key("mix-12_34")`}}, {"mix-12_34=0", `{"mix-12_34": 0}`, ``},
{"-hey_good_Lookin123-", []string{`key("-hey_good_Lookin123-")`}}, {"-hey_good_Lookin123-=0", `{"-hey_good_Lookin123-": 0}`, ``},
{"wrong!", []string{`key("wrong")`, `Error: unexpected input (expected end of file) at line 1, column 6`}}, {"wrong!=0", `{}`, `unexpected input (expected a value assignment) at line 1, column 6`},
{"key1.", []string{`key("key1")`, `Error: unexpected end of file (expected a key name) at line 1, column 6`}}, {"key1.=0", `{}`, `unexpected input (expected a key name) at line 1, column 6`},
{"key1.key2", []string{`key("key1", "key2")`}}, {"key1.key2=0", `{"key1": {"key2": 0}}`, ``},
{"key . with . spaces", []string{`key("key", "with", "spaces")`}}, {"key . with . spaces=0", `{"key": {"with": {"spaces": 0}}}`, ``},
{"key \t . \twithtabs\t . \tandspaces", []string{`key("key", "withtabs", "andspaces")`}}, {"key \t . \twithtabs\t . \tandspaces=0", `{"key": {"withtabs": {"andspaces": 0}}}`, ``},
// Single quoted key tests // Single quoted key tests
{"''", []string{`key("")`}}, {"''=0", `{"": 0}`, ``},
{"'single quoted'", []string{`key("single quoted")`}}, {"'single quoted'=0", `{"single quoted": 0}`, ``},
{`'escape\s are literal'`, []string{`key("escape\\s are literal")`}}, {`'escape\s are literal'=0`, `{"escape\\s are literal": 0}`, ``},
{`'"using inner quotes"'`, []string{`key("\"using inner quotes\"")`}}, {`'"using inner quotes"'=0`, `{"\"using inner quotes\"": 0}`, ``},
// Double quoted key tests // Double quoted key tests
{`""`, []string{`key("")`}}, {`""=0`, `{"": 0}`, ``},
{`"double quoted"`, []string{`key("double quoted")`}}, {`"double quoted"=0`, `{"double quoted": 0}`, ``},
{`"escapes are in\terpreted"`, []string{`key("escapes are in\terpreted")`}}, {`"escapes are in\terpreted"=0`, `{"escapes are in\terpreted": 0}`, ``},
{`"using 'inner' \"quotes\""`, []string{`key("using 'inner' \"quotes\"")`}}, {`"using 'inner' \"quotes\""=0`, `{"using 'inner' \"quotes\"": 0}`, ``},
// Mixed key types // Mixed key types
{`this.'i\s'."madness\t".''`, []string{`key("this", "i\\s", "madness\t", "")`}}, {`this.'i\s'."madness\t".''=0`, `{"this": {"i\\s": {"madness\t": {"": 0}}}}`, ``},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startKey, test) testParseToAST(t, p, p.startKeyValuePair, test)
}
}
func TestAssignment(t *testing.T) {
for _, test := range []parseTest{
{"", []string{`Error: unexpected end of file (expected a value assignment) at start of file`}},
{"=", []string{`=`}},
{" \t = \t ", []string{`=`}},
{" \n = \n ", []string{`Error: unexpected input (expected a value assignment) at start of file`}},
} {
p := newParser()
testParseHandler(t, p, p.startAssignment, test)
} }
} }
func TestKeyValuePair(t *testing.T) { func TestKeyValuePair(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseToASTTest{
{"", []string{}}, {``, `{}`, ``},
{" ", []string{}}, {` `, `{}`, ``},
{" \t ", []string{}}, {" \t ", `{}`, ``},
{" key ", []string{`key("key")`, `Error: unexpected input (expected a value assignment) at line 1, column 5`}}, {" key ", `{}`, `unexpected input (expected a value assignment) at line 1, column 5`},
{" key \t=", []string{`key("key")`, `=`, `Error: unexpected end of file (expected a value) at line 1, column 8`}}, {" key \t=", `{}`, `unexpected end of file (expected a value) at line 1, column 8`},
{"key = # INVALID", []string{`key("key")`, `=`, `Error: unexpected input (expected a value) at line 1, column 7`}}, {"key = # INVALID", `{}`, `unexpected input (expected a value) at line 1, column 7`},
{" key \t =\t \"The Value\" \r\n", []string{`key("key")`, `=`, `"The Value"`}}, {" key \t =\t \"The Value\" \r\n", `{"key": "The Value"}`, ``},
{`3.14159 = "pi"`, []string{`key("3", "14159")`, `=`, `"pi"`}}, {`3.14159 = "pi"`, `{"3": {"14159": "pi"}}`, ``},
{`"ʎǝʞ" = "value"`, []string{`key("ʎǝʞ")`, `=`, `"value"`}}, {`"ʎǝʞ" = "value"`, `{"ʎǝʞ": "value"}`, ``},
{`key = "value" # This is a comment at the end of a line`, []string{`key("key")`, `=`, `"value"`, `comment("# This is a comment at the end of a line")`}}, {`key = "value" # This is a comment at the end of a line`, `{"key": "value"}`, ``},
{`another = "# This is not a comment"`, []string{`key("another")`, `=`, `"# This is not a comment"`}}, {`another = "# This is not a comment"`, `{"another": "# This is not a comment"}`, ``},
{"key1=\"value1\"key2=\"value2\"\r\nkey3a.key3b=\"value3\"", []string{ {"key1=\"value1\"key2=\"value2\"\r\nkey3a.key3b=\"value3\"", `{"key1": "value1", "key2": "value2", "key3a": {"key3b": "value3"}}`, ``},
`key("key1")`, `=`, `"value1"`, {"with=\"comments\"# boring \nanother.cool =\"one\" \t # to the end\r\n", `{"another": {"cool": "one"}, "with": "comments"}`, ``},
`key("key2")`, `=`, `"value2"`,
`key("key3a", "key3b")`, `=`, `"value3"`}},
{"with=\"comments\"# boring \nanother.cool =\"one\" \t # to the end\r\n", []string{
`key("with")`, `=`, `"comments"`, `comment("# boring ")`,
`key("another", "cool")`, `=`, `"one"`, `comment("# to the end")`}},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startKeyValuePair, test) testParseToAST(t, p, p.startKeyValuePair, test)
} }
} }
func TestKeyValuePair_ForAllTypes(t *testing.T) { func TestKeyValuePair_ForAllTypes(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseToASTTest{
{"string='literal'", []string{`key("string")`, `=`, `"literal"`}}, {"string='literal'", `{"string": "literal"}`, ``},
{"string='''literal\nmulti-line'''", []string{`key("string")`, `=`, `"literal\nmulti-line"`}}, {"string='''literal\nmulti-line'''", `{"string": "literal\nmulti-line"}`, ``},
{`string="basic"`, []string{`key("string")`, `=`, `"basic"`}}, {`string="basic"`, `{"string": "basic"}`, ``},
{"string=\"\"\"basic\nmulti-line\"\"\"", []string{`key("string")`, `=`, `"basic\nmulti-line"`}}, {"string=\"\"\"basic\nmulti-line\"\"\"", `{"string": "basic\nmulti-line"}`, ``},
{"integer=1_234_567", []string{`key("integer")`, `=`, `1234567`}}, {"integer=1_234_567", `{"integer": 1234567}`, ``},
{"integer=42", []string{`key("integer")`, `=`, `42`}}, {"integer=42", `{"integer": 42}`, ``},
{"integer=0x42", []string{`key("integer")`, `=`, `66`}}, {"integer=0x42", `{"integer": 66}`, ``},
{"integer=0o42", []string{`key("integer")`, `=`, `34`}}, {"integer=0o42", `{"integer": 34}`, ``},
{"integer=0b101010", []string{`key("integer")`, `=`, `42`}}, {"integer=0b101010", `{"integer": 42}`, ``},
{"float=42.37", []string{`key("float")`, `=`, `42.37`}}, {"float=42.37", `{"float": 42.37}`, ``},
{"float=42e+37", []string{`key("float")`, `=`, `4.2e+38`}}, {"float=42e+37", `{"float": 4.2e+38}`, ``},
{"float=42.37e-11", []string{`key("float")`, `=`, `4.237e-10`}}, {"float=42.37e-11", `{"float": 4.237e-10}`, ``},
{"boolean=true", []string{`key("boolean")`, `=`, `true`}}, {"boolean=true", `{"boolean": true}`, ``},
{"boolean=false", []string{`key("boolean")`, `=`, `false`}}, {"boolean=false", `{"boolean": false}`, ``},
{"date=2019-01-01", []string{`key("date")`, `=`, `2019-01-01`}}, {"date=2019-01-01", `{"date": 2019-01-01}`, ``},
{"time=15:03:11", []string{`key("time")`, `=`, `15:03:11`}}, {"time=15:03:11", `{"time": 15:03:11}`, ``},
{"datetime=2021-02-01 15:03:11.123", []string{`key("datetime")`, `=`, `2021-02-01 15:03:11.123`}}, {"datetime=2021-02-01 15:03:11.123", `{"datetime": 2021-02-01 15:03:11.123}`, ``},
{"offset_datetime=1111-11-11 11:11:11.111111111+11:11", []string{`key("offset_datetime")`, `=`, `1111-11-11T11:11:11.111111111+11:11`}}, {"offset_datetime=1111-11-11 11:11:11.111111111+11:11", `{"offset_datetime": 1111-11-11T11:11:11.111111111+11:11}`, ``},
{"static_array=['a', 'static', 'array']", []string{`key("static_array")`, `=`, `["a", "static", "array"]`}}, {"static_array=['a', 'static', 'array']", `{"static_array": ["a", "static", "array"]}`, ``},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startKeyValuePair, test) testParseToAST(t, p, p.startKeyValuePair, test)
} }
} }
func TestKeyValuePair_ExamplesFromSpecification(t *testing.T) { func TestKeyValuePair_ExamplesFromSpecification(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseToASTTest{
{"int1 = +99", []string{`key("int1")`, `=`, `99`}}, {"int1 = +99", `{"int1": 99}`, ``},
{"int2 = 42", []string{`key("int2")`, `=`, `42`}}, {"int2 = 42", `{"int2": 42}`, ``},
{"int3 = 0", []string{`key("int3")`, `=`, `0`}}, {"int3 = 0", `{"int3": 0}`, ``},
{"int4 = -17", []string{`key("int4")`, `=`, `-17`}}, {"int4 = -17", `{"int4": -17}`, ``},
{"int5 = 1_000", []string{`key("int5")`, `=`, `1000`}}, {"int5 = 1_000", `{"int5": 1000}`, ``},
{"int6 = 5_349_221", []string{`key("int6")`, `=`, `5349221`}}, {"int6 = 5_349_221", `{"int6": 5349221}`, ``},
{"int7 = 1_2_3_4_5 # VALID but discouraged", []string{`key("int7")`, `=`, `12345`, `comment("# VALID but discouraged")`}}, {"int7 = 1_2_3_4_5 # VALID but discouraged", `{"int7": 12345}`, ``},
{"hex1 = 0xDEADBEEF", []string{`key("hex1")`, `=`, `3735928559`}}, {"hex1 = 0xDEADBEEF", `{"hex1": 3735928559}`, ``},
{"hex2 = 0xdeadbeef", []string{`key("hex2")`, `=`, `3735928559`}}, {"hex2 = 0xdeadbeef", `{"hex2": 3735928559}`, ``},
{"hex3 = 0xdead_beef", []string{`key("hex3")`, `=`, `3735928559`}}, {"hex3 = 0xdead_beef", `{"hex3": 3735928559}`, ``},
{"oct1 = 0o01234567", []string{`key("oct1")`, `=`, `342391`}}, {"oct1 = 0o01234567", `{"oct1": 342391}`, ``},
{"oct2 = 0o755", []string{`key("oct2")`, `=`, `493`}}, {"oct2 = 0o755", `{"oct2": 493}`, ``},
{"bin1 = 0b11010110", []string{`key("bin1")`, `=`, `214`}}, {"bin1 = 0b11010110", `{"bin1": 214}`, ``},
{"flt1 = +1.0", []string{`key("flt1")`, `=`, `1`}}, {"flt1 = +1.0", `{"flt1": 1}`, ``},
{"flt2 = 3.1415", []string{`key("flt2")`, `=`, `3.1415`}}, {"flt2 = 3.1415", `{"flt2": 3.1415}`, ``},
{"flt3 = -0.01", []string{`key("flt3")`, `=`, `-0.01`}}, {"flt3 = -0.01", `{"flt3": -0.01}`, ``},
{"flt4 = 5e+22", []string{`key("flt4")`, `=`, `5e+22`}}, {"flt4 = 5e+22", `{"flt4": 5e+22}`, ``},
{"flt5 = 1e6", []string{`key("flt5")`, `=`, `1e+06`}}, {"flt5 = 1e6", `{"flt5": 1e+06}`, ``},
{"flt6 = -2E-2", []string{`key("flt6")`, `=`, `-0.02`}}, {"flt6 = -2E-2", `{"flt6": -0.02}`, ``},
{"flt7 = 6.626e-34", []string{`key("flt7")`, `=`, `6.626e-34`}}, {"flt7 = 6.626e-34", `{"flt7": 6.626e-34}`, ``},
{"flt8 = 224_617.445_991_228", []string{`key("flt8")`, `=`, `224617.445991228`}}, {"flt8 = 224_617.445_991_228", `{"flt8": 224617.445991228}`, ``},
{"sf1 = inf # positive infinity", []string{`key("sf1")`, `=`, `+Inf`, `comment("# positive infinity")`}}, {"sf1 = inf # positive infinity", `{"sf1": +Inf}`, ``},
{"sf2 = +inf # positive infinity", []string{`key("sf2")`, `=`, `+Inf`, `comment("# positive infinity")`}}, {"sf2 = +inf # positive infinity", `{"sf2": +Inf}`, ``},
{"sf3 = -inf # negative infinity", []string{`key("sf3")`, `=`, `-Inf`, `comment("# negative infinity")`}}, {"sf3 = -inf # negative infinity", `{"sf3": -Inf}`, ``},
{"sf4 = nan # actual sNaN/qNaN encoding is implementation-specific", []string{`key("sf4")`, `=`, `NaN`, `comment("# actual sNaN/qNaN encoding is implementation-specific")`}}, {"sf4 = nan # actual sNaN/qNaN encoding is implementation-specific", `{"sf4": NaN}`, ``},
{"sf5 = +nan # same as `nan`", []string{`key("sf5")`, `=`, `NaN`, "comment(\"# same as `nan`\")"}}, {"sf5 = +nan # same as `nan`", `{"sf5": NaN}`, ``},
{"sf6 = -nan # valid, actual encoding is implementation-specific", []string{`key("sf6")`, `=`, `NaN`, `comment("# valid, actual encoding is implementation-specific")`}}, {"sf6 = -nan # valid, actual encoding is implementation-specific", `{"sf6": NaN}`, ``},
{"bool1 = true", []string{`key("bool1")`, `=`, `true`}}, {"bool1 = true", `{"bool1": true}`, ``},
{"bool2 = false", []string{`key("bool2")`, `=`, `false`}}, {"bool2 = false", `{"bool2": false}`, ``},
{"odt1 = 1979-05-27T07:32:00Z", []string{`key("odt1")`, `=`, `1979-05-27T07:32:00Z`}}, {"odt1 = 1979-05-27T07:32:00Z", `{"odt1": 1979-05-27T07:32:00Z}`, ``},
{"odt2 = 1979-05-27T00:32:00-07:00", []string{`key("odt2")`, `=`, `1979-05-27T00:32:00-07:00`}}, {"odt2 = 1979-05-27T00:32:00-07:00", `{"odt2": 1979-05-27T00:32:00-07:00}`, ``},
{"odt3 = 1979-05-27T00:32:00.999999-07:00", []string{`key("odt3")`, `=`, `1979-05-27T00:32:00.999999-07:00`}}, {"odt3 = 1979-05-27T00:32:00.999999-07:00", `{"odt3": 1979-05-27T00:32:00.999999-07:00}`, ``},
{"odt4 = 1979-05-27 07:32:00Z", []string{`key("odt4")`, `=`, `1979-05-27T07:32:00Z`}}, {"odt4 = 1979-05-27 07:32:00Z", `{"odt4": 1979-05-27T07:32:00Z}`, ``},
{"ldt1 = 1979-05-27T07:32:00", []string{`key("ldt1")`, `=`, `1979-05-27 07:32:00`}}, {"ldt1 = 1979-05-27T07:32:00", `{"ldt1": 1979-05-27 07:32:00}`, ``},
{"ldt2 = 1979-05-27T00:32:00.999999", []string{`key("ldt2")`, `=`, `1979-05-27 00:32:00.999999`}}, {"ldt2 = 1979-05-27T00:32:00.999999", `{"ldt2": 1979-05-27 00:32:00.999999}`, ``},
{"ld1 = 1979-05-27", []string{`key("ld1")`, `=`, `1979-05-27`}}, {"ld1 = 1979-05-27", `{"ld1": 1979-05-27}`, ``},
{"lt1 = 07:32:00", []string{`key("lt1")`, `=`, `07:32:00`}}, {"lt1 = 07:32:00", `{"lt1": 07:32:00}`, ``},
{"lt2 = 00:32:00.999999", []string{`key("lt2")`, `=`, `00:32:00.999999`}}, {"lt2 = 00:32:00.999999", `{"lt2": 00:32:00.999999}`, ``},
{"arr1 = [ 1, 2, 3 ]", []string{`key("arr1")`, `=`, `[1, 2, 3]`}}, {"arr1 = [ 1, 2, 3 ]", `{"arr1": [1, 2, 3]}`, ``},
{`arr2 = [ "red", "yellow", "green" ]`, []string{`key("arr2")`, `=`, `["red", "yellow", "green"]`}}, {`arr2 = [ "red", "yellow", "green" ]`, `{"arr2": ["red", "yellow", "green"]}`, ``},
{`arr3 = [ [ 1, 2 ], [3, 4, 5] ]`, []string{`key("arr3")`, `=`, `[[1, 2], [3, 4, 5]]`}}, {`arr3 = [ [ 1, 2 ], [3, 4, 5] ]`, `{"arr3": [[1, 2], [3, 4, 5]]}`, ``},
{`arr4 = [ "all", 'strings', """are the same""", '''type''']`, []string{`key("arr4")`, `=`, `["all", "strings", "are the same", "type"]`}}, {`arr4 = [ "all", 'strings', """are the same""", '''type''']`, `{"arr4": ["all", "strings", "are the same", "type"]}`, ``},
{`arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]`, []string{`key("arr5")`, `=`, `[[1, 2], ["a", "b", "c"]]`}}, {`arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]`, `{"arr5": [[1, 2], ["a", "b", "c"]]}`, ``},
{`arr6 = [ 1, 2.0 ] # INVALID`, []string{`key("arr6")`, `=`, `Error: type mismatch in array of integers: found an item of type float at line 1, column 16`}}, {`arr6 = [ 1, 2.0 ] # INVALID`, `{}`, `type mismatch in array of integers: found an item of type float at line 1, column 16`},
{"arr7 = [\n 1, 2, 3\n]", []string{`key("arr7")`, `=`, `[1, 2, 3]`}}, {"arr7 = [\n 1, 2, 3\n]", `{"arr7": [1, 2, 3]}`, ``},
{"arr8 = [\n 1,\n 2, # this is ok\n]", []string{`key("arr8")`, `=`, `[1, 2]`}}, {"arr8 = [\n 1,\n 2, # this is ok\n]", `{"arr8": [1, 2]}`, ``},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startKeyValuePair, test) testParseToAST(t, p, p.startKeyValuePair, test)
} }
} }

40
value.go
View File

@ -4,27 +4,33 @@ import (
"git.makaay.nl/mauricem/go-parsekit/parse" "git.makaay.nl/mauricem/go-parsekit/parse"
) )
var (
detectString = a.SingleQuote.Or(a.DoubleQuote)
detectBoolean = a.Str("true").Or(a.Str("false"))
detectNumberSpecials = c.Any(a.Plus, a.Minus, a.Str("inf"), a.Str("nan"))
detectDateTime = a.Digits.Then(a.Minus.Or(a.Colon))
detectNumber = a.Digit
detectArray = a.SquareOpen
)
// Values must be of the following types: String, Integer, Float, Boolean, // Values must be of the following types: String, Integer, Float, Boolean,
// Datetime, Array, or Inline Table. Unspecified values are invalid. // Datetime, Array, or Inline Table. Unspecified values are invalid.
func (t *parser) startValue(p *parse.API) { func (t *parser) parseValue(p *parse.API) (*item, bool) {
switch { switch {
case p.Peek(c.Any(a.SingleQuote, a.DoubleQuote)): case p.Peek(detectString):
p.Handle(t.startString) return t.parseString(p)
case p.Peek(a.Runes('t', 'f')): case p.Peek(detectBoolean):
p.Handle(t.startBoolean) return t.parseBoolean(p)
case p.Peek(a.Plus.Or(a.Minus)): case p.Peek(detectNumberSpecials):
p.Handle(t.startNumber) return t.parseNumber(p)
case p.Peek(a.Runes('i', 'n')): case p.Peek(detectDateTime):
p.Handle(t.startNumber) return t.parseDateTime(p)
case p.Peek(a.Digit): case p.Peek(detectNumber):
if p.Peek(a.Digits.Then(a.Minus.Or(a.Colon))) { return t.parseNumber(p)
p.Handle(t.startDateTime) case p.Peek(detectArray):
} else { return t.parseArray(p)
p.Handle(t.startNumber)
}
case p.Peek(a.SquareOpen):
p.Handle(t.startArray)
default: default:
p.Expected("a value") p.Expected("a value")
return nil, false
} }
} }

36
value_array.go
View File

@ -36,51 +36,45 @@ var (
arrayClose = c.Seq(c.Optional(arraySpace.Then(a.Comma)), arraySpace, a.SquareClose) arrayClose = c.Seq(c.Optional(arraySpace.Then(a.Comma)), arraySpace, a.SquareClose)
) )
func (t *parser) startArray(p *parse.API) { func (t *parser) parseArray(p *parse.API) (*item, bool) {
// Check for the start of the array. // Check for the start of the array.
if !p.Accept(arrayOpen) { if !p.Accept(arrayOpen) {
p.Expected("an array") p.Expected("an array")
return return nil, false
} }
items := []item{}
// Check for an empty array. // Check for an empty array.
if p.Accept(arrayClose) { if p.Accept(arrayClose) {
t.addParsedItem(pStaticArray, items) return newItem(pArray), true
return
} }
// Not an empty array, parse the items. // Not an empty array, parse the array items.
items := []interface{}{}
for { for {
// Check for a valid item. // Check for a value item.
if !p.Handle(t.startValue) { value, ok := t.parseValue(p)
return if !ok {
return nil, false
} }
// Pop the item from the value parsing and append it to the array items.
parseItems, item := t.Items[0:len(t.Items)-1], t.Items[len(t.Items)-1]
t.Items = parseItems
// Data types may not be mixed (different ways to define strings should be // Data types may not be mixed (different ways to define strings should be
// considered the same type, and so should arrays with different element types). // considered the same type, and so should arrays with different element types).
if len(items) > 0 && item.Type != items[0].Type { if len(items) > 0 && value.Type != items[0].(*item).Type {
p.Error("type mismatch in array of %ss: found an item of type %s", items[0].Type, item.Type) p.Error("type mismatch in array of %ss: found an item of type %s", items[0].(*item).Type, value.Type)
return return nil, false
} }
items = append(items, item) items = append(items, value)
// Check for the end of the array. // Check for the end of the array.
if p.Accept(arrayClose) { if p.Accept(arrayClose) {
t.addParsedItem(pStaticArray, items) return newItem(pArray, items...), true
return
} }
// Not the end of the array? Then we should find an array separator. // Not the end of the array? Then we should find an array separator.
if !p.Accept(arraySeparator) { if !p.Accept(arraySeparator) {
p.Expected("an array separator") p.Expected("an array separator")
return return nil, false
} }
} }
} }

65
value_array_test.go
View File

@ -2,39 +2,46 @@ package toml
import ( import (
"testing" "testing"
"git.makaay.nl/mauricem/go-parsekit/parse"
) )
func TestArrayStart(t *testing.T) {
parser := newParser()
wrapper := func(p *parse.API) { parser.parseArray(p) }
testParseToAST(t, parser, wrapper, parseToASTTest{"INVALID", "{}", "unexpected input (expected an array) at start of file"})
}
func TestArray(t *testing.T) { func TestArray(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseToASTTest{
{"", []string{`Error: unexpected end of file (expected an array) at start of file`}}, {"x=[ape", `{}`, `unexpected input (expected a value) at line 1, column 4`},
{"[ape", []string{`Error: unexpected input (expected a value) at line 1, column 2`}}, {"x=[1", `{}`, `unexpected end of file (expected an array separator) at line 1, column 5`},
{"[1", []string{`Error: unexpected end of file (expected an array separator) at line 1, column 3`}}, {"x=[]", `{"x": []}`, ``},
{"[]", []string{`[]`}}, {"x=[\n]", `{"x": []}`, ``},
{"[\n]", []string{`[]`}}, {"x=[,]", `{"x": []}`, ``},
{"[,]", []string{`[]`}}, {"x=[ , ]", `{"x": []}`, ``},
{"[ , ]", []string{`[]`}}, {"x=[ \n , \r\n ]", `{"x": []}`, ``},
{"[ \n , \r\n ]", []string{`[]`}}, {"x=[ \t , \t ]", `{"x": []}`, ``},
{"[ \t , \t ]", []string{`[]`}}, {"x=[\r\n\r\n , \r\n \t\n]", `{"x": []}`, ``},
{"[\r\n\r\n , \r\n \t\n]", []string{`[]`}}, {"x=[\n#comment on its own line\n]", `{"x": []}`, ``},
{"[\n#comment on its own line\n]", []string{`[]`}}, {"x=[#comment before close\n]", `{"x": []}`, ``},
{"[#comment before close\n]", []string{`[]`}}, {"x=[,#comment after separator\n]", `{"x": []}`, ``},
{"[,#comment after separator\n]", []string{`[]`}}, {"x=[#comment before separator\n,]", `{"x": []}`, ``},
{"[#comment before separator\n,]", []string{`[]`}}, {"x=[#comment before value\n1]", `{"x": [1]}`, ``},
{"[#comment before value\n1]", []string{`[1]`}}, {"x=[1#comment after value\n]", `{"x": [1]}`, ``},
{"[1#comment after value\n]", []string{`[1]`}}, {"x=[1\n#comment on its own line after value\n]", `{"x": [1]}`, ``},
{"[1\n#comment on its own line after value\n]", []string{`[1]`}}, {"x=[1#comment 1\n#comment 2\n#comment 3\n , \n2]", `{"x": [1, 2]}`, ``},
{"[1#comment 1\n#comment 2\n#comment 3\n , \n2]", []string{`[1, 2]`}}, {"x=[1]", `{"x": [1]}`, ``},
{"[1]", []string{`[1]`}}, {"x=[1,0x2, 0b11, 0o4]", `{"x": [1, 2, 3, 4]}`, ``},
{"[1,0x2, 0b11, 0o4]", []string{`[1, 2, 3, 4]`}}, {"x=[0.1,0.2,3e-1,0.04e+1, nan, inf]", `{"x": [0.1, 0.2, 0.3, 0.4, NaN, +Inf]}`, ``},
{"[0.1,0.2,3e-1,0.04e+1, nan, inf]", []string{`[0.1, 0.2, 0.3, 0.4, NaN, +Inf]`}}, {"x=[\n\t 'a', \"b\", '''c''', \"\"\"d\ne\"\"\",\n \t]", `{"x": ["a", "b", "c", "d\ne"]}`, ``},
{"[\n\t 'a', \"b\", '''c''', \"\"\"d\ne\"\"\",\n \t]", []string{`["a", "b", "c", "d\ne"]`}}, {`x=[1, 2, 3, "four"]`, `{}`, `type mismatch in array of integers: found an item of type string at line 1, column 19`},
{`[1, 2, 3, "four"]`, []string{`Error: type mismatch in array of integers: found an item of type string at line 1, column 17`}}, {`x=[[1],['a']]`, `{"x": [[1], ["a"]]}`, ``},
{`[[1],['a']]`, []string{`[[1], ["a"]]`}}, {`x=[[[],[]],[]]`, `{"x": [[[], []], []]}`, ``},
{`[[[],[]],[]]`, []string{`[[[], []], []]`}}, {"x=[\r\n\r\n \t\n [\r\n\r\n\t [],[\t]],\t\n[]\t \t \n ]", `{"x": [[[], []], []]}`, ``},
{"[\r\n\r\n \t\n [\r\n\r\n\t [],[\t]],\t\n[]\t \t \n ]", []string{`[[[], []], []]`}}, {`x=[[1],'a']`, `{}`, `type mismatch in array of static arrays: found an item of type string at line 1, column 11`},
{`[[1],'a']`, []string{`Error: type mismatch in array of static arrays: found an item of type string at line 1, column 9`}},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startArray, test) testParseToAST(t, p, p.startKeyValuePair, test)
} }
} }

10
value_boolean.go
View File

@ -4,14 +4,18 @@ import (
"git.makaay.nl/mauricem/go-parsekit/parse" "git.makaay.nl/mauricem/go-parsekit/parse"
) )
var falseItem = newItem(pBoolean, false)
var trueItem = newItem(pBoolean, true)
// Booleans are just the tokens you're used to. Always lowercase. // Booleans are just the tokens you're used to. Always lowercase.
func (t *parser) startBoolean(p *parse.API) { func (t *parser) parseBoolean(p *parse.API) (*item, bool) {
switch { switch {
case p.Accept(a.Str("true")): case p.Accept(a.Str("true")):
t.addParsedItem(pBoolean, true) return trueItem, true
case p.Accept(a.Str("false")): case p.Accept(a.Str("false")):
t.addParsedItem(pBoolean, false) return falseItem, true
default: default:
p.Expected("true or false") p.Expected("true or false")
return nil, false
} }
} }

25
value_boolean_test.go
View File

@ -2,19 +2,26 @@ package toml
import ( import (
"testing" "testing"
"git.makaay.nl/mauricem/go-parsekit/parse"
) )
func TestBooleanStart(t *testing.T) {
parser := newParser()
wrapper := func(p *parse.API) { parser.parseBoolean(p) }
testParseToAST(t, parser, wrapper, parseToASTTest{"INVALID", "{}", "unexpected input (expected true or false) at start of file"})
}
func TestBoolean(t *testing.T) { func TestBoolean(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseToASTTest{
{``, []string{`Error: unexpected end of file (expected true or false) at start of file`}}, {`x=true`, `{"x": true}`, ``},
{`true`, []string{`true`}}, {`x=false`, `{"x": false}`, ``},
{`false`, []string{`false`}}, {`x=yes`, `{}`, `unexpected input (expected a value) at line 1, column 3`},
{`yes`, []string{`Error: unexpected input (expected true or false) at start of file`}}, {`x=no`, `{}`, `unexpected input (expected a value) at line 1, column 3`},
{`no`, []string{`Error: unexpected input (expected true or false) at start of file`}}, {`x=1`, `{"x": 1}`, ``},
{`1`, []string{`Error: unexpected input (expected true or false) at start of file`}}, {`x=0`, `{"x": 0}`, ``},
{`0`, []string{`Error: unexpected input (expected true or false) at start of file`}},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startBoolean, test) testParseToAST(t, p, p.startKeyValuePair, test)
} }
} }

9
value_datetime.go
View File

@ -73,18 +73,19 @@ var (
datetime = c.Any(offsetDateTime, localDateTime, localDate, localTime) datetime = c.Any(offsetDateTime, localDateTime, localDate, localTime)
) )
func (t *parser) startDateTime(p *parse.API) { func (t *parser) parseDateTime(p *parse.API) (*item, bool) {
if !p.Accept(datetime) { if !p.Accept(datetime) {
p.Expected("a date and/or time") p.Expected("a date and/or time")
return return nil, false
} }
tokens := p.Result().Tokens() tokens := p.Result().Tokens()
valueType := getDateTimeValueType(&tokens) valueType := getDateTimeValueType(&tokens)
input, value, err := getDateTimeValue(&tokens) input, value, err := getDateTimeValue(&tokens)
if err == nil { if err == nil {
t.addParsedItem(valueType, value) return newItem(valueType, value), true
} else { } else {
p.Error("Cannot parse value 0%s: %s", input, err) p.Error("invalid date/time value %s: %s", input, err)
return nil, false
} }
} }

49
value_datetime_test.go
View File

@ -2,32 +2,39 @@ package toml
import ( import (
"testing" "testing"
"git.makaay.nl/mauricem/go-parsekit/parse"
) )
func TestDateTimeStart(t *testing.T) {
parser := newParser()
wrapper := func(p *parse.API) { parser.parseDateTime(p) }
testParseToAST(t, parser, wrapper, parseToASTTest{"INVALID", "{}", "unexpected input (expected a date and/or time) at start of file"})
}
func TestDateTime(t *testing.T) { func TestDateTime(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseToASTTest{
{``, []string{`Error: unexpected end of file (expected a date and/or time) at start of file`}}, {`x=1979-05-27`, `{"x": 1979-05-27}`, ``},
{`1979-05-27`, []string{`1979-05-27`}}, {`x=00:00:00`, `{"x": 00:00:00}`, ``},
{`00:00:00`, []string{`00:00:00`}}, {`x=23:59:59`, `{"x": 23:59:59}`, ``},
{`23:59:59`, []string{`23:59:59`}}, {`x=12:10:08.12121212121212`, `{"x": 12:10:08.121212121}`, ``},
{`12:10:08.12121212121212`, []string{`12:10:08.121212121`}}, {`x=1979-05-28T01:01:01`, `{"x": 1979-05-28 01:01:01}`, ``},
{`1979-05-28T01:01:01`, []string{`1979-05-28 01:01:01`}}, {`x=1979-05-28 01:01:01`, `{"x": 1979-05-28 01:01:01}`, ``},
{`1979-05-28 01:01:01`, []string{`1979-05-28 01:01:01`}}, {`x=1979-05-27T07:32:00Z`, `{"x": 1979-05-27T07:32:00Z}`, ``},
{`1979-05-27T07:32:00Z`, []string{`1979-05-27T07:32:00Z`}}, {`x=1979-05-27 07:33:00Z`, `{"x": 1979-05-27T07:33:00Z}`, ``},
{`1979-05-27 07:33:00Z`, []string{`1979-05-27T07:33:00Z`}}, {`x=1979-05-27 07:34:00+07:00`, `{"x": 1979-05-27T07:34:00+07:00}`, ``},
{`1979-05-27 07:34:00+07:00`, []string{`1979-05-27T07:34:00+07:00`}}, {`x=1979-05-27 07:34:00-07:00`, `{"x": 1979-05-27T07:34:00-07:00}`, ``},
{`1979-05-27 07:34:00-07:00`, []string{`1979-05-27T07:34:00-07:00`}}, {`x=1985-03-31 23:59:59+00:00`, `{"x": 1985-03-31T23:59:59Z}`, ``},
{`1985-03-31 23:59:59+00:00`, []string{`1985-03-31T23:59:59Z`}}, {`x=2000-09-10 00:00:00.000000000+00:00`, `{"x": 2000-09-10T00:00:00Z}`, ``},
{`2000-09-10 00:00:00.000000000+00:00`, []string{`2000-09-10T00:00:00Z`}}, {`x=2003-11-01 01:02:03.999999999999+10:00`, `{"x": 2003-11-01T01:02:03.999999999+10:00}`, ``},
{`2003-11-01 01:02:03.999999999999+10:00`, []string{`2003-11-01T01:02:03.999999999+10:00`}}, {`x=2007-12-25 04:00:04.1111-10:30`, `{"x": 2007-12-25T04:00:04.1111-10:30}`, ``},
{`2007-12-25 04:00:04.1111-10:30`, []string{`2007-12-25T04:00:04.1111-10:30`}}, {`x=2021-02-01 10:10:10.101010203040Z`, `{"x": 2021-02-01T10:10:10.101010203Z}`, ``},
{`2021-02-01 10:10:10.101010203040Z`, []string{`2021-02-01T10:10:10.101010203Z`}},
// TODO ugly column, should be at start or at the actual wrong part // TODO ugly column, should be at start or at the actual wrong part
{`2000-13-01`, []string{`Error: Cannot parse value 02000-13-01: parsing time "2000-13-01": month out of range at line 1, column 11`}}, {`x=2000-13-01`, `{}`, `invalid date/time value 2000-13-01: parsing time "2000-13-01": month out of range at line 1, column 13`},
{`2000-02-31`, []string{`Error: Cannot parse value 02000-02-31: parsing time "2000-02-31": day out of range at line 1, column 11`}}, {`x=2000-02-31`, `{}`, `invalid date/time value 2000-02-31: parsing time "2000-02-31": day out of range at line 1, column 13`},
{`25:01:01`, []string{`Error: Cannot parse value 025:01:01: parsing time "25:01:01": hour out of range at line 1, column 9`}}, {`x=25:01:01`, `{}`, `invalid date/time value 25:01:01: parsing time "25:01:01": hour out of range at line 1, column 11`},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startDateTime, test) testParseToAST(t, p, p.startKeyValuePair, test)
} }
} }

27
value_number.go
View File

@ -65,28 +65,28 @@ var (
nan = a.Signed(a.Str("nan")) nan = a.Signed(a.Str("nan"))
) )
func (t *parser) startNumber(p *parse.API) { func (t *parser) parseNumber(p *parse.API) (*item, bool) {
switch { switch {
case p.Accept(tok.Float64(nil, float)): case p.Accept(tok.Float64(nil, float)):
t.addParsedItem(pFloat, p.Result().Value(0).(float64)) return newItem(pFloat, p.Result().Value(0).(float64)), true
case p.Accept(nan): case p.Accept(nan):
t.addParsedItem(pFloat, math.NaN()) return newItem(pFloat, math.NaN()), true
case p.Accept(inf): case p.Accept(inf):
if p.Result().Rune(0) == '-' { if p.Result().Rune(0) == '-' {
t.addParsedItem(pFloat, math.Inf(-1)) return newItem(pFloat, math.Inf(-1)), true
} else {
t.addParsedItem(pFloat, math.Inf(+1))
} }
return newItem(pFloat, math.Inf(+1)), true
case p.Accept(a.Zero): case p.Accept(a.Zero):
p.Handle(t.startIntegerStartingWithZero) return t.parseIntegerStartingWithZero(p)
case p.Accept(tok.Int64(nil, integer)): case p.Accept(tok.Int64(nil, integer)):
t.addParsedItem(pInteger, p.Result().Value(0).(int64)) return newItem(pInteger, p.Result().Value(0).(int64)), true
default: default:
p.Expected("a number") p.Expected("a number")
return nil, false
} }
} }
func (t *parser) startIntegerStartingWithZero(p *parse.API) { func (t *parser) parseIntegerStartingWithZero(p *parse.API) (*item, bool) {
var value int64 var value int64
var err error var err error
switch { switch {
@ -97,12 +97,11 @@ func (t *parser) startIntegerStartingWithZero(p *parse.API) {
case p.Accept(binary): case p.Accept(binary):
value, err = strconv.ParseInt(p.Result().Value(0).(string), 2, 64) value, err = strconv.ParseInt(p.Result().Value(0).(string), 2, 64)
default: default:
t.addParsedItem(pInteger, int64(0)) return newItem(pInteger, int64(0)), true
return
} }
if err == nil { if err == nil {
t.addParsedItem(pInteger, value) return newItem(pInteger, value), true
} else {
p.Error("Cannot parse value 0%s: %s", p.Result().String(), err)
} }
p.Error("invalid integer value 0%s: %s", p.Result().String(), err)
return nil, false
} }

165
value_number_test.go
View File

@ -2,106 +2,105 @@ package toml
import ( import (
"testing" "testing"
"git.makaay.nl/mauricem/go-parsekit/parse"
) )
func TestNumberStart(t *testing.T) {
parser := newParser()
wrapper := func(p *parse.API) { parser.parseNumber(p) }
testParseToAST(t, parser, wrapper, parseToASTTest{"INVALID", "{}", "unexpected input (expected a number) at start of file"})
}
func TestInteger(t *testing.T) { func TestInteger(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseToASTTest{
{``, []string{`Error: unexpected end of file (expected a number) at start of file`}},
// Decimal // Decimal
{`0`, []string{`0`}}, {`x=0`, `{"x": 0}`, ``},
{`+0`, []string{`0`}}, {`x=+0`, `{"x": 0}`, ``},
{`-0`, []string{`0`}}, {`x=-0`, `{"x": 0}`, ``},
{`1`, []string{`1`}}, {`x=1`, `{"x": 1}`, ``},
{`42`, []string{`42`}}, {`x=42`, `{"x": 42}`, ``},
{`+99`, []string{`99`}}, {`x=+99`, `{"x": 99}`, ``},
{`-17`, []string{`-17`}}, {`x=-17`, `{"x": -17}`, ``},
{`1234`, []string{`1234`}}, {`x=1234`, `{"x": 1234}`, ``},
{`_`, []string{`Error: unexpected input (expected a number) at start of file`}}, {`x=_`, `{}`, `unexpected input (expected a value) at line 1, column 3`},
{`1_`, []string{`1`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, {`x=1_`, `{"x": 1}`, `unexpected end of file (expected a value assignment) at line 1, column 5`},
{`1_000`, []string{`1000`}}, {`x=1_000`, `{"x": 1000}`, ``},
{`5_349_221`, []string{`5349221`}}, {`x=5_349_221`, `{"x": 5349221}`, ``},
{`1_2_3_4_5`, []string{`12345`}}, {`x=1_2_3_4_5`, `{"x": 12345}`, ``},
{`9_223_372_036_854_775_807`, []string{`9223372036854775807`}}, {`x=9_223_372_036_854_775_807`, `{"x": 9223372036854775807}`, ``},
{`9_223_372_036_854_775_808`, []string{ {`x=9_223_372_036_854_775_808`, `{}`,
`Panic: Handler error: MakeInt64Token cannot handle input "9223372036854775808": ` + `Panic: Handler error: MakeInt64Token cannot handle input "9223372036854775808": ` +
`strconv.ParseInt: parsing "9223372036854775808": value out of range (only use a ` + `strconv.ParseInt: parsing "9223372036854775808": value out of range ` +
`type conversion token maker, when the input has been validated on beforehand)`}}, `(only use a type conversion token maker, when the input has been validated on beforehand)`},
{`-9_223_372_036_854_775_808`, []string{`-9223372036854775808`}}, {`x=-9_223_372_036_854_775_808`, `{"x": -9223372036854775808}`, ``},
// TODO make the use of the same kind of handling for panics and for errors between parsekit and TOML. // TODO make the use of the same kind of handling for panics and for errors between parsekit and TOML.
{`-9_223_372_036_854_775_809`, []string{ {`x=-9_223_372_036_854_775_809`, `{}`,
`Panic: Handler error: MakeInt64Token cannot handle input "-9223372036854775809": ` + `Panic: Handler error: MakeInt64Token cannot handle input "-9223372036854775809": ` +
`strconv.ParseInt: parsing "-9223372036854775809": value out of range (only use a ` + `strconv.ParseInt: parsing "-9223372036854775809": value out of range ` +
`type conversion token maker, when the input has been validated on beforehand)`}}, `(only use a type conversion token maker, when the input has been validated on beforehand)`},
// Hexadecimal // Hexadecimal
{`0x0`, []string{`0`}}, {`x=0x0`, `{"x": 0}`, ``},
{`0x1`, []string{`1`}}, {`x=0x1`, `{"x": 1}`, ``},
{`0x01`, []string{`1`}}, {`x=0x01`, `{"x": 1}`, ``},
{`0x00fF`, []string{`255`}}, {`x=0x00fF`, `{"x": 255}`, ``},
{`0xf_f`, []string{`255`}}, {`x=0xf_f`, `{"x": 255}`, ``},
{`0x0_0_f_f`, []string{`255`}}, {`x=0x0_0_f_f`, `{"x": 255}`, ``},
{`0xdead_beef`, []string{`3735928559`}}, {`x=0xdead_beef`, `{"x": 3735928559}`, ``},
{`0xgood_beef`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, {`x=0xgood_beef`, `{"x": 0}`, `unexpected end of file (expected a value assignment) at line 1, column 14`},
{`0x7FFFFFFFFFFFFFFF`, []string{`9223372036854775807`}}, {`x=0x7FFFFFFFFFFFFFFF`, `{"x": 9223372036854775807}`, ``},
{`0x8000000000000000`, []string{ {`x=0x8000000000000000`, `{}`, `invalid integer value 0x8000000000000000: strconv.ParseInt: parsing "8000000000000000": value out of range at line 1, column 21`},
`Error: Cannot parse value 0x8000000000000000: strconv.ParseInt: parsing "8000000000000000": ` +
`value out of range at line 1, column 19`}},
//Octal //Octal
{`0o0`, []string{`0`}}, {`x=0o0`, `{"x": 0}`, ``},
{`0o1`, []string{`1`}}, {`x=0o1`, `{"x": 1}`, ``},
{`0o01`, []string{`1`}}, {`x=0o01`, `{"x": 1}`, ``},
{`0o10`, []string{`8`}}, {`x=0o10`, `{"x": 8}`, ``},
{`0o1_6`, []string{`14`}}, {`x=0o1_6`, `{"x": 14}`, ``},
{`0o0_0_1_1_1`, []string{`73`}}, {`x=0o0_0_1_1_1`, `{"x": 73}`, ``},
{`0o9`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, {`x=0o9`, `{"x": 0}`, `unexpected end of file (expected a value assignment) at line 1, column 6`},
{`0o777777777777777777777`, []string{`9223372036854775807`}}, {`x=0o777777777777777777777`, `{"x": 9223372036854775807}`, ``},
{`0o1000000000000000000000`, []string{ {`x=0o1000000000000000000000`, `{}`, `invalid integer value 0o1000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000": value out of range at line 1, column 27`},
`Error: Cannot parse value 0o1000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000": ` +
`value out of range at line 1, column 25`}},
// Binary // Binary
{`0b0`, []string{`0`}}, {`x=0b0`, `{"x": 0}`, ``},
{`0b1`, []string{`1`}}, {`x=0b1`, `{"x": 1}`, ``},
{`0b01`, []string{`1`}}, {`x=0b01`, `{"x": 1}`, ``},
{`0b10`, []string{`2`}}, {`x=0b10`, `{"x": 2}`, ``},
{`0b0100`, []string{`4`}}, {`x=0b0100`, `{"x": 4}`, ``},
{`0b00001000`, []string{`8`}}, {`x=0b00001000`, `{"x": 8}`, ``},
{`0b0001_0000`, []string{`16`}}, {`x=0b0001_0000`, `{"x": 16}`, ``},
{`0b9`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, {`x=0b9`, `{"x": 0}`, `unexpected end of file (expected a value assignment) at line 1, column 6`},
{`0b1_1_0_1_1`, []string{`27`}}, {`x=0b1_1_0_1_1`, `{"x": 27}`, ``},
{`0b11111111_11111111`, []string{`65535`}}, {`x=0b11111111_11111111`, `{"x": 65535}`, ``},
{`0b01111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111`, []string{`9223372036854775807`}}, {`x=0b01111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111`, `{"x": 9223372036854775807}`, ``},
{`0b10000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000`, []string{ {`x=0b10000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000`, `{}`, `invalid integer value 0b1000000000000000000000000000000000000000000000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000000000000000000000000000000000000000000000": value out of range at line 1, column 76`},
`Error: Cannot parse value 0b1000000000000000000000000000000000000000000000000000000000000000: ` +
`strconv.ParseInt: parsing "1000000000000000000000000000000000000000000000000000000000000000": ` +
`value out of range at line 1, column 74`}},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startNumber, test) testParseToAST(t, p, p.startKeyValuePair, test)
} }
} }
func TestFloat(t *testing.T) { func TestFloat(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseToASTTest{
{``, []string{`Error: unexpected end of file (expected a number) at start of file`}}, {`x=0.0`, `{"x": 0}`, ``},
{`0.0`, []string{`0`}}, {`x=+0.0`, `{"x": 0}`, ``},
{`+0.0`, []string{`0`}}, {`x=-0.0`, `{"x": -0}`, ``},
{`-0.0`, []string{`-0`}}, {`x=+1.0`, `{"x": 1}`, ``},
{`+1.0`, []string{`1`}}, {`x=3.1415`, `{"x": 3.1415}`, ``},
{`3.1415`, []string{`3.1415`}}, {`x=-0.01`, `{"x": -0.01}`, ``},
{`-0.01`, []string{`-0.01`}}, {`x=5e+22`, `{"x": 5e+22}`, ``},
{`5e+22`, []string{`5e+22`}}, {`x=1E6`, `{"x": 1e+06}`, ``},
{`1E6`, []string{`1e+06`}}, {`x=-2E-2`, `{"x": -0.02}`, ``},
{`-2E-2`, []string{`-0.02`}}, {`x=6.626e-34`, `{"x": 6.626e-34}`, ``},
{`6.626e-34`, []string{`6.626e-34`}}, {`x=224_617.445_991_228`, `{"x": 224617.445991228}`, ``},
{`224_617.445_991_228`, []string{`224617.445991228`}}, {`x=12_345.111_222e+1_2_3`, `{"x": 1.2345111222e+127}`, ``},
{`12_345.111_222e+1_2_3`, []string{`1.2345111222e+127`}}, {`x=+nan`, `{"x": NaN}`, ``},
{`+nan`, []string{`NaN`}}, {`x=-nan`, `{"x": NaN}`, ``},
{`-nan`, []string{`NaN`}}, {`x=nan`, `{"x": NaN}`, ``},
{`nan`, []string{`NaN`}}, {`x=inf`, `{"x": +Inf}`, ``},
{`inf`, []string{`+Inf`}}, {`x=+inf`, `{"x": +Inf}`, ``},
{`+inf`, []string{`+Inf`}}, {`x=-inf`, `{"x": -Inf}`, ``},
{`-inf`, []string{`-Inf`}},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startNumber, test) testParseToAST(t, p, p.startKeyValuePair, test)
} }
} }

58
value_string.go
View File

@ -43,19 +43,25 @@ var (
// There are four ways to express strings: basic, multi-line basic, literal and // There are four ways to express strings: basic, multi-line basic, literal and
// multi-line literal. All strings must contain only valid UTF-8 characters. // multi-line literal. All strings must contain only valid UTF-8 characters.
func (t *parser) startString(p *parse.API) { func (t *parser) parseString(p *parse.API) (*item, bool) {
var value string
var ok bool
switch { switch {
case p.Peek(doubleQuote3): case p.Peek(doubleQuote3):
p.Handle(t.startMultiLineBasipString) value, ok = t.parseMultiLineBasicString(p)
case p.Peek(a.DoubleQuote): case p.Peek(a.DoubleQuote):
p.Handle(t.startBasipString) value, ok = t.parseBasicString("string value", p)
case p.Peek(singleQuote3): case p.Peek(singleQuote3):
p.Handle(t.startMultiLineLiteralString) value, ok = t.parseMultiLineLiteralString(p)
case p.Peek(a.SingleQuote): case p.Peek(a.SingleQuote):
p.Handle(t.startLiteralString) value, ok = t.parseLiteralString("string value", p)
default: default:
p.Expected("a string value") p.Expected("a string value")
} }
if ok {
return newItem(pString, value), ok
}
return nil, false
} }
// Specific handling of input for basic strings. // Specific handling of input for basic strings.
@ -69,13 +75,7 @@ func (t *parser) startString(p *parse.API) {
// • No additional \escape sequences are allowed. What the spec say about this: // • No additional \escape sequences are allowed. What the spec say about this:
// "All other escape sequences [..] are reserved and, if used, TOML should // "All other escape sequences [..] are reserved and, if used, TOML should
// produce an error."" // produce an error.""
func (t *parser) startBasipString(p *parse.API) { func (t *parser) parseBasicString(name string, p *parse.API) (string, bool) {
if str, ok := t.parseBasipString("basic string", p); ok {
t.addParsedItem(pString, str)
}
}
func (t *parser) parseBasipString(name string, p *parse.API) (string, bool) {
if !p.Accept(a.DoubleQuote) { if !p.Accept(a.DoubleQuote) {
p.Expected(`opening quotation marks`) p.Expected(`opening quotation marks`)
return "", false return "", false
@ -112,12 +112,6 @@ func (t *parser) parseBasipString(name string, p *parse.API) (string, bool) {
// • Like basic strings, they must appear on a single line. // • Like basic strings, they must appear on a single line.
// //
// • Control characters other than tab are not permitted in a literal string. // • Control characters other than tab are not permitted in a literal string.
func (t *parser) startLiteralString(p *parse.API) {
if str, ok := t.parseLiteralString("literal string", p); ok {
t.addParsedItem(pString, str)
}
}
func (t *parser) parseLiteralString(name string, p *parse.API) (string, bool) { func (t *parser) parseLiteralString(name string, p *parse.API) (string, bool) {
if !p.Accept(a.SingleQuote) { if !p.Accept(a.SingleQuote) {
p.Expected("opening single quote") p.Expected("opening single quote")
@ -168,10 +162,10 @@ func (t *parser) parseLiteralString(name string, p *parse.API) (string, bool) {
// "line ending backslash". When the last non-whitespace character on a line is // "line ending backslash". When the last non-whitespace character on a line is
// a \, it will be trimmed along with all whitespace (including newlines) up to // a \, it will be trimmed along with all whitespace (including newlines) up to
// the next non-whitespace character or closing delimiter. // the next non-whitespace character or closing delimiter.
func (t *parser) startMultiLineBasipString(p *parse.API) { func (t *parser) parseMultiLineBasicString(p *parse.API) (string, bool) {
if !p.Accept(doubleQuote3.Then(newline.Optional())) { if !p.Accept(doubleQuote3.Then(newline.Optional())) {
p.Expected("opening three quotation marks") p.Expected("opening three quotation marks")
return return "", false
} }
sb := &strings.Builder{} sb := &strings.Builder{}
for { for {
@ -180,25 +174,24 @@ func (t *parser) startMultiLineBasipString(p *parse.API) {
sb.WriteString("\n") sb.WriteString("\n")
case p.Peek(controlCharacter): case p.Peek(controlCharacter):
p.Error("invalid character in multi-line basic string: %q (must be escaped)", p.Result().Rune(0)) p.Error("invalid character in multi-line basic string: %q (must be escaped)", p.Result().Rune(0))
return return sb.String(), false
case p.Accept(tok.StrInterpreted(nil, c.OneOrMore(validEscape))): case p.Accept(tok.StrInterpreted(nil, c.OneOrMore(validEscape))):
sb.WriteString(p.Result().Value(0).(string)) sb.WriteString(p.Result().Value(0).(string))
case p.Accept(lineEndingBackslash): case p.Accept(lineEndingBackslash):
// NOOP, the line-ending backslash sequence is skipped. // NOOP, the line-ending backslash sequence is skipped.
case p.Peek(a.Backslash): case p.Peek(a.Backslash):
p.Error("invalid escape sequence") p.Error("invalid escape sequence")
return return sb.String(), false
case p.Accept(m.Drop(doubleQuote3)): case p.Accept(m.Drop(doubleQuote3)):
t.addParsedItem(pString, sb.String()) return sb.String(), true
return
case p.Accept(a.ValidRune): case p.Accept(a.ValidRune):
sb.WriteString(p.Result().String()) sb.WriteString(p.Result().String())
case p.Peek(a.InvalidRune): case p.Peek(a.InvalidRune):
p.Error("invalid UTF8 rune") p.Error("invalid UTF8 rune")
return return sb.String(), false
default: default:
p.Expected("closing three quotation marks") p.Expected("closing three quotation marks")
return return sb.String(), false
} }
} }
} }
@ -216,32 +209,31 @@ func (t *parser) startMultiLineBasipString(p *parse.API) {
// sense for their platform. // sense for their platform.
// //
// • Control characters other than tab and newline are not permitted in a multi-line literal string. // • Control characters other than tab and newline are not permitted in a multi-line literal string.
func (t *parser) startMultiLineLiteralString(p *parse.API) { func (t *parser) parseMultiLineLiteralString(p *parse.API) (string, bool) {
if !p.Accept(singleQuote3.Then(newline.Optional())) { if !p.Accept(singleQuote3.Then(newline.Optional())) {
p.Expected("opening three single quotes") p.Expected("opening three single quotes")
return return "", false
} }
sb := &strings.Builder{} sb := &strings.Builder{}
for { for {
switch { switch {
case p.Accept(m.Drop(singleQuote3)): case p.Accept(m.Drop(singleQuote3)):
t.addParsedItem(pString, sb.String()) return sb.String(), true
return
case p.Accept(a.Tab): case p.Accept(a.Tab):
sb.WriteString("\t") sb.WriteString("\t")
case p.Accept(newline): case p.Accept(newline):
sb.WriteString("\n") sb.WriteString("\n")
case p.Peek(controlCharacter): case p.Peek(controlCharacter):
p.Error("invalid character in literal string: %q (no control chars allowed, except for tab and newline)", p.Result().Rune(0)) p.Error("invalid character in literal string: %q (no control chars allowed, except for tab and newline)", p.Result().Rune(0))
return return sb.String(), false
case p.Accept(a.ValidRune): case p.Accept(a.ValidRune):
sb.WriteString(p.Result().String()) sb.WriteString(p.Result().String())
case p.Peek(a.InvalidRune): case p.Peek(a.InvalidRune):
p.Error("invalid UTF8 rune") p.Error("invalid UTF8 rune")
return return sb.String(), false
default: default:
p.Expected("closing three single quotes") p.Expected("closing three single quotes")
return return sb.String(), false
} }
} }
} }

118
value_string_test.go
View File

@ -6,91 +6,85 @@ import (
) )
func TestString(t *testing.T) { func TestString(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseToASTTest{
{``, []string{`Error: unexpected end of file (expected a string value) at start of file`}}, {`x=no start quote"`, `{}`, `unexpected input (expected a value) at line 1, column 3`},
{`no start quote"`, []string{`Error: unexpected input (expected a string value) at start of file`}}, {`x="basic s\tring"`, `{"x": "basic s\tring"}`, ``},
{`"basic s\tring"`, []string{`"basic s\tring"`}}, {"x=\"\"\"\n basic multi-line\n string value\n\"\"\"", `{"x": " basic multi-line\n string value\n"}`, ``},
{"\"\"\"\n basic multi-line\n string value\n\"\"\"", []string{`" basic multi-line\n string value\n"`}}, {`x='literal s\tring'`, `{"x": "literal s\\tring"}`, ``},
{`'literal s\tring'`, []string{`"literal s\\tring"`}}, {"x='''\n literal multi-line\n string value\n'''", `{"x": " literal multi-line\n string value\n"}`, ``},
{"'''\n literal multi-line\n string value\n'''", []string{`" literal multi-line\n string value\n"`}},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startString, test) testParseToAST(t, p, p.startKeyValuePair, test)
} }
} }
func TestBasipString(t *testing.T) { func TestBasipString(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseToASTTest{
{``, []string{`Error: unexpected end of file (expected opening quotation marks) at start of file`}}, {`x="no end quote`, `{}`, `unexpected end of file (expected closing quotation marks) at line 1, column 16`},
{`no start quote"`, []string{`Error: unexpected input (expected opening quotation marks) at start of file`}}, {`x=""`, `{"x": ""}`, ``},
{`"no end quote`, []string{`Error: unexpected end of file (expected closing quotation marks) at line 1, column 14`}}, {`x="simple string"`, `{"x": "simple string"}`, ``},
{`""`, []string{`""`}}, {`x="with\tsome\r\nvalid escapes\b"`, `{"x": "with\tsome\r\nvalid escapes\b"}`, ``},
{`"simple string"`, []string{`"simple string"`}}, {`x="with an \invalid escape"`, `{}`, `invalid escape sequence at line 1, column 12`},
{`"with\tsome\r\nvalid escapes\b"`, []string{`"with\tsome\r\nvalid escapes\b"`}}, {`x="A cool UTF8 ƃuıɹʇs"`, `{"x": "A cool UTF8 ƃuıɹʇs"}`, ``},
{`"with an \invalid escape"`, []string{`Error: invalid escape sequence at line 1, column 10`}}, {`x="A string with UTF8 escape \u2318"`, `{"x": "A string with UTF8 escape ⌘"}`, ``},
{`"A cool UTF8 ƃuıɹʇs"`, []string{`"A cool UTF8 ƃuıɹʇs"`}}, {"x=\"Invalid character for UTF \xcd\"", `{}`, `invalid UTF8 rune at line 1, column 30`},
{`"A string with UTF8 escape \u2318"`, []string{`"A string with UTF8 escape ⌘"`}}, {"x=\"Character that mus\t be escaped\"", `{}`, `invalid character in string value: '\t' (must be escaped) at line 1, column 22`},
{"\"Invalid character for UTF \xcd\"", []string{`Error: invalid UTF8 rune at line 1, column 28`}}, {"x=\"Character that must be escaped \u0000\"", `{}`, `invalid character in string value: '\x00' (must be escaped) at line 1, column 35`},
{"\"Character that mus\t be escaped\"", []string{`Error: invalid character in basic string: '\t' (must be escaped) at line 1, column 20`}}, {"x=\"Character that must be escaped \x7f\"", `{}`, `invalid character in string value: '\u007f' (must be escaped) at line 1, column 35`},
{"\"Character that must be escaped \u0000\"", []string{`Error: invalid character in basic string: '\x00' (must be escaped) at line 1, column 33`}},
{"\"Character that must be escaped \x7f\"", []string{`Error: invalid character in basic string: '\u007f' (must be escaped) at line 1, column 33`}},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startBasipString, test) testParseToAST(t, p, p.startKeyValuePair, test)
} }
} }
func TestMultiLineBasipString(t *testing.T) { func TestMultiLineBasipString(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseToASTTest{
{``, []string{`Error: unexpected end of file (expected opening three quotation marks) at start of file`}}, {`x="""missing close quote""`, `{}`, `unexpected end of file (expected closing three quotation marks) at line 1, column 27`},
{`"""missing close quote""`, []string{`Error: unexpected end of file (expected closing three quotation marks) at line 1, column 25`}}, {`x=""""""`, `{"x": ""}`, ``},
{`""""""`, []string{`""`}}, {"x=\"\"\"\n\"\"\"", `{"x": ""}`, ``},
{"\"\"\"\n\"\"\"", []string{`""`}}, {"x=\"\"\"\r\n\r\n\"\"\"", `{"x": "\n"}`, ``},
{"\"\"\"\r\n\r\n\"\"\"", []string{`"\n"`}}, {`x="""\"\"\"\""""`, `{"x": "\"\"\"\""}`, ``},
{`"""\"\"\"\""""`, []string{`"\"\"\"\""`}}, {"x=\"\"\"\nThe quick brown \\\n\n\n \t fox jumps over \\\n\t the lazy dog.\\\n \"\"\"", `{"x": "The quick brown fox jumps over the lazy dog."}`, ``},
{"\"\"\"\nThe quick brown \\\n\n\n \t fox jumps over \\\n\t the lazy dog.\\\n \"\"\"", []string{`"The quick brown fox jumps over the lazy dog."`}}, {"x=\"\"\"No control chars \f allowed\"\"\"", `{}`, `invalid character in multi-line basic string: '\f' (must be escaped) at line 1, column 23`},
{"\"\"\"No control chars \f allowed\"\"\"", []string{`Error: invalid character in multi-line basic string: '\f' (must be escaped) at line 1, column 21`}}, {"x=\"\"\"Escaping control chars\\nis valid\"\"\"", `{"x": "Escaping control chars\nis valid"}`, ``},
{"\"\"\"Escaping control chars\\nis valid\"\"\"", []string{`"Escaping control chars\nis valid"`}}, {"x=\"\"\"Invalid escaping \\is not allowed\"\"\"", `{}`, `invalid escape sequence at line 1, column 23`},
{"\"\"\"Invalid escaping \\is not allowed\"\"\"", []string{`Error: invalid escape sequence at line 1, column 21`}}, {"x=\"\"\"Invalid rune \xcd\"\"\"", `{}`, `invalid UTF8 rune at line 1, column 19`},
{"\"\"\"Invalid rune \xcd\"\"\"", []string{`Error: invalid UTF8 rune at line 1, column 17`}},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startMultiLineBasipString, test) testParseToAST(t, p, p.startKeyValuePair, test)
} }
} }
func TestLiteralString(t *testing.T) { func TestLiteralString(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseToASTTest{
{``, []string{`Error: unexpected end of file (expected opening single quote) at start of file`}}, {`x='missing close quote`, `{}`, `unexpected end of file (expected closing single quote) at line 1, column 23`},
{`'missing close quote`, []string{`Error: unexpected end of file (expected closing single quote) at line 1, column 21`}}, {`x=''`, `{"x": ""}`, ``},
{`''`, []string{`""`}}, {`x='simple'`, `{"x": "simple"}`, ``},
{`'simple'`, []string{`"simple"`}}, {`x='C:\Users\nodejs\templates'`, `{"x": "C:\\Users\\nodejs\\templates"}`, ``},
{`'C:\Users\nodejs\templates'`, []string{`"C:\\Users\\nodejs\\templates"`}}, {`x='\\ServerX\admin$\system32\'`, `{"x": "\\\\ServerX\\admin$\\system32\\"}`, ``},
{`'\\ServerX\admin$\system32\'`, []string{`"\\\\ServerX\\admin$\\system32\\"`}}, {`x='Tom "Dubs" Preston-Werner'`, `{"x": "Tom \"Dubs\" Preston-Werner"}`, ``},
{`'Tom "Dubs" Preston-Werner'`, []string{`"Tom \"Dubs\" Preston-Werner"`}}, {`x='<\i\c*\s*>'`, `{"x": "<\\i\\c*\\s*>"}`, ``},
{`'<\i\c*\s*>'`, []string{`"<\\i\\c*\\s*>"`}}, {"x='No cont\rol chars allowed'", `{}`, `invalid character in string value: '\r' (no control chars allowed, except for tab) at line 1, column 11`},
{"'No cont\rol chars allowed'", []string{`Error: invalid character in literal string: '\r' (no control chars allowed, except for tab) at line 1, column 9`}}, {"x='Except\tfor\ttabs'", `{"x": "Except\tfor\ttabs"}`, ``},
{"'Except\tfor\ttabs'", []string{`"Except\tfor\ttabs"`}}, {"x='Invalid rune \xcd'", `{}`, `invalid UTF8 rune at line 1, column 17`},
{"'Invalid rune \xcd'", []string{`Error: invalid UTF8 rune at line 1, column 15`}},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startLiteralString, test) testParseToAST(t, p, p.startKeyValuePair, test)
} }
} }
func TestMultiLineLiteralString(t *testing.T) { func TestMultiLineLiteralString(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseToASTTest{
{``, []string{`Error: unexpected end of file (expected opening three single quotes) at start of file`}}, {`x='''missing close quote''`, `{}`, `unexpected end of file (expected closing three single quotes) at line 1, column 27`},
{`'''missing close quote''`, []string{`Error: unexpected end of file (expected closing three single quotes) at line 1, column 25`}}, {`x=''''''`, `{"x": ""}`, ``},
{`''''''`, []string{`""`}}, {"x='''\n'''", `{"x": ""}`, ``},
{"'''\n'''", []string{`""`}}, {`x='''I [dw]on't need \d{2} apples'''`, `{"x": "I [dw]on't need \\d{2} apples"}`, ``},
{`'''I [dw]on't need \d{2} apples'''`, []string{`"I [dw]on't need \\d{2} apples"`}}, {"x='''\nThere can\nbe newlines\r\nand \ttabs!\r\n'''", `{"x": "There can\nbe newlines\nand \ttabs!\n"}`, ``},
{"'''\nThere can\nbe newlines\r\nand \ttabs!\r\n'''", []string{`"There can\nbe newlines\nand \ttabs!\n"`}}, {"x='''No other \f control characters'''", `{}`, `invalid character in literal string: '\f' (no control chars allowed, except for tab and newline) at line 1, column 15`},
{"'''No other \f control characters'''", []string{`Error: invalid character in literal string: '\f' (no control chars allowed, except for tab and newline) at line 1, column 13`}}, {"x='''No invalid runes allowed \xcd'''", `{}`, `invalid UTF8 rune at line 1, column 31`},
{"'''No invalid runes allowed \xcd'''", []string{"Error: invalid UTF8 rune at line 1, column 29"}},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startMultiLineLiteralString, test) testParseToAST(t, p, p.startKeyValuePair, test)
} }
} }
@ -99,8 +93,8 @@ func TestBasipStringWithUnescapedControlCharacters(t *testing.T) {
// The missing one (\x7f) is covered in the previous test. // The missing one (\x7f) is covered in the previous test.
for i := 0x00; i <= 0x1F; i++ { for i := 0x00; i <= 0x1F; i++ {
p := newParser() p := newParser()
input := fmt.Sprintf(`"%c"`, rune(i)) input := fmt.Sprintf(`x="%c"`, rune(i))
expected := fmt.Sprintf(`Error: invalid character in basic string: %q (must be escaped) at line 1, column 2`, rune(i)) expected := fmt.Sprintf(`invalid character in string value: %q (must be escaped) at line 1, column 4`, rune(i))
testParseHandler(t, p, p.startString, parseTest{input, []string{expected}}) testParseToAST(t, p, p.startKeyValuePair, parseToASTTest{input, "{}", expected})
} }
} }

57
value_table.go
View File

@ -86,11 +86,32 @@ var (
// [[fruit.variety]] // [[fruit.variety]]
// name = "plantain" // name = "plantain"
tableArrayOpen = c.Seq(dropBlanks, a.SquareOpen, a.SquareOpen, dropBlanks) tableArrayOpen = c.Seq(dropBlanks, a.SquareOpen, a.SquareOpen, dropBlanks)
tableArrayClose = c.Seq(dropBlanks, a.SquareClose, a.SquareClose, a.EndOfLine.Or(comment)) tableArrayClose = c.Seq(dropBlanks, a.SquareClose, a.SquareClose, dropBlanks, a.EndOfLine.Or(comment))
// Inline tables provide a more compact syntax for expressing tables.
// They are especially useful for grouped data that can otherwise quickly
// become verbose. Inline tables are enclosed in curly braces { and }.
// Within the braces, zero or more comma separated key/value pairs may appear.
// Key/value pairs take the same form as key/value pairs in standard tables.
// All value types are allowed, including inline tables.
//
// Inline tables are intended to appear on a single line. No newlines are
// allowed between the curly braces unless they are valid within a value.
// Even so, it is strongly discouraged to break an inline table onto multiple
// lines. If you find yourself gripped with this desire, it means you should
// be using standard tables.
//
// name = { first = "Tom", last = "Preston-Werner" }
// point = { x = 1, y = 2 }
// animal = { type.name = "pug" }
inlineTableOpen = c.Seq(dropBlanks, a.CurlyOpen, dropBlanks)
inlineTableClose = c.Seq(dropBlanks, a.CurlyClose, dropBlanks, a.EndOfLine.Or(comment))
) )
func (t *parser) startTable(p *parse.API) { func (t *parser) startTable(p *parse.API) {
switch { switch {
case p.Accept(tableArrayOpen):
p.Handle(t.startArrayOfTables)
case p.Accept(tableOpen): case p.Accept(tableOpen):
p.Handle(t.startPlainTable) p.Handle(t.startPlainTable)
default: default:
@ -98,14 +119,30 @@ func (t *parser) startTable(p *parse.API) {
} }
} }
func (t *parser) startPlainTable(p *parse.API) { func (t *parser) startArrayOfTables(p *parse.API) {
if !p.Handle(t.startKey) { if key, ok := t.parseKey(p, []string{}); ok {
return if !p.Accept(tableArrayClose) {
p.Expected("closing ']]' for array of tables name")
return
}
if err := t.openArrayOfTables(key); err != nil {
p.Error("%s", err)
return
}
p.Handle(t.startKeyValuePair)
}
}
func (t *parser) startPlainTable(p *parse.API) {
if key, ok := t.parseKey(p, []string{}); ok {
if !p.Accept(tableClose) {
p.Expected("closing ']' for table name")
return
}
if err := t.openTable(key); err != nil {
p.Error("%s", err)
return
}
p.Handle(t.startKeyValuePair)
} }
if !p.Accept(tableClose) {
p.Expected("closing ']' for table name")
}
key := t.Items[0]
t.Items = t.Items[1:]
t.openTable(key)
} }

55
value_table_test.go
View File

@ -4,20 +4,71 @@ import (
"testing" "testing"
) )
func TestTable(t *testing.T) { func TestTableKey(t *testing.T) {
for _, test := range []parseTest{ for _, test := range []parseTest{
{"", []string{`Error: unexpected end of file (expected a table) at start of file`}}, {"", []string{`Error: unexpected end of file (expected a table) at start of file`}},
{"[", []string{`Error: unexpected end of file (expected a key name) at line 1, column 2`}}, {"[", []string{`Error: unexpected end of file (expected a key name) at line 1, column 2`}},
{" \t [", []string{`Error: unexpected end of file (expected a key name) at line 1, column 5`}}, {" \t [", []string{`Error: unexpected end of file (expected a key name) at line 1, column 5`}},
{" \t [key", []string{`Error: unexpected end of file (expected closing ']' for table name) at line 1, column 8`}}, {" \t [key", []string{`Error: unexpected end of file (expected closing ']' for table name) at line 1, column 8`}},
{" \t [key.", []string{`key("key")`, `Error: unexpected end of file (expected a key name) at line 1, column 9`}}, {" \t [key.", []string{`Error: unexpected end of file (expected a key name) at line 1, column 9`}},
{" \t [key.'sub key'", []string{`Error: unexpected end of file (expected closing ']' for table name) at line 1, column 18`}}, {" \t [key.'sub key'", []string{`Error: unexpected end of file (expected closing ']' for table name) at line 1, column 18`}},
{" \t [key.'sub key' \t]", []string{}}, {" \t [key.'sub key' \t]", []string{}},
{" \t [key.'sub key' \t] \t ", []string{}}, {" \t [key.'sub key' \t] \t ", []string{}},
{" \t [key.'sub key' \t] \t \n", []string{}}, {" \t [key.'sub key' \t] \t \n", []string{}},
{" \t [key.'sub key' \t]# with comment\n", []string{}},
{" \t [key.'sub key' \t] \t # with comment\n", []string{}}, {" \t [key.'sub key' \t] \t # with comment\n", []string{}},
} { } {
p := newParser() p := newParser()
testParseHandler(t, p, p.startTable, test) testParseHandler(t, p, p.startTable, test)
} }
} }
func TestTable(t *testing.T) {
for _, test := range []parseToASTTest{
{"[a]", `{"a": {}}`, ``},
{"['a key']", `{"a key": {}}`, ``},
{"[\"a key\"]", `{"a key": {}}`, ``},
{"[\"a key\".'sub'.key]", `{"a key": {"sub": {"key": {}}}}`, ``},
{"[a]\nx=1234", `{"a": {"x": 1234}}`, ``},
{"[a]\nx=1234\ny='string'", `{"a": {"x": 1234, "y": "string"}}`, ``},
{"[a]\n[b]", `{"a": {}, "b": {}}`, ``},
{"[a]\n[a.b]\n[a.b.c]\n[a.d]\nx=1", `{"a": {"b": {"c": {}}, "d": {"x": 1}}}`, ``},
{"[a]\n[b] #another table \na=1\n", `{"a": {}, "b": {"a": 1}}`, ``},
{"[a]\nx=1\ny=2\n[b] #another table \nx=1\ny=2021-01-01", `{"a": {"x": 1, "y": 2}, "b": {"x": 1, "y": 2021-01-01}}`, ``},
{"[a]\nx=1\ny=2\n[a.b] #subtable \nx=1\ny=2021-01-01", `{"a": {"b": {"x": 1, "y": 2021-01-01}, "x": 1, "y": 2}}`, ``},
} {
p := newParser()
testParseToAST(t, p, p.startTable, test)
}
}
func TestArrayOfTableKey(t *testing.T) {
for _, test := range []parseTest{
{"[[", []string{`Error: unexpected end of file (expected a key name) at line 1, column 3`}},
{" \t [[", []string{`Error: unexpected end of file (expected a key name) at line 1, column 6`}},
{" \t [[key", []string{`Error: unexpected end of file (expected closing ']]' for array of tables name) at line 1, column 9`}},
{" \t [[key.", []string{`Error: unexpected end of file (expected a key name) at line 1, column 10`}},
{" \t [[key.'sub key'", []string{`Error: unexpected end of file (expected closing ']]' for array of tables name) at line 1, column 19`}},
{" \t [[key.'sub key' \t]]", []string{}},
{" \t [[key.'sub key' \t]] \t ", []string{}},
{" \t [[key.'sub key' \t]] \t \n", []string{}},
{" \t [[key.'sub key' \t]]# with comment\n", []string{}},
{" \t [[key.'sub key' \t]] \t # with comment\n", []string{}},
} {
p := newParser()
testParseHandler(t, p, p.startTable, test)
}
}
func TestArrayOfTables(t *testing.T) {
for _, test := range []parseToASTTest{
{"[[a]]", `{"a": [{}]}`, ``},
{"[[a]]\n[['a']]\n[[\"a\"]]", `{"a": [{}, {}, {}]}`, ``},
{"[[a]]\n[['a']]\n[b]\n[[\"a\"]]", `{"a": [{}, {}, {}], "b": {}}`, ``},
{"[[a]]\nx=1\n[['a']]\nx=2\ny=3\n[[\"a\"]]", `{"a": [{"x": 1}, {"x": 2, "y": 3}, {}]}`, ``},
{"[a]\n[[a.b]]\nx=1\n[[a.b]]\nx=2\n[a.c]\ny=1234", `{"a": {"b": [{"x": 1}, {"x": 2}], "c": {"y": 1234}}}`, ``},
} {
p := newParser()
testParseToAST(t, p, p.startTable, test)
}
}

84
value_test.go
View File

@ -1,46 +1,42 @@
package toml package toml
import ( // func TestValue(t *testing.T) {
"testing" // for _, test := range []parseTest{
) // {``, []string{`Error: unexpected end of file (expected a value) at start of file`}},
// {`"basic s\tring value"`, []string{`"basic s\tring value"`}},
func TestValue(t *testing.T) { // {`'literal s\tring value'`, []string{`"literal s\\tring value"`}},
for _, test := range []parseTest{ // {"\"\"\"basic multi-line\nstring value\"\"\"", []string{`"basic multi-line\nstring value"`}},
{``, []string{`Error: unexpected end of file (expected a value) at start of file`}}, // {"'''literal multi-line\nstring value'''", []string{`"literal multi-line\nstring value"`}},
{`"basic s\tring value"`, []string{`"basic s\tring value"`}}, // {"true", []string{`true`}},
{`'literal s\tring value'`, []string{`"literal s\\tring value"`}}, // {"false", []string{`false`}},
{"\"\"\"basic multi-line\nstring value\"\"\"", []string{`"basic multi-line\nstring value"`}}, // {"0", []string{`0`}},
{"'''literal multi-line\nstring value'''", []string{`"literal multi-line\nstring value"`}}, // {"+0", []string{`0`}},
{"true", []string{`true`}}, // {"-0", []string{`0`}},
{"false", []string{`false`}}, // {"0.0", []string{`0`}},
{"0", []string{`0`}}, // {"+0.0", []string{`0`}},
{"+0", []string{`0`}}, // {"-0.0", []string{`-0`}},
{"-0", []string{`0`}}, // {"1234", []string{`1234`}},
{"0.0", []string{`0`}}, // {"-1234", []string{`-1234`}},
{"+0.0", []string{`0`}}, // {"+9_8_7.6_5_4e-321", []string{`9.8765e-319`}},
{"-0.0", []string{`-0`}}, // {"-1_234.5678e-33", []string{`-1.2345678e-30`}},
{"1234", []string{`1234`}}, // {"inf", []string{`+Inf`}},
{"-1234", []string{`-1234`}}, // {"+inf", []string{`+Inf`}},
{"+9_8_7.6_5_4e-321", []string{`9.8765e-319`}}, // {"-inf", []string{`-Inf`}},
{"-1_234.5678e-33", []string{`-1.2345678e-30`}}, // {"nan", []string{`NaN`}},
{"inf", []string{`+Inf`}}, // {"+nan", []string{`NaN`}},
{"+inf", []string{`+Inf`}}, // {"-nan", []string{`NaN`}},
{"-inf", []string{`-Inf`}}, // {"2019-06-19", []string{`2019-06-19`}},
{"nan", []string{`NaN`}}, // {"08:38:54", []string{`08:38:54`}},
{"+nan", []string{`NaN`}}, // {"08:38:54.8765487654876", []string{`08:38:54.876548765`}},
{"-nan", []string{`NaN`}}, // {"2019-06-19 08:38:54", []string{`2019-06-19 08:38:54`}},
{"2019-06-19", []string{`2019-06-19`}}, // {"2019-06-19T08:38:54", []string{`2019-06-19 08:38:54`}},
{"08:38:54", []string{`08:38:54`}}, // {"2019-06-19T08:38:54.88888", []string{`2019-06-19 08:38:54.88888`}},
{"08:38:54.8765487654876", []string{`08:38:54.876548765`}}, // {"1979-05-27T07:32:00Z", []string{`1979-05-27T07:32:00Z`}},
{"2019-06-19 08:38:54", []string{`2019-06-19 08:38:54`}}, // {"1979-05-27T00:32:00-07:00", []string{`1979-05-27T00:32:00-07:00`}},
{"2019-06-19T08:38:54", []string{`2019-06-19 08:38:54`}}, // {"1979-05-27T00:32:00.999999-07:00", []string{`1979-05-27T00:32:00.999999-07:00`}},
{"2019-06-19T08:38:54.88888", []string{`2019-06-19 08:38:54.88888`}}, // {"[1,2,3]", []string{`[1, 2, 3]`}},
{"1979-05-27T07:32:00Z", []string{`1979-05-27T07:32:00Z`}}, // } {
{"1979-05-27T00:32:00-07:00", []string{`1979-05-27T00:32:00-07:00`}}, // p := newParser()
{"1979-05-27T00:32:00.999999-07:00", []string{`1979-05-27T00:32:00.999999-07:00`}}, // testParseHandler(t, p, p.startValue, test)
{"[1,2,3]", []string{`[1, 2, 3]`}}, // }
} { // }
p := newParser()
testParseHandler(t, p, p.startValue, test)
}
}