OpenSourceProjects
/
go-toml
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Huge overhaul in AST handling. The AST is now fully integrated with the parser, which has been simplified quite a bit because of this.

This commit is contained in:
Maurice Makaay 2019-06-25 21:29:05 +00:00
parent 15560b29b0
commit c536dd1243
21 changed files with 928 additions and 690 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

308
ast.go
View File

@ -2,6 +2,7 @@ package toml
import (
"fmt"
"regexp"
"sort"
"strings"
"time"
@ -14,18 +15,12 @@ type item struct {
}
// table represents a TOML table.
type table map[string]item
type table map[string]*item
// itemType identifies the semantic role of a TOML item.
type itemType string
const (
// TODO Would be nice to not need these in the end.
pComment itemType = "comment"
pKey itemType = "key"
pAssign itemType = "assign"
// TODO and just use these data types.
pString itemType = "string" // "various", 'types', """of""", '''strings'''
pInteger itemType = "integer" // 12345, 0xffee12a0, 0o0755, 0b101101011
pFloat itemType = "float" // 10.1234, 143E-12, 43.28377e+4, +inf, -inf, nan
@ -34,14 +29,176 @@ const (
pLocalDateTime itemType = "datetime" // 2018-12-25 12:12:18.876772533
pLocalDate itemType = "date" // 2017-05-17
pLocalTime itemType = "time" // 23:01:22
pArray itemType = "array" // defined using an [[array.of.tables]]
pStaticArray itemType = "static array" // defined using ["an", "inline", "array"]
pArrayOfTables itemType = "array" // defined using an [[array.of.tables]]
pArray itemType = "static array" // defined using ["an", "inline", "array"]
pTable itemType = "table" // defined using { "inline" = "table" } or [standard.table]
)
// newItem instantiates a new item struct.
func newItem(itemType itemType, data ...interface{}) item {
return item{Type: itemType, Data: data}
func newItem(itemType itemType, data ...interface{}) *item {
return &item{Type: itemType, Data: data}
}
// newKey instantiates a new key.
func newKey(key ...string) []string {
return key
}
// parser holds the state for the TOML parser. All parsing functions are
// methods of this struct.
type parser struct {
Items []*item // a buffer for holding parsed items
Root table // the root-level TOML table (each TOML doc is implicitly a table)
Current table // the currently active TOML table
CurrentKey []string // the key for the currently active TOML table
}
func newParser() *parser {
p := &parser{Root: make(table)}
p.Current = p.Root
return p
}
func (t *parser) setKeyValuePair(key []string, value *item) error {
// First make sure the table structure for storing the value exists.
node, lastKeyPart, err := t.makeTablePath(key)
if err != nil {
return fmt.Errorf("invalid key/value pair: %s", err)
}
// Check if the key is still free for use.
if existing, ok := node[lastKeyPart]; ok {
path := t.formatKeyPath(key, len(key)-1)
return fmt.Errorf("invalid key/value pair: %s item already exists at key %s", existing.Type, path)
}
// It is, store the value in the table.
node[lastKeyPart] = value
return nil
}
func (t *parser) openTable(key []string) error {
t.CurrentKey = nil
t.Current = t.Root
// Go over all requested levels of the key. For all levels, except the last
// one, it is okay if a table already exists. For at least the last level,
// no table or value must exist, because that would mean we are overwriting
// an existing key/value pair, which is not allowed.
node, lastKeyPart, err := t.makeTablePath(key)
if err != nil {
return fmt.Errorf("invalid table: %s", err)
}
// Check if the key is still free for use.
if existing, ok := node[lastKeyPart]; ok {
path := t.formatKeyPath(key, len(key)-1)
return fmt.Errorf("invalid table: %s item already exists at key %s", existing.Type, path)
}
// The subtable does not exist yet. Create the subtable.
subTable := make(table)
node[lastKeyPart] = newItem(pTable, subTable)
node = subTable
// From here on, key/value pairs are added to the newly defined table.
t.Current = node
t.CurrentKey = key
return nil
}
func (t *parser) openArrayOfTables(key []string) error {
t.CurrentKey = nil
t.Current = t.Root
// Go over all requested levels of the key. For all levels, except the last
// one, it is okay if a table already exists. For the last level, either
// no item must exist (in which case a table array will be created), or a
// table array must exist.
// Other cases would mean we are overwriting an existing key/value pair,
// which is not allowed.
node, lastKeyPart, err := t.makeTablePath(key)
if err != nil {
return fmt.Errorf("invalid table array: %s", err)
}
// At the last key position, there must be either no value yet, or the
// existing value must be a table array. Other values are invalid.
if existing, ok := node[lastKeyPart]; ok {
if existing.Type != pArrayOfTables {
path := t.formatKeyPath(key, len(key)-1)
return fmt.Errorf("invalid table array: %s item already exists at key %s", existing.Type, path)
}
// A table array exists. Add a new table to this array.
array := node[lastKeyPart]
subTable := make(table)
tables := array.Data
tables = append(tables, newItem(pTable, subTable))
array.Data = tables
node = subTable
} else {
// No value exists at the defined key path. Create a new table array.
subTable := make(table)
node[lastKeyPart] = newItem(pArrayOfTables, newItem(pTable, subTable))
node = subTable
}
// From here on, key/value pairs are added to the newly defined table.
t.Current = node
t.CurrentKey = key
return nil
}
func (t *parser) makeTablePath(key []string) (table, string, error) {
node := t.Current
for i, keyPart := range key {
// Arrived at the last key part? Then the path towards that key is
// setup correctly. Return the last part, so the caller can use it.
isLast := i == len(key)-1
if isLast {
return node, keyPart, nil
}
if subItem, ok := node[keyPart]; ok {
// You cannot overwrite an already defined key, regardless its value.
// When a value already exists at the current key, this can only be a table.
if subItem.Type != pTable {
path := t.formatKeyPath(key, i)
return nil, "", fmt.Errorf("%s item already exists at key %s", subItem.Type, path)
}
// All is okay, traverse to the subtable.
node = subItem.Data[0].(table)
} else {
// The subtable does not exist yet. Create the subtable.
subTable := make(table)
node[keyPart] = newItem(pTable, subTable)
node = subTable
}
}
panic("makeTablePath(): empty key provided; a key must have at least one key part")
}
func (t *parser) formatKeyPath(key []string, end int) string {
var sb strings.Builder
sb.WriteRune('[')
if t.CurrentKey != nil {
for i, keyPart := range t.CurrentKey {
if i > 0 {
sb.WriteString("->")
}
sb.WriteString(formatKeyName(keyPart))
}
}
for i, keyPart := range key {
if t.CurrentKey != nil || i > 0 {
sb.WriteString("->")
}
sb.WriteString(formatKeyName(keyPart))
if i == end {
break
}
}
sb.WriteRune(']')
return sb.String()
}
func formatKeyName(key string) string {
if ok, _ := regexp.Match(`^\w+$`, []byte(key)); ok {
return key
}
return fmt.Sprintf("%q", key)
}
func (t table) String() string {
@ -52,12 +209,6 @@ func (parseItem item) String() string {
switch parseItem.Type {
case pString:
return fmt.Sprintf("%q", parseItem.Data[0])
case pInteger:
return fmt.Sprintf("%v", parseItem.Data[0])
case pFloat:
return fmt.Sprintf("%v", parseItem.Data[0])
case pBoolean:
return fmt.Sprintf("%v", parseItem.Data[0])
case pOffsetDateTime:
return parseItem.Data[0].(time.Time).Format(time.RFC3339Nano)
case pLocalDateTime:
@ -66,12 +217,12 @@ func (parseItem item) String() string {
return parseItem.Data[0].(time.Time).Format("2006-01-02")
case pLocalTime:
return parseItem.Data[0].(time.Time).Format("15:04:05.999999999")
case pArray:
case pArrayOfTables:
fallthrough
case pStaticArray:
items := make([]string, len(parseItem.Data[0].([]item)))
for i, d := range parseItem.Data[0].([]item) {
items[i] = d.String()
case pArray:
items := make([]string, len(parseItem.Data))
for i, value := range parseItem.Data {
items[i] = value.(*item).String()
}
return fmt.Sprintf("[%s]", strings.Join(items, ", "))
case pTable:
@ -88,118 +239,7 @@ func (parseItem item) String() string {
items[i] = fmt.Sprintf("%q: %s", k, pairs[k].String())
}
return fmt.Sprintf("{%s}", strings.Join(items, ", "))
case pComment:
return fmt.Sprintf("comment(%q)", parseItem.Data[0])
case pKey:
items := make([]string, len(parseItem.Data))
for i, e := range parseItem.Data {
items[i] = fmt.Sprintf("%q", e)
}
return fmt.Sprintf("key(%s)", strings.Join(items, ", "))
case pAssign:
return "="
default:
panic(fmt.Sprintf("Missing String() formatting for item type '%s'", parseItem.Type))
return fmt.Sprintf("%v", parseItem.Data[0])
}
}
// parser holds the state for the TOML parser. All parsing functions are
// methods of this struct.
type parser struct {
Items []item // a buffer for holding parsed items
Root table // the root-level TOML table (each TOML doc is implicitly a table)
Current table // the currently active TOML table
}
func newParser() *parser {
p := &parser{Root: make(table)}
p.Current = p.Root
return p
}
func (t *parser) addParsedItem(itemType itemType, data ...interface{}) {
t.Items = append(t.Items, newItem(itemType, data...))
}
func (t *parser) setValue(key item, value item) error {
// When the key has multiple elements, then first make sure the table structure
// for storing the value exists.
var valueKey string
node := t.Current
l := len(key.Data)
if l > 1 {
pathKeys := key.Data[0 : l-1]
valueKey = key.Data[l-1].(string)
for i, name := range pathKeys {
name := name.(string)
if subItem, ok := node[name]; ok {
// An item was found at the current key. It is expected to be a table.
if subItem.Type != pTable {
path := formatKeyPath(key, i)
return fmt.Errorf("invalid key used: %s item already exists at key %s", subItem.Type, path)
}
node = subItem.Data[0].(table)
} else {
// No item was found at the current key. Create a new subtable.
subTable := make(table)
node[name] = newItem(pTable, subTable)
node = subTable
}
}
} else {
valueKey = key.Data[0].(string)
node = t.Current
}
if existing, ok := node[valueKey]; ok {
path := "moetnog"
return fmt.Errorf("Cannot store value: %s item already exists at key %s", existing.Type, path)
}
node[valueKey] = value
return nil
}
func (t *parser) openTable(key item) (table, error) {
node := t.Root
// Go over all requested levels of the key. For all levels, except the last
// one, it is okay if a table already exists. For at least the last level,
// no table or value must exist, because that would mean we are overwriting
// an existing key/value pair, which is not allowed.
for i, value := range key.Data {
keyName := value.(string)
if subItem, ok := node[keyName]; ok {
// You cannot overwrite an already defined key, regardless its value.
if subItem.Type != pTable {
path := formatKeyPath(key, i)
return nil, fmt.Errorf("Cannot create table: %s item already exists at key %s", subItem.Type, path)
}
// Like keys, you cannot define any table more than once. Doing so is invalid.
isLast := i == len(key.Data)-1
if isLast {
path := formatKeyPath(key, i)
return nil, fmt.Errorf("Cannot create table: table for key %s already exists", path)
}
node = subItem.Data[0].(table)
} else {
// Create the subtable.
subTable := make(table)
node[keyName] = newItem(pTable, subTable)
node = subTable
}
}
// From here on, key/value pairs are added to the newly defined table.
t.Current = node
return node, nil
}
func formatKeyPath(key item, end int) string {
var sb strings.Builder
for i := 0; i <= end; i++ {
if i > 0 {
sb.WriteRune('.')
}
sb.WriteString(fmt.Sprintf("%q", key.Data[i].(string)))
}
return sb.String()
}

174
ast_test.go
View File

@ -7,63 +7,171 @@ import (
func TestAST_ConstructStructure(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.setValue(newItem(pKey, "ding"), newItem(pInteger, 10))
p.setValue(newItem(pKey, "dong"), newItem(pString, "not a song"))
p.openTable(newItem(pKey, "key1", "key2 a"))
p.setValue(newItem(pKey, "dooh"), newItem(pBoolean, true))
p.setValue(newItem(pKey, "dah"), newItem(pBoolean, false))
p.openTable(newItem(pKey, "key1", "key2 b"))
p.setValue(newItem(pKey, "dieh"), newItem(pFloat, 1.111))
p.setValue(newItem(pKey, "duh"), newItem(pFloat, 1.18e-12))
p.setKeyValuePair(newKey("ding"), newItem(pInteger, 10))
p.setKeyValuePair(newKey("dong"), newItem(pString, "not a song"))
p.openTable(newKey("key1", "key2 a"))
p.setKeyValuePair(newKey("dooh"), newItem(pBoolean, true))
p.setKeyValuePair(newKey("dah"), newItem(pBoolean, false))
p.openTable(newKey("key1", "key2 b"))
p.setKeyValuePair(newKey("dieh"), newItem(pFloat, 1.111))
p.setKeyValuePair(newKey("duh"), newItem(pFloat, 1.18e-12))
p.setKeyValuePair(newKey("foo", "bar"), newItem(pArrayOfTables, newItem(pInteger, 1), newItem(pInteger, 2)))
p.openArrayOfTables(newKey("aaah", "table array"))
p.setKeyValuePair(newKey("a"), newItem(pFloat, 1.234))
p.openArrayOfTables(newKey("aaah", "table array"))
p.setKeyValuePair(newKey("b"), newItem(pFloat, 2.345))
p.setKeyValuePair(newKey("c"), newItem(pString, "bingo!"))
p.openArrayOfTables(newKey("aaah", "table array"))
return nil, p
}, "", `{"ding": 10, "dong": "not a song", "key1": {"key2 a": {"dah": false, "dooh": true}, "key2 b": {"dieh": 1.111, "duh": 1.18e-12}}}`)
},
"",
`{"aaah": {"table array": [{"a": 1.234}, {"b": 2.345, "c": "bingo!"}, {}]}, `+
`"ding": 10, "dong": "not a song", `+
`"key1": {"key2 a": {"dah": false, "dooh": true}, "key2 b": {"dieh": 1.111, "duh": 1.18e-12, "foo": {"bar": [1, 2]}}}}`)
}
func TestAST_EmptyKeyForCreatingTablePath_Panics(t *testing.T) {
defer func() {
r := recover()
if r.(string) != "makeTablePath(): empty key provided; a key must have at least one key part" {
t.Fatalf("Did not get the expected panic message")
}
}()
p := newParser()
p.openTable(newKey())
}
func TestAST_StoreValueInRootTable(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.setValue(newItem(pKey, "key1"), newItem(pString, "value1"))
return p.setValue(newItem(pKey, "key2"), newItem(pString, "value2")), p
}, "", `{"key1": "value1", "key2": "value2"}`)
p.setKeyValuePair(newKey("key1"), newItem(pString, "value1"))
return p.setKeyValuePair(newKey("key2"), newItem(pString, "value2")), p
},
"",
`{"key1": "value1", "key2": "value2"}`)
}
func TestAST_StoreValueWithMultipartKey_CreatesTableHierarchy(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
return p.setValue(newItem(pKey, "key1", "key2", "key3"), newItem(pString, "value")), p
}, "", `{"key1": {"key2": {"key3": "value"}}}`)
}
func TestAST_StoreValueWithMultipartKey_UnderSubtable_CreatesTableHierarchy(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newItem(pKey, "tablekey1", "tablekey2"))
return p.setValue(newItem(pKey, "valuekey1", "valuekey2", "valuekey3"), newItem(pString, "value")), p
}, "", `{"tablekey1": {"tablekey2": {"valuekey1": {"valuekey2": {"valuekey3": "value"}}}}}`)
return p.setKeyValuePair(newKey("key1", "key2", "key3"), newItem(pString, "value")), p
},
"",
`{"key1": {"key2": {"key3": "value"}}}`)
}
func TestAST_StoreDuplicateKeyInRootTable_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.setValue(newItem(pKey, "key"), newItem(pString, "value"))
return p.setValue(newItem(pKey, "key"), newItem(pInteger, 321)), p
}, `Cannot store value: string item already exists at key "key"`, "")
p.setKeyValuePair(newKey("key"), newItem(pString, "value"))
return p.setKeyValuePair(newKey("key"), newItem(pInteger, 321)), p
},
`invalid key/value pair: string item already exists at key [key]`,
`{"key": "value"}`)
}
func TestAST_StoreValueWithMultipartKey_UnderSubtable_CreatesTableHierarchy(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("tablekey1", "tablekey2"))
return p.setKeyValuePair(newKey("valuekey1", "valuekey2", "valuekey3"), newItem(pString, "value")), p
},
"",
`{"tablekey1": {"tablekey2": {"valuekey1": {"valuekey2": {"valuekey3": "value"}}}}}`)
}
func TestAST_StoreKeyPathWherePathContainsNonTableAlready_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("key1"))
p.setKeyValuePair(newKey("key2"), newItem(pInteger, 0))
return p.setKeyValuePair(newKey("key2", "key3"), newItem(pString, "value")), p
},
`invalid key/value pair: integer item already exists at key [key1->key2]`,
`{"key1": {"key2": 0}}`)
}
func TestAST_GivenExistingTableAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newItem(pKey, "key1", "key2"))
_, err := p.openTable(newItem(pKey, "key1", "key2"))
return err, p
}, `Cannot create table: table for key "key1"."key2" already exists`, "")
p.openTable(newKey("key1", "key2"))
return p.openTable(newKey("key1", "key2")), p
},
`invalid table: table item already exists at key [key1->key2]`,
`{"key1": {"key2": {}}}`)
}
func TestAST_GivenExistingItemAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.Root["key"] = newItem(pString, "value")
_, err := p.openTable(newItem(pKey, "key"))
return err, p
}, `Cannot create table: string item already exists at key "key"`, "")
p.setKeyValuePair(newKey("key"), newItem(pString, "value"))
return p.openTable(newKey("key")), p
},
`invalid table: string item already exists at key [key]`,
`{"key": "value"}`)
}
func TestAST_GivenExistingItemInKeyPath_CreatingTable_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.setKeyValuePair(newKey("key"), newItem(pString, "value"))
return p.openTable(newKey("key", "subkey")), p
},
`invalid table: string item already exists at key [key]`,
`{"key": "value"}`)
}
func TestAST_GivenExistingItemAtDeepKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("deep", "table"))
p.setKeyValuePair(newKey("key"), newItem(pInteger, 0))
return p.openTable(newKey("deep", "table", "key")), p
},
`invalid table: integer item already exists at key [deep->table->key]`,
`{"deep": {"table": {"key": 0}}}`)
}
func TestAST_GivenExistingItemAtDeepKeyFromSubTable_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("deep", "table"))
p.setKeyValuePair(newKey("key1", "key2"), newItem(pInteger, 0))
return p.setKeyValuePair(newKey("key1", "key2"), newItem(pBoolean, true)), p
},
// This test mainly tests the formatting of [deep->table->key1->key2], being a concatenation
// of the currently active table plus the multipart key for setKeyValuePair().
`invalid key/value pair: integer item already exists at key [deep->table->key1->key2]`,
`{"deep": {"table": {"key1": {"key2": 0}}}}`)
}
func TestAST_FormattingOfQuotedPathPartInError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.openTable(newKey("must be quoted"))
p.setKeyValuePair(newKey("this one too"), newItem(pInteger, 0))
return p.setKeyValuePair(newKey("this one too"), newItem(pInteger, 0)), p
},
`invalid key/value pair: integer item already exists at key ["must be quoted"->"this one too"]`,
`{"must be quoted": {"this one too": 0}}`)
}
func TestAST_GivenExistingItemAtKey_CreatingArrayOfTablesAtSameKey_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.Root["key"] = newItem(pString, "value")
return p.openArrayOfTables(newKey("key")), p
},
`invalid table array: string item already exists at key [key]`,
`{"key": "value"}`)
}
func TestAST_GivenExistingItemInKeyPath_CreatingArrayOfTables_ReturnsError(t *testing.T) {
testAST(t, func() (error, *parser) {
p := newParser()
p.Root["key"] = newItem(pString, "value")
return p.openArrayOfTables(newKey("key", "subkey")), p
},
`invalid table array: string item already exists at key [key]`,
`{"key": "value"}`)
}

4
comment.go
View File

@ -9,9 +9,7 @@ import (
var comment = c.Seq(a.Hash, c.ZeroOrMore(c.Not(a.EndOfLine)), m.Drop(a.EndOfLine))
func (t *parser) startComment(p *parse.API) {
if p.Accept(comment) {
t.addParsedItem(pComment, p.Result().String())
} else {
if !p.Accept(comment) {
p.Expected("comment")
}
}

23
comment_test.go
View File

@ -5,20 +5,17 @@ import (
)
func TestComment2(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected comment) at start of file`}},
{`#`, []string{`comment("#")`}},
{`# `, []string{`comment("# ")`}},
{`# with data`, []string{`comment("# with data")`}},
{"# ending in EOL & EOF\r\n", []string{`comment("# ending in EOL & EOF")`}},
{`# \xxx/ \u can't escape/`, []string{`comment("# \\xxx/ \\u can't escape/")`}},
{"# \tlexe\r accepts embedded ca\r\riage \returns\r\n", []string{
`comment("# \tlexe\r accepts embedded ca\r\riage \returns")`}},
{"# with data and newline\ncode continues here", []string{
`comment("# with data and newline")`,
`Error: unexpected input (expected end of file) at line 2, column 1`}},
for _, test := range []parseToASTTest{
{``, `{}`, `unexpected end of file (expected comment) at start of file`},
{`#`, `{}`, ``},
{`# `, `{}`, ``},
{`# with data`, `{}`, ``},
{"# ending in EOL & EOF\r\n", `{}`, ``},
{`# \xxx/ \u can't escape/`, `{}`, ``},
{"# \tlexe\r accepts embedded ca\r\riage \returns\r\n", `{}`, ``},
{"# with data and newline\ncode continues here", `{}`, `unexpected input (expected end of file) at line 2, column 1`},
} {
p := newParser()
testParseHandler(t, p, p.startComment, test)
testParseToAST(t, p, p.startComment, test)
}
}

27
helpers_test.go
View File

@ -74,3 +74,30 @@ func testAST(t *testing.T, code func() (error, *parser), expectedError string, e
t.Fatalf("Unexpected data after parsing:\nexpected: %s\nactual: %s\n", expectedData, p.Root.String())
}
}
type parseToASTTest struct {
input interface{}
expected string
expectedError string
}
func testParseToAST(t *testing.T, p *parser, handler parse.Handler, test parseToASTTest) {
var err error
defer func() {
recovered := recover()
if recovered != nil {
err = fmt.Errorf("Panic: %s", recovered.(string))
}
if err != nil && test.expectedError == "" {
t.Errorf("Unexpected error for input %q: %s", test.input, err)
} else if err != nil && test.expectedError != err.Error() {
t.Errorf("Unexpected error for input %q:\nexpected: %s\nactual: %s\n", test.input, test.expectedError, err.Error())
} else {
result := p.Root.String()
if test.expected != result {
t.Errorf("Unexpected result for input %q:\nexpected: %s\nactual: %s\n", test.input, test.expected, result)
}
}
}()
err = parse.New(handler)(test.input)
}

58
keyvaluepair.go
View File

@ -32,6 +32,9 @@ var (
// around dot-separated parts are ignored, however, best practice is to
// not use any extraneous blanks.
keySeparatorDot = c.Seq(dropBlanks, a.Dot, dropBlanks)
// Both [tables] and [[arrays of tables]] start with a square open bracket.
startOfTableOrArrayOfTables = a.SquareOpen
)
func (t *parser) startKeyValuePair(p *parse.API) {
@ -40,8 +43,15 @@ func (t *parser) startKeyValuePair(p *parse.API) {
switch {
case p.Peek(a.Hash):
p.Handle(t.startComment)
case p.Peek(startOfTableOrArrayOfTables):
p.Handle(t.startTable)
case p.Peek(startOfKey):
p.Handle(t.startKey, t.startAssignment, t.startValue)
key, ok := t.parseKey(p, []string{})
if ok && p.Handle(t.startAssignment) {
if value, ok := t.parseValue(p); ok {
t.setKeyValuePair(key, value)
}
}
default:
p.ExpectEndOfFile()
return
@ -61,58 +71,40 @@ func (t *parser) startKeyValuePair(p *parse.API) {
// is to use bare keys except when absolutely necessary.
// A bare key must be non-empty, but an empty quoted key is allowed (though
// discouraged).
func (t *parser) startKey(p *parse.API) {
var key string
func (t *parser) parseKey(p *parse.API, key []string) ([]string, bool) {
var keyPart string
var ok bool
switch {
case p.Accept(bareKey):
key, ok = p.Result().String(), true
keyPart, ok = p.Result().String(), true
case p.Peek(a.SingleQuote):
key, ok = t.parseLiteralString("key", p)
keyPart, ok = t.parseLiteralString("key", p)
case p.Peek(a.DoubleQuote):
key, ok = t.parseBasipString("key", p)
keyPart, ok = t.parseBasicString("key", p)
default:
p.Expected("a key name")
return
return nil, false
}
if ok {
t.addParsedItem(pKey, key)
p.Handle(t.endOfKeyOrDot)
if !ok {
return nil, false
}
key = append(key, keyPart)
return t.parseEndOfKeyOrDot(p, key)
}
// Dotted keys are a sequence of bare or quoted keys joined with a dot.
// This allows for grouping similar properties together.
// Whitespace around dot-separated parts is ignored, however, best
// practice is to not use any extraneous whitespace.
func (t *parser) endOfKeyOrDot(p *parse.API) {
func (t *parser) parseEndOfKeyOrDot(p *parse.API, key []string) ([]string, bool) {
if p.Accept(keySeparatorDot) {
p.Handle(t.startKey)
return
return t.parseKey(p, key)
}
// TODO not sure if we really need this index.
// Can't we alway simply use the full item list, given that we'll feed all
// results to the parser's table state?
// Do we even need to emit a key here? Shouldn't we just create the
// table structure in the parser, ready for followup calls to fill the data?
keyStart := len(t.Items) - 1
for keyStart > 0 && t.Items[keyStart-1].Type == pKey {
keyStart--
}
keyLen := len(t.Items) - keyStart
key := make([]interface{}, keyLen)
for i := 0; i < keyLen; i++ {
key[i] = t.Items[keyStart+i].Data[0].(string)
}
t.Items = t.Items[0:keyStart]
t.addParsedItem(pKey, key...)
return key, true
}
func (t *parser) startAssignment(p *parse.API) {
if p.Accept(keyAssignment) {
t.addParsedItem(pAssign)
} else {
if !p.Accept(keyAssignment) {
p.Expected("a value assignment")
}
}

226
keyvaluepair_test.go
View File

@ -3,151 +3,133 @@ package toml
import "testing"
func TestKey(t *testing.T) {
for _, test := range []parseTest{
{"", []string{`Error: unexpected end of file (expected a key name) at start of file`}},
for _, test := range []parseToASTTest{
// Bare key tests
{"barekey", []string{`key("barekey")`}},
{"1234567", []string{`key("1234567")`}},
{"mix-12_34", []string{`key("mix-12_34")`}},
{"-hey_good_Lookin123-", []string{`key("-hey_good_Lookin123-")`}},
{"wrong!", []string{`key("wrong")`, `Error: unexpected input (expected end of file) at line 1, column 6`}},
{"key1.", []string{`key("key1")`, `Error: unexpected end of file (expected a key name) at line 1, column 6`}},
{"key1.key2", []string{`key("key1", "key2")`}},
{"key . with . spaces", []string{`key("key", "with", "spaces")`}},
{"key \t . \twithtabs\t . \tandspaces", []string{`key("key", "withtabs", "andspaces")`}},
{"barekey=0", `{"barekey": 0}`, ``},
{"1234567=0", `{"1234567": 0}`, ``},
{"mix-12_34=0", `{"mix-12_34": 0}`, ``},
{"-hey_good_Lookin123-=0", `{"-hey_good_Lookin123-": 0}`, ``},
{"wrong!=0", `{}`, `unexpected input (expected a value assignment) at line 1, column 6`},
{"key1.=0", `{}`, `unexpected input (expected a key name) at line 1, column 6`},
{"key1.key2=0", `{"key1": {"key2": 0}}`, ``},
{"key . with . spaces=0", `{"key": {"with": {"spaces": 0}}}`, ``},
{"key \t . \twithtabs\t . \tandspaces=0", `{"key": {"withtabs": {"andspaces": 0}}}`, ``},
// Single quoted key tests
{"''", []string{`key("")`}},
{"'single quoted'", []string{`key("single quoted")`}},
{`'escape\s are literal'`, []string{`key("escape\\s are literal")`}},
{`'"using inner quotes"'`, []string{`key("\"using inner quotes\"")`}},
{"''=0", `{"": 0}`, ``},
{"'single quoted'=0", `{"single quoted": 0}`, ``},
{`'escape\s are literal'=0`, `{"escape\\s are literal": 0}`, ``},
{`'"using inner quotes"'=0`, `{"\"using inner quotes\"": 0}`, ``},
// Double quoted key tests
{`""`, []string{`key("")`}},
{`"double quoted"`, []string{`key("double quoted")`}},
{`"escapes are in\terpreted"`, []string{`key("escapes are in\terpreted")`}},
{`"using 'inner' \"quotes\""`, []string{`key("using 'inner' \"quotes\"")`}},
{`""=0`, `{"": 0}`, ``},
{`"double quoted"=0`, `{"double quoted": 0}`, ``},
{`"escapes are in\terpreted"=0`, `{"escapes are in\terpreted": 0}`, ``},
{`"using 'inner' \"quotes\""=0`, `{"using 'inner' \"quotes\"": 0}`, ``},
// Mixed key types
{`this.'i\s'."madness\t".''`, []string{`key("this", "i\\s", "madness\t", "")`}},
{`this.'i\s'."madness\t".''=0`, `{"this": {"i\\s": {"madness\t": {"": 0}}}}`, ``},
} {
p := newParser()
testParseHandler(t, p, p.startKey, test)
}
}
func TestAssignment(t *testing.T) {
for _, test := range []parseTest{
{"", []string{`Error: unexpected end of file (expected a value assignment) at start of file`}},
{"=", []string{`=`}},
{" \t = \t ", []string{`=`}},
{" \n = \n ", []string{`Error: unexpected input (expected a value assignment) at start of file`}},
} {
p := newParser()
testParseHandler(t, p, p.startAssignment, test)
testParseToAST(t, p, p.startKeyValuePair, test)
}
}
func TestKeyValuePair(t *testing.T) {
for _, test := range []parseTest{
{"", []string{}},
{" ", []string{}},
{" \t ", []string{}},
{" key ", []string{`key("key")`, `Error: unexpected input (expected a value assignment) at line 1, column 5`}},
{" key \t=", []string{`key("key")`, `=`, `Error: unexpected end of file (expected a value) at line 1, column 8`}},
{"key = # INVALID", []string{`key("key")`, `=`, `Error: unexpected input (expected a value) at line 1, column 7`}},
{" key \t =\t \"The Value\" \r\n", []string{`key("key")`, `=`, `"The Value"`}},
{`3.14159 = "pi"`, []string{`key("3", "14159")`, `=`, `"pi"`}},
{`"ʎǝʞ" = "value"`, []string{`key("ʎǝʞ")`, `=`, `"value"`}},
{`key = "value" # This is a comment at the end of a line`, []string{`key("key")`, `=`, `"value"`, `comment("# This is a comment at the end of a line")`}},
{`another = "# This is not a comment"`, []string{`key("another")`, `=`, `"# This is not a comment"`}},
{"key1=\"value1\"key2=\"value2\"\r\nkey3a.key3b=\"value3\"", []string{
`key("key1")`, `=`, `"value1"`,
`key("key2")`, `=`, `"value2"`,
`key("key3a", "key3b")`, `=`, `"value3"`}},
{"with=\"comments\"# boring \nanother.cool =\"one\" \t # to the end\r\n", []string{
`key("with")`, `=`, `"comments"`, `comment("# boring ")`,
`key("another", "cool")`, `=`, `"one"`, `comment("# to the end")`}},
for _, test := range []parseToASTTest{
{``, `{}`, ``},
{` `, `{}`, ``},
{" \t ", `{}`, ``},
{" key ", `{}`, `unexpected input (expected a value assignment) at line 1, column 5`},
{" key \t=", `{}`, `unexpected end of file (expected a value) at line 1, column 8`},
{"key = # INVALID", `{}`, `unexpected input (expected a value) at line 1, column 7`},
{" key \t =\t \"The Value\" \r\n", `{"key": "The Value"}`, ``},
{`3.14159 = "pi"`, `{"3": {"14159": "pi"}}`, ``},
{`"ʎǝʞ" = "value"`, `{"ʎǝʞ": "value"}`, ``},
{`key = "value" # This is a comment at the end of a line`, `{"key": "value"}`, ``},
{`another = "# This is not a comment"`, `{"another": "# This is not a comment"}`, ``},
{"key1=\"value1\"key2=\"value2\"\r\nkey3a.key3b=\"value3\"", `{"key1": "value1", "key2": "value2", "key3a": {"key3b": "value3"}}`, ``},
{"with=\"comments\"# boring \nanother.cool =\"one\" \t # to the end\r\n", `{"another": {"cool": "one"}, "with": "comments"}`, ``},
} {
p := newParser()
testParseHandler(t, p, p.startKeyValuePair, test)
testParseToAST(t, p, p.startKeyValuePair, test)
}
}
func TestKeyValuePair_ForAllTypes(t *testing.T) {
for _, test := range []parseTest{
{"string='literal'", []string{`key("string")`, `=`, `"literal"`}},
{"string='''literal\nmulti-line'''", []string{`key("string")`, `=`, `"literal\nmulti-line"`}},
{`string="basic"`, []string{`key("string")`, `=`, `"basic"`}},
{"string=\"\"\"basic\nmulti-line\"\"\"", []string{`key("string")`, `=`, `"basic\nmulti-line"`}},
{"integer=1_234_567", []string{`key("integer")`, `=`, `1234567`}},
{"integer=42", []string{`key("integer")`, `=`, `42`}},
{"integer=0x42", []string{`key("integer")`, `=`, `66`}},
{"integer=0o42", []string{`key("integer")`, `=`, `34`}},
{"integer=0b101010", []string{`key("integer")`, `=`, `42`}},
{"float=42.37", []string{`key("float")`, `=`, `42.37`}},
{"float=42e+37", []string{`key("float")`, `=`, `4.2e+38`}},
{"float=42.37e-11", []string{`key("float")`, `=`, `4.237e-10`}},
{"boolean=true", []string{`key("boolean")`, `=`, `true`}},
{"boolean=false", []string{`key("boolean")`, `=`, `false`}},
{"date=2019-01-01", []string{`key("date")`, `=`, `2019-01-01`}},
{"time=15:03:11", []string{`key("time")`, `=`, `15:03:11`}},
{"datetime=2021-02-01 15:03:11.123", []string{`key("datetime")`, `=`, `2021-02-01 15:03:11.123`}},
{"offset_datetime=1111-11-11 11:11:11.111111111+11:11", []string{`key("offset_datetime")`, `=`, `1111-11-11T11:11:11.111111111+11:11`}},
{"static_array=['a', 'static', 'array']", []string{`key("static_array")`, `=`, `["a", "static", "array"]`}},
for _, test := range []parseToASTTest{
{"string='literal'", `{"string": "literal"}`, ``},
{"string='''literal\nmulti-line'''", `{"string": "literal\nmulti-line"}`, ``},
{`string="basic"`, `{"string": "basic"}`, ``},
{"string=\"\"\"basic\nmulti-line\"\"\"", `{"string": "basic\nmulti-line"}`, ``},
{"integer=1_234_567", `{"integer": 1234567}`, ``},
{"integer=42", `{"integer": 42}`, ``},
{"integer=0x42", `{"integer": 66}`, ``},
{"integer=0o42", `{"integer": 34}`, ``},
{"integer=0b101010", `{"integer": 42}`, ``},
{"float=42.37", `{"float": 42.37}`, ``},
{"float=42e+37", `{"float": 4.2e+38}`, ``},
{"float=42.37e-11", `{"float": 4.237e-10}`, ``},
{"boolean=true", `{"boolean": true}`, ``},
{"boolean=false", `{"boolean": false}`, ``},
{"date=2019-01-01", `{"date": 2019-01-01}`, ``},
{"time=15:03:11", `{"time": 15:03:11}`, ``},
{"datetime=2021-02-01 15:03:11.123", `{"datetime": 2021-02-01 15:03:11.123}`, ``},
{"offset_datetime=1111-11-11 11:11:11.111111111+11:11", `{"offset_datetime": 1111-11-11T11:11:11.111111111+11:11}`, ``},
{"static_array=['a', 'static', 'array']", `{"static_array": ["a", "static", "array"]}`, ``},
} {
p := newParser()
testParseHandler(t, p, p.startKeyValuePair, test)
testParseToAST(t, p, p.startKeyValuePair, test)
}
}
func TestKeyValuePair_ExamplesFromSpecification(t *testing.T) {
for _, test := range []parseTest{
{"int1 = +99", []string{`key("int1")`, `=`, `99`}},
{"int2 = 42", []string{`key("int2")`, `=`, `42`}},
{"int3 = 0", []string{`key("int3")`, `=`, `0`}},
{"int4 = -17", []string{`key("int4")`, `=`, `-17`}},
{"int5 = 1_000", []string{`key("int5")`, `=`, `1000`}},
{"int6 = 5_349_221", []string{`key("int6")`, `=`, `5349221`}},
{"int7 = 1_2_3_4_5 # VALID but discouraged", []string{`key("int7")`, `=`, `12345`, `comment("# VALID but discouraged")`}},
{"hex1 = 0xDEADBEEF", []string{`key("hex1")`, `=`, `3735928559`}},
{"hex2 = 0xdeadbeef", []string{`key("hex2")`, `=`, `3735928559`}},
{"hex3 = 0xdead_beef", []string{`key("hex3")`, `=`, `3735928559`}},
{"oct1 = 0o01234567", []string{`key("oct1")`, `=`, `342391`}},
{"oct2 = 0o755", []string{`key("oct2")`, `=`, `493`}},
{"bin1 = 0b11010110", []string{`key("bin1")`, `=`, `214`}},
{"flt1 = +1.0", []string{`key("flt1")`, `=`, `1`}},
{"flt2 = 3.1415", []string{`key("flt2")`, `=`, `3.1415`}},
{"flt3 = -0.01", []string{`key("flt3")`, `=`, `-0.01`}},
{"flt4 = 5e+22", []string{`key("flt4")`, `=`, `5e+22`}},
{"flt5 = 1e6", []string{`key("flt5")`, `=`, `1e+06`}},
{"flt6 = -2E-2", []string{`key("flt6")`, `=`, `-0.02`}},
{"flt7 = 6.626e-34", []string{`key("flt7")`, `=`, `6.626e-34`}},
{"flt8 = 224_617.445_991_228", []string{`key("flt8")`, `=`, `224617.445991228`}},
{"sf1 = inf # positive infinity", []string{`key("sf1")`, `=`, `+Inf`, `comment("# positive infinity")`}},
{"sf2 = +inf # positive infinity", []string{`key("sf2")`, `=`, `+Inf`, `comment("# positive infinity")`}},
{"sf3 = -inf # negative infinity", []string{`key("sf3")`, `=`, `-Inf`, `comment("# negative infinity")`}},
{"sf4 = nan # actual sNaN/qNaN encoding is implementation-specific", []string{`key("sf4")`, `=`, `NaN`, `comment("# actual sNaN/qNaN encoding is implementation-specific")`}},
{"sf5 = +nan # same as `nan`", []string{`key("sf5")`, `=`, `NaN`, "comment(\"# same as `nan`\")"}},
{"sf6 = -nan # valid, actual encoding is implementation-specific", []string{`key("sf6")`, `=`, `NaN`, `comment("# valid, actual encoding is implementation-specific")`}},
{"bool1 = true", []string{`key("bool1")`, `=`, `true`}},
{"bool2 = false", []string{`key("bool2")`, `=`, `false`}},
{"odt1 = 1979-05-27T07:32:00Z", []string{`key("odt1")`, `=`, `1979-05-27T07:32:00Z`}},
{"odt2 = 1979-05-27T00:32:00-07:00", []string{`key("odt2")`, `=`, `1979-05-27T00:32:00-07:00`}},
{"odt3 = 1979-05-27T00:32:00.999999-07:00", []string{`key("odt3")`, `=`, `1979-05-27T00:32:00.999999-07:00`}},
{"odt4 = 1979-05-27 07:32:00Z", []string{`key("odt4")`, `=`, `1979-05-27T07:32:00Z`}},
{"ldt1 = 1979-05-27T07:32:00", []string{`key("ldt1")`, `=`, `1979-05-27 07:32:00`}},
{"ldt2 = 1979-05-27T00:32:00.999999", []string{`key("ldt2")`, `=`, `1979-05-27 00:32:00.999999`}},
{"ld1 = 1979-05-27", []string{`key("ld1")`, `=`, `1979-05-27`}},
{"lt1 = 07:32:00", []string{`key("lt1")`, `=`, `07:32:00`}},
{"lt2 = 00:32:00.999999", []string{`key("lt2")`, `=`, `00:32:00.999999`}},
{"arr1 = [ 1, 2, 3 ]", []string{`key("arr1")`, `=`, `[1, 2, 3]`}},
{`arr2 = [ "red", "yellow", "green" ]`, []string{`key("arr2")`, `=`, `["red", "yellow", "green"]`}},
{`arr3 = [ [ 1, 2 ], [3, 4, 5] ]`, []string{`key("arr3")`, `=`, `[[1, 2], [3, 4, 5]]`}},
{`arr4 = [ "all", 'strings', """are the same""", '''type''']`, []string{`key("arr4")`, `=`, `["all", "strings", "are the same", "type"]`}},
{`arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]`, []string{`key("arr5")`, `=`, `[[1, 2], ["a", "b", "c"]]`}},
{`arr6 = [ 1, 2.0 ] # INVALID`, []string{`key("arr6")`, `=`, `Error: type mismatch in array of integers: found an item of type float at line 1, column 16`}},
{"arr7 = [\n 1, 2, 3\n]", []string{`key("arr7")`, `=`, `[1, 2, 3]`}},
{"arr8 = [\n 1,\n 2, # this is ok\n]", []string{`key("arr8")`, `=`, `[1, 2]`}},
for _, test := range []parseToASTTest{
{"int1 = +99", `{"int1": 99}`, ``},
{"int2 = 42", `{"int2": 42}`, ``},
{"int3 = 0", `{"int3": 0}`, ``},
{"int4 = -17", `{"int4": -17}`, ``},
{"int5 = 1_000", `{"int5": 1000}`, ``},
{"int6 = 5_349_221", `{"int6": 5349221}`, ``},
{"int7 = 1_2_3_4_5 # VALID but discouraged", `{"int7": 12345}`, ``},
{"hex1 = 0xDEADBEEF", `{"hex1": 3735928559}`, ``},
{"hex2 = 0xdeadbeef", `{"hex2": 3735928559}`, ``},
{"hex3 = 0xdead_beef", `{"hex3": 3735928559}`, ``},
{"oct1 = 0o01234567", `{"oct1": 342391}`, ``},
{"oct2 = 0o755", `{"oct2": 493}`, ``},
{"bin1 = 0b11010110", `{"bin1": 214}`, ``},
{"flt1 = +1.0", `{"flt1": 1}`, ``},
{"flt2 = 3.1415", `{"flt2": 3.1415}`, ``},
{"flt3 = -0.01", `{"flt3": -0.01}`, ``},
{"flt4 = 5e+22", `{"flt4": 5e+22}`, ``},
{"flt5 = 1e6", `{"flt5": 1e+06}`, ``},
{"flt6 = -2E-2", `{"flt6": -0.02}`, ``},
{"flt7 = 6.626e-34", `{"flt7": 6.626e-34}`, ``},
{"flt8 = 224_617.445_991_228", `{"flt8": 224617.445991228}`, ``},
{"sf1 = inf # positive infinity", `{"sf1": +Inf}`, ``},
{"sf2 = +inf # positive infinity", `{"sf2": +Inf}`, ``},
{"sf3 = -inf # negative infinity", `{"sf3": -Inf}`, ``},
{"sf4 = nan # actual sNaN/qNaN encoding is implementation-specific", `{"sf4": NaN}`, ``},
{"sf5 = +nan # same as `nan`", `{"sf5": NaN}`, ``},
{"sf6 = -nan # valid, actual encoding is implementation-specific", `{"sf6": NaN}`, ``},
{"bool1 = true", `{"bool1": true}`, ``},
{"bool2 = false", `{"bool2": false}`, ``},
{"odt1 = 1979-05-27T07:32:00Z", `{"odt1": 1979-05-27T07:32:00Z}`, ``},
{"odt2 = 1979-05-27T00:32:00-07:00", `{"odt2": 1979-05-27T00:32:00-07:00}`, ``},
{"odt3 = 1979-05-27T00:32:00.999999-07:00", `{"odt3": 1979-05-27T00:32:00.999999-07:00}`, ``},
{"odt4 = 1979-05-27 07:32:00Z", `{"odt4": 1979-05-27T07:32:00Z}`, ``},
{"ldt1 = 1979-05-27T07:32:00", `{"ldt1": 1979-05-27 07:32:00}`, ``},
{"ldt2 = 1979-05-27T00:32:00.999999", `{"ldt2": 1979-05-27 00:32:00.999999}`, ``},
{"ld1 = 1979-05-27", `{"ld1": 1979-05-27}`, ``},
{"lt1 = 07:32:00", `{"lt1": 07:32:00}`, ``},
{"lt2 = 00:32:00.999999", `{"lt2": 00:32:00.999999}`, ``},
{"arr1 = [ 1, 2, 3 ]", `{"arr1": [1, 2, 3]}`, ``},
{`arr2 = [ "red", "yellow", "green" ]`, `{"arr2": ["red", "yellow", "green"]}`, ``},
{`arr3 = [ [ 1, 2 ], [3, 4, 5] ]`, `{"arr3": [[1, 2], [3, 4, 5]]}`, ``},
{`arr4 = [ "all", 'strings', """are the same""", '''type''']`, `{"arr4": ["all", "strings", "are the same", "type"]}`, ``},
{`arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]`, `{"arr5": [[1, 2], ["a", "b", "c"]]}`, ``},
{`arr6 = [ 1, 2.0 ] # INVALID`, `{}`, `type mismatch in array of integers: found an item of type float at line 1, column 16`},
{"arr7 = [\n 1, 2, 3\n]", `{"arr7": [1, 2, 3]}`, ``},
{"arr8 = [\n 1,\n 2, # this is ok\n]", `{"arr8": [1, 2]}`, ``},
} {
p := newParser()
testParseHandler(t, p, p.startKeyValuePair, test)
testParseToAST(t, p, p.startKeyValuePair, test)
}
}

40
value.go
View File

@ -4,27 +4,33 @@ import (
"git.makaay.nl/mauricem/go-parsekit/parse"
)
var (
detectString = a.SingleQuote.Or(a.DoubleQuote)
detectBoolean = a.Str("true").Or(a.Str("false"))
detectNumberSpecials = c.Any(a.Plus, a.Minus, a.Str("inf"), a.Str("nan"))
detectDateTime = a.Digits.Then(a.Minus.Or(a.Colon))
detectNumber = a.Digit
detectArray = a.SquareOpen
)
// Values must be of the following types: String, Integer, Float, Boolean,
// Datetime, Array, or Inline Table. Unspecified values are invalid.
func (t *parser) startValue(p *parse.API) {
func (t *parser) parseValue(p *parse.API) (*item, bool) {
switch {
case p.Peek(c.Any(a.SingleQuote, a.DoubleQuote)):
p.Handle(t.startString)
case p.Peek(a.Runes('t', 'f')):
p.Handle(t.startBoolean)
case p.Peek(a.Plus.Or(a.Minus)):
p.Handle(t.startNumber)
case p.Peek(a.Runes('i', 'n')):
p.Handle(t.startNumber)
case p.Peek(a.Digit):
if p.Peek(a.Digits.Then(a.Minus.Or(a.Colon))) {
p.Handle(t.startDateTime)
} else {
p.Handle(t.startNumber)
}
case p.Peek(a.SquareOpen):
p.Handle(t.startArray)
case p.Peek(detectString):
return t.parseString(p)
case p.Peek(detectBoolean):
return t.parseBoolean(p)
case p.Peek(detectNumberSpecials):
return t.parseNumber(p)
case p.Peek(detectDateTime):
return t.parseDateTime(p)
case p.Peek(detectNumber):
return t.parseNumber(p)
case p.Peek(detectArray):
return t.parseArray(p)
default:
p.Expected("a value")
return nil, false
}
}

36
value_array.go
View File

@ -36,51 +36,45 @@ var (
arrayClose = c.Seq(c.Optional(arraySpace.Then(a.Comma)), arraySpace, a.SquareClose)
)
func (t *parser) startArray(p *parse.API) {
func (t *parser) parseArray(p *parse.API) (*item, bool) {
// Check for the start of the array.
if !p.Accept(arrayOpen) {
p.Expected("an array")
return
return nil, false
}
items := []item{}
// Check for an empty array.
if p.Accept(arrayClose) {
t.addParsedItem(pStaticArray, items)
return
return newItem(pArray), true
}
// Not an empty array, parse the items.
// Not an empty array, parse the array items.
items := []interface{}{}
for {
// Check for a valid item.
if !p.Handle(t.startValue) {
return
// Check for a value item.
value, ok := t.parseValue(p)
if !ok {
return nil, false
}
// Pop the item from the value parsing and append it to the array items.
parseItems, item := t.Items[0:len(t.Items)-1], t.Items[len(t.Items)-1]
t.Items = parseItems
// Data types may not be mixed (different ways to define strings should be
// considered the same type, and so should arrays with different element types).
if len(items) > 0 && item.Type != items[0].Type {
p.Error("type mismatch in array of %ss: found an item of type %s", items[0].Type, item.Type)
return
if len(items) > 0 && value.Type != items[0].(*item).Type {
p.Error("type mismatch in array of %ss: found an item of type %s", items[0].(*item).Type, value.Type)
return nil, false
}
items = append(items, item)
items = append(items, value)
// Check for the end of the array.
if p.Accept(arrayClose) {
t.addParsedItem(pStaticArray, items)
return
return newItem(pArray, items...), true
}
// Not the end of the array? Then we should find an array separator.
if !p.Accept(arraySeparator) {
p.Expected("an array separator")
return
return nil, false
}
}
}

65
value_array_test.go
View File

@ -2,39 +2,46 @@ package toml
import (
"testing"
"git.makaay.nl/mauricem/go-parsekit/parse"
)
func TestArrayStart(t *testing.T) {
parser := newParser()
wrapper := func(p *parse.API) { parser.parseArray(p) }
testParseToAST(t, parser, wrapper, parseToASTTest{"INVALID", "{}", "unexpected input (expected an array) at start of file"})
}
func TestArray(t *testing.T) {
for _, test := range []parseTest{
{"", []string{`Error: unexpected end of file (expected an array) at start of file`}},
{"[ape", []string{`Error: unexpected input (expected a value) at line 1, column 2`}},
{"[1", []string{`Error: unexpected end of file (expected an array separator) at line 1, column 3`}},
{"[]", []string{`[]`}},
{"[\n]", []string{`[]`}},
{"[,]", []string{`[]`}},
{"[ , ]", []string{`[]`}},
{"[ \n , \r\n ]", []string{`[]`}},
{"[ \t , \t ]", []string{`[]`}},
{"[\r\n\r\n , \r\n \t\n]", []string{`[]`}},
{"[\n#comment on its own line\n]", []string{`[]`}},
{"[#comment before close\n]", []string{`[]`}},
{"[,#comment after separator\n]", []string{`[]`}},
{"[#comment before separator\n,]", []string{`[]`}},
{"[#comment before value\n1]", []string{`[1]`}},
{"[1#comment after value\n]", []string{`[1]`}},
{"[1\n#comment on its own line after value\n]", []string{`[1]`}},
{"[1#comment 1\n#comment 2\n#comment 3\n , \n2]", []string{`[1, 2]`}},
{"[1]", []string{`[1]`}},
{"[1,0x2, 0b11, 0o4]", []string{`[1, 2, 3, 4]`}},
{"[0.1,0.2,3e-1,0.04e+1, nan, inf]", []string{`[0.1, 0.2, 0.3, 0.4, NaN, +Inf]`}},
{"[\n\t 'a', \"b\", '''c''', \"\"\"d\ne\"\"\",\n \t]", []string{`["a", "b", "c", "d\ne"]`}},
{`[1, 2, 3, "four"]`, []string{`Error: type mismatch in array of integers: found an item of type string at line 1, column 17`}},
{`[[1],['a']]`, []string{`[[1], ["a"]]`}},
{`[[[],[]],[]]`, []string{`[[[], []], []]`}},
{"[\r\n\r\n \t\n [\r\n\r\n\t [],[\t]],\t\n[]\t \t \n ]", []string{`[[[], []], []]`}},
{`[[1],'a']`, []string{`Error: type mismatch in array of static arrays: found an item of type string at line 1, column 9`}},
for _, test := range []parseToASTTest{
{"x=[ape", `{}`, `unexpected input (expected a value) at line 1, column 4`},
{"x=[1", `{}`, `unexpected end of file (expected an array separator) at line 1, column 5`},
{"x=[]", `{"x": []}`, ``},
{"x=[\n]", `{"x": []}`, ``},
{"x=[,]", `{"x": []}`, ``},
{"x=[ , ]", `{"x": []}`, ``},
{"x=[ \n , \r\n ]", `{"x": []}`, ``},
{"x=[ \t , \t ]", `{"x": []}`, ``},
{"x=[\r\n\r\n , \r\n \t\n]", `{"x": []}`, ``},
{"x=[\n#comment on its own line\n]", `{"x": []}`, ``},
{"x=[#comment before close\n]", `{"x": []}`, ``},
{"x=[,#comment after separator\n]", `{"x": []}`, ``},
{"x=[#comment before separator\n,]", `{"x": []}`, ``},
{"x=[#comment before value\n1]", `{"x": [1]}`, ``},
{"x=[1#comment after value\n]", `{"x": [1]}`, ``},
{"x=[1\n#comment on its own line after value\n]", `{"x": [1]}`, ``},
{"x=[1#comment 1\n#comment 2\n#comment 3\n , \n2]", `{"x": [1, 2]}`, ``},
{"x=[1]", `{"x": [1]}`, ``},
{"x=[1,0x2, 0b11, 0o4]", `{"x": [1, 2, 3, 4]}`, ``},
{"x=[0.1,0.2,3e-1,0.04e+1, nan, inf]", `{"x": [0.1, 0.2, 0.3, 0.4, NaN, +Inf]}`, ``},
{"x=[\n\t 'a', \"b\", '''c''', \"\"\"d\ne\"\"\",\n \t]", `{"x": ["a", "b", "c", "d\ne"]}`, ``},
{`x=[1, 2, 3, "four"]`, `{}`, `type mismatch in array of integers: found an item of type string at line 1, column 19`},
{`x=[[1],['a']]`, `{"x": [[1], ["a"]]}`, ``},
{`x=[[[],[]],[]]`, `{"x": [[[], []], []]}`, ``},
{"x=[\r\n\r\n \t\n [\r\n\r\n\t [],[\t]],\t\n[]\t \t \n ]", `{"x": [[[], []], []]}`, ``},
{`x=[[1],'a']`, `{}`, `type mismatch in array of static arrays: found an item of type string at line 1, column 11`},
} {
p := newParser()
testParseHandler(t, p, p.startArray, test)
testParseToAST(t, p, p.startKeyValuePair, test)
}
}

10
value_boolean.go
View File

@ -4,14 +4,18 @@ import (
"git.makaay.nl/mauricem/go-parsekit/parse"
)
var falseItem = newItem(pBoolean, false)
var trueItem = newItem(pBoolean, true)
// Booleans are just the tokens you're used to. Always lowercase.
func (t *parser) startBoolean(p *parse.API) {
func (t *parser) parseBoolean(p *parse.API) (*item, bool) {
switch {
case p.Accept(a.Str("true")):
t.addParsedItem(pBoolean, true)
return trueItem, true
case p.Accept(a.Str("false")):
t.addParsedItem(pBoolean, false)
return falseItem, true
default:
p.Expected("true or false")
return nil, false
}
}

25
value_boolean_test.go
View File

@ -2,19 +2,26 @@ package toml
import (
"testing"
"git.makaay.nl/mauricem/go-parsekit/parse"
)
func TestBooleanStart(t *testing.T) {
parser := newParser()
wrapper := func(p *parse.API) { parser.parseBoolean(p) }
testParseToAST(t, parser, wrapper, parseToASTTest{"INVALID", "{}", "unexpected input (expected true or false) at start of file"})
}
func TestBoolean(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected true or false) at start of file`}},
{`true`, []string{`true`}},
{`false`, []string{`false`}},
{`yes`, []string{`Error: unexpected input (expected true or false) at start of file`}},
{`no`, []string{`Error: unexpected input (expected true or false) at start of file`}},
{`1`, []string{`Error: unexpected input (expected true or false) at start of file`}},
{`0`, []string{`Error: unexpected input (expected true or false) at start of file`}},
for _, test := range []parseToASTTest{
{`x=true`, `{"x": true}`, ``},
{`x=false`, `{"x": false}`, ``},
{`x=yes`, `{}`, `unexpected input (expected a value) at line 1, column 3`},
{`x=no`, `{}`, `unexpected input (expected a value) at line 1, column 3`},
{`x=1`, `{"x": 1}`, ``},
{`x=0`, `{"x": 0}`, ``},
} {
p := newParser()
testParseHandler(t, p, p.startBoolean, test)
testParseToAST(t, p, p.startKeyValuePair, test)
}
}

9
value_datetime.go
View File

@ -73,18 +73,19 @@ var (
datetime = c.Any(offsetDateTime, localDateTime, localDate, localTime)
)
func (t *parser) startDateTime(p *parse.API) {
func (t *parser) parseDateTime(p *parse.API) (*item, bool) {
if !p.Accept(datetime) {
p.Expected("a date and/or time")
return
return nil, false
}
tokens := p.Result().Tokens()
valueType := getDateTimeValueType(&tokens)
input, value, err := getDateTimeValue(&tokens)
if err == nil {
t.addParsedItem(valueType, value)
return newItem(valueType, value), true
} else {
p.Error("Cannot parse value 0%s: %s", input, err)
p.Error("invalid date/time value %s: %s", input, err)
return nil, false
}
}

49
value_datetime_test.go
View File

@ -2,32 +2,39 @@ package toml
import (
"testing"
"git.makaay.nl/mauricem/go-parsekit/parse"
)
func TestDateTimeStart(t *testing.T) {
parser := newParser()
wrapper := func(p *parse.API) { parser.parseDateTime(p) }
testParseToAST(t, parser, wrapper, parseToASTTest{"INVALID", "{}", "unexpected input (expected a date and/or time) at start of file"})
}
func TestDateTime(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected a date and/or time) at start of file`}},
{`1979-05-27`, []string{`1979-05-27`}},
{`00:00:00`, []string{`00:00:00`}},
{`23:59:59`, []string{`23:59:59`}},
{`12:10:08.12121212121212`, []string{`12:10:08.121212121`}},
{`1979-05-28T01:01:01`, []string{`1979-05-28 01:01:01`}},
{`1979-05-28 01:01:01`, []string{`1979-05-28 01:01:01`}},
{`1979-05-27T07:32:00Z`, []string{`1979-05-27T07:32:00Z`}},
{`1979-05-27 07:33:00Z`, []string{`1979-05-27T07:33:00Z`}},
{`1979-05-27 07:34:00+07:00`, []string{`1979-05-27T07:34:00+07:00`}},
{`1979-05-27 07:34:00-07:00`, []string{`1979-05-27T07:34:00-07:00`}},
{`1985-03-31 23:59:59+00:00`, []string{`1985-03-31T23:59:59Z`}},
{`2000-09-10 00:00:00.000000000+00:00`, []string{`2000-09-10T00:00:00Z`}},
{`2003-11-01 01:02:03.999999999999+10:00`, []string{`2003-11-01T01:02:03.999999999+10:00`}},
{`2007-12-25 04:00:04.1111-10:30`, []string{`2007-12-25T04:00:04.1111-10:30`}},
{`2021-02-01 10:10:10.101010203040Z`, []string{`2021-02-01T10:10:10.101010203Z`}},
for _, test := range []parseToASTTest{
{`x=1979-05-27`, `{"x": 1979-05-27}`, ``},
{`x=00:00:00`, `{"x": 00:00:00}`, ``},
{`x=23:59:59`, `{"x": 23:59:59}`, ``},
{`x=12:10:08.12121212121212`, `{"x": 12:10:08.121212121}`, ``},
{`x=1979-05-28T01:01:01`, `{"x": 1979-05-28 01:01:01}`, ``},
{`x=1979-05-28 01:01:01`, `{"x": 1979-05-28 01:01:01}`, ``},
{`x=1979-05-27T07:32:00Z`, `{"x": 1979-05-27T07:32:00Z}`, ``},
{`x=1979-05-27 07:33:00Z`, `{"x": 1979-05-27T07:33:00Z}`, ``},
{`x=1979-05-27 07:34:00+07:00`, `{"x": 1979-05-27T07:34:00+07:00}`, ``},
{`x=1979-05-27 07:34:00-07:00`, `{"x": 1979-05-27T07:34:00-07:00}`, ``},
{`x=1985-03-31 23:59:59+00:00`, `{"x": 1985-03-31T23:59:59Z}`, ``},
{`x=2000-09-10 00:00:00.000000000+00:00`, `{"x": 2000-09-10T00:00:00Z}`, ``},
{`x=2003-11-01 01:02:03.999999999999+10:00`, `{"x": 2003-11-01T01:02:03.999999999+10:00}`, ``},
{`x=2007-12-25 04:00:04.1111-10:30`, `{"x": 2007-12-25T04:00:04.1111-10:30}`, ``},
{`x=2021-02-01 10:10:10.101010203040Z`, `{"x": 2021-02-01T10:10:10.101010203Z}`, ``},
// TODO ugly column, should be at start or at the actual wrong part
{`2000-13-01`, []string{`Error: Cannot parse value 02000-13-01: parsing time "2000-13-01": month out of range at line 1, column 11`}},
{`2000-02-31`, []string{`Error: Cannot parse value 02000-02-31: parsing time "2000-02-31": day out of range at line 1, column 11`}},
{`25:01:01`, []string{`Error: Cannot parse value 025:01:01: parsing time "25:01:01": hour out of range at line 1, column 9`}},
{`x=2000-13-01`, `{}`, `invalid date/time value 2000-13-01: parsing time "2000-13-01": month out of range at line 1, column 13`},
{`x=2000-02-31`, `{}`, `invalid date/time value 2000-02-31: parsing time "2000-02-31": day out of range at line 1, column 13`},
{`x=25:01:01`, `{}`, `invalid date/time value 25:01:01: parsing time "25:01:01": hour out of range at line 1, column 11`},
} {
p := newParser()
testParseHandler(t, p, p.startDateTime, test)
testParseToAST(t, p, p.startKeyValuePair, test)
}
}

27
value_number.go
View File

@ -65,28 +65,28 @@ var (
nan = a.Signed(a.Str("nan"))
)
func (t *parser) startNumber(p *parse.API) {
func (t *parser) parseNumber(p *parse.API) (*item, bool) {
switch {
case p.Accept(tok.Float64(nil, float)):
t.addParsedItem(pFloat, p.Result().Value(0).(float64))
return newItem(pFloat, p.Result().Value(0).(float64)), true
case p.Accept(nan):
t.addParsedItem(pFloat, math.NaN())
return newItem(pFloat, math.NaN()), true
case p.Accept(inf):
if p.Result().Rune(0) == '-' {
t.addParsedItem(pFloat, math.Inf(-1))
} else {
t.addParsedItem(pFloat, math.Inf(+1))
return newItem(pFloat, math.Inf(-1)), true
}
return newItem(pFloat, math.Inf(+1)), true
case p.Accept(a.Zero):
p.Handle(t.startIntegerStartingWithZero)
return t.parseIntegerStartingWithZero(p)
case p.Accept(tok.Int64(nil, integer)):
t.addParsedItem(pInteger, p.Result().Value(0).(int64))
return newItem(pInteger, p.Result().Value(0).(int64)), true
default:
p.Expected("a number")
return nil, false
}
}
func (t *parser) startIntegerStartingWithZero(p *parse.API) {
func (t *parser) parseIntegerStartingWithZero(p *parse.API) (*item, bool) {
var value int64
var err error
switch {
@ -97,12 +97,11 @@ func (t *parser) startIntegerStartingWithZero(p *parse.API) {
case p.Accept(binary):
value, err = strconv.ParseInt(p.Result().Value(0).(string), 2, 64)
default:
t.addParsedItem(pInteger, int64(0))
return
return newItem(pInteger, int64(0)), true
}
if err == nil {
t.addParsedItem(pInteger, value)
} else {
p.Error("Cannot parse value 0%s: %s", p.Result().String(), err)
return newItem(pInteger, value), true
}
p.Error("invalid integer value 0%s: %s", p.Result().String(), err)
return nil, false
}

165
value_number_test.go
View File

@ -2,106 +2,105 @@ package toml
import (
"testing"
"git.makaay.nl/mauricem/go-parsekit/parse"
)
func TestNumberStart(t *testing.T) {
parser := newParser()
wrapper := func(p *parse.API) { parser.parseNumber(p) }
testParseToAST(t, parser, wrapper, parseToASTTest{"INVALID", "{}", "unexpected input (expected a number) at start of file"})
}
func TestInteger(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected a number) at start of file`}},
for _, test := range []parseToASTTest{
// Decimal
{`0`, []string{`0`}},
{`+0`, []string{`0`}},
{`-0`, []string{`0`}},
{`1`, []string{`1`}},
{`42`, []string{`42`}},
{`+99`, []string{`99`}},
{`-17`, []string{`-17`}},
{`1234`, []string{`1234`}},
{`_`, []string{`Error: unexpected input (expected a number) at start of file`}},
{`1_`, []string{`1`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
{`1_000`, []string{`1000`}},
{`5_349_221`, []string{`5349221`}},
{`1_2_3_4_5`, []string{`12345`}},
{`9_223_372_036_854_775_807`, []string{`9223372036854775807`}},
{`9_223_372_036_854_775_808`, []string{
{`x=0`, `{"x": 0}`, ``},
{`x=+0`, `{"x": 0}`, ``},
{`x=-0`, `{"x": 0}`, ``},
{`x=1`, `{"x": 1}`, ``},
{`x=42`, `{"x": 42}`, ``},
{`x=+99`, `{"x": 99}`, ``},
{`x=-17`, `{"x": -17}`, ``},
{`x=1234`, `{"x": 1234}`, ``},
{`x=_`, `{}`, `unexpected input (expected a value) at line 1, column 3`},
{`x=1_`, `{"x": 1}`, `unexpected end of file (expected a value assignment) at line 1, column 5`},
{`x=1_000`, `{"x": 1000}`, ``},
{`x=5_349_221`, `{"x": 5349221}`, ``},
{`x=1_2_3_4_5`, `{"x": 12345}`, ``},
{`x=9_223_372_036_854_775_807`, `{"x": 9223372036854775807}`, ``},
{`x=9_223_372_036_854_775_808`, `{}`,
`Panic: Handler error: MakeInt64Token cannot handle input "9223372036854775808": ` +
`strconv.ParseInt: parsing "9223372036854775808": value out of range (only use a ` +
`type conversion token maker, when the input has been validated on beforehand)`}},
{`-9_223_372_036_854_775_808`, []string{`-9223372036854775808`}},
`strconv.ParseInt: parsing "9223372036854775808": value out of range ` +
`(only use a type conversion token maker, when the input has been validated on beforehand)`},
{`x=-9_223_372_036_854_775_808`, `{"x": -9223372036854775808}`, ``},
// TODO make the use of the same kind of handling for panics and for errors between parsekit and TOML.
{`-9_223_372_036_854_775_809`, []string{
{`x=-9_223_372_036_854_775_809`, `{}`,
`Panic: Handler error: MakeInt64Token cannot handle input "-9223372036854775809": ` +
`strconv.ParseInt: parsing "-9223372036854775809": value out of range (only use a ` +
`type conversion token maker, when the input has been validated on beforehand)`}},
`strconv.ParseInt: parsing "-9223372036854775809": value out of range ` +
`(only use a type conversion token maker, when the input has been validated on beforehand)`},
// Hexadecimal
{`0x0`, []string{`0`}},
{`0x1`, []string{`1`}},
{`0x01`, []string{`1`}},
{`0x00fF`, []string{`255`}},
{`0xf_f`, []string{`255`}},
{`0x0_0_f_f`, []string{`255`}},
{`0xdead_beef`, []string{`3735928559`}},
{`0xgood_beef`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
{`0x7FFFFFFFFFFFFFFF`, []string{`9223372036854775807`}},
{`0x8000000000000000`, []string{
`Error: Cannot parse value 0x8000000000000000: strconv.ParseInt: parsing "8000000000000000": ` +
`value out of range at line 1, column 19`}},
{`x=0x0`, `{"x": 0}`, ``},
{`x=0x1`, `{"x": 1}`, ``},
{`x=0x01`, `{"x": 1}`, ``},
{`x=0x00fF`, `{"x": 255}`, ``},
{`x=0xf_f`, `{"x": 255}`, ``},
{`x=0x0_0_f_f`, `{"x": 255}`, ``},
{`x=0xdead_beef`, `{"x": 3735928559}`, ``},
{`x=0xgood_beef`, `{"x": 0}`, `unexpected end of file (expected a value assignment) at line 1, column 14`},
{`x=0x7FFFFFFFFFFFFFFF`, `{"x": 9223372036854775807}`, ``},
{`x=0x8000000000000000`, `{}`, `invalid integer value 0x8000000000000000: strconv.ParseInt: parsing "8000000000000000": value out of range at line 1, column 21`},
//Octal
{`0o0`, []string{`0`}},
{`0o1`, []string{`1`}},
{`0o01`, []string{`1`}},
{`0o10`, []string{`8`}},
{`0o1_6`, []string{`14`}},
{`0o0_0_1_1_1`, []string{`73`}},
{`0o9`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
{`0o777777777777777777777`, []string{`9223372036854775807`}},
{`0o1000000000000000000000`, []string{
`Error: Cannot parse value 0o1000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000": ` +
`value out of range at line 1, column 25`}},
{`x=0o0`, `{"x": 0}`, ``},
{`x=0o1`, `{"x": 1}`, ``},
{`x=0o01`, `{"x": 1}`, ``},
{`x=0o10`, `{"x": 8}`, ``},
{`x=0o1_6`, `{"x": 14}`, ``},
{`x=0o0_0_1_1_1`, `{"x": 73}`, ``},
{`x=0o9`, `{"x": 0}`, `unexpected end of file (expected a value assignment) at line 1, column 6`},
{`x=0o777777777777777777777`, `{"x": 9223372036854775807}`, ``},
{`x=0o1000000000000000000000`, `{}`, `invalid integer value 0o1000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000": value out of range at line 1, column 27`},
// Binary
{`0b0`, []string{`0`}},
{`0b1`, []string{`1`}},
{`0b01`, []string{`1`}},
{`0b10`, []string{`2`}},
{`0b0100`, []string{`4`}},
{`0b00001000`, []string{`8`}},
{`0b0001_0000`, []string{`16`}},
{`0b9`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
{`0b1_1_0_1_1`, []string{`27`}},
{`0b11111111_11111111`, []string{`65535`}},
{`0b01111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111`, []string{`9223372036854775807`}},
{`0b10000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000`, []string{
`Error: Cannot parse value 0b1000000000000000000000000000000000000000000000000000000000000000: ` +
`strconv.ParseInt: parsing "1000000000000000000000000000000000000000000000000000000000000000": ` +
`value out of range at line 1, column 74`}},
{`x=0b0`, `{"x": 0}`, ``},
{`x=0b1`, `{"x": 1}`, ``},
{`x=0b01`, `{"x": 1}`, ``},
{`x=0b10`, `{"x": 2}`, ``},
{`x=0b0100`, `{"x": 4}`, ``},
{`x=0b00001000`, `{"x": 8}`, ``},
{`x=0b0001_0000`, `{"x": 16}`, ``},
{`x=0b9`, `{"x": 0}`, `unexpected end of file (expected a value assignment) at line 1, column 6`},
{`x=0b1_1_0_1_1`, `{"x": 27}`, ``},
{`x=0b11111111_11111111`, `{"x": 65535}`, ``},
{`x=0b01111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111`, `{"x": 9223372036854775807}`, ``},
{`x=0b10000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000`, `{}`, `invalid integer value 0b1000000000000000000000000000000000000000000000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000000000000000000000000000000000000000000000": value out of range at line 1, column 76`},
} {
p := newParser()
testParseHandler(t, p, p.startNumber, test)
testParseToAST(t, p, p.startKeyValuePair, test)
}
}
func TestFloat(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected a number) at start of file`}},
{`0.0`, []string{`0`}},
{`+0.0`, []string{`0`}},
{`-0.0`, []string{`-0`}},
{`+1.0`, []string{`1`}},
{`3.1415`, []string{`3.1415`}},
{`-0.01`, []string{`-0.01`}},
{`5e+22`, []string{`5e+22`}},
{`1E6`, []string{`1e+06`}},
{`-2E-2`, []string{`-0.02`}},
{`6.626e-34`, []string{`6.626e-34`}},
{`224_617.445_991_228`, []string{`224617.445991228`}},
{`12_345.111_222e+1_2_3`, []string{`1.2345111222e+127`}},
{`+nan`, []string{`NaN`}},
{`-nan`, []string{`NaN`}},
{`nan`, []string{`NaN`}},
{`inf`, []string{`+Inf`}},
{`+inf`, []string{`+Inf`}},
{`-inf`, []string{`-Inf`}},
for _, test := range []parseToASTTest{
{`x=0.0`, `{"x": 0}`, ``},
{`x=+0.0`, `{"x": 0}`, ``},
{`x=-0.0`, `{"x": -0}`, ``},
{`x=+1.0`, `{"x": 1}`, ``},
{`x=3.1415`, `{"x": 3.1415}`, ``},
{`x=-0.01`, `{"x": -0.01}`, ``},
{`x=5e+22`, `{"x": 5e+22}`, ``},
{`x=1E6`, `{"x": 1e+06}`, ``},
{`x=-2E-2`, `{"x": -0.02}`, ``},
{`x=6.626e-34`, `{"x": 6.626e-34}`, ``},
{`x=224_617.445_991_228`, `{"x": 224617.445991228}`, ``},
{`x=12_345.111_222e+1_2_3`, `{"x": 1.2345111222e+127}`, ``},
{`x=+nan`, `{"x": NaN}`, ``},
{`x=-nan`, `{"x": NaN}`, ``},
{`x=nan`, `{"x": NaN}`, ``},
{`x=inf`, `{"x": +Inf}`, ``},
{`x=+inf`, `{"x": +Inf}`, ``},
{`x=-inf`, `{"x": -Inf}`, ``},
} {
p := newParser()
testParseHandler(t, p, p.startNumber, test)
testParseToAST(t, p, p.startKeyValuePair, test)
}
}

58
value_string.go
View File

@ -43,19 +43,25 @@ var (
// There are four ways to express strings: basic, multi-line basic, literal and
// multi-line literal. All strings must contain only valid UTF-8 characters.
func (t *parser) startString(p *parse.API) {
func (t *parser) parseString(p *parse.API) (*item, bool) {
var value string
var ok bool
switch {
case p.Peek(doubleQuote3):
p.Handle(t.startMultiLineBasipString)
value, ok = t.parseMultiLineBasicString(p)
case p.Peek(a.DoubleQuote):
p.Handle(t.startBasipString)
value, ok = t.parseBasicString("string value", p)
case p.Peek(singleQuote3):
p.Handle(t.startMultiLineLiteralString)
value, ok = t.parseMultiLineLiteralString(p)
case p.Peek(a.SingleQuote):
p.Handle(t.startLiteralString)
value, ok = t.parseLiteralString("string value", p)
default:
p.Expected("a string value")
}
if ok {
return newItem(pString, value), ok
}
return nil, false
}
// Specific handling of input for basic strings.
@ -69,13 +75,7 @@ func (t *parser) startString(p *parse.API) {
// • No additional \escape sequences are allowed. What the spec say about this:
// "All other escape sequences [..] are reserved and, if used, TOML should
// produce an error.""
func (t *parser) startBasipString(p *parse.API) {
if str, ok := t.parseBasipString("basic string", p); ok {
t.addParsedItem(pString, str)
}
}
func (t *parser) parseBasipString(name string, p *parse.API) (string, bool) {
func (t *parser) parseBasicString(name string, p *parse.API) (string, bool) {
if !p.Accept(a.DoubleQuote) {
p.Expected(`opening quotation marks`)
return "", false
@ -112,12 +112,6 @@ func (t *parser) parseBasipString(name string, p *parse.API) (string, bool) {
// • Like basic strings, they must appear on a single line.
//
// • Control characters other than tab are not permitted in a literal string.
func (t *parser) startLiteralString(p *parse.API) {
if str, ok := t.parseLiteralString("literal string", p); ok {
t.addParsedItem(pString, str)
}
}
func (t *parser) parseLiteralString(name string, p *parse.API) (string, bool) {
if !p.Accept(a.SingleQuote) {
p.Expected("opening single quote")
@ -168,10 +162,10 @@ func (t *parser) parseLiteralString(name string, p *parse.API) (string, bool) {
// "line ending backslash". When the last non-whitespace character on a line is
// a \, it will be trimmed along with all whitespace (including newlines) up to
// the next non-whitespace character or closing delimiter.
func (t *parser) startMultiLineBasipString(p *parse.API) {
func (t *parser) parseMultiLineBasicString(p *parse.API) (string, bool) {
if !p.Accept(doubleQuote3.Then(newline.Optional())) {
p.Expected("opening three quotation marks")
return
return "", false
}
sb := &strings.Builder{}
for {
@ -180,25 +174,24 @@ func (t *parser) startMultiLineBasipString(p *parse.API) {
sb.WriteString("\n")
case p.Peek(controlCharacter):
p.Error("invalid character in multi-line basic string: %q (must be escaped)", p.Result().Rune(0))
return
return sb.String(), false
case p.Accept(tok.StrInterpreted(nil, c.OneOrMore(validEscape))):
sb.WriteString(p.Result().Value(0).(string))
case p.Accept(lineEndingBackslash):
// NOOP, the line-ending backslash sequence is skipped.
case p.Peek(a.Backslash):
p.Error("invalid escape sequence")
return
return sb.String(), false
case p.Accept(m.Drop(doubleQuote3)):
t.addParsedItem(pString, sb.String())
return
return sb.String(), true
case p.Accept(a.ValidRune):
sb.WriteString(p.Result().String())
case p.Peek(a.InvalidRune):
p.Error("invalid UTF8 rune")
return
return sb.String(), false
default:
p.Expected("closing three quotation marks")
return
return sb.String(), false
}
}
}
@ -216,32 +209,31 @@ func (t *parser) startMultiLineBasipString(p *parse.API) {
// sense for their platform.
//
// • Control characters other than tab and newline are not permitted in a multi-line literal string.
func (t *parser) startMultiLineLiteralString(p *parse.API) {
func (t *parser) parseMultiLineLiteralString(p *parse.API) (string, bool) {
if !p.Accept(singleQuote3.Then(newline.Optional())) {
p.Expected("opening three single quotes")
return
return "", false
}
sb := &strings.Builder{}
for {
switch {
case p.Accept(m.Drop(singleQuote3)):
t.addParsedItem(pString, sb.String())
return
return sb.String(), true
case p.Accept(a.Tab):
sb.WriteString("\t")
case p.Accept(newline):
sb.WriteString("\n")
case p.Peek(controlCharacter):
p.Error("invalid character in literal string: %q (no control chars allowed, except for tab and newline)", p.Result().Rune(0))
return
return sb.String(), false
case p.Accept(a.ValidRune):
sb.WriteString(p.Result().String())
case p.Peek(a.InvalidRune):
p.Error("invalid UTF8 rune")
return
return sb.String(), false
default:
p.Expected("closing three single quotes")
return
return sb.String(), false
}
}
}

118
value_string_test.go
View File

@ -6,91 +6,85 @@ import (
)
func TestString(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected a string value) at start of file`}},
{`no start quote"`, []string{`Error: unexpected input (expected a string value) at start of file`}},
{`"basic s\tring"`, []string{`"basic s\tring"`}},
{"\"\"\"\n basic multi-line\n string value\n\"\"\"", []string{`" basic multi-line\n string value\n"`}},
{`'literal s\tring'`, []string{`"literal s\\tring"`}},
{"'''\n literal multi-line\n string value\n'''", []string{`" literal multi-line\n string value\n"`}},
for _, test := range []parseToASTTest{
{`x=no start quote"`, `{}`, `unexpected input (expected a value) at line 1, column 3`},
{`x="basic s\tring"`, `{"x": "basic s\tring"}`, ``},
{"x=\"\"\"\n basic multi-line\n string value\n\"\"\"", `{"x": " basic multi-line\n string value\n"}`, ``},
{`x='literal s\tring'`, `{"x": "literal s\\tring"}`, ``},
{"x='''\n literal multi-line\n string value\n'''", `{"x": " literal multi-line\n string value\n"}`, ``},
} {
p := newParser()
testParseHandler(t, p, p.startString, test)
testParseToAST(t, p, p.startKeyValuePair, test)
}
}
func TestBasipString(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected opening quotation marks) at start of file`}},
{`no start quote"`, []string{`Error: unexpected input (expected opening quotation marks) at start of file`}},
{`"no end quote`, []string{`Error: unexpected end of file (expected closing quotation marks) at line 1, column 14`}},
{`""`, []string{`""`}},
{`"simple string"`, []string{`"simple string"`}},
{`"with\tsome\r\nvalid escapes\b"`, []string{`"with\tsome\r\nvalid escapes\b"`}},
{`"with an \invalid escape"`, []string{`Error: invalid escape sequence at line 1, column 10`}},
{`"A cool UTF8 ƃuıɹʇs"`, []string{`"A cool UTF8 ƃuıɹʇs"`}},
{`"A string with UTF8 escape \u2318"`, []string{`"A string with UTF8 escape ⌘"`}},
{"\"Invalid character for UTF \xcd\"", []string{`Error: invalid UTF8 rune at line 1, column 28`}},
{"\"Character that mus\t be escaped\"", []string{`Error: invalid character in basic string: '\t' (must be escaped) at line 1, column 20`}},
{"\"Character that must be escaped \u0000\"", []string{`Error: invalid character in basic string: '\x00' (must be escaped) at line 1, column 33`}},
{"\"Character that must be escaped \x7f\"", []string{`Error: invalid character in basic string: '\u007f' (must be escaped) at line 1, column 33`}},
for _, test := range []parseToASTTest{
{`x="no end quote`, `{}`, `unexpected end of file (expected closing quotation marks) at line 1, column 16`},
{`x=""`, `{"x": ""}`, ``},
{`x="simple string"`, `{"x": "simple string"}`, ``},
{`x="with\tsome\r\nvalid escapes\b"`, `{"x": "with\tsome\r\nvalid escapes\b"}`, ``},
{`x="with an \invalid escape"`, `{}`, `invalid escape sequence at line 1, column 12`},
{`x="A cool UTF8 ƃuıɹʇs"`, `{"x": "A cool UTF8 ƃuıɹʇs"}`, ``},
{`x="A string with UTF8 escape \u2318"`, `{"x": "A string with UTF8 escape ⌘"}`, ``},
{"x=\"Invalid character for UTF \xcd\"", `{}`, `invalid UTF8 rune at line 1, column 30`},
{"x=\"Character that mus\t be escaped\"", `{}`, `invalid character in string value: '\t' (must be escaped) at line 1, column 22`},
{"x=\"Character that must be escaped \u0000\"", `{}`, `invalid character in string value: '\x00' (must be escaped) at line 1, column 35`},
{"x=\"Character that must be escaped \x7f\"", `{}`, `invalid character in string value: '\u007f' (must be escaped) at line 1, column 35`},
} {
p := newParser()
testParseHandler(t, p, p.startBasipString, test)
testParseToAST(t, p, p.startKeyValuePair, test)
}
}
func TestMultiLineBasipString(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected opening three quotation marks) at start of file`}},
{`"""missing close quote""`, []string{`Error: unexpected end of file (expected closing three quotation marks) at line 1, column 25`}},
{`""""""`, []string{`""`}},
{"\"\"\"\n\"\"\"", []string{`""`}},
{"\"\"\"\r\n\r\n\"\"\"", []string{`"\n"`}},
{`"""\"\"\"\""""`, []string{`"\"\"\"\""`}},
{"\"\"\"\nThe quick brown \\\n\n\n \t fox jumps over \\\n\t the lazy dog.\\\n \"\"\"", []string{`"The quick brown fox jumps over the lazy dog."`}},
{"\"\"\"No control chars \f allowed\"\"\"", []string{`Error: invalid character in multi-line basic string: '\f' (must be escaped) at line 1, column 21`}},
{"\"\"\"Escaping control chars\\nis valid\"\"\"", []string{`"Escaping control chars\nis valid"`}},
{"\"\"\"Invalid escaping \\is not allowed\"\"\"", []string{`Error: invalid escape sequence at line 1, column 21`}},
{"\"\"\"Invalid rune \xcd\"\"\"", []string{`Error: invalid UTF8 rune at line 1, column 17`}},
for _, test := range []parseToASTTest{
{`x="""missing close quote""`, `{}`, `unexpected end of file (expected closing three quotation marks) at line 1, column 27`},
{`x=""""""`, `{"x": ""}`, ``},
{"x=\"\"\"\n\"\"\"", `{"x": ""}`, ``},
{"x=\"\"\"\r\n\r\n\"\"\"", `{"x": "\n"}`, ``},
{`x="""\"\"\"\""""`, `{"x": "\"\"\"\""}`, ``},
{"x=\"\"\"\nThe quick brown \\\n\n\n \t fox jumps over \\\n\t the lazy dog.\\\n \"\"\"", `{"x": "The quick brown fox jumps over the lazy dog."}`, ``},
{"x=\"\"\"No control chars \f allowed\"\"\"", `{}`, `invalid character in multi-line basic string: '\f' (must be escaped) at line 1, column 23`},
{"x=\"\"\"Escaping control chars\\nis valid\"\"\"", `{"x": "Escaping control chars\nis valid"}`, ``},
{"x=\"\"\"Invalid escaping \\is not allowed\"\"\"", `{}`, `invalid escape sequence at line 1, column 23`},
{"x=\"\"\"Invalid rune \xcd\"\"\"", `{}`, `invalid UTF8 rune at line 1, column 19`},
} {
p := newParser()
testParseHandler(t, p, p.startMultiLineBasipString, test)
testParseToAST(t, p, p.startKeyValuePair, test)
}
}
func TestLiteralString(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected opening single quote) at start of file`}},
{`'missing close quote`, []string{`Error: unexpected end of file (expected closing single quote) at line 1, column 21`}},
{`''`, []string{`""`}},
{`'simple'`, []string{`"simple"`}},
{`'C:\Users\nodejs\templates'`, []string{`"C:\\Users\\nodejs\\templates"`}},
{`'\\ServerX\admin$\system32\'`, []string{`"\\\\ServerX\\admin$\\system32\\"`}},
{`'Tom "Dubs" Preston-Werner'`, []string{`"Tom \"Dubs\" Preston-Werner"`}},
{`'<\i\c*\s*>'`, []string{`"<\\i\\c*\\s*>"`}},
{"'No cont\rol chars allowed'", []string{`Error: invalid character in literal string: '\r' (no control chars allowed, except for tab) at line 1, column 9`}},
{"'Except\tfor\ttabs'", []string{`"Except\tfor\ttabs"`}},
{"'Invalid rune \xcd'", []string{`Error: invalid UTF8 rune at line 1, column 15`}},
for _, test := range []parseToASTTest{
{`x='missing close quote`, `{}`, `unexpected end of file (expected closing single quote) at line 1, column 23`},
{`x=''`, `{"x": ""}`, ``},
{`x='simple'`, `{"x": "simple"}`, ``},
{`x='C:\Users\nodejs\templates'`, `{"x": "C:\\Users\\nodejs\\templates"}`, ``},
{`x='\\ServerX\admin$\system32\'`, `{"x": "\\\\ServerX\\admin$\\system32\\"}`, ``},
{`x='Tom "Dubs" Preston-Werner'`, `{"x": "Tom \"Dubs\" Preston-Werner"}`, ``},
{`x='<\i\c*\s*>'`, `{"x": "<\\i\\c*\\s*>"}`, ``},
{"x='No cont\rol chars allowed'", `{}`, `invalid character in string value: '\r' (no control chars allowed, except for tab) at line 1, column 11`},
{"x='Except\tfor\ttabs'", `{"x": "Except\tfor\ttabs"}`, ``},
{"x='Invalid rune \xcd'", `{}`, `invalid UTF8 rune at line 1, column 17`},
} {
p := newParser()
testParseHandler(t, p, p.startLiteralString, test)
testParseToAST(t, p, p.startKeyValuePair, test)
}
}
func TestMultiLineLiteralString(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected opening three single quotes) at start of file`}},
{`'''missing close quote''`, []string{`Error: unexpected end of file (expected closing three single quotes) at line 1, column 25`}},
{`''''''`, []string{`""`}},
{"'''\n'''", []string{`""`}},
{`'''I [dw]on't need \d{2} apples'''`, []string{`"I [dw]on't need \\d{2} apples"`}},
{"'''\nThere can\nbe newlines\r\nand \ttabs!\r\n'''", []string{`"There can\nbe newlines\nand \ttabs!\n"`}},
{"'''No other \f control characters'''", []string{`Error: invalid character in literal string: '\f' (no control chars allowed, except for tab and newline) at line 1, column 13`}},
{"'''No invalid runes allowed \xcd'''", []string{"Error: invalid UTF8 rune at line 1, column 29"}},
for _, test := range []parseToASTTest{
{`x='''missing close quote''`, `{}`, `unexpected end of file (expected closing three single quotes) at line 1, column 27`},
{`x=''''''`, `{"x": ""}`, ``},
{"x='''\n'''", `{"x": ""}`, ``},
{`x='''I [dw]on't need \d{2} apples'''`, `{"x": "I [dw]on't need \\d{2} apples"}`, ``},
{"x='''\nThere can\nbe newlines\r\nand \ttabs!\r\n'''", `{"x": "There can\nbe newlines\nand \ttabs!\n"}`, ``},
{"x='''No other \f control characters'''", `{}`, `invalid character in literal string: '\f' (no control chars allowed, except for tab and newline) at line 1, column 15`},
{"x='''No invalid runes allowed \xcd'''", `{}`, `invalid UTF8 rune at line 1, column 31`},
} {
p := newParser()
testParseHandler(t, p, p.startMultiLineLiteralString, test)
testParseToAST(t, p, p.startKeyValuePair, test)
}
}
@ -99,8 +93,8 @@ func TestBasipStringWithUnescapedControlCharacters(t *testing.T) {
// The missing one (\x7f) is covered in the previous test.
for i := 0x00; i <= 0x1F; i++ {
p := newParser()
input := fmt.Sprintf(`"%c"`, rune(i))
expected := fmt.Sprintf(`Error: invalid character in basic string: %q (must be escaped) at line 1, column 2`, rune(i))
testParseHandler(t, p, p.startString, parseTest{input, []string{expected}})
input := fmt.Sprintf(`x="%c"`, rune(i))
expected := fmt.Sprintf(`invalid character in string value: %q (must be escaped) at line 1, column 4`, rune(i))
testParseToAST(t, p, p.startKeyValuePair, parseToASTTest{input, "{}", expected})
}
}

57
value_table.go
View File

@ -86,11 +86,32 @@ var (
// [[fruit.variety]]
// name = "plantain"
tableArrayOpen = c.Seq(dropBlanks, a.SquareOpen, a.SquareOpen, dropBlanks)
tableArrayClose = c.Seq(dropBlanks, a.SquareClose, a.SquareClose, a.EndOfLine.Or(comment))
tableArrayClose = c.Seq(dropBlanks, a.SquareClose, a.SquareClose, dropBlanks, a.EndOfLine.Or(comment))
// Inline tables provide a more compact syntax for expressing tables.
// They are especially useful for grouped data that can otherwise quickly
// become verbose. Inline tables are enclosed in curly braces { and }.
// Within the braces, zero or more comma separated key/value pairs may appear.
// Key/value pairs take the same form as key/value pairs in standard tables.
// All value types are allowed, including inline tables.
//
// Inline tables are intended to appear on a single line. No newlines are
// allowed between the curly braces unless they are valid within a value.
// Even so, it is strongly discouraged to break an inline table onto multiple
// lines. If you find yourself gripped with this desire, it means you should
// be using standard tables.
//
// name = { first = "Tom", last = "Preston-Werner" }
// point = { x = 1, y = 2 }
// animal = { type.name = "pug" }
inlineTableOpen = c.Seq(dropBlanks, a.CurlyOpen, dropBlanks)
inlineTableClose = c.Seq(dropBlanks, a.CurlyClose, dropBlanks, a.EndOfLine.Or(comment))
)
func (t *parser) startTable(p *parse.API) {
switch {
case p.Accept(tableArrayOpen):
p.Handle(t.startArrayOfTables)
case p.Accept(tableOpen):
p.Handle(t.startPlainTable)
default:
@ -98,14 +119,30 @@ func (t *parser) startTable(p *parse.API) {
}
}
func (t *parser) startPlainTable(p *parse.API) {
if !p.Handle(t.startKey) {
return
func (t *parser) startArrayOfTables(p *parse.API) {
if key, ok := t.parseKey(p, []string{}); ok {
if !p.Accept(tableArrayClose) {
p.Expected("closing ']]' for array of tables name")
return
}
if err := t.openArrayOfTables(key); err != nil {
p.Error("%s", err)
return
}
p.Handle(t.startKeyValuePair)
}
}
func (t *parser) startPlainTable(p *parse.API) {
if key, ok := t.parseKey(p, []string{}); ok {
if !p.Accept(tableClose) {
p.Expected("closing ']' for table name")
return
}
if err := t.openTable(key); err != nil {
p.Error("%s", err)
return
}
p.Handle(t.startKeyValuePair)
}
if !p.Accept(tableClose) {
p.Expected("closing ']' for table name")
}
key := t.Items[0]
t.Items = t.Items[1:]
t.openTable(key)
}

55
value_table_test.go
View File

@ -4,20 +4,71 @@ import (
"testing"
)
func TestTable(t *testing.T) {
func TestTableKey(t *testing.T) {
for _, test := range []parseTest{
{"", []string{`Error: unexpected end of file (expected a table) at start of file`}},
{"[", []string{`Error: unexpected end of file (expected a key name) at line 1, column 2`}},
{" \t [", []string{`Error: unexpected end of file (expected a key name) at line 1, column 5`}},
{" \t [key", []string{`Error: unexpected end of file (expected closing ']' for table name) at line 1, column 8`}},
{" \t [key.", []string{`key("key")`, `Error: unexpected end of file (expected a key name) at line 1, column 9`}},
{" \t [key.", []string{`Error: unexpected end of file (expected a key name) at line 1, column 9`}},
{" \t [key.'sub key'", []string{`Error: unexpected end of file (expected closing ']' for table name) at line 1, column 18`}},
{" \t [key.'sub key' \t]", []string{}},
{" \t [key.'sub key' \t] \t ", []string{}},
{" \t [key.'sub key' \t] \t \n", []string{}},
{" \t [key.'sub key' \t]# with comment\n", []string{}},
{" \t [key.'sub key' \t] \t # with comment\n", []string{}},
} {
p := newParser()
testParseHandler(t, p, p.startTable, test)
}
}
func TestTable(t *testing.T) {
for _, test := range []parseToASTTest{
{"[a]", `{"a": {}}`, ``},
{"['a key']", `{"a key": {}}`, ``},
{"[\"a key\"]", `{"a key": {}}`, ``},
{"[\"a key\".'sub'.key]", `{"a key": {"sub": {"key": {}}}}`, ``},
{"[a]\nx=1234", `{"a": {"x": 1234}}`, ``},
{"[a]\nx=1234\ny='string'", `{"a": {"x": 1234, "y": "string"}}`, ``},
{"[a]\n[b]", `{"a": {}, "b": {}}`, ``},
{"[a]\n[a.b]\n[a.b.c]\n[a.d]\nx=1", `{"a": {"b": {"c": {}}, "d": {"x": 1}}}`, ``},
{"[a]\n[b] #another table \na=1\n", `{"a": {}, "b": {"a": 1}}`, ``},
{"[a]\nx=1\ny=2\n[b] #another table \nx=1\ny=2021-01-01", `{"a": {"x": 1, "y": 2}, "b": {"x": 1, "y": 2021-01-01}}`, ``},
{"[a]\nx=1\ny=2\n[a.b] #subtable \nx=1\ny=2021-01-01", `{"a": {"b": {"x": 1, "y": 2021-01-01}, "x": 1, "y": 2}}`, ``},
} {
p := newParser()
testParseToAST(t, p, p.startTable, test)
}
}
func TestArrayOfTableKey(t *testing.T) {
for _, test := range []parseTest{
{"[[", []string{`Error: unexpected end of file (expected a key name) at line 1, column 3`}},
{" \t [[", []string{`Error: unexpected end of file (expected a key name) at line 1, column 6`}},
{" \t [[key", []string{`Error: unexpected end of file (expected closing ']]' for array of tables name) at line 1, column 9`}},
{" \t [[key.", []string{`Error: unexpected end of file (expected a key name) at line 1, column 10`}},
{" \t [[key.'sub key'", []string{`Error: unexpected end of file (expected closing ']]' for array of tables name) at line 1, column 19`}},
{" \t [[key.'sub key' \t]]", []string{}},
{" \t [[key.'sub key' \t]] \t ", []string{}},
{" \t [[key.'sub key' \t]] \t \n", []string{}},
{" \t [[key.'sub key' \t]]# with comment\n", []string{}},
{" \t [[key.'sub key' \t]] \t # with comment\n", []string{}},
} {
p := newParser()
testParseHandler(t, p, p.startTable, test)
}
}
func TestArrayOfTables(t *testing.T) {
for _, test := range []parseToASTTest{
{"[[a]]", `{"a": [{}]}`, ``},
{"[[a]]\n[['a']]\n[[\"a\"]]", `{"a": [{}, {}, {}]}`, ``},
{"[[a]]\n[['a']]\n[b]\n[[\"a\"]]", `{"a": [{}, {}, {}], "b": {}}`, ``},
{"[[a]]\nx=1\n[['a']]\nx=2\ny=3\n[[\"a\"]]", `{"a": [{"x": 1}, {"x": 2, "y": 3}, {}]}`, ``},
{"[a]\n[[a.b]]\nx=1\n[[a.b]]\nx=2\n[a.c]\ny=1234", `{"a": {"b": [{"x": 1}, {"x": 2}], "c": {"y": 1234}}}`, ``},
} {
p := newParser()
testParseToAST(t, p, p.startTable, test)
}
}

84
value_test.go
View File

@ -1,46 +1,42 @@
package toml
import (
"testing"
)
func TestValue(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected a value) at start of file`}},
{`"basic s\tring value"`, []string{`"basic s\tring value"`}},
{`'literal s\tring value'`, []string{`"literal s\\tring value"`}},
{"\"\"\"basic multi-line\nstring value\"\"\"", []string{`"basic multi-line\nstring value"`}},
{"'''literal multi-line\nstring value'''", []string{`"literal multi-line\nstring value"`}},
{"true", []string{`true`}},
{"false", []string{`false`}},
{"0", []string{`0`}},
{"+0", []string{`0`}},
{"-0", []string{`0`}},
{"0.0", []string{`0`}},
{"+0.0", []string{`0`}},
{"-0.0", []string{`-0`}},
{"1234", []string{`1234`}},
{"-1234", []string{`-1234`}},
{"+9_8_7.6_5_4e-321", []string{`9.8765e-319`}},
{"-1_234.5678e-33", []string{`-1.2345678e-30`}},
{"inf", []string{`+Inf`}},
{"+inf", []string{`+Inf`}},
{"-inf", []string{`-Inf`}},
{"nan", []string{`NaN`}},
{"+nan", []string{`NaN`}},
{"-nan", []string{`NaN`}},
{"2019-06-19", []string{`2019-06-19`}},
{"08:38:54", []string{`08:38:54`}},
{"08:38:54.8765487654876", []string{`08:38:54.876548765`}},
{"2019-06-19 08:38:54", []string{`2019-06-19 08:38:54`}},
{"2019-06-19T08:38:54", []string{`2019-06-19 08:38:54`}},
{"2019-06-19T08:38:54.88888", []string{`2019-06-19 08:38:54.88888`}},
{"1979-05-27T07:32:00Z", []string{`1979-05-27T07:32:00Z`}},
{"1979-05-27T00:32:00-07:00", []string{`1979-05-27T00:32:00-07:00`}},
{"1979-05-27T00:32:00.999999-07:00", []string{`1979-05-27T00:32:00.999999-07:00`}},
{"[1,2,3]", []string{`[1, 2, 3]`}},
} {
p := newParser()
testParseHandler(t, p, p.startValue, test)
}
}
// func TestValue(t *testing.T) {
// for _, test := range []parseTest{
// {``, []string{`Error: unexpected end of file (expected a value) at start of file`}},
// {`"basic s\tring value"`, []string{`"basic s\tring value"`}},
// {`'literal s\tring value'`, []string{`"literal s\\tring value"`}},
// {"\"\"\"basic multi-line\nstring value\"\"\"", []string{`"basic multi-line\nstring value"`}},
// {"'''literal multi-line\nstring value'''", []string{`"literal multi-line\nstring value"`}},
// {"true", []string{`true`}},
// {"false", []string{`false`}},
// {"0", []string{`0`}},
// {"+0", []string{`0`}},
// {"-0", []string{`0`}},
// {"0.0", []string{`0`}},
// {"+0.0", []string{`0`}},
// {"-0.0", []string{`-0`}},
// {"1234", []string{`1234`}},
// {"-1234", []string{`-1234`}},
// {"+9_8_7.6_5_4e-321", []string{`9.8765e-319`}},
// {"-1_234.5678e-33", []string{`-1.2345678e-30`}},
// {"inf", []string{`+Inf`}},
// {"+inf", []string{`+Inf`}},
// {"-inf", []string{`-Inf`}},
// {"nan", []string{`NaN`}},
// {"+nan", []string{`NaN`}},
// {"-nan", []string{`NaN`}},
// {"2019-06-19", []string{`2019-06-19`}},
// {"08:38:54", []string{`08:38:54`}},
// {"08:38:54.8765487654876", []string{`08:38:54.876548765`}},
// {"2019-06-19 08:38:54", []string{`2019-06-19 08:38:54`}},
// {"2019-06-19T08:38:54", []string{`2019-06-19 08:38:54`}},
// {"2019-06-19T08:38:54.88888", []string{`2019-06-19 08:38:54.88888`}},
// {"1979-05-27T07:32:00Z", []string{`1979-05-27T07:32:00Z`}},
// {"1979-05-27T00:32:00-07:00", []string{`1979-05-27T00:32:00-07:00`}},
// {"1979-05-27T00:32:00.999999-07:00", []string{`1979-05-27T00:32:00.999999-07:00`}},
// {"[1,2,3]", []string{`[1, 2, 3]`}},
// } {
// p := newParser()
// testParseHandler(t, p, p.startValue, test)
// }
// }