diff --git a/ast.go b/ast.go index 081a9af..554ab70 100644 --- a/ast.go +++ b/ast.go @@ -2,6 +2,7 @@ package toml import ( "fmt" + "regexp" "sort" "strings" "time" @@ -14,18 +15,12 @@ type item struct { } // table represents a TOML table. -type table map[string]item +type table map[string]*item // itemType identifies the semantic role of a TOML item. type itemType string const ( - // TODO Would be nice to not need these in the end. - pComment itemType = "comment" - pKey itemType = "key" - pAssign itemType = "assign" - - // TODO and just use these data types. pString itemType = "string" // "various", 'types', """of""", '''strings''' pInteger itemType = "integer" // 12345, 0xffee12a0, 0o0755, 0b101101011 pFloat itemType = "float" // 10.1234, 143E-12, 43.28377e+4, +inf, -inf, nan @@ -34,14 +29,176 @@ const ( pLocalDateTime itemType = "datetime" // 2018-12-25 12:12:18.876772533 pLocalDate itemType = "date" // 2017-05-17 pLocalTime itemType = "time" // 23:01:22 - pArray itemType = "array" // defined using an [[array.of.tables]] - pStaticArray itemType = "static array" // defined using ["an", "inline", "array"] + pArrayOfTables itemType = "array" // defined using an [[array.of.tables]] + pArray itemType = "static array" // defined using ["an", "inline", "array"] pTable itemType = "table" // defined using { "inline" = "table" } or [standard.table] ) // newItem instantiates a new item struct. -func newItem(itemType itemType, data ...interface{}) item { - return item{Type: itemType, Data: data} +func newItem(itemType itemType, data ...interface{}) *item { + return &item{Type: itemType, Data: data} +} + +// newKey instantiates a new key. +func newKey(key ...string) []string { + return key +} + +// parser holds the state for the TOML parser. All parsing functions are +// methods of this struct. +type parser struct { + Items []*item // a buffer for holding parsed items + Root table // the root-level TOML table (each TOML doc is implicitly a table) + Current table // the currently active TOML table + CurrentKey []string // the key for the currently active TOML table +} + +func newParser() *parser { + p := &parser{Root: make(table)} + p.Current = p.Root + return p +} + +func (t *parser) setKeyValuePair(key []string, value *item) error { + // First make sure the table structure for storing the value exists. + node, lastKeyPart, err := t.makeTablePath(key) + if err != nil { + return fmt.Errorf("invalid key/value pair: %s", err) + } + // Check if the key is still free for use. + if existing, ok := node[lastKeyPart]; ok { + path := t.formatKeyPath(key, len(key)-1) + return fmt.Errorf("invalid key/value pair: %s item already exists at key %s", existing.Type, path) + } + // It is, store the value in the table. + node[lastKeyPart] = value + return nil +} + +func (t *parser) openTable(key []string) error { + t.CurrentKey = nil + t.Current = t.Root + // Go over all requested levels of the key. For all levels, except the last + // one, it is okay if a table already exists. For at least the last level, + // no table or value must exist, because that would mean we are overwriting + // an existing key/value pair, which is not allowed. + node, lastKeyPart, err := t.makeTablePath(key) + if err != nil { + return fmt.Errorf("invalid table: %s", err) + } + // Check if the key is still free for use. + if existing, ok := node[lastKeyPart]; ok { + path := t.formatKeyPath(key, len(key)-1) + return fmt.Errorf("invalid table: %s item already exists at key %s", existing.Type, path) + } + // The subtable does not exist yet. Create the subtable. + subTable := make(table) + node[lastKeyPart] = newItem(pTable, subTable) + node = subTable + + // From here on, key/value pairs are added to the newly defined table. + t.Current = node + t.CurrentKey = key + return nil +} + +func (t *parser) openArrayOfTables(key []string) error { + t.CurrentKey = nil + t.Current = t.Root + // Go over all requested levels of the key. For all levels, except the last + // one, it is okay if a table already exists. For the last level, either + // no item must exist (in which case a table array will be created), or a + // table array must exist. + // Other cases would mean we are overwriting an existing key/value pair, + // which is not allowed. + node, lastKeyPart, err := t.makeTablePath(key) + if err != nil { + return fmt.Errorf("invalid table array: %s", err) + } + // At the last key position, there must be either no value yet, or the + // existing value must be a table array. Other values are invalid. + if existing, ok := node[lastKeyPart]; ok { + if existing.Type != pArrayOfTables { + path := t.formatKeyPath(key, len(key)-1) + return fmt.Errorf("invalid table array: %s item already exists at key %s", existing.Type, path) + } + // A table array exists. Add a new table to this array. + array := node[lastKeyPart] + subTable := make(table) + tables := array.Data + tables = append(tables, newItem(pTable, subTable)) + array.Data = tables + node = subTable + } else { + // No value exists at the defined key path. Create a new table array. + subTable := make(table) + node[lastKeyPart] = newItem(pArrayOfTables, newItem(pTable, subTable)) + node = subTable + } + + // From here on, key/value pairs are added to the newly defined table. + t.Current = node + t.CurrentKey = key + return nil +} + +func (t *parser) makeTablePath(key []string) (table, string, error) { + node := t.Current + for i, keyPart := range key { + // Arrived at the last key part? Then the path towards that key is + // setup correctly. Return the last part, so the caller can use it. + isLast := i == len(key)-1 + if isLast { + return node, keyPart, nil + } + if subItem, ok := node[keyPart]; ok { + // You cannot overwrite an already defined key, regardless its value. + // When a value already exists at the current key, this can only be a table. + if subItem.Type != pTable { + path := t.formatKeyPath(key, i) + return nil, "", fmt.Errorf("%s item already exists at key %s", subItem.Type, path) + } + // All is okay, traverse to the subtable. + node = subItem.Data[0].(table) + } else { + // The subtable does not exist yet. Create the subtable. + subTable := make(table) + node[keyPart] = newItem(pTable, subTable) + node = subTable + } + } + panic("makeTablePath(): empty key provided; a key must have at least one key part") +} + +func (t *parser) formatKeyPath(key []string, end int) string { + var sb strings.Builder + sb.WriteRune('[') + if t.CurrentKey != nil { + for i, keyPart := range t.CurrentKey { + if i > 0 { + sb.WriteString("->") + } + sb.WriteString(formatKeyName(keyPart)) + } + } + for i, keyPart := range key { + if t.CurrentKey != nil || i > 0 { + sb.WriteString("->") + } + sb.WriteString(formatKeyName(keyPart)) + if i == end { + break + } + } + sb.WriteRune(']') + return sb.String() +} + +func formatKeyName(key string) string { + if ok, _ := regexp.Match(`^\w+$`, []byte(key)); ok { + return key + } + return fmt.Sprintf("%q", key) } func (t table) String() string { @@ -52,12 +209,6 @@ func (parseItem item) String() string { switch parseItem.Type { case pString: return fmt.Sprintf("%q", parseItem.Data[0]) - case pInteger: - return fmt.Sprintf("%v", parseItem.Data[0]) - case pFloat: - return fmt.Sprintf("%v", parseItem.Data[0]) - case pBoolean: - return fmt.Sprintf("%v", parseItem.Data[0]) case pOffsetDateTime: return parseItem.Data[0].(time.Time).Format(time.RFC3339Nano) case pLocalDateTime: @@ -66,12 +217,12 @@ func (parseItem item) String() string { return parseItem.Data[0].(time.Time).Format("2006-01-02") case pLocalTime: return parseItem.Data[0].(time.Time).Format("15:04:05.999999999") - case pArray: + case pArrayOfTables: fallthrough - case pStaticArray: - items := make([]string, len(parseItem.Data[0].([]item))) - for i, d := range parseItem.Data[0].([]item) { - items[i] = d.String() + case pArray: + items := make([]string, len(parseItem.Data)) + for i, value := range parseItem.Data { + items[i] = value.(*item).String() } return fmt.Sprintf("[%s]", strings.Join(items, ", ")) case pTable: @@ -88,118 +239,7 @@ func (parseItem item) String() string { items[i] = fmt.Sprintf("%q: %s", k, pairs[k].String()) } return fmt.Sprintf("{%s}", strings.Join(items, ", ")) - case pComment: - return fmt.Sprintf("comment(%q)", parseItem.Data[0]) - case pKey: - items := make([]string, len(parseItem.Data)) - for i, e := range parseItem.Data { - items[i] = fmt.Sprintf("%q", e) - } - return fmt.Sprintf("key(%s)", strings.Join(items, ", ")) - case pAssign: - return "=" default: - panic(fmt.Sprintf("Missing String() formatting for item type '%s'", parseItem.Type)) + return fmt.Sprintf("%v", parseItem.Data[0]) } } - -// parser holds the state for the TOML parser. All parsing functions are -// methods of this struct. -type parser struct { - Items []item // a buffer for holding parsed items - Root table // the root-level TOML table (each TOML doc is implicitly a table) - Current table // the currently active TOML table -} - -func newParser() *parser { - p := &parser{Root: make(table)} - p.Current = p.Root - return p -} - -func (t *parser) addParsedItem(itemType itemType, data ...interface{}) { - t.Items = append(t.Items, newItem(itemType, data...)) -} - -func (t *parser) setValue(key item, value item) error { - // When the key has multiple elements, then first make sure the table structure - // for storing the value exists. - var valueKey string - node := t.Current - l := len(key.Data) - if l > 1 { - pathKeys := key.Data[0 : l-1] - valueKey = key.Data[l-1].(string) - for i, name := range pathKeys { - name := name.(string) - if subItem, ok := node[name]; ok { - // An item was found at the current key. It is expected to be a table. - if subItem.Type != pTable { - path := formatKeyPath(key, i) - return fmt.Errorf("invalid key used: %s item already exists at key %s", subItem.Type, path) - } - node = subItem.Data[0].(table) - } else { - // No item was found at the current key. Create a new subtable. - subTable := make(table) - node[name] = newItem(pTable, subTable) - node = subTable - } - } - } else { - valueKey = key.Data[0].(string) - node = t.Current - } - - if existing, ok := node[valueKey]; ok { - path := "moetnog" - return fmt.Errorf("Cannot store value: %s item already exists at key %s", existing.Type, path) - } - - node[valueKey] = value - return nil -} - -func (t *parser) openTable(key item) (table, error) { - node := t.Root - // Go over all requested levels of the key. For all levels, except the last - // one, it is okay if a table already exists. For at least the last level, - // no table or value must exist, because that would mean we are overwriting - // an existing key/value pair, which is not allowed. - for i, value := range key.Data { - keyName := value.(string) - if subItem, ok := node[keyName]; ok { - // You cannot overwrite an already defined key, regardless its value. - if subItem.Type != pTable { - path := formatKeyPath(key, i) - return nil, fmt.Errorf("Cannot create table: %s item already exists at key %s", subItem.Type, path) - } - // Like keys, you cannot define any table more than once. Doing so is invalid. - isLast := i == len(key.Data)-1 - if isLast { - path := formatKeyPath(key, i) - return nil, fmt.Errorf("Cannot create table: table for key %s already exists", path) - } - node = subItem.Data[0].(table) - } else { - // Create the subtable. - subTable := make(table) - node[keyName] = newItem(pTable, subTable) - node = subTable - } - } - // From here on, key/value pairs are added to the newly defined table. - t.Current = node - return node, nil -} - -func formatKeyPath(key item, end int) string { - var sb strings.Builder - for i := 0; i <= end; i++ { - if i > 0 { - sb.WriteRune('.') - } - sb.WriteString(fmt.Sprintf("%q", key.Data[i].(string))) - } - return sb.String() -} diff --git a/ast_test.go b/ast_test.go index 245a0d7..45883cb 100644 --- a/ast_test.go +++ b/ast_test.go @@ -7,63 +7,171 @@ import ( func TestAST_ConstructStructure(t *testing.T) { testAST(t, func() (error, *parser) { p := newParser() - p.setValue(newItem(pKey, "ding"), newItem(pInteger, 10)) - p.setValue(newItem(pKey, "dong"), newItem(pString, "not a song")) - p.openTable(newItem(pKey, "key1", "key2 a")) - p.setValue(newItem(pKey, "dooh"), newItem(pBoolean, true)) - p.setValue(newItem(pKey, "dah"), newItem(pBoolean, false)) - p.openTable(newItem(pKey, "key1", "key2 b")) - p.setValue(newItem(pKey, "dieh"), newItem(pFloat, 1.111)) - p.setValue(newItem(pKey, "duh"), newItem(pFloat, 1.18e-12)) + p.setKeyValuePair(newKey("ding"), newItem(pInteger, 10)) + p.setKeyValuePair(newKey("dong"), newItem(pString, "not a song")) + p.openTable(newKey("key1", "key2 a")) + p.setKeyValuePair(newKey("dooh"), newItem(pBoolean, true)) + p.setKeyValuePair(newKey("dah"), newItem(pBoolean, false)) + p.openTable(newKey("key1", "key2 b")) + p.setKeyValuePair(newKey("dieh"), newItem(pFloat, 1.111)) + p.setKeyValuePair(newKey("duh"), newItem(pFloat, 1.18e-12)) + p.setKeyValuePair(newKey("foo", "bar"), newItem(pArrayOfTables, newItem(pInteger, 1), newItem(pInteger, 2))) + p.openArrayOfTables(newKey("aaah", "table array")) + p.setKeyValuePair(newKey("a"), newItem(pFloat, 1.234)) + p.openArrayOfTables(newKey("aaah", "table array")) + p.setKeyValuePair(newKey("b"), newItem(pFloat, 2.345)) + p.setKeyValuePair(newKey("c"), newItem(pString, "bingo!")) + p.openArrayOfTables(newKey("aaah", "table array")) return nil, p - }, "", `{"ding": 10, "dong": "not a song", "key1": {"key2 a": {"dah": false, "dooh": true}, "key2 b": {"dieh": 1.111, "duh": 1.18e-12}}}`) + }, + "", + `{"aaah": {"table array": [{"a": 1.234}, {"b": 2.345, "c": "bingo!"}, {}]}, `+ + `"ding": 10, "dong": "not a song", `+ + `"key1": {"key2 a": {"dah": false, "dooh": true}, "key2 b": {"dieh": 1.111, "duh": 1.18e-12, "foo": {"bar": [1, 2]}}}}`) +} + +func TestAST_EmptyKeyForCreatingTablePath_Panics(t *testing.T) { + defer func() { + r := recover() + if r.(string) != "makeTablePath(): empty key provided; a key must have at least one key part" { + t.Fatalf("Did not get the expected panic message") + } + }() + p := newParser() + p.openTable(newKey()) } func TestAST_StoreValueInRootTable(t *testing.T) { testAST(t, func() (error, *parser) { p := newParser() - p.setValue(newItem(pKey, "key1"), newItem(pString, "value1")) - return p.setValue(newItem(pKey, "key2"), newItem(pString, "value2")), p - }, "", `{"key1": "value1", "key2": "value2"}`) + p.setKeyValuePair(newKey("key1"), newItem(pString, "value1")) + return p.setKeyValuePair(newKey("key2"), newItem(pString, "value2")), p + }, + "", + `{"key1": "value1", "key2": "value2"}`) } func TestAST_StoreValueWithMultipartKey_CreatesTableHierarchy(t *testing.T) { testAST(t, func() (error, *parser) { p := newParser() - return p.setValue(newItem(pKey, "key1", "key2", "key3"), newItem(pString, "value")), p - }, "", `{"key1": {"key2": {"key3": "value"}}}`) -} - -func TestAST_StoreValueWithMultipartKey_UnderSubtable_CreatesTableHierarchy(t *testing.T) { - testAST(t, func() (error, *parser) { - p := newParser() - p.openTable(newItem(pKey, "tablekey1", "tablekey2")) - return p.setValue(newItem(pKey, "valuekey1", "valuekey2", "valuekey3"), newItem(pString, "value")), p - }, "", `{"tablekey1": {"tablekey2": {"valuekey1": {"valuekey2": {"valuekey3": "value"}}}}}`) + return p.setKeyValuePair(newKey("key1", "key2", "key3"), newItem(pString, "value")), p + }, + "", + `{"key1": {"key2": {"key3": "value"}}}`) } func TestAST_StoreDuplicateKeyInRootTable_ReturnsError(t *testing.T) { testAST(t, func() (error, *parser) { p := newParser() - p.setValue(newItem(pKey, "key"), newItem(pString, "value")) - return p.setValue(newItem(pKey, "key"), newItem(pInteger, 321)), p - }, `Cannot store value: string item already exists at key "key"`, "") + p.setKeyValuePair(newKey("key"), newItem(pString, "value")) + return p.setKeyValuePair(newKey("key"), newItem(pInteger, 321)), p + }, + `invalid key/value pair: string item already exists at key [key]`, + `{"key": "value"}`) +} + +func TestAST_StoreValueWithMultipartKey_UnderSubtable_CreatesTableHierarchy(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.openTable(newKey("tablekey1", "tablekey2")) + return p.setKeyValuePair(newKey("valuekey1", "valuekey2", "valuekey3"), newItem(pString, "value")), p + }, + "", + `{"tablekey1": {"tablekey2": {"valuekey1": {"valuekey2": {"valuekey3": "value"}}}}}`) +} + +func TestAST_StoreKeyPathWherePathContainsNonTableAlready_ReturnsError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.openTable(newKey("key1")) + p.setKeyValuePair(newKey("key2"), newItem(pInteger, 0)) + return p.setKeyValuePair(newKey("key2", "key3"), newItem(pString, "value")), p + }, + `invalid key/value pair: integer item already exists at key [key1->key2]`, + `{"key1": {"key2": 0}}`) } func TestAST_GivenExistingTableAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { testAST(t, func() (error, *parser) { p := newParser() - p.openTable(newItem(pKey, "key1", "key2")) - _, err := p.openTable(newItem(pKey, "key1", "key2")) - return err, p - }, `Cannot create table: table for key "key1"."key2" already exists`, "") + p.openTable(newKey("key1", "key2")) + return p.openTable(newKey("key1", "key2")), p + }, + `invalid table: table item already exists at key [key1->key2]`, + `{"key1": {"key2": {}}}`) } func TestAST_GivenExistingItemAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { testAST(t, func() (error, *parser) { p := newParser() - p.Root["key"] = newItem(pString, "value") - _, err := p.openTable(newItem(pKey, "key")) - return err, p - }, `Cannot create table: string item already exists at key "key"`, "") + p.setKeyValuePair(newKey("key"), newItem(pString, "value")) + return p.openTable(newKey("key")), p + }, + `invalid table: string item already exists at key [key]`, + `{"key": "value"}`) +} + +func TestAST_GivenExistingItemInKeyPath_CreatingTable_ReturnsError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.setKeyValuePair(newKey("key"), newItem(pString, "value")) + return p.openTable(newKey("key", "subkey")), p + }, + `invalid table: string item already exists at key [key]`, + `{"key": "value"}`) +} + +func TestAST_GivenExistingItemAtDeepKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.openTable(newKey("deep", "table")) + p.setKeyValuePair(newKey("key"), newItem(pInteger, 0)) + return p.openTable(newKey("deep", "table", "key")), p + }, + `invalid table: integer item already exists at key [deep->table->key]`, + `{"deep": {"table": {"key": 0}}}`) +} + +func TestAST_GivenExistingItemAtDeepKeyFromSubTable_CreatingTableAtSameKey_ReturnsError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.openTable(newKey("deep", "table")) + p.setKeyValuePair(newKey("key1", "key2"), newItem(pInteger, 0)) + return p.setKeyValuePair(newKey("key1", "key2"), newItem(pBoolean, true)), p + }, + // This test mainly tests the formatting of [deep->table->key1->key2], being a concatenation + // of the currently active table plus the multipart key for setKeyValuePair(). + `invalid key/value pair: integer item already exists at key [deep->table->key1->key2]`, + `{"deep": {"table": {"key1": {"key2": 0}}}}`) +} + +func TestAST_FormattingOfQuotedPathPartInError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.openTable(newKey("must be quoted")) + p.setKeyValuePair(newKey("this one too"), newItem(pInteger, 0)) + return p.setKeyValuePair(newKey("this one too"), newItem(pInteger, 0)), p + }, + `invalid key/value pair: integer item already exists at key ["must be quoted"->"this one too"]`, + `{"must be quoted": {"this one too": 0}}`) +} + +func TestAST_GivenExistingItemAtKey_CreatingArrayOfTablesAtSameKey_ReturnsError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.Root["key"] = newItem(pString, "value") + return p.openArrayOfTables(newKey("key")), p + }, + `invalid table array: string item already exists at key [key]`, + `{"key": "value"}`) +} + +func TestAST_GivenExistingItemInKeyPath_CreatingArrayOfTables_ReturnsError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.Root["key"] = newItem(pString, "value") + return p.openArrayOfTables(newKey("key", "subkey")), p + }, + `invalid table array: string item already exists at key [key]`, + `{"key": "value"}`) } diff --git a/comment.go b/comment.go index d8d54e9..d892f1b 100644 --- a/comment.go +++ b/comment.go @@ -9,9 +9,7 @@ import ( var comment = c.Seq(a.Hash, c.ZeroOrMore(c.Not(a.EndOfLine)), m.Drop(a.EndOfLine)) func (t *parser) startComment(p *parse.API) { - if p.Accept(comment) { - t.addParsedItem(pComment, p.Result().String()) - } else { + if !p.Accept(comment) { p.Expected("comment") } } diff --git a/comment_test.go b/comment_test.go index 5979e29..3b9898a 100644 --- a/comment_test.go +++ b/comment_test.go @@ -5,20 +5,17 @@ import ( ) func TestComment2(t *testing.T) { - for _, test := range []parseTest{ - {``, []string{`Error: unexpected end of file (expected comment) at start of file`}}, - {`#`, []string{`comment("#")`}}, - {`# `, []string{`comment("# ")`}}, - {`# with data`, []string{`comment("# with data")`}}, - {"# ending in EOL & EOF\r\n", []string{`comment("# ending in EOL & EOF")`}}, - {`# \xxx/ \u can't escape/`, []string{`comment("# \\xxx/ \\u can't escape/")`}}, - {"# \tlexe\r accepts embedded ca\r\riage \returns\r\n", []string{ - `comment("# \tlexe\r accepts embedded ca\r\riage \returns")`}}, - {"# with data and newline\ncode continues here", []string{ - `comment("# with data and newline")`, - `Error: unexpected input (expected end of file) at line 2, column 1`}}, + for _, test := range []parseToASTTest{ + {``, `{}`, `unexpected end of file (expected comment) at start of file`}, + {`#`, `{}`, ``}, + {`# `, `{}`, ``}, + {`# with data`, `{}`, ``}, + {"# ending in EOL & EOF\r\n", `{}`, ``}, + {`# \xxx/ \u can't escape/`, `{}`, ``}, + {"# \tlexe\r accepts embedded ca\r\riage \returns\r\n", `{}`, ``}, + {"# with data and newline\ncode continues here", `{}`, `unexpected input (expected end of file) at line 2, column 1`}, } { p := newParser() - testParseHandler(t, p, p.startComment, test) + testParseToAST(t, p, p.startComment, test) } } diff --git a/helpers_test.go b/helpers_test.go index eaa5094..2d9c5b5 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -74,3 +74,30 @@ func testAST(t *testing.T, code func() (error, *parser), expectedError string, e t.Fatalf("Unexpected data after parsing:\nexpected: %s\nactual: %s\n", expectedData, p.Root.String()) } } + +type parseToASTTest struct { + input interface{} + expected string + expectedError string +} + +func testParseToAST(t *testing.T, p *parser, handler parse.Handler, test parseToASTTest) { + var err error + defer func() { + recovered := recover() + if recovered != nil { + err = fmt.Errorf("Panic: %s", recovered.(string)) + } + if err != nil && test.expectedError == "" { + t.Errorf("Unexpected error for input %q: %s", test.input, err) + } else if err != nil && test.expectedError != err.Error() { + t.Errorf("Unexpected error for input %q:\nexpected: %s\nactual: %s\n", test.input, test.expectedError, err.Error()) + } else { + result := p.Root.String() + if test.expected != result { + t.Errorf("Unexpected result for input %q:\nexpected: %s\nactual: %s\n", test.input, test.expected, result) + } + } + }() + err = parse.New(handler)(test.input) +} diff --git a/keyvaluepair.go b/keyvaluepair.go index 9eed3a2..26550f1 100644 --- a/keyvaluepair.go +++ b/keyvaluepair.go @@ -32,6 +32,9 @@ var ( // around dot-separated parts are ignored, however, best practice is to // not use any extraneous blanks. keySeparatorDot = c.Seq(dropBlanks, a.Dot, dropBlanks) + + // Both [tables] and [[arrays of tables]] start with a square open bracket. + startOfTableOrArrayOfTables = a.SquareOpen ) func (t *parser) startKeyValuePair(p *parse.API) { @@ -40,8 +43,15 @@ func (t *parser) startKeyValuePair(p *parse.API) { switch { case p.Peek(a.Hash): p.Handle(t.startComment) + case p.Peek(startOfTableOrArrayOfTables): + p.Handle(t.startTable) case p.Peek(startOfKey): - p.Handle(t.startKey, t.startAssignment, t.startValue) + key, ok := t.parseKey(p, []string{}) + if ok && p.Handle(t.startAssignment) { + if value, ok := t.parseValue(p); ok { + t.setKeyValuePair(key, value) + } + } default: p.ExpectEndOfFile() return @@ -61,58 +71,40 @@ func (t *parser) startKeyValuePair(p *parse.API) { // is to use bare keys except when absolutely necessary. // A bare key must be non-empty, but an empty quoted key is allowed (though // discouraged). -func (t *parser) startKey(p *parse.API) { - var key string +func (t *parser) parseKey(p *parse.API, key []string) ([]string, bool) { + var keyPart string var ok bool switch { case p.Accept(bareKey): - key, ok = p.Result().String(), true + keyPart, ok = p.Result().String(), true case p.Peek(a.SingleQuote): - key, ok = t.parseLiteralString("key", p) + keyPart, ok = t.parseLiteralString("key", p) case p.Peek(a.DoubleQuote): - key, ok = t.parseBasipString("key", p) + keyPart, ok = t.parseBasicString("key", p) default: p.Expected("a key name") - return + return nil, false } - if ok { - t.addParsedItem(pKey, key) - p.Handle(t.endOfKeyOrDot) + if !ok { + return nil, false } + key = append(key, keyPart) + return t.parseEndOfKeyOrDot(p, key) } // Dotted keys are a sequence of bare or quoted keys joined with a dot. // This allows for grouping similar properties together. // Whitespace around dot-separated parts is ignored, however, best // practice is to not use any extraneous whitespace. -func (t *parser) endOfKeyOrDot(p *parse.API) { +func (t *parser) parseEndOfKeyOrDot(p *parse.API, key []string) ([]string, bool) { if p.Accept(keySeparatorDot) { - p.Handle(t.startKey) - return + return t.parseKey(p, key) } - - // TODO not sure if we really need this index. - // Can't we alway simply use the full item list, given that we'll feed all - // results to the parser's table state? - // Do we even need to emit a key here? Shouldn't we just create the - // table structure in the parser, ready for followup calls to fill the data? - keyStart := len(t.Items) - 1 - for keyStart > 0 && t.Items[keyStart-1].Type == pKey { - keyStart-- - } - keyLen := len(t.Items) - keyStart - key := make([]interface{}, keyLen) - for i := 0; i < keyLen; i++ { - key[i] = t.Items[keyStart+i].Data[0].(string) - } - t.Items = t.Items[0:keyStart] - t.addParsedItem(pKey, key...) + return key, true } func (t *parser) startAssignment(p *parse.API) { - if p.Accept(keyAssignment) { - t.addParsedItem(pAssign) - } else { + if !p.Accept(keyAssignment) { p.Expected("a value assignment") } } diff --git a/keyvaluepair_test.go b/keyvaluepair_test.go index 99c4000..f19d93b 100644 --- a/keyvaluepair_test.go +++ b/keyvaluepair_test.go @@ -3,151 +3,133 @@ package toml import "testing" func TestKey(t *testing.T) { - for _, test := range []parseTest{ - {"", []string{`Error: unexpected end of file (expected a key name) at start of file`}}, + for _, test := range []parseToASTTest{ // Bare key tests - {"barekey", []string{`key("barekey")`}}, - {"1234567", []string{`key("1234567")`}}, - {"mix-12_34", []string{`key("mix-12_34")`}}, - {"-hey_good_Lookin123-", []string{`key("-hey_good_Lookin123-")`}}, - {"wrong!", []string{`key("wrong")`, `Error: unexpected input (expected end of file) at line 1, column 6`}}, - {"key1.", []string{`key("key1")`, `Error: unexpected end of file (expected a key name) at line 1, column 6`}}, - {"key1.key2", []string{`key("key1", "key2")`}}, - {"key . with . spaces", []string{`key("key", "with", "spaces")`}}, - {"key \t . \twithtabs\t . \tandspaces", []string{`key("key", "withtabs", "andspaces")`}}, + {"barekey=0", `{"barekey": 0}`, ``}, + {"1234567=0", `{"1234567": 0}`, ``}, + {"mix-12_34=0", `{"mix-12_34": 0}`, ``}, + {"-hey_good_Lookin123-=0", `{"-hey_good_Lookin123-": 0}`, ``}, + {"wrong!=0", `{}`, `unexpected input (expected a value assignment) at line 1, column 6`}, + {"key1.=0", `{}`, `unexpected input (expected a key name) at line 1, column 6`}, + {"key1.key2=0", `{"key1": {"key2": 0}}`, ``}, + {"key . with . spaces=0", `{"key": {"with": {"spaces": 0}}}`, ``}, + {"key \t . \twithtabs\t . \tandspaces=0", `{"key": {"withtabs": {"andspaces": 0}}}`, ``}, // Single quoted key tests - {"''", []string{`key("")`}}, - {"'single quoted'", []string{`key("single quoted")`}}, - {`'escape\s are literal'`, []string{`key("escape\\s are literal")`}}, - {`'"using inner quotes"'`, []string{`key("\"using inner quotes\"")`}}, + {"''=0", `{"": 0}`, ``}, + {"'single quoted'=0", `{"single quoted": 0}`, ``}, + {`'escape\s are literal'=0`, `{"escape\\s are literal": 0}`, ``}, + {`'"using inner quotes"'=0`, `{"\"using inner quotes\"": 0}`, ``}, // Double quoted key tests - {`""`, []string{`key("")`}}, - {`"double quoted"`, []string{`key("double quoted")`}}, - {`"escapes are in\terpreted"`, []string{`key("escapes are in\terpreted")`}}, - {`"using 'inner' \"quotes\""`, []string{`key("using 'inner' \"quotes\"")`}}, + {`""=0`, `{"": 0}`, ``}, + {`"double quoted"=0`, `{"double quoted": 0}`, ``}, + {`"escapes are in\terpreted"=0`, `{"escapes are in\terpreted": 0}`, ``}, + {`"using 'inner' \"quotes\""=0`, `{"using 'inner' \"quotes\"": 0}`, ``}, // Mixed key types - {`this.'i\s'."madness\t".''`, []string{`key("this", "i\\s", "madness\t", "")`}}, + {`this.'i\s'."madness\t".''=0`, `{"this": {"i\\s": {"madness\t": {"": 0}}}}`, ``}, } { p := newParser() - testParseHandler(t, p, p.startKey, test) - } -} - -func TestAssignment(t *testing.T) { - for _, test := range []parseTest{ - {"", []string{`Error: unexpected end of file (expected a value assignment) at start of file`}}, - {"=", []string{`=`}}, - {" \t = \t ", []string{`=`}}, - {" \n = \n ", []string{`Error: unexpected input (expected a value assignment) at start of file`}}, - } { - p := newParser() - testParseHandler(t, p, p.startAssignment, test) + testParseToAST(t, p, p.startKeyValuePair, test) } } func TestKeyValuePair(t *testing.T) { - for _, test := range []parseTest{ - {"", []string{}}, - {" ", []string{}}, - {" \t ", []string{}}, - {" key ", []string{`key("key")`, `Error: unexpected input (expected a value assignment) at line 1, column 5`}}, - {" key \t=", []string{`key("key")`, `=`, `Error: unexpected end of file (expected a value) at line 1, column 8`}}, - {"key = # INVALID", []string{`key("key")`, `=`, `Error: unexpected input (expected a value) at line 1, column 7`}}, - {" key \t =\t \"The Value\" \r\n", []string{`key("key")`, `=`, `"The Value"`}}, - {`3.14159 = "pi"`, []string{`key("3", "14159")`, `=`, `"pi"`}}, - {`"ʎǝʞ" = "value"`, []string{`key("ʎǝʞ")`, `=`, `"value"`}}, - {`key = "value" # This is a comment at the end of a line`, []string{`key("key")`, `=`, `"value"`, `comment("# This is a comment at the end of a line")`}}, - {`another = "# This is not a comment"`, []string{`key("another")`, `=`, `"# This is not a comment"`}}, - {"key1=\"value1\"key2=\"value2\"\r\nkey3a.key3b=\"value3\"", []string{ - `key("key1")`, `=`, `"value1"`, - `key("key2")`, `=`, `"value2"`, - `key("key3a", "key3b")`, `=`, `"value3"`}}, - {"with=\"comments\"# boring \nanother.cool =\"one\" \t # to the end\r\n", []string{ - `key("with")`, `=`, `"comments"`, `comment("# boring ")`, - `key("another", "cool")`, `=`, `"one"`, `comment("# to the end")`}}, + for _, test := range []parseToASTTest{ + {``, `{}`, ``}, + {` `, `{}`, ``}, + {" \t ", `{}`, ``}, + {" key ", `{}`, `unexpected input (expected a value assignment) at line 1, column 5`}, + {" key \t=", `{}`, `unexpected end of file (expected a value) at line 1, column 8`}, + {"key = # INVALID", `{}`, `unexpected input (expected a value) at line 1, column 7`}, + {" key \t =\t \"The Value\" \r\n", `{"key": "The Value"}`, ``}, + {`3.14159 = "pi"`, `{"3": {"14159": "pi"}}`, ``}, + {`"ʎǝʞ" = "value"`, `{"ʎǝʞ": "value"}`, ``}, + {`key = "value" # This is a comment at the end of a line`, `{"key": "value"}`, ``}, + {`another = "# This is not a comment"`, `{"another": "# This is not a comment"}`, ``}, + {"key1=\"value1\"key2=\"value2\"\r\nkey3a.key3b=\"value3\"", `{"key1": "value1", "key2": "value2", "key3a": {"key3b": "value3"}}`, ``}, + {"with=\"comments\"# boring \nanother.cool =\"one\" \t # to the end\r\n", `{"another": {"cool": "one"}, "with": "comments"}`, ``}, } { p := newParser() - testParseHandler(t, p, p.startKeyValuePair, test) + testParseToAST(t, p, p.startKeyValuePair, test) } } func TestKeyValuePair_ForAllTypes(t *testing.T) { - for _, test := range []parseTest{ - {"string='literal'", []string{`key("string")`, `=`, `"literal"`}}, - {"string='''literal\nmulti-line'''", []string{`key("string")`, `=`, `"literal\nmulti-line"`}}, - {`string="basic"`, []string{`key("string")`, `=`, `"basic"`}}, - {"string=\"\"\"basic\nmulti-line\"\"\"", []string{`key("string")`, `=`, `"basic\nmulti-line"`}}, - {"integer=1_234_567", []string{`key("integer")`, `=`, `1234567`}}, - {"integer=42", []string{`key("integer")`, `=`, `42`}}, - {"integer=0x42", []string{`key("integer")`, `=`, `66`}}, - {"integer=0o42", []string{`key("integer")`, `=`, `34`}}, - {"integer=0b101010", []string{`key("integer")`, `=`, `42`}}, - {"float=42.37", []string{`key("float")`, `=`, `42.37`}}, - {"float=42e+37", []string{`key("float")`, `=`, `4.2e+38`}}, - {"float=42.37e-11", []string{`key("float")`, `=`, `4.237e-10`}}, - {"boolean=true", []string{`key("boolean")`, `=`, `true`}}, - {"boolean=false", []string{`key("boolean")`, `=`, `false`}}, - {"date=2019-01-01", []string{`key("date")`, `=`, `2019-01-01`}}, - {"time=15:03:11", []string{`key("time")`, `=`, `15:03:11`}}, - {"datetime=2021-02-01 15:03:11.123", []string{`key("datetime")`, `=`, `2021-02-01 15:03:11.123`}}, - {"offset_datetime=1111-11-11 11:11:11.111111111+11:11", []string{`key("offset_datetime")`, `=`, `1111-11-11T11:11:11.111111111+11:11`}}, - {"static_array=['a', 'static', 'array']", []string{`key("static_array")`, `=`, `["a", "static", "array"]`}}, + for _, test := range []parseToASTTest{ + {"string='literal'", `{"string": "literal"}`, ``}, + {"string='''literal\nmulti-line'''", `{"string": "literal\nmulti-line"}`, ``}, + {`string="basic"`, `{"string": "basic"}`, ``}, + {"string=\"\"\"basic\nmulti-line\"\"\"", `{"string": "basic\nmulti-line"}`, ``}, + {"integer=1_234_567", `{"integer": 1234567}`, ``}, + {"integer=42", `{"integer": 42}`, ``}, + {"integer=0x42", `{"integer": 66}`, ``}, + {"integer=0o42", `{"integer": 34}`, ``}, + {"integer=0b101010", `{"integer": 42}`, ``}, + {"float=42.37", `{"float": 42.37}`, ``}, + {"float=42e+37", `{"float": 4.2e+38}`, ``}, + {"float=42.37e-11", `{"float": 4.237e-10}`, ``}, + {"boolean=true", `{"boolean": true}`, ``}, + {"boolean=false", `{"boolean": false}`, ``}, + {"date=2019-01-01", `{"date": 2019-01-01}`, ``}, + {"time=15:03:11", `{"time": 15:03:11}`, ``}, + {"datetime=2021-02-01 15:03:11.123", `{"datetime": 2021-02-01 15:03:11.123}`, ``}, + {"offset_datetime=1111-11-11 11:11:11.111111111+11:11", `{"offset_datetime": 1111-11-11T11:11:11.111111111+11:11}`, ``}, + {"static_array=['a', 'static', 'array']", `{"static_array": ["a", "static", "array"]}`, ``}, } { p := newParser() - testParseHandler(t, p, p.startKeyValuePair, test) + testParseToAST(t, p, p.startKeyValuePair, test) } } func TestKeyValuePair_ExamplesFromSpecification(t *testing.T) { - for _, test := range []parseTest{ - {"int1 = +99", []string{`key("int1")`, `=`, `99`}}, - {"int2 = 42", []string{`key("int2")`, `=`, `42`}}, - {"int3 = 0", []string{`key("int3")`, `=`, `0`}}, - {"int4 = -17", []string{`key("int4")`, `=`, `-17`}}, - {"int5 = 1_000", []string{`key("int5")`, `=`, `1000`}}, - {"int6 = 5_349_221", []string{`key("int6")`, `=`, `5349221`}}, - {"int7 = 1_2_3_4_5 # VALID but discouraged", []string{`key("int7")`, `=`, `12345`, `comment("# VALID but discouraged")`}}, - {"hex1 = 0xDEADBEEF", []string{`key("hex1")`, `=`, `3735928559`}}, - {"hex2 = 0xdeadbeef", []string{`key("hex2")`, `=`, `3735928559`}}, - {"hex3 = 0xdead_beef", []string{`key("hex3")`, `=`, `3735928559`}}, - {"oct1 = 0o01234567", []string{`key("oct1")`, `=`, `342391`}}, - {"oct2 = 0o755", []string{`key("oct2")`, `=`, `493`}}, - {"bin1 = 0b11010110", []string{`key("bin1")`, `=`, `214`}}, - {"flt1 = +1.0", []string{`key("flt1")`, `=`, `1`}}, - {"flt2 = 3.1415", []string{`key("flt2")`, `=`, `3.1415`}}, - {"flt3 = -0.01", []string{`key("flt3")`, `=`, `-0.01`}}, - {"flt4 = 5e+22", []string{`key("flt4")`, `=`, `5e+22`}}, - {"flt5 = 1e6", []string{`key("flt5")`, `=`, `1e+06`}}, - {"flt6 = -2E-2", []string{`key("flt6")`, `=`, `-0.02`}}, - {"flt7 = 6.626e-34", []string{`key("flt7")`, `=`, `6.626e-34`}}, - {"flt8 = 224_617.445_991_228", []string{`key("flt8")`, `=`, `224617.445991228`}}, - {"sf1 = inf # positive infinity", []string{`key("sf1")`, `=`, `+Inf`, `comment("# positive infinity")`}}, - {"sf2 = +inf # positive infinity", []string{`key("sf2")`, `=`, `+Inf`, `comment("# positive infinity")`}}, - {"sf3 = -inf # negative infinity", []string{`key("sf3")`, `=`, `-Inf`, `comment("# negative infinity")`}}, - {"sf4 = nan # actual sNaN/qNaN encoding is implementation-specific", []string{`key("sf4")`, `=`, `NaN`, `comment("# actual sNaN/qNaN encoding is implementation-specific")`}}, - {"sf5 = +nan # same as `nan`", []string{`key("sf5")`, `=`, `NaN`, "comment(\"# same as `nan`\")"}}, - {"sf6 = -nan # valid, actual encoding is implementation-specific", []string{`key("sf6")`, `=`, `NaN`, `comment("# valid, actual encoding is implementation-specific")`}}, - {"bool1 = true", []string{`key("bool1")`, `=`, `true`}}, - {"bool2 = false", []string{`key("bool2")`, `=`, `false`}}, - {"odt1 = 1979-05-27T07:32:00Z", []string{`key("odt1")`, `=`, `1979-05-27T07:32:00Z`}}, - {"odt2 = 1979-05-27T00:32:00-07:00", []string{`key("odt2")`, `=`, `1979-05-27T00:32:00-07:00`}}, - {"odt3 = 1979-05-27T00:32:00.999999-07:00", []string{`key("odt3")`, `=`, `1979-05-27T00:32:00.999999-07:00`}}, - {"odt4 = 1979-05-27 07:32:00Z", []string{`key("odt4")`, `=`, `1979-05-27T07:32:00Z`}}, - {"ldt1 = 1979-05-27T07:32:00", []string{`key("ldt1")`, `=`, `1979-05-27 07:32:00`}}, - {"ldt2 = 1979-05-27T00:32:00.999999", []string{`key("ldt2")`, `=`, `1979-05-27 00:32:00.999999`}}, - {"ld1 = 1979-05-27", []string{`key("ld1")`, `=`, `1979-05-27`}}, - {"lt1 = 07:32:00", []string{`key("lt1")`, `=`, `07:32:00`}}, - {"lt2 = 00:32:00.999999", []string{`key("lt2")`, `=`, `00:32:00.999999`}}, - {"arr1 = [ 1, 2, 3 ]", []string{`key("arr1")`, `=`, `[1, 2, 3]`}}, - {`arr2 = [ "red", "yellow", "green" ]`, []string{`key("arr2")`, `=`, `["red", "yellow", "green"]`}}, - {`arr3 = [ [ 1, 2 ], [3, 4, 5] ]`, []string{`key("arr3")`, `=`, `[[1, 2], [3, 4, 5]]`}}, - {`arr4 = [ "all", 'strings', """are the same""", '''type''']`, []string{`key("arr4")`, `=`, `["all", "strings", "are the same", "type"]`}}, - {`arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]`, []string{`key("arr5")`, `=`, `[[1, 2], ["a", "b", "c"]]`}}, - {`arr6 = [ 1, 2.0 ] # INVALID`, []string{`key("arr6")`, `=`, `Error: type mismatch in array of integers: found an item of type float at line 1, column 16`}}, - {"arr7 = [\n 1, 2, 3\n]", []string{`key("arr7")`, `=`, `[1, 2, 3]`}}, - {"arr8 = [\n 1,\n 2, # this is ok\n]", []string{`key("arr8")`, `=`, `[1, 2]`}}, + for _, test := range []parseToASTTest{ + {"int1 = +99", `{"int1": 99}`, ``}, + {"int2 = 42", `{"int2": 42}`, ``}, + {"int3 = 0", `{"int3": 0}`, ``}, + {"int4 = -17", `{"int4": -17}`, ``}, + {"int5 = 1_000", `{"int5": 1000}`, ``}, + {"int6 = 5_349_221", `{"int6": 5349221}`, ``}, + {"int7 = 1_2_3_4_5 # VALID but discouraged", `{"int7": 12345}`, ``}, + {"hex1 = 0xDEADBEEF", `{"hex1": 3735928559}`, ``}, + {"hex2 = 0xdeadbeef", `{"hex2": 3735928559}`, ``}, + {"hex3 = 0xdead_beef", `{"hex3": 3735928559}`, ``}, + {"oct1 = 0o01234567", `{"oct1": 342391}`, ``}, + {"oct2 = 0o755", `{"oct2": 493}`, ``}, + {"bin1 = 0b11010110", `{"bin1": 214}`, ``}, + {"flt1 = +1.0", `{"flt1": 1}`, ``}, + {"flt2 = 3.1415", `{"flt2": 3.1415}`, ``}, + {"flt3 = -0.01", `{"flt3": -0.01}`, ``}, + {"flt4 = 5e+22", `{"flt4": 5e+22}`, ``}, + {"flt5 = 1e6", `{"flt5": 1e+06}`, ``}, + {"flt6 = -2E-2", `{"flt6": -0.02}`, ``}, + {"flt7 = 6.626e-34", `{"flt7": 6.626e-34}`, ``}, + {"flt8 = 224_617.445_991_228", `{"flt8": 224617.445991228}`, ``}, + {"sf1 = inf # positive infinity", `{"sf1": +Inf}`, ``}, + {"sf2 = +inf # positive infinity", `{"sf2": +Inf}`, ``}, + {"sf3 = -inf # negative infinity", `{"sf3": -Inf}`, ``}, + {"sf4 = nan # actual sNaN/qNaN encoding is implementation-specific", `{"sf4": NaN}`, ``}, + {"sf5 = +nan # same as `nan`", `{"sf5": NaN}`, ``}, + {"sf6 = -nan # valid, actual encoding is implementation-specific", `{"sf6": NaN}`, ``}, + {"bool1 = true", `{"bool1": true}`, ``}, + {"bool2 = false", `{"bool2": false}`, ``}, + {"odt1 = 1979-05-27T07:32:00Z", `{"odt1": 1979-05-27T07:32:00Z}`, ``}, + {"odt2 = 1979-05-27T00:32:00-07:00", `{"odt2": 1979-05-27T00:32:00-07:00}`, ``}, + {"odt3 = 1979-05-27T00:32:00.999999-07:00", `{"odt3": 1979-05-27T00:32:00.999999-07:00}`, ``}, + {"odt4 = 1979-05-27 07:32:00Z", `{"odt4": 1979-05-27T07:32:00Z}`, ``}, + {"ldt1 = 1979-05-27T07:32:00", `{"ldt1": 1979-05-27 07:32:00}`, ``}, + {"ldt2 = 1979-05-27T00:32:00.999999", `{"ldt2": 1979-05-27 00:32:00.999999}`, ``}, + {"ld1 = 1979-05-27", `{"ld1": 1979-05-27}`, ``}, + {"lt1 = 07:32:00", `{"lt1": 07:32:00}`, ``}, + {"lt2 = 00:32:00.999999", `{"lt2": 00:32:00.999999}`, ``}, + {"arr1 = [ 1, 2, 3 ]", `{"arr1": [1, 2, 3]}`, ``}, + {`arr2 = [ "red", "yellow", "green" ]`, `{"arr2": ["red", "yellow", "green"]}`, ``}, + {`arr3 = [ [ 1, 2 ], [3, 4, 5] ]`, `{"arr3": [[1, 2], [3, 4, 5]]}`, ``}, + {`arr4 = [ "all", 'strings', """are the same""", '''type''']`, `{"arr4": ["all", "strings", "are the same", "type"]}`, ``}, + {`arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]`, `{"arr5": [[1, 2], ["a", "b", "c"]]}`, ``}, + {`arr6 = [ 1, 2.0 ] # INVALID`, `{}`, `type mismatch in array of integers: found an item of type float at line 1, column 16`}, + {"arr7 = [\n 1, 2, 3\n]", `{"arr7": [1, 2, 3]}`, ``}, + {"arr8 = [\n 1,\n 2, # this is ok\n]", `{"arr8": [1, 2]}`, ``}, } { p := newParser() - testParseHandler(t, p, p.startKeyValuePair, test) + testParseToAST(t, p, p.startKeyValuePair, test) } } diff --git a/value.go b/value.go index c712d10..00b325e 100644 --- a/value.go +++ b/value.go @@ -4,27 +4,33 @@ import ( "git.makaay.nl/mauricem/go-parsekit/parse" ) +var ( + detectString = a.SingleQuote.Or(a.DoubleQuote) + detectBoolean = a.Str("true").Or(a.Str("false")) + detectNumberSpecials = c.Any(a.Plus, a.Minus, a.Str("inf"), a.Str("nan")) + detectDateTime = a.Digits.Then(a.Minus.Or(a.Colon)) + detectNumber = a.Digit + detectArray = a.SquareOpen +) + // Values must be of the following types: String, Integer, Float, Boolean, // Datetime, Array, or Inline Table. Unspecified values are invalid. -func (t *parser) startValue(p *parse.API) { +func (t *parser) parseValue(p *parse.API) (*item, bool) { switch { - case p.Peek(c.Any(a.SingleQuote, a.DoubleQuote)): - p.Handle(t.startString) - case p.Peek(a.Runes('t', 'f')): - p.Handle(t.startBoolean) - case p.Peek(a.Plus.Or(a.Minus)): - p.Handle(t.startNumber) - case p.Peek(a.Runes('i', 'n')): - p.Handle(t.startNumber) - case p.Peek(a.Digit): - if p.Peek(a.Digits.Then(a.Minus.Or(a.Colon))) { - p.Handle(t.startDateTime) - } else { - p.Handle(t.startNumber) - } - case p.Peek(a.SquareOpen): - p.Handle(t.startArray) + case p.Peek(detectString): + return t.parseString(p) + case p.Peek(detectBoolean): + return t.parseBoolean(p) + case p.Peek(detectNumberSpecials): + return t.parseNumber(p) + case p.Peek(detectDateTime): + return t.parseDateTime(p) + case p.Peek(detectNumber): + return t.parseNumber(p) + case p.Peek(detectArray): + return t.parseArray(p) default: p.Expected("a value") + return nil, false } } diff --git a/value_array.go b/value_array.go index 459f04a..6ca0d62 100644 --- a/value_array.go +++ b/value_array.go @@ -36,51 +36,45 @@ var ( arrayClose = c.Seq(c.Optional(arraySpace.Then(a.Comma)), arraySpace, a.SquareClose) ) -func (t *parser) startArray(p *parse.API) { +func (t *parser) parseArray(p *parse.API) (*item, bool) { // Check for the start of the array. if !p.Accept(arrayOpen) { p.Expected("an array") - return + return nil, false } - items := []item{} - // Check for an empty array. if p.Accept(arrayClose) { - t.addParsedItem(pStaticArray, items) - return + return newItem(pArray), true } - // Not an empty array, parse the items. + // Not an empty array, parse the array items. + items := []interface{}{} for { - // Check for a valid item. - if !p.Handle(t.startValue) { - return + // Check for a value item. + value, ok := t.parseValue(p) + if !ok { + return nil, false } - // Pop the item from the value parsing and append it to the array items. - parseItems, item := t.Items[0:len(t.Items)-1], t.Items[len(t.Items)-1] - t.Items = parseItems - // Data types may not be mixed (different ways to define strings should be // considered the same type, and so should arrays with different element types). - if len(items) > 0 && item.Type != items[0].Type { - p.Error("type mismatch in array of %ss: found an item of type %s", items[0].Type, item.Type) - return + if len(items) > 0 && value.Type != items[0].(*item).Type { + p.Error("type mismatch in array of %ss: found an item of type %s", items[0].(*item).Type, value.Type) + return nil, false } - items = append(items, item) + items = append(items, value) // Check for the end of the array. if p.Accept(arrayClose) { - t.addParsedItem(pStaticArray, items) - return + return newItem(pArray, items...), true } // Not the end of the array? Then we should find an array separator. if !p.Accept(arraySeparator) { p.Expected("an array separator") - return + return nil, false } } } diff --git a/value_array_test.go b/value_array_test.go index d01767b..8548bd5 100644 --- a/value_array_test.go +++ b/value_array_test.go @@ -2,39 +2,46 @@ package toml import ( "testing" + + "git.makaay.nl/mauricem/go-parsekit/parse" ) +func TestArrayStart(t *testing.T) { + parser := newParser() + wrapper := func(p *parse.API) { parser.parseArray(p) } + testParseToAST(t, parser, wrapper, parseToASTTest{"INVALID", "{}", "unexpected input (expected an array) at start of file"}) +} + func TestArray(t *testing.T) { - for _, test := range []parseTest{ - {"", []string{`Error: unexpected end of file (expected an array) at start of file`}}, - {"[ape", []string{`Error: unexpected input (expected a value) at line 1, column 2`}}, - {"[1", []string{`Error: unexpected end of file (expected an array separator) at line 1, column 3`}}, - {"[]", []string{`[]`}}, - {"[\n]", []string{`[]`}}, - {"[,]", []string{`[]`}}, - {"[ , ]", []string{`[]`}}, - {"[ \n , \r\n ]", []string{`[]`}}, - {"[ \t , \t ]", []string{`[]`}}, - {"[\r\n\r\n , \r\n \t\n]", []string{`[]`}}, - {"[\n#comment on its own line\n]", []string{`[]`}}, - {"[#comment before close\n]", []string{`[]`}}, - {"[,#comment after separator\n]", []string{`[]`}}, - {"[#comment before separator\n,]", []string{`[]`}}, - {"[#comment before value\n1]", []string{`[1]`}}, - {"[1#comment after value\n]", []string{`[1]`}}, - {"[1\n#comment on its own line after value\n]", []string{`[1]`}}, - {"[1#comment 1\n#comment 2\n#comment 3\n , \n2]", []string{`[1, 2]`}}, - {"[1]", []string{`[1]`}}, - {"[1,0x2, 0b11, 0o4]", []string{`[1, 2, 3, 4]`}}, - {"[0.1,0.2,3e-1,0.04e+1, nan, inf]", []string{`[0.1, 0.2, 0.3, 0.4, NaN, +Inf]`}}, - {"[\n\t 'a', \"b\", '''c''', \"\"\"d\ne\"\"\",\n \t]", []string{`["a", "b", "c", "d\ne"]`}}, - {`[1, 2, 3, "four"]`, []string{`Error: type mismatch in array of integers: found an item of type string at line 1, column 17`}}, - {`[[1],['a']]`, []string{`[[1], ["a"]]`}}, - {`[[[],[]],[]]`, []string{`[[[], []], []]`}}, - {"[\r\n\r\n \t\n [\r\n\r\n\t [],[\t]],\t\n[]\t \t \n ]", []string{`[[[], []], []]`}}, - {`[[1],'a']`, []string{`Error: type mismatch in array of static arrays: found an item of type string at line 1, column 9`}}, + for _, test := range []parseToASTTest{ + {"x=[ape", `{}`, `unexpected input (expected a value) at line 1, column 4`}, + {"x=[1", `{}`, `unexpected end of file (expected an array separator) at line 1, column 5`}, + {"x=[]", `{"x": []}`, ``}, + {"x=[\n]", `{"x": []}`, ``}, + {"x=[,]", `{"x": []}`, ``}, + {"x=[ , ]", `{"x": []}`, ``}, + {"x=[ \n , \r\n ]", `{"x": []}`, ``}, + {"x=[ \t , \t ]", `{"x": []}`, ``}, + {"x=[\r\n\r\n , \r\n \t\n]", `{"x": []}`, ``}, + {"x=[\n#comment on its own line\n]", `{"x": []}`, ``}, + {"x=[#comment before close\n]", `{"x": []}`, ``}, + {"x=[,#comment after separator\n]", `{"x": []}`, ``}, + {"x=[#comment before separator\n,]", `{"x": []}`, ``}, + {"x=[#comment before value\n1]", `{"x": [1]}`, ``}, + {"x=[1#comment after value\n]", `{"x": [1]}`, ``}, + {"x=[1\n#comment on its own line after value\n]", `{"x": [1]}`, ``}, + {"x=[1#comment 1\n#comment 2\n#comment 3\n , \n2]", `{"x": [1, 2]}`, ``}, + {"x=[1]", `{"x": [1]}`, ``}, + {"x=[1,0x2, 0b11, 0o4]", `{"x": [1, 2, 3, 4]}`, ``}, + {"x=[0.1,0.2,3e-1,0.04e+1, nan, inf]", `{"x": [0.1, 0.2, 0.3, 0.4, NaN, +Inf]}`, ``}, + {"x=[\n\t 'a', \"b\", '''c''', \"\"\"d\ne\"\"\",\n \t]", `{"x": ["a", "b", "c", "d\ne"]}`, ``}, + {`x=[1, 2, 3, "four"]`, `{}`, `type mismatch in array of integers: found an item of type string at line 1, column 19`}, + {`x=[[1],['a']]`, `{"x": [[1], ["a"]]}`, ``}, + {`x=[[[],[]],[]]`, `{"x": [[[], []], []]}`, ``}, + {"x=[\r\n\r\n \t\n [\r\n\r\n\t [],[\t]],\t\n[]\t \t \n ]", `{"x": [[[], []], []]}`, ``}, + {`x=[[1],'a']`, `{}`, `type mismatch in array of static arrays: found an item of type string at line 1, column 11`}, } { p := newParser() - testParseHandler(t, p, p.startArray, test) + testParseToAST(t, p, p.startKeyValuePair, test) } } diff --git a/value_boolean.go b/value_boolean.go index d050e75..b906c38 100644 --- a/value_boolean.go +++ b/value_boolean.go @@ -4,14 +4,18 @@ import ( "git.makaay.nl/mauricem/go-parsekit/parse" ) +var falseItem = newItem(pBoolean, false) +var trueItem = newItem(pBoolean, true) + // Booleans are just the tokens you're used to. Always lowercase. -func (t *parser) startBoolean(p *parse.API) { +func (t *parser) parseBoolean(p *parse.API) (*item, bool) { switch { case p.Accept(a.Str("true")): - t.addParsedItem(pBoolean, true) + return trueItem, true case p.Accept(a.Str("false")): - t.addParsedItem(pBoolean, false) + return falseItem, true default: p.Expected("true or false") + return nil, false } } diff --git a/value_boolean_test.go b/value_boolean_test.go index 0eb14a6..5394a3d 100644 --- a/value_boolean_test.go +++ b/value_boolean_test.go @@ -2,19 +2,26 @@ package toml import ( "testing" + + "git.makaay.nl/mauricem/go-parsekit/parse" ) +func TestBooleanStart(t *testing.T) { + parser := newParser() + wrapper := func(p *parse.API) { parser.parseBoolean(p) } + testParseToAST(t, parser, wrapper, parseToASTTest{"INVALID", "{}", "unexpected input (expected true or false) at start of file"}) +} + func TestBoolean(t *testing.T) { - for _, test := range []parseTest{ - {``, []string{`Error: unexpected end of file (expected true or false) at start of file`}}, - {`true`, []string{`true`}}, - {`false`, []string{`false`}}, - {`yes`, []string{`Error: unexpected input (expected true or false) at start of file`}}, - {`no`, []string{`Error: unexpected input (expected true or false) at start of file`}}, - {`1`, []string{`Error: unexpected input (expected true or false) at start of file`}}, - {`0`, []string{`Error: unexpected input (expected true or false) at start of file`}}, + for _, test := range []parseToASTTest{ + {`x=true`, `{"x": true}`, ``}, + {`x=false`, `{"x": false}`, ``}, + {`x=yes`, `{}`, `unexpected input (expected a value) at line 1, column 3`}, + {`x=no`, `{}`, `unexpected input (expected a value) at line 1, column 3`}, + {`x=1`, `{"x": 1}`, ``}, + {`x=0`, `{"x": 0}`, ``}, } { p := newParser() - testParseHandler(t, p, p.startBoolean, test) + testParseToAST(t, p, p.startKeyValuePair, test) } } diff --git a/value_datetime.go b/value_datetime.go index 9b64c07..93df40a 100644 --- a/value_datetime.go +++ b/value_datetime.go @@ -73,18 +73,19 @@ var ( datetime = c.Any(offsetDateTime, localDateTime, localDate, localTime) ) -func (t *parser) startDateTime(p *parse.API) { +func (t *parser) parseDateTime(p *parse.API) (*item, bool) { if !p.Accept(datetime) { p.Expected("a date and/or time") - return + return nil, false } tokens := p.Result().Tokens() valueType := getDateTimeValueType(&tokens) input, value, err := getDateTimeValue(&tokens) if err == nil { - t.addParsedItem(valueType, value) + return newItem(valueType, value), true } else { - p.Error("Cannot parse value 0%s: %s", input, err) + p.Error("invalid date/time value %s: %s", input, err) + return nil, false } } diff --git a/value_datetime_test.go b/value_datetime_test.go index 5454fda..7168dd0 100644 --- a/value_datetime_test.go +++ b/value_datetime_test.go @@ -2,32 +2,39 @@ package toml import ( "testing" + + "git.makaay.nl/mauricem/go-parsekit/parse" ) +func TestDateTimeStart(t *testing.T) { + parser := newParser() + wrapper := func(p *parse.API) { parser.parseDateTime(p) } + testParseToAST(t, parser, wrapper, parseToASTTest{"INVALID", "{}", "unexpected input (expected a date and/or time) at start of file"}) +} + func TestDateTime(t *testing.T) { - for _, test := range []parseTest{ - {``, []string{`Error: unexpected end of file (expected a date and/or time) at start of file`}}, - {`1979-05-27`, []string{`1979-05-27`}}, - {`00:00:00`, []string{`00:00:00`}}, - {`23:59:59`, []string{`23:59:59`}}, - {`12:10:08.12121212121212`, []string{`12:10:08.121212121`}}, - {`1979-05-28T01:01:01`, []string{`1979-05-28 01:01:01`}}, - {`1979-05-28 01:01:01`, []string{`1979-05-28 01:01:01`}}, - {`1979-05-27T07:32:00Z`, []string{`1979-05-27T07:32:00Z`}}, - {`1979-05-27 07:33:00Z`, []string{`1979-05-27T07:33:00Z`}}, - {`1979-05-27 07:34:00+07:00`, []string{`1979-05-27T07:34:00+07:00`}}, - {`1979-05-27 07:34:00-07:00`, []string{`1979-05-27T07:34:00-07:00`}}, - {`1985-03-31 23:59:59+00:00`, []string{`1985-03-31T23:59:59Z`}}, - {`2000-09-10 00:00:00.000000000+00:00`, []string{`2000-09-10T00:00:00Z`}}, - {`2003-11-01 01:02:03.999999999999+10:00`, []string{`2003-11-01T01:02:03.999999999+10:00`}}, - {`2007-12-25 04:00:04.1111-10:30`, []string{`2007-12-25T04:00:04.1111-10:30`}}, - {`2021-02-01 10:10:10.101010203040Z`, []string{`2021-02-01T10:10:10.101010203Z`}}, + for _, test := range []parseToASTTest{ + {`x=1979-05-27`, `{"x": 1979-05-27}`, ``}, + {`x=00:00:00`, `{"x": 00:00:00}`, ``}, + {`x=23:59:59`, `{"x": 23:59:59}`, ``}, + {`x=12:10:08.12121212121212`, `{"x": 12:10:08.121212121}`, ``}, + {`x=1979-05-28T01:01:01`, `{"x": 1979-05-28 01:01:01}`, ``}, + {`x=1979-05-28 01:01:01`, `{"x": 1979-05-28 01:01:01}`, ``}, + {`x=1979-05-27T07:32:00Z`, `{"x": 1979-05-27T07:32:00Z}`, ``}, + {`x=1979-05-27 07:33:00Z`, `{"x": 1979-05-27T07:33:00Z}`, ``}, + {`x=1979-05-27 07:34:00+07:00`, `{"x": 1979-05-27T07:34:00+07:00}`, ``}, + {`x=1979-05-27 07:34:00-07:00`, `{"x": 1979-05-27T07:34:00-07:00}`, ``}, + {`x=1985-03-31 23:59:59+00:00`, `{"x": 1985-03-31T23:59:59Z}`, ``}, + {`x=2000-09-10 00:00:00.000000000+00:00`, `{"x": 2000-09-10T00:00:00Z}`, ``}, + {`x=2003-11-01 01:02:03.999999999999+10:00`, `{"x": 2003-11-01T01:02:03.999999999+10:00}`, ``}, + {`x=2007-12-25 04:00:04.1111-10:30`, `{"x": 2007-12-25T04:00:04.1111-10:30}`, ``}, + {`x=2021-02-01 10:10:10.101010203040Z`, `{"x": 2021-02-01T10:10:10.101010203Z}`, ``}, // TODO ugly column, should be at start or at the actual wrong part - {`2000-13-01`, []string{`Error: Cannot parse value 02000-13-01: parsing time "2000-13-01": month out of range at line 1, column 11`}}, - {`2000-02-31`, []string{`Error: Cannot parse value 02000-02-31: parsing time "2000-02-31": day out of range at line 1, column 11`}}, - {`25:01:01`, []string{`Error: Cannot parse value 025:01:01: parsing time "25:01:01": hour out of range at line 1, column 9`}}, + {`x=2000-13-01`, `{}`, `invalid date/time value 2000-13-01: parsing time "2000-13-01": month out of range at line 1, column 13`}, + {`x=2000-02-31`, `{}`, `invalid date/time value 2000-02-31: parsing time "2000-02-31": day out of range at line 1, column 13`}, + {`x=25:01:01`, `{}`, `invalid date/time value 25:01:01: parsing time "25:01:01": hour out of range at line 1, column 11`}, } { p := newParser() - testParseHandler(t, p, p.startDateTime, test) + testParseToAST(t, p, p.startKeyValuePair, test) } } diff --git a/value_number.go b/value_number.go index 40c4a5b..f3d6248 100644 --- a/value_number.go +++ b/value_number.go @@ -65,28 +65,28 @@ var ( nan = a.Signed(a.Str("nan")) ) -func (t *parser) startNumber(p *parse.API) { +func (t *parser) parseNumber(p *parse.API) (*item, bool) { switch { case p.Accept(tok.Float64(nil, float)): - t.addParsedItem(pFloat, p.Result().Value(0).(float64)) + return newItem(pFloat, p.Result().Value(0).(float64)), true case p.Accept(nan): - t.addParsedItem(pFloat, math.NaN()) + return newItem(pFloat, math.NaN()), true case p.Accept(inf): if p.Result().Rune(0) == '-' { - t.addParsedItem(pFloat, math.Inf(-1)) - } else { - t.addParsedItem(pFloat, math.Inf(+1)) + return newItem(pFloat, math.Inf(-1)), true } + return newItem(pFloat, math.Inf(+1)), true case p.Accept(a.Zero): - p.Handle(t.startIntegerStartingWithZero) + return t.parseIntegerStartingWithZero(p) case p.Accept(tok.Int64(nil, integer)): - t.addParsedItem(pInteger, p.Result().Value(0).(int64)) + return newItem(pInteger, p.Result().Value(0).(int64)), true default: p.Expected("a number") + return nil, false } } -func (t *parser) startIntegerStartingWithZero(p *parse.API) { +func (t *parser) parseIntegerStartingWithZero(p *parse.API) (*item, bool) { var value int64 var err error switch { @@ -97,12 +97,11 @@ func (t *parser) startIntegerStartingWithZero(p *parse.API) { case p.Accept(binary): value, err = strconv.ParseInt(p.Result().Value(0).(string), 2, 64) default: - t.addParsedItem(pInteger, int64(0)) - return + return newItem(pInteger, int64(0)), true } if err == nil { - t.addParsedItem(pInteger, value) - } else { - p.Error("Cannot parse value 0%s: %s", p.Result().String(), err) + return newItem(pInteger, value), true } + p.Error("invalid integer value 0%s: %s", p.Result().String(), err) + return nil, false } diff --git a/value_number_test.go b/value_number_test.go index 243e954..e3e8525 100644 --- a/value_number_test.go +++ b/value_number_test.go @@ -2,106 +2,105 @@ package toml import ( "testing" + + "git.makaay.nl/mauricem/go-parsekit/parse" ) +func TestNumberStart(t *testing.T) { + parser := newParser() + wrapper := func(p *parse.API) { parser.parseNumber(p) } + testParseToAST(t, parser, wrapper, parseToASTTest{"INVALID", "{}", "unexpected input (expected a number) at start of file"}) +} + func TestInteger(t *testing.T) { - for _, test := range []parseTest{ - {``, []string{`Error: unexpected end of file (expected a number) at start of file`}}, + for _, test := range []parseToASTTest{ // Decimal - {`0`, []string{`0`}}, - {`+0`, []string{`0`}}, - {`-0`, []string{`0`}}, - {`1`, []string{`1`}}, - {`42`, []string{`42`}}, - {`+99`, []string{`99`}}, - {`-17`, []string{`-17`}}, - {`1234`, []string{`1234`}}, - {`_`, []string{`Error: unexpected input (expected a number) at start of file`}}, - {`1_`, []string{`1`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, - {`1_000`, []string{`1000`}}, - {`5_349_221`, []string{`5349221`}}, - {`1_2_3_4_5`, []string{`12345`}}, - {`9_223_372_036_854_775_807`, []string{`9223372036854775807`}}, - {`9_223_372_036_854_775_808`, []string{ + {`x=0`, `{"x": 0}`, ``}, + {`x=+0`, `{"x": 0}`, ``}, + {`x=-0`, `{"x": 0}`, ``}, + {`x=1`, `{"x": 1}`, ``}, + {`x=42`, `{"x": 42}`, ``}, + {`x=+99`, `{"x": 99}`, ``}, + {`x=-17`, `{"x": -17}`, ``}, + {`x=1234`, `{"x": 1234}`, ``}, + {`x=_`, `{}`, `unexpected input (expected a value) at line 1, column 3`}, + {`x=1_`, `{"x": 1}`, `unexpected end of file (expected a value assignment) at line 1, column 5`}, + {`x=1_000`, `{"x": 1000}`, ``}, + {`x=5_349_221`, `{"x": 5349221}`, ``}, + {`x=1_2_3_4_5`, `{"x": 12345}`, ``}, + {`x=9_223_372_036_854_775_807`, `{"x": 9223372036854775807}`, ``}, + {`x=9_223_372_036_854_775_808`, `{}`, `Panic: Handler error: MakeInt64Token cannot handle input "9223372036854775808": ` + - `strconv.ParseInt: parsing "9223372036854775808": value out of range (only use a ` + - `type conversion token maker, when the input has been validated on beforehand)`}}, - {`-9_223_372_036_854_775_808`, []string{`-9223372036854775808`}}, + `strconv.ParseInt: parsing "9223372036854775808": value out of range ` + + `(only use a type conversion token maker, when the input has been validated on beforehand)`}, + {`x=-9_223_372_036_854_775_808`, `{"x": -9223372036854775808}`, ``}, // TODO make the use of the same kind of handling for panics and for errors between parsekit and TOML. - {`-9_223_372_036_854_775_809`, []string{ + {`x=-9_223_372_036_854_775_809`, `{}`, `Panic: Handler error: MakeInt64Token cannot handle input "-9223372036854775809": ` + - `strconv.ParseInt: parsing "-9223372036854775809": value out of range (only use a ` + - `type conversion token maker, when the input has been validated on beforehand)`}}, + `strconv.ParseInt: parsing "-9223372036854775809": value out of range ` + + `(only use a type conversion token maker, when the input has been validated on beforehand)`}, // Hexadecimal - {`0x0`, []string{`0`}}, - {`0x1`, []string{`1`}}, - {`0x01`, []string{`1`}}, - {`0x00fF`, []string{`255`}}, - {`0xf_f`, []string{`255`}}, - {`0x0_0_f_f`, []string{`255`}}, - {`0xdead_beef`, []string{`3735928559`}}, - {`0xgood_beef`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, - {`0x7FFFFFFFFFFFFFFF`, []string{`9223372036854775807`}}, - {`0x8000000000000000`, []string{ - `Error: Cannot parse value 0x8000000000000000: strconv.ParseInt: parsing "8000000000000000": ` + - `value out of range at line 1, column 19`}}, + {`x=0x0`, `{"x": 0}`, ``}, + {`x=0x1`, `{"x": 1}`, ``}, + {`x=0x01`, `{"x": 1}`, ``}, + {`x=0x00fF`, `{"x": 255}`, ``}, + {`x=0xf_f`, `{"x": 255}`, ``}, + {`x=0x0_0_f_f`, `{"x": 255}`, ``}, + {`x=0xdead_beef`, `{"x": 3735928559}`, ``}, + {`x=0xgood_beef`, `{"x": 0}`, `unexpected end of file (expected a value assignment) at line 1, column 14`}, + {`x=0x7FFFFFFFFFFFFFFF`, `{"x": 9223372036854775807}`, ``}, + {`x=0x8000000000000000`, `{}`, `invalid integer value 0x8000000000000000: strconv.ParseInt: parsing "8000000000000000": value out of range at line 1, column 21`}, //Octal - {`0o0`, []string{`0`}}, - {`0o1`, []string{`1`}}, - {`0o01`, []string{`1`}}, - {`0o10`, []string{`8`}}, - {`0o1_6`, []string{`14`}}, - {`0o0_0_1_1_1`, []string{`73`}}, - {`0o9`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, - {`0o777777777777777777777`, []string{`9223372036854775807`}}, - {`0o1000000000000000000000`, []string{ - `Error: Cannot parse value 0o1000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000": ` + - `value out of range at line 1, column 25`}}, + {`x=0o0`, `{"x": 0}`, ``}, + {`x=0o1`, `{"x": 1}`, ``}, + {`x=0o01`, `{"x": 1}`, ``}, + {`x=0o10`, `{"x": 8}`, ``}, + {`x=0o1_6`, `{"x": 14}`, ``}, + {`x=0o0_0_1_1_1`, `{"x": 73}`, ``}, + {`x=0o9`, `{"x": 0}`, `unexpected end of file (expected a value assignment) at line 1, column 6`}, + {`x=0o777777777777777777777`, `{"x": 9223372036854775807}`, ``}, + {`x=0o1000000000000000000000`, `{}`, `invalid integer value 0o1000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000": value out of range at line 1, column 27`}, // Binary - {`0b0`, []string{`0`}}, - {`0b1`, []string{`1`}}, - {`0b01`, []string{`1`}}, - {`0b10`, []string{`2`}}, - {`0b0100`, []string{`4`}}, - {`0b00001000`, []string{`8`}}, - {`0b0001_0000`, []string{`16`}}, - {`0b9`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, - {`0b1_1_0_1_1`, []string{`27`}}, - {`0b11111111_11111111`, []string{`65535`}}, - {`0b01111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111`, []string{`9223372036854775807`}}, - {`0b10000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000`, []string{ - `Error: Cannot parse value 0b1000000000000000000000000000000000000000000000000000000000000000: ` + - `strconv.ParseInt: parsing "1000000000000000000000000000000000000000000000000000000000000000": ` + - `value out of range at line 1, column 74`}}, + {`x=0b0`, `{"x": 0}`, ``}, + {`x=0b1`, `{"x": 1}`, ``}, + {`x=0b01`, `{"x": 1}`, ``}, + {`x=0b10`, `{"x": 2}`, ``}, + {`x=0b0100`, `{"x": 4}`, ``}, + {`x=0b00001000`, `{"x": 8}`, ``}, + {`x=0b0001_0000`, `{"x": 16}`, ``}, + {`x=0b9`, `{"x": 0}`, `unexpected end of file (expected a value assignment) at line 1, column 6`}, + {`x=0b1_1_0_1_1`, `{"x": 27}`, ``}, + {`x=0b11111111_11111111`, `{"x": 65535}`, ``}, + {`x=0b01111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111`, `{"x": 9223372036854775807}`, ``}, + {`x=0b10000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000`, `{}`, `invalid integer value 0b1000000000000000000000000000000000000000000000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000000000000000000000000000000000000000000000": value out of range at line 1, column 76`}, } { p := newParser() - testParseHandler(t, p, p.startNumber, test) + testParseToAST(t, p, p.startKeyValuePair, test) } } func TestFloat(t *testing.T) { - for _, test := range []parseTest{ - {``, []string{`Error: unexpected end of file (expected a number) at start of file`}}, - {`0.0`, []string{`0`}}, - {`+0.0`, []string{`0`}}, - {`-0.0`, []string{`-0`}}, - {`+1.0`, []string{`1`}}, - {`3.1415`, []string{`3.1415`}}, - {`-0.01`, []string{`-0.01`}}, - {`5e+22`, []string{`5e+22`}}, - {`1E6`, []string{`1e+06`}}, - {`-2E-2`, []string{`-0.02`}}, - {`6.626e-34`, []string{`6.626e-34`}}, - {`224_617.445_991_228`, []string{`224617.445991228`}}, - {`12_345.111_222e+1_2_3`, []string{`1.2345111222e+127`}}, - {`+nan`, []string{`NaN`}}, - {`-nan`, []string{`NaN`}}, - {`nan`, []string{`NaN`}}, - {`inf`, []string{`+Inf`}}, - {`+inf`, []string{`+Inf`}}, - {`-inf`, []string{`-Inf`}}, + for _, test := range []parseToASTTest{ + {`x=0.0`, `{"x": 0}`, ``}, + {`x=+0.0`, `{"x": 0}`, ``}, + {`x=-0.0`, `{"x": -0}`, ``}, + {`x=+1.0`, `{"x": 1}`, ``}, + {`x=3.1415`, `{"x": 3.1415}`, ``}, + {`x=-0.01`, `{"x": -0.01}`, ``}, + {`x=5e+22`, `{"x": 5e+22}`, ``}, + {`x=1E6`, `{"x": 1e+06}`, ``}, + {`x=-2E-2`, `{"x": -0.02}`, ``}, + {`x=6.626e-34`, `{"x": 6.626e-34}`, ``}, + {`x=224_617.445_991_228`, `{"x": 224617.445991228}`, ``}, + {`x=12_345.111_222e+1_2_3`, `{"x": 1.2345111222e+127}`, ``}, + {`x=+nan`, `{"x": NaN}`, ``}, + {`x=-nan`, `{"x": NaN}`, ``}, + {`x=nan`, `{"x": NaN}`, ``}, + {`x=inf`, `{"x": +Inf}`, ``}, + {`x=+inf`, `{"x": +Inf}`, ``}, + {`x=-inf`, `{"x": -Inf}`, ``}, } { p := newParser() - testParseHandler(t, p, p.startNumber, test) + testParseToAST(t, p, p.startKeyValuePair, test) } } diff --git a/value_string.go b/value_string.go index 837f26c..eb34d7f 100644 --- a/value_string.go +++ b/value_string.go @@ -43,19 +43,25 @@ var ( // There are four ways to express strings: basic, multi-line basic, literal and // multi-line literal. All strings must contain only valid UTF-8 characters. -func (t *parser) startString(p *parse.API) { +func (t *parser) parseString(p *parse.API) (*item, bool) { + var value string + var ok bool switch { case p.Peek(doubleQuote3): - p.Handle(t.startMultiLineBasipString) + value, ok = t.parseMultiLineBasicString(p) case p.Peek(a.DoubleQuote): - p.Handle(t.startBasipString) + value, ok = t.parseBasicString("string value", p) case p.Peek(singleQuote3): - p.Handle(t.startMultiLineLiteralString) + value, ok = t.parseMultiLineLiteralString(p) case p.Peek(a.SingleQuote): - p.Handle(t.startLiteralString) + value, ok = t.parseLiteralString("string value", p) default: p.Expected("a string value") } + if ok { + return newItem(pString, value), ok + } + return nil, false } // Specific handling of input for basic strings. @@ -69,13 +75,7 @@ func (t *parser) startString(p *parse.API) { // • No additional \escape sequences are allowed. What the spec say about this: // "All other escape sequences [..] are reserved and, if used, TOML should // produce an error."" -func (t *parser) startBasipString(p *parse.API) { - if str, ok := t.parseBasipString("basic string", p); ok { - t.addParsedItem(pString, str) - } -} - -func (t *parser) parseBasipString(name string, p *parse.API) (string, bool) { +func (t *parser) parseBasicString(name string, p *parse.API) (string, bool) { if !p.Accept(a.DoubleQuote) { p.Expected(`opening quotation marks`) return "", false @@ -112,12 +112,6 @@ func (t *parser) parseBasipString(name string, p *parse.API) (string, bool) { // • Like basic strings, they must appear on a single line. // // • Control characters other than tab are not permitted in a literal string. -func (t *parser) startLiteralString(p *parse.API) { - if str, ok := t.parseLiteralString("literal string", p); ok { - t.addParsedItem(pString, str) - } -} - func (t *parser) parseLiteralString(name string, p *parse.API) (string, bool) { if !p.Accept(a.SingleQuote) { p.Expected("opening single quote") @@ -168,10 +162,10 @@ func (t *parser) parseLiteralString(name string, p *parse.API) (string, bool) { // "line ending backslash". When the last non-whitespace character on a line is // a \, it will be trimmed along with all whitespace (including newlines) up to // the next non-whitespace character or closing delimiter. -func (t *parser) startMultiLineBasipString(p *parse.API) { +func (t *parser) parseMultiLineBasicString(p *parse.API) (string, bool) { if !p.Accept(doubleQuote3.Then(newline.Optional())) { p.Expected("opening three quotation marks") - return + return "", false } sb := &strings.Builder{} for { @@ -180,25 +174,24 @@ func (t *parser) startMultiLineBasipString(p *parse.API) { sb.WriteString("\n") case p.Peek(controlCharacter): p.Error("invalid character in multi-line basic string: %q (must be escaped)", p.Result().Rune(0)) - return + return sb.String(), false case p.Accept(tok.StrInterpreted(nil, c.OneOrMore(validEscape))): sb.WriteString(p.Result().Value(0).(string)) case p.Accept(lineEndingBackslash): // NOOP, the line-ending backslash sequence is skipped. case p.Peek(a.Backslash): p.Error("invalid escape sequence") - return + return sb.String(), false case p.Accept(m.Drop(doubleQuote3)): - t.addParsedItem(pString, sb.String()) - return + return sb.String(), true case p.Accept(a.ValidRune): sb.WriteString(p.Result().String()) case p.Peek(a.InvalidRune): p.Error("invalid UTF8 rune") - return + return sb.String(), false default: p.Expected("closing three quotation marks") - return + return sb.String(), false } } } @@ -216,32 +209,31 @@ func (t *parser) startMultiLineBasipString(p *parse.API) { // sense for their platform. // // • Control characters other than tab and newline are not permitted in a multi-line literal string. -func (t *parser) startMultiLineLiteralString(p *parse.API) { +func (t *parser) parseMultiLineLiteralString(p *parse.API) (string, bool) { if !p.Accept(singleQuote3.Then(newline.Optional())) { p.Expected("opening three single quotes") - return + return "", false } sb := &strings.Builder{} for { switch { case p.Accept(m.Drop(singleQuote3)): - t.addParsedItem(pString, sb.String()) - return + return sb.String(), true case p.Accept(a.Tab): sb.WriteString("\t") case p.Accept(newline): sb.WriteString("\n") case p.Peek(controlCharacter): p.Error("invalid character in literal string: %q (no control chars allowed, except for tab and newline)", p.Result().Rune(0)) - return + return sb.String(), false case p.Accept(a.ValidRune): sb.WriteString(p.Result().String()) case p.Peek(a.InvalidRune): p.Error("invalid UTF8 rune") - return + return sb.String(), false default: p.Expected("closing three single quotes") - return + return sb.String(), false } } } diff --git a/value_string_test.go b/value_string_test.go index dca2695..a8a7143 100644 --- a/value_string_test.go +++ b/value_string_test.go @@ -6,91 +6,85 @@ import ( ) func TestString(t *testing.T) { - for _, test := range []parseTest{ - {``, []string{`Error: unexpected end of file (expected a string value) at start of file`}}, - {`no start quote"`, []string{`Error: unexpected input (expected a string value) at start of file`}}, - {`"basic s\tring"`, []string{`"basic s\tring"`}}, - {"\"\"\"\n basic multi-line\n string value\n\"\"\"", []string{`" basic multi-line\n string value\n"`}}, - {`'literal s\tring'`, []string{`"literal s\\tring"`}}, - {"'''\n literal multi-line\n string value\n'''", []string{`" literal multi-line\n string value\n"`}}, + for _, test := range []parseToASTTest{ + {`x=no start quote"`, `{}`, `unexpected input (expected a value) at line 1, column 3`}, + {`x="basic s\tring"`, `{"x": "basic s\tring"}`, ``}, + {"x=\"\"\"\n basic multi-line\n string value\n\"\"\"", `{"x": " basic multi-line\n string value\n"}`, ``}, + {`x='literal s\tring'`, `{"x": "literal s\\tring"}`, ``}, + {"x='''\n literal multi-line\n string value\n'''", `{"x": " literal multi-line\n string value\n"}`, ``}, } { p := newParser() - testParseHandler(t, p, p.startString, test) + testParseToAST(t, p, p.startKeyValuePair, test) } } func TestBasipString(t *testing.T) { - for _, test := range []parseTest{ - {``, []string{`Error: unexpected end of file (expected opening quotation marks) at start of file`}}, - {`no start quote"`, []string{`Error: unexpected input (expected opening quotation marks) at start of file`}}, - {`"no end quote`, []string{`Error: unexpected end of file (expected closing quotation marks) at line 1, column 14`}}, - {`""`, []string{`""`}}, - {`"simple string"`, []string{`"simple string"`}}, - {`"with\tsome\r\nvalid escapes\b"`, []string{`"with\tsome\r\nvalid escapes\b"`}}, - {`"with an \invalid escape"`, []string{`Error: invalid escape sequence at line 1, column 10`}}, - {`"A cool UTF8 ƃuıɹʇs"`, []string{`"A cool UTF8 ƃuıɹʇs"`}}, - {`"A string with UTF8 escape \u2318"`, []string{`"A string with UTF8 escape ⌘"`}}, - {"\"Invalid character for UTF \xcd\"", []string{`Error: invalid UTF8 rune at line 1, column 28`}}, - {"\"Character that mus\t be escaped\"", []string{`Error: invalid character in basic string: '\t' (must be escaped) at line 1, column 20`}}, - {"\"Character that must be escaped \u0000\"", []string{`Error: invalid character in basic string: '\x00' (must be escaped) at line 1, column 33`}}, - {"\"Character that must be escaped \x7f\"", []string{`Error: invalid character in basic string: '\u007f' (must be escaped) at line 1, column 33`}}, + for _, test := range []parseToASTTest{ + {`x="no end quote`, `{}`, `unexpected end of file (expected closing quotation marks) at line 1, column 16`}, + {`x=""`, `{"x": ""}`, ``}, + {`x="simple string"`, `{"x": "simple string"}`, ``}, + {`x="with\tsome\r\nvalid escapes\b"`, `{"x": "with\tsome\r\nvalid escapes\b"}`, ``}, + {`x="with an \invalid escape"`, `{}`, `invalid escape sequence at line 1, column 12`}, + {`x="A cool UTF8 ƃuıɹʇs"`, `{"x": "A cool UTF8 ƃuıɹʇs"}`, ``}, + {`x="A string with UTF8 escape \u2318"`, `{"x": "A string with UTF8 escape ⌘"}`, ``}, + {"x=\"Invalid character for UTF \xcd\"", `{}`, `invalid UTF8 rune at line 1, column 30`}, + {"x=\"Character that mus\t be escaped\"", `{}`, `invalid character in string value: '\t' (must be escaped) at line 1, column 22`}, + {"x=\"Character that must be escaped \u0000\"", `{}`, `invalid character in string value: '\x00' (must be escaped) at line 1, column 35`}, + {"x=\"Character that must be escaped \x7f\"", `{}`, `invalid character in string value: '\u007f' (must be escaped) at line 1, column 35`}, } { p := newParser() - testParseHandler(t, p, p.startBasipString, test) + testParseToAST(t, p, p.startKeyValuePair, test) } } func TestMultiLineBasipString(t *testing.T) { - for _, test := range []parseTest{ - {``, []string{`Error: unexpected end of file (expected opening three quotation marks) at start of file`}}, - {`"""missing close quote""`, []string{`Error: unexpected end of file (expected closing three quotation marks) at line 1, column 25`}}, - {`""""""`, []string{`""`}}, - {"\"\"\"\n\"\"\"", []string{`""`}}, - {"\"\"\"\r\n\r\n\"\"\"", []string{`"\n"`}}, - {`"""\"\"\"\""""`, []string{`"\"\"\"\""`}}, - {"\"\"\"\nThe quick brown \\\n\n\n \t fox jumps over \\\n\t the lazy dog.\\\n \"\"\"", []string{`"The quick brown fox jumps over the lazy dog."`}}, - {"\"\"\"No control chars \f allowed\"\"\"", []string{`Error: invalid character in multi-line basic string: '\f' (must be escaped) at line 1, column 21`}}, - {"\"\"\"Escaping control chars\\nis valid\"\"\"", []string{`"Escaping control chars\nis valid"`}}, - {"\"\"\"Invalid escaping \\is not allowed\"\"\"", []string{`Error: invalid escape sequence at line 1, column 21`}}, - {"\"\"\"Invalid rune \xcd\"\"\"", []string{`Error: invalid UTF8 rune at line 1, column 17`}}, + for _, test := range []parseToASTTest{ + {`x="""missing close quote""`, `{}`, `unexpected end of file (expected closing three quotation marks) at line 1, column 27`}, + {`x=""""""`, `{"x": ""}`, ``}, + {"x=\"\"\"\n\"\"\"", `{"x": ""}`, ``}, + {"x=\"\"\"\r\n\r\n\"\"\"", `{"x": "\n"}`, ``}, + {`x="""\"\"\"\""""`, `{"x": "\"\"\"\""}`, ``}, + {"x=\"\"\"\nThe quick brown \\\n\n\n \t fox jumps over \\\n\t the lazy dog.\\\n \"\"\"", `{"x": "The quick brown fox jumps over the lazy dog."}`, ``}, + {"x=\"\"\"No control chars \f allowed\"\"\"", `{}`, `invalid character in multi-line basic string: '\f' (must be escaped) at line 1, column 23`}, + {"x=\"\"\"Escaping control chars\\nis valid\"\"\"", `{"x": "Escaping control chars\nis valid"}`, ``}, + {"x=\"\"\"Invalid escaping \\is not allowed\"\"\"", `{}`, `invalid escape sequence at line 1, column 23`}, + {"x=\"\"\"Invalid rune \xcd\"\"\"", `{}`, `invalid UTF8 rune at line 1, column 19`}, } { p := newParser() - testParseHandler(t, p, p.startMultiLineBasipString, test) + testParseToAST(t, p, p.startKeyValuePair, test) } } func TestLiteralString(t *testing.T) { - for _, test := range []parseTest{ - {``, []string{`Error: unexpected end of file (expected opening single quote) at start of file`}}, - {`'missing close quote`, []string{`Error: unexpected end of file (expected closing single quote) at line 1, column 21`}}, - {`''`, []string{`""`}}, - {`'simple'`, []string{`"simple"`}}, - {`'C:\Users\nodejs\templates'`, []string{`"C:\\Users\\nodejs\\templates"`}}, - {`'\\ServerX\admin$\system32\'`, []string{`"\\\\ServerX\\admin$\\system32\\"`}}, - {`'Tom "Dubs" Preston-Werner'`, []string{`"Tom \"Dubs\" Preston-Werner"`}}, - {`'<\i\c*\s*>'`, []string{`"<\\i\\c*\\s*>"`}}, - {"'No cont\rol chars allowed'", []string{`Error: invalid character in literal string: '\r' (no control chars allowed, except for tab) at line 1, column 9`}}, - {"'Except\tfor\ttabs'", []string{`"Except\tfor\ttabs"`}}, - {"'Invalid rune \xcd'", []string{`Error: invalid UTF8 rune at line 1, column 15`}}, + for _, test := range []parseToASTTest{ + {`x='missing close quote`, `{}`, `unexpected end of file (expected closing single quote) at line 1, column 23`}, + {`x=''`, `{"x": ""}`, ``}, + {`x='simple'`, `{"x": "simple"}`, ``}, + {`x='C:\Users\nodejs\templates'`, `{"x": "C:\\Users\\nodejs\\templates"}`, ``}, + {`x='\\ServerX\admin$\system32\'`, `{"x": "\\\\ServerX\\admin$\\system32\\"}`, ``}, + {`x='Tom "Dubs" Preston-Werner'`, `{"x": "Tom \"Dubs\" Preston-Werner"}`, ``}, + {`x='<\i\c*\s*>'`, `{"x": "<\\i\\c*\\s*>"}`, ``}, + {"x='No cont\rol chars allowed'", `{}`, `invalid character in string value: '\r' (no control chars allowed, except for tab) at line 1, column 11`}, + {"x='Except\tfor\ttabs'", `{"x": "Except\tfor\ttabs"}`, ``}, + {"x='Invalid rune \xcd'", `{}`, `invalid UTF8 rune at line 1, column 17`}, } { p := newParser() - testParseHandler(t, p, p.startLiteralString, test) + testParseToAST(t, p, p.startKeyValuePair, test) } } func TestMultiLineLiteralString(t *testing.T) { - for _, test := range []parseTest{ - {``, []string{`Error: unexpected end of file (expected opening three single quotes) at start of file`}}, - {`'''missing close quote''`, []string{`Error: unexpected end of file (expected closing three single quotes) at line 1, column 25`}}, - {`''''''`, []string{`""`}}, - {"'''\n'''", []string{`""`}}, - {`'''I [dw]on't need \d{2} apples'''`, []string{`"I [dw]on't need \\d{2} apples"`}}, - {"'''\nThere can\nbe newlines\r\nand \ttabs!\r\n'''", []string{`"There can\nbe newlines\nand \ttabs!\n"`}}, - {"'''No other \f control characters'''", []string{`Error: invalid character in literal string: '\f' (no control chars allowed, except for tab and newline) at line 1, column 13`}}, - {"'''No invalid runes allowed \xcd'''", []string{"Error: invalid UTF8 rune at line 1, column 29"}}, + for _, test := range []parseToASTTest{ + {`x='''missing close quote''`, `{}`, `unexpected end of file (expected closing three single quotes) at line 1, column 27`}, + {`x=''''''`, `{"x": ""}`, ``}, + {"x='''\n'''", `{"x": ""}`, ``}, + {`x='''I [dw]on't need \d{2} apples'''`, `{"x": "I [dw]on't need \\d{2} apples"}`, ``}, + {"x='''\nThere can\nbe newlines\r\nand \ttabs!\r\n'''", `{"x": "There can\nbe newlines\nand \ttabs!\n"}`, ``}, + {"x='''No other \f control characters'''", `{}`, `invalid character in literal string: '\f' (no control chars allowed, except for tab and newline) at line 1, column 15`}, + {"x='''No invalid runes allowed \xcd'''", `{}`, `invalid UTF8 rune at line 1, column 31`}, } { p := newParser() - testParseHandler(t, p, p.startMultiLineLiteralString, test) + testParseToAST(t, p, p.startKeyValuePair, test) } } @@ -99,8 +93,8 @@ func TestBasipStringWithUnescapedControlCharacters(t *testing.T) { // The missing one (\x7f) is covered in the previous test. for i := 0x00; i <= 0x1F; i++ { p := newParser() - input := fmt.Sprintf(`"%c"`, rune(i)) - expected := fmt.Sprintf(`Error: invalid character in basic string: %q (must be escaped) at line 1, column 2`, rune(i)) - testParseHandler(t, p, p.startString, parseTest{input, []string{expected}}) + input := fmt.Sprintf(`x="%c"`, rune(i)) + expected := fmt.Sprintf(`invalid character in string value: %q (must be escaped) at line 1, column 4`, rune(i)) + testParseToAST(t, p, p.startKeyValuePair, parseToASTTest{input, "{}", expected}) } } diff --git a/value_table.go b/value_table.go index 4e17265..92017fb 100644 --- a/value_table.go +++ b/value_table.go @@ -86,11 +86,32 @@ var ( // [[fruit.variety]] // name = "plantain" tableArrayOpen = c.Seq(dropBlanks, a.SquareOpen, a.SquareOpen, dropBlanks) - tableArrayClose = c.Seq(dropBlanks, a.SquareClose, a.SquareClose, a.EndOfLine.Or(comment)) + tableArrayClose = c.Seq(dropBlanks, a.SquareClose, a.SquareClose, dropBlanks, a.EndOfLine.Or(comment)) + + // Inline tables provide a more compact syntax for expressing tables. + // They are especially useful for grouped data that can otherwise quickly + // become verbose. Inline tables are enclosed in curly braces { and }. + // Within the braces, zero or more comma separated key/value pairs may appear. + // Key/value pairs take the same form as key/value pairs in standard tables. + // All value types are allowed, including inline tables. + // + // Inline tables are intended to appear on a single line. No newlines are + // allowed between the curly braces unless they are valid within a value. + // Even so, it is strongly discouraged to break an inline table onto multiple + // lines. If you find yourself gripped with this desire, it means you should + // be using standard tables. + // + // name = { first = "Tom", last = "Preston-Werner" } + // point = { x = 1, y = 2 } + // animal = { type.name = "pug" } + inlineTableOpen = c.Seq(dropBlanks, a.CurlyOpen, dropBlanks) + inlineTableClose = c.Seq(dropBlanks, a.CurlyClose, dropBlanks, a.EndOfLine.Or(comment)) ) func (t *parser) startTable(p *parse.API) { switch { + case p.Accept(tableArrayOpen): + p.Handle(t.startArrayOfTables) case p.Accept(tableOpen): p.Handle(t.startPlainTable) default: @@ -98,14 +119,30 @@ func (t *parser) startTable(p *parse.API) { } } -func (t *parser) startPlainTable(p *parse.API) { - if !p.Handle(t.startKey) { - return +func (t *parser) startArrayOfTables(p *parse.API) { + if key, ok := t.parseKey(p, []string{}); ok { + if !p.Accept(tableArrayClose) { + p.Expected("closing ']]' for array of tables name") + return + } + if err := t.openArrayOfTables(key); err != nil { + p.Error("%s", err) + return + } + p.Handle(t.startKeyValuePair) + } +} + +func (t *parser) startPlainTable(p *parse.API) { + if key, ok := t.parseKey(p, []string{}); ok { + if !p.Accept(tableClose) { + p.Expected("closing ']' for table name") + return + } + if err := t.openTable(key); err != nil { + p.Error("%s", err) + return + } + p.Handle(t.startKeyValuePair) } - if !p.Accept(tableClose) { - p.Expected("closing ']' for table name") - } - key := t.Items[0] - t.Items = t.Items[1:] - t.openTable(key) } diff --git a/value_table_test.go b/value_table_test.go index 3095c6e..aff7efa 100644 --- a/value_table_test.go +++ b/value_table_test.go @@ -4,20 +4,71 @@ import ( "testing" ) -func TestTable(t *testing.T) { +func TestTableKey(t *testing.T) { for _, test := range []parseTest{ {"", []string{`Error: unexpected end of file (expected a table) at start of file`}}, {"[", []string{`Error: unexpected end of file (expected a key name) at line 1, column 2`}}, {" \t [", []string{`Error: unexpected end of file (expected a key name) at line 1, column 5`}}, {" \t [key", []string{`Error: unexpected end of file (expected closing ']' for table name) at line 1, column 8`}}, - {" \t [key.", []string{`key("key")`, `Error: unexpected end of file (expected a key name) at line 1, column 9`}}, + {" \t [key.", []string{`Error: unexpected end of file (expected a key name) at line 1, column 9`}}, {" \t [key.'sub key'", []string{`Error: unexpected end of file (expected closing ']' for table name) at line 1, column 18`}}, {" \t [key.'sub key' \t]", []string{}}, {" \t [key.'sub key' \t] \t ", []string{}}, {" \t [key.'sub key' \t] \t \n", []string{}}, + {" \t [key.'sub key' \t]# with comment\n", []string{}}, {" \t [key.'sub key' \t] \t # with comment\n", []string{}}, } { p := newParser() testParseHandler(t, p, p.startTable, test) } } + +func TestTable(t *testing.T) { + for _, test := range []parseToASTTest{ + {"[a]", `{"a": {}}`, ``}, + {"['a key']", `{"a key": {}}`, ``}, + {"[\"a key\"]", `{"a key": {}}`, ``}, + {"[\"a key\".'sub'.key]", `{"a key": {"sub": {"key": {}}}}`, ``}, + {"[a]\nx=1234", `{"a": {"x": 1234}}`, ``}, + {"[a]\nx=1234\ny='string'", `{"a": {"x": 1234, "y": "string"}}`, ``}, + {"[a]\n[b]", `{"a": {}, "b": {}}`, ``}, + {"[a]\n[a.b]\n[a.b.c]\n[a.d]\nx=1", `{"a": {"b": {"c": {}}, "d": {"x": 1}}}`, ``}, + {"[a]\n[b] #another table \na=1\n", `{"a": {}, "b": {"a": 1}}`, ``}, + {"[a]\nx=1\ny=2\n[b] #another table \nx=1\ny=2021-01-01", `{"a": {"x": 1, "y": 2}, "b": {"x": 1, "y": 2021-01-01}}`, ``}, + {"[a]\nx=1\ny=2\n[a.b] #subtable \nx=1\ny=2021-01-01", `{"a": {"b": {"x": 1, "y": 2021-01-01}, "x": 1, "y": 2}}`, ``}, + } { + p := newParser() + testParseToAST(t, p, p.startTable, test) + } +} + +func TestArrayOfTableKey(t *testing.T) { + for _, test := range []parseTest{ + {"[[", []string{`Error: unexpected end of file (expected a key name) at line 1, column 3`}}, + {" \t [[", []string{`Error: unexpected end of file (expected a key name) at line 1, column 6`}}, + {" \t [[key", []string{`Error: unexpected end of file (expected closing ']]' for array of tables name) at line 1, column 9`}}, + {" \t [[key.", []string{`Error: unexpected end of file (expected a key name) at line 1, column 10`}}, + {" \t [[key.'sub key'", []string{`Error: unexpected end of file (expected closing ']]' for array of tables name) at line 1, column 19`}}, + {" \t [[key.'sub key' \t]]", []string{}}, + {" \t [[key.'sub key' \t]] \t ", []string{}}, + {" \t [[key.'sub key' \t]] \t \n", []string{}}, + {" \t [[key.'sub key' \t]]# with comment\n", []string{}}, + {" \t [[key.'sub key' \t]] \t # with comment\n", []string{}}, + } { + p := newParser() + testParseHandler(t, p, p.startTable, test) + } +} + +func TestArrayOfTables(t *testing.T) { + for _, test := range []parseToASTTest{ + {"[[a]]", `{"a": [{}]}`, ``}, + {"[[a]]\n[['a']]\n[[\"a\"]]", `{"a": [{}, {}, {}]}`, ``}, + {"[[a]]\n[['a']]\n[b]\n[[\"a\"]]", `{"a": [{}, {}, {}], "b": {}}`, ``}, + {"[[a]]\nx=1\n[['a']]\nx=2\ny=3\n[[\"a\"]]", `{"a": [{"x": 1}, {"x": 2, "y": 3}, {}]}`, ``}, + {"[a]\n[[a.b]]\nx=1\n[[a.b]]\nx=2\n[a.c]\ny=1234", `{"a": {"b": [{"x": 1}, {"x": 2}], "c": {"y": 1234}}}`, ``}, + } { + p := newParser() + testParseToAST(t, p, p.startTable, test) + } +} diff --git a/value_test.go b/value_test.go index d2c7115..f141074 100644 --- a/value_test.go +++ b/value_test.go @@ -1,46 +1,42 @@ package toml -import ( - "testing" -) - -func TestValue(t *testing.T) { - for _, test := range []parseTest{ - {``, []string{`Error: unexpected end of file (expected a value) at start of file`}}, - {`"basic s\tring value"`, []string{`"basic s\tring value"`}}, - {`'literal s\tring value'`, []string{`"literal s\\tring value"`}}, - {"\"\"\"basic multi-line\nstring value\"\"\"", []string{`"basic multi-line\nstring value"`}}, - {"'''literal multi-line\nstring value'''", []string{`"literal multi-line\nstring value"`}}, - {"true", []string{`true`}}, - {"false", []string{`false`}}, - {"0", []string{`0`}}, - {"+0", []string{`0`}}, - {"-0", []string{`0`}}, - {"0.0", []string{`0`}}, - {"+0.0", []string{`0`}}, - {"-0.0", []string{`-0`}}, - {"1234", []string{`1234`}}, - {"-1234", []string{`-1234`}}, - {"+9_8_7.6_5_4e-321", []string{`9.8765e-319`}}, - {"-1_234.5678e-33", []string{`-1.2345678e-30`}}, - {"inf", []string{`+Inf`}}, - {"+inf", []string{`+Inf`}}, - {"-inf", []string{`-Inf`}}, - {"nan", []string{`NaN`}}, - {"+nan", []string{`NaN`}}, - {"-nan", []string{`NaN`}}, - {"2019-06-19", []string{`2019-06-19`}}, - {"08:38:54", []string{`08:38:54`}}, - {"08:38:54.8765487654876", []string{`08:38:54.876548765`}}, - {"2019-06-19 08:38:54", []string{`2019-06-19 08:38:54`}}, - {"2019-06-19T08:38:54", []string{`2019-06-19 08:38:54`}}, - {"2019-06-19T08:38:54.88888", []string{`2019-06-19 08:38:54.88888`}}, - {"1979-05-27T07:32:00Z", []string{`1979-05-27T07:32:00Z`}}, - {"1979-05-27T00:32:00-07:00", []string{`1979-05-27T00:32:00-07:00`}}, - {"1979-05-27T00:32:00.999999-07:00", []string{`1979-05-27T00:32:00.999999-07:00`}}, - {"[1,2,3]", []string{`[1, 2, 3]`}}, - } { - p := newParser() - testParseHandler(t, p, p.startValue, test) - } -} +// func TestValue(t *testing.T) { +// for _, test := range []parseTest{ +// {``, []string{`Error: unexpected end of file (expected a value) at start of file`}}, +// {`"basic s\tring value"`, []string{`"basic s\tring value"`}}, +// {`'literal s\tring value'`, []string{`"literal s\\tring value"`}}, +// {"\"\"\"basic multi-line\nstring value\"\"\"", []string{`"basic multi-line\nstring value"`}}, +// {"'''literal multi-line\nstring value'''", []string{`"literal multi-line\nstring value"`}}, +// {"true", []string{`true`}}, +// {"false", []string{`false`}}, +// {"0", []string{`0`}}, +// {"+0", []string{`0`}}, +// {"-0", []string{`0`}}, +// {"0.0", []string{`0`}}, +// {"+0.0", []string{`0`}}, +// {"-0.0", []string{`-0`}}, +// {"1234", []string{`1234`}}, +// {"-1234", []string{`-1234`}}, +// {"+9_8_7.6_5_4e-321", []string{`9.8765e-319`}}, +// {"-1_234.5678e-33", []string{`-1.2345678e-30`}}, +// {"inf", []string{`+Inf`}}, +// {"+inf", []string{`+Inf`}}, +// {"-inf", []string{`-Inf`}}, +// {"nan", []string{`NaN`}}, +// {"+nan", []string{`NaN`}}, +// {"-nan", []string{`NaN`}}, +// {"2019-06-19", []string{`2019-06-19`}}, +// {"08:38:54", []string{`08:38:54`}}, +// {"08:38:54.8765487654876", []string{`08:38:54.876548765`}}, +// {"2019-06-19 08:38:54", []string{`2019-06-19 08:38:54`}}, +// {"2019-06-19T08:38:54", []string{`2019-06-19 08:38:54`}}, +// {"2019-06-19T08:38:54.88888", []string{`2019-06-19 08:38:54.88888`}}, +// {"1979-05-27T07:32:00Z", []string{`1979-05-27T07:32:00Z`}}, +// {"1979-05-27T00:32:00-07:00", []string{`1979-05-27T00:32:00-07:00`}}, +// {"1979-05-27T00:32:00.999999-07:00", []string{`1979-05-27T00:32:00.999999-07:00`}}, +// {"[1,2,3]", []string{`[1, 2, 3]`}}, +// } { +// p := newParser() +// testParseHandler(t, p, p.startValue, test) +// } +// }