From 688894dbf258b95cd60405092c19387433bba2c3 Mon Sep 17 00:00:00 2001 From: Maurice Makaay Date: Wed, 26 Jun 2019 13:57:22 +0000 Subject: [PATCH] Extracted the toml AST to its own subpackage. Preparation for having encoder and decoder as well, not all inside the main toml dir. --- ast/ast.go | 203 +++++++++++++++++++++++++++++++++++++++++++++ ast/ast_test.go | 198 +++++++++++++++++++++++++++++++++++++++++++ ast/string.go | 49 +++++++++++ ast/string_test.go | 37 +++++++++ 4 files changed, 487 insertions(+) create mode 100644 ast/ast.go create mode 100644 ast/ast_test.go create mode 100644 ast/string.go create mode 100644 ast/string_test.go diff --git a/ast/ast.go b/ast/ast.go new file mode 100644 index 0000000..644783c --- /dev/null +++ b/ast/ast.go @@ -0,0 +1,203 @@ +package ast + +import ( + "fmt" + "regexp" + "strings" +) + +// item represents a TOML item. +type item struct { + Type valueType + Data []interface{} +} + +// table represents a TOML table. +type table map[string]*item + +// valueType identifies the type of a TOML value. +type valueType string + +const ( + tString valueType = "string" // "various", 'types', """of""", '''strings''' + tInteger valueType = "integer" // 12345, 0xffee12a0, 0o0755, 0b101101011 + tFloat valueType = "float" // 10.1234, 143E-12, 43.28377e+4, +inf, -inf, nan + tBoolean valueType = "boolean" // true or false + tOffsetDateTime valueType = "offset datetime" // 2019-06-18 10:32:15.173645362+0200 + tLocalDateTime valueType = "datetime" // 2018-12-25 12:12:18.876772533 + tLocalDate valueType = "date" // 2017-05-17 + tLocalTime valueType = "time" // 23:01:22 + tArrayOfTables valueType = "array" // defined using an [[array.of.tables]] + tArray valueType = "static array" // defined using ["an", "inline", "array"] + tTable valueType = "table" // defined using { "inline" = "table" } or [standard.table] +) + +// newItem instantiates a new item struct. +func newItem(valueType valueType, data ...interface{}) *item { + return &item{Type: valueType, Data: data} +} + +// newKey instantiates a new key. +func newKey(key ...string) []string { + return key +} + +// parser holds the state for the TOML parser. All parsing functions are +// methods of this struct. +type parser struct { + Root table // the root-level TOML table (each TOML doc is implicitly a table) + Current table // the currently active TOML table + CurrentKey []string // the key for the currently active TOML table +} + +func newParser() *parser { + p := &parser{Root: make(table)} + p.Current = p.Root + return p +} + +func (t *parser) setKeyValuePair(key []string, value *item) error { + // First make sure the table structure for storing the value exists. + node, lastKeyPart, err := t.makeTablePath(key) + if err != nil { + return fmt.Errorf("invalid key/value pair: %s", err) + } + // Check if the key is still free for use. + if existing, ok := node[lastKeyPart]; ok { + path := t.formatKeyPath(key, len(key)-1) + return fmt.Errorf("invalid key/value pair: %s item already exists at key %s", existing.Type, path) + } + // It is, store the value in the table. + node[lastKeyPart] = value + return nil +} + +func (t *parser) openTable(key []string) error { + t.CurrentKey = nil + t.Current = t.Root + // Go over all requested levels of the key. For all levels, except the last + // one, it is okay if a table already exists. For at least the last level, + // no table or value must exist, because that would mean we are overwriting + // an existing key/value pair, which is not allowed. + node, lastKeyPart, err := t.makeTablePath(key) + if err != nil { + return fmt.Errorf("invalid table: %s", err) + } + // Check if the key is still free for use. + if existing, ok := node[lastKeyPart]; ok { + path := t.formatKeyPath(key, len(key)-1) + return fmt.Errorf("invalid table: %s item already exists at key %s", existing.Type, path) + } + // The subtable does not exist yet. Create the subtable. + subTable := make(table) + node[lastKeyPart] = newItem(tTable, subTable) + node = subTable + + // From here on, key/value pairs are added to the newly defined table. + t.Current = node + t.CurrentKey = key + return nil +} + +func (t *parser) openArrayOfTables(key []string) error { + t.CurrentKey = nil + t.Current = t.Root + // Go over all requested levels of the key. For all levels, except the last + // one, it is okay if a table already exists. For the last level, either + // no item must exist (in which case a table array will be created), or a + // table array must exist. + // Other cases would mean we are overwriting an existing key/value pair, + // which is not allowed. + node, lastKeyPart, err := t.makeTablePath(key) + if err != nil { + return fmt.Errorf("invalid table array: %s", err) + } + // At the last key position, there must be either no value yet, or the + // existing value must be a table array. Other values are invalid. + if existing, ok := node[lastKeyPart]; ok { + if existing.Type != tArrayOfTables { + path := t.formatKeyPath(key, len(key)-1) + return fmt.Errorf("invalid table array: %s item already exists at key %s", existing.Type, path) + } + // A table array exists. Add a new table to this array. + array := node[lastKeyPart] + subTable := make(table) + tables := array.Data + tables = append(tables, newItem(tTable, subTable)) + array.Data = tables + node = subTable + } else { + // No value exists at the defined key path. Create a new table array. + subTable := make(table) + node[lastKeyPart] = newItem(tArrayOfTables, newItem(tTable, subTable)) + node = subTable + } + + // From here on, key/value pairs are added to the newly defined table. + t.Current = node + t.CurrentKey = key + return nil +} + +func (t *parser) makeTablePath(key []string) (table, string, error) { + node := t.Current + for i, keyPart := range key { + // Arrived at the last key part? Then the path towards that key is + // setup correctly. Return the last part, so the caller can use it. + isLast := i == len(key)-1 + if isLast { + return node, keyPart, nil + } + if subItem, ok := node[keyPart]; ok { + // You cannot overwrite an already defined key, regardless its value. + // When a value already exists at the current key, this can only be a table. + if subItem.Type != tTable { + path := t.formatKeyPath(key, i) + return nil, "", fmt.Errorf("%s item already exists at key %s", subItem.Type, path) + } + // All is okay, traverse to the subtable. + node = subItem.Data[0].(table) + } else { + // The subtable does not exist yet. Create the subtable. + subTable := make(table) + node[keyPart] = newItem(tTable, subTable) + node = subTable + } + } + panic("makeTablePath(): empty key provided; a key must have at least one key part") +} + +func (t *parser) formatKeyPath(key []string, end int) string { + var sb strings.Builder + sb.WriteRune('[') + if t.CurrentKey != nil { + for i, keyPart := range t.CurrentKey { + if i > 0 { + sb.WriteString("->") + } + sb.WriteString(formatKeyName(keyPart)) + } + } + for i, keyPart := range key { + if t.CurrentKey != nil || i > 0 { + sb.WriteString("->") + } + sb.WriteString(formatKeyName(keyPart)) + if i == end { + break + } + } + sb.WriteRune(']') + return sb.String() +} + +func formatKeyName(key string) string { + if ok, _ := regexp.Match(`^\w+$`, []byte(key)); ok { + return key + } + return fmt.Sprintf("%q", key) +} + +func (t table) String() string { + return newItem(tTable, t).String() +} diff --git a/ast/ast_test.go b/ast/ast_test.go new file mode 100644 index 0000000..a94ae55 --- /dev/null +++ b/ast/ast_test.go @@ -0,0 +1,198 @@ +package ast + +import ( + "testing" +) + +func Test_ConstructSlightlyComplexStructure(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.setKeyValuePair(newKey("ding"), newItem(tInteger, 10)) + p.setKeyValuePair(newKey("dong"), newItem(tString, "not a song")) + p.openTable(newKey("key1", "key2 a")) + p.setKeyValuePair(newKey("dooh"), newItem(tBoolean, true)) + p.setKeyValuePair(newKey("dah"), newItem(tBoolean, false)) + p.openTable(newKey("key1", "key2 b")) + p.setKeyValuePair(newKey("dieh"), newItem(tFloat, 1.111)) + p.setKeyValuePair(newKey("duh"), newItem(tFloat, 1.18e-12)) + p.setKeyValuePair(newKey("foo", "bar"), newItem(tArrayOfTables, newItem(tInteger, 1), newItem(tInteger, 2))) + p.openArrayOfTables(newKey("aaah", "table array")) + p.setKeyValuePair(newKey("a"), newItem(tFloat, 1.234)) + p.openArrayOfTables(newKey("aaah", "table array")) + p.setKeyValuePair(newKey("b"), newItem(tFloat, 2.345)) + p.setKeyValuePair(newKey("c"), newItem(tString, "bingo!")) + p.openArrayOfTables(newKey("aaah", "table array")) + return nil, p + }, + "", + `{"aaah": {"table array": [{"a": 1.234}, {"b": 2.345, "c": "bingo!"}, {}]}, `+ + `"ding": 10, "dong": "not a song", `+ + `"key1": {"key2 a": {"dah": false, "dooh": true}, "key2 b": {"dieh": 1.111, "duh": 1.18e-12, "foo": {"bar": [1, 2]}}}}`) +} + +func Test_EmptyKeyForCreatingTablePath_Panics(t *testing.T) { + defer func() { + r := recover() + if r.(string) != "makeTablePath(): empty key provided; a key must have at least one key part" { + t.Fatalf("Did not get the expected panic message") + } + }() + p := newParser() + p.openTable(newKey()) +} + +func Test_StoreValueInRootTable(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.setKeyValuePair(newKey("key1"), newItem(tString, "value1")) + return p.setKeyValuePair(newKey("key2"), newItem(tString, "value2")), p + }, + "", + `{"key1": "value1", "key2": "value2"}`) +} + +func Test_StoreValueWithMultipartKey_CreatesTableHierarchy(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + return p.setKeyValuePair(newKey("key1", "key2", "key3"), newItem(tString, "value")), p + }, + "", + `{"key1": {"key2": {"key3": "value"}}}`) +} + +func Test_StoreDuplicateKeyInRootTable_ReturnsError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.setKeyValuePair(newKey("key"), newItem(tString, "value")) + return p.setKeyValuePair(newKey("key"), newItem(tInteger, 321)), p + }, + `invalid key/value pair: string item already exists at key [key]`, + `{"key": "value"}`) +} + +func Test_StoreValueWithMultipartKey_UnderSubtable_CreatesTableHierarchy(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.openTable(newKey("tablekey1", "tablekey2")) + return p.setKeyValuePair(newKey("valuekey1", "valuekey2", "valuekey3"), newItem(tString, "value")), p + }, + "", + `{"tablekey1": {"tablekey2": {"valuekey1": {"valuekey2": {"valuekey3": "value"}}}}}`) +} + +func Test_StoreKeyPathWherePathContainsNonTableAlready_ReturnsError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.openTable(newKey("key1")) + p.setKeyValuePair(newKey("key2"), newItem(tInteger, 0)) + return p.setKeyValuePair(newKey("key2", "key3"), newItem(tString, "value")), p + }, + `invalid key/value pair: integer item already exists at key [key1->key2]`, + `{"key1": {"key2": 0}}`) +} + +func Test_GivenExistingTableAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.openTable(newKey("key1", "key2")) + return p.openTable(newKey("key1", "key2")), p + }, + `invalid table: table item already exists at key [key1->key2]`, + `{"key1": {"key2": {}}}`) +} + +func Test_GivenExistingItemAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.setKeyValuePair(newKey("key"), newItem(tString, "value")) + return p.openTable(newKey("key")), p + }, + `invalid table: string item already exists at key [key]`, + `{"key": "value"}`) +} + +func Test_GivenExistingItemInKeyPath_CreatingTable_ReturnsError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.setKeyValuePair(newKey("key"), newItem(tString, "value")) + return p.openTable(newKey("key", "subkey")), p + }, + `invalid table: string item already exists at key [key]`, + `{"key": "value"}`) +} + +func Test_GivenExistingItemAtDeepKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.openTable(newKey("deep", "table")) + p.setKeyValuePair(newKey("key"), newItem(tInteger, 0)) + return p.openTable(newKey("deep", "table", "key")), p + }, + `invalid table: integer item already exists at key [deep->table->key]`, + `{"deep": {"table": {"key": 0}}}`) +} + +func Test_GivenExistingItemAtDeepKeyFromSubTable_CreatingTableAtSameKey_ReturnsError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.openTable(newKey("deep", "table")) + p.setKeyValuePair(newKey("key1", "key2"), newItem(tInteger, 0)) + return p.setKeyValuePair(newKey("key1", "key2"), newItem(tBoolean, true)), p + }, + // This test mainly tests the formatting of [deep->table->key1->key2], being a concatenation + // of the currently active table plus the multipart key for setKeyValuePair(). + `invalid key/value pair: integer item already exists at key [deep->table->key1->key2]`, + `{"deep": {"table": {"key1": {"key2": 0}}}}`) +} + +func Test_FormattingOfQuotedPathPartInError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.openTable(newKey("must be quoted")) + p.setKeyValuePair(newKey("this one too"), newItem(tInteger, 0)) + return p.setKeyValuePair(newKey("this one too"), newItem(tInteger, 0)), p + }, + `invalid key/value pair: integer item already exists at key ["must be quoted"->"this one too"]`, + `{"must be quoted": {"this one too": 0}}`) +} + +func Test_GivenExistingItemAtKey_CreatingArrayOfTablesAtSameKey_ReturnsError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.Root["key"] = newItem(tString, "value") + return p.openArrayOfTables(newKey("key")), p + }, + `invalid table array: string item already exists at key [key]`, + `{"key": "value"}`) +} + +func Test_GivenExistingItemInKeyPath_CreatingArrayOfTables_ReturnsError(t *testing.T) { + testAST(t, func() (error, *parser) { + p := newParser() + p.Root["key"] = newItem(tString, "value") + return p.openArrayOfTables(newKey("key", "subkey")), p + }, + `invalid table array: string item already exists at key [key]`, + `{"key": "value"}`) +} + +func testAST(t *testing.T, code func() (error, *parser), expectedError string, expectedData string) { + err, p := code() + if expectedError == "" { + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + } else { + if err == nil { + t.Fatalf("An error was expected, but no error was returned") + } else if err.Error() != expectedError { + t.Fatalf("Unexpected error:\nexpected: %s\nactual: %s\n", expectedError, err.Error()) + } + } + if expectedData == "" { + return + } + if expectedData != p.Root.String() { + t.Fatalf("Unexpected data after parsing:\nexpected: %s\nactual: %s\n", expectedData, p.Root.String()) + } +} diff --git a/ast/string.go b/ast/string.go new file mode 100644 index 0000000..8b5d056 --- /dev/null +++ b/ast/string.go @@ -0,0 +1,49 @@ +package ast + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// String() produces a JSON-like (but not JSON) string representation of the value. +// This string version is useful for testing and debugging purposes. +func (parseItem item) String() string { + switch parseItem.Type { + case tString: + return fmt.Sprintf("%q", parseItem.Data[0]) + case tOffsetDateTime: + return parseItem.Data[0].(time.Time).Format(time.RFC3339Nano) + case tLocalDateTime: + return parseItem.Data[0].(time.Time).Format("2006-01-02 15:04:05.999999999") + case tLocalDate: + return parseItem.Data[0].(time.Time).Format("2006-01-02") + case tLocalTime: + return parseItem.Data[0].(time.Time).Format("15:04:05.999999999") + case tArrayOfTables: + fallthrough + case tArray: + items := make([]string, len(parseItem.Data)) + for i, value := range parseItem.Data { + items[i] = value.(*item).String() + } + return fmt.Sprintf("[%s]", strings.Join(items, ", ")) + case tTable: + pairs := parseItem.Data[0].(table) + keys := make([]string, len(pairs)) + i := 0 + for k := range pairs { + keys[i] = k + i++ + } + sort.Strings(keys) + items := make([]string, len(pairs)) + for i, k := range keys { + items[i] = fmt.Sprintf("%q: %s", k, pairs[k].String()) + } + return fmt.Sprintf("{%s}", strings.Join(items, ", ")) + default: + return fmt.Sprintf("%v", parseItem.Data[0]) + } +} diff --git a/ast/string_test.go b/ast/string_test.go new file mode 100644 index 0000000..c67e73a --- /dev/null +++ b/ast/string_test.go @@ -0,0 +1,37 @@ +package ast + +import ( + "testing" + "time" +) + +func Test_StringFormatting(t *testing.T) { + testAST(t, func() (error, *parser) { + tableData := make(table) + tableData["x"] = newItem(tBoolean, true) + tableData["y"] = newItem(tInteger, 42) + dateTime, _ := time.Parse(time.RFC3339Nano, "2003-11-01T01:02:03.999999999+10:00") + p := newParser() + p.setKeyValuePair(newKey("a"), newItem(tInteger, 1)) + p.setKeyValuePair(newKey("b"), newItem(tFloat, 2.3)) + p.setKeyValuePair(newKey("c"), newItem(tBoolean, true)) + p.setKeyValuePair(newKey("d"), newItem(tString, "foo")) + p.setKeyValuePair(newKey("e"), newItem(tArray, newItem(tInteger, 1), newItem(tInteger, 2))) + p.setKeyValuePair(newKey("f"), newItem(tTable, tableData)) + p.setKeyValuePair(newKey("g"), newItem(tOffsetDateTime, dateTime)) + p.setKeyValuePair(newKey("h"), newItem(tLocalDateTime, dateTime)) + p.setKeyValuePair(newKey("i"), newItem(tLocalDate, dateTime)) + p.setKeyValuePair(newKey("j"), newItem(tLocalTime, dateTime)) + return nil, p + }, "", + `{"a": 1, `+ + `"b": 2.3, `+ + `"c": true, `+ + `"d": "foo", `+ + `"e": [1, 2], `+ + `"f": {"x": true, "y": 42}, `+ + `"g": 2003-11-01T01:02:03.999999999+10:00, `+ + `"h": 2003-11-01 01:02:03.999999999, `+ + `"i": 2003-11-01, `+ + `"j": 01:02:03.999999999}`) +}