From b95f255ae7a3e2b57d08f0c711ab38e3391fbb40 Mon Sep 17 00:00:00 2001 From: Maurice Makaay Date: Wed, 26 Jun 2019 21:51:42 +0000 Subject: [PATCH] Moved all parser code into subpackage 'parse'. --- .gitignore | 6 +- ast.go | 245 ------------------ ast/ast.go | 8 +- ast/ast_test.go | 12 +- ast/string.go | 22 +- ast_test.go | 177 ------------- comment.go | 15 -- go.sum | 4 +- comment_test.go => parse/comment_test.go | 8 +- parse/document.go | 32 +++ helpers_test.go => parse/helpers_test.go | 23 +- keyvaluepair.go => parse/keyvaluepair.go | 46 +--- .../keyvaluepair_test.go | 16 +- parse/parse.go | 56 ++++ parse/parse_test.go | 23 ++ value.go => parse/value.go | 5 +- value_array.go => parse/value_array.go | 21 +- .../value_array_test.go | 4 +- parse/value_boolean.go | 22 ++ .../value_boolean_test.go | 4 +- value_datetime.go => parse/value_datetime.go | 19 +- .../value_datetime_test.go | 4 +- value_number.go => parse/value_number.go | 21 +- .../value_number_test.go | 6 +- value_string.go => parse/value_string.go | 7 +- .../value_string_test.go | 14 +- value_table.go => parse/value_table.go | 31 +-- .../value_table_test.go | 12 +- value_test.go => parse/value_test.go | 4 +- toml.go | 22 -- value_boolean.go | 21 -- 31 files changed, 264 insertions(+), 646 deletions(-) delete mode 100644 ast.go delete mode 100644 ast_test.go delete mode 100644 comment.go rename comment_test.go => parse/comment_test.go (74%) create mode 100644 parse/document.go rename helpers_test.go => parse/helpers_test.go (59%) rename keyvaluepair.go => parse/keyvaluepair.go (77%) rename keyvaluepair_test.go => parse/keyvaluepair_test.go (92%) create mode 100644 parse/parse.go create mode 100644 parse/parse_test.go rename value.go => parse/value.go (89%) rename value_array.go => parse/value_array.go (79%) rename value_array_test.go => parse/value_array_test.go (97%) create mode 100644 parse/value_boolean.go rename value_boolean_test.go => parse/value_boolean_test.go (92%) rename value_datetime.go => parse/value_datetime.go (85%) rename value_datetime_test.go => parse/value_datetime_test.go (97%) rename value_number.go => parse/value_number.go (86%) rename value_number_test.go => parse/value_number_test.go (97%) rename value_string.go => parse/value_string.go (98%) rename value_string_test.go => parse/value_string_test.go (95%) rename value_table.go => parse/value_table.go (89%) rename value_table_test.go => parse/value_table_test.go (91%) rename value_test.go => parse/value_test.go (96%) delete mode 100644 toml.go delete mode 100644 value_boolean.go diff --git a/.gitignore b/.gitignore index 0558153..fd2ea3e 100644 --- a/.gitignore +++ b/.gitignore @@ -32,9 +32,5 @@ tags *.out # ---> VisualStudioCode -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json +.vscode diff --git a/ast.go b/ast.go deleted file mode 100644 index b07ab5f..0000000 --- a/ast.go +++ /dev/null @@ -1,245 +0,0 @@ -package toml - -import ( - "fmt" - "regexp" - "sort" - "strings" - "time" -) - -// item represents a TOML item. -type item struct { - Type valueType - Data []interface{} -} - -// table represents a TOML table. -type table map[string]*item - -// valueType identifies the type of a TOML value. -type valueType string - -const ( - tString valueType = "string" // "various", 'types', """of""", '''strings''' - tInteger valueType = "integer" // 12345, 0xffee12a0, 0o0755, 0b101101011 - tFloat valueType = "float" // 10.1234, 143E-12, 43.28377e+4, +inf, -inf, nan - tBoolean valueType = "boolean" // true or false - tOffsetDateTime valueType = "offset datetime" // 2019-06-18 10:32:15.173645362+0200 - tLocalDateTime valueType = "datetime" // 2018-12-25 12:12:18.876772533 - tLocalDate valueType = "date" // 2017-05-17 - tLocalTime valueType = "time" // 23:01:22 - tArrayOfTables valueType = "array" // defined using an [[array.of.tables]] - tArray valueType = "static array" // defined using ["an", "inline", "array"] - tTable valueType = "table" // defined using { "inline" = "table" } or [standard.table] -) - -// newItem instantiates a new item struct. -func newItem(valueType valueType, data ...interface{}) *item { - return &item{Type: valueType, Data: data} -} - -// newKey instantiates a new key. -func newKey(key ...string) []string { - return key -} - -// parser holds the state for the TOML parser. All parsing functions are -// methods of this struct. -type parser struct { - Root table // the root-level TOML table (each TOML doc is implicitly a table) - Current table // the currently active TOML table - CurrentKey []string // the key for the currently active TOML table -} - -func newParser() *parser { - p := &parser{Root: make(table)} - p.Current = p.Root - return p -} - -func (t *parser) setKeyValuePair(key []string, value *item) error { - // First make sure the table structure for storing the value exists. - node, lastKeyPart, err := t.makeTablePath(key) - if err != nil { - return fmt.Errorf("invalid key/value pair: %s", err) - } - // Check if the key is still free for use. - if existing, ok := node[lastKeyPart]; ok { - path := t.formatKeyPath(key, len(key)-1) - return fmt.Errorf("invalid key/value pair: %s item already exists at key %s", existing.Type, path) - } - // It is, store the value in the table. - node[lastKeyPart] = value - return nil -} - -func (t *parser) openTable(key []string) error { - t.CurrentKey = nil - t.Current = t.Root - // Go over all requested levels of the key. For all levels, except the last - // one, it is okay if a table already exists. For at least the last level, - // no table or value must exist, because that would mean we are overwriting - // an existing key/value pair, which is not allowed. - node, lastKeyPart, err := t.makeTablePath(key) - if err != nil { - return fmt.Errorf("invalid table: %s", err) - } - // Check if the key is still free for use. - if existing, ok := node[lastKeyPart]; ok { - path := t.formatKeyPath(key, len(key)-1) - return fmt.Errorf("invalid table: %s item already exists at key %s", existing.Type, path) - } - // The subtable does not exist yet. Create the subtable. - subTable := make(table) - node[lastKeyPart] = newItem(tTable, subTable) - node = subTable - - // From here on, key/value pairs are added to the newly defined table. - t.Current = node - t.CurrentKey = key - return nil -} - -func (t *parser) openArrayOfTables(key []string) error { - t.CurrentKey = nil - t.Current = t.Root - // Go over all requested levels of the key. For all levels, except the last - // one, it is okay if a table already exists. For the last level, either - // no item must exist (in which case a table array will be created), or a - // table array must exist. - // Other cases would mean we are overwriting an existing key/value pair, - // which is not allowed. - node, lastKeyPart, err := t.makeTablePath(key) - if err != nil { - return fmt.Errorf("invalid table array: %s", err) - } - // At the last key position, there must be either no value yet, or the - // existing value must be a table array. Other values are invalid. - if existing, ok := node[lastKeyPart]; ok { - if existing.Type != tArrayOfTables { - path := t.formatKeyPath(key, len(key)-1) - return fmt.Errorf("invalid table array: %s item already exists at key %s", existing.Type, path) - } - // A table array exists. Add a new table to this array. - array := node[lastKeyPart] - subTable := make(table) - tables := array.Data - tables = append(tables, newItem(tTable, subTable)) - array.Data = tables - node = subTable - } else { - // No value exists at the defined key path. Create a new table array. - subTable := make(table) - node[lastKeyPart] = newItem(tArrayOfTables, newItem(tTable, subTable)) - node = subTable - } - - // From here on, key/value pairs are added to the newly defined table. - t.Current = node - t.CurrentKey = key - return nil -} - -func (t *parser) makeTablePath(key []string) (table, string, error) { - node := t.Current - for i, keyPart := range key { - // Arrived at the last key part? Then the path towards that key is - // setup correctly. Return the last part, so the caller can use it. - isLast := i == len(key)-1 - if isLast { - return node, keyPart, nil - } - if subItem, ok := node[keyPart]; ok { - // You cannot overwrite an already defined key, regardless its value. - // When a value already exists at the current key, this can only be a table. - if subItem.Type != tTable { - path := t.formatKeyPath(key, i) - return nil, "", fmt.Errorf("%s item already exists at key %s", subItem.Type, path) - } - // All is okay, traverse to the subtable. - node = subItem.Data[0].(table) - } else { - // The subtable does not exist yet. Create the subtable. - subTable := make(table) - node[keyPart] = newItem(tTable, subTable) - node = subTable - } - } - panic("makeTablePath(): empty key provided; a key must have at least one key part") -} - -func (t *parser) formatKeyPath(key []string, end int) string { - var sb strings.Builder - sb.WriteRune('[') - if t.CurrentKey != nil { - for i, keyPart := range t.CurrentKey { - if i > 0 { - sb.WriteString("->") - } - sb.WriteString(formatKeyName(keyPart)) - } - } - for i, keyPart := range key { - if t.CurrentKey != nil || i > 0 { - sb.WriteString("->") - } - sb.WriteString(formatKeyName(keyPart)) - if i == end { - break - } - } - sb.WriteRune(']') - return sb.String() -} - -func formatKeyName(key string) string { - if ok, _ := regexp.Match(`^\w+$`, []byte(key)); ok { - return key - } - return fmt.Sprintf("%q", key) -} - -func (t table) String() string { - return newItem(tTable, t).String() -} - -// String() produces a JSON-like (but not JSON) string representation of the value. -func (parseItem item) String() string { - switch parseItem.Type { - case tString: - return fmt.Sprintf("%q", parseItem.Data[0]) - case tOffsetDateTime: - return parseItem.Data[0].(time.Time).Format(time.RFC3339Nano) - case tLocalDateTime: - return parseItem.Data[0].(time.Time).Format("2006-01-02 15:04:05.999999999") - case tLocalDate: - return parseItem.Data[0].(time.Time).Format("2006-01-02") - case tLocalTime: - return parseItem.Data[0].(time.Time).Format("15:04:05.999999999") - case tArrayOfTables: - fallthrough - case tArray: - items := make([]string, len(parseItem.Data)) - for i, value := range parseItem.Data { - items[i] = value.(*item).String() - } - return fmt.Sprintf("[%s]", strings.Join(items, ", ")) - case tTable: - pairs := parseItem.Data[0].(table) - keys := make([]string, len(pairs)) - i := 0 - for k := range pairs { - keys[i] = k - i++ - } - sort.Strings(keys) - items := make([]string, len(pairs)) - for i, k := range keys { - items[i] = fmt.Sprintf("%q: %s", k, pairs[k].String()) - } - return fmt.Sprintf("{%s}", strings.Join(items, ", ")) - default: - return fmt.Sprintf("%v", parseItem.Data[0]) - } -} diff --git a/ast/ast.go b/ast/ast.go index 3e02459..3f04fba 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -191,15 +191,15 @@ func (doc *Document) makeTablePath(key Key) (Table, string, error) { if isLast { return node, keyPart, nil } - if subItem, ok := node[keyPart]; ok { + if subValue, ok := node[keyPart]; ok { // You cannot overwrite an already defined key, regardless its value. // When a value already exists at the current key, this can only be a table. - if subItem.Type != TypeTable { + if subValue.Type != TypeTable { path := doc.formatKeyPath(key, i) - return nil, "", fmt.Errorf("%s value already exists at key %s", subItem.Type, path) + return nil, "", fmt.Errorf("%s value already exists at key %s", subValue.Type, path) } // All is okay, traverse to the subtable. - node = subItem.Data[0].(Table) + node = subValue.Data[0].(Table) } else { // The subtable does not exist yet. Create the subtable. subTable := make(Table) diff --git a/ast/ast_test.go b/ast/ast_test.go index f0b333a..8769a1d 100644 --- a/ast/ast_test.go +++ b/ast/ast_test.go @@ -103,7 +103,7 @@ func Test_GivenExistingTableAtKey_CreatingTableAtSameKey_ReturnsError(t *testing `{"key1": {"key2": {}}}`) } -func Test_GivenExistingItemAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { +func Test_GivenExistingValueAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { testAST(t, func() (error, *ast.Document) { p := ast.NewDocument() p.SetKeyValuePair(ast.NewKey("key"), ast.NewValue(ast.TypeString, "value")) @@ -113,7 +113,7 @@ func Test_GivenExistingItemAtKey_CreatingTableAtSameKey_ReturnsError(t *testing. `{"key": "value"}`) } -func Test_GivenExistingItemInKeyPath_CreatingTable_ReturnsError(t *testing.T) { +func Test_GivenExistingValueInKeyPath_CreatingTable_ReturnsError(t *testing.T) { testAST(t, func() (error, *ast.Document) { p := ast.NewDocument() p.SetKeyValuePair(ast.NewKey("key"), ast.NewValue(ast.TypeString, "value")) @@ -123,7 +123,7 @@ func Test_GivenExistingItemInKeyPath_CreatingTable_ReturnsError(t *testing.T) { `{"key": "value"}`) } -func Test_GivenExistingItemAtDeepKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { +func Test_GivenExistingValueAtDeepKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { testAST(t, func() (error, *ast.Document) { p := ast.NewDocument() p.OpenTable(ast.NewKey("deep", "table")) @@ -134,7 +134,7 @@ func Test_GivenExistingItemAtDeepKey_CreatingTableAtSameKey_ReturnsError(t *test `{"deep": {"table": {"key": 0}}}`) } -func Test_GivenExistingItemAtDeepKeyFromSubTable_CreatingTableAtSameKey_ReturnsError(t *testing.T) { +func Test_GivenExistingValueAtDeepKeyFromSubTable_CreatingTableAtSameKey_ReturnsError(t *testing.T) { testAST(t, func() (error, *ast.Document) { p := ast.NewDocument() p.OpenTable(ast.NewKey("deep", "table")) @@ -158,7 +158,7 @@ func Test_FormattingOfQuotedPathPartInError(t *testing.T) { `{"must be quoted": {"this one too": 0}}`) } -func Test_GivenExistingItemAtKey_CreatingArrayOfTablesAtSameKey_ReturnsError(t *testing.T) { +func Test_GivenExistingValueAtKey_CreatingArrayOfTablesAtSameKey_ReturnsError(t *testing.T) { testAST(t, func() (error, *ast.Document) { p := ast.NewDocument() p.Root["key"] = ast.NewValue(ast.TypeString, "value") @@ -168,7 +168,7 @@ func Test_GivenExistingItemAtKey_CreatingArrayOfTablesAtSameKey_ReturnsError(t * `{"key": "value"}`) } -func Test_GivenExistingItemInKeyPath_CreatingArrayOfTables_ReturnsError(t *testing.T) { +func Test_GivenExistingValueInKeyPath_CreatingArrayOfTables_ReturnsError(t *testing.T) { testAST(t, func() (error, *ast.Document) { p := ast.NewDocument() p.Root["key"] = ast.NewValue(ast.TypeString, "value") diff --git a/ast/string.go b/ast/string.go index 08b0816..a4a3bba 100644 --- a/ast/string.go +++ b/ast/string.go @@ -15,28 +15,28 @@ func (t Table) String() string { // String() produces a JSON-like (but not JSON) string representation of the value. // This string version is mainly useful for testing and debugging purposes. -func (parseItem Value) String() string { - switch parseItem.Type { +func (value Value) String() string { + switch value.Type { case TypeString: - return fmt.Sprintf("%q", parseItem.Data[0]) + return fmt.Sprintf("%q", value.Data[0]) case TypeOffsetDateTime: - return parseItem.Data[0].(time.Time).Format(time.RFC3339Nano) + return value.Data[0].(time.Time).Format(time.RFC3339Nano) case TypeLocalDateTime: - return parseItem.Data[0].(time.Time).Format("2006-01-02 15:04:05.999999999") + return value.Data[0].(time.Time).Format("2006-01-02 15:04:05.999999999") case TypeLocalDate: - return parseItem.Data[0].(time.Time).Format("2006-01-02") + return value.Data[0].(time.Time).Format("2006-01-02") case TypeLocalTime: - return parseItem.Data[0].(time.Time).Format("15:04:05.999999999") + return value.Data[0].(time.Time).Format("15:04:05.999999999") case TypeArrayOfTables: fallthrough case TypeArray: - values := make(Key, len(parseItem.Data)) - for i, value := range parseItem.Data { + values := make(Key, len(value.Data)) + for i, value := range value.Data { values[i] = value.(*Value).String() } return fmt.Sprintf("[%s]", strings.Join(values, ", ")) case TypeTable: - pairs := parseItem.Data[0].(Table) + pairs := value.Data[0].(Table) keys := make([]string, len(pairs)) i := 0 for k := range pairs { @@ -50,6 +50,6 @@ func (parseItem Value) String() string { } return fmt.Sprintf("{%s}", strings.Join(values, ", ")) default: - return fmt.Sprintf("%v", parseItem.Data[0]) + return fmt.Sprintf("%v", value.Data[0]) } } diff --git a/ast_test.go b/ast_test.go deleted file mode 100644 index 50f5953..0000000 --- a/ast_test.go +++ /dev/null @@ -1,177 +0,0 @@ -package toml - -import ( - "testing" -) - -func TestAST_ConstructStructure(t *testing.T) { - testAST(t, func() (error, *parser) { - p := newParser() - p.setKeyValuePair(newKey("ding"), newItem(tInteger, 10)) - p.setKeyValuePair(newKey("dong"), newItem(tString, "not a song")) - p.openTable(newKey("key1", "key2 a")) - p.setKeyValuePair(newKey("dooh"), newItem(tBoolean, true)) - p.setKeyValuePair(newKey("dah"), newItem(tBoolean, false)) - p.openTable(newKey("key1", "key2 b")) - p.setKeyValuePair(newKey("dieh"), newItem(tFloat, 1.111)) - p.setKeyValuePair(newKey("duh"), newItem(tFloat, 1.18e-12)) - p.setKeyValuePair(newKey("foo", "bar"), newItem(tArrayOfTables, newItem(tInteger, 1), newItem(tInteger, 2))) - p.openArrayOfTables(newKey("aaah", "table array")) - p.setKeyValuePair(newKey("a"), newItem(tFloat, 1.234)) - p.openArrayOfTables(newKey("aaah", "table array")) - p.setKeyValuePair(newKey("b"), newItem(tFloat, 2.345)) - p.setKeyValuePair(newKey("c"), newItem(tString, "bingo!")) - p.openArrayOfTables(newKey("aaah", "table array")) - return nil, p - }, - "", - `{"aaah": {"table array": [{"a": 1.234}, {"b": 2.345, "c": "bingo!"}, {}]}, `+ - `"ding": 10, "dong": "not a song", `+ - `"key1": {"key2 a": {"dah": false, "dooh": true}, "key2 b": {"dieh": 1.111, "duh": 1.18e-12, "foo": {"bar": [1, 2]}}}}`) -} - -func TestAST_EmptyKeyForCreatingTablePath_Panics(t *testing.T) { - defer func() { - r := recover() - if r.(string) != "makeTablePath(): empty key provided; a key must have at least one key part" { - t.Fatalf("Did not get the expected panic message") - } - }() - p := newParser() - p.openTable(newKey()) -} - -func TestAST_StoreValueInRootTable(t *testing.T) { - testAST(t, func() (error, *parser) { - p := newParser() - p.setKeyValuePair(newKey("key1"), newItem(tString, "value1")) - return p.setKeyValuePair(newKey("key2"), newItem(tString, "value2")), p - }, - "", - `{"key1": "value1", "key2": "value2"}`) -} - -func TestAST_StoreValueWithMultipartKey_CreatesTableHierarchy(t *testing.T) { - testAST(t, func() (error, *parser) { - p := newParser() - return p.setKeyValuePair(newKey("key1", "key2", "key3"), newItem(tString, "value")), p - }, - "", - `{"key1": {"key2": {"key3": "value"}}}`) -} - -func TestAST_StoreDuplicateKeyInRootTable_ReturnsError(t *testing.T) { - testAST(t, func() (error, *parser) { - p := newParser() - p.setKeyValuePair(newKey("key"), newItem(tString, "value")) - return p.setKeyValuePair(newKey("key"), newItem(tInteger, 321)), p - }, - `invalid key/value pair: string item already exists at key [key]`, - `{"key": "value"}`) -} - -func TestAST_StoreValueWithMultipartKey_UnderSubtable_CreatesTableHierarchy(t *testing.T) { - testAST(t, func() (error, *parser) { - p := newParser() - p.openTable(newKey("tablekey1", "tablekey2")) - return p.setKeyValuePair(newKey("valuekey1", "valuekey2", "valuekey3"), newItem(tString, "value")), p - }, - "", - `{"tablekey1": {"tablekey2": {"valuekey1": {"valuekey2": {"valuekey3": "value"}}}}}`) -} - -func TestAST_StoreKeyPathWherePathContainsNonTableAlready_ReturnsError(t *testing.T) { - testAST(t, func() (error, *parser) { - p := newParser() - p.openTable(newKey("key1")) - p.setKeyValuePair(newKey("key2"), newItem(tInteger, 0)) - return p.setKeyValuePair(newKey("key2", "key3"), newItem(tString, "value")), p - }, - `invalid key/value pair: integer item already exists at key [key1->key2]`, - `{"key1": {"key2": 0}}`) -} - -func TestAST_GivenExistingTableAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { - testAST(t, func() (error, *parser) { - p := newParser() - p.openTable(newKey("key1", "key2")) - return p.openTable(newKey("key1", "key2")), p - }, - `invalid table: table item already exists at key [key1->key2]`, - `{"key1": {"key2": {}}}`) -} - -func TestAST_GivenExistingItemAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { - testAST(t, func() (error, *parser) { - p := newParser() - p.setKeyValuePair(newKey("key"), newItem(tString, "value")) - return p.openTable(newKey("key")), p - }, - `invalid table: string item already exists at key [key]`, - `{"key": "value"}`) -} - -func TestAST_GivenExistingItemInKeyPath_CreatingTable_ReturnsError(t *testing.T) { - testAST(t, func() (error, *parser) { - p := newParser() - p.setKeyValuePair(newKey("key"), newItem(tString, "value")) - return p.openTable(newKey("key", "subkey")), p - }, - `invalid table: string item already exists at key [key]`, - `{"key": "value"}`) -} - -func TestAST_GivenExistingItemAtDeepKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { - testAST(t, func() (error, *parser) { - p := newParser() - p.openTable(newKey("deep", "table")) - p.setKeyValuePair(newKey("key"), newItem(tInteger, 0)) - return p.openTable(newKey("deep", "table", "key")), p - }, - `invalid table: integer item already exists at key [deep->table->key]`, - `{"deep": {"table": {"key": 0}}}`) -} - -func TestAST_GivenExistingItemAtDeepKeyFromSubTable_CreatingTableAtSameKey_ReturnsError(t *testing.T) { - testAST(t, func() (error, *parser) { - p := newParser() - p.openTable(newKey("deep", "table")) - p.setKeyValuePair(newKey("key1", "key2"), newItem(tInteger, 0)) - return p.setKeyValuePair(newKey("key1", "key2"), newItem(tBoolean, true)), p - }, - // This test mainly tests the formatting of [deep->table->key1->key2], being a concatenation - // of the currently active table plus the multipart key for setKeyValuePair(). - `invalid key/value pair: integer item already exists at key [deep->table->key1->key2]`, - `{"deep": {"table": {"key1": {"key2": 0}}}}`) -} - -func TestAST_FormattingOfQuotedPathPartInError(t *testing.T) { - testAST(t, func() (error, *parser) { - p := newParser() - p.openTable(newKey("must be quoted")) - p.setKeyValuePair(newKey("this one too"), newItem(tInteger, 0)) - return p.setKeyValuePair(newKey("this one too"), newItem(tInteger, 0)), p - }, - `invalid key/value pair: integer item already exists at key ["must be quoted"->"this one too"]`, - `{"must be quoted": {"this one too": 0}}`) -} - -func TestAST_GivenExistingItemAtKey_CreatingArrayOfTablesAtSameKey_ReturnsError(t *testing.T) { - testAST(t, func() (error, *parser) { - p := newParser() - p.Root["key"] = newItem(tString, "value") - return p.openArrayOfTables(newKey("key")), p - }, - `invalid table array: string item already exists at key [key]`, - `{"key": "value"}`) -} - -func TestAST_GivenExistingItemInKeyPath_CreatingArrayOfTables_ReturnsError(t *testing.T) { - testAST(t, func() (error, *parser) { - p := newParser() - p.Root["key"] = newItem(tString, "value") - return p.openArrayOfTables(newKey("key", "subkey")), p - }, - `invalid table array: string item already exists at key [key]`, - `{"key": "value"}`) -} diff --git a/comment.go b/comment.go deleted file mode 100644 index d892f1b..0000000 --- a/comment.go +++ /dev/null @@ -1,15 +0,0 @@ -package toml - -import ( - "git.makaay.nl/mauricem/go-parsekit/parse" -) - -// A '#' hash symbol marks the rest of the line as a comment. -// All characters up to the end of the line are included in the comment. -var comment = c.Seq(a.Hash, c.ZeroOrMore(c.Not(a.EndOfLine)), m.Drop(a.EndOfLine)) - -func (t *parser) startComment(p *parse.API) { - if !p.Accept(comment) { - p.Expected("comment") - } -} diff --git a/go.sum b/go.sum index 391dbee..fc70406 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ git.makaay.nl/mauricem/go-parsekit v0.0.0-20190521150537-747456517939 h1:cMBHhfSJR2BZgVN7NmP+c2agNlXDef4Iz6+XQp5AqdU= -git.makaay.nl/mauricem/go-parsekit v0.0.0-20190521150537-747456517939/go.mod h1:/mo+aM5Im5rkBqBvXTAsVR0//OfsAAiFyvuxxcxGGlU= \ No newline at end of file +git.makaay.nl/mauricem/go-parsekit v0.0.0-20190521150537-747456517939/go.mod h1:/mo+aM5Im5rkBqBvXTAsVR0//OfsAAiFyvuxxcxGGlU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml-test v0.1.0/go.mod h1:xhCXZ5sE5Y4uZrhYxoOUTfGFH+wVyuKlcOC371elHo8= diff --git a/comment_test.go b/parse/comment_test.go similarity index 74% rename from comment_test.go rename to parse/comment_test.go index 058adce..a4ec959 100644 --- a/comment_test.go +++ b/parse/comment_test.go @@ -1,10 +1,10 @@ -package toml +package parse import ( "testing" ) -func TestComment2(t *testing.T) { +func TestComment(t *testing.T) { for _, test := range []parseTest{ {``, `{}`, `unexpected end of file (expected comment) at start of file`}, {`#`, `{}`, ``}, @@ -13,9 +13,9 @@ func TestComment2(t *testing.T) { {"# ending in EOL & EOF\r\n", `{}`, ``}, {`# \xxx/ \u can't escape/`, `{}`, ``}, {"# \tlexe\r accepts embedded ca\r\riage \returns\r\n", `{}`, ``}, - {"# with data and newline\ncode continues here", `{}`, `unexpected input (expected end of file) at line 2, column 1`}, + {"# with data and newline\ncode continues here", `{}`, `unexpected input (expected a value assignment) at line 2, column 5`}, } { p := newParser() - testParse(t, p, p.startComment, test) + testParse(t, p, p.startDocument, test) } } diff --git a/parse/document.go b/parse/document.go new file mode 100644 index 0000000..dbc0979 --- /dev/null +++ b/parse/document.go @@ -0,0 +1,32 @@ +package parse + +import ( + "git.makaay.nl/mauricem/go-parsekit/parse" +) + +var ( + // Keys may be either bare or quoted. + detectKey = c.Any(bareKeyRune, a.SingleQuote, a.DoubleQuote) + + // Both [tables] and [[arrays of tables]] start with a square open bracket. + detectTableOrArrayOfTables = a.SquareOpen +) + +func (t *parser) startDocument(p *parse.API) { + for { + p.Accept(dropWhitespace) + p.Accept(dropComment) + switch { + case p.Peek(detectTableOrArrayOfTables): + p.Handle(t.startTable) + case p.Peek(detectKey): + p.Handle(t.startKeyValuePair) + default: + p.ExpectEndOfFile() + return + } + if p.IsStoppedOrInError() { + return + } + } +} diff --git a/helpers_test.go b/parse/helpers_test.go similarity index 59% rename from helpers_test.go rename to parse/helpers_test.go index 7ea856e..0dbaee6 100644 --- a/helpers_test.go +++ b/parse/helpers_test.go @@ -1,4 +1,4 @@ -package toml +package parse import ( "fmt" @@ -7,27 +7,6 @@ import ( "git.makaay.nl/mauricem/go-parsekit/parse" ) -func testAST(t *testing.T, code func() (error, *parser), expectedError string, expectedData string) { - err, p := code() - if expectedError == "" { - if err != nil { - t.Fatalf("Unexpected error: %s", err) - } - } else { - if err == nil { - t.Fatalf("An error was expected, but no error was returned") - } else if err.Error() != expectedError { - t.Fatalf("Unexpected error:\nexpected: %s\nactual: %s\n", expectedError, err.Error()) - } - } - if expectedData == "" { - return - } - if expectedData != p.Root.String() { - t.Fatalf("Unexpected data after parsing:\nexpected: %s\nactual: %s\n", expectedData, p.Root.String()) - } -} - type parseTest struct { input interface{} expected string diff --git a/keyvaluepair.go b/parse/keyvaluepair.go similarity index 77% rename from keyvaluepair.go rename to parse/keyvaluepair.go index 070f019..d4f7a36 100644 --- a/keyvaluepair.go +++ b/parse/keyvaluepair.go @@ -1,10 +1,11 @@ -package toml +package parse import ( "git.makaay.nl/mauricem/go-parsekit/parse" ) -// The primary building block of a TOML document is the key/value pair. +// The primary building block of a TOML document is the table. +// Tables are filled with key/value pairs. var ( // Keys are on the left of the equals sign and values are on the right. @@ -13,55 +14,34 @@ var ( // broken over multiple lines). keyAssignment = c.Seq(dropBlanks, a.Equal, dropBlanks) - // A key may be either bare, quoted or dotted. Bare keys may only + // A key may be either bare, quoted or dotted. Bare keys may only // contain ASCII letters, ASCII digits, underscores, and dashes // (A-Za-z0-9_-). Note that bare keys are allowed to be composed of only // ASCII digits, e.g. 1234, but are always interpreted as strings. - bareKeyRune = c.Any(a.ASCIILower, a.ASCIIUpper, a.Digit, a.Underscore, a.Minus) - bareKey = c.OneOrMore(bareKeyRune) - + // // Quoted keys follow the exact same rules as either basic strings or // literal strings and allow you to use a much broader set of key names. // Best practice is to use bare keys except when absolutely necessary. // A bare key must be non-empty, but an empty quoted key is allowed // (though discouraged). - startOfKey = c.Any(bareKeyRune, a.SingleQuote, a.DoubleQuote) + bareKeyRune = c.Any(a.ASCIILower, a.ASCIIUpper, a.Digit, a.Underscore, a.Minus) + bareKey = c.OneOrMore(bareKeyRune) // Dotted keys are a sequence of bare or quoted keys joined with a dot. // This allows for grouping similar properties together. Blanks // around dot-separated parts are ignored, however, best practice is to // not use any extraneous blanks. keySeparatorDot = c.Seq(dropBlanks, a.Dot, dropBlanks) - - // Both [tables] and [[arrays of tables]] start with a square open bracket. - startOfTableOrArrayOfTables = a.SquareOpen ) func (t *parser) startKeyValuePair(p *parse.API) { - for { - p.Accept(dropWhitespace) - switch { - case p.Peek(a.Hash): - p.Handle(t.startComment) - case p.Peek(startOfTableOrArrayOfTables): - p.Handle(t.startTable) - case p.Peek(startOfKey): - key, ok := t.parseKey(p, []string{}) - if ok && p.Handle(t.startAssignment) { - if value, ok := t.parseValue(p); ok { - err := t.setKeyValuePair(key, value) - if err != nil { - p.Error("%s", err) - return - } - } + key, ok := t.parseKey(p, []string{}) + if ok && p.Handle(t.startAssignment) { + if value, ok := t.parseValue(p); ok { + err := t.SetKeyValuePair(key, value) + if err != nil { + p.Error("%s", err) } - default: - p.ExpectEndOfFile() - return - } - if p.IsStoppedOrInError() { - return } } } diff --git a/keyvaluepair_test.go b/parse/keyvaluepair_test.go similarity index 92% rename from keyvaluepair_test.go rename to parse/keyvaluepair_test.go index f0ca46e..585d6c7 100644 --- a/keyvaluepair_test.go +++ b/parse/keyvaluepair_test.go @@ -1,4 +1,4 @@ -package toml +package parse import "testing" @@ -14,6 +14,7 @@ func TestKey(t *testing.T) { {"key1.key2=0", `{"key1": {"key2": 0}}`, ``}, {"key . with . spaces=0", `{"key": {"with": {"spaces": 0}}}`, ``}, {"key \t . \twithtabs\t . \tandspaces=0", `{"key": {"withtabs": {"andspaces": 0}}}`, ``}, + {"key1='value1' key2='value2' # on same line", `{"key1": "value1", "key2": "value2"}`, ``}, // Single quoted key tests {"''=0", `{"": 0}`, ``}, {"'single quoted'=0", `{"single quoted": 0}`, ``}, @@ -29,7 +30,7 @@ func TestKey(t *testing.T) { {`this.'i\s'."madness\t".''=0`, `{"this": {"i\\s": {"madness\t": {"": 0}}}}`, ``}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } @@ -48,10 +49,10 @@ func TestKeyValuePair(t *testing.T) { {`another = "# This is not a comment"`, `{"another": "# This is not a comment"}`, ``}, {"key1=\"value1\"key2=\"value2\"\r\nkey3a.key3b=\"value3\"", `{"key1": "value1", "key2": "value2", "key3a": {"key3b": "value3"}}`, ``}, {"with=\"comments\"# boring \nanother.cool =\"one\" \t # to the end\r\n", `{"another": {"cool": "one"}, "with": "comments"}`, ``}, - {"key='value'\nkey='another value'", `{"key": "value"}`, `invalid key/value pair: string item already exists at key [key] at line 2, column 20`}, + {"key='value'\nkey='another value'", `{"key": "value"}`, `invalid key/value pair: string value already exists at key [key] at line 2, column 20`}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } @@ -75,10 +76,11 @@ func TestKeyValuePair_ForAllTypes(t *testing.T) { {"time=15:03:11", `{"time": 15:03:11}`, ``}, {"datetime=2021-02-01 15:03:11.123", `{"datetime": 2021-02-01 15:03:11.123}`, ``}, {"offset_datetime=1111-11-11 11:11:11.111111111+11:11", `{"offset_datetime": 1111-11-11T11:11:11.111111111+11:11}`, ``}, - {"static_array=['a', 'static', 'array']", `{"static_array": ["a", "static", "array"]}`, ``}, + {"inline_array=['a', 'inline', 'array']", `{"inline_array": ["a", "inline", "array"]}`, ``}, + {`inline_table={"inline"="table"}`, `{"inline_table": {"inline": "table"}}`, ``}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } @@ -140,6 +142,6 @@ func TestKeyValuePair_ExamplesFromSpecification(t *testing.T) { `{"points": [{"x": 1, "y": 2, "z": 3}, {"x": 7, "y": 8, "z": 9}, {"x": 2, "y": 4, "z": 8}]}`, ``}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } diff --git a/parse/parse.go b/parse/parse.go new file mode 100644 index 0000000..5b91a35 --- /dev/null +++ b/parse/parse.go @@ -0,0 +1,56 @@ +package parse + +import ( + "git.makaay.nl/mauricem/go-parsekit/parse" + "git.makaay.nl/mauricem/go-parsekit/tokenize" + "git.makaay.nl/mauricem/go-toml/ast" +) + +// Some globally useful tokenizer definitions. +var ( + c, a, m, tok = tokenize.C, tokenize.A, tokenize.M, tokenize.T + + // From the specs: "Whitespace means tab (0x09) or space (0x20)." + // In this package, we name this a blank, to be in line with the + // terminology as used in parsekit. + blank = a.Runes('\t', ' ') + + // Newline means LF (0x0A) or CRLF (0x0D0A). + // This matches the default newline as defined by parsekit. + newline = a.Newline + + dropBlanks = m.Drop(c.ZeroOrMore(blank)) + dropWhitespace = m.Drop(c.ZeroOrMore(blank.Or(newline))) + + // A '#' hash symbol marks the rest of the line as a comment. + // All characters up to the end of the line are included in the comment. + dropComment = m.Drop(c.Seq(a.Hash, c.ZeroOrMore(c.Not(a.EndOfLine)), m.Drop(a.EndOfLine))) +) + +// parser embeds the TOML ast.Document, so it can be extended with methods +// that implement the parsing logic. This makes the ast.Document part of +// the state of the parser, making it possible to let parsing code call +// ast.Document methods directly to build the abstract syntax tree for the +// parsed TOML input. +type parser struct { + *ast.Document +} + +func newParser() *parser { + doc := ast.NewDocument() + return &parser{doc} +} + +// Run the TOML parser against the provided input data. +// +// For an overview of allowed inputs, take a look at the documentation for +// parsekit.read.New(). +// +// This function returns a TOML ast structure and an error (or nil when +// no error occurred). When an error occurred, the TOML ast struct will +// contain the data that could be parsed up to the error. +func Run(input interface{}) (ast.Table, error) { + p := newParser() + err := parse.New(p.startDocument)(input) + return p.Root, err +} diff --git a/parse/parse_test.go b/parse/parse_test.go new file mode 100644 index 0000000..9c25955 --- /dev/null +++ b/parse/parse_test.go @@ -0,0 +1,23 @@ +package parse_test + +import ( + "fmt" + + "git.makaay.nl/mauricem/go-toml/parse" +) + +func ExampleRun() { + doc, err := parse.Run("key = 'value' key2 = 'another value'") + fmt.Println(doc, err) + + doc, err = parse.Run("key = 'value'\n[table]\nanother_key = 'another one'") + fmt.Println(doc, err) + + doc, err = parse.Run("key1 = 'valid' key2 = invalid") + fmt.Println(doc, err) + + // Output: + // {"key": "value", "key2": "another value"} + // {"key": "value", "table": {"another_key": "another one"}} + // {"key1": "valid"} unexpected input (expected a value) at line 1, column 23 +} diff --git a/value.go b/parse/value.go similarity index 89% rename from value.go rename to parse/value.go index 86d955c..0d46342 100644 --- a/value.go +++ b/parse/value.go @@ -1,7 +1,8 @@ -package toml +package parse import ( "git.makaay.nl/mauricem/go-parsekit/parse" + "git.makaay.nl/mauricem/go-toml/ast" ) var ( @@ -16,7 +17,7 @@ var ( // Values must be of the following types: String, Integer, Float, Boolean, // Datetime, Array, or Inline Table. Unspecified values are invalid. -func (t *parser) parseValue(p *parse.API) (*item, bool) { +func (t *parser) parseValue(p *parse.API) (*ast.Value, bool) { switch { case p.Peek(detectString): return t.parseString(p) diff --git a/value_array.go b/parse/value_array.go similarity index 79% rename from value_array.go rename to parse/value_array.go index 53e13b0..b4bbca2 100644 --- a/value_array.go +++ b/parse/value_array.go @@ -1,7 +1,8 @@ -package toml +package parse import ( "git.makaay.nl/mauricem/go-parsekit/parse" + "git.makaay.nl/mauricem/go-toml/ast" ) // Arrays are square brackets with values inside. Whitespace is ignored. @@ -30,13 +31,13 @@ import ( // 2, # this is ok // ] var ( - arraySpace = c.ZeroOrMore(c.Any(blank, newline, comment)) + arraySpace = c.ZeroOrMore(c.Any(blank, newline, dropComment)) arrayOpen = a.SquareOpen.Then(arraySpace) arraySeparator = c.Seq(arraySpace, a.Comma, arraySpace) arrayClose = c.Seq(c.Optional(arraySpace.Then(a.Comma)), arraySpace, a.SquareClose) ) -func (t *parser) parseArray(p *parse.API) (*item, bool) { +func (t *parser) parseArray(p *parse.API) (*ast.Value, bool) { // Check for the start of the array. if !p.Accept(arrayOpen) { p.Expected("an array") @@ -45,11 +46,11 @@ func (t *parser) parseArray(p *parse.API) (*item, bool) { // Check for an empty array. if p.Accept(arrayClose) { - return newItem(tArray), true + return ast.NewValue(ast.TypeArray), true } - // Not an empty array, parse the array items. - items := []interface{}{} + // Not an empty array, parse the array values. + values := []interface{}{} for { // Check for a value item. value, ok := t.parseValue(p) @@ -59,16 +60,16 @@ func (t *parser) parseArray(p *parse.API) (*item, bool) { // Data types may not be mixed (different ways to define strings should be // considered the same type, and so should arrays with different element types). - if len(items) > 0 && value.Type != items[0].(*item).Type { - p.Error("type mismatch in array of %ss: found an item of type %s", items[0].(*item).Type, value.Type) + if len(values) > 0 && value.Type != values[0].(*ast.Value).Type { + p.Error("type mismatch in array of %ss: found an item of type %s", values[0].(*ast.Value).Type, value.Type) return nil, false } - items = append(items, value) + values = append(values, value) // Check for the end of the array. if p.Accept(arrayClose) { - return newItem(tArray, items...), true + return ast.NewValue(ast.TypeArray, values...), true } // Not the end of the array? Then we should find an array separator. diff --git a/value_array_test.go b/parse/value_array_test.go similarity index 97% rename from value_array_test.go rename to parse/value_array_test.go index 23748d5..806e54c 100644 --- a/value_array_test.go +++ b/parse/value_array_test.go @@ -1,4 +1,4 @@ -package toml +package parse import ( "testing" @@ -42,6 +42,6 @@ func TestArray(t *testing.T) { {`x=[[1],'a']`, `{}`, `type mismatch in array of static arrays: found an item of type string at line 1, column 11`}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } diff --git a/parse/value_boolean.go b/parse/value_boolean.go new file mode 100644 index 0000000..88a8191 --- /dev/null +++ b/parse/value_boolean.go @@ -0,0 +1,22 @@ +package parse + +import ( + "git.makaay.nl/mauricem/go-parsekit/parse" + "git.makaay.nl/mauricem/go-toml/ast" +) + +var falseValue = ast.NewValue(ast.TypeBoolean, false) +var trueValue = ast.NewValue(ast.TypeBoolean, true) + +// Booleans are just the tokens you're used to. Always lowercase. +func (t *parser) parseBoolean(p *parse.API) (*ast.Value, bool) { + switch { + case p.Accept(a.Str("true")): + return trueValue, true + case p.Accept(a.Str("false")): + return falseValue, true + default: + p.Expected("true or false") + return nil, false + } +} diff --git a/value_boolean_test.go b/parse/value_boolean_test.go similarity index 92% rename from value_boolean_test.go rename to parse/value_boolean_test.go index 2c03f99..21c5ca6 100644 --- a/value_boolean_test.go +++ b/parse/value_boolean_test.go @@ -1,4 +1,4 @@ -package toml +package parse import ( "testing" @@ -22,6 +22,6 @@ func TestBoolean(t *testing.T) { {`x=0`, `{"x": 0}`, ``}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } diff --git a/value_datetime.go b/parse/value_datetime.go similarity index 85% rename from value_datetime.go rename to parse/value_datetime.go index 48a4e0f..c39bed2 100644 --- a/value_datetime.go +++ b/parse/value_datetime.go @@ -1,10 +1,11 @@ -package toml +package parse import ( "time" "git.makaay.nl/mauricem/go-parsekit/parse" "git.makaay.nl/mauricem/go-parsekit/tokenize" + "git.makaay.nl/mauricem/go-toml/ast" ) var ( @@ -66,14 +67,14 @@ var ( // The full date/time parse format, based on the above definitions. // The first token denotes the type of date/time value. // The rest of the tokens contain layout fragments for time.Parse(). - offsetDateTime = tok.Str(tOffsetDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok, tzTok)) - localDateTime = tok.Str(tLocalDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok)) - localDate = tok.Str(tLocalDate, dateTok) - localTime = tok.Str(tLocalTime, c.Seq(timeTok, microTok)) + offsetDateTime = tok.Str(ast.TypeOffsetDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok, tzTok)) + localDateTime = tok.Str(ast.TypeLocalDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok)) + localDate = tok.Str(ast.TypeLocalDate, dateTok) + localTime = tok.Str(ast.TypeLocalTime, c.Seq(timeTok, microTok)) datetime = c.Any(offsetDateTime, localDateTime, localDate, localTime) ) -func (t *parser) parseDateTime(p *parse.API) (*item, bool) { +func (t *parser) parseDateTime(p *parse.API) (*ast.Value, bool) { if !p.Accept(datetime) { p.Expected("a date and/or time") return nil, false @@ -82,7 +83,7 @@ func (t *parser) parseDateTime(p *parse.API) (*item, bool) { valueType := getDateTimeValueType(&tokens) input, value, err := getDateTimeValue(&tokens) if err == nil { - return newItem(valueType, value), true + return ast.NewValue(valueType, value), true } p.Error("invalid date/time value %s: %s", input, err) return nil, false @@ -90,8 +91,8 @@ func (t *parser) parseDateTime(p *parse.API) (*item, bool) { // The first token is a token that wraps the complete date/time input. // Its type denotes the type of date/time value that it wraps. -func getDateTimeValueType(tokens *[]*tokenize.Token) valueType { - return (*tokens)[0].Type.(valueType) +func getDateTimeValueType(tokens *[]*tokenize.Token) ast.ValueType { + return (*tokens)[0].Type.(ast.ValueType) } // The rest of the tokens contain fragments that can be used with diff --git a/value_datetime_test.go b/parse/value_datetime_test.go similarity index 97% rename from value_datetime_test.go rename to parse/value_datetime_test.go index 7c49b54..24b6ac3 100644 --- a/value_datetime_test.go +++ b/parse/value_datetime_test.go @@ -1,4 +1,4 @@ -package toml +package parse import ( "testing" @@ -35,6 +35,6 @@ func TestDateTime(t *testing.T) { {`x=25:01:01`, `{}`, `invalid date/time value 25:01:01: parsing time "25:01:01": hour out of range at line 1, column 11`}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } diff --git a/value_number.go b/parse/value_number.go similarity index 86% rename from value_number.go rename to parse/value_number.go index e86e6ee..425c7ec 100644 --- a/value_number.go +++ b/parse/value_number.go @@ -1,10 +1,11 @@ -package toml +package parse import ( "math" "strconv" "git.makaay.nl/mauricem/go-parsekit/parse" + "git.makaay.nl/mauricem/go-toml/ast" ) var ( @@ -65,28 +66,28 @@ var ( nan = a.Signed(a.Str("nan")) ) -func (t *parser) parseNumber(p *parse.API) (*item, bool) { +func (t *parser) parseNumber(p *parse.API) (*ast.Value, bool) { switch { case p.Accept(tok.Float64(nil, float)): - return newItem(tFloat, p.Result().Value(0).(float64)), true + return ast.NewValue(ast.TypeFloat, p.Result().Value(0).(float64)), true case p.Accept(nan): - return newItem(tFloat, math.NaN()), true + return ast.NewValue(ast.TypeFloat, math.NaN()), true case p.Accept(inf): if p.Result().Rune(0) == '-' { - return newItem(tFloat, math.Inf(-1)), true + return ast.NewValue(ast.TypeFloat, math.Inf(-1)), true } - return newItem(tFloat, math.Inf(+1)), true + return ast.NewValue(ast.TypeFloat, math.Inf(+1)), true case p.Accept(a.Zero): return t.parseIntegerStartingWithZero(p) case p.Accept(tok.Int64(nil, integer)): - return newItem(tInteger, p.Result().Value(0).(int64)), true + return ast.NewValue(ast.TypeInteger, p.Result().Value(0).(int64)), true default: p.Expected("a number") return nil, false } } -func (t *parser) parseIntegerStartingWithZero(p *parse.API) (*item, bool) { +func (t *parser) parseIntegerStartingWithZero(p *parse.API) (*ast.Value, bool) { var value int64 var err error switch { @@ -97,10 +98,10 @@ func (t *parser) parseIntegerStartingWithZero(p *parse.API) (*item, bool) { case p.Accept(binary): value, err = strconv.ParseInt(p.Result().Value(0).(string), 2, 64) default: - return newItem(tInteger, int64(0)), true + return ast.NewValue(ast.TypeInteger, int64(0)), true } if err == nil { - return newItem(tInteger, value), true + return ast.NewValue(ast.TypeInteger, value), true } p.Error("invalid integer value 0%s: %s", p.Result().String(), err) return nil, false diff --git a/value_number_test.go b/parse/value_number_test.go similarity index 97% rename from value_number_test.go rename to parse/value_number_test.go index a547d72..8965596 100644 --- a/value_number_test.go +++ b/parse/value_number_test.go @@ -1,4 +1,4 @@ -package toml +package parse import ( "testing" @@ -75,7 +75,7 @@ func TestInteger(t *testing.T) { {`x=0b10000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000`, `{}`, `invalid integer value 0b1000000000000000000000000000000000000000000000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000000000000000000000000000000000000000000000": value out of range at line 1, column 76`}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } @@ -101,6 +101,6 @@ func TestFloat(t *testing.T) { {`x=-inf`, `{"x": -Inf}`, ``}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } diff --git a/value_string.go b/parse/value_string.go similarity index 98% rename from value_string.go rename to parse/value_string.go index 7d140db..4744ad4 100644 --- a/value_string.go +++ b/parse/value_string.go @@ -1,9 +1,10 @@ -package toml +package parse import ( "strings" "git.makaay.nl/mauricem/go-parsekit/parse" + "git.makaay.nl/mauricem/go-toml/ast" ) var ( @@ -43,7 +44,7 @@ var ( // There are four ways to express strings: basic, multi-line basic, literal and // multi-line literal. All strings must contain only valid UTF-8 characters. -func (t *parser) parseString(p *parse.API) (*item, bool) { +func (t *parser) parseString(p *parse.API) (*ast.Value, bool) { var value string var ok bool switch { @@ -59,7 +60,7 @@ func (t *parser) parseString(p *parse.API) (*item, bool) { p.Expected("a string value") } if ok { - return newItem(tString, value), ok + return ast.NewValue(ast.TypeString, value), ok } return nil, false } diff --git a/value_string_test.go b/parse/value_string_test.go similarity index 95% rename from value_string_test.go rename to parse/value_string_test.go index 65da32e..35e95c5 100644 --- a/value_string_test.go +++ b/parse/value_string_test.go @@ -1,4 +1,4 @@ -package toml +package parse import ( "fmt" @@ -46,7 +46,7 @@ func TestString(t *testing.T) { {"x='''\n literal multi-line\n string value\n'''", `{"x": " literal multi-line\n string value\n"}`, ``}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } @@ -65,7 +65,7 @@ func TestBasipString(t *testing.T) { {"x=\"Character that must be escaped \x7f\"", `{}`, `invalid character in string value: '\u007f' (must be escaped) at line 1, column 35`}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } @@ -83,7 +83,7 @@ func TestMultiLineBasipString(t *testing.T) { {"x=\"\"\"Invalid rune \xcd\"\"\"", `{}`, `invalid UTF8 rune at line 1, column 19`}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } @@ -101,7 +101,7 @@ func TestLiteralString(t *testing.T) { {"x='Invalid rune \xcd'", `{}`, `invalid UTF8 rune at line 1, column 17`}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } @@ -116,7 +116,7 @@ func TestMultiLineLiteralString(t *testing.T) { {"x='''No invalid runes allowed \xcd'''", `{}`, `invalid UTF8 rune at line 1, column 31`}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } @@ -127,6 +127,6 @@ func TestBasipStringWithUnescapedControlCharacters(t *testing.T) { p := newParser() input := fmt.Sprintf(`x="%c"`, rune(i)) expected := fmt.Sprintf(`invalid character in string value: %q (must be escaped) at line 1, column 4`, rune(i)) - testParse(t, p, p.startKeyValuePair, parseTest{input, "{}", expected}) + testParse(t, p, p.startDocument, parseTest{input, "{}", expected}) } } diff --git a/value_table.go b/parse/value_table.go similarity index 89% rename from value_table.go rename to parse/value_table.go index b9dce02..367f4fe 100644 --- a/value_table.go +++ b/parse/value_table.go @@ -1,17 +1,18 @@ -package toml +package parse import ( "git.makaay.nl/mauricem/go-parsekit/parse" + "git.makaay.nl/mauricem/go-toml/ast" ) var ( // Opener and closer for [table]. tableOpen = c.Seq(dropBlanks, a.SquareOpen, dropBlanks) - tableClose = c.Seq(dropBlanks, a.SquareClose, dropBlanks, a.EndOfLine.Or(comment)) + tableClose = c.Seq(dropBlanks, a.SquareClose, dropBlanks, a.EndOfLine.Or(dropComment)) // Opener and closer for [[array.of.tables]]. tableArrayOpen = c.Seq(dropBlanks, a.SquareOpen, a.SquareOpen, dropBlanks) - tableArrayClose = c.Seq(dropBlanks, a.SquareClose, a.SquareClose, dropBlanks, a.EndOfLine.Or(comment)) + tableArrayClose = c.Seq(dropBlanks, a.SquareClose, a.SquareClose, dropBlanks, a.EndOfLine.Or(dropComment)) // Opener, separator and closer for { inline: "tables" }. inlineTableOpen = c.Seq(dropBlanks, a.CurlyOpen, dropBlanks) @@ -74,11 +75,11 @@ func (t *parser) startArrayOfTables(p *parse.API) { p.Expected("closing ']]' for array of tables name") return } - if err := t.openArrayOfTables(key); err != nil { + if err := t.OpenArrayOfTables(key); err != nil { p.Error("%s", err) return } - p.Handle(t.startKeyValuePair) + p.Handle(t.startDocument) } } @@ -126,11 +127,11 @@ func (t *parser) startPlainTable(p *parse.API) { p.Expected("closing ']' for table name") return } - if err := t.openTable(key); err != nil { + if err := t.OpenTable(key); err != nil { p.Error("%s", err) return } - p.Handle(t.startKeyValuePair) + p.Handle(t.startDocument) } } @@ -150,34 +151,34 @@ func (t *parser) startPlainTable(p *parse.API) { // name = { first = "Tom", last = "Preston-Werner" } // point = { x = 1, y = 2 } // animal = { type.name = "pug" } -func (t *parser) parseInlineTable(p *parse.API) (*item, bool) { +func (t *parser) parseInlineTable(p *parse.API) (*ast.Value, bool) { // Check for the start of the array. if !p.Accept(inlineTableOpen) { p.Expected("an inline table") return nil, false } - subt := newParser() + subdoc := newParser() // Check for an empty inline table. if p.Accept(inlineTableClose) { - return newItem(tTable, subt.Root), true + return ast.NewValue(ast.TypeTable, subdoc.Root), true } // Not an empty table, parse the table data. for { - key, ok := subt.parseKey(p, []string{}) + key, ok := subdoc.parseKey(p, []string{}) if !ok { return nil, false } - if !p.Handle(subt.startAssignment) { + if !p.Handle(subdoc.startAssignment) { return nil, false } - value, ok := subt.parseValue(p) + value, ok := subdoc.parseValue(p) if !ok { return nil, false } - err := subt.setKeyValuePair(key, value) + err := subdoc.SetKeyValuePair(key, value) if err != nil { p.Error("%s", err) return nil, false @@ -185,7 +186,7 @@ func (t *parser) parseInlineTable(p *parse.API) (*item, bool) { // Check for the end of the inline table. if p.Accept(inlineTableClose) { - return newItem(tTable, subt.Root), true + return ast.NewValue(ast.TypeTable, subdoc.Root), true } // Not the end of the inline table? Then we should find a key/value pair separator. diff --git a/value_table_test.go b/parse/value_table_test.go similarity index 91% rename from value_table_test.go rename to parse/value_table_test.go index 1865624..9c69dce 100644 --- a/value_table_test.go +++ b/parse/value_table_test.go @@ -1,4 +1,4 @@ -package toml +package parse import ( "testing" @@ -43,7 +43,7 @@ func TestTable(t *testing.T) { {"[a]\n[b] #another table \na=1\n", `{"a": {}, "b": {"a": 1}}`, ``}, {"[a]\nx=1\ny=2\n[b] #another table \nx=1\ny=2021-01-01", `{"a": {"x": 1, "y": 2}, "b": {"x": 1, "y": 2021-01-01}}`, ``}, {"[a]\nx=1\ny=2\n[a.b] #subtable \nx=1\ny=2021-01-01", `{"a": {"b": {"x": 1, "y": 2021-01-01}, "x": 1, "y": 2}}`, ``}, - {"[a]\n[a]", `{"a": {}}`, `invalid table: table item already exists at key [a] at line 2, column 4`}, + {"[a]\n[a]", `{"a": {}}`, `invalid table: table value already exists at key [a] at line 2, column 4`}, } { p := newParser() testParse(t, p, p.startTable, test) @@ -75,7 +75,7 @@ func TestArrayOfTables(t *testing.T) { {"[[a]]\n[['a']]\n[b]\n[[\"a\"]]", `{"a": [{}, {}, {}], "b": {}}`, ``}, {"[[a]]\nx=1\n[['a']]\nx=2\ny=3\n[[\"a\"]]", `{"a": [{"x": 1}, {"x": 2, "y": 3}, {}]}`, ``}, {"[a]\n[[a.b]]\nx=1\n[[a.b]]\nx=2\n[a.c]\ny=1234", `{"a": {"b": [{"x": 1}, {"x": 2}], "c": {"y": 1234}}}`, ``}, - {"[a]\n[[a]]", `{"a": {}}`, `invalid table array: table item already exists at key [a] at line 2, column 6`}, + {"[a]\n[[a]]", `{"a": {}}`, `invalid table array: table value already exists at key [a] at line 2, column 6`}, } { p := newParser() testParse(t, p, p.startTable, test) @@ -106,10 +106,10 @@ func TestInlineTable(t *testing.T) { {"x={a=1", `{}`, `unexpected end of file (expected an array separator) at line 1, column 7`}, {"x={a=1,", `{}`, `unexpected end of file (expected a key name) at line 1, column 8`}, {"x={a=1,}", `{}`, `unexpected input (expected a key name) at line 1, column 8`}, - {"x={a=1,a=2}", `{}`, `invalid key/value pair: integer item already exists at key [a] at line 1, column 11`}, - {"x={a={b=1,b=2}}", `{}`, `invalid key/value pair: integer item already exists at key [b] at line 1, column 14`}, + {"x={a=1,a=2}", `{}`, `invalid key/value pair: integer value already exists at key [a] at line 1, column 11`}, + {"x={a={b=1,b=2}}", `{}`, `invalid key/value pair: integer value already exists at key [b] at line 1, column 14`}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } diff --git a/value_test.go b/parse/value_test.go similarity index 96% rename from value_test.go rename to parse/value_test.go index f086d8a..c30aff1 100644 --- a/value_test.go +++ b/parse/value_test.go @@ -1,4 +1,4 @@ -package toml +package parse import "testing" @@ -40,6 +40,6 @@ func TestValue(t *testing.T) { {"x={1=1,2=2,3=3}", `{"x": {"1": 1, "2": 2, "3": 3}}`, ``}, } { p := newParser() - testParse(t, p, p.startKeyValuePair, test) + testParse(t, p, p.startDocument, test) } } diff --git a/toml.go b/toml.go deleted file mode 100644 index b80fcf5..0000000 --- a/toml.go +++ /dev/null @@ -1,22 +0,0 @@ -package toml - -import ( - "git.makaay.nl/mauricem/go-parsekit/tokenize" -) - -// Some globally useful tokenizer definitions. -var ( - c, a, m, tok = tokenize.C, tokenize.A, tokenize.M, tokenize.T - - // From the specs: "Whitespace means tab (0x09) or space (0x20)." - // In this package, we name this a blank, to be in line with the - // terminology as used in parsekit. - blank = a.Runes('\t', ' ') - - // Newline means LF (0x0A) or CRLF (0x0D0A). - // This matches the default newline as defined by parsekit. - newline = a.Newline - - dropBlanks = m.Drop(c.ZeroOrMore(blank)) - dropWhitespace = m.Drop(c.ZeroOrMore(blank.Or(newline))) -) diff --git a/value_boolean.go b/value_boolean.go deleted file mode 100644 index fc954c0..0000000 --- a/value_boolean.go +++ /dev/null @@ -1,21 +0,0 @@ -package toml - -import ( - "git.makaay.nl/mauricem/go-parsekit/parse" -) - -var falseItem = newItem(tBoolean, false) -var trueItem = newItem(tBoolean, true) - -// Booleans are just the tokens you're used to. Always lowercase. -func (t *parser) parseBoolean(p *parse.API) (*item, bool) { - switch { - case p.Accept(a.Str("true")): - return trueItem, true - case p.Accept(a.Str("false")): - return falseItem, true - default: - p.Expected("true or false") - return nil, false - } -}