From 8838dc9c4499924b74da11cfa22e85f1b8654495 Mon Sep 17 00:00:00 2001 From: Maurice Makaay Date: Sun, 23 Jun 2019 12:05:52 +0000 Subject: [PATCH] Backup work. --- ast.go | 199 +++++++++++++++++++++++++++++++++++++++++ ast_test.go | 72 +++++++++++++++ comment.go | 4 +- comment_test.go | 4 +- helpers_test.go | 25 ++++-- keyvaluepair.go | 30 +++++-- keyvaluepair_test.go | 173 ++++++++++++++++++----------------- toml.go | 73 +++------------ toml_test.go | 34 ------- value.go | 4 +- value_array.go | 86 ++++++++++++++++++ value_array_test.go | 40 +++++++++ value_boolean.go | 6 +- value_boolean_test.go | 8 +- value_datetime.go | 16 ++-- value_datetime_test.go | 34 +++---- value_number.go | 16 ++-- value_number_test.go | 126 +++++++++++++------------- value_string.go | 35 ++++---- value_string_test.go | 76 ++++++++-------- value_table.go | 111 +++++++++++++++++++++++ value_table_test.go | 23 +++++ value_test.go | 67 +++++++------- 23 files changed, 876 insertions(+), 386 deletions(-) create mode 100644 ast.go create mode 100644 ast_test.go delete mode 100644 toml_test.go create mode 100644 value_array.go create mode 100644 value_array_test.go create mode 100644 value_table.go create mode 100644 value_table_test.go diff --git a/ast.go b/ast.go new file mode 100644 index 0000000..cfcc6a7 --- /dev/null +++ b/ast.go @@ -0,0 +1,199 @@ +package toml + +import ( + "fmt" + "strings" + "time" +) + +// item represents a TOML item. +type item struct { + Type itemType + Data []interface{} +} + +// table represents a TOML table. +type table map[string]item + +// itemType identifies the semantic role of a TOML item. +type itemType string + +const ( + // TODO Would be nice to not need these in the end. + pComment itemType = "comment" + pKey itemType = "key" + pAssign itemType = "assign" + + // TODO and just use these data types. + pString itemType = "string" // "various", 'types', """of""", '''strings''' + pInteger itemType = "integer" // 12345, 0xffee12a0, 0o0755, 0b101101011 + pFloat itemType = "float" // 10.1234, 143E-12, 43.28377e+4, +inf, -inf, nan + pBoolean itemType = "boolean" // true or false + pOffsetDateTime itemType = "offset datetime" // 2019-06-18 10:32:15.173645362+0200 + pLocalDateTime itemType = "datetime" // 2018-12-25 12:12:18.876772533 + pLocalDate itemType = "date" // 2017-05-17 + pLocalTime itemType = "time" // 23:01:22 + pArray itemType = "array" // defined using an [[array.of.tables]] + pStaticArray itemType = "static array" // defined using ["an", "inline", "array"] + pTable itemType = "table" // defined using { "inline" = "table" } or [standard.table] +) + +// newItem instantiates a new item struct. +func newItem(itemType itemType, data ...interface{}) item { + return item{Type: itemType, Data: data} +} + +func (t table) String() string { + return newItem(pTable, t).String() +} + +func (parseItem item) String() string { + switch parseItem.Type { + case pString: + return fmt.Sprintf("%q", parseItem.Data[0]) + case pInteger: + return fmt.Sprintf("%v", parseItem.Data[0]) + case pFloat: + return fmt.Sprintf("%v", parseItem.Data[0]) + case pBoolean: + return fmt.Sprintf("%v", parseItem.Data[0]) + case pOffsetDateTime: + return parseItem.Data[0].(time.Time).Format(time.RFC3339Nano) + case pLocalDateTime: + return parseItem.Data[0].(time.Time).Format("2006-01-02 15:04:05.999999999") + case pLocalDate: + return parseItem.Data[0].(time.Time).Format("2006-01-02") + case pLocalTime: + return parseItem.Data[0].(time.Time).Format("15:04:05.999999999") + case pArray: + fallthrough + case pStaticArray: + items := make([]string, len(parseItem.Data[0].([]item))) + for i, d := range parseItem.Data[0].([]item) { + items[i] = d.String() + } + return fmt.Sprintf("[%s]", strings.Join(items, ", ")) + case pTable: + items := make([]string, len(parseItem.Data)) + pairs := parseItem.Data[0].(table) + i := 0 + for k, v := range pairs { + items[i] = fmt.Sprintf("%q: %s", k, v.String()) + i++ + } + return fmt.Sprintf("{%s}", strings.Join(items, ", ")) + case pComment: + return fmt.Sprintf("comment(%q)", parseItem.Data[0]) + case pKey: + items := make([]string, len(parseItem.Data)) + for i, e := range parseItem.Data { + items[i] = fmt.Sprintf("%q", e) + } + return fmt.Sprintf("key(%s)", strings.Join(items, ", ")) + case pAssign: + return "=" + default: + panic(fmt.Sprintf("Missing String() formatting for item type '%s'", parseItem.Type)) + } +} + +// parser holds the state for the TOML parser. All parsing functions are +// methods of this struct. +type parser struct { + Items []item // a buffer for holding parsed items + Root table // the root-level TOML table (each TOML doc is implicitly a table) + Current table // the currently active TOML table +} + +func newParser() *parser { + p := &parser{Root: make(table)} + p.Current = p.Root + return p +} + +func (t *parser) addParsedItem(itemType itemType, data ...interface{}) { + t.Items = append(t.Items, newItem(itemType, data...)) +} + +func (t *parser) setValue(key item, value item) error { + // When the key has multiple elements, then first make sure the table structure + // for storing the value exists. + var valueKey string + node := t.Current + l := len(key.Data) + if l > 1 { + pathKeys := key.Data[0 : l-1] + valueKey = key.Data[l-1].(string) + for i, name := range pathKeys { + name := name.(string) + if subItem, ok := node[name]; ok { + // An item was found at the current key. It is expected to be a table. + if subItem.Type != pTable { + path := formatKeyPath(key, i) + return fmt.Errorf("invalid key used: %s item already exists at key %s", subItem.Type, path) + } + node = subItem.Data[0].(table) + } else { + // No item was found at the current key. Create a new subtable. + subTable := make(table) + node[name] = newItem(pTable, subTable) + node = subTable + } + } + } else { + valueKey = key.Data[0].(string) + node = t.Current + } + + if existing, ok := node[valueKey]; ok { + path := "moetnog" + return fmt.Errorf("Cannot store value: %s item already exists at key %s", existing.Type, path) + } + + node[valueKey] = value + return nil +} + +func (t *parser) newTable(key item) (table, error) { + node := t.Root + // Go over all requested levels of the key. For all levels, except the last + // one, it is okay if a table already exists. For at least the last level, + // no table or value must exist, because that would mean we are overwriting + // an existing key/value pair, which is not allowed. + for i, e := range key.Data { + name := e.(string) + if subItem, ok := node[name]; ok { + // You cannot overwrite an already defined key, regardless its value. + if subItem.Type != pTable { + path := formatKeyPath(key, i) + return nil, fmt.Errorf("Cannot create table: %s item already exists at key %s", subItem.Type, path) + } + // Like keys, you cannot define any table more than once. Doing so is invalid. + isLast := i == len(key.Data)-1 + if isLast { + path := formatKeyPath(key, i) + return nil, fmt.Errorf("Cannot create table: table for key %s already exists", path) + } + node = subItem.Data[0].(table) + } else { + // Create the subtable. + subTable := make(table) + node[name] = newItem(pTable, subTable) + node = subTable + } + } + // From here on, key/value pairs are added to the newly defined table. + t.Current = node + return node, nil +} + +func formatKeyPath(key item, end int) string { + var sb strings.Builder + for i := 0; i <= end; i++ { + if i > 0 { + sb.WriteRune('.') + } + sb.WriteString(fmt.Sprintf("%q", key.Data[i].(string))) + } + return sb.String() +} diff --git a/ast_test.go b/ast_test.go new file mode 100644 index 0000000..d3405bc --- /dev/null +++ b/ast_test.go @@ -0,0 +1,72 @@ +package toml + +import ( + "fmt" + "testing" +) + +func TestAST_ConstructStructure(t *testing.T) { + p := newParser() + p.Root["ding"] = newItem(pInteger, 10) + p.Root["dong"] = newItem(pString, "not a song") + subTable1, _ := p.newTable(newItem(pKey, "key1", "key2 a")) + subTable1["dooh"] = newItem(pBoolean, true) + subTable1["dah"] = newItem(pBoolean, false) + subTable2, _ := p.newTable(newItem(pKey, "key1", "key2 b")) + subTable2["dieh"] = newItem(pFloat, 1.111) + subTable2["duhh"] = newItem(pFloat, 1.18e-12) +} + +func TestAST_StoreValueInRootTable(t *testing.T) { + testError(t, func() error { + p := newParser() + p.setValue(newItem(pKey, "key1"), newItem(pString, "value1")) + return p.setValue(newItem(pKey, "key2"), newItem(pString, "value2")) + }, "") +} + +func TestAST_StoreValueWithMultipartKey_CreatesTableHierarchy(t *testing.T) { + testError(t, func() error { + p := newParser() + return p.setValue(newItem(pKey, "key1", "key2", "key3"), newItem(pString, "value")) + }, "") + // TODO an actual test assertion +} + +func TestAST_StoreValueWithMultipartKey_UnderSubtable_CreatesTableHierarchy(t *testing.T) { + testError(t, func() error { + p := newParser() + p.newTable(newItem(pKey, "subkey1", "subkey2")) + err := p.setValue(newItem(pKey, "key1", "key2", "key3"), newItem(pString, "value")) + fmt.Printf("%s", p.Root) + return err + }, "") + t.Fail() + // TODO an actual test assertion +} + +func TestAST_StoreDuplicateKeyInRootTable_ReturnsError(t *testing.T) { + testError(t, func() error { + p := newParser() + p.setValue(newItem(pKey, "key"), newItem(pString, "value")) + return p.setValue(newItem(pKey, "key"), newItem(pInteger, 321)) + }, `Cannot store value: string item already exists at key "key"`) +} + +func TestAST_GivenExistingTableAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { + testError(t, func() error { + p := newParser() + p.newTable(newItem(pKey, "key1", "key2")) + _, err := p.newTable(newItem(pKey, "key1", "key2")) + return err + }, `Cannot create table: table for key "key1"."key2" already exists`) +} + +func TestAST_GivenExistingItemAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) { + testError(t, func() error { + p := newParser() + p.Root["key"] = newItem(pString, "value") + _, err := p.newTable(newItem(pKey, "key")) + return err + }, `Cannot create table: string item already exists at key "key"`) +} diff --git a/comment.go b/comment.go index a7f5835..d8d54e9 100644 --- a/comment.go +++ b/comment.go @@ -1,4 +1,4 @@ -package parser +package toml import ( "git.makaay.nl/mauricem/go-parsekit/parse" @@ -10,7 +10,7 @@ var comment = c.Seq(a.Hash, c.ZeroOrMore(c.Not(a.EndOfLine)), m.Drop(a.EndOfLine func (t *parser) startComment(p *parse.API) { if p.Accept(comment) { - t.emitCommand(cComment, p.Result().String()) + t.addParsedItem(pComment, p.Result().String()) } else { p.Expected("comment") } diff --git a/comment_test.go b/comment_test.go index 4b7cd3d..5979e29 100644 --- a/comment_test.go +++ b/comment_test.go @@ -1,4 +1,4 @@ -package parser +package toml import ( "testing" @@ -18,7 +18,7 @@ func TestComment2(t *testing.T) { `comment("# with data and newline")`, `Error: unexpected input (expected end of file) at line 2, column 1`}}, } { - p := &parser{} + p := newParser() testParseHandler(t, p, p.startComment, test) } } diff --git a/helpers_test.go b/helpers_test.go index 6a3db51..9f76c0e 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -1,4 +1,4 @@ -package parser +package toml import ( "fmt" @@ -24,8 +24,8 @@ func testParseHandler(t *testing.T, p *parser, handler parse.Handler, test parse defer func() { recovered := recover() results := []string{} - for _, cmd := range p.commands { - results = append(results, cmd.String()) + for _, item := range p.Items { + results = append(results, item.String()) } if err != nil { results = append(results, fmt.Sprintf("Error: %s", err)) @@ -36,12 +36,12 @@ func testParseHandler(t *testing.T, p *parser, handler parse.Handler, test parse for i, e := range test.expected { if i > len(results)-1 { - t.Errorf("No result at index %d, expected: %s", i, e) + t.Errorf("No result at index %d for input %q, expected: %s", i, test.input, e) continue } r := results[i] if e != r { - t.Errorf("Unexpected result at index %d:\nexpected: %s\nactual: %s\n", i, e, r) + t.Errorf("Unexpected result at index %d for input %q:\nexpected: %s\nactual: %s\n", i, test.input, e, r) } } if len(results) > len(test.expected) { @@ -53,3 +53,18 @@ func testParseHandler(t *testing.T, p *parser, handler parse.Handler, test parse }() err = parse.New(handler)(test.input) } + +func testError(t *testing.T, code func() error, expected string) { + err := code() + if expected == "" { + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + } else { + if err == nil { + t.Fatalf("An error was expected, but no error was returned") + } else if err.Error() != expected { + t.Fatalf("Unexpected error:\nexpected: %s\nactual: %s\n", expected, err.Error()) + } + } +} diff --git a/keyvaluepair.go b/keyvaluepair.go index 71c47af..9eed3a2 100644 --- a/keyvaluepair.go +++ b/keyvaluepair.go @@ -1,4 +1,4 @@ -package parser +package toml import ( "git.makaay.nl/mauricem/go-parsekit/parse" @@ -7,9 +7,6 @@ import ( // The primary building block of a TOML document is the key/value pair. var ( - dropWhitespace = m.Drop(a.Whitespace.Optional()) - dropBlanks = m.Drop(a.Blanks.Optional()) - // Keys are on the left of the equals sign and values are on the right. // Blank is ignored around key names and values. The key, equals // sign, and value must be on the same line (though some values can be @@ -73,13 +70,13 @@ func (t *parser) startKey(p *parse.API) { case p.Peek(a.SingleQuote): key, ok = t.parseLiteralString("key", p) case p.Peek(a.DoubleQuote): - key, ok = t.parseBasicString("key", p) + key, ok = t.parseBasipString("key", p) default: p.Expected("a key name") return } if ok { - t.emitCommand(cKey, key) + t.addParsedItem(pKey, key) p.Handle(t.endOfKeyOrDot) } } @@ -90,14 +87,31 @@ func (t *parser) startKey(p *parse.API) { // practice is to not use any extraneous whitespace. func (t *parser) endOfKeyOrDot(p *parse.API) { if p.Accept(keySeparatorDot) { - t.emitCommand(cKeyDot) p.Handle(t.startKey) + return } + + // TODO not sure if we really need this index. + // Can't we alway simply use the full item list, given that we'll feed all + // results to the parser's table state? + // Do we even need to emit a key here? Shouldn't we just create the + // table structure in the parser, ready for followup calls to fill the data? + keyStart := len(t.Items) - 1 + for keyStart > 0 && t.Items[keyStart-1].Type == pKey { + keyStart-- + } + keyLen := len(t.Items) - keyStart + key := make([]interface{}, keyLen) + for i := 0; i < keyLen; i++ { + key[i] = t.Items[keyStart+i].Data[0].(string) + } + t.Items = t.Items[0:keyStart] + t.addParsedItem(pKey, key...) } func (t *parser) startAssignment(p *parse.API) { if p.Accept(keyAssignment) { - t.emitCommand(cAssign) + t.addParsedItem(pAssign) } else { p.Expected("a value assignment") } diff --git a/keyvaluepair_test.go b/keyvaluepair_test.go index c18c98f..99c4000 100644 --- a/keyvaluepair_test.go +++ b/keyvaluepair_test.go @@ -1,4 +1,4 @@ -package parser +package toml import "testing" @@ -11,10 +11,10 @@ func TestKey(t *testing.T) { {"mix-12_34", []string{`key("mix-12_34")`}}, {"-hey_good_Lookin123-", []string{`key("-hey_good_Lookin123-")`}}, {"wrong!", []string{`key("wrong")`, `Error: unexpected input (expected end of file) at line 1, column 6`}}, - {"key1.", []string{`key("key1")`, `keydot`, `Error: unexpected end of file (expected a key name) at line 1, column 6`}}, - {"key1.key2", []string{`key("key1")`, `keydot`, `key("key2")`}}, - {"key . with . spaces", []string{`key("key")`, `keydot`, `key("with")`, `keydot`, `key("spaces")`}}, - {"key \t . \twithtabs\t . \tandspaces", []string{`key("key")`, `keydot`, `key("withtabs")`, `keydot`, `key("andspaces")`}}, + {"key1.", []string{`key("key1")`, `Error: unexpected end of file (expected a key name) at line 1, column 6`}}, + {"key1.key2", []string{`key("key1", "key2")`}}, + {"key . with . spaces", []string{`key("key", "with", "spaces")`}}, + {"key \t . \twithtabs\t . \tandspaces", []string{`key("key", "withtabs", "andspaces")`}}, // Single quoted key tests {"''", []string{`key("")`}}, {"'single quoted'", []string{`key("single quoted")`}}, @@ -26,9 +26,9 @@ func TestKey(t *testing.T) { {`"escapes are in\terpreted"`, []string{`key("escapes are in\terpreted")`}}, {`"using 'inner' \"quotes\""`, []string{`key("using 'inner' \"quotes\"")`}}, // Mixed key types - {`this.'i\s'."madness\t".''`, []string{`key("this")`, `keydot`, `key("i\\s")`, `keydot`, `key("madness\t")`, `keydot`, `key("")`}}, + {`this.'i\s'."madness\t".''`, []string{`key("this", "i\\s", "madness\t", "")`}}, } { - p := &parser{} + p := newParser() testParseHandler(t, p, p.startKey, test) } } @@ -36,11 +36,11 @@ func TestKey(t *testing.T) { func TestAssignment(t *testing.T) { for _, test := range []parseTest{ {"", []string{`Error: unexpected end of file (expected a value assignment) at start of file`}}, - {"=", []string{`assign`}}, - {" \t = \t ", []string{`assign`}}, + {"=", []string{`=`}}, + {" \t = \t ", []string{`=`}}, {" \n = \n ", []string{`Error: unexpected input (expected a value assignment) at start of file`}}, } { - p := &parser{} + p := newParser() testParseHandler(t, p, p.startAssignment, test) } } @@ -51,94 +51,103 @@ func TestKeyValuePair(t *testing.T) { {" ", []string{}}, {" \t ", []string{}}, {" key ", []string{`key("key")`, `Error: unexpected input (expected a value assignment) at line 1, column 5`}}, - {" key \t=", []string{`key("key")`, `assign`, `Error: unexpected end of file (expected a value) at line 1, column 8`}}, - {"key = # INVALID", []string{`key("key")`, `assign`, `Error: unexpected input (expected a value) at line 1, column 7`}}, - {" key \t =\t \"The Value\" \r\n", []string{`key("key")`, `assign`, `string("The Value")`}}, - {`3.14159 = "pi"`, []string{`key("3")`, `keydot`, `key("14159")`, `assign`, `string("pi")`}}, - {`"ʎǝʞ" = "value"`, []string{`key("ʎǝʞ")`, `assign`, `string("value")`}}, - {`key = "value" # This is a comment at the end of a line`, []string{`key("key")`, `assign`, `string("value")`, `comment("# This is a comment at the end of a line")`}}, - {`another = "# This is not a comment"`, []string{`key("another")`, `assign`, `string("# This is not a comment")`}}, - {"key1=\"value1\"key2=\"value2\"\r\nkey3=\"value3\"", []string{ - `key("key1")`, `assign`, `string("value1")`, - `key("key2")`, `assign`, `string("value2")`, - `key("key3")`, `assign`, `string("value3")`}}, + {" key \t=", []string{`key("key")`, `=`, `Error: unexpected end of file (expected a value) at line 1, column 8`}}, + {"key = # INVALID", []string{`key("key")`, `=`, `Error: unexpected input (expected a value) at line 1, column 7`}}, + {" key \t =\t \"The Value\" \r\n", []string{`key("key")`, `=`, `"The Value"`}}, + {`3.14159 = "pi"`, []string{`key("3", "14159")`, `=`, `"pi"`}}, + {`"ʎǝʞ" = "value"`, []string{`key("ʎǝʞ")`, `=`, `"value"`}}, + {`key = "value" # This is a comment at the end of a line`, []string{`key("key")`, `=`, `"value"`, `comment("# This is a comment at the end of a line")`}}, + {`another = "# This is not a comment"`, []string{`key("another")`, `=`, `"# This is not a comment"`}}, + {"key1=\"value1\"key2=\"value2\"\r\nkey3a.key3b=\"value3\"", []string{ + `key("key1")`, `=`, `"value1"`, + `key("key2")`, `=`, `"value2"`, + `key("key3a", "key3b")`, `=`, `"value3"`}}, {"with=\"comments\"# boring \nanother.cool =\"one\" \t # to the end\r\n", []string{ - `key("with")`, `assign`, `string("comments")`, `comment("# boring ")`, - `key("another")`, `keydot`, `key("cool")`, `assign`, `string("one")`, `comment("# to the end")`}}, + `key("with")`, `=`, `"comments"`, `comment("# boring ")`, + `key("another", "cool")`, `=`, `"one"`, `comment("# to the end")`}}, } { - p := &parser{} + p := newParser() testParseHandler(t, p, p.startKeyValuePair, test) } } func TestKeyValuePair_ForAllTypes(t *testing.T) { for _, test := range []parseTest{ - {"string='literal'", []string{`key("string")`, `assign`, `string("literal")`}}, - {"string='''literal\nmulti-line'''", []string{`key("string")`, `assign`, `string("literal\nmulti-line")`}}, - {`string="basic"`, []string{`key("string")`, `assign`, `string("basic")`}}, - {"string=\"\"\"basic\nmulti-line\"\"\"", []string{`key("string")`, `assign`, `string("basic\nmulti-line")`}}, - {"integer=1_234_567", []string{`key("integer")`, `assign`, `integer(1234567)`}}, - {"integer=42", []string{`key("integer")`, `assign`, `integer(42)`}}, - {"integer=0x42", []string{`key("integer")`, `assign`, `integer(66)`}}, - {"integer=0o42", []string{`key("integer")`, `assign`, `integer(34)`}}, - {"integer=0b101010", []string{`key("integer")`, `assign`, `integer(42)`}}, - {"float=42.37", []string{`key("float")`, `assign`, `float(42.37)`}}, - {"float=42e+37", []string{`key("float")`, `assign`, `float(4.2e+38)`}}, - {"float=42.37e-11", []string{`key("float")`, `assign`, `float(4.237e-10)`}}, - {"boolean=true", []string{`key("boolean")`, `assign`, `boolean(true)`}}, - {"boolean=false", []string{`key("boolean")`, `assign`, `boolean(false)`}}, - {"date=2019-01-01", []string{`key("date")`, `assign`, `date(2019-01-01 00:00:00 +0000 UTC)`}}, - {"time=15:03:11", []string{`key("time")`, `assign`, `time(0000-01-01 15:03:11 +0000 UTC)`}}, - {"datetime=2021-02-01 15:03:11.123", []string{`key("datetime")`, `assign`, `datetime(2021-02-01 15:03:11.123 +0000 UTC)`}}, - {"offset_datetime=1111-11-11 11:11:11.111111111+11:11", []string{`key("offset_datetime")`, `assign`, `offset_datetime(1111-11-11 11:11:11.111111111 +1111 +1111)`}}, + {"string='literal'", []string{`key("string")`, `=`, `"literal"`}}, + {"string='''literal\nmulti-line'''", []string{`key("string")`, `=`, `"literal\nmulti-line"`}}, + {`string="basic"`, []string{`key("string")`, `=`, `"basic"`}}, + {"string=\"\"\"basic\nmulti-line\"\"\"", []string{`key("string")`, `=`, `"basic\nmulti-line"`}}, + {"integer=1_234_567", []string{`key("integer")`, `=`, `1234567`}}, + {"integer=42", []string{`key("integer")`, `=`, `42`}}, + {"integer=0x42", []string{`key("integer")`, `=`, `66`}}, + {"integer=0o42", []string{`key("integer")`, `=`, `34`}}, + {"integer=0b101010", []string{`key("integer")`, `=`, `42`}}, + {"float=42.37", []string{`key("float")`, `=`, `42.37`}}, + {"float=42e+37", []string{`key("float")`, `=`, `4.2e+38`}}, + {"float=42.37e-11", []string{`key("float")`, `=`, `4.237e-10`}}, + {"boolean=true", []string{`key("boolean")`, `=`, `true`}}, + {"boolean=false", []string{`key("boolean")`, `=`, `false`}}, + {"date=2019-01-01", []string{`key("date")`, `=`, `2019-01-01`}}, + {"time=15:03:11", []string{`key("time")`, `=`, `15:03:11`}}, + {"datetime=2021-02-01 15:03:11.123", []string{`key("datetime")`, `=`, `2021-02-01 15:03:11.123`}}, + {"offset_datetime=1111-11-11 11:11:11.111111111+11:11", []string{`key("offset_datetime")`, `=`, `1111-11-11T11:11:11.111111111+11:11`}}, + {"static_array=['a', 'static', 'array']", []string{`key("static_array")`, `=`, `["a", "static", "array"]`}}, } { - p := &parser{} + p := newParser() testParseHandler(t, p, p.startKeyValuePair, test) } } func TestKeyValuePair_ExamplesFromSpecification(t *testing.T) { for _, test := range []parseTest{ - {"int1 = +99", []string{`key("int1")`, `assign`, `integer(99)`}}, - {"int2 = 42", []string{`key("int2")`, `assign`, `integer(42)`}}, - {"int3 = 0", []string{`key("int3")`, `assign`, `integer(0)`}}, - {"int4 = -17", []string{`key("int4")`, `assign`, `integer(-17)`}}, - {"int5 = 1_000", []string{`key("int5")`, `assign`, `integer(1000)`}}, - {"int6 = 5_349_221", []string{`key("int6")`, `assign`, `integer(5349221)`}}, - {"int7 = 1_2_3_4_5 # VALID but discouraged", []string{`key("int7")`, `assign`, `integer(12345)`, `comment("# VALID but discouraged")`}}, - {"hex1 = 0xDEADBEEF", []string{`key("hex1")`, `assign`, `integer(3735928559)`}}, - {"hex2 = 0xdeadbeef", []string{`key("hex2")`, `assign`, `integer(3735928559)`}}, - {"hex3 = 0xdead_beef", []string{`key("hex3")`, `assign`, `integer(3735928559)`}}, - {"oct1 = 0o01234567", []string{`key("oct1")`, `assign`, `integer(342391)`}}, - {"oct2 = 0o755", []string{`key("oct2")`, `assign`, `integer(493)`}}, - {"bin1 = 0b11010110", []string{`key("bin1")`, `assign`, `integer(214)`}}, - {"flt1 = +1.0", []string{`key("flt1")`, `assign`, `float(1)`}}, - {"flt2 = 3.1415", []string{`key("flt2")`, `assign`, `float(3.1415)`}}, - {"flt3 = -0.01", []string{`key("flt3")`, `assign`, `float(-0.01)`}}, - {"flt4 = 5e+22", []string{`key("flt4")`, `assign`, `float(5e+22)`}}, - {"flt5 = 1e6", []string{`key("flt5")`, `assign`, `float(1e+06)`}}, - {"flt6 = -2E-2", []string{`key("flt6")`, `assign`, `float(-0.02)`}}, - {"flt7 = 6.626e-34", []string{`key("flt7")`, `assign`, `float(6.626e-34)`}}, - {"flt8 = 224_617.445_991_228", []string{`key("flt8")`, `assign`, `float(224617.445991228)`}}, - {"sf1 = inf # positive infinity", []string{`key("sf1")`, `assign`, `float(+Inf)`, `comment("# positive infinity")`}}, - {"sf2 = +inf # positive infinity", []string{`key("sf2")`, `assign`, `float(+Inf)`, `comment("# positive infinity")`}}, - {"sf3 = -inf # negative infinity", []string{`key("sf3")`, `assign`, `float(-Inf)`, `comment("# negative infinity")`}}, - {"sf4 = nan # actual sNaN/qNaN encoding is implementation-specific", []string{`key("sf4")`, `assign`, `float(NaN)`, `comment("# actual sNaN/qNaN encoding is implementation-specific")`}}, - {"sf5 = +nan # same as `nan`", []string{`key("sf5")`, `assign`, `float(NaN)`, "comment(\"# same as `nan`\")"}}, - {"sf6 = -nan # valid, actual encoding is implementation-specific", []string{`key("sf6")`, `assign`, `float(NaN)`, `comment("# valid, actual encoding is implementation-specific")`}}, - {"bool1 = true", []string{`key("bool1")`, `assign`, `boolean(true)`}}, - {"bool2 = false", []string{`key("bool2")`, `assign`, `boolean(false)`}}, - {"odt1 = 1979-05-27T07:32:00Z", []string{`key("odt1")`, `assign`, `offset_datetime(1979-05-27 07:32:00 +0000 UTC)`}}, - {"odt2 = 1979-05-27T00:32:00-07:00", []string{`key("odt2")`, `assign`, `offset_datetime(1979-05-27 00:32:00 -0700 -0700)`}}, - {"odt3 = 1979-05-27T00:32:00.999999-07:00", []string{`key("odt3")`, `assign`, `offset_datetime(1979-05-27 00:32:00.999999 -0700 -0700)`}}, - {"odt4 = 1979-05-27 07:32:00Z", []string{`key("odt4")`, `assign`, `offset_datetime(1979-05-27 07:32:00 +0000 UTC)`}}, - {"ldt1 = 1979-05-27T07:32:00", []string{`key("ldt1")`, `assign`, `datetime(1979-05-27 07:32:00 +0000 UTC)`}}, - {"ldt2 = 1979-05-27T00:32:00.999999", []string{`key("ldt2")`, `assign`, `datetime(1979-05-27 00:32:00.999999 +0000 UTC)`}}, - {"ld1 = 1979-05-27", []string{`key("ld1")`, `assign`, `date(1979-05-27 00:00:00 +0000 UTC)`}}, - {"lt1 = 07:32:00", []string{`key("lt1")`, `assign`, `time(0000-01-01 07:32:00 +0000 UTC)`}}, - {"lt2 = 00:32:00.999999", []string{`key("lt2")`, `assign`, `time(0000-01-01 00:32:00.999999 +0000 UTC)`}}, + {"int1 = +99", []string{`key("int1")`, `=`, `99`}}, + {"int2 = 42", []string{`key("int2")`, `=`, `42`}}, + {"int3 = 0", []string{`key("int3")`, `=`, `0`}}, + {"int4 = -17", []string{`key("int4")`, `=`, `-17`}}, + {"int5 = 1_000", []string{`key("int5")`, `=`, `1000`}}, + {"int6 = 5_349_221", []string{`key("int6")`, `=`, `5349221`}}, + {"int7 = 1_2_3_4_5 # VALID but discouraged", []string{`key("int7")`, `=`, `12345`, `comment("# VALID but discouraged")`}}, + {"hex1 = 0xDEADBEEF", []string{`key("hex1")`, `=`, `3735928559`}}, + {"hex2 = 0xdeadbeef", []string{`key("hex2")`, `=`, `3735928559`}}, + {"hex3 = 0xdead_beef", []string{`key("hex3")`, `=`, `3735928559`}}, + {"oct1 = 0o01234567", []string{`key("oct1")`, `=`, `342391`}}, + {"oct2 = 0o755", []string{`key("oct2")`, `=`, `493`}}, + {"bin1 = 0b11010110", []string{`key("bin1")`, `=`, `214`}}, + {"flt1 = +1.0", []string{`key("flt1")`, `=`, `1`}}, + {"flt2 = 3.1415", []string{`key("flt2")`, `=`, `3.1415`}}, + {"flt3 = -0.01", []string{`key("flt3")`, `=`, `-0.01`}}, + {"flt4 = 5e+22", []string{`key("flt4")`, `=`, `5e+22`}}, + {"flt5 = 1e6", []string{`key("flt5")`, `=`, `1e+06`}}, + {"flt6 = -2E-2", []string{`key("flt6")`, `=`, `-0.02`}}, + {"flt7 = 6.626e-34", []string{`key("flt7")`, `=`, `6.626e-34`}}, + {"flt8 = 224_617.445_991_228", []string{`key("flt8")`, `=`, `224617.445991228`}}, + {"sf1 = inf # positive infinity", []string{`key("sf1")`, `=`, `+Inf`, `comment("# positive infinity")`}}, + {"sf2 = +inf # positive infinity", []string{`key("sf2")`, `=`, `+Inf`, `comment("# positive infinity")`}}, + {"sf3 = -inf # negative infinity", []string{`key("sf3")`, `=`, `-Inf`, `comment("# negative infinity")`}}, + {"sf4 = nan # actual sNaN/qNaN encoding is implementation-specific", []string{`key("sf4")`, `=`, `NaN`, `comment("# actual sNaN/qNaN encoding is implementation-specific")`}}, + {"sf5 = +nan # same as `nan`", []string{`key("sf5")`, `=`, `NaN`, "comment(\"# same as `nan`\")"}}, + {"sf6 = -nan # valid, actual encoding is implementation-specific", []string{`key("sf6")`, `=`, `NaN`, `comment("# valid, actual encoding is implementation-specific")`}}, + {"bool1 = true", []string{`key("bool1")`, `=`, `true`}}, + {"bool2 = false", []string{`key("bool2")`, `=`, `false`}}, + {"odt1 = 1979-05-27T07:32:00Z", []string{`key("odt1")`, `=`, `1979-05-27T07:32:00Z`}}, + {"odt2 = 1979-05-27T00:32:00-07:00", []string{`key("odt2")`, `=`, `1979-05-27T00:32:00-07:00`}}, + {"odt3 = 1979-05-27T00:32:00.999999-07:00", []string{`key("odt3")`, `=`, `1979-05-27T00:32:00.999999-07:00`}}, + {"odt4 = 1979-05-27 07:32:00Z", []string{`key("odt4")`, `=`, `1979-05-27T07:32:00Z`}}, + {"ldt1 = 1979-05-27T07:32:00", []string{`key("ldt1")`, `=`, `1979-05-27 07:32:00`}}, + {"ldt2 = 1979-05-27T00:32:00.999999", []string{`key("ldt2")`, `=`, `1979-05-27 00:32:00.999999`}}, + {"ld1 = 1979-05-27", []string{`key("ld1")`, `=`, `1979-05-27`}}, + {"lt1 = 07:32:00", []string{`key("lt1")`, `=`, `07:32:00`}}, + {"lt2 = 00:32:00.999999", []string{`key("lt2")`, `=`, `00:32:00.999999`}}, + {"arr1 = [ 1, 2, 3 ]", []string{`key("arr1")`, `=`, `[1, 2, 3]`}}, + {`arr2 = [ "red", "yellow", "green" ]`, []string{`key("arr2")`, `=`, `["red", "yellow", "green"]`}}, + {`arr3 = [ [ 1, 2 ], [3, 4, 5] ]`, []string{`key("arr3")`, `=`, `[[1, 2], [3, 4, 5]]`}}, + {`arr4 = [ "all", 'strings', """are the same""", '''type''']`, []string{`key("arr4")`, `=`, `["all", "strings", "are the same", "type"]`}}, + {`arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]`, []string{`key("arr5")`, `=`, `[[1, 2], ["a", "b", "c"]]`}}, + {`arr6 = [ 1, 2.0 ] # INVALID`, []string{`key("arr6")`, `=`, `Error: type mismatch in array of integers: found an item of type float at line 1, column 16`}}, + {"arr7 = [\n 1, 2, 3\n]", []string{`key("arr7")`, `=`, `[1, 2, 3]`}}, + {"arr8 = [\n 1,\n 2, # this is ok\n]", []string{`key("arr8")`, `=`, `[1, 2]`}}, } { - p := &parser{} + p := newParser() testParseHandler(t, p, p.startKeyValuePair, test) } } diff --git a/toml.go b/toml.go index 9e68e15..b80fcf5 100644 --- a/toml.go +++ b/toml.go @@ -1,67 +1,22 @@ -package parser +package toml import ( - "fmt" - "strings" - "git.makaay.nl/mauricem/go-parsekit/tokenize" ) -// Easy access to the parsekit.tokenize definitions. -var c, a, m, tok = tokenize.C, tokenize.A, tokenize.M, tokenize.T +// Some globally useful tokenizer definitions. +var ( + c, a, m, tok = tokenize.C, tokenize.A, tokenize.M, tokenize.T -type cmdType string + // From the specs: "Whitespace means tab (0x09) or space (0x20)." + // In this package, we name this a blank, to be in line with the + // terminology as used in parsekit. + blank = a.Runes('\t', ' ') -// Command types that are emitted by the parser. -const ( - cComment cmdType = "comment" // a # comment at the end of the line - cKey cmdType = "key" // set key name - cKeyDot cmdType = "keydot" // new key stack level - cAssign cmdType = "assign" // assign a value - csetStrVal cmdType = "string" // set a string value - csetIntVal cmdType = "integer" // set an integer value - csetFloatVal cmdType = "float" // set a float value - csetBoolVal cmdType = "boolean" // set a boolean value - coffsetDateTime cmdType = "offset_datetime" // set a date/time value with timezone information - clocalDateTime cmdType = "datetime" // set a local date/time value - clocalDate cmdType = "date" // set a local date value - clocalTime cmdType = "time" // set a local time value + // Newline means LF (0x0A) or CRLF (0x0D0A). + // This matches the default newline as defined by parsekit. + newline = a.Newline + + dropBlanks = m.Drop(c.ZeroOrMore(blank)) + dropWhitespace = m.Drop(c.ZeroOrMore(blank.Or(newline))) ) - -type parser struct { - commands []cmd - keyStack []string -} - -type cmd struct { - command cmdType - args []interface{} -} - -func (cmd *cmd) String() string { - if len(cmd.args) == 0 { - return fmt.Sprintf("%s", cmd.command) - } - args := make([]string, len(cmd.args)) - for i, arg := range cmd.args { - switch arg.(type) { - case string: - args[i] = fmt.Sprintf("%q", arg) - default: - args[i] = fmt.Sprintf("%v", arg) - } - } - return fmt.Sprintf("%s(%s)", cmd.command, strings.Join(args, ", ")) -} - -func (p *parser) emitCommand(command cmdType, args ...interface{}) { - c := cmd{command: command, args: args} - p.commands = append(p.commands, c) -} - -// Parse starts the parser for the provided input. -// func Parse(input interface{}) []cmd { -// p := &parser{} -// parse.New(p.startKeyValuePair)(input) -// return p.commands -// } diff --git a/toml_test.go b/toml_test.go deleted file mode 100644 index 0437c2d..0000000 --- a/toml_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package parser_test - -// func TestEmptyInput(t *testing.T) { -// runStatesT(t, statesT{"empty string", "", "", ""}) -// } - -// func TestFullIncludesLineAndRowPosition(t *testing.T) { -// p := toml.Parse("# 12345 abcde\t\n\n\n# 67890\r\n# 12345\n +") -// _, err := parseItemsToArray(p) -// actual := err.Error() -// expected := "unexpected input (expected end of file) at line 6, column 3" -// if actual != expected { -// t.Errorf("Unexpected error message:\nexpected: %s\nactual: %s\n", expected, actual) -// } -// } - -// func TestInvalidUTF8Data(t *testing.T) { -// runStatesTs(t, []statesT{ -// {"bare key 1", "\xbc", "", "invalid UTF8 character in input (expected end of file)"}, -// {"bare key 2", "key\xbc", "[key]", "invalid UTF8 character in input (expected a value assignment)"}, -// {"start of value", "key=\xbc", "[key]=", "invalid UTF8 character in input (expected a value)"}, -// {"basic string value", "a=\"\xbc\"", "[a]=", "invalid UTF8 character in input (expected string contents)"}, -// }) -// } - -// func TestWhiteSpaceAndNewlines(t *testing.T) { -// runStatesTs(t, []statesT{ -// {"space", " ", "", ""}, -// {"tab", "\t", "", ""}, -// {"newline", "\n", "", ""}, -// {"all blanks and newlines", " \t \t \r\n\n \n \t", "", ""}, -// {"bare carriage return", "\r", "", "unexpected character '\\r' (expected end of file)"}, -// }) -// } diff --git a/value.go b/value.go index fe2c45d..c712d10 100644 --- a/value.go +++ b/value.go @@ -1,4 +1,4 @@ -package parser +package toml import ( "git.makaay.nl/mauricem/go-parsekit/parse" @@ -22,6 +22,8 @@ func (t *parser) startValue(p *parse.API) { } else { p.Handle(t.startNumber) } + case p.Peek(a.SquareOpen): + p.Handle(t.startArray) default: p.Expected("a value") } diff --git a/value_array.go b/value_array.go new file mode 100644 index 0000000..459f04a --- /dev/null +++ b/value_array.go @@ -0,0 +1,86 @@ +package toml + +import ( + "git.makaay.nl/mauricem/go-parsekit/parse" +) + +// Arrays are square brackets with values inside. Whitespace is ignored. +// Elements are separated by commas. Data types may not be mixed (different +// ways to define strings should be considered the same type, and so should +// arrays with different element types). +// +// arr1 = [ 1, 2, 3 ] +// arr2 = [ "red", "yellow", "green" ] +// arr3 = [ [ 1, 2 ], [3, 4, 5] ] +// arr4 = [ "all", 'strings', """are the same""", '''type'''] +// arr5 = [ [ 1, 2 ], ["a", "b", "c"] ] +// +// arr6 = [ 1, 2.0 ] # INVALID +// +// Arrays can also be multiline. A terminating comma (also called trailing +// comma) is ok after the last value of the array. There can be an arbitrary +// number of newlines and comments before a value and before the closing bracket. +// +// arr7 = [ +// 1, 2, 3 +// ] +// +// arr8 = [ +// 1, +// 2, # this is ok +// ] +var ( + arraySpace = c.ZeroOrMore(c.Any(blank, newline, comment)) + arrayOpen = a.SquareOpen.Then(arraySpace) + arraySeparator = c.Seq(arraySpace, a.Comma, arraySpace) + arrayClose = c.Seq(c.Optional(arraySpace.Then(a.Comma)), arraySpace, a.SquareClose) +) + +func (t *parser) startArray(p *parse.API) { + // Check for the start of the array. + if !p.Accept(arrayOpen) { + p.Expected("an array") + return + } + + items := []item{} + + // Check for an empty array. + if p.Accept(arrayClose) { + t.addParsedItem(pStaticArray, items) + return + } + + // Not an empty array, parse the items. + for { + // Check for a valid item. + if !p.Handle(t.startValue) { + return + } + + // Pop the item from the value parsing and append it to the array items. + parseItems, item := t.Items[0:len(t.Items)-1], t.Items[len(t.Items)-1] + t.Items = parseItems + + // Data types may not be mixed (different ways to define strings should be + // considered the same type, and so should arrays with different element types). + if len(items) > 0 && item.Type != items[0].Type { + p.Error("type mismatch in array of %ss: found an item of type %s", items[0].Type, item.Type) + return + } + + items = append(items, item) + + // Check for the end of the array. + if p.Accept(arrayClose) { + t.addParsedItem(pStaticArray, items) + return + } + + // Not the end of the array? Then we should find an array separator. + if !p.Accept(arraySeparator) { + p.Expected("an array separator") + return + } + } +} diff --git a/value_array_test.go b/value_array_test.go new file mode 100644 index 0000000..d01767b --- /dev/null +++ b/value_array_test.go @@ -0,0 +1,40 @@ +package toml + +import ( + "testing" +) + +func TestArray(t *testing.T) { + for _, test := range []parseTest{ + {"", []string{`Error: unexpected end of file (expected an array) at start of file`}}, + {"[ape", []string{`Error: unexpected input (expected a value) at line 1, column 2`}}, + {"[1", []string{`Error: unexpected end of file (expected an array separator) at line 1, column 3`}}, + {"[]", []string{`[]`}}, + {"[\n]", []string{`[]`}}, + {"[,]", []string{`[]`}}, + {"[ , ]", []string{`[]`}}, + {"[ \n , \r\n ]", []string{`[]`}}, + {"[ \t , \t ]", []string{`[]`}}, + {"[\r\n\r\n , \r\n \t\n]", []string{`[]`}}, + {"[\n#comment on its own line\n]", []string{`[]`}}, + {"[#comment before close\n]", []string{`[]`}}, + {"[,#comment after separator\n]", []string{`[]`}}, + {"[#comment before separator\n,]", []string{`[]`}}, + {"[#comment before value\n1]", []string{`[1]`}}, + {"[1#comment after value\n]", []string{`[1]`}}, + {"[1\n#comment on its own line after value\n]", []string{`[1]`}}, + {"[1#comment 1\n#comment 2\n#comment 3\n , \n2]", []string{`[1, 2]`}}, + {"[1]", []string{`[1]`}}, + {"[1,0x2, 0b11, 0o4]", []string{`[1, 2, 3, 4]`}}, + {"[0.1,0.2,3e-1,0.04e+1, nan, inf]", []string{`[0.1, 0.2, 0.3, 0.4, NaN, +Inf]`}}, + {"[\n\t 'a', \"b\", '''c''', \"\"\"d\ne\"\"\",\n \t]", []string{`["a", "b", "c", "d\ne"]`}}, + {`[1, 2, 3, "four"]`, []string{`Error: type mismatch in array of integers: found an item of type string at line 1, column 17`}}, + {`[[1],['a']]`, []string{`[[1], ["a"]]`}}, + {`[[[],[]],[]]`, []string{`[[[], []], []]`}}, + {"[\r\n\r\n \t\n [\r\n\r\n\t [],[\t]],\t\n[]\t \t \n ]", []string{`[[[], []], []]`}}, + {`[[1],'a']`, []string{`Error: type mismatch in array of static arrays: found an item of type string at line 1, column 9`}}, + } { + p := newParser() + testParseHandler(t, p, p.startArray, test) + } +} diff --git a/value_boolean.go b/value_boolean.go index 7996347..d050e75 100644 --- a/value_boolean.go +++ b/value_boolean.go @@ -1,4 +1,4 @@ -package parser +package toml import ( "git.makaay.nl/mauricem/go-parsekit/parse" @@ -8,9 +8,9 @@ import ( func (t *parser) startBoolean(p *parse.API) { switch { case p.Accept(a.Str("true")): - t.emitCommand(csetBoolVal, true) + t.addParsedItem(pBoolean, true) case p.Accept(a.Str("false")): - t.emitCommand(csetBoolVal, false) + t.addParsedItem(pBoolean, false) default: p.Expected("true or false") } diff --git a/value_boolean_test.go b/value_boolean_test.go index 182e252..0eb14a6 100644 --- a/value_boolean_test.go +++ b/value_boolean_test.go @@ -1,4 +1,4 @@ -package parser +package toml import ( "testing" @@ -7,14 +7,14 @@ import ( func TestBoolean(t *testing.T) { for _, test := range []parseTest{ {``, []string{`Error: unexpected end of file (expected true or false) at start of file`}}, - {`true`, []string{`boolean(true)`}}, - {`false`, []string{`boolean(false)`}}, + {`true`, []string{`true`}}, + {`false`, []string{`false`}}, {`yes`, []string{`Error: unexpected input (expected true or false) at start of file`}}, {`no`, []string{`Error: unexpected input (expected true or false) at start of file`}}, {`1`, []string{`Error: unexpected input (expected true or false) at start of file`}}, {`0`, []string{`Error: unexpected input (expected true or false) at start of file`}}, } { - p := &parser{} + p := newParser() testParseHandler(t, p, p.startBoolean, test) } } diff --git a/value_datetime.go b/value_datetime.go index 25e7196..9b64c07 100644 --- a/value_datetime.go +++ b/value_datetime.go @@ -1,4 +1,4 @@ -package parser +package toml import ( "time" @@ -66,10 +66,10 @@ var ( // The full date/time parse format, based on the above definitions. // The first token denotes the type of date/time value. // The rest of the tokens contain layout fragments for time.Parse(). - offsetDateTime = tok.Str(coffsetDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok, tzTok)) - localDateTime = tok.Str(clocalDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok)) - localDate = tok.Str(clocalDate, dateTok) - localTime = tok.Str(clocalTime, c.Seq(timeTok, microTok)) + offsetDateTime = tok.Str(pOffsetDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok, tzTok)) + localDateTime = tok.Str(pLocalDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok)) + localDate = tok.Str(pLocalDate, dateTok) + localTime = tok.Str(pLocalTime, c.Seq(timeTok, microTok)) datetime = c.Any(offsetDateTime, localDateTime, localDate, localTime) ) @@ -82,7 +82,7 @@ func (t *parser) startDateTime(p *parse.API) { valueType := getDateTimeValueType(&tokens) input, value, err := getDateTimeValue(&tokens) if err == nil { - t.emitCommand(valueType, value) + t.addParsedItem(valueType, value) } else { p.Error("Cannot parse value 0%s: %s", input, err) } @@ -90,8 +90,8 @@ func (t *parser) startDateTime(p *parse.API) { // The first token is a token that wraps the complete date/time input. // Its type denotes the type of date/time value that it wraps. -func getDateTimeValueType(tokens *[]*tokenize.Token) cmdType { - return (*tokens)[0].Type.(cmdType) +func getDateTimeValueType(tokens *[]*tokenize.Token) itemType { + return (*tokens)[0].Type.(itemType) } // The rest of the tokens contain fragments that can be used with diff --git a/value_datetime_test.go b/value_datetime_test.go index 553a4f2..5454fda 100644 --- a/value_datetime_test.go +++ b/value_datetime_test.go @@ -1,4 +1,4 @@ -package parser +package toml import ( "testing" @@ -7,27 +7,27 @@ import ( func TestDateTime(t *testing.T) { for _, test := range []parseTest{ {``, []string{`Error: unexpected end of file (expected a date and/or time) at start of file`}}, - {`1979-05-27`, []string{`date(1979-05-27 00:00:00 +0000 UTC)`}}, - {`00:00:00`, []string{`time(0000-01-01 00:00:00 +0000 UTC)`}}, - {`23:59:59`, []string{`time(0000-01-01 23:59:59 +0000 UTC)`}}, - {`12:10:08.12121212121212`, []string{`time(0000-01-01 12:10:08.121212121 +0000 UTC)`}}, - {`1979-05-28T01:01:01`, []string{`datetime(1979-05-28 01:01:01 +0000 UTC)`}}, - {`1979-05-28 01:01:01`, []string{`datetime(1979-05-28 01:01:01 +0000 UTC)`}}, - {`1979-05-27T07:32:00Z`, []string{`offset_datetime(1979-05-27 07:32:00 +0000 UTC)`}}, - {`1979-05-27 07:33:00Z`, []string{`offset_datetime(1979-05-27 07:33:00 +0000 UTC)`}}, - {`1979-05-27 07:34:00+07:00`, []string{`offset_datetime(1979-05-27 07:34:00 +0700 +0700)`}}, - {`1979-05-27 07:34:00-07:00`, []string{`offset_datetime(1979-05-27 07:34:00 -0700 -0700)`}}, - {`1985-03-31 23:59:59+00:00`, []string{`offset_datetime(1985-03-31 23:59:59 +0000 UTC)`}}, - {`2000-09-10 00:00:00.000000000+00:00`, []string{`offset_datetime(2000-09-10 00:00:00 +0000 UTC)`}}, - {`2003-11-01 01:02:03.999999999999+10:00`, []string{`offset_datetime(2003-11-01 01:02:03.999999999 +1000 +1000)`}}, - {`2007-12-25 04:00:04.1111-10:30`, []string{`offset_datetime(2007-12-25 04:00:04.1111 -1030 -1030)`}}, - {`2021-02-01 10:10:10.101010203040Z`, []string{`offset_datetime(2021-02-01 10:10:10.101010203 +0000 UTC)`}}, + {`1979-05-27`, []string{`1979-05-27`}}, + {`00:00:00`, []string{`00:00:00`}}, + {`23:59:59`, []string{`23:59:59`}}, + {`12:10:08.12121212121212`, []string{`12:10:08.121212121`}}, + {`1979-05-28T01:01:01`, []string{`1979-05-28 01:01:01`}}, + {`1979-05-28 01:01:01`, []string{`1979-05-28 01:01:01`}}, + {`1979-05-27T07:32:00Z`, []string{`1979-05-27T07:32:00Z`}}, + {`1979-05-27 07:33:00Z`, []string{`1979-05-27T07:33:00Z`}}, + {`1979-05-27 07:34:00+07:00`, []string{`1979-05-27T07:34:00+07:00`}}, + {`1979-05-27 07:34:00-07:00`, []string{`1979-05-27T07:34:00-07:00`}}, + {`1985-03-31 23:59:59+00:00`, []string{`1985-03-31T23:59:59Z`}}, + {`2000-09-10 00:00:00.000000000+00:00`, []string{`2000-09-10T00:00:00Z`}}, + {`2003-11-01 01:02:03.999999999999+10:00`, []string{`2003-11-01T01:02:03.999999999+10:00`}}, + {`2007-12-25 04:00:04.1111-10:30`, []string{`2007-12-25T04:00:04.1111-10:30`}}, + {`2021-02-01 10:10:10.101010203040Z`, []string{`2021-02-01T10:10:10.101010203Z`}}, // TODO ugly column, should be at start or at the actual wrong part {`2000-13-01`, []string{`Error: Cannot parse value 02000-13-01: parsing time "2000-13-01": month out of range at line 1, column 11`}}, {`2000-02-31`, []string{`Error: Cannot parse value 02000-02-31: parsing time "2000-02-31": day out of range at line 1, column 11`}}, {`25:01:01`, []string{`Error: Cannot parse value 025:01:01: parsing time "25:01:01": hour out of range at line 1, column 9`}}, } { - p := &parser{} + p := newParser() testParseHandler(t, p, p.startDateTime, test) } } diff --git a/value_number.go b/value_number.go index b3702bf..40c4a5b 100644 --- a/value_number.go +++ b/value_number.go @@ -1,4 +1,4 @@ -package parser +package toml import ( "math" @@ -68,19 +68,19 @@ var ( func (t *parser) startNumber(p *parse.API) { switch { case p.Accept(tok.Float64(nil, float)): - t.emitCommand(csetFloatVal, p.Result().Value(0).(float64)) + t.addParsedItem(pFloat, p.Result().Value(0).(float64)) case p.Accept(nan): - t.emitCommand(csetFloatVal, math.NaN()) + t.addParsedItem(pFloat, math.NaN()) case p.Accept(inf): if p.Result().Rune(0) == '-' { - t.emitCommand(csetFloatVal, math.Inf(-1)) + t.addParsedItem(pFloat, math.Inf(-1)) } else { - t.emitCommand(csetFloatVal, math.Inf(+1)) + t.addParsedItem(pFloat, math.Inf(+1)) } case p.Accept(a.Zero): p.Handle(t.startIntegerStartingWithZero) case p.Accept(tok.Int64(nil, integer)): - t.emitCommand(csetIntVal, p.Result().Value(0).(int64)) + t.addParsedItem(pInteger, p.Result().Value(0).(int64)) default: p.Expected("a number") } @@ -97,11 +97,11 @@ func (t *parser) startIntegerStartingWithZero(p *parse.API) { case p.Accept(binary): value, err = strconv.ParseInt(p.Result().Value(0).(string), 2, 64) default: - t.emitCommand(csetIntVal, int64(0)) + t.addParsedItem(pInteger, int64(0)) return } if err == nil { - t.emitCommand(csetIntVal, value) + t.addParsedItem(pInteger, value) } else { p.Error("Cannot parse value 0%s: %s", p.Result().String(), err) } diff --git a/value_number_test.go b/value_number_test.go index 2868aef..243e954 100644 --- a/value_number_test.go +++ b/value_number_test.go @@ -1,4 +1,4 @@ -package parser +package toml import ( "testing" @@ -8,73 +8,73 @@ func TestInteger(t *testing.T) { for _, test := range []parseTest{ {``, []string{`Error: unexpected end of file (expected a number) at start of file`}}, // Decimal - {`0`, []string{`integer(0)`}}, - {`+0`, []string{`integer(0)`}}, - {`-0`, []string{`integer(0)`}}, - {`1`, []string{`integer(1)`}}, - {`42`, []string{`integer(42)`}}, - {`+99`, []string{`integer(99)`}}, - {`-17`, []string{`integer(-17)`}}, - {`1234`, []string{`integer(1234)`}}, + {`0`, []string{`0`}}, + {`+0`, []string{`0`}}, + {`-0`, []string{`0`}}, + {`1`, []string{`1`}}, + {`42`, []string{`42`}}, + {`+99`, []string{`99`}}, + {`-17`, []string{`-17`}}, + {`1234`, []string{`1234`}}, {`_`, []string{`Error: unexpected input (expected a number) at start of file`}}, - {`1_`, []string{`integer(1)`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, - {`1_000`, []string{`integer(1000)`}}, - {`5_349_221`, []string{`integer(5349221)`}}, - {`1_2_3_4_5`, []string{`integer(12345)`}}, - {`9_223_372_036_854_775_807`, []string{`integer(9223372036854775807)`}}, + {`1_`, []string{`1`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, + {`1_000`, []string{`1000`}}, + {`5_349_221`, []string{`5349221`}}, + {`1_2_3_4_5`, []string{`12345`}}, + {`9_223_372_036_854_775_807`, []string{`9223372036854775807`}}, {`9_223_372_036_854_775_808`, []string{ `Panic: Handler error: MakeInt64Token cannot handle input "9223372036854775808": ` + `strconv.ParseInt: parsing "9223372036854775808": value out of range (only use a ` + `type conversion token maker, when the input has been validated on beforehand)`}}, - {`-9_223_372_036_854_775_808`, []string{`integer(-9223372036854775808)`}}, + {`-9_223_372_036_854_775_808`, []string{`-9223372036854775808`}}, // TODO make the use of the same kind of handling for panics and for errors between parsekit and TOML. {`-9_223_372_036_854_775_809`, []string{ `Panic: Handler error: MakeInt64Token cannot handle input "-9223372036854775809": ` + `strconv.ParseInt: parsing "-9223372036854775809": value out of range (only use a ` + `type conversion token maker, when the input has been validated on beforehand)`}}, // Hexadecimal - {`0x0`, []string{`integer(0)`}}, - {`0x1`, []string{`integer(1)`}}, - {`0x01`, []string{`integer(1)`}}, - {`0x00fF`, []string{`integer(255)`}}, - {`0xf_f`, []string{`integer(255)`}}, - {`0x0_0_f_f`, []string{`integer(255)`}}, - {`0xdead_beef`, []string{`integer(3735928559)`}}, - {`0xgood_beef`, []string{`integer(0)`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, - {`0x7FFFFFFFFFFFFFFF`, []string{`integer(9223372036854775807)`}}, + {`0x0`, []string{`0`}}, + {`0x1`, []string{`1`}}, + {`0x01`, []string{`1`}}, + {`0x00fF`, []string{`255`}}, + {`0xf_f`, []string{`255`}}, + {`0x0_0_f_f`, []string{`255`}}, + {`0xdead_beef`, []string{`3735928559`}}, + {`0xgood_beef`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, + {`0x7FFFFFFFFFFFFFFF`, []string{`9223372036854775807`}}, {`0x8000000000000000`, []string{ `Error: Cannot parse value 0x8000000000000000: strconv.ParseInt: parsing "8000000000000000": ` + `value out of range at line 1, column 19`}}, //Octal - {`0o0`, []string{`integer(0)`}}, - {`0o1`, []string{`integer(1)`}}, - {`0o01`, []string{`integer(1)`}}, - {`0o10`, []string{`integer(8)`}}, - {`0o1_6`, []string{`integer(14)`}}, - {`0o0_0_1_1_1`, []string{`integer(73)`}}, - {`0o9`, []string{`integer(0)`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, - {`0o777777777777777777777`, []string{`integer(9223372036854775807)`}}, + {`0o0`, []string{`0`}}, + {`0o1`, []string{`1`}}, + {`0o01`, []string{`1`}}, + {`0o10`, []string{`8`}}, + {`0o1_6`, []string{`14`}}, + {`0o0_0_1_1_1`, []string{`73`}}, + {`0o9`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, + {`0o777777777777777777777`, []string{`9223372036854775807`}}, {`0o1000000000000000000000`, []string{ `Error: Cannot parse value 0o1000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000": ` + `value out of range at line 1, column 25`}}, // Binary - {`0b0`, []string{`integer(0)`}}, - {`0b1`, []string{`integer(1)`}}, - {`0b01`, []string{`integer(1)`}}, - {`0b10`, []string{`integer(2)`}}, - {`0b0100`, []string{`integer(4)`}}, - {`0b00001000`, []string{`integer(8)`}}, - {`0b0001_0000`, []string{`integer(16)`}}, - {`0b9`, []string{`integer(0)`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, - {`0b1_1_0_1_1`, []string{`integer(27)`}}, - {`0b11111111_11111111`, []string{`integer(65535)`}}, - {`0b01111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111`, []string{`integer(9223372036854775807)`}}, + {`0b0`, []string{`0`}}, + {`0b1`, []string{`1`}}, + {`0b01`, []string{`1`}}, + {`0b10`, []string{`2`}}, + {`0b0100`, []string{`4`}}, + {`0b00001000`, []string{`8`}}, + {`0b0001_0000`, []string{`16`}}, + {`0b9`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, + {`0b1_1_0_1_1`, []string{`27`}}, + {`0b11111111_11111111`, []string{`65535`}}, + {`0b01111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111`, []string{`9223372036854775807`}}, {`0b10000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000`, []string{ `Error: Cannot parse value 0b1000000000000000000000000000000000000000000000000000000000000000: ` + `strconv.ParseInt: parsing "1000000000000000000000000000000000000000000000000000000000000000": ` + `value out of range at line 1, column 74`}}, } { - p := &parser{} + p := newParser() testParseHandler(t, p, p.startNumber, test) } } @@ -82,26 +82,26 @@ func TestInteger(t *testing.T) { func TestFloat(t *testing.T) { for _, test := range []parseTest{ {``, []string{`Error: unexpected end of file (expected a number) at start of file`}}, - {`0.0`, []string{`float(0)`}}, - {`+0.0`, []string{`float(0)`}}, - {`-0.0`, []string{`float(-0)`}}, - {`+1.0`, []string{`float(1)`}}, - {`3.1415`, []string{`float(3.1415)`}}, - {`-0.01`, []string{`float(-0.01)`}}, - {`5e+22`, []string{`float(5e+22)`}}, - {`1E6`, []string{`float(1e+06)`}}, - {`-2E-2`, []string{`float(-0.02)`}}, - {`6.626e-34`, []string{`float(6.626e-34)`}}, - {`224_617.445_991_228`, []string{`float(224617.445991228)`}}, - {`12_345.111_222e+1_2_3`, []string{`float(1.2345111222e+127)`}}, - {`+nan`, []string{`float(NaN)`}}, - {`-nan`, []string{`float(NaN)`}}, - {`nan`, []string{`float(NaN)`}}, - {`inf`, []string{`float(+Inf)`}}, - {`+inf`, []string{`float(+Inf)`}}, - {`-inf`, []string{`float(-Inf)`}}, + {`0.0`, []string{`0`}}, + {`+0.0`, []string{`0`}}, + {`-0.0`, []string{`-0`}}, + {`+1.0`, []string{`1`}}, + {`3.1415`, []string{`3.1415`}}, + {`-0.01`, []string{`-0.01`}}, + {`5e+22`, []string{`5e+22`}}, + {`1E6`, []string{`1e+06`}}, + {`-2E-2`, []string{`-0.02`}}, + {`6.626e-34`, []string{`6.626e-34`}}, + {`224_617.445_991_228`, []string{`224617.445991228`}}, + {`12_345.111_222e+1_2_3`, []string{`1.2345111222e+127`}}, + {`+nan`, []string{`NaN`}}, + {`-nan`, []string{`NaN`}}, + {`nan`, []string{`NaN`}}, + {`inf`, []string{`+Inf`}}, + {`+inf`, []string{`+Inf`}}, + {`-inf`, []string{`-Inf`}}, } { - p := &parser{} + p := newParser() testParseHandler(t, p, p.startNumber, test) } } diff --git a/value_string.go b/value_string.go index 43d39a3..837f26c 100644 --- a/value_string.go +++ b/value_string.go @@ -1,4 +1,4 @@ -package parser +package toml import ( "strings" @@ -38,10 +38,7 @@ var ( // "line ending backslash". When the last non-whitespace character on a line is // a \, it will be trimmed along with all whitespace (including newlines) up to // the next non-whitespace character or closing delimiter. - lineEndingBackslash = a.Backslash. - Then(c.ZeroOrMore(a.Blanks)). - Then(a.Newline). - Then(c.ZeroOrMore(a.Whitespace)) + lineEndingBackslash = a.Backslash.Then(dropBlanks).Then(newline).Then(dropWhitespace) ) // There are four ways to express strings: basic, multi-line basic, literal and @@ -49,9 +46,9 @@ var ( func (t *parser) startString(p *parse.API) { switch { case p.Peek(doubleQuote3): - p.Handle(t.startMultiLineBasicString) + p.Handle(t.startMultiLineBasipString) case p.Peek(a.DoubleQuote): - p.Handle(t.startBasicString) + p.Handle(t.startBasipString) case p.Peek(singleQuote3): p.Handle(t.startMultiLineLiteralString) case p.Peek(a.SingleQuote): @@ -72,13 +69,13 @@ func (t *parser) startString(p *parse.API) { // • No additional \escape sequences are allowed. What the spec say about this: // "All other escape sequences [..] are reserved and, if used, TOML should // produce an error."" -func (t *parser) startBasicString(p *parse.API) { - if str, ok := t.parseBasicString("basic string", p); ok { - t.emitCommand(csetStrVal, str) +func (t *parser) startBasipString(p *parse.API) { + if str, ok := t.parseBasipString("basic string", p); ok { + t.addParsedItem(pString, str) } } -func (t *parser) parseBasicString(name string, p *parse.API) (string, bool) { +func (t *parser) parseBasipString(name string, p *parse.API) (string, bool) { if !p.Accept(a.DoubleQuote) { p.Expected(`opening quotation marks`) return "", false @@ -117,7 +114,7 @@ func (t *parser) parseBasicString(name string, p *parse.API) (string, bool) { // • Control characters other than tab are not permitted in a literal string. func (t *parser) startLiteralString(p *parse.API) { if str, ok := t.parseLiteralString("literal string", p); ok { - t.emitCommand(csetStrVal, str) + t.addParsedItem(pString, str) } } @@ -171,15 +168,15 @@ func (t *parser) parseLiteralString(name string, p *parse.API) (string, bool) { // "line ending backslash". When the last non-whitespace character on a line is // a \, it will be trimmed along with all whitespace (including newlines) up to // the next non-whitespace character or closing delimiter. -func (t *parser) startMultiLineBasicString(p *parse.API) { - if !p.Accept(doubleQuote3.Then(a.Newline.Optional())) { +func (t *parser) startMultiLineBasipString(p *parse.API) { + if !p.Accept(doubleQuote3.Then(newline.Optional())) { p.Expected("opening three quotation marks") return } sb := &strings.Builder{} for { switch { - case p.Accept(a.Newline): + case p.Accept(newline): sb.WriteString("\n") case p.Peek(controlCharacter): p.Error("invalid character in multi-line basic string: %q (must be escaped)", p.Result().Rune(0)) @@ -192,7 +189,7 @@ func (t *parser) startMultiLineBasicString(p *parse.API) { p.Error("invalid escape sequence") return case p.Accept(m.Drop(doubleQuote3)): - t.emitCommand(csetStrVal, sb.String()) + t.addParsedItem(pString, sb.String()) return case p.Accept(a.ValidRune): sb.WriteString(p.Result().String()) @@ -220,7 +217,7 @@ func (t *parser) startMultiLineBasicString(p *parse.API) { // // • Control characters other than tab and newline are not permitted in a multi-line literal string. func (t *parser) startMultiLineLiteralString(p *parse.API) { - if !p.Accept(singleQuote3.Then(a.Newline.Optional())) { + if !p.Accept(singleQuote3.Then(newline.Optional())) { p.Expected("opening three single quotes") return } @@ -228,11 +225,11 @@ func (t *parser) startMultiLineLiteralString(p *parse.API) { for { switch { case p.Accept(m.Drop(singleQuote3)): - t.emitCommand(csetStrVal, sb.String()) + t.addParsedItem(pString, sb.String()) return case p.Accept(a.Tab): sb.WriteString("\t") - case p.Accept(a.Newline): + case p.Accept(newline): sb.WriteString("\n") case p.Peek(controlCharacter): p.Error("invalid character in literal string: %q (no control chars allowed, except for tab and newline)", p.Result().Rune(0)) diff --git a/value_string_test.go b/value_string_test.go index e4c6323..dca2695 100644 --- a/value_string_test.go +++ b/value_string_test.go @@ -1,4 +1,4 @@ -package parser +package toml import ( "fmt" @@ -9,53 +9,53 @@ func TestString(t *testing.T) { for _, test := range []parseTest{ {``, []string{`Error: unexpected end of file (expected a string value) at start of file`}}, {`no start quote"`, []string{`Error: unexpected input (expected a string value) at start of file`}}, - {`"basic s\tring"`, []string{`string("basic s\tring")`}}, - {"\"\"\"\n basic multi-line\n string value\n\"\"\"", []string{`string(" basic multi-line\n string value\n")`}}, - {`'literal s\tring'`, []string{`string("literal s\\tring")`}}, - {"'''\n literal multi-line\n string value\n'''", []string{`string(" literal multi-line\n string value\n")`}}, + {`"basic s\tring"`, []string{`"basic s\tring"`}}, + {"\"\"\"\n basic multi-line\n string value\n\"\"\"", []string{`" basic multi-line\n string value\n"`}}, + {`'literal s\tring'`, []string{`"literal s\\tring"`}}, + {"'''\n literal multi-line\n string value\n'''", []string{`" literal multi-line\n string value\n"`}}, } { - p := &parser{} + p := newParser() testParseHandler(t, p, p.startString, test) } } -func TestBasicString(t *testing.T) { +func TestBasipString(t *testing.T) { for _, test := range []parseTest{ {``, []string{`Error: unexpected end of file (expected opening quotation marks) at start of file`}}, {`no start quote"`, []string{`Error: unexpected input (expected opening quotation marks) at start of file`}}, {`"no end quote`, []string{`Error: unexpected end of file (expected closing quotation marks) at line 1, column 14`}}, - {`""`, []string{`string("")`}}, - {`"simple string"`, []string{`string("simple string")`}}, - {`"with\tsome\r\nvalid escapes\b"`, []string{`string("with\tsome\r\nvalid escapes\b")`}}, + {`""`, []string{`""`}}, + {`"simple string"`, []string{`"simple string"`}}, + {`"with\tsome\r\nvalid escapes\b"`, []string{`"with\tsome\r\nvalid escapes\b"`}}, {`"with an \invalid escape"`, []string{`Error: invalid escape sequence at line 1, column 10`}}, - {`"A cool UTF8 ƃuıɹʇs"`, []string{`string("A cool UTF8 ƃuıɹʇs")`}}, - {`"A string with UTF8 escape \u2318"`, []string{`string("A string with UTF8 escape ⌘")`}}, + {`"A cool UTF8 ƃuıɹʇs"`, []string{`"A cool UTF8 ƃuıɹʇs"`}}, + {`"A string with UTF8 escape \u2318"`, []string{`"A string with UTF8 escape ⌘"`}}, {"\"Invalid character for UTF \xcd\"", []string{`Error: invalid UTF8 rune at line 1, column 28`}}, {"\"Character that mus\t be escaped\"", []string{`Error: invalid character in basic string: '\t' (must be escaped) at line 1, column 20`}}, {"\"Character that must be escaped \u0000\"", []string{`Error: invalid character in basic string: '\x00' (must be escaped) at line 1, column 33`}}, {"\"Character that must be escaped \x7f\"", []string{`Error: invalid character in basic string: '\u007f' (must be escaped) at line 1, column 33`}}, } { - p := &parser{} - testParseHandler(t, p, p.startBasicString, test) + p := newParser() + testParseHandler(t, p, p.startBasipString, test) } } -func TestMultiLineBasicString(t *testing.T) { +func TestMultiLineBasipString(t *testing.T) { for _, test := range []parseTest{ {``, []string{`Error: unexpected end of file (expected opening three quotation marks) at start of file`}}, {`"""missing close quote""`, []string{`Error: unexpected end of file (expected closing three quotation marks) at line 1, column 25`}}, - {`""""""`, []string{`string("")`}}, - {"\"\"\"\n\"\"\"", []string{`string("")`}}, - {"\"\"\"\r\n\r\n\"\"\"", []string{`string("\n")`}}, - {`"""\"\"\"\""""`, []string{`string("\"\"\"\"")`}}, - {"\"\"\"\nThe quick brown \\\n\n\n \t fox jumps over \\\n\t the lazy dog.\\\n \"\"\"", []string{`string("The quick brown fox jumps over the lazy dog.")`}}, + {`""""""`, []string{`""`}}, + {"\"\"\"\n\"\"\"", []string{`""`}}, + {"\"\"\"\r\n\r\n\"\"\"", []string{`"\n"`}}, + {`"""\"\"\"\""""`, []string{`"\"\"\"\""`}}, + {"\"\"\"\nThe quick brown \\\n\n\n \t fox jumps over \\\n\t the lazy dog.\\\n \"\"\"", []string{`"The quick brown fox jumps over the lazy dog."`}}, {"\"\"\"No control chars \f allowed\"\"\"", []string{`Error: invalid character in multi-line basic string: '\f' (must be escaped) at line 1, column 21`}}, - {"\"\"\"Escaping control chars\\nis valid\"\"\"", []string{`string("Escaping control chars\nis valid")`}}, + {"\"\"\"Escaping control chars\\nis valid\"\"\"", []string{`"Escaping control chars\nis valid"`}}, {"\"\"\"Invalid escaping \\is not allowed\"\"\"", []string{`Error: invalid escape sequence at line 1, column 21`}}, {"\"\"\"Invalid rune \xcd\"\"\"", []string{`Error: invalid UTF8 rune at line 1, column 17`}}, } { - p := &parser{} - testParseHandler(t, p, p.startMultiLineBasicString, test) + p := newParser() + testParseHandler(t, p, p.startMultiLineBasipString, test) } } @@ -63,17 +63,17 @@ func TestLiteralString(t *testing.T) { for _, test := range []parseTest{ {``, []string{`Error: unexpected end of file (expected opening single quote) at start of file`}}, {`'missing close quote`, []string{`Error: unexpected end of file (expected closing single quote) at line 1, column 21`}}, - {`''`, []string{`string("")`}}, - {`'simple'`, []string{`string("simple")`}}, - {`'C:\Users\nodejs\templates'`, []string{`string("C:\\Users\\nodejs\\templates")`}}, - {`'\\ServerX\admin$\system32\'`, []string{`string("\\\\ServerX\\admin$\\system32\\")`}}, - {`'Tom "Dubs" Preston-Werner'`, []string{`string("Tom \"Dubs\" Preston-Werner")`}}, - {`'<\i\c*\s*>'`, []string{`string("<\\i\\c*\\s*>")`}}, + {`''`, []string{`""`}}, + {`'simple'`, []string{`"simple"`}}, + {`'C:\Users\nodejs\templates'`, []string{`"C:\\Users\\nodejs\\templates"`}}, + {`'\\ServerX\admin$\system32\'`, []string{`"\\\\ServerX\\admin$\\system32\\"`}}, + {`'Tom "Dubs" Preston-Werner'`, []string{`"Tom \"Dubs\" Preston-Werner"`}}, + {`'<\i\c*\s*>'`, []string{`"<\\i\\c*\\s*>"`}}, {"'No cont\rol chars allowed'", []string{`Error: invalid character in literal string: '\r' (no control chars allowed, except for tab) at line 1, column 9`}}, - {"'Except\tfor\ttabs'", []string{`string("Except\tfor\ttabs")`}}, + {"'Except\tfor\ttabs'", []string{`"Except\tfor\ttabs"`}}, {"'Invalid rune \xcd'", []string{`Error: invalid UTF8 rune at line 1, column 15`}}, } { - p := &parser{} + p := newParser() testParseHandler(t, p, p.startLiteralString, test) } } @@ -82,23 +82,23 @@ func TestMultiLineLiteralString(t *testing.T) { for _, test := range []parseTest{ {``, []string{`Error: unexpected end of file (expected opening three single quotes) at start of file`}}, {`'''missing close quote''`, []string{`Error: unexpected end of file (expected closing three single quotes) at line 1, column 25`}}, - {`''''''`, []string{`string("")`}}, - {"'''\n'''", []string{`string("")`}}, - {`'''I [dw]on't need \d{2} apples'''`, []string{`string("I [dw]on't need \\d{2} apples")`}}, - {"'''\nThere can\nbe newlines\r\nand \ttabs!\r\n'''", []string{`string("There can\nbe newlines\nand \ttabs!\n")`}}, + {`''''''`, []string{`""`}}, + {"'''\n'''", []string{`""`}}, + {`'''I [dw]on't need \d{2} apples'''`, []string{`"I [dw]on't need \\d{2} apples"`}}, + {"'''\nThere can\nbe newlines\r\nand \ttabs!\r\n'''", []string{`"There can\nbe newlines\nand \ttabs!\n"`}}, {"'''No other \f control characters'''", []string{`Error: invalid character in literal string: '\f' (no control chars allowed, except for tab and newline) at line 1, column 13`}}, {"'''No invalid runes allowed \xcd'''", []string{"Error: invalid UTF8 rune at line 1, column 29"}}, } { - p := &parser{} + p := newParser() testParseHandler(t, p, p.startMultiLineLiteralString, test) } } -func TestBasicStringWithUnescapedControlCharacters(t *testing.T) { +func TestBasipStringWithUnescapedControlCharacters(t *testing.T) { // A quick check for almost all characters that must be escaped. // The missing one (\x7f) is covered in the previous test. for i := 0x00; i <= 0x1F; i++ { - p := &parser{} + p := newParser() input := fmt.Sprintf(`"%c"`, rune(i)) expected := fmt.Sprintf(`Error: invalid character in basic string: %q (must be escaped) at line 1, column 2`, rune(i)) testParseHandler(t, p, p.startString, parseTest{input, []string{expected}}) diff --git a/value_table.go b/value_table.go new file mode 100644 index 0000000..b66800f --- /dev/null +++ b/value_table.go @@ -0,0 +1,111 @@ +package toml + +import ( + "git.makaay.nl/mauricem/go-parsekit/parse" +) + +var ( + // Tables (also known as hash tables or dictionaries) are collections of + // key/value pairs. They appear in square brackets on a line by themselves. + // You can tell them apart from arrays because arrays are only ever values. + // + // Under that, and until the next table or EOF are the key/values of that + // table. Key/value pairs within tables are not guaranteed to be in any + // specific order. + // + // [table-1] + // key1 = "some string" + // key2 = 123 + // + // [table-2] + // key1 = "another string" + // key2 = 456 + // + // Naming rules for tables are the same as for keys. + // + // [dog."tater.man"] + // type.name = "pug" + // + // Whitespace around the key is ignored, however, best practice is to not + // use any extraneous whitespace. + // + // [a.b.c] # this is best practice + // [ d.e.f ] # same as [d.e.f] + // [ g . h . i ] # same as [g.h.i] + // [ j . "ʞ" . 'l' ] # same as [j."ʞ".'l'] + // + // You don't need to specify all the super-tables if you don't want to. + // TOML knows how to do it for you. + // + // # [x] you + // # [x.y] don't + // # [x.y.z] need these + // [x.y.z.w] # for this to work + // + // Empty tables are allowed and simply have no key/value pairs within them. + // + tableOpen = c.Seq(dropBlanks, a.SquareOpen, dropBlanks) + tableClose = c.Seq(dropBlanks, a.SquareClose, dropBlanks, a.EndOfLine.Or(comment)) + + // Arrays of tables can be expressed by using a table name in double brackets. + // Each table with the same double bracketed name will be an element in the + // array. The tables are inserted in the order encountered. A double bracketed + // table without any key/value pairs will be considered an empty table. + // + // [[products]] + // name = "Hammer" + // sku = 738594937 + // + // [[products]] + // + // [[products]] + // name = "Nail" + // sku = 284758393 + // color = "gray" + // + // You can create nested arrays of tables as well. Just use the same double + // bracket syntax on sub-tables. Each double-bracketed sub-table will belong + // to the most recently defined table element above it. + // + // [[fruit]] + // name = "apple" + // + // [fruit.physical] + // color = "red" + // shape = "round" + // + // [[fruit.variety]] + // name = "red delicious" + // + // [[fruit.variety]] + // name = "granny smith" + // + // [[fruit]] + // name = "banana" + // + // [[fruit.variety]] + // name = "plantain" + tableArrayOpen = c.Seq(dropBlanks, a.SquareOpen, a.SquareOpen, dropBlanks) + tableArrayClose = c.Seq(dropBlanks, a.SquareClose, a.SquareClose, a.EndOfLine.Or(comment)) +) + +func (t *parser) startTable(p *parse.API) { + switch { + case p.Accept(tableOpen): + p.Handle(t.startPlainTable) + default: + p.Expected("a table") + } +} + +func (t *parser) startPlainTable(p *parse.API) { + if !p.Handle(t.startKey) { + return + } + if !p.Accept(tableClose) { + p.Expected("closing ']' for table name") + } + key := t.Items[0] + t.Items = t.Items[1:] + t.newTable(key) +} diff --git a/value_table_test.go b/value_table_test.go new file mode 100644 index 0000000..3095c6e --- /dev/null +++ b/value_table_test.go @@ -0,0 +1,23 @@ +package toml + +import ( + "testing" +) + +func TestTable(t *testing.T) { + for _, test := range []parseTest{ + {"", []string{`Error: unexpected end of file (expected a table) at start of file`}}, + {"[", []string{`Error: unexpected end of file (expected a key name) at line 1, column 2`}}, + {" \t [", []string{`Error: unexpected end of file (expected a key name) at line 1, column 5`}}, + {" \t [key", []string{`Error: unexpected end of file (expected closing ']' for table name) at line 1, column 8`}}, + {" \t [key.", []string{`key("key")`, `Error: unexpected end of file (expected a key name) at line 1, column 9`}}, + {" \t [key.'sub key'", []string{`Error: unexpected end of file (expected closing ']' for table name) at line 1, column 18`}}, + {" \t [key.'sub key' \t]", []string{}}, + {" \t [key.'sub key' \t] \t ", []string{}}, + {" \t [key.'sub key' \t] \t \n", []string{}}, + {" \t [key.'sub key' \t] \t # with comment\n", []string{}}, + } { + p := newParser() + testParseHandler(t, p, p.startTable, test) + } +} diff --git a/value_test.go b/value_test.go index 4453c2e..d2c7115 100644 --- a/value_test.go +++ b/value_test.go @@ -1,4 +1,4 @@ -package parser +package toml import ( "testing" @@ -7,39 +7,40 @@ import ( func TestValue(t *testing.T) { for _, test := range []parseTest{ {``, []string{`Error: unexpected end of file (expected a value) at start of file`}}, - {`"basic s\tring value"`, []string{`string("basic s\tring value")`}}, - {`'literal s\tring value'`, []string{`string("literal s\\tring value")`}}, - {"\"\"\"basic multi-line\nstring value\"\"\"", []string{`string("basic multi-line\nstring value")`}}, - {"'''literal multi-line\nstring value'''", []string{`string("literal multi-line\nstring value")`}}, - {"true", []string{`boolean(true)`}}, - {"false", []string{`boolean(false)`}}, - {"0", []string{`integer(0)`}}, - {"+0", []string{`integer(0)`}}, - {"-0", []string{`integer(0)`}}, - {"0.0", []string{`float(0)`}}, - {"+0.0", []string{`float(0)`}}, - {"-0.0", []string{`float(-0)`}}, - {"1234", []string{`integer(1234)`}}, - {"-1234", []string{`integer(-1234)`}}, - {"+9_8_7.6_5_4e-321", []string{`float(9.8765e-319)`}}, - {"-1_234.5678e-33", []string{`float(-1.2345678e-30)`}}, - {"inf", []string{`float(+Inf)`}}, - {"+inf", []string{`float(+Inf)`}}, - {"-inf", []string{`float(-Inf)`}}, - {"nan", []string{`float(NaN)`}}, - {"+nan", []string{`float(NaN)`}}, - {"-nan", []string{`float(NaN)`}}, - {"2019-06-19", []string{`date(2019-06-19 00:00:00 +0000 UTC)`}}, - {"08:38:54", []string{`time(0000-01-01 08:38:54 +0000 UTC)`}}, - {"2019-06-19 08:38:54", []string{`datetime(2019-06-19 08:38:54 +0000 UTC)`}}, - {"2019-06-19T08:38:54", []string{`datetime(2019-06-19 08:38:54 +0000 UTC)`}}, - {"2019-06-19 08:38:54", []string{`datetime(2019-06-19 08:38:54 +0000 UTC)`}}, - {"2019-06-19T08:38:54.88888", []string{`datetime(2019-06-19 08:38:54.88888 +0000 UTC)`}}, - {"1979-05-27T07:32:00Z", []string{`offset_datetime(1979-05-27 07:32:00 +0000 UTC)`}}, - {"1979-05-27T00:32:00-07:00", []string{`offset_datetime(1979-05-27 00:32:00 -0700 -0700)`}}, - {"1979-05-27T00:32:00.999999-07:00", []string{`offset_datetime(1979-05-27 00:32:00.999999 -0700 -0700)`}}, + {`"basic s\tring value"`, []string{`"basic s\tring value"`}}, + {`'literal s\tring value'`, []string{`"literal s\\tring value"`}}, + {"\"\"\"basic multi-line\nstring value\"\"\"", []string{`"basic multi-line\nstring value"`}}, + {"'''literal multi-line\nstring value'''", []string{`"literal multi-line\nstring value"`}}, + {"true", []string{`true`}}, + {"false", []string{`false`}}, + {"0", []string{`0`}}, + {"+0", []string{`0`}}, + {"-0", []string{`0`}}, + {"0.0", []string{`0`}}, + {"+0.0", []string{`0`}}, + {"-0.0", []string{`-0`}}, + {"1234", []string{`1234`}}, + {"-1234", []string{`-1234`}}, + {"+9_8_7.6_5_4e-321", []string{`9.8765e-319`}}, + {"-1_234.5678e-33", []string{`-1.2345678e-30`}}, + {"inf", []string{`+Inf`}}, + {"+inf", []string{`+Inf`}}, + {"-inf", []string{`-Inf`}}, + {"nan", []string{`NaN`}}, + {"+nan", []string{`NaN`}}, + {"-nan", []string{`NaN`}}, + {"2019-06-19", []string{`2019-06-19`}}, + {"08:38:54", []string{`08:38:54`}}, + {"08:38:54.8765487654876", []string{`08:38:54.876548765`}}, + {"2019-06-19 08:38:54", []string{`2019-06-19 08:38:54`}}, + {"2019-06-19T08:38:54", []string{`2019-06-19 08:38:54`}}, + {"2019-06-19T08:38:54.88888", []string{`2019-06-19 08:38:54.88888`}}, + {"1979-05-27T07:32:00Z", []string{`1979-05-27T07:32:00Z`}}, + {"1979-05-27T00:32:00-07:00", []string{`1979-05-27T00:32:00-07:00`}}, + {"1979-05-27T00:32:00.999999-07:00", []string{`1979-05-27T00:32:00.999999-07:00`}}, + {"[1,2,3]", []string{`[1, 2, 3]`}}, } { - p := &parser{} + p := newParser() testParseHandler(t, p, p.startValue, test) } }