Backup work.

ast.go

@@ -0,0 +1,199 @@
+package toml
+
+import (
+	"fmt"
+	"strings"
+	"time"
+)
+
+// item represents a TOML item.
+type item struct {
+	Type itemType
+	Data []interface{}
+}
+
+// table represents a TOML table.
+type table map[string]item
+
+// itemType identifies the semantic role of a TOML item.
+type itemType string
+
+const (
+	// TODO Would be nice to not need these in the end.
+	pComment itemType = "comment"
+	pKey     itemType = "key"
+	pAssign  itemType = "assign"
+
+	// TODO and just use these data types.
+	pString         itemType = "string"          // "various", 'types', """of""", '''strings'''
+	pInteger        itemType = "integer"         // 12345, 0xffee12a0, 0o0755, 0b101101011
+	pFloat          itemType = "float"           // 10.1234, 143E-12, 43.28377e+4, +inf, -inf, nan
+	pBoolean        itemType = "boolean"         // true or false
+	pOffsetDateTime itemType = "offset datetime" // 2019-06-18 10:32:15.173645362+0200
+	pLocalDateTime  itemType = "datetime"        // 2018-12-25 12:12:18.876772533
+	pLocalDate      itemType = "date"            // 2017-05-17
+	pLocalTime      itemType = "time"            // 23:01:22
+	pArray          itemType = "array"           // defined using an [[array.of.tables]]
+	pStaticArray    itemType = "static array"    // defined using ["an", "inline", "array"]
+	pTable          itemType = "table"           // defined using { "inline" = "table" } or [standard.table]
+)
+
+// newItem instantiates a new item struct.
+func newItem(itemType itemType, data ...interface{}) item {
+	return item{Type: itemType, Data: data}
+}
+
+func (t table) String() string {
+	return newItem(pTable, t).String()
+}
+
+func (parseItem item) String() string {
+	switch parseItem.Type {
+	case pString:
+		return fmt.Sprintf("%q", parseItem.Data[0])
+	case pInteger:
+		return fmt.Sprintf("%v", parseItem.Data[0])
+	case pFloat:
+		return fmt.Sprintf("%v", parseItem.Data[0])
+	case pBoolean:
+		return fmt.Sprintf("%v", parseItem.Data[0])
+	case pOffsetDateTime:
+		return parseItem.Data[0].(time.Time).Format(time.RFC3339Nano)
+	case pLocalDateTime:
+		return parseItem.Data[0].(time.Time).Format("2006-01-02 15:04:05.999999999")
+	case pLocalDate:
+		return parseItem.Data[0].(time.Time).Format("2006-01-02")
+	case pLocalTime:
+		return parseItem.Data[0].(time.Time).Format("15:04:05.999999999")
+	case pArray:
+		fallthrough
+	case pStaticArray:
+		items := make([]string, len(parseItem.Data[0].([]item)))
+		for i, d := range parseItem.Data[0].([]item) {
+			items[i] = d.String()
+		}
+		return fmt.Sprintf("[%s]", strings.Join(items, ", "))
+	case pTable:
+		items := make([]string, len(parseItem.Data))
+		pairs := parseItem.Data[0].(table)
+		i := 0
+		for k, v := range pairs {
+			items[i] = fmt.Sprintf("%q: %s", k, v.String())
+			i++
+		}
+		return fmt.Sprintf("{%s}", strings.Join(items, ", "))
+	case pComment:
+		return fmt.Sprintf("comment(%q)", parseItem.Data[0])
+	case pKey:
+		items := make([]string, len(parseItem.Data))
+		for i, e := range parseItem.Data {
+			items[i] = fmt.Sprintf("%q", e)
+		}
+		return fmt.Sprintf("key(%s)", strings.Join(items, ", "))
+	case pAssign:
+		return "="
+	default:
+		panic(fmt.Sprintf("Missing String() formatting for item type '%s'", parseItem.Type))
+	}
+}
+
+// parser holds the state for the TOML parser. All parsing functions are
+// methods of this struct.
+type parser struct {
+	Items   []item // a buffer for holding parsed items
+	Root    table  // the root-level TOML table (each TOML doc is implicitly a table)
+	Current table  // the currently active TOML table
+}
+
+func newParser() *parser {
+	p := &parser{Root: make(table)}
+	p.Current = p.Root
+	return p
+}
+
+func (t *parser) addParsedItem(itemType itemType, data ...interface{}) {
+	t.Items = append(t.Items, newItem(itemType, data...))
+}
+
+func (t *parser) setValue(key item, value item) error {
+	// When the key has multiple elements, then first make sure the table structure
+	// for storing the value exists.
+	var valueKey string
+	node := t.Current
+	l := len(key.Data)
+	if l > 1 {
+		pathKeys := key.Data[0 : l-1]
+		valueKey = key.Data[l-1].(string)
+		for i, name := range pathKeys {
+			name := name.(string)
+			if subItem, ok := node[name]; ok {
+				// An item was found at the current key. It is expected to be a table.
+				if subItem.Type != pTable {
+					path := formatKeyPath(key, i)
+					return fmt.Errorf("invalid key used: %s item already exists at key %s", subItem.Type, path)
+				}
+				node = subItem.Data[0].(table)
+			} else {
+				// No item was found at the current key. Create a new subtable.
+				subTable := make(table)
+				node[name] = newItem(pTable, subTable)
+				node = subTable
+			}
+		}
+	} else {
+		valueKey = key.Data[0].(string)
+		node = t.Current
+	}
+
+	if existing, ok := node[valueKey]; ok {
+		path := "moetnog"
+		return fmt.Errorf("Cannot store value: %s item already exists at key %s", existing.Type, path)
+	}
+
+	node[valueKey] = value
+	return nil
+}
+
+func (t *parser) newTable(key item) (table, error) {
+	node := t.Root
+	// Go over all requested levels of the key. For all levels, except the last
+	// one, it is okay if a table already exists. For at least the last level,
+	// no table or value must exist, because that would mean we are overwriting
+	// an existing key/value pair, which is not allowed.
+	for i, e := range key.Data {
+		name := e.(string)
+		if subItem, ok := node[name]; ok {
+			// You cannot overwrite an already defined key, regardless its value.
+			if subItem.Type != pTable {
+				path := formatKeyPath(key, i)
+				return nil, fmt.Errorf("Cannot create table: %s item already exists at key %s", subItem.Type, path)
+			}
+			// Like keys, you cannot define any table more than once. Doing so is invalid.
+			isLast := i == len(key.Data)-1
+			if isLast {
+				path := formatKeyPath(key, i)
+				return nil, fmt.Errorf("Cannot create table: table for key %s already exists", path)
+			}
+			node = subItem.Data[0].(table)
+		} else {
+			// Create the subtable.
+			subTable := make(table)
+			node[name] = newItem(pTable, subTable)
+			node = subTable
+		}
+	}
+	// From here on, key/value pairs are added to the newly defined table.
+	t.Current = node
+	return node, nil
+}
+
+func formatKeyPath(key item, end int) string {
+	var sb strings.Builder
+	for i := 0; i <= end; i++ {
+		if i > 0 {
+			sb.WriteRune('.')
+		}
+		sb.WriteString(fmt.Sprintf("%q", key.Data[i].(string)))
+	}
+	return sb.String()
+}

ast_test.go

@@ -0,0 +1,72 @@
+package toml
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestAST_ConstructStructure(t *testing.T) {
+	p := newParser()
+	p.Root["ding"] = newItem(pInteger, 10)
+	p.Root["dong"] = newItem(pString, "not a song")
+	subTable1, _ := p.newTable(newItem(pKey, "key1", "key2 a"))
+	subTable1["dooh"] = newItem(pBoolean, true)
+	subTable1["dah"] = newItem(pBoolean, false)
+	subTable2, _ := p.newTable(newItem(pKey, "key1", "key2 b"))
+	subTable2["dieh"] = newItem(pFloat, 1.111)
+	subTable2["duhh"] = newItem(pFloat, 1.18e-12)
+}
+
+func TestAST_StoreValueInRootTable(t *testing.T) {
+	testError(t, func() error {
+		p := newParser()
+		p.setValue(newItem(pKey, "key1"), newItem(pString, "value1"))
+		return p.setValue(newItem(pKey, "key2"), newItem(pString, "value2"))
+	}, "")
+}
+
+func TestAST_StoreValueWithMultipartKey_CreatesTableHierarchy(t *testing.T) {
+	testError(t, func() error {
+		p := newParser()
+		return p.setValue(newItem(pKey, "key1", "key2", "key3"), newItem(pString, "value"))
+	}, "")
+	// TODO an actual test assertion
+}
+
+func TestAST_StoreValueWithMultipartKey_UnderSubtable_CreatesTableHierarchy(t *testing.T) {
+	testError(t, func() error {
+		p := newParser()
+		p.newTable(newItem(pKey, "subkey1", "subkey2"))
+		err := p.setValue(newItem(pKey, "key1", "key2", "key3"), newItem(pString, "value"))
+		fmt.Printf("%s", p.Root)
+		return err
+	}, "")
+	t.Fail()
+	// TODO an actual test assertion
+}
+
+func TestAST_StoreDuplicateKeyInRootTable_ReturnsError(t *testing.T) {
+	testError(t, func() error {
+		p := newParser()
+		p.setValue(newItem(pKey, "key"), newItem(pString, "value"))
+		return p.setValue(newItem(pKey, "key"), newItem(pInteger, 321))
+	}, `Cannot store value: string item already exists at key "key"`)
+}
+
+func TestAST_GivenExistingTableAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
+	testError(t, func() error {
+		p := newParser()
+		p.newTable(newItem(pKey, "key1", "key2"))
+		_, err := p.newTable(newItem(pKey, "key1", "key2"))
+		return err
+	}, `Cannot create table: table for key "key1"."key2" already exists`)
+}
+
+func TestAST_GivenExistingItemAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
+	testError(t, func() error {
+		p := newParser()
+		p.Root["key"] = newItem(pString, "value")
+		_, err := p.newTable(newItem(pKey, "key"))
+		return err
+	}, `Cannot create table: string item already exists at key "key"`)
+}

comment.go

@@ -1,4 +1,4 @@
-package parser
+package toml
 
 import (
 	"git.makaay.nl/mauricem/go-parsekit/parse"
@@ -10,7 +10,7 @@ var comment = c.Seq(a.Hash, c.ZeroOrMore(c.Not(a.EndOfLine)), m.Drop(a.EndOfLine
 
 func (t *parser) startComment(p *parse.API) {
 	if p.Accept(comment) {
-		t.emitCommand(cComment, p.Result().String())
+		t.addParsedItem(pComment, p.Result().String())
 	} else {
 		p.Expected("comment")
 	}

comment_test.go

@@ -1,4 +1,4 @@
-package parser
+package toml
 
 import (
 	"testing"
@@ -18,7 +18,7 @@ func TestComment2(t *testing.T) {
 			`comment("# with data and newline")`,
 			`Error: unexpected input (expected end of file) at line 2, column 1`}},
 	} {
-		p := &parser{}
+		p := newParser()
 		testParseHandler(t, p, p.startComment, test)
 	}
 }

helpers_test.go

@@ -1,4 +1,4 @@
-package parser
+package toml
 
 import (
 	"fmt"
@@ -24,8 +24,8 @@ func testParseHandler(t *testing.T, p *parser, handler parse.Handler, test parse
 	defer func() {
 		recovered := recover()
 		results := []string{}
-		for _, cmd := range p.commands {
+		for _, item := range p.Items {
-			results = append(results, cmd.String())
+			results = append(results, item.String())
 		}
 		if err != nil {
 			results = append(results, fmt.Sprintf("Error: %s", err))
@@ -36,12 +36,12 @@ func testParseHandler(t *testing.T, p *parser, handler parse.Handler, test parse
 
 		for i, e := range test.expected {
 			if i > len(results)-1 {
-				t.Errorf("No result at index %d, expected: %s", i, e)
+				t.Errorf("No result at index %d for input %q, expected: %s", i, test.input, e)
 				continue
 			}
 			r := results[i]
 			if e != r {
-				t.Errorf("Unexpected result at index %d:\nexpected: %s\nactual: %s\n", i, e, r)
+				t.Errorf("Unexpected result at index %d for input %q:\nexpected: %s\nactual: %s\n", i, test.input, e, r)
 			}
 		}
 		if len(results) > len(test.expected) {
@@ -53,3 +53,18 @@ func testParseHandler(t *testing.T, p *parser, handler parse.Handler, test parse
 	}()
 	err = parse.New(handler)(test.input)
 }
+
+func testError(t *testing.T, code func() error, expected string) {
+	err := code()
+	if expected == "" {
+		if err != nil {
+			t.Fatalf("Unexpected error: %s", err)
+		}
+	} else {
+		if err == nil {
+			t.Fatalf("An error was expected, but no error was returned")
+		} else if err.Error() != expected {
+			t.Fatalf("Unexpected error:\nexpected: %s\nactual: %s\n", expected, err.Error())
+		}
+	}
+}

keyvaluepair.go

@@ -1,4 +1,4 @@
-package parser
+package toml
 
 import (
 	"git.makaay.nl/mauricem/go-parsekit/parse"
@@ -7,9 +7,6 @@ import (
 // The primary building block of a TOML document is the key/value pair.
 
 var (
-	dropWhitespace = m.Drop(a.Whitespace.Optional())
-	dropBlanks     = m.Drop(a.Blanks.Optional())
-
 	// Keys are on the left of the equals sign and values are on the right.
 	// Blank is ignored around key names and values. The key, equals
 	// sign, and value must be on the same line (though some values can be
@@ -73,13 +70,13 @@ func (t *parser) startKey(p *parse.API) {
 	case p.Peek(a.SingleQuote):
 		key, ok = t.parseLiteralString("key", p)
 	case p.Peek(a.DoubleQuote):
-		key, ok = t.parseBasicString("key", p)
+		key, ok = t.parseBasipString("key", p)
 	default:
 		p.Expected("a key name")
 		return
 	}
 	if ok {
-		t.emitCommand(cKey, key)
+		t.addParsedItem(pKey, key)
 		p.Handle(t.endOfKeyOrDot)
 	}
 }
@@ -90,14 +87,31 @@ func (t *parser) startKey(p *parse.API) {
 // practice is to not use any extraneous whitespace.
 func (t *parser) endOfKeyOrDot(p *parse.API) {
 	if p.Accept(keySeparatorDot) {
-		t.emitCommand(cKeyDot)
 		p.Handle(t.startKey)
+		return
 	}
+
+	// TODO not sure if we really need this index.
+	// Can't we alway simply use the full item list, given that we'll feed all
+	// results to the parser's table state?
+	// Do we even need to emit a key here? Shouldn't we just create the
+	// table structure in the parser, ready for followup calls to fill the data?
+	keyStart := len(t.Items) - 1
+	for keyStart > 0 && t.Items[keyStart-1].Type == pKey {
+		keyStart--
+	}
+	keyLen := len(t.Items) - keyStart
+	key := make([]interface{}, keyLen)
+	for i := 0; i < keyLen; i++ {
+		key[i] = t.Items[keyStart+i].Data[0].(string)
+	}
+	t.Items = t.Items[0:keyStart]
+	t.addParsedItem(pKey, key...)
 }
 
 func (t *parser) startAssignment(p *parse.API) {
 	if p.Accept(keyAssignment) {
-		t.emitCommand(cAssign)
+		t.addParsedItem(pAssign)
 	} else {
 		p.Expected("a value assignment")
 	}

keyvaluepair_test.go

@@ -1,4 +1,4 @@
-package parser
+package toml
 
 import "testing"
 
@@ -11,10 +11,10 @@ func TestKey(t *testing.T) {
 		{"mix-12_34", []string{`key("mix-12_34")`}},
 		{"-hey_good_Lookin123-", []string{`key("-hey_good_Lookin123-")`}},
 		{"wrong!", []string{`key("wrong")`, `Error: unexpected input (expected end of file) at line 1, column 6`}},
-		{"key1.", []string{`key("key1")`, `keydot`, `Error: unexpected end of file (expected a key name) at line 1, column 6`}},
+		{"key1.", []string{`key("key1")`, `Error: unexpected end of file (expected a key name) at line 1, column 6`}},
-		{"key1.key2", []string{`key("key1")`, `keydot`, `key("key2")`}},
+		{"key1.key2", []string{`key("key1", "key2")`}},
-		{"key . with . spaces", []string{`key("key")`, `keydot`, `key("with")`, `keydot`, `key("spaces")`}},
+		{"key . with . spaces", []string{`key("key", "with", "spaces")`}},
-		{"key  \t  . \twithtabs\t . \tandspaces", []string{`key("key")`, `keydot`, `key("withtabs")`, `keydot`, `key("andspaces")`}},
+		{"key  \t  . \twithtabs\t . \tandspaces", []string{`key("key", "withtabs", "andspaces")`}},
 		// Single quoted key tests
 		{"''", []string{`key("")`}},
 		{"'single quoted'", []string{`key("single quoted")`}},
@@ -26,9 +26,9 @@ func TestKey(t *testing.T) {
 		{`"escapes are in\terpreted"`, []string{`key("escapes are in\terpreted")`}},
 		{`"using 'inner' \"quotes\""`, []string{`key("using 'inner' \"quotes\"")`}},
 		// Mixed key types
-		{`this.'i\s'."madness\t".''`, []string{`key("this")`, `keydot`, `key("i\\s")`, `keydot`, `key("madness\t")`, `keydot`, `key("")`}},
+		{`this.'i\s'."madness\t".''`, []string{`key("this", "i\\s", "madness\t", "")`}},
 	} {
-		p := &parser{}
+		p := newParser()
 		testParseHandler(t, p, p.startKey, test)
 	}
 }
@@ -36,11 +36,11 @@ func TestKey(t *testing.T) {
 func TestAssignment(t *testing.T) {
 	for _, test := range []parseTest{
 		{"", []string{`Error: unexpected end of file (expected a value assignment) at start of file`}},
-		{"=", []string{`assign`}},
+		{"=", []string{`=`}},
-		{" \t = \t ", []string{`assign`}},
+		{" \t = \t ", []string{`=`}},
 		{" \n = \n ", []string{`Error: unexpected input (expected a value assignment) at start of file`}},
 	} {
-		p := &parser{}
+		p := newParser()
 		testParseHandler(t, p, p.startAssignment, test)
 	}
 }
@@ -51,94 +51,103 @@ func TestKeyValuePair(t *testing.T) {
 		{" ", []string{}},
 		{" \t ", []string{}},
 		{" key ", []string{`key("key")`, `Error: unexpected input (expected a value assignment) at line 1, column 5`}},
-		{" key \t=", []string{`key("key")`, `assign`, `Error: unexpected end of file (expected a value) at line 1, column 8`}},
+		{" key \t=", []string{`key("key")`, `=`, `Error: unexpected end of file (expected a value) at line 1, column 8`}},
-		{"key = # INVALID", []string{`key("key")`, `assign`, `Error: unexpected input (expected a value) at line 1, column 7`}},
+		{"key = # INVALID", []string{`key("key")`, `=`, `Error: unexpected input (expected a value) at line 1, column 7`}},
-		{" key \t =\t \"The Value\"  \r\n", []string{`key("key")`, `assign`, `string("The Value")`}},
+		{" key \t =\t \"The Value\"  \r\n", []string{`key("key")`, `=`, `"The Value"`}},
-		{`3.14159 = "pi"`, []string{`key("3")`, `keydot`, `key("14159")`, `assign`, `string("pi")`}},
+		{`3.14159 = "pi"`, []string{`key("3", "14159")`, `=`, `"pi"`}},
-		{`"ʎǝʞ" = "value"`, []string{`key("ʎǝʞ")`, `assign`, `string("value")`}},
+		{`"ʎǝʞ" = "value"`, []string{`key("ʎǝʞ")`, `=`, `"value"`}},
-		{`key = "value"  # This is a comment at the end of a line`, []string{`key("key")`, `assign`, `string("value")`, `comment("# This is a comment at the end of a line")`}},
+		{`key = "value"  # This is a comment at the end of a line`, []string{`key("key")`, `=`, `"value"`, `comment("# This is a comment at the end of a line")`}},
-		{`another = "# This is not a comment"`, []string{`key("another")`, `assign`, `string("# This is not a comment")`}},
+		{`another = "# This is not a comment"`, []string{`key("another")`, `=`, `"# This is not a comment"`}},
-		{"key1=\"value1\"key2=\"value2\"\r\nkey3=\"value3\"", []string{
+		{"key1=\"value1\"key2=\"value2\"\r\nkey3a.key3b=\"value3\"", []string{
-			`key("key1")`, `assign`, `string("value1")`,
+			`key("key1")`, `=`, `"value1"`,
-			`key("key2")`, `assign`, `string("value2")`,
+			`key("key2")`, `=`, `"value2"`,
-			`key("key3")`, `assign`, `string("value3")`}},
+			`key("key3a", "key3b")`, `=`, `"value3"`}},
 		{"with=\"comments\"# boring \nanother.cool =\"one\" \t # to the end\r\n", []string{
-			`key("with")`, `assign`, `string("comments")`, `comment("# boring ")`,
+			`key("with")`, `=`, `"comments"`, `comment("# boring ")`,
-			`key("another")`, `keydot`, `key("cool")`, `assign`, `string("one")`, `comment("# to the end")`}},
+			`key("another", "cool")`, `=`, `"one"`, `comment("# to the end")`}},
 	} {
-		p := &parser{}
+		p := newParser()
 		testParseHandler(t, p, p.startKeyValuePair, test)
 	}
 }
 
 func TestKeyValuePair_ForAllTypes(t *testing.T) {
 	for _, test := range []parseTest{
-		{"string='literal'", []string{`key("string")`, `assign`, `string("literal")`}},
+		{"string='literal'", []string{`key("string")`, `=`, `"literal"`}},
-		{"string='''literal\nmulti-line'''", []string{`key("string")`, `assign`, `string("literal\nmulti-line")`}},
+		{"string='''literal\nmulti-line'''", []string{`key("string")`, `=`, `"literal\nmulti-line"`}},
-		{`string="basic"`, []string{`key("string")`, `assign`, `string("basic")`}},
+		{`string="basic"`, []string{`key("string")`, `=`, `"basic"`}},
-		{"string=\"\"\"basic\nmulti-line\"\"\"", []string{`key("string")`, `assign`, `string("basic\nmulti-line")`}},
+		{"string=\"\"\"basic\nmulti-line\"\"\"", []string{`key("string")`, `=`, `"basic\nmulti-line"`}},
-		{"integer=1_234_567", []string{`key("integer")`, `assign`, `integer(1234567)`}},
+		{"integer=1_234_567", []string{`key("integer")`, `=`, `1234567`}},
-		{"integer=42", []string{`key("integer")`, `assign`, `integer(42)`}},
+		{"integer=42", []string{`key("integer")`, `=`, `42`}},
-		{"integer=0x42", []string{`key("integer")`, `assign`, `integer(66)`}},
+		{"integer=0x42", []string{`key("integer")`, `=`, `66`}},
-		{"integer=0o42", []string{`key("integer")`, `assign`, `integer(34)`}},
+		{"integer=0o42", []string{`key("integer")`, `=`, `34`}},
-		{"integer=0b101010", []string{`key("integer")`, `assign`, `integer(42)`}},
+		{"integer=0b101010", []string{`key("integer")`, `=`, `42`}},
-		{"float=42.37", []string{`key("float")`, `assign`, `float(42.37)`}},
+		{"float=42.37", []string{`key("float")`, `=`, `42.37`}},
-		{"float=42e+37", []string{`key("float")`, `assign`, `float(4.2e+38)`}},
+		{"float=42e+37", []string{`key("float")`, `=`, `4.2e+38`}},
-		{"float=42.37e-11", []string{`key("float")`, `assign`, `float(4.237e-10)`}},
+		{"float=42.37e-11", []string{`key("float")`, `=`, `4.237e-10`}},
-		{"boolean=true", []string{`key("boolean")`, `assign`, `boolean(true)`}},
+		{"boolean=true", []string{`key("boolean")`, `=`, `true`}},
-		{"boolean=false", []string{`key("boolean")`, `assign`, `boolean(false)`}},
+		{"boolean=false", []string{`key("boolean")`, `=`, `false`}},
-		{"date=2019-01-01", []string{`key("date")`, `assign`, `date(2019-01-01 00:00:00 +0000 UTC)`}},
+		{"date=2019-01-01", []string{`key("date")`, `=`, `2019-01-01`}},
-		{"time=15:03:11", []string{`key("time")`, `assign`, `time(0000-01-01 15:03:11 +0000 UTC)`}},
+		{"time=15:03:11", []string{`key("time")`, `=`, `15:03:11`}},
-		{"datetime=2021-02-01 15:03:11.123", []string{`key("datetime")`, `assign`, `datetime(2021-02-01 15:03:11.123 +0000 UTC)`}},
+		{"datetime=2021-02-01 15:03:11.123", []string{`key("datetime")`, `=`, `2021-02-01 15:03:11.123`}},
-		{"offset_datetime=1111-11-11 11:11:11.111111111+11:11", []string{`key("offset_datetime")`, `assign`, `offset_datetime(1111-11-11 11:11:11.111111111 +1111 +1111)`}},
+		{"offset_datetime=1111-11-11 11:11:11.111111111+11:11", []string{`key("offset_datetime")`, `=`, `1111-11-11T11:11:11.111111111+11:11`}},
+		{"static_array=['a', 'static', 'array']", []string{`key("static_array")`, `=`, `["a", "static", "array"]`}},
 	} {
-		p := &parser{}
+		p := newParser()
 		testParseHandler(t, p, p.startKeyValuePair, test)
 	}
 }
 
 func TestKeyValuePair_ExamplesFromSpecification(t *testing.T) {
 	for _, test := range []parseTest{
-		{"int1 = +99", []string{`key("int1")`, `assign`, `integer(99)`}},
+		{"int1 = +99", []string{`key("int1")`, `=`, `99`}},
-		{"int2 = 42", []string{`key("int2")`, `assign`, `integer(42)`}},
+		{"int2 = 42", []string{`key("int2")`, `=`, `42`}},
-		{"int3 = 0", []string{`key("int3")`, `assign`, `integer(0)`}},
+		{"int3 = 0", []string{`key("int3")`, `=`, `0`}},
-		{"int4 = -17", []string{`key("int4")`, `assign`, `integer(-17)`}},
+		{"int4 = -17", []string{`key("int4")`, `=`, `-17`}},
-		{"int5 = 1_000", []string{`key("int5")`, `assign`, `integer(1000)`}},
+		{"int5 = 1_000", []string{`key("int5")`, `=`, `1000`}},
-		{"int6 = 5_349_221", []string{`key("int6")`, `assign`, `integer(5349221)`}},
+		{"int6 = 5_349_221", []string{`key("int6")`, `=`, `5349221`}},
-		{"int7 = 1_2_3_4_5  # VALID but discouraged", []string{`key("int7")`, `assign`, `integer(12345)`, `comment("# VALID but discouraged")`}},
+		{"int7 = 1_2_3_4_5  # VALID but discouraged", []string{`key("int7")`, `=`, `12345`, `comment("# VALID but discouraged")`}},
-		{"hex1 = 0xDEADBEEF", []string{`key("hex1")`, `assign`, `integer(3735928559)`}},
+		{"hex1 = 0xDEADBEEF", []string{`key("hex1")`, `=`, `3735928559`}},
-		{"hex2 = 0xdeadbeef", []string{`key("hex2")`, `assign`, `integer(3735928559)`}},
+		{"hex2 = 0xdeadbeef", []string{`key("hex2")`, `=`, `3735928559`}},
-		{"hex3 = 0xdead_beef", []string{`key("hex3")`, `assign`, `integer(3735928559)`}},
+		{"hex3 = 0xdead_beef", []string{`key("hex3")`, `=`, `3735928559`}},
-		{"oct1 = 0o01234567", []string{`key("oct1")`, `assign`, `integer(342391)`}},
+		{"oct1 = 0o01234567", []string{`key("oct1")`, `=`, `342391`}},
-		{"oct2 = 0o755", []string{`key("oct2")`, `assign`, `integer(493)`}},
+		{"oct2 = 0o755", []string{`key("oct2")`, `=`, `493`}},
-		{"bin1 = 0b11010110", []string{`key("bin1")`, `assign`, `integer(214)`}},
+		{"bin1 = 0b11010110", []string{`key("bin1")`, `=`, `214`}},
-		{"flt1 = +1.0", []string{`key("flt1")`, `assign`, `float(1)`}},
+		{"flt1 = +1.0", []string{`key("flt1")`, `=`, `1`}},
-		{"flt2 = 3.1415", []string{`key("flt2")`, `assign`, `float(3.1415)`}},
+		{"flt2 = 3.1415", []string{`key("flt2")`, `=`, `3.1415`}},
-		{"flt3 = -0.01", []string{`key("flt3")`, `assign`, `float(-0.01)`}},
+		{"flt3 = -0.01", []string{`key("flt3")`, `=`, `-0.01`}},
-		{"flt4 = 5e+22", []string{`key("flt4")`, `assign`, `float(5e+22)`}},
+		{"flt4 = 5e+22", []string{`key("flt4")`, `=`, `5e+22`}},
-		{"flt5 = 1e6", []string{`key("flt5")`, `assign`, `float(1e+06)`}},
+		{"flt5 = 1e6", []string{`key("flt5")`, `=`, `1e+06`}},
-		{"flt6 = -2E-2", []string{`key("flt6")`, `assign`, `float(-0.02)`}},
+		{"flt6 = -2E-2", []string{`key("flt6")`, `=`, `-0.02`}},
-		{"flt7 = 6.626e-34", []string{`key("flt7")`, `assign`, `float(6.626e-34)`}},
+		{"flt7 = 6.626e-34", []string{`key("flt7")`, `=`, `6.626e-34`}},
-		{"flt8 = 224_617.445_991_228", []string{`key("flt8")`, `assign`, `float(224617.445991228)`}},
+		{"flt8 = 224_617.445_991_228", []string{`key("flt8")`, `=`, `224617.445991228`}},
-		{"sf1 = inf  # positive infinity", []string{`key("sf1")`, `assign`, `float(+Inf)`, `comment("# positive infinity")`}},
+		{"sf1 = inf  # positive infinity", []string{`key("sf1")`, `=`, `+Inf`, `comment("# positive infinity")`}},
-		{"sf2 = +inf # positive infinity", []string{`key("sf2")`, `assign`, `float(+Inf)`, `comment("# positive infinity")`}},
+		{"sf2 = +inf # positive infinity", []string{`key("sf2")`, `=`, `+Inf`, `comment("# positive infinity")`}},
-		{"sf3 = -inf # negative infinity", []string{`key("sf3")`, `assign`, `float(-Inf)`, `comment("# negative infinity")`}},
+		{"sf3 = -inf # negative infinity", []string{`key("sf3")`, `=`, `-Inf`, `comment("# negative infinity")`}},
-		{"sf4 = nan  # actual sNaN/qNaN encoding is implementation-specific", []string{`key("sf4")`, `assign`, `float(NaN)`, `comment("# actual sNaN/qNaN encoding is implementation-specific")`}},
+		{"sf4 = nan  # actual sNaN/qNaN encoding is implementation-specific", []string{`key("sf4")`, `=`, `NaN`, `comment("# actual sNaN/qNaN encoding is implementation-specific")`}},
-		{"sf5 = +nan # same as `nan`", []string{`key("sf5")`, `assign`, `float(NaN)`, "comment(\"# same as `nan`\")"}},
+		{"sf5 = +nan # same as `nan`", []string{`key("sf5")`, `=`, `NaN`, "comment(\"# same as `nan`\")"}},
-		{"sf6 = -nan # valid, actual encoding is implementation-specific", []string{`key("sf6")`, `assign`, `float(NaN)`, `comment("# valid, actual encoding is implementation-specific")`}},
+		{"sf6 = -nan # valid, actual encoding is implementation-specific", []string{`key("sf6")`, `=`, `NaN`, `comment("# valid, actual encoding is implementation-specific")`}},
-		{"bool1 = true", []string{`key("bool1")`, `assign`, `boolean(true)`}},
+		{"bool1 = true", []string{`key("bool1")`, `=`, `true`}},
-		{"bool2 = false", []string{`key("bool2")`, `assign`, `boolean(false)`}},
+		{"bool2 = false", []string{`key("bool2")`, `=`, `false`}},
-		{"odt1 = 1979-05-27T07:32:00Z", []string{`key("odt1")`, `assign`, `offset_datetime(1979-05-27 07:32:00 +0000 UTC)`}},
+		{"odt1 = 1979-05-27T07:32:00Z", []string{`key("odt1")`, `=`, `1979-05-27T07:32:00Z`}},
-		{"odt2 = 1979-05-27T00:32:00-07:00", []string{`key("odt2")`, `assign`, `offset_datetime(1979-05-27 00:32:00 -0700 -0700)`}},
+		{"odt2 = 1979-05-27T00:32:00-07:00", []string{`key("odt2")`, `=`, `1979-05-27T00:32:00-07:00`}},
-		{"odt3 = 1979-05-27T00:32:00.999999-07:00", []string{`key("odt3")`, `assign`, `offset_datetime(1979-05-27 00:32:00.999999 -0700 -0700)`}},
+		{"odt3 = 1979-05-27T00:32:00.999999-07:00", []string{`key("odt3")`, `=`, `1979-05-27T00:32:00.999999-07:00`}},
-		{"odt4 = 1979-05-27 07:32:00Z", []string{`key("odt4")`, `assign`, `offset_datetime(1979-05-27 07:32:00 +0000 UTC)`}},
+		{"odt4 = 1979-05-27 07:32:00Z", []string{`key("odt4")`, `=`, `1979-05-27T07:32:00Z`}},
-		{"ldt1 = 1979-05-27T07:32:00", []string{`key("ldt1")`, `assign`, `datetime(1979-05-27 07:32:00 +0000 UTC)`}},
+		{"ldt1 = 1979-05-27T07:32:00", []string{`key("ldt1")`, `=`, `1979-05-27 07:32:00`}},
-		{"ldt2 = 1979-05-27T00:32:00.999999", []string{`key("ldt2")`, `assign`, `datetime(1979-05-27 00:32:00.999999 +0000 UTC)`}},
+		{"ldt2 = 1979-05-27T00:32:00.999999", []string{`key("ldt2")`, `=`, `1979-05-27 00:32:00.999999`}},
-		{"ld1 = 1979-05-27", []string{`key("ld1")`, `assign`, `date(1979-05-27 00:00:00 +0000 UTC)`}},
+		{"ld1 = 1979-05-27", []string{`key("ld1")`, `=`, `1979-05-27`}},
-		{"lt1 = 07:32:00", []string{`key("lt1")`, `assign`, `time(0000-01-01 07:32:00 +0000 UTC)`}},
+		{"lt1 = 07:32:00", []string{`key("lt1")`, `=`, `07:32:00`}},
-		{"lt2 = 00:32:00.999999", []string{`key("lt2")`, `assign`, `time(0000-01-01 00:32:00.999999 +0000 UTC)`}},
+		{"lt2 = 00:32:00.999999", []string{`key("lt2")`, `=`, `00:32:00.999999`}},
+		{"arr1 = [ 1, 2, 3 ]", []string{`key("arr1")`, `=`, `[1, 2, 3]`}},
+		{`arr2 = [ "red", "yellow", "green" ]`, []string{`key("arr2")`, `=`, `["red", "yellow", "green"]`}},
+		{`arr3 = [ [ 1, 2 ], [3, 4, 5] ]`, []string{`key("arr3")`, `=`, `[[1, 2], [3, 4, 5]]`}},
+		{`arr4 = [ "all", 'strings', """are the same""", '''type''']`, []string{`key("arr4")`, `=`, `["all", "strings", "are the same", "type"]`}},
+		{`arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]`, []string{`key("arr5")`, `=`, `[[1, 2], ["a", "b", "c"]]`}},
+		{`arr6 = [ 1, 2.0 ] # INVALID`, []string{`key("arr6")`, `=`, `Error: type mismatch in array of integers: found an item of type float at line 1, column 16`}},
+		{"arr7 = [\n  1, 2, 3\n]", []string{`key("arr7")`, `=`, `[1, 2, 3]`}},
+		{"arr8 = [\n  1,\n  2, # this is ok\n]", []string{`key("arr8")`, `=`, `[1, 2]`}},
 	} {
-		p := &parser{}
+		p := newParser()
 		testParseHandler(t, p, p.startKeyValuePair, test)
 	}
 }

toml.go

@@ -1,67 +1,22 @@
-package parser
+package toml
 
 import (
-	"fmt"
-	"strings"
-
 	"git.makaay.nl/mauricem/go-parsekit/tokenize"
 )
 
-// Easy access to the parsekit.tokenize definitions.
+// Some globally useful tokenizer definitions.
-var c, a, m, tok = tokenize.C, tokenize.A, tokenize.M, tokenize.T
+var (
+	c, a, m, tok = tokenize.C, tokenize.A, tokenize.M, tokenize.T
 
-type cmdType string
+	// From the specs: "Whitespace means tab (0x09) or space (0x20)."
+	// In this package, we name this a blank, to be in line with the
+	// terminology as used in parsekit.
+	blank = a.Runes('\t', ' ')
 
-// Command types that are emitted by the parser.
+	// Newline means LF (0x0A) or CRLF (0x0D0A).
-const (
+	// This matches the default newline as defined by parsekit.
-	cComment        cmdType = "comment"         // a # comment at the end of the line
+	newline = a.Newline
-	cKey            cmdType = "key"             // set key name
+
-	cKeyDot         cmdType = "keydot"          // new key stack level
+	dropBlanks     = m.Drop(c.ZeroOrMore(blank))
-	cAssign         cmdType = "assign"          // assign a value
+	dropWhitespace = m.Drop(c.ZeroOrMore(blank.Or(newline)))
-	csetStrVal      cmdType = "string"          // set a string value
-	csetIntVal      cmdType = "integer"         // set an integer value
-	csetFloatVal    cmdType = "float"           // set a float value
-	csetBoolVal     cmdType = "boolean"         // set a boolean value
-	coffsetDateTime cmdType = "offset_datetime" // set a date/time value with timezone information
-	clocalDateTime  cmdType = "datetime"  // set a local date/time value
-	clocalDate      cmdType = "date"      // set a local date value
-	clocalTime      cmdType = "time"      // set a local time value
 )
-
-type parser struct {
-	commands []cmd
-	keyStack []string
-}
-
-type cmd struct {
-	command cmdType
-	args    []interface{}
-}
-
-func (cmd *cmd) String() string {
-	if len(cmd.args) == 0 {
-		return fmt.Sprintf("%s", cmd.command)
-	}
-	args := make([]string, len(cmd.args))
-	for i, arg := range cmd.args {
-		switch arg.(type) {
-		case string:
-			args[i] = fmt.Sprintf("%q", arg)
-		default:
-			args[i] = fmt.Sprintf("%v", arg)
-		}
-	}
-	return fmt.Sprintf("%s(%s)", cmd.command, strings.Join(args, ", "))
-}
-
-func (p *parser) emitCommand(command cmdType, args ...interface{}) {
-	c := cmd{command: command, args: args}
-	p.commands = append(p.commands, c)
-}
-
-// Parse starts the parser for the provided input.
-// func Parse(input interface{}) []cmd {
-// 	p := &parser{}
-// 	parse.New(p.startKeyValuePair)(input)
-// 	return p.commands
-// }

toml_test.go

@@ -1,34 +0,0 @@
-package parser_test
-
-// func TestEmptyInput(t *testing.T) {
-// 	runStatesT(t, statesT{"empty string", "", "", ""})
-// }
-
-// func TestFullIncludesLineAndRowPosition(t *testing.T) {
-// 	p := toml.Parse("# 12345 abcde\t\n\n\n# 67890\r\n# 12345\n   +")
-// 	_, err := parseItemsToArray(p)
-// 	actual := err.Error()
-// 	expected := "unexpected input (expected end of file) at line 6, column 3"
-// 	if actual != expected {
-// 		t.Errorf("Unexpected error message:\nexpected: %s\nactual: %s\n", expected, actual)
-// 	}
-// }
-
-// func TestInvalidUTF8Data(t *testing.T) {
-// 	runStatesTs(t, []statesT{
-// 		{"bare key 1", "\xbc", "", "invalid UTF8 character in input (expected end of file)"},
-// 		{"bare key 2", "key\xbc", "[key]", "invalid UTF8 character in input (expected a value assignment)"},
-// 		{"start of value", "key=\xbc", "[key]=", "invalid UTF8 character in input (expected a value)"},
-// 		{"basic string value", "a=\"\xbc\"", "[a]=", "invalid UTF8 character in input (expected string contents)"},
-// 	})
-// }
-
-// func TestWhiteSpaceAndNewlines(t *testing.T) {
-// 	runStatesTs(t, []statesT{
-// 		{"space", " ", "", ""},
-// 		{"tab", "\t", "", ""},
-// 		{"newline", "\n", "", ""},
-// 		{"all blanks and newlines", " \t \t \r\n\n  \n \t", "", ""},
-// 		{"bare carriage return", "\r", "", "unexpected character '\\r' (expected end of file)"},
-// 	})
-// }

value.go

@@ -1,4 +1,4 @@
-package parser
+package toml
 
 import (
 	"git.makaay.nl/mauricem/go-parsekit/parse"
@@ -22,6 +22,8 @@ func (t *parser) startValue(p *parse.API) {
 		} else {
 			p.Handle(t.startNumber)
 		}
+	case p.Peek(a.SquareOpen):
+		p.Handle(t.startArray)
 	default:
 		p.Expected("a value")
 	}

value_array.go

@@ -0,0 +1,86 @@
+package toml
+
+import (
+	"git.makaay.nl/mauricem/go-parsekit/parse"
+)
+
+// Arrays are square brackets with values inside. Whitespace is ignored.
+// Elements are separated by commas. Data types may not be mixed (different
+// ways to define strings should be considered the same type, and so should
+// arrays with different element types).
+//
+//     arr1 = [ 1, 2, 3 ]
+//     arr2 = [ "red", "yellow", "green" ]
+//     arr3 = [ [ 1, 2 ], [3, 4, 5] ]
+//     arr4 = [ "all", 'strings', """are the same""", '''type''']
+//     arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]
+//
+//     arr6 = [ 1, 2.0 ] # INVALID
+//
+// Arrays can also be multiline. A terminating comma (also called trailing
+// comma) is ok after the last value of the array. There can be an arbitrary
+// number of newlines and comments before a value and before the closing bracket.
+//
+//     arr7 = [
+//       1, 2, 3
+//     ]
+//
+//     arr8 = [
+//       1,
+//       2, # this is ok
+//     ]
+var (
+	arraySpace     = c.ZeroOrMore(c.Any(blank, newline, comment))
+	arrayOpen      = a.SquareOpen.Then(arraySpace)
+	arraySeparator = c.Seq(arraySpace, a.Comma, arraySpace)
+	arrayClose     = c.Seq(c.Optional(arraySpace.Then(a.Comma)), arraySpace, a.SquareClose)
+)
+
+func (t *parser) startArray(p *parse.API) {
+	// Check for the start of the array.
+	if !p.Accept(arrayOpen) {
+		p.Expected("an array")
+		return
+	}
+
+	items := []item{}
+
+	// Check for an empty array.
+	if p.Accept(arrayClose) {
+		t.addParsedItem(pStaticArray, items)
+		return
+	}
+
+	// Not an empty array, parse the items.
+	for {
+		// Check for a valid item.
+		if !p.Handle(t.startValue) {
+			return
+		}
+
+		// Pop the item from the value parsing and append it to the array items.
+		parseItems, item := t.Items[0:len(t.Items)-1], t.Items[len(t.Items)-1]
+		t.Items = parseItems
+
+		// Data types may not be mixed (different ways to define strings should be
+		// considered the same type, and so should arrays with different element types).
+		if len(items) > 0 && item.Type != items[0].Type {
+			p.Error("type mismatch in array of %ss: found an item of type %s", items[0].Type, item.Type)
+			return
+		}
+
+		items = append(items, item)
+
+		// Check for the end of the array.
+		if p.Accept(arrayClose) {
+			t.addParsedItem(pStaticArray, items)
+			return
+		}
+
+		// Not the end of the array? Then we should find an array separator.
+		if !p.Accept(arraySeparator) {
+			p.Expected("an array separator")
+			return
+		}
+	}
+}

value_array_test.go

@@ -0,0 +1,40 @@
+package toml
+
+import (
+	"testing"
+)
+
+func TestArray(t *testing.T) {
+	for _, test := range []parseTest{
+		{"", []string{`Error: unexpected end of file (expected an array) at start of file`}},
+		{"[ape", []string{`Error: unexpected input (expected a value) at line 1, column 2`}},
+		{"[1", []string{`Error: unexpected end of file (expected an array separator) at line 1, column 3`}},
+		{"[]", []string{`[]`}},
+		{"[\n]", []string{`[]`}},
+		{"[,]", []string{`[]`}},
+		{"[ , ]", []string{`[]`}},
+		{"[ \n , \r\n ]", []string{`[]`}},
+		{"[ \t , \t ]", []string{`[]`}},
+		{"[\r\n\r\n   ,   \r\n  \t\n]", []string{`[]`}},
+		{"[\n#comment on its own line\n]", []string{`[]`}},
+		{"[#comment before close\n]", []string{`[]`}},
+		{"[,#comment after separator\n]", []string{`[]`}},
+		{"[#comment before separator\n,]", []string{`[]`}},
+		{"[#comment before value\n1]", []string{`[1]`}},
+		{"[1#comment after value\n]", []string{`[1]`}},
+		{"[1\n#comment on its own line after value\n]", []string{`[1]`}},
+		{"[1#comment 1\n#comment 2\n#comment 3\n , \n2]", []string{`[1, 2]`}},
+		{"[1]", []string{`[1]`}},
+		{"[1,0x2, 0b11, 0o4]", []string{`[1, 2, 3, 4]`}},
+		{"[0.1,0.2,3e-1,0.04e+1, nan, inf]", []string{`[0.1, 0.2, 0.3, 0.4, NaN, +Inf]`}},
+		{"[\n\t 'a', \"b\", '''c''', \"\"\"d\ne\"\"\",\n \t]", []string{`["a", "b", "c", "d\ne"]`}},
+		{`[1, 2, 3, "four"]`, []string{`Error: type mismatch in array of integers: found an item of type string at line 1, column 17`}},
+		{`[[1],['a']]`, []string{`[[1], ["a"]]`}},
+		{`[[[],[]],[]]`, []string{`[[[], []], []]`}},
+		{"[\r\n\r\n \t\n [\r\n\r\n\t  [],[\t]],\t\n[]\t \t \n   ]", []string{`[[[], []], []]`}},
+		{`[[1],'a']`, []string{`Error: type mismatch in array of static arrays: found an item of type string at line 1, column 9`}},
+	} {
+		p := newParser()
+		testParseHandler(t, p, p.startArray, test)
+	}
+}

value_boolean.go

@@ -1,4 +1,4 @@
-package parser
+package toml
 
 import (
 	"git.makaay.nl/mauricem/go-parsekit/parse"
@@ -8,9 +8,9 @@ import (
 func (t *parser) startBoolean(p *parse.API) {
 	switch {
 	case p.Accept(a.Str("true")):
-		t.emitCommand(csetBoolVal, true)
+		t.addParsedItem(pBoolean, true)
 	case p.Accept(a.Str("false")):
-		t.emitCommand(csetBoolVal, false)
+		t.addParsedItem(pBoolean, false)
 	default:
 		p.Expected("true or false")
 	}

value_boolean_test.go

@@ -1,4 +1,4 @@
-package parser
+package toml
 
 import (
 	"testing"
@@ -7,14 +7,14 @@ import (
 func TestBoolean(t *testing.T) {
 	for _, test := range []parseTest{
 		{``, []string{`Error: unexpected end of file (expected true or false) at start of file`}},
-		{`true`, []string{`boolean(true)`}},
+		{`true`, []string{`true`}},
-		{`false`, []string{`boolean(false)`}},
+		{`false`, []string{`false`}},
 		{`yes`, []string{`Error: unexpected input (expected true or false) at start of file`}},
 		{`no`, []string{`Error: unexpected input (expected true or false) at start of file`}},
 		{`1`, []string{`Error: unexpected input (expected true or false) at start of file`}},
 		{`0`, []string{`Error: unexpected input (expected true or false) at start of file`}},
 	} {
-		p := &parser{}
+		p := newParser()
 		testParseHandler(t, p, p.startBoolean, test)
 	}
 }

value_datetime.go

@@ -1,4 +1,4 @@
-package parser
+package toml
 
 import (
 	"time"
@@ -66,10 +66,10 @@ var (
 	// The full date/time parse format, based on the above definitions.
 	// The first token denotes the type of date/time value.
 	// The rest of the tokens contain layout fragments for time.Parse().
-	offsetDateTime = tok.Str(coffsetDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok, tzTok))
+	offsetDateTime = tok.Str(pOffsetDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok, tzTok))
-	localDateTime  = tok.Str(clocalDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok))
+	localDateTime  = tok.Str(pLocalDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok))
-	localDate      = tok.Str(clocalDate, dateTok)
+	localDate      = tok.Str(pLocalDate, dateTok)
-	localTime      = tok.Str(clocalTime, c.Seq(timeTok, microTok))
+	localTime      = tok.Str(pLocalTime, c.Seq(timeTok, microTok))
 	datetime       = c.Any(offsetDateTime, localDateTime, localDate, localTime)
 )
 
@@ -82,7 +82,7 @@ func (t *parser) startDateTime(p *parse.API) {
 	valueType := getDateTimeValueType(&tokens)
 	input, value, err := getDateTimeValue(&tokens)
 	if err == nil {
-		t.emitCommand(valueType, value)
+		t.addParsedItem(valueType, value)
 	} else {
 		p.Error("Cannot parse value 0%s: %s", input, err)
 	}
@@ -90,8 +90,8 @@ func (t *parser) startDateTime(p *parse.API) {
 
 // The first token is a token that wraps the complete date/time input.
 // Its type denotes the type of date/time value that it wraps.
-func getDateTimeValueType(tokens *[]*tokenize.Token) cmdType {
+func getDateTimeValueType(tokens *[]*tokenize.Token) itemType {
-	return (*tokens)[0].Type.(cmdType)
+	return (*tokens)[0].Type.(itemType)
 }
 
 // The rest of the tokens contain fragments that can be used with

value_datetime_test.go

@@ -1,4 +1,4 @@
-package parser
+package toml
 
 import (
 	"testing"
@@ -7,27 +7,27 @@ import (
 func TestDateTime(t *testing.T) {
 	for _, test := range []parseTest{
 		{``, []string{`Error: unexpected end of file (expected a date and/or time) at start of file`}},
-		{`1979-05-27`, []string{`date(1979-05-27 00:00:00 +0000 UTC)`}},
+		{`1979-05-27`, []string{`1979-05-27`}},
-		{`00:00:00`, []string{`time(0000-01-01 00:00:00 +0000 UTC)`}},
+		{`00:00:00`, []string{`00:00:00`}},
-		{`23:59:59`, []string{`time(0000-01-01 23:59:59 +0000 UTC)`}},
+		{`23:59:59`, []string{`23:59:59`}},
-		{`12:10:08.12121212121212`, []string{`time(0000-01-01 12:10:08.121212121 +0000 UTC)`}},
+		{`12:10:08.12121212121212`, []string{`12:10:08.121212121`}},
-		{`1979-05-28T01:01:01`, []string{`datetime(1979-05-28 01:01:01 +0000 UTC)`}},
+		{`1979-05-28T01:01:01`, []string{`1979-05-28 01:01:01`}},
-		{`1979-05-28 01:01:01`, []string{`datetime(1979-05-28 01:01:01 +0000 UTC)`}},
+		{`1979-05-28 01:01:01`, []string{`1979-05-28 01:01:01`}},
-		{`1979-05-27T07:32:00Z`, []string{`offset_datetime(1979-05-27 07:32:00 +0000 UTC)`}},
+		{`1979-05-27T07:32:00Z`, []string{`1979-05-27T07:32:00Z`}},
-		{`1979-05-27 07:33:00Z`, []string{`offset_datetime(1979-05-27 07:33:00 +0000 UTC)`}},
+		{`1979-05-27 07:33:00Z`, []string{`1979-05-27T07:33:00Z`}},
-		{`1979-05-27 07:34:00+07:00`, []string{`offset_datetime(1979-05-27 07:34:00 +0700 +0700)`}},
+		{`1979-05-27 07:34:00+07:00`, []string{`1979-05-27T07:34:00+07:00`}},
-		{`1979-05-27 07:34:00-07:00`, []string{`offset_datetime(1979-05-27 07:34:00 -0700 -0700)`}},
+		{`1979-05-27 07:34:00-07:00`, []string{`1979-05-27T07:34:00-07:00`}},
-		{`1985-03-31 23:59:59+00:00`, []string{`offset_datetime(1985-03-31 23:59:59 +0000 UTC)`}},
+		{`1985-03-31 23:59:59+00:00`, []string{`1985-03-31T23:59:59Z`}},
-		{`2000-09-10 00:00:00.000000000+00:00`, []string{`offset_datetime(2000-09-10 00:00:00 +0000 UTC)`}},
+		{`2000-09-10 00:00:00.000000000+00:00`, []string{`2000-09-10T00:00:00Z`}},
-		{`2003-11-01 01:02:03.999999999999+10:00`, []string{`offset_datetime(2003-11-01 01:02:03.999999999 +1000 +1000)`}},
+		{`2003-11-01 01:02:03.999999999999+10:00`, []string{`2003-11-01T01:02:03.999999999+10:00`}},
-		{`2007-12-25 04:00:04.1111-10:30`, []string{`offset_datetime(2007-12-25 04:00:04.1111 -1030 -1030)`}},
+		{`2007-12-25 04:00:04.1111-10:30`, []string{`2007-12-25T04:00:04.1111-10:30`}},
-		{`2021-02-01 10:10:10.101010203040Z`, []string{`offset_datetime(2021-02-01 10:10:10.101010203 +0000 UTC)`}},
+		{`2021-02-01 10:10:10.101010203040Z`, []string{`2021-02-01T10:10:10.101010203Z`}},
 		// TODO ugly column, should be at start or at the actual wrong part
 		{`2000-13-01`, []string{`Error: Cannot parse value 02000-13-01: parsing time "2000-13-01": month out of range at line 1, column 11`}},
 		{`2000-02-31`, []string{`Error: Cannot parse value 02000-02-31: parsing time "2000-02-31": day out of range at line 1, column 11`}},
 		{`25:01:01`, []string{`Error: Cannot parse value 025:01:01: parsing time "25:01:01": hour out of range at line 1, column 9`}},
 	} {
-		p := &parser{}
+		p := newParser()
 		testParseHandler(t, p, p.startDateTime, test)
 	}
 }

value_number.go

@@ -1,4 +1,4 @@
-package parser
+package toml
 
 import (
 	"math"
@@ -68,19 +68,19 @@ var (
 func (t *parser) startNumber(p *parse.API) {
 	switch {
 	case p.Accept(tok.Float64(nil, float)):
-		t.emitCommand(csetFloatVal, p.Result().Value(0).(float64))
+		t.addParsedItem(pFloat, p.Result().Value(0).(float64))
 	case p.Accept(nan):
-		t.emitCommand(csetFloatVal, math.NaN())
+		t.addParsedItem(pFloat, math.NaN())
 	case p.Accept(inf):
 		if p.Result().Rune(0) == '-' {
-			t.emitCommand(csetFloatVal, math.Inf(-1))
+			t.addParsedItem(pFloat, math.Inf(-1))
 		} else {
-			t.emitCommand(csetFloatVal, math.Inf(+1))
+			t.addParsedItem(pFloat, math.Inf(+1))
 		}
 	case p.Accept(a.Zero):
 		p.Handle(t.startIntegerStartingWithZero)
 	case p.Accept(tok.Int64(nil, integer)):
-		t.emitCommand(csetIntVal, p.Result().Value(0).(int64))
+		t.addParsedItem(pInteger, p.Result().Value(0).(int64))
 	default:
 		p.Expected("a number")
 	}
@@ -97,11 +97,11 @@ func (t *parser) startIntegerStartingWithZero(p *parse.API) {
 	case p.Accept(binary):
 		value, err = strconv.ParseInt(p.Result().Value(0).(string), 2, 64)
 	default:
-		t.emitCommand(csetIntVal, int64(0))
+		t.addParsedItem(pInteger, int64(0))
 		return
 	}
 	if err == nil {
-		t.emitCommand(csetIntVal, value)
+		t.addParsedItem(pInteger, value)
 	} else {
 		p.Error("Cannot parse value 0%s: %s", p.Result().String(), err)
 	}

value_number_test.go

@@ -1,4 +1,4 @@
-package parser
+package toml
 
 import (
 	"testing"
@@ -8,73 +8,73 @@ func TestInteger(t *testing.T) {
 	for _, test := range []parseTest{
 		{``, []string{`Error: unexpected end of file (expected a number) at start of file`}},
 		// Decimal
-		{`0`, []string{`integer(0)`}},
+		{`0`, []string{`0`}},
-		{`+0`, []string{`integer(0)`}},
+		{`+0`, []string{`0`}},
-		{`-0`, []string{`integer(0)`}},
+		{`-0`, []string{`0`}},
-		{`1`, []string{`integer(1)`}},
+		{`1`, []string{`1`}},
-		{`42`, []string{`integer(42)`}},
+		{`42`, []string{`42`}},
-		{`+99`, []string{`integer(99)`}},
+		{`+99`, []string{`99`}},
-		{`-17`, []string{`integer(-17)`}},
+		{`-17`, []string{`-17`}},
-		{`1234`, []string{`integer(1234)`}},
+		{`1234`, []string{`1234`}},
 		{`_`, []string{`Error: unexpected input (expected a number) at start of file`}},
-		{`1_`, []string{`integer(1)`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
+		{`1_`, []string{`1`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
-		{`1_000`, []string{`integer(1000)`}},
+		{`1_000`, []string{`1000`}},
-		{`5_349_221`, []string{`integer(5349221)`}},
+		{`5_349_221`, []string{`5349221`}},
-		{`1_2_3_4_5`, []string{`integer(12345)`}},
+		{`1_2_3_4_5`, []string{`12345`}},
-		{`9_223_372_036_854_775_807`, []string{`integer(9223372036854775807)`}},
+		{`9_223_372_036_854_775_807`, []string{`9223372036854775807`}},
 		{`9_223_372_036_854_775_808`, []string{
 			`Panic: Handler error: MakeInt64Token cannot handle input "9223372036854775808": ` +
 				`strconv.ParseInt: parsing "9223372036854775808": value out of range (only use a ` +
 				`type conversion token maker, when the input has been validated on beforehand)`}},
-		{`-9_223_372_036_854_775_808`, []string{`integer(-9223372036854775808)`}},
+		{`-9_223_372_036_854_775_808`, []string{`-9223372036854775808`}},
 		// TODO make the use of the same kind of handling for panics and for errors between parsekit and TOML.
 		{`-9_223_372_036_854_775_809`, []string{
 			`Panic: Handler error: MakeInt64Token cannot handle input "-9223372036854775809": ` +
 				`strconv.ParseInt: parsing "-9223372036854775809": value out of range (only use a ` +
 				`type conversion token maker, when the input has been validated on beforehand)`}},
 		// Hexadecimal
-		{`0x0`, []string{`integer(0)`}},
+		{`0x0`, []string{`0`}},
-		{`0x1`, []string{`integer(1)`}},
+		{`0x1`, []string{`1`}},
-		{`0x01`, []string{`integer(1)`}},
+		{`0x01`, []string{`1`}},
-		{`0x00fF`, []string{`integer(255)`}},
+		{`0x00fF`, []string{`255`}},
-		{`0xf_f`, []string{`integer(255)`}},
+		{`0xf_f`, []string{`255`}},
-		{`0x0_0_f_f`, []string{`integer(255)`}},
+		{`0x0_0_f_f`, []string{`255`}},
-		{`0xdead_beef`, []string{`integer(3735928559)`}},
+		{`0xdead_beef`, []string{`3735928559`}},
-		{`0xgood_beef`, []string{`integer(0)`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
+		{`0xgood_beef`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
-		{`0x7FFFFFFFFFFFFFFF`, []string{`integer(9223372036854775807)`}},
+		{`0x7FFFFFFFFFFFFFFF`, []string{`9223372036854775807`}},
 		{`0x8000000000000000`, []string{
 			`Error: Cannot parse value 0x8000000000000000: strconv.ParseInt: parsing "8000000000000000": ` +
 				`value out of range at line 1, column 19`}},
 		//Octal
-		{`0o0`, []string{`integer(0)`}},
+		{`0o0`, []string{`0`}},
-		{`0o1`, []string{`integer(1)`}},
+		{`0o1`, []string{`1`}},
-		{`0o01`, []string{`integer(1)`}},
+		{`0o01`, []string{`1`}},
-		{`0o10`, []string{`integer(8)`}},
+		{`0o10`, []string{`8`}},
-		{`0o1_6`, []string{`integer(14)`}},
+		{`0o1_6`, []string{`14`}},
-		{`0o0_0_1_1_1`, []string{`integer(73)`}},
+		{`0o0_0_1_1_1`, []string{`73`}},
-		{`0o9`, []string{`integer(0)`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
+		{`0o9`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
-		{`0o777777777777777777777`, []string{`integer(9223372036854775807)`}},
+		{`0o777777777777777777777`, []string{`9223372036854775807`}},
 		{`0o1000000000000000000000`, []string{
 			`Error: Cannot parse value 0o1000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000": ` +
 				`value out of range at line 1, column 25`}},
 		// Binary
-		{`0b0`, []string{`integer(0)`}},
+		{`0b0`, []string{`0`}},
-		{`0b1`, []string{`integer(1)`}},
+		{`0b1`, []string{`1`}},
-		{`0b01`, []string{`integer(1)`}},
+		{`0b01`, []string{`1`}},
-		{`0b10`, []string{`integer(2)`}},
+		{`0b10`, []string{`2`}},
-		{`0b0100`, []string{`integer(4)`}},
+		{`0b0100`, []string{`4`}},
-		{`0b00001000`, []string{`integer(8)`}},
+		{`0b00001000`, []string{`8`}},
-		{`0b0001_0000`, []string{`integer(16)`}},
+		{`0b0001_0000`, []string{`16`}},
-		{`0b9`, []string{`integer(0)`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
+		{`0b9`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
-		{`0b1_1_0_1_1`, []string{`integer(27)`}},
+		{`0b1_1_0_1_1`, []string{`27`}},
-		{`0b11111111_11111111`, []string{`integer(65535)`}},
+		{`0b11111111_11111111`, []string{`65535`}},
-		{`0b01111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111`, []string{`integer(9223372036854775807)`}},
+		{`0b01111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111`, []string{`9223372036854775807`}},
 		{`0b10000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000`, []string{
 			`Error: Cannot parse value 0b1000000000000000000000000000000000000000000000000000000000000000: ` +
 				`strconv.ParseInt: parsing "1000000000000000000000000000000000000000000000000000000000000000": ` +
 				`value out of range at line 1, column 74`}},
 	} {
-		p := &parser{}
+		p := newParser()
 		testParseHandler(t, p, p.startNumber, test)
 	}
 }
@@ -82,26 +82,26 @@ func TestInteger(t *testing.T) {
 func TestFloat(t *testing.T) {
 	for _, test := range []parseTest{
 		{``, []string{`Error: unexpected end of file (expected a number) at start of file`}},
-		{`0.0`, []string{`float(0)`}},
+		{`0.0`, []string{`0`}},
-		{`+0.0`, []string{`float(0)`}},
+		{`+0.0`, []string{`0`}},
-		{`-0.0`, []string{`float(-0)`}},
+		{`-0.0`, []string{`-0`}},
-		{`+1.0`, []string{`float(1)`}},
+		{`+1.0`, []string{`1`}},
-		{`3.1415`, []string{`float(3.1415)`}},
+		{`3.1415`, []string{`3.1415`}},
-		{`-0.01`, []string{`float(-0.01)`}},
+		{`-0.01`, []string{`-0.01`}},
-		{`5e+22`, []string{`float(5e+22)`}},
+		{`5e+22`, []string{`5e+22`}},
-		{`1E6`, []string{`float(1e+06)`}},
+		{`1E6`, []string{`1e+06`}},
-		{`-2E-2`, []string{`float(-0.02)`}},
+		{`-2E-2`, []string{`-0.02`}},
-		{`6.626e-34`, []string{`float(6.626e-34)`}},
+		{`6.626e-34`, []string{`6.626e-34`}},
-		{`224_617.445_991_228`, []string{`float(224617.445991228)`}},
+		{`224_617.445_991_228`, []string{`224617.445991228`}},
-		{`12_345.111_222e+1_2_3`, []string{`float(1.2345111222e+127)`}},
+		{`12_345.111_222e+1_2_3`, []string{`1.2345111222e+127`}},
-		{`+nan`, []string{`float(NaN)`}},
+		{`+nan`, []string{`NaN`}},
-		{`-nan`, []string{`float(NaN)`}},
+		{`-nan`, []string{`NaN`}},
-		{`nan`, []string{`float(NaN)`}},
+		{`nan`, []string{`NaN`}},
-		{`inf`, []string{`float(+Inf)`}},
+		{`inf`, []string{`+Inf`}},
-		{`+inf`, []string{`float(+Inf)`}},
+		{`+inf`, []string{`+Inf`}},
-		{`-inf`, []string{`float(-Inf)`}},
+		{`-inf`, []string{`-Inf`}},
 	} {
-		p := &parser{}
+		p := newParser()
 		testParseHandler(t, p, p.startNumber, test)
 	}
 }

value_string.go

@@ -1,4 +1,4 @@
-package parser
+package toml
 
 import (
 	"strings"
@@ -38,10 +38,7 @@ var (
 	// "line ending backslash". When the last non-whitespace character on a line is
 	// a \, it will be trimmed along with all whitespace (including newlines) up to
 	// the next non-whitespace character or closing delimiter.
-	lineEndingBackslash = a.Backslash.
+	lineEndingBackslash = a.Backslash.Then(dropBlanks).Then(newline).Then(dropWhitespace)
-				Then(c.ZeroOrMore(a.Blanks)).
-				Then(a.Newline).
-				Then(c.ZeroOrMore(a.Whitespace))
 )
 
 // There are four ways to express strings: basic, multi-line basic, literal and
@@ -49,9 +46,9 @@ var (
 func (t *parser) startString(p *parse.API) {
 	switch {
 	case p.Peek(doubleQuote3):
-		p.Handle(t.startMultiLineBasicString)
+		p.Handle(t.startMultiLineBasipString)
 	case p.Peek(a.DoubleQuote):
-		p.Handle(t.startBasicString)
+		p.Handle(t.startBasipString)
 	case p.Peek(singleQuote3):
 		p.Handle(t.startMultiLineLiteralString)
 	case p.Peek(a.SingleQuote):
@@ -72,13 +69,13 @@ func (t *parser) startString(p *parse.API) {
 // • No additional \escape sequences are allowed. What the spec say about this:
 // "All other escape sequences [..] are reserved and, if used, TOML should
 // produce an error.""
-func (t *parser) startBasicString(p *parse.API) {
+func (t *parser) startBasipString(p *parse.API) {
-	if str, ok := t.parseBasicString("basic string", p); ok {
+	if str, ok := t.parseBasipString("basic string", p); ok {
-		t.emitCommand(csetStrVal, str)
+		t.addParsedItem(pString, str)
 	}
 }
 
-func (t *parser) parseBasicString(name string, p *parse.API) (string, bool) {
+func (t *parser) parseBasipString(name string, p *parse.API) (string, bool) {
 	if !p.Accept(a.DoubleQuote) {
 		p.Expected(`opening quotation marks`)
 		return "", false
@@ -117,7 +114,7 @@ func (t *parser) parseBasicString(name string, p *parse.API) (string, bool) {
 // • Control characters other than tab are not permitted in a literal string.
 func (t *parser) startLiteralString(p *parse.API) {
 	if str, ok := t.parseLiteralString("literal string", p); ok {
-		t.emitCommand(csetStrVal, str)
+		t.addParsedItem(pString, str)
 	}
 }
 
@@ -171,15 +168,15 @@ func (t *parser) parseLiteralString(name string, p *parse.API) (string, bool) {
 // "line ending backslash". When the last non-whitespace character on a line is
 // a \, it will be trimmed along with all whitespace (including newlines) up to
 // the next non-whitespace character or closing delimiter.
-func (t *parser) startMultiLineBasicString(p *parse.API) {
+func (t *parser) startMultiLineBasipString(p *parse.API) {
-	if !p.Accept(doubleQuote3.Then(a.Newline.Optional())) {
+	if !p.Accept(doubleQuote3.Then(newline.Optional())) {
 		p.Expected("opening three quotation marks")
 		return
 	}
 	sb := &strings.Builder{}
 	for {
 		switch {
-		case p.Accept(a.Newline):
+		case p.Accept(newline):
 			sb.WriteString("\n")
 		case p.Peek(controlCharacter):
 			p.Error("invalid character in multi-line basic string: %q (must be escaped)", p.Result().Rune(0))
@@ -192,7 +189,7 @@ func (t *parser) startMultiLineBasicString(p *parse.API) {
 			p.Error("invalid escape sequence")
 			return
 		case p.Accept(m.Drop(doubleQuote3)):
-			t.emitCommand(csetStrVal, sb.String())
+			t.addParsedItem(pString, sb.String())
 			return
 		case p.Accept(a.ValidRune):
 			sb.WriteString(p.Result().String())
@@ -220,7 +217,7 @@ func (t *parser) startMultiLineBasicString(p *parse.API) {
 //
 // • Control characters other than tab and newline are not permitted in a multi-line literal string.
 func (t *parser) startMultiLineLiteralString(p *parse.API) {
-	if !p.Accept(singleQuote3.Then(a.Newline.Optional())) {
+	if !p.Accept(singleQuote3.Then(newline.Optional())) {
 		p.Expected("opening three single quotes")
 		return
 	}
@@ -228,11 +225,11 @@ func (t *parser) startMultiLineLiteralString(p *parse.API) {
 	for {
 		switch {
 		case p.Accept(m.Drop(singleQuote3)):
-			t.emitCommand(csetStrVal, sb.String())
+			t.addParsedItem(pString, sb.String())
 			return
 		case p.Accept(a.Tab):
 			sb.WriteString("\t")
-		case p.Accept(a.Newline):
+		case p.Accept(newline):
 			sb.WriteString("\n")
 		case p.Peek(controlCharacter):
 			p.Error("invalid character in literal string: %q (no control chars allowed, except for tab and newline)", p.Result().Rune(0))

value_string_test.go

@@ -1,4 +1,4 @@
-package parser
+package toml
 
 import (
 	"fmt"
@@ -9,53 +9,53 @@ func TestString(t *testing.T) {
 	for _, test := range []parseTest{
 		{``, []string{`Error: unexpected end of file (expected a string value) at start of file`}},
 		{`no start quote"`, []string{`Error: unexpected input (expected a string value) at start of file`}},
-		{`"basic s\tring"`, []string{`string("basic s\tring")`}},
+		{`"basic s\tring"`, []string{`"basic s\tring"`}},
-		{"\"\"\"\n  basic multi-line\n  string value\n\"\"\"", []string{`string("  basic multi-line\n  string value\n")`}},
+		{"\"\"\"\n  basic multi-line\n  string value\n\"\"\"", []string{`"  basic multi-line\n  string value\n"`}},
-		{`'literal s\tring'`, []string{`string("literal s\\tring")`}},
+		{`'literal s\tring'`, []string{`"literal s\\tring"`}},
-		{"'''\n  literal multi-line\n  string value\n'''", []string{`string("  literal multi-line\n  string value\n")`}},
+		{"'''\n  literal multi-line\n  string value\n'''", []string{`"  literal multi-line\n  string value\n"`}},
 	} {
-		p := &parser{}
+		p := newParser()
 		testParseHandler(t, p, p.startString, test)
 	}
 }
 
-func TestBasicString(t *testing.T) {
+func TestBasipString(t *testing.T) {
 	for _, test := range []parseTest{
 		{``, []string{`Error: unexpected end of file (expected opening quotation marks) at start of file`}},
 		{`no start quote"`, []string{`Error: unexpected input (expected opening quotation marks) at start of file`}},
 		{`"no end quote`, []string{`Error: unexpected end of file (expected closing quotation marks) at line 1, column 14`}},
-		{`""`, []string{`string("")`}},
+		{`""`, []string{`""`}},
-		{`"simple string"`, []string{`string("simple string")`}},
+		{`"simple string"`, []string{`"simple string"`}},
-		{`"with\tsome\r\nvalid escapes\b"`, []string{`string("with\tsome\r\nvalid escapes\b")`}},
+		{`"with\tsome\r\nvalid escapes\b"`, []string{`"with\tsome\r\nvalid escapes\b"`}},
 		{`"with an \invalid escape"`, []string{`Error: invalid escape sequence at line 1, column 10`}},
-		{`"A cool UTF8 ƃuıɹʇs"`, []string{`string("A cool UTF8 ƃuıɹʇs")`}},
+		{`"A cool UTF8 ƃuıɹʇs"`, []string{`"A cool UTF8 ƃuıɹʇs"`}},
-		{`"A string with UTF8 escape \u2318"`, []string{`string("A string with UTF8 escape ⌘")`}},
+		{`"A string with UTF8 escape \u2318"`, []string{`"A string with UTF8 escape ⌘"`}},
 		{"\"Invalid character for UTF \xcd\"", []string{`Error: invalid UTF8 rune at line 1, column 28`}},
 		{"\"Character that mus\t be escaped\"", []string{`Error: invalid character in basic string: '\t' (must be escaped) at line 1, column 20`}},
 		{"\"Character that must be escaped \u0000\"", []string{`Error: invalid character in basic string: '\x00' (must be escaped) at line 1, column 33`}},
 		{"\"Character that must be escaped \x7f\"", []string{`Error: invalid character in basic string: '\u007f' (must be escaped) at line 1, column 33`}},
 	} {
-		p := &parser{}
+		p := newParser()
-		testParseHandler(t, p, p.startBasicString, test)
+		testParseHandler(t, p, p.startBasipString, test)
 	}
 }
 
-func TestMultiLineBasicString(t *testing.T) {
+func TestMultiLineBasipString(t *testing.T) {
 	for _, test := range []parseTest{
 		{``, []string{`Error: unexpected end of file (expected opening three quotation marks) at start of file`}},
 		{`"""missing close quote""`, []string{`Error: unexpected end of file (expected closing three quotation marks) at line 1, column 25`}},
-		{`""""""`, []string{`string("")`}},
+		{`""""""`, []string{`""`}},
-		{"\"\"\"\n\"\"\"", []string{`string("")`}},
+		{"\"\"\"\n\"\"\"", []string{`""`}},
-		{"\"\"\"\r\n\r\n\"\"\"", []string{`string("\n")`}},
+		{"\"\"\"\r\n\r\n\"\"\"", []string{`"\n"`}},
-		{`"""\"\"\"\""""`, []string{`string("\"\"\"\"")`}},
+		{`"""\"\"\"\""""`, []string{`"\"\"\"\""`}},
-		{"\"\"\"\nThe quick brown \\\n\n\n  \t  fox jumps over \\\n\t the lazy dog.\\\n   \"\"\"", []string{`string("The quick brown fox jumps over the lazy dog.")`}},
+		{"\"\"\"\nThe quick brown \\\n\n\n  \t  fox jumps over \\\n\t the lazy dog.\\\n   \"\"\"", []string{`"The quick brown fox jumps over the lazy dog."`}},
 		{"\"\"\"No control chars \f allowed\"\"\"", []string{`Error: invalid character in multi-line basic string: '\f' (must be escaped) at line 1, column 21`}},
-		{"\"\"\"Escaping control chars\\nis valid\"\"\"", []string{`string("Escaping control chars\nis valid")`}},
+		{"\"\"\"Escaping control chars\\nis valid\"\"\"", []string{`"Escaping control chars\nis valid"`}},
 		{"\"\"\"Invalid escaping \\is not allowed\"\"\"", []string{`Error: invalid escape sequence at line 1, column 21`}},
 		{"\"\"\"Invalid rune \xcd\"\"\"", []string{`Error: invalid UTF8 rune at line 1, column 17`}},
 	} {
-		p := &parser{}
+		p := newParser()
-		testParseHandler(t, p, p.startMultiLineBasicString, test)
+		testParseHandler(t, p, p.startMultiLineBasipString, test)
 	}
 }
 
@@ -63,17 +63,17 @@ func TestLiteralString(t *testing.T) {
 	for _, test := range []parseTest{
 		{``, []string{`Error: unexpected end of file (expected opening single quote) at start of file`}},
 		{`'missing close quote`, []string{`Error: unexpected end of file (expected closing single quote) at line 1, column 21`}},
-		{`''`, []string{`string("")`}},
+		{`''`, []string{`""`}},
-		{`'simple'`, []string{`string("simple")`}},
+		{`'simple'`, []string{`"simple"`}},
-		{`'C:\Users\nodejs\templates'`, []string{`string("C:\\Users\\nodejs\\templates")`}},
+		{`'C:\Users\nodejs\templates'`, []string{`"C:\\Users\\nodejs\\templates"`}},
-		{`'\\ServerX\admin$\system32\'`, []string{`string("\\\\ServerX\\admin$\\system32\\")`}},
+		{`'\\ServerX\admin$\system32\'`, []string{`"\\\\ServerX\\admin$\\system32\\"`}},
-		{`'Tom "Dubs" Preston-Werner'`, []string{`string("Tom \"Dubs\" Preston-Werner")`}},
+		{`'Tom "Dubs" Preston-Werner'`, []string{`"Tom \"Dubs\" Preston-Werner"`}},
-		{`'<\i\c*\s*>'`, []string{`string("<\\i\\c*\\s*>")`}},
+		{`'<\i\c*\s*>'`, []string{`"<\\i\\c*\\s*>"`}},
 		{"'No cont\rol chars allowed'", []string{`Error: invalid character in literal string: '\r' (no control chars allowed, except for tab) at line 1, column 9`}},
-		{"'Except\tfor\ttabs'", []string{`string("Except\tfor\ttabs")`}},
+		{"'Except\tfor\ttabs'", []string{`"Except\tfor\ttabs"`}},
 		{"'Invalid rune \xcd'", []string{`Error: invalid UTF8 rune at line 1, column 15`}},
 	} {
-		p := &parser{}
+		p := newParser()
 		testParseHandler(t, p, p.startLiteralString, test)
 	}
 }
@@ -82,23 +82,23 @@ func TestMultiLineLiteralString(t *testing.T) {
 	for _, test := range []parseTest{
 		{``, []string{`Error: unexpected end of file (expected opening three single quotes) at start of file`}},
 		{`'''missing close quote''`, []string{`Error: unexpected end of file (expected closing three single quotes) at line 1, column 25`}},
-		{`''''''`, []string{`string("")`}},
+		{`''''''`, []string{`""`}},
-		{"'''\n'''", []string{`string("")`}},
+		{"'''\n'''", []string{`""`}},
-		{`'''I [dw]on't need \d{2} apples'''`, []string{`string("I [dw]on't need \\d{2} apples")`}},
+		{`'''I [dw]on't need \d{2} apples'''`, []string{`"I [dw]on't need \\d{2} apples"`}},
-		{"'''\nThere can\nbe newlines\r\nand \ttabs!\r\n'''", []string{`string("There can\nbe newlines\nand \ttabs!\n")`}},
+		{"'''\nThere can\nbe newlines\r\nand \ttabs!\r\n'''", []string{`"There can\nbe newlines\nand \ttabs!\n"`}},
 		{"'''No other \f control characters'''", []string{`Error: invalid character in literal string: '\f' (no control chars allowed, except for tab and newline) at line 1, column 13`}},
 		{"'''No invalid runes allowed \xcd'''", []string{"Error: invalid UTF8 rune at line 1, column 29"}},
 	} {
-		p := &parser{}
+		p := newParser()
 		testParseHandler(t, p, p.startMultiLineLiteralString, test)
 	}
 }
 
-func TestBasicStringWithUnescapedControlCharacters(t *testing.T) {
+func TestBasipStringWithUnescapedControlCharacters(t *testing.T) {
 	// A quick check for almost all characters that must be escaped.
 	// The missing one (\x7f) is covered in the previous test.
 	for i := 0x00; i <= 0x1F; i++ {
-		p := &parser{}
+		p := newParser()
 		input := fmt.Sprintf(`"%c"`, rune(i))
 		expected := fmt.Sprintf(`Error: invalid character in basic string: %q (must be escaped) at line 1, column 2`, rune(i))
 		testParseHandler(t, p, p.startString, parseTest{input, []string{expected}})

value_table.go

@@ -0,0 +1,111 @@
+package toml
+
+import (
+	"git.makaay.nl/mauricem/go-parsekit/parse"
+)
+
+var (
+	// Tables (also known as hash tables or dictionaries) are collections of
+	// key/value pairs. They appear in square brackets on a line by themselves.
+	// You can tell them apart from arrays because arrays are only ever values.
+	//
+	// Under that, and until the next table or EOF are the key/values of that
+	// table. Key/value pairs within tables are not guaranteed to be in any
+	// specific order.
+	//
+	//     [table-1]
+	//     key1 = "some string"
+	//     key2 = 123
+	//
+	//     [table-2]
+	//     key1 = "another string"
+	//     key2 = 456
+	//
+	// Naming rules for tables are the same as for keys.
+	//
+	//     [dog."tater.man"]
+	//     type.name = "pug"
+	//
+	// Whitespace around the key is ignored, however, best practice is to not
+	// use any extraneous whitespace.
+	//
+	//     [a.b.c]            # this is best practice
+	//     [ d.e.f ]          # same as [d.e.f]
+	//     [ g .  h  . i ]    # same as [g.h.i]
+	//     [ j . "ʞ" . 'l' ]  # same as [j."ʞ".'l']
+	//
+	// You don't need to specify all the super-tables if you don't want to.
+	// TOML knows how to do it for you.
+	//
+	//     # [x] you
+	//     # [x.y] don't
+	//     # [x.y.z] need these
+	//     [x.y.z.w] # for this to work
+	//
+	// Empty tables are allowed and simply have no key/value pairs within them.
+	//
+	tableOpen  = c.Seq(dropBlanks, a.SquareOpen, dropBlanks)
+	tableClose = c.Seq(dropBlanks, a.SquareClose, dropBlanks, a.EndOfLine.Or(comment))
+
+	// Arrays of tables can be expressed by using a table name in double brackets.
+	// Each table with the same double bracketed name will be an element in the
+	// array. The tables are inserted in the order encountered. A double bracketed
+	// table without any key/value pairs will be considered an empty table.
+	//
+	//     [[products]]
+	//     name = "Hammer"
+	//     sku = 738594937
+	//
+	//     [[products]]
+	//
+	//     [[products]]
+	//     name = "Nail"
+	//     sku = 284758393
+	//     color = "gray"
+	//
+	// You can create nested arrays of tables as well. Just use the same double
+	// bracket syntax on sub-tables. Each double-bracketed sub-table will belong
+	// to the most recently defined table element above it.
+	//
+	// 	    [[fruit]]
+	// 	    name = "apple"
+	//
+	//        [fruit.physical]
+	//        color = "red"
+	// 	      shape = "round"
+	//
+	// 	      [[fruit.variety]]
+	// 	      name = "red delicious"
+	//
+	// 	      [[fruit.variety]]
+	// 	      name = "granny smith"
+	//
+	//      [[fruit]]
+	// 	    name = "banana"
+	//
+	// 	      [[fruit.variety]]
+	// 	      name = "plantain"
+	tableArrayOpen  = c.Seq(dropBlanks, a.SquareOpen, a.SquareOpen, dropBlanks)
+	tableArrayClose = c.Seq(dropBlanks, a.SquareClose, a.SquareClose, a.EndOfLine.Or(comment))
+)
+
+func (t *parser) startTable(p *parse.API) {
+	switch {
+	case p.Accept(tableOpen):
+		p.Handle(t.startPlainTable)
+	default:
+		p.Expected("a table")
+	}
+}
+
+func (t *parser) startPlainTable(p *parse.API) {
+	if !p.Handle(t.startKey) {
+		return
+	}
+	if !p.Accept(tableClose) {
+		p.Expected("closing ']' for table name")
+	}
+	key := t.Items[0]
+	t.Items = t.Items[1:]
+	t.newTable(key)
+}

value_table_test.go

@@ -0,0 +1,23 @@
+package toml
+
+import (
+	"testing"
+)
+
+func TestTable(t *testing.T) {
+	for _, test := range []parseTest{
+		{"", []string{`Error: unexpected end of file (expected a table) at start of file`}},
+		{"[", []string{`Error: unexpected end of file (expected a key name) at line 1, column 2`}},
+		{" \t [", []string{`Error: unexpected end of file (expected a key name) at line 1, column 5`}},
+		{" \t [key", []string{`Error: unexpected end of file (expected closing ']' for table name) at line 1, column 8`}},
+		{" \t [key.", []string{`key("key")`, `Error: unexpected end of file (expected a key name) at line 1, column 9`}},
+		{" \t [key.'sub key'", []string{`Error: unexpected end of file (expected closing ']' for table name) at line 1, column 18`}},
+		{" \t [key.'sub key'  \t]", []string{}},
+		{" \t [key.'sub key'  \t]  \t  ", []string{}},
+		{" \t [key.'sub key'  \t]  \t  \n", []string{}},
+		{" \t [key.'sub key'  \t]  \t  # with comment\n", []string{}},
+	} {
+		p := newParser()
+		testParseHandler(t, p, p.startTable, test)
+	}
+}

value_test.go

@@ -1,4 +1,4 @@
-package parser
+package toml
 
 import (
 	"testing"
@@ -7,39 +7,40 @@ import (
 func TestValue(t *testing.T) {
 	for _, test := range []parseTest{
 		{``, []string{`Error: unexpected end of file (expected a value) at start of file`}},
-		{`"basic s\tring value"`, []string{`string("basic s\tring value")`}},
+		{`"basic s\tring value"`, []string{`"basic s\tring value"`}},
-		{`'literal s\tring value'`, []string{`string("literal s\\tring value")`}},
+		{`'literal s\tring value'`, []string{`"literal s\\tring value"`}},
-		{"\"\"\"basic multi-line\nstring value\"\"\"", []string{`string("basic multi-line\nstring value")`}},
+		{"\"\"\"basic multi-line\nstring value\"\"\"", []string{`"basic multi-line\nstring value"`}},
-		{"'''literal multi-line\nstring value'''", []string{`string("literal multi-line\nstring value")`}},
+		{"'''literal multi-line\nstring value'''", []string{`"literal multi-line\nstring value"`}},
-		{"true", []string{`boolean(true)`}},
+		{"true", []string{`true`}},
-		{"false", []string{`boolean(false)`}},
+		{"false", []string{`false`}},
-		{"0", []string{`integer(0)`}},
+		{"0", []string{`0`}},
-		{"+0", []string{`integer(0)`}},
+		{"+0", []string{`0`}},
-		{"-0", []string{`integer(0)`}},
+		{"-0", []string{`0`}},
-		{"0.0", []string{`float(0)`}},
+		{"0.0", []string{`0`}},
-		{"+0.0", []string{`float(0)`}},
+		{"+0.0", []string{`0`}},
-		{"-0.0", []string{`float(-0)`}},
+		{"-0.0", []string{`-0`}},
-		{"1234", []string{`integer(1234)`}},
+		{"1234", []string{`1234`}},
-		{"-1234", []string{`integer(-1234)`}},
+		{"-1234", []string{`-1234`}},
-		{"+9_8_7.6_5_4e-321", []string{`float(9.8765e-319)`}},
+		{"+9_8_7.6_5_4e-321", []string{`9.8765e-319`}},
-		{"-1_234.5678e-33", []string{`float(-1.2345678e-30)`}},
+		{"-1_234.5678e-33", []string{`-1.2345678e-30`}},
-		{"inf", []string{`float(+Inf)`}},
+		{"inf", []string{`+Inf`}},
-		{"+inf", []string{`float(+Inf)`}},
+		{"+inf", []string{`+Inf`}},
-		{"-inf", []string{`float(-Inf)`}},
+		{"-inf", []string{`-Inf`}},
-		{"nan", []string{`float(NaN)`}},
+		{"nan", []string{`NaN`}},
-		{"+nan", []string{`float(NaN)`}},
+		{"+nan", []string{`NaN`}},
-		{"-nan", []string{`float(NaN)`}},
+		{"-nan", []string{`NaN`}},
-		{"2019-06-19", []string{`date(2019-06-19 00:00:00 +0000 UTC)`}},
+		{"2019-06-19", []string{`2019-06-19`}},
-		{"08:38:54", []string{`time(0000-01-01 08:38:54 +0000 UTC)`}},
+		{"08:38:54", []string{`08:38:54`}},
-		{"2019-06-19 08:38:54", []string{`datetime(2019-06-19 08:38:54 +0000 UTC)`}},
+		{"08:38:54.8765487654876", []string{`08:38:54.876548765`}},
-		{"2019-06-19T08:38:54", []string{`datetime(2019-06-19 08:38:54 +0000 UTC)`}},
+		{"2019-06-19 08:38:54", []string{`2019-06-19 08:38:54`}},
-		{"2019-06-19 08:38:54", []string{`datetime(2019-06-19 08:38:54 +0000 UTC)`}},
+		{"2019-06-19T08:38:54", []string{`2019-06-19 08:38:54`}},
-		{"2019-06-19T08:38:54.88888", []string{`datetime(2019-06-19 08:38:54.88888 +0000 UTC)`}},
+		{"2019-06-19T08:38:54.88888", []string{`2019-06-19 08:38:54.88888`}},
-		{"1979-05-27T07:32:00Z", []string{`offset_datetime(1979-05-27 07:32:00 +0000 UTC)`}},
+		{"1979-05-27T07:32:00Z", []string{`1979-05-27T07:32:00Z`}},
-		{"1979-05-27T00:32:00-07:00", []string{`offset_datetime(1979-05-27 00:32:00 -0700 -0700)`}},
+		{"1979-05-27T00:32:00-07:00", []string{`1979-05-27T00:32:00-07:00`}},
-		{"1979-05-27T00:32:00.999999-07:00", []string{`offset_datetime(1979-05-27 00:32:00.999999 -0700 -0700)`}},
+		{"1979-05-27T00:32:00.999999-07:00", []string{`1979-05-27T00:32:00.999999-07:00`}},
+		{"[1,2,3]", []string{`[1, 2, 3]`}},
 	} {
-		p := &parser{}
+		p := newParser()
 		testParseHandler(t, p, p.startValue, test)
 	}
 }