OpenSourceProjects
/
go-toml
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Backup work.

This commit is contained in:
Maurice Makaay 2019-06-23 12:05:52 +00:00
parent 726b5a377b
commit 8838dc9c44
23 changed files with 876 additions and 386 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

199
ast.go Normal file
View File

@ -0,0 +1,199 @@
package toml
import (
"fmt"
"strings"
"time"
)
// item represents a TOML item.
type item struct {
Type itemType
Data []interface{}
}
// table represents a TOML table.
type table map[string]item
// itemType identifies the semantic role of a TOML item.
type itemType string
const (
// TODO Would be nice to not need these in the end.
pComment itemType = "comment"
pKey itemType = "key"
pAssign itemType = "assign"
// TODO and just use these data types.
pString itemType = "string" // "various", 'types', """of""", '''strings'''
pInteger itemType = "integer" // 12345, 0xffee12a0, 0o0755, 0b101101011
pFloat itemType = "float" // 10.1234, 143E-12, 43.28377e+4, +inf, -inf, nan
pBoolean itemType = "boolean" // true or false
pOffsetDateTime itemType = "offset datetime" // 2019-06-18 10:32:15.173645362+0200
pLocalDateTime itemType = "datetime" // 2018-12-25 12:12:18.876772533
pLocalDate itemType = "date" // 2017-05-17
pLocalTime itemType = "time" // 23:01:22
pArray itemType = "array" // defined using an [[array.of.tables]]
pStaticArray itemType = "static array" // defined using ["an", "inline", "array"]
pTable itemType = "table" // defined using { "inline" = "table" } or [standard.table]
)
// newItem instantiates a new item struct.
func newItem(itemType itemType, data ...interface{}) item {
return item{Type: itemType, Data: data}
}
func (t table) String() string {
return newItem(pTable, t).String()
}
func (parseItem item) String() string {
switch parseItem.Type {
case pString:
return fmt.Sprintf("%q", parseItem.Data[0])
case pInteger:
return fmt.Sprintf("%v", parseItem.Data[0])
case pFloat:
return fmt.Sprintf("%v", parseItem.Data[0])
case pBoolean:
return fmt.Sprintf("%v", parseItem.Data[0])
case pOffsetDateTime:
return parseItem.Data[0].(time.Time).Format(time.RFC3339Nano)
case pLocalDateTime:
return parseItem.Data[0].(time.Time).Format("2006-01-02 15:04:05.999999999")
case pLocalDate:
return parseItem.Data[0].(time.Time).Format("2006-01-02")
case pLocalTime:
return parseItem.Data[0].(time.Time).Format("15:04:05.999999999")
case pArray:
fallthrough
case pStaticArray:
items := make([]string, len(parseItem.Data[0].([]item)))
for i, d := range parseItem.Data[0].([]item) {
items[i] = d.String()
}
return fmt.Sprintf("[%s]", strings.Join(items, ", "))
case pTable:
items := make([]string, len(parseItem.Data))
pairs := parseItem.Data[0].(table)
i := 0
for k, v := range pairs {
items[i] = fmt.Sprintf("%q: %s", k, v.String())
i++
}
return fmt.Sprintf("{%s}", strings.Join(items, ", "))
case pComment:
return fmt.Sprintf("comment(%q)", parseItem.Data[0])
case pKey:
items := make([]string, len(parseItem.Data))
for i, e := range parseItem.Data {
items[i] = fmt.Sprintf("%q", e)
}
return fmt.Sprintf("key(%s)", strings.Join(items, ", "))
case pAssign:
return "="
default:
panic(fmt.Sprintf("Missing String() formatting for item type '%s'", parseItem.Type))
}
}
// parser holds the state for the TOML parser. All parsing functions are
// methods of this struct.
type parser struct {
Items []item // a buffer for holding parsed items
Root table // the root-level TOML table (each TOML doc is implicitly a table)
Current table // the currently active TOML table
}
func newParser() *parser {
p := &parser{Root: make(table)}
p.Current = p.Root
return p
}
func (t *parser) addParsedItem(itemType itemType, data ...interface{}) {
t.Items = append(t.Items, newItem(itemType, data...))
}
func (t *parser) setValue(key item, value item) error {
// When the key has multiple elements, then first make sure the table structure
// for storing the value exists.
var valueKey string
node := t.Current
l := len(key.Data)
if l > 1 {
pathKeys := key.Data[0 : l-1]
valueKey = key.Data[l-1].(string)
for i, name := range pathKeys {
name := name.(string)
if subItem, ok := node[name]; ok {
// An item was found at the current key. It is expected to be a table.
if subItem.Type != pTable {
path := formatKeyPath(key, i)
return fmt.Errorf("invalid key used: %s item already exists at key %s", subItem.Type, path)
}
node = subItem.Data[0].(table)
} else {
// No item was found at the current key. Create a new subtable.
subTable := make(table)
node[name] = newItem(pTable, subTable)
node = subTable
}
}
} else {
valueKey = key.Data[0].(string)
node = t.Current
}
if existing, ok := node[valueKey]; ok {
path := "moetnog"
return fmt.Errorf("Cannot store value: %s item already exists at key %s", existing.Type, path)
}
node[valueKey] = value
return nil
}
func (t *parser) newTable(key item) (table, error) {
node := t.Root
// Go over all requested levels of the key. For all levels, except the last
// one, it is okay if a table already exists. For at least the last level,
// no table or value must exist, because that would mean we are overwriting
// an existing key/value pair, which is not allowed.
for i, e := range key.Data {
name := e.(string)
if subItem, ok := node[name]; ok {
// You cannot overwrite an already defined key, regardless its value.
if subItem.Type != pTable {
path := formatKeyPath(key, i)
return nil, fmt.Errorf("Cannot create table: %s item already exists at key %s", subItem.Type, path)
}
// Like keys, you cannot define any table more than once. Doing so is invalid.
isLast := i == len(key.Data)-1
if isLast {
path := formatKeyPath(key, i)
return nil, fmt.Errorf("Cannot create table: table for key %s already exists", path)
}
node = subItem.Data[0].(table)
} else {
// Create the subtable.
subTable := make(table)
node[name] = newItem(pTable, subTable)
node = subTable
}
}
// From here on, key/value pairs are added to the newly defined table.
t.Current = node
return node, nil
}
func formatKeyPath(key item, end int) string {
var sb strings.Builder
for i := 0; i <= end; i++ {
if i > 0 {
sb.WriteRune('.')
}
sb.WriteString(fmt.Sprintf("%q", key.Data[i].(string)))
}
return sb.String()
}

72
ast_test.go Normal file
View File

@ -0,0 +1,72 @@
package toml
import (
"fmt"
"testing"
)
func TestAST_ConstructStructure(t *testing.T) {
p := newParser()
p.Root["ding"] = newItem(pInteger, 10)
p.Root["dong"] = newItem(pString, "not a song")
subTable1, _ := p.newTable(newItem(pKey, "key1", "key2 a"))
subTable1["dooh"] = newItem(pBoolean, true)
subTable1["dah"] = newItem(pBoolean, false)
subTable2, _ := p.newTable(newItem(pKey, "key1", "key2 b"))
subTable2["dieh"] = newItem(pFloat, 1.111)
subTable2["duhh"] = newItem(pFloat, 1.18e-12)
}
func TestAST_StoreValueInRootTable(t *testing.T) {
testError(t, func() error {
p := newParser()
p.setValue(newItem(pKey, "key1"), newItem(pString, "value1"))
return p.setValue(newItem(pKey, "key2"), newItem(pString, "value2"))
}, "")
}
func TestAST_StoreValueWithMultipartKey_CreatesTableHierarchy(t *testing.T) {
testError(t, func() error {
p := newParser()
return p.setValue(newItem(pKey, "key1", "key2", "key3"), newItem(pString, "value"))
}, "")
// TODO an actual test assertion
}
func TestAST_StoreValueWithMultipartKey_UnderSubtable_CreatesTableHierarchy(t *testing.T) {
testError(t, func() error {
p := newParser()
p.newTable(newItem(pKey, "subkey1", "subkey2"))
err := p.setValue(newItem(pKey, "key1", "key2", "key3"), newItem(pString, "value"))
fmt.Printf("%s", p.Root)
return err
}, "")
t.Fail()
// TODO an actual test assertion
}
func TestAST_StoreDuplicateKeyInRootTable_ReturnsError(t *testing.T) {
testError(t, func() error {
p := newParser()
p.setValue(newItem(pKey, "key"), newItem(pString, "value"))
return p.setValue(newItem(pKey, "key"), newItem(pInteger, 321))
}, `Cannot store value: string item already exists at key "key"`)
}
func TestAST_GivenExistingTableAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testError(t, func() error {
p := newParser()
p.newTable(newItem(pKey, "key1", "key2"))
_, err := p.newTable(newItem(pKey, "key1", "key2"))
return err
}, `Cannot create table: table for key "key1"."key2" already exists`)
}
func TestAST_GivenExistingItemAtKey_CreatingTableAtSameKey_ReturnsError(t *testing.T) {
testError(t, func() error {
p := newParser()
p.Root["key"] = newItem(pString, "value")
_, err := p.newTable(newItem(pKey, "key"))
return err
}, `Cannot create table: string item already exists at key "key"`)
}

4
comment.go
View File

@ -1,4 +1,4 @@
package parser
package toml
import (
"git.makaay.nl/mauricem/go-parsekit/parse"
@ -10,7 +10,7 @@ var comment = c.Seq(a.Hash, c.ZeroOrMore(c.Not(a.EndOfLine)), m.Drop(a.EndOfLine
func (t *parser) startComment(p *parse.API) {
if p.Accept(comment) {
t.emitCommand(cComment, p.Result().String())
t.addParsedItem(pComment, p.Result().String())
} else {
p.Expected("comment")
}

4
comment_test.go
View File

@ -1,4 +1,4 @@
package parser
package toml
import (
"testing"
@ -18,7 +18,7 @@ func TestComment2(t *testing.T) {
`comment("# with data and newline")`,
`Error: unexpected input (expected end of file) at line 2, column 1`}},
} {
p := &parser{}
p := newParser()
testParseHandler(t, p, p.startComment, test)
}
}

25
helpers_test.go
View File

@ -1,4 +1,4 @@
package parser
package toml
import (
"fmt"
@ -24,8 +24,8 @@ func testParseHandler(t *testing.T, p *parser, handler parse.Handler, test parse
defer func() {
recovered := recover()
results := []string{}
for _, cmd := range p.commands {
results = append(results, cmd.String())
for _, item := range p.Items {
results = append(results, item.String())
}
if err != nil {
results = append(results, fmt.Sprintf("Error: %s", err))
@ -36,12 +36,12 @@ func testParseHandler(t *testing.T, p *parser, handler parse.Handler, test parse
for i, e := range test.expected {
if i > len(results)-1 {
t.Errorf("No result at index %d, expected: %s", i, e)
t.Errorf("No result at index %d for input %q, expected: %s", i, test.input, e)
continue
}
r := results[i]
if e != r {
t.Errorf("Unexpected result at index %d:\nexpected: %s\nactual: %s\n", i, e, r)
t.Errorf("Unexpected result at index %d for input %q:\nexpected: %s\nactual: %s\n", i, test.input, e, r)
}
}
if len(results) > len(test.expected) {
@ -53,3 +53,18 @@ func testParseHandler(t *testing.T, p *parser, handler parse.Handler, test parse
}()
err = parse.New(handler)(test.input)
}
func testError(t *testing.T, code func() error, expected string) {
err := code()
if expected == "" {
if err != nil {
t.Fatalf("Unexpected error: %s", err)
}
} else {
if err == nil {
t.Fatalf("An error was expected, but no error was returned")
} else if err.Error() != expected {
t.Fatalf("Unexpected error:\nexpected: %s\nactual: %s\n", expected, err.Error())
}
}
}

30
keyvaluepair.go
View File

@ -1,4 +1,4 @@
package parser
package toml
import (
"git.makaay.nl/mauricem/go-parsekit/parse"
@ -7,9 +7,6 @@ import (
// The primary building block of a TOML document is the key/value pair.
var (
dropWhitespace = m.Drop(a.Whitespace.Optional())
dropBlanks = m.Drop(a.Blanks.Optional())
// Keys are on the left of the equals sign and values are on the right.
// Blank is ignored around key names and values. The key, equals
// sign, and value must be on the same line (though some values can be
@ -73,13 +70,13 @@ func (t *parser) startKey(p *parse.API) {
case p.Peek(a.SingleQuote):
key, ok = t.parseLiteralString("key", p)
case p.Peek(a.DoubleQuote):
key, ok = t.parseBasicString("key", p)
key, ok = t.parseBasipString("key", p)
default:
p.Expected("a key name")
return
}
if ok {
t.emitCommand(cKey, key)
t.addParsedItem(pKey, key)
p.Handle(t.endOfKeyOrDot)
}
}
@ -90,14 +87,31 @@ func (t *parser) startKey(p *parse.API) {
// practice is to not use any extraneous whitespace.
func (t *parser) endOfKeyOrDot(p *parse.API) {
if p.Accept(keySeparatorDot) {
t.emitCommand(cKeyDot)
p.Handle(t.startKey)
return
}
// TODO not sure if we really need this index.
// Can't we alway simply use the full item list, given that we'll feed all
// results to the parser's table state?
// Do we even need to emit a key here? Shouldn't we just create the
// table structure in the parser, ready for followup calls to fill the data?
keyStart := len(t.Items) - 1
for keyStart > 0 && t.Items[keyStart-1].Type == pKey {
keyStart--
}
keyLen := len(t.Items) - keyStart
key := make([]interface{}, keyLen)
for i := 0; i < keyLen; i++ {
key[i] = t.Items[keyStart+i].Data[0].(string)
}
t.Items = t.Items[0:keyStart]
t.addParsedItem(pKey, key...)
}
func (t *parser) startAssignment(p *parse.API) {
if p.Accept(keyAssignment) {
t.emitCommand(cAssign)
t.addParsedItem(pAssign)
} else {
p.Expected("a value assignment")
}

173
keyvaluepair_test.go
View File

@ -1,4 +1,4 @@
package parser
package toml
import "testing"
@ -11,10 +11,10 @@ func TestKey(t *testing.T) {
{"mix-12_34", []string{`key("mix-12_34")`}},
{"-hey_good_Lookin123-", []string{`key("-hey_good_Lookin123-")`}},
{"wrong!", []string{`key("wrong")`, `Error: unexpected input (expected end of file) at line 1, column 6`}},
{"key1.", []string{`key("key1")`, `keydot`, `Error: unexpected end of file (expected a key name) at line 1, column 6`}},
{"key1.key2", []string{`key("key1")`, `keydot`, `key("key2")`}},
{"key . with . spaces", []string{`key("key")`, `keydot`, `key("with")`, `keydot`, `key("spaces")`}},
{"key \t . \twithtabs\t . \tandspaces", []string{`key("key")`, `keydot`, `key("withtabs")`, `keydot`, `key("andspaces")`}},
{"key1.", []string{`key("key1")`, `Error: unexpected end of file (expected a key name) at line 1, column 6`}},
{"key1.key2", []string{`key("key1", "key2")`}},
{"key . with . spaces", []string{`key("key", "with", "spaces")`}},
{"key \t . \twithtabs\t . \tandspaces", []string{`key("key", "withtabs", "andspaces")`}},
// Single quoted key tests
{"''", []string{`key("")`}},
{"'single quoted'", []string{`key("single quoted")`}},
@ -26,9 +26,9 @@ func TestKey(t *testing.T) {
{`"escapes are in\terpreted"`, []string{`key("escapes are in\terpreted")`}},
{`"using 'inner' \"quotes\""`, []string{`key("using 'inner' \"quotes\"")`}},
// Mixed key types
{`this.'i\s'."madness\t".''`, []string{`key("this")`, `keydot`, `key("i\\s")`, `keydot`, `key("madness\t")`, `keydot`, `key("")`}},
{`this.'i\s'."madness\t".''`, []string{`key("this", "i\\s", "madness\t", "")`}},
} {
p := &parser{}
p := newParser()
testParseHandler(t, p, p.startKey, test)
}
}
@ -36,11 +36,11 @@ func TestKey(t *testing.T) {
func TestAssignment(t *testing.T) {
for _, test := range []parseTest{
{"", []string{`Error: unexpected end of file (expected a value assignment) at start of file`}},
{"=", []string{`assign`}},
{" \t = \t ", []string{`assign`}},
{"=", []string{`=`}},
{" \t = \t ", []string{`=`}},
{" \n = \n ", []string{`Error: unexpected input (expected a value assignment) at start of file`}},
} {
p := &parser{}
p := newParser()
testParseHandler(t, p, p.startAssignment, test)
}
}
@ -51,94 +51,103 @@ func TestKeyValuePair(t *testing.T) {
{" ", []string{}},
{" \t ", []string{}},
{" key ", []string{`key("key")`, `Error: unexpected input (expected a value assignment) at line 1, column 5`}},
{" key \t=", []string{`key("key")`, `assign`, `Error: unexpected end of file (expected a value) at line 1, column 8`}},
{"key = # INVALID", []string{`key("key")`, `assign`, `Error: unexpected input (expected a value) at line 1, column 7`}},
{" key \t =\t \"The Value\" \r\n", []string{`key("key")`, `assign`, `string("The Value")`}},
{`3.14159 = "pi"`, []string{`key("3")`, `keydot`, `key("14159")`, `assign`, `string("pi")`}},
{`"ʎǝʞ" = "value"`, []string{`key("ʎǝʞ")`, `assign`, `string("value")`}},
{`key = "value" # This is a comment at the end of a line`, []string{`key("key")`, `assign`, `string("value")`, `comment("# This is a comment at the end of a line")`}},
{`another = "# This is not a comment"`, []string{`key("another")`, `assign`, `string("# This is not a comment")`}},
{"key1=\"value1\"key2=\"value2\"\r\nkey3=\"value3\"", []string{
`key("key1")`, `assign`, `string("value1")`,
`key("key2")`, `assign`, `string("value2")`,
`key("key3")`, `assign`, `string("value3")`}},
{" key \t=", []string{`key("key")`, `=`, `Error: unexpected end of file (expected a value) at line 1, column 8`}},
{"key = # INVALID", []string{`key("key")`, `=`, `Error: unexpected input (expected a value) at line 1, column 7`}},
{" key \t =\t \"The Value\" \r\n", []string{`key("key")`, `=`, `"The Value"`}},
{`3.14159 = "pi"`, []string{`key("3", "14159")`, `=`, `"pi"`}},
{`"ʎǝʞ" = "value"`, []string{`key("ʎǝʞ")`, `=`, `"value"`}},
{`key = "value" # This is a comment at the end of a line`, []string{`key("key")`, `=`, `"value"`, `comment("# This is a comment at the end of a line")`}},
{`another = "# This is not a comment"`, []string{`key("another")`, `=`, `"# This is not a comment"`}},
{"key1=\"value1\"key2=\"value2\"\r\nkey3a.key3b=\"value3\"", []string{
`key("key1")`, `=`, `"value1"`,
`key("key2")`, `=`, `"value2"`,
`key("key3a", "key3b")`, `=`, `"value3"`}},
{"with=\"comments\"# boring \nanother.cool =\"one\" \t # to the end\r\n", []string{
`key("with")`, `assign`, `string("comments")`, `comment("# boring ")`,
`key("another")`, `keydot`, `key("cool")`, `assign`, `string("one")`, `comment("# to the end")`}},
`key("with")`, `=`, `"comments"`, `comment("# boring ")`,
`key("another", "cool")`, `=`, `"one"`, `comment("# to the end")`}},
} {
p := &parser{}
p := newParser()
testParseHandler(t, p, p.startKeyValuePair, test)
}
}
func TestKeyValuePair_ForAllTypes(t *testing.T) {
for _, test := range []parseTest{
{"string='literal'", []string{`key("string")`, `assign`, `string("literal")`}},
{"string='''literal\nmulti-line'''", []string{`key("string")`, `assign`, `string("literal\nmulti-line")`}},
{`string="basic"`, []string{`key("string")`, `assign`, `string("basic")`}},
{"string=\"\"\"basic\nmulti-line\"\"\"", []string{`key("string")`, `assign`, `string("basic\nmulti-line")`}},
{"integer=1_234_567", []string{`key("integer")`, `assign`, `integer(1234567)`}},
{"integer=42", []string{`key("integer")`, `assign`, `integer(42)`}},
{"integer=0x42", []string{`key("integer")`, `assign`, `integer(66)`}},
{"integer=0o42", []string{`key("integer")`, `assign`, `integer(34)`}},
{"integer=0b101010", []string{`key("integer")`, `assign`, `integer(42)`}},
{"float=42.37", []string{`key("float")`, `assign`, `float(42.37)`}},
{"float=42e+37", []string{`key("float")`, `assign`, `float(4.2e+38)`}},
{"float=42.37e-11", []string{`key("float")`, `assign`, `float(4.237e-10)`}},
{"boolean=true", []string{`key("boolean")`, `assign`, `boolean(true)`}},
{"boolean=false", []string{`key("boolean")`, `assign`, `boolean(false)`}},
{"date=2019-01-01", []string{`key("date")`, `assign`, `date(2019-01-01 00:00:00 +0000 UTC)`}},
{"time=15:03:11", []string{`key("time")`, `assign`, `time(0000-01-01 15:03:11 +0000 UTC)`}},
{"datetime=2021-02-01 15:03:11.123", []string{`key("datetime")`, `assign`, `datetime(2021-02-01 15:03:11.123 +0000 UTC)`}},
{"offset_datetime=1111-11-11 11:11:11.111111111+11:11", []string{`key("offset_datetime")`, `assign`, `offset_datetime(1111-11-11 11:11:11.111111111 +1111 +1111)`}},
{"string='literal'", []string{`key("string")`, `=`, `"literal"`}},
{"string='''literal\nmulti-line'''", []string{`key("string")`, `=`, `"literal\nmulti-line"`}},
{`string="basic"`, []string{`key("string")`, `=`, `"basic"`}},
{"string=\"\"\"basic\nmulti-line\"\"\"", []string{`key("string")`, `=`, `"basic\nmulti-line"`}},
{"integer=1_234_567", []string{`key("integer")`, `=`, `1234567`}},
{"integer=42", []string{`key("integer")`, `=`, `42`}},
{"integer=0x42", []string{`key("integer")`, `=`, `66`}},
{"integer=0o42", []string{`key("integer")`, `=`, `34`}},
{"integer=0b101010", []string{`key("integer")`, `=`, `42`}},
{"float=42.37", []string{`key("float")`, `=`, `42.37`}},
{"float=42e+37", []string{`key("float")`, `=`, `4.2e+38`}},
{"float=42.37e-11", []string{`key("float")`, `=`, `4.237e-10`}},
{"boolean=true", []string{`key("boolean")`, `=`, `true`}},
{"boolean=false", []string{`key("boolean")`, `=`, `false`}},
{"date=2019-01-01", []string{`key("date")`, `=`, `2019-01-01`}},
{"time=15:03:11", []string{`key("time")`, `=`, `15:03:11`}},
{"datetime=2021-02-01 15:03:11.123", []string{`key("datetime")`, `=`, `2021-02-01 15:03:11.123`}},
{"offset_datetime=1111-11-11 11:11:11.111111111+11:11", []string{`key("offset_datetime")`, `=`, `1111-11-11T11:11:11.111111111+11:11`}},
{"static_array=['a', 'static', 'array']", []string{`key("static_array")`, `=`, `["a", "static", "array"]`}},
} {
p := &parser{}
p := newParser()
testParseHandler(t, p, p.startKeyValuePair, test)
}
}
func TestKeyValuePair_ExamplesFromSpecification(t *testing.T) {
for _, test := range []parseTest{
{"int1 = +99", []string{`key("int1")`, `assign`, `integer(99)`}},
{"int2 = 42", []string{`key("int2")`, `assign`, `integer(42)`}},
{"int3 = 0", []string{`key("int3")`, `assign`, `integer(0)`}},
{"int4 = -17", []string{`key("int4")`, `assign`, `integer(-17)`}},
{"int5 = 1_000", []string{`key("int5")`, `assign`, `integer(1000)`}},
{"int6 = 5_349_221", []string{`key("int6")`, `assign`, `integer(5349221)`}},
{"int7 = 1_2_3_4_5 # VALID but discouraged", []string{`key("int7")`, `assign`, `integer(12345)`, `comment("# VALID but discouraged")`}},
{"hex1 = 0xDEADBEEF", []string{`key("hex1")`, `assign`, `integer(3735928559)`}},
{"hex2 = 0xdeadbeef", []string{`key("hex2")`, `assign`, `integer(3735928559)`}},
{"hex3 = 0xdead_beef", []string{`key("hex3")`, `assign`, `integer(3735928559)`}},
{"oct1 = 0o01234567", []string{`key("oct1")`, `assign`, `integer(342391)`}},
{"oct2 = 0o755", []string{`key("oct2")`, `assign`, `integer(493)`}},
{"bin1 = 0b11010110", []string{`key("bin1")`, `assign`, `integer(214)`}},
{"flt1 = +1.0", []string{`key("flt1")`, `assign`, `float(1)`}},
{"flt2 = 3.1415", []string{`key("flt2")`, `assign`, `float(3.1415)`}},
{"flt3 = -0.01", []string{`key("flt3")`, `assign`, `float(-0.01)`}},
{"flt4 = 5e+22", []string{`key("flt4")`, `assign`, `float(5e+22)`}},
{"flt5 = 1e6", []string{`key("flt5")`, `assign`, `float(1e+06)`}},
{"flt6 = -2E-2", []string{`key("flt6")`, `assign`, `float(-0.02)`}},
{"flt7 = 6.626e-34", []string{`key("flt7")`, `assign`, `float(6.626e-34)`}},
{"flt8 = 224_617.445_991_228", []string{`key("flt8")`, `assign`, `float(224617.445991228)`}},
{"sf1 = inf # positive infinity", []string{`key("sf1")`, `assign`, `float(+Inf)`, `comment("# positive infinity")`}},
{"sf2 = +inf # positive infinity", []string{`key("sf2")`, `assign`, `float(+Inf)`, `comment("# positive infinity")`}},
{"sf3 = -inf # negative infinity", []string{`key("sf3")`, `assign`, `float(-Inf)`, `comment("# negative infinity")`}},
{"sf4 = nan # actual sNaN/qNaN encoding is implementation-specific", []string{`key("sf4")`, `assign`, `float(NaN)`, `comment("# actual sNaN/qNaN encoding is implementation-specific")`}},
{"sf5 = +nan # same as `nan`", []string{`key("sf5")`, `assign`, `float(NaN)`, "comment(\"# same as `nan`\")"}},
{"sf6 = -nan # valid, actual encoding is implementation-specific", []string{`key("sf6")`, `assign`, `float(NaN)`, `comment("# valid, actual encoding is implementation-specific")`}},
{"bool1 = true", []string{`key("bool1")`, `assign`, `boolean(true)`}},
{"bool2 = false", []string{`key("bool2")`, `assign`, `boolean(false)`}},
{"odt1 = 1979-05-27T07:32:00Z", []string{`key("odt1")`, `assign`, `offset_datetime(1979-05-27 07:32:00 +0000 UTC)`}},
{"odt2 = 1979-05-27T00:32:00-07:00", []string{`key("odt2")`, `assign`, `offset_datetime(1979-05-27 00:32:00 -0700 -0700)`}},
{"odt3 = 1979-05-27T00:32:00.999999-07:00", []string{`key("odt3")`, `assign`, `offset_datetime(1979-05-27 00:32:00.999999 -0700 -0700)`}},
{"odt4 = 1979-05-27 07:32:00Z", []string{`key("odt4")`, `assign`, `offset_datetime(1979-05-27 07:32:00 +0000 UTC)`}},
{"ldt1 = 1979-05-27T07:32:00", []string{`key("ldt1")`, `assign`, `datetime(1979-05-27 07:32:00 +0000 UTC)`}},
{"ldt2 = 1979-05-27T00:32:00.999999", []string{`key("ldt2")`, `assign`, `datetime(1979-05-27 00:32:00.999999 +0000 UTC)`}},
{"ld1 = 1979-05-27", []string{`key("ld1")`, `assign`, `date(1979-05-27 00:00:00 +0000 UTC)`}},
{"lt1 = 07:32:00", []string{`key("lt1")`, `assign`, `time(0000-01-01 07:32:00 +0000 UTC)`}},
{"lt2 = 00:32:00.999999", []string{`key("lt2")`, `assign`, `time(0000-01-01 00:32:00.999999 +0000 UTC)`}},
{"int1 = +99", []string{`key("int1")`, `=`, `99`}},
{"int2 = 42", []string{`key("int2")`, `=`, `42`}},
{"int3 = 0", []string{`key("int3")`, `=`, `0`}},
{"int4 = -17", []string{`key("int4")`, `=`, `-17`}},
{"int5 = 1_000", []string{`key("int5")`, `=`, `1000`}},
{"int6 = 5_349_221", []string{`key("int6")`, `=`, `5349221`}},
{"int7 = 1_2_3_4_5 # VALID but discouraged", []string{`key("int7")`, `=`, `12345`, `comment("# VALID but discouraged")`}},
{"hex1 = 0xDEADBEEF", []string{`key("hex1")`, `=`, `3735928559`}},
{"hex2 = 0xdeadbeef", []string{`key("hex2")`, `=`, `3735928559`}},
{"hex3 = 0xdead_beef", []string{`key("hex3")`, `=`, `3735928559`}},
{"oct1 = 0o01234567", []string{`key("oct1")`, `=`, `342391`}},
{"oct2 = 0o755", []string{`key("oct2")`, `=`, `493`}},
{"bin1 = 0b11010110", []string{`key("bin1")`, `=`, `214`}},
{"flt1 = +1.0", []string{`key("flt1")`, `=`, `1`}},
{"flt2 = 3.1415", []string{`key("flt2")`, `=`, `3.1415`}},
{"flt3 = -0.01", []string{`key("flt3")`, `=`, `-0.01`}},
{"flt4 = 5e+22", []string{`key("flt4")`, `=`, `5e+22`}},
{"flt5 = 1e6", []string{`key("flt5")`, `=`, `1e+06`}},
{"flt6 = -2E-2", []string{`key("flt6")`, `=`, `-0.02`}},
{"flt7 = 6.626e-34", []string{`key("flt7")`, `=`, `6.626e-34`}},
{"flt8 = 224_617.445_991_228", []string{`key("flt8")`, `=`, `224617.445991228`}},
{"sf1 = inf # positive infinity", []string{`key("sf1")`, `=`, `+Inf`, `comment("# positive infinity")`}},
{"sf2 = +inf # positive infinity", []string{`key("sf2")`, `=`, `+Inf`, `comment("# positive infinity")`}},
{"sf3 = -inf # negative infinity", []string{`key("sf3")`, `=`, `-Inf`, `comment("# negative infinity")`}},
{"sf4 = nan # actual sNaN/qNaN encoding is implementation-specific", []string{`key("sf4")`, `=`, `NaN`, `comment("# actual sNaN/qNaN encoding is implementation-specific")`}},
{"sf5 = +nan # same as `nan`", []string{`key("sf5")`, `=`, `NaN`, "comment(\"# same as `nan`\")"}},
{"sf6 = -nan # valid, actual encoding is implementation-specific", []string{`key("sf6")`, `=`, `NaN`, `comment("# valid, actual encoding is implementation-specific")`}},
{"bool1 = true", []string{`key("bool1")`, `=`, `true`}},
{"bool2 = false", []string{`key("bool2")`, `=`, `false`}},
{"odt1 = 1979-05-27T07:32:00Z", []string{`key("odt1")`, `=`, `1979-05-27T07:32:00Z`}},
{"odt2 = 1979-05-27T00:32:00-07:00", []string{`key("odt2")`, `=`, `1979-05-27T00:32:00-07:00`}},
{"odt3 = 1979-05-27T00:32:00.999999-07:00", []string{`key("odt3")`, `=`, `1979-05-27T00:32:00.999999-07:00`}},
{"odt4 = 1979-05-27 07:32:00Z", []string{`key("odt4")`, `=`, `1979-05-27T07:32:00Z`}},
{"ldt1 = 1979-05-27T07:32:00", []string{`key("ldt1")`, `=`, `1979-05-27 07:32:00`}},
{"ldt2 = 1979-05-27T00:32:00.999999", []string{`key("ldt2")`, `=`, `1979-05-27 00:32:00.999999`}},
{"ld1 = 1979-05-27", []string{`key("ld1")`, `=`, `1979-05-27`}},
{"lt1 = 07:32:00", []string{`key("lt1")`, `=`, `07:32:00`}},
{"lt2 = 00:32:00.999999", []string{`key("lt2")`, `=`, `00:32:00.999999`}},
{"arr1 = [ 1, 2, 3 ]", []string{`key("arr1")`, `=`, `[1, 2, 3]`}},
{`arr2 = [ "red", "yellow", "green" ]`, []string{`key("arr2")`, `=`, `["red", "yellow", "green"]`}},
{`arr3 = [ [ 1, 2 ], [3, 4, 5] ]`, []string{`key("arr3")`, `=`, `[[1, 2], [3, 4, 5]]`}},
{`arr4 = [ "all", 'strings', """are the same""", '''type''']`, []string{`key("arr4")`, `=`, `["all", "strings", "are the same", "type"]`}},
{`arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]`, []string{`key("arr5")`, `=`, `[[1, 2], ["a", "b", "c"]]`}},
{`arr6 = [ 1, 2.0 ] # INVALID`, []string{`key("arr6")`, `=`, `Error: type mismatch in array of integers: found an item of type float at line 1, column 16`}},
{"arr7 = [\n 1, 2, 3\n]", []string{`key("arr7")`, `=`, `[1, 2, 3]`}},
{"arr8 = [\n 1,\n 2, # this is ok\n]", []string{`key("arr8")`, `=`, `[1, 2]`}},
} {
p := &parser{}
p := newParser()
testParseHandler(t, p, p.startKeyValuePair, test)
}
}

73
toml.go
View File

@ -1,67 +1,22 @@
package parser
package toml
import (
"fmt"
"strings"
"git.makaay.nl/mauricem/go-parsekit/tokenize"
)
// Easy access to the parsekit.tokenize definitions.
var c, a, m, tok = tokenize.C, tokenize.A, tokenize.M, tokenize.T
// Some globally useful tokenizer definitions.
var (
c, a, m, tok = tokenize.C, tokenize.A, tokenize.M, tokenize.T
type cmdType string
// From the specs: "Whitespace means tab (0x09) or space (0x20)."
// In this package, we name this a blank, to be in line with the
// terminology as used in parsekit.
blank = a.Runes('\t', ' ')
// Command types that are emitted by the parser.
const (
cComment cmdType = "comment" // a # comment at the end of the line
cKey cmdType = "key" // set key name
cKeyDot cmdType = "keydot" // new key stack level
cAssign cmdType = "assign" // assign a value
csetStrVal cmdType = "string" // set a string value
csetIntVal cmdType = "integer" // set an integer value
csetFloatVal cmdType = "float" // set a float value
csetBoolVal cmdType = "boolean" // set a boolean value
coffsetDateTime cmdType = "offset_datetime" // set a date/time value with timezone information
clocalDateTime cmdType = "datetime" // set a local date/time value
clocalDate cmdType = "date" // set a local date value
clocalTime cmdType = "time" // set a local time value
// Newline means LF (0x0A) or CRLF (0x0D0A).
// This matches the default newline as defined by parsekit.
newline = a.Newline
dropBlanks = m.Drop(c.ZeroOrMore(blank))
dropWhitespace = m.Drop(c.ZeroOrMore(blank.Or(newline)))
)
type parser struct {
commands []cmd
keyStack []string
}
type cmd struct {
command cmdType
args []interface{}
}
func (cmd *cmd) String() string {
if len(cmd.args) == 0 {
return fmt.Sprintf("%s", cmd.command)
}
args := make([]string, len(cmd.args))
for i, arg := range cmd.args {
switch arg.(type) {
case string:
args[i] = fmt.Sprintf("%q", arg)
default:
args[i] = fmt.Sprintf("%v", arg)
}
}
return fmt.Sprintf("%s(%s)", cmd.command, strings.Join(args, ", "))
}
func (p *parser) emitCommand(command cmdType, args ...interface{}) {
c := cmd{command: command, args: args}
p.commands = append(p.commands, c)
}
// Parse starts the parser for the provided input.
// func Parse(input interface{}) []cmd {
// p := &parser{}
// parse.New(p.startKeyValuePair)(input)
// return p.commands
// }

34
toml_test.go
View File

@ -1,34 +0,0 @@
package parser_test
// func TestEmptyInput(t *testing.T) {
// runStatesT(t, statesT{"empty string", "", "", ""})
// }
// func TestFullIncludesLineAndRowPosition(t *testing.T) {
// p := toml.Parse("# 12345 abcde\t\n\n\n# 67890\r\n# 12345\n +")
// _, err := parseItemsToArray(p)
// actual := err.Error()
// expected := "unexpected input (expected end of file) at line 6, column 3"
// if actual != expected {
// t.Errorf("Unexpected error message:\nexpected: %s\nactual: %s\n", expected, actual)
// }
// }
// func TestInvalidUTF8Data(t *testing.T) {
// runStatesTs(t, []statesT{
// {"bare key 1", "\xbc", "", "invalid UTF8 character in input (expected end of file)"},
// {"bare key 2", "key\xbc", "[key]", "invalid UTF8 character in input (expected a value assignment)"},
// {"start of value", "key=\xbc", "[key]=", "invalid UTF8 character in input (expected a value)"},
// {"basic string value", "a=\"\xbc\"", "[a]=", "invalid UTF8 character in input (expected string contents)"},
// })
// }
// func TestWhiteSpaceAndNewlines(t *testing.T) {
// runStatesTs(t, []statesT{
// {"space", " ", "", ""},
// {"tab", "\t", "", ""},
// {"newline", "\n", "", ""},
// {"all blanks and newlines", " \t \t \r\n\n \n \t", "", ""},
// {"bare carriage return", "\r", "", "unexpected character '\\r' (expected end of file)"},
// })
// }

4
value.go
View File

@ -1,4 +1,4 @@
package parser
package toml
import (
"git.makaay.nl/mauricem/go-parsekit/parse"
@ -22,6 +22,8 @@ func (t *parser) startValue(p *parse.API) {
} else {
p.Handle(t.startNumber)
}
case p.Peek(a.SquareOpen):
p.Handle(t.startArray)
default:
p.Expected("a value")
}

86
value_array.go Normal file
View File

@ -0,0 +1,86 @@
package toml
import (
"git.makaay.nl/mauricem/go-parsekit/parse"
)
// Arrays are square brackets with values inside. Whitespace is ignored.
// Elements are separated by commas. Data types may not be mixed (different
// ways to define strings should be considered the same type, and so should
// arrays with different element types).
//
// arr1 = [ 1, 2, 3 ]
// arr2 = [ "red", "yellow", "green" ]
// arr3 = [ [ 1, 2 ], [3, 4, 5] ]
// arr4 = [ "all", 'strings', """are the same""", '''type''']
// arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]
//
// arr6 = [ 1, 2.0 ] # INVALID
//
// Arrays can also be multiline. A terminating comma (also called trailing
// comma) is ok after the last value of the array. There can be an arbitrary
// number of newlines and comments before a value and before the closing bracket.
//
// arr7 = [
// 1, 2, 3
// ]
//
// arr8 = [
// 1,
// 2, # this is ok
// ]
var (
arraySpace = c.ZeroOrMore(c.Any(blank, newline, comment))
arrayOpen = a.SquareOpen.Then(arraySpace)
arraySeparator = c.Seq(arraySpace, a.Comma, arraySpace)
arrayClose = c.Seq(c.Optional(arraySpace.Then(a.Comma)), arraySpace, a.SquareClose)
)
func (t *parser) startArray(p *parse.API) {
// Check for the start of the array.
if !p.Accept(arrayOpen) {
p.Expected("an array")
return
}
items := []item{}
// Check for an empty array.
if p.Accept(arrayClose) {
t.addParsedItem(pStaticArray, items)
return
}
// Not an empty array, parse the items.
for {
// Check for a valid item.
if !p.Handle(t.startValue) {
return
}
// Pop the item from the value parsing and append it to the array items.
parseItems, item := t.Items[0:len(t.Items)-1], t.Items[len(t.Items)-1]
t.Items = parseItems
// Data types may not be mixed (different ways to define strings should be
// considered the same type, and so should arrays with different element types).
if len(items) > 0 && item.Type != items[0].Type {
p.Error("type mismatch in array of %ss: found an item of type %s", items[0].Type, item.Type)
return
}
items = append(items, item)
// Check for the end of the array.
if p.Accept(arrayClose) {
t.addParsedItem(pStaticArray, items)
return
}
// Not the end of the array? Then we should find an array separator.
if !p.Accept(arraySeparator) {
p.Expected("an array separator")
return
}
}
}

40
value_array_test.go Normal file
View File

@ -0,0 +1,40 @@
package toml
import (
"testing"
)
func TestArray(t *testing.T) {
for _, test := range []parseTest{
{"", []string{`Error: unexpected end of file (expected an array) at start of file`}},
{"[ape", []string{`Error: unexpected input (expected a value) at line 1, column 2`}},
{"[1", []string{`Error: unexpected end of file (expected an array separator) at line 1, column 3`}},
{"[]", []string{`[]`}},
{"[\n]", []string{`[]`}},
{"[,]", []string{`[]`}},
{"[ , ]", []string{`[]`}},
{"[ \n , \r\n ]", []string{`[]`}},
{"[ \t , \t ]", []string{`[]`}},
{"[\r\n\r\n , \r\n \t\n]", []string{`[]`}},
{"[\n#comment on its own line\n]", []string{`[]`}},
{"[#comment before close\n]", []string{`[]`}},
{"[,#comment after separator\n]", []string{`[]`}},
{"[#comment before separator\n,]", []string{`[]`}},
{"[#comment before value\n1]", []string{`[1]`}},
{"[1#comment after value\n]", []string{`[1]`}},
{"[1\n#comment on its own line after value\n]", []string{`[1]`}},
{"[1#comment 1\n#comment 2\n#comment 3\n , \n2]", []string{`[1, 2]`}},
{"[1]", []string{`[1]`}},
{"[1,0x2, 0b11, 0o4]", []string{`[1, 2, 3, 4]`}},
{"[0.1,0.2,3e-1,0.04e+1, nan, inf]", []string{`[0.1, 0.2, 0.3, 0.4, NaN, +Inf]`}},
{"[\n\t 'a', \"b\", '''c''', \"\"\"d\ne\"\"\",\n \t]", []string{`["a", "b", "c", "d\ne"]`}},
{`[1, 2, 3, "four"]`, []string{`Error: type mismatch in array of integers: found an item of type string at line 1, column 17`}},
{`[[1],['a']]`, []string{`[[1], ["a"]]`}},
{`[[[],[]],[]]`, []string{`[[[], []], []]`}},
{"[\r\n\r\n \t\n [\r\n\r\n\t [],[\t]],\t\n[]\t \t \n ]", []string{`[[[], []], []]`}},
{`[[1],'a']`, []string{`Error: type mismatch in array of static arrays: found an item of type string at line 1, column 9`}},
} {
p := newParser()
testParseHandler(t, p, p.startArray, test)
}
}

6
value_boolean.go
View File

@ -1,4 +1,4 @@
package parser
package toml
import (
"git.makaay.nl/mauricem/go-parsekit/parse"
@ -8,9 +8,9 @@ import (
func (t *parser) startBoolean(p *parse.API) {
switch {
case p.Accept(a.Str("true")):
t.emitCommand(csetBoolVal, true)
t.addParsedItem(pBoolean, true)
case p.Accept(a.Str("false")):
t.emitCommand(csetBoolVal, false)
t.addParsedItem(pBoolean, false)
default:
p.Expected("true or false")
}

8
value_boolean_test.go
View File

@ -1,4 +1,4 @@
package parser
package toml
import (
"testing"
@ -7,14 +7,14 @@ import (
func TestBoolean(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected true or false) at start of file`}},
{`true`, []string{`boolean(true)`}},
{`false`, []string{`boolean(false)`}},
{`true`, []string{`true`}},
{`false`, []string{`false`}},
{`yes`, []string{`Error: unexpected input (expected true or false) at start of file`}},
{`no`, []string{`Error: unexpected input (expected true or false) at start of file`}},
{`1`, []string{`Error: unexpected input (expected true or false) at start of file`}},
{`0`, []string{`Error: unexpected input (expected true or false) at start of file`}},
} {
p := &parser{}
p := newParser()
testParseHandler(t, p, p.startBoolean, test)
}
}

16
value_datetime.go
View File

@ -1,4 +1,4 @@
package parser
package toml
import (
"time"
@ -66,10 +66,10 @@ var (
// The full date/time parse format, based on the above definitions.
// The first token denotes the type of date/time value.
// The rest of the tokens contain layout fragments for time.Parse().
offsetDateTime = tok.Str(coffsetDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok, tzTok))
localDateTime = tok.Str(clocalDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok))
localDate = tok.Str(clocalDate, dateTok)
localTime = tok.Str(clocalTime, c.Seq(timeTok, microTok))
offsetDateTime = tok.Str(pOffsetDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok, tzTok))
localDateTime = tok.Str(pLocalDateTime, c.Seq(dateTok, tdelimTok, timeTok, microTok))
localDate = tok.Str(pLocalDate, dateTok)
localTime = tok.Str(pLocalTime, c.Seq(timeTok, microTok))
datetime = c.Any(offsetDateTime, localDateTime, localDate, localTime)
)
@ -82,7 +82,7 @@ func (t *parser) startDateTime(p *parse.API) {
valueType := getDateTimeValueType(&tokens)
input, value, err := getDateTimeValue(&tokens)
if err == nil {
t.emitCommand(valueType, value)
t.addParsedItem(valueType, value)
} else {
p.Error("Cannot parse value 0%s: %s", input, err)
}
@ -90,8 +90,8 @@ func (t *parser) startDateTime(p *parse.API) {
// The first token is a token that wraps the complete date/time input.
// Its type denotes the type of date/time value that it wraps.
func getDateTimeValueType(tokens *[]*tokenize.Token) cmdType {
return (*tokens)[0].Type.(cmdType)
func getDateTimeValueType(tokens *[]*tokenize.Token) itemType {
return (*tokens)[0].Type.(itemType)
}
// The rest of the tokens contain fragments that can be used with

34
value_datetime_test.go
View File

@ -1,4 +1,4 @@
package parser
package toml
import (
"testing"
@ -7,27 +7,27 @@ import (
func TestDateTime(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected a date and/or time) at start of file`}},
{`1979-05-27`, []string{`date(1979-05-27 00:00:00 +0000 UTC)`}},
{`00:00:00`, []string{`time(0000-01-01 00:00:00 +0000 UTC)`}},
{`23:59:59`, []string{`time(0000-01-01 23:59:59 +0000 UTC)`}},
{`12:10:08.12121212121212`, []string{`time(0000-01-01 12:10:08.121212121 +0000 UTC)`}},
{`1979-05-28T01:01:01`, []string{`datetime(1979-05-28 01:01:01 +0000 UTC)`}},
{`1979-05-28 01:01:01`, []string{`datetime(1979-05-28 01:01:01 +0000 UTC)`}},
{`1979-05-27T07:32:00Z`, []string{`offset_datetime(1979-05-27 07:32:00 +0000 UTC)`}},
{`1979-05-27 07:33:00Z`, []string{`offset_datetime(1979-05-27 07:33:00 +0000 UTC)`}},
{`1979-05-27 07:34:00+07:00`, []string{`offset_datetime(1979-05-27 07:34:00 +0700 +0700)`}},
{`1979-05-27 07:34:00-07:00`, []string{`offset_datetime(1979-05-27 07:34:00 -0700 -0700)`}},
{`1985-03-31 23:59:59+00:00`, []string{`offset_datetime(1985-03-31 23:59:59 +0000 UTC)`}},
{`2000-09-10 00:00:00.000000000+00:00`, []string{`offset_datetime(2000-09-10 00:00:00 +0000 UTC)`}},
{`2003-11-01 01:02:03.999999999999+10:00`, []string{`offset_datetime(2003-11-01 01:02:03.999999999 +1000 +1000)`}},
{`2007-12-25 04:00:04.1111-10:30`, []string{`offset_datetime(2007-12-25 04:00:04.1111 -1030 -1030)`}},
{`2021-02-01 10:10:10.101010203040Z`, []string{`offset_datetime(2021-02-01 10:10:10.101010203 +0000 UTC)`}},
{`1979-05-27`, []string{`1979-05-27`}},
{`00:00:00`, []string{`00:00:00`}},
{`23:59:59`, []string{`23:59:59`}},
{`12:10:08.12121212121212`, []string{`12:10:08.121212121`}},
{`1979-05-28T01:01:01`, []string{`1979-05-28 01:01:01`}},
{`1979-05-28 01:01:01`, []string{`1979-05-28 01:01:01`}},
{`1979-05-27T07:32:00Z`, []string{`1979-05-27T07:32:00Z`}},
{`1979-05-27 07:33:00Z`, []string{`1979-05-27T07:33:00Z`}},
{`1979-05-27 07:34:00+07:00`, []string{`1979-05-27T07:34:00+07:00`}},
{`1979-05-27 07:34:00-07:00`, []string{`1979-05-27T07:34:00-07:00`}},
{`1985-03-31 23:59:59+00:00`, []string{`1985-03-31T23:59:59Z`}},
{`2000-09-10 00:00:00.000000000+00:00`, []string{`2000-09-10T00:00:00Z`}},
{`2003-11-01 01:02:03.999999999999+10:00`, []string{`2003-11-01T01:02:03.999999999+10:00`}},
{`2007-12-25 04:00:04.1111-10:30`, []string{`2007-12-25T04:00:04.1111-10:30`}},
{`2021-02-01 10:10:10.101010203040Z`, []string{`2021-02-01T10:10:10.101010203Z`}},
// TODO ugly column, should be at start or at the actual wrong part
{`2000-13-01`, []string{`Error: Cannot parse value 02000-13-01: parsing time "2000-13-01": month out of range at line 1, column 11`}},
{`2000-02-31`, []string{`Error: Cannot parse value 02000-02-31: parsing time "2000-02-31": day out of range at line 1, column 11`}},
{`25:01:01`, []string{`Error: Cannot parse value 025:01:01: parsing time "25:01:01": hour out of range at line 1, column 9`}},
} {
p := &parser{}
p := newParser()
testParseHandler(t, p, p.startDateTime, test)
}
}

16
value_number.go
View File

@ -1,4 +1,4 @@
package parser
package toml
import (
"math"
@ -68,19 +68,19 @@ var (
func (t *parser) startNumber(p *parse.API) {
switch {
case p.Accept(tok.Float64(nil, float)):
t.emitCommand(csetFloatVal, p.Result().Value(0).(float64))
t.addParsedItem(pFloat, p.Result().Value(0).(float64))
case p.Accept(nan):
t.emitCommand(csetFloatVal, math.NaN())
t.addParsedItem(pFloat, math.NaN())
case p.Accept(inf):
if p.Result().Rune(0) == '-' {
t.emitCommand(csetFloatVal, math.Inf(-1))
t.addParsedItem(pFloat, math.Inf(-1))
} else {
t.emitCommand(csetFloatVal, math.Inf(+1))
t.addParsedItem(pFloat, math.Inf(+1))
}
case p.Accept(a.Zero):
p.Handle(t.startIntegerStartingWithZero)
case p.Accept(tok.Int64(nil, integer)):
t.emitCommand(csetIntVal, p.Result().Value(0).(int64))
t.addParsedItem(pInteger, p.Result().Value(0).(int64))
default:
p.Expected("a number")
}
@ -97,11 +97,11 @@ func (t *parser) startIntegerStartingWithZero(p *parse.API) {
case p.Accept(binary):
value, err = strconv.ParseInt(p.Result().Value(0).(string), 2, 64)
default:
t.emitCommand(csetIntVal, int64(0))
t.addParsedItem(pInteger, int64(0))
return
}
if err == nil {
t.emitCommand(csetIntVal, value)
t.addParsedItem(pInteger, value)
} else {
p.Error("Cannot parse value 0%s: %s", p.Result().String(), err)
}

126
value_number_test.go
View File

@ -1,4 +1,4 @@
package parser
package toml
import (
"testing"
@ -8,73 +8,73 @@ func TestInteger(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected a number) at start of file`}},
// Decimal
{`0`, []string{`integer(0)`}},
{`+0`, []string{`integer(0)`}},
{`-0`, []string{`integer(0)`}},
{`1`, []string{`integer(1)`}},
{`42`, []string{`integer(42)`}},
{`+99`, []string{`integer(99)`}},
{`-17`, []string{`integer(-17)`}},
{`1234`, []string{`integer(1234)`}},
{`0`, []string{`0`}},
{`+0`, []string{`0`}},
{`-0`, []string{`0`}},
{`1`, []string{`1`}},
{`42`, []string{`42`}},
{`+99`, []string{`99`}},
{`-17`, []string{`-17`}},
{`1234`, []string{`1234`}},
{`_`, []string{`Error: unexpected input (expected a number) at start of file`}},
{`1_`, []string{`integer(1)`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
{`1_000`, []string{`integer(1000)`}},
{`5_349_221`, []string{`integer(5349221)`}},
{`1_2_3_4_5`, []string{`integer(12345)`}},
{`9_223_372_036_854_775_807`, []string{`integer(9223372036854775807)`}},
{`1_`, []string{`1`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
{`1_000`, []string{`1000`}},
{`5_349_221`, []string{`5349221`}},
{`1_2_3_4_5`, []string{`12345`}},
{`9_223_372_036_854_775_807`, []string{`9223372036854775807`}},
{`9_223_372_036_854_775_808`, []string{
`Panic: Handler error: MakeInt64Token cannot handle input "9223372036854775808": ` +
`strconv.ParseInt: parsing "9223372036854775808": value out of range (only use a ` +
`type conversion token maker, when the input has been validated on beforehand)`}},
{`-9_223_372_036_854_775_808`, []string{`integer(-9223372036854775808)`}},
{`-9_223_372_036_854_775_808`, []string{`-9223372036854775808`}},
// TODO make the use of the same kind of handling for panics and for errors between parsekit and TOML.
{`-9_223_372_036_854_775_809`, []string{
`Panic: Handler error: MakeInt64Token cannot handle input "-9223372036854775809": ` +
`strconv.ParseInt: parsing "-9223372036854775809": value out of range (only use a ` +
`type conversion token maker, when the input has been validated on beforehand)`}},
// Hexadecimal
{`0x0`, []string{`integer(0)`}},
{`0x1`, []string{`integer(1)`}},
{`0x01`, []string{`integer(1)`}},
{`0x00fF`, []string{`integer(255)`}},
{`0xf_f`, []string{`integer(255)`}},
{`0x0_0_f_f`, []string{`integer(255)`}},
{`0xdead_beef`, []string{`integer(3735928559)`}},
{`0xgood_beef`, []string{`integer(0)`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
{`0x7FFFFFFFFFFFFFFF`, []string{`integer(9223372036854775807)`}},
{`0x0`, []string{`0`}},
{`0x1`, []string{`1`}},
{`0x01`, []string{`1`}},
{`0x00fF`, []string{`255`}},
{`0xf_f`, []string{`255`}},
{`0x0_0_f_f`, []string{`255`}},
{`0xdead_beef`, []string{`3735928559`}},
{`0xgood_beef`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
{`0x7FFFFFFFFFFFFFFF`, []string{`9223372036854775807`}},
{`0x8000000000000000`, []string{
`Error: Cannot parse value 0x8000000000000000: strconv.ParseInt: parsing "8000000000000000": ` +
`value out of range at line 1, column 19`}},
//Octal
{`0o0`, []string{`integer(0)`}},
{`0o1`, []string{`integer(1)`}},
{`0o01`, []string{`integer(1)`}},
{`0o10`, []string{`integer(8)`}},
{`0o1_6`, []string{`integer(14)`}},
{`0o0_0_1_1_1`, []string{`integer(73)`}},
{`0o9`, []string{`integer(0)`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
{`0o777777777777777777777`, []string{`integer(9223372036854775807)`}},
{`0o0`, []string{`0`}},
{`0o1`, []string{`1`}},
{`0o01`, []string{`1`}},
{`0o10`, []string{`8`}},
{`0o1_6`, []string{`14`}},
{`0o0_0_1_1_1`, []string{`73`}},
{`0o9`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
{`0o777777777777777777777`, []string{`9223372036854775807`}},
{`0o1000000000000000000000`, []string{
`Error: Cannot parse value 0o1000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000": ` +
`value out of range at line 1, column 25`}},
// Binary
{`0b0`, []string{`integer(0)`}},
{`0b1`, []string{`integer(1)`}},
{`0b01`, []string{`integer(1)`}},
{`0b10`, []string{`integer(2)`}},
{`0b0100`, []string{`integer(4)`}},
{`0b00001000`, []string{`integer(8)`}},
{`0b0001_0000`, []string{`integer(16)`}},
{`0b9`, []string{`integer(0)`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
{`0b1_1_0_1_1`, []string{`integer(27)`}},
{`0b11111111_11111111`, []string{`integer(65535)`}},
{`0b01111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111`, []string{`integer(9223372036854775807)`}},
{`0b0`, []string{`0`}},
{`0b1`, []string{`1`}},
{`0b01`, []string{`1`}},
{`0b10`, []string{`2`}},
{`0b0100`, []string{`4`}},
{`0b00001000`, []string{`8`}},
{`0b0001_0000`, []string{`16`}},
{`0b9`, []string{`0`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
{`0b1_1_0_1_1`, []string{`27`}},
{`0b11111111_11111111`, []string{`65535`}},
{`0b01111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111`, []string{`9223372036854775807`}},
{`0b10000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000`, []string{
`Error: Cannot parse value 0b1000000000000000000000000000000000000000000000000000000000000000: ` +
`strconv.ParseInt: parsing "1000000000000000000000000000000000000000000000000000000000000000": ` +
`value out of range at line 1, column 74`}},
} {
p := &parser{}
p := newParser()
testParseHandler(t, p, p.startNumber, test)
}
}
@ -82,26 +82,26 @@ func TestInteger(t *testing.T) {
func TestFloat(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected a number) at start of file`}},
{`0.0`, []string{`float(0)`}},
{`+0.0`, []string{`float(0)`}},
{`-0.0`, []string{`float(-0)`}},
{`+1.0`, []string{`float(1)`}},
{`3.1415`, []string{`float(3.1415)`}},
{`-0.01`, []string{`float(-0.01)`}},
{`5e+22`, []string{`float(5e+22)`}},
{`1E6`, []string{`float(1e+06)`}},
{`-2E-2`, []string{`float(-0.02)`}},
{`6.626e-34`, []string{`float(6.626e-34)`}},
{`224_617.445_991_228`, []string{`float(224617.445991228)`}},
{`12_345.111_222e+1_2_3`, []string{`float(1.2345111222e+127)`}},
{`+nan`, []string{`float(NaN)`}},
{`-nan`, []string{`float(NaN)`}},
{`nan`, []string{`float(NaN)`}},
{`inf`, []string{`float(+Inf)`}},
{`+inf`, []string{`float(+Inf)`}},
{`-inf`, []string{`float(-Inf)`}},
{`0.0`, []string{`0`}},
{`+0.0`, []string{`0`}},
{`-0.0`, []string{`-0`}},
{`+1.0`, []string{`1`}},
{`3.1415`, []string{`3.1415`}},
{`-0.01`, []string{`-0.01`}},
{`5e+22`, []string{`5e+22`}},
{`1E6`, []string{`1e+06`}},
{`-2E-2`, []string{`-0.02`}},
{`6.626e-34`, []string{`6.626e-34`}},
{`224_617.445_991_228`, []string{`224617.445991228`}},
{`12_345.111_222e+1_2_3`, []string{`1.2345111222e+127`}},
{`+nan`, []string{`NaN`}},
{`-nan`, []string{`NaN`}},
{`nan`, []string{`NaN`}},
{`inf`, []string{`+Inf`}},
{`+inf`, []string{`+Inf`}},
{`-inf`, []string{`-Inf`}},
} {
p := &parser{}
p := newParser()
testParseHandler(t, p, p.startNumber, test)
}
}

35
value_string.go
View File

@ -1,4 +1,4 @@
package parser
package toml
import (
"strings"
@ -38,10 +38,7 @@ var (
// "line ending backslash". When the last non-whitespace character on a line is
// a \, it will be trimmed along with all whitespace (including newlines) up to
// the next non-whitespace character or closing delimiter.
lineEndingBackslash = a.Backslash.
Then(c.ZeroOrMore(a.Blanks)).
Then(a.Newline).
Then(c.ZeroOrMore(a.Whitespace))
lineEndingBackslash = a.Backslash.Then(dropBlanks).Then(newline).Then(dropWhitespace)
)
// There are four ways to express strings: basic, multi-line basic, literal and
@ -49,9 +46,9 @@ var (
func (t *parser) startString(p *parse.API) {
switch {
case p.Peek(doubleQuote3):
p.Handle(t.startMultiLineBasicString)
p.Handle(t.startMultiLineBasipString)
case p.Peek(a.DoubleQuote):
p.Handle(t.startBasicString)
p.Handle(t.startBasipString)
case p.Peek(singleQuote3):
p.Handle(t.startMultiLineLiteralString)
case p.Peek(a.SingleQuote):
@ -72,13 +69,13 @@ func (t *parser) startString(p *parse.API) {
// • No additional \escape sequences are allowed. What the spec say about this:
// "All other escape sequences [..] are reserved and, if used, TOML should
// produce an error.""
func (t *parser) startBasicString(p *parse.API) {
if str, ok := t.parseBasicString("basic string", p); ok {
t.emitCommand(csetStrVal, str)
func (t *parser) startBasipString(p *parse.API) {
if str, ok := t.parseBasipString("basic string", p); ok {
t.addParsedItem(pString, str)
}
}
func (t *parser) parseBasicString(name string, p *parse.API) (string, bool) {
func (t *parser) parseBasipString(name string, p *parse.API) (string, bool) {
if !p.Accept(a.DoubleQuote) {
p.Expected(`opening quotation marks`)
return "", false
@ -117,7 +114,7 @@ func (t *parser) parseBasicString(name string, p *parse.API) (string, bool) {
// • Control characters other than tab are not permitted in a literal string.
func (t *parser) startLiteralString(p *parse.API) {
if str, ok := t.parseLiteralString("literal string", p); ok {
t.emitCommand(csetStrVal, str)
t.addParsedItem(pString, str)
}
}
@ -171,15 +168,15 @@ func (t *parser) parseLiteralString(name string, p *parse.API) (string, bool) {
// "line ending backslash". When the last non-whitespace character on a line is
// a \, it will be trimmed along with all whitespace (including newlines) up to
// the next non-whitespace character or closing delimiter.
func (t *parser) startMultiLineBasicString(p *parse.API) {
if !p.Accept(doubleQuote3.Then(a.Newline.Optional())) {
func (t *parser) startMultiLineBasipString(p *parse.API) {
if !p.Accept(doubleQuote3.Then(newline.Optional())) {
p.Expected("opening three quotation marks")
return
}
sb := &strings.Builder{}
for {
switch {
case p.Accept(a.Newline):
case p.Accept(newline):
sb.WriteString("\n")
case p.Peek(controlCharacter):
p.Error("invalid character in multi-line basic string: %q (must be escaped)", p.Result().Rune(0))
@ -192,7 +189,7 @@ func (t *parser) startMultiLineBasicString(p *parse.API) {
p.Error("invalid escape sequence")
return
case p.Accept(m.Drop(doubleQuote3)):
t.emitCommand(csetStrVal, sb.String())
t.addParsedItem(pString, sb.String())
return
case p.Accept(a.ValidRune):
sb.WriteString(p.Result().String())
@ -220,7 +217,7 @@ func (t *parser) startMultiLineBasicString(p *parse.API) {
//
// • Control characters other than tab and newline are not permitted in a multi-line literal string.
func (t *parser) startMultiLineLiteralString(p *parse.API) {
if !p.Accept(singleQuote3.Then(a.Newline.Optional())) {
if !p.Accept(singleQuote3.Then(newline.Optional())) {
p.Expected("opening three single quotes")
return
}
@ -228,11 +225,11 @@ func (t *parser) startMultiLineLiteralString(p *parse.API) {
for {
switch {
case p.Accept(m.Drop(singleQuote3)):
t.emitCommand(csetStrVal, sb.String())
t.addParsedItem(pString, sb.String())
return
case p.Accept(a.Tab):
sb.WriteString("\t")
case p.Accept(a.Newline):
case p.Accept(newline):
sb.WriteString("\n")
case p.Peek(controlCharacter):
p.Error("invalid character in literal string: %q (no control chars allowed, except for tab and newline)", p.Result().Rune(0))

76
value_string_test.go
View File

@ -1,4 +1,4 @@
package parser
package toml
import (
"fmt"
@ -9,53 +9,53 @@ func TestString(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected a string value) at start of file`}},
{`no start quote"`, []string{`Error: unexpected input (expected a string value) at start of file`}},
{`"basic s\tring"`, []string{`string("basic s\tring")`}},
{"\"\"\"\n basic multi-line\n string value\n\"\"\"", []string{`string(" basic multi-line\n string value\n")`}},
{`'literal s\tring'`, []string{`string("literal s\\tring")`}},
{"'''\n literal multi-line\n string value\n'''", []string{`string(" literal multi-line\n string value\n")`}},
{`"basic s\tring"`, []string{`"basic s\tring"`}},
{"\"\"\"\n basic multi-line\n string value\n\"\"\"", []string{`" basic multi-line\n string value\n"`}},
{`'literal s\tring'`, []string{`"literal s\\tring"`}},
{"'''\n literal multi-line\n string value\n'''", []string{`" literal multi-line\n string value\n"`}},
} {
p := &parser{}
p := newParser()
testParseHandler(t, p, p.startString, test)
}
}
func TestBasicString(t *testing.T) {
func TestBasipString(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected opening quotation marks) at start of file`}},
{`no start quote"`, []string{`Error: unexpected input (expected opening quotation marks) at start of file`}},
{`"no end quote`, []string{`Error: unexpected end of file (expected closing quotation marks) at line 1, column 14`}},
{`""`, []string{`string("")`}},
{`"simple string"`, []string{`string("simple string")`}},
{`"with\tsome\r\nvalid escapes\b"`, []string{`string("with\tsome\r\nvalid escapes\b")`}},
{`""`, []string{`""`}},
{`"simple string"`, []string{`"simple string"`}},
{`"with\tsome\r\nvalid escapes\b"`, []string{`"with\tsome\r\nvalid escapes\b"`}},
{`"with an \invalid escape"`, []string{`Error: invalid escape sequence at line 1, column 10`}},
{`"A cool UTF8 ƃuıɹʇs"`, []string{`string("A cool UTF8 ƃuıɹʇs")`}},
{`"A string with UTF8 escape \u2318"`, []string{`string("A string with UTF8 escape ⌘")`}},
{`"A cool UTF8 ƃuıɹʇs"`, []string{`"A cool UTF8 ƃuıɹʇs"`}},
{`"A string with UTF8 escape \u2318"`, []string{`"A string with UTF8 escape ⌘"`}},
{"\"Invalid character for UTF \xcd\"", []string{`Error: invalid UTF8 rune at line 1, column 28`}},
{"\"Character that mus\t be escaped\"", []string{`Error: invalid character in basic string: '\t' (must be escaped) at line 1, column 20`}},
{"\"Character that must be escaped \u0000\"", []string{`Error: invalid character in basic string: '\x00' (must be escaped) at line 1, column 33`}},
{"\"Character that must be escaped \x7f\"", []string{`Error: invalid character in basic string: '\u007f' (must be escaped) at line 1, column 33`}},
} {
p := &parser{}
testParseHandler(t, p, p.startBasicString, test)
p := newParser()
testParseHandler(t, p, p.startBasipString, test)
}
}
func TestMultiLineBasicString(t *testing.T) {
func TestMultiLineBasipString(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected opening three quotation marks) at start of file`}},
{`"""missing close quote""`, []string{`Error: unexpected end of file (expected closing three quotation marks) at line 1, column 25`}},
{`""""""`, []string{`string("")`}},
{"\"\"\"\n\"\"\"", []string{`string("")`}},
{"\"\"\"\r\n\r\n\"\"\"", []string{`string("\n")`}},
{`"""\"\"\"\""""`, []string{`string("\"\"\"\"")`}},
{"\"\"\"\nThe quick brown \\\n\n\n \t fox jumps over \\\n\t the lazy dog.\\\n \"\"\"", []string{`string("The quick brown fox jumps over the lazy dog.")`}},
{`""""""`, []string{`""`}},
{"\"\"\"\n\"\"\"", []string{`""`}},
{"\"\"\"\r\n\r\n\"\"\"", []string{`"\n"`}},
{`"""\"\"\"\""""`, []string{`"\"\"\"\""`}},
{"\"\"\"\nThe quick brown \\\n\n\n \t fox jumps over \\\n\t the lazy dog.\\\n \"\"\"", []string{`"The quick brown fox jumps over the lazy dog."`}},
{"\"\"\"No control chars \f allowed\"\"\"", []string{`Error: invalid character in multi-line basic string: '\f' (must be escaped) at line 1, column 21`}},
{"\"\"\"Escaping control chars\\nis valid\"\"\"", []string{`string("Escaping control chars\nis valid")`}},
{"\"\"\"Escaping control chars\\nis valid\"\"\"", []string{`"Escaping control chars\nis valid"`}},
{"\"\"\"Invalid escaping \\is not allowed\"\"\"", []string{`Error: invalid escape sequence at line 1, column 21`}},
{"\"\"\"Invalid rune \xcd\"\"\"", []string{`Error: invalid UTF8 rune at line 1, column 17`}},
} {
p := &parser{}
testParseHandler(t, p, p.startMultiLineBasicString, test)
p := newParser()
testParseHandler(t, p, p.startMultiLineBasipString, test)
}
}
@ -63,17 +63,17 @@ func TestLiteralString(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected opening single quote) at start of file`}},
{`'missing close quote`, []string{`Error: unexpected end of file (expected closing single quote) at line 1, column 21`}},
{`''`, []string{`string("")`}},
{`'simple'`, []string{`string("simple")`}},
{`'C:\Users\nodejs\templates'`, []string{`string("C:\\Users\\nodejs\\templates")`}},
{`'\\ServerX\admin$\system32\'`, []string{`string("\\\\ServerX\\admin$\\system32\\")`}},
{`'Tom "Dubs" Preston-Werner'`, []string{`string("Tom \"Dubs\" Preston-Werner")`}},
{`'<\i\c*\s*>'`, []string{`string("<\\i\\c*\\s*>")`}},
{`''`, []string{`""`}},
{`'simple'`, []string{`"simple"`}},
{`'C:\Users\nodejs\templates'`, []string{`"C:\\Users\\nodejs\\templates"`}},
{`'\\ServerX\admin$\system32\'`, []string{`"\\\\ServerX\\admin$\\system32\\"`}},
{`'Tom "Dubs" Preston-Werner'`, []string{`"Tom \"Dubs\" Preston-Werner"`}},
{`'<\i\c*\s*>'`, []string{`"<\\i\\c*\\s*>"`}},
{"'No cont\rol chars allowed'", []string{`Error: invalid character in literal string: '\r' (no control chars allowed, except for tab) at line 1, column 9`}},
{"'Except\tfor\ttabs'", []string{`string("Except\tfor\ttabs")`}},
{"'Except\tfor\ttabs'", []string{`"Except\tfor\ttabs"`}},
{"'Invalid rune \xcd'", []string{`Error: invalid UTF8 rune at line 1, column 15`}},
} {
p := &parser{}
p := newParser()
testParseHandler(t, p, p.startLiteralString, test)
}
}
@ -82,23 +82,23 @@ func TestMultiLineLiteralString(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected opening three single quotes) at start of file`}},
{`'''missing close quote''`, []string{`Error: unexpected end of file (expected closing three single quotes) at line 1, column 25`}},
{`''''''`, []string{`string("")`}},
{"'''\n'''", []string{`string("")`}},
{`'''I [dw]on't need \d{2} apples'''`, []string{`string("I [dw]on't need \\d{2} apples")`}},
{"'''\nThere can\nbe newlines\r\nand \ttabs!\r\n'''", []string{`string("There can\nbe newlines\nand \ttabs!\n")`}},
{`''''''`, []string{`""`}},
{"'''\n'''", []string{`""`}},
{`'''I [dw]on't need \d{2} apples'''`, []string{`"I [dw]on't need \\d{2} apples"`}},
{"'''\nThere can\nbe newlines\r\nand \ttabs!\r\n'''", []string{`"There can\nbe newlines\nand \ttabs!\n"`}},
{"'''No other \f control characters'''", []string{`Error: invalid character in literal string: '\f' (no control chars allowed, except for tab and newline) at line 1, column 13`}},
{"'''No invalid runes allowed \xcd'''", []string{"Error: invalid UTF8 rune at line 1, column 29"}},
} {
p := &parser{}
p := newParser()
testParseHandler(t, p, p.startMultiLineLiteralString, test)
}
}
func TestBasicStringWithUnescapedControlCharacters(t *testing.T) {
func TestBasipStringWithUnescapedControlCharacters(t *testing.T) {
// A quick check for almost all characters that must be escaped.
// The missing one (\x7f) is covered in the previous test.
for i := 0x00; i <= 0x1F; i++ {
p := &parser{}
p := newParser()
input := fmt.Sprintf(`"%c"`, rune(i))
expected := fmt.Sprintf(`Error: invalid character in basic string: %q (must be escaped) at line 1, column 2`, rune(i))
testParseHandler(t, p, p.startString, parseTest{input, []string{expected}})

111
value_table.go Normal file
View File

@ -0,0 +1,111 @@
package toml
import (
"git.makaay.nl/mauricem/go-parsekit/parse"
)
var (
// Tables (also known as hash tables or dictionaries) are collections of
// key/value pairs. They appear in square brackets on a line by themselves.
// You can tell them apart from arrays because arrays are only ever values.
//
// Under that, and until the next table or EOF are the key/values of that
// table. Key/value pairs within tables are not guaranteed to be in any
// specific order.
//
// [table-1]
// key1 = "some string"
// key2 = 123
//
// [table-2]
// key1 = "another string"
// key2 = 456
//
// Naming rules for tables are the same as for keys.
//
// [dog."tater.man"]
// type.name = "pug"
//
// Whitespace around the key is ignored, however, best practice is to not
// use any extraneous whitespace.
//
// [a.b.c] # this is best practice
// [ d.e.f ] # same as [d.e.f]
// [ g . h . i ] # same as [g.h.i]
// [ j . "ʞ" . 'l' ] # same as [j."ʞ".'l']
//
// You don't need to specify all the super-tables if you don't want to.
// TOML knows how to do it for you.
//
// # [x] you
// # [x.y] don't
// # [x.y.z] need these
// [x.y.z.w] # for this to work
//
// Empty tables are allowed and simply have no key/value pairs within them.
//
tableOpen = c.Seq(dropBlanks, a.SquareOpen, dropBlanks)
tableClose = c.Seq(dropBlanks, a.SquareClose, dropBlanks, a.EndOfLine.Or(comment))
// Arrays of tables can be expressed by using a table name in double brackets.
// Each table with the same double bracketed name will be an element in the
// array. The tables are inserted in the order encountered. A double bracketed
// table without any key/value pairs will be considered an empty table.
//
// [[products]]
// name = "Hammer"
// sku = 738594937
//
// [[products]]
//
// [[products]]
// name = "Nail"
// sku = 284758393
// color = "gray"
//
// You can create nested arrays of tables as well. Just use the same double
// bracket syntax on sub-tables. Each double-bracketed sub-table will belong
// to the most recently defined table element above it.
//
// [[fruit]]
// name = "apple"
//
// [fruit.physical]
// color = "red"
// shape = "round"
//
// [[fruit.variety]]
// name = "red delicious"
//
// [[fruit.variety]]
// name = "granny smith"
//
// [[fruit]]
// name = "banana"
//
// [[fruit.variety]]
// name = "plantain"
tableArrayOpen = c.Seq(dropBlanks, a.SquareOpen, a.SquareOpen, dropBlanks)
tableArrayClose = c.Seq(dropBlanks, a.SquareClose, a.SquareClose, a.EndOfLine.Or(comment))
)
func (t *parser) startTable(p *parse.API) {
switch {
case p.Accept(tableOpen):
p.Handle(t.startPlainTable)
default:
p.Expected("a table")
}
}
func (t *parser) startPlainTable(p *parse.API) {
if !p.Handle(t.startKey) {
return
}
if !p.Accept(tableClose) {
p.Expected("closing ']' for table name")
}
key := t.Items[0]
t.Items = t.Items[1:]
t.newTable(key)
}

23
value_table_test.go Normal file
View File

@ -0,0 +1,23 @@
package toml
import (
"testing"
)
func TestTable(t *testing.T) {
for _, test := range []parseTest{
{"", []string{`Error: unexpected end of file (expected a table) at start of file`}},
{"[", []string{`Error: unexpected end of file (expected a key name) at line 1, column 2`}},
{" \t [", []string{`Error: unexpected end of file (expected a key name) at line 1, column 5`}},
{" \t [key", []string{`Error: unexpected end of file (expected closing ']' for table name) at line 1, column 8`}},
{" \t [key.", []string{`key("key")`, `Error: unexpected end of file (expected a key name) at line 1, column 9`}},
{" \t [key.'sub key'", []string{`Error: unexpected end of file (expected closing ']' for table name) at line 1, column 18`}},
{" \t [key.'sub key' \t]", []string{}},
{" \t [key.'sub key' \t] \t ", []string{}},
{" \t [key.'sub key' \t] \t \n", []string{}},
{" \t [key.'sub key' \t] \t # with comment\n", []string{}},
} {
p := newParser()
testParseHandler(t, p, p.startTable, test)
}
}

67
value_test.go
View File

@ -1,4 +1,4 @@
package parser
package toml
import (
"testing"
@ -7,39 +7,40 @@ import (
func TestValue(t *testing.T) {
for _, test := range []parseTest{
{``, []string{`Error: unexpected end of file (expected a value) at start of file`}},
{`"basic s\tring value"`, []string{`string("basic s\tring value")`}},
{`'literal s\tring value'`, []string{`string("literal s\\tring value")`}},
{"\"\"\"basic multi-line\nstring value\"\"\"", []string{`string("basic multi-line\nstring value")`}},
{"'''literal multi-line\nstring value'''", []string{`string("literal multi-line\nstring value")`}},
{"true", []string{`boolean(true)`}},
{"false", []string{`boolean(false)`}},
{"0", []string{`integer(0)`}},
{"+0", []string{`integer(0)`}},
{"-0", []string{`integer(0)`}},
{"0.0", []string{`float(0)`}},
{"+0.0", []string{`float(0)`}},
{"-0.0", []string{`float(-0)`}},
{"1234", []string{`integer(1234)`}},
{"-1234", []string{`integer(-1234)`}},
{"+9_8_7.6_5_4e-321", []string{`float(9.8765e-319)`}},
{"-1_234.5678e-33", []string{`float(-1.2345678e-30)`}},
{"inf", []string{`float(+Inf)`}},
{"+inf", []string{`float(+Inf)`}},
{"-inf", []string{`float(-Inf)`}},
{"nan", []string{`float(NaN)`}},
{"+nan", []string{`float(NaN)`}},
{"-nan", []string{`float(NaN)`}},
{"2019-06-19", []string{`date(2019-06-19 00:00:00 +0000 UTC)`}},
{"08:38:54", []string{`time(0000-01-01 08:38:54 +0000 UTC)`}},
{"2019-06-19 08:38:54", []string{`datetime(2019-06-19 08:38:54 +0000 UTC)`}},
{"2019-06-19T08:38:54", []string{`datetime(2019-06-19 08:38:54 +0000 UTC)`}},
{"2019-06-19 08:38:54", []string{`datetime(2019-06-19 08:38:54 +0000 UTC)`}},
{"2019-06-19T08:38:54.88888", []string{`datetime(2019-06-19 08:38:54.88888 +0000 UTC)`}},
{"1979-05-27T07:32:00Z", []string{`offset_datetime(1979-05-27 07:32:00 +0000 UTC)`}},
{"1979-05-27T00:32:00-07:00", []string{`offset_datetime(1979-05-27 00:32:00 -0700 -0700)`}},
{"1979-05-27T00:32:00.999999-07:00", []string{`offset_datetime(1979-05-27 00:32:00.999999 -0700 -0700)`}},
{`"basic s\tring value"`, []string{`"basic s\tring value"`}},
{`'literal s\tring value'`, []string{`"literal s\\tring value"`}},
{"\"\"\"basic multi-line\nstring value\"\"\"", []string{`"basic multi-line\nstring value"`}},
{"'''literal multi-line\nstring value'''", []string{`"literal multi-line\nstring value"`}},
{"true", []string{`true`}},
{"false", []string{`false`}},
{"0", []string{`0`}},
{"+0", []string{`0`}},
{"-0", []string{`0`}},
{"0.0", []string{`0`}},
{"+0.0", []string{`0`}},
{"-0.0", []string{`-0`}},
{"1234", []string{`1234`}},
{"-1234", []string{`-1234`}},
{"+9_8_7.6_5_4e-321", []string{`9.8765e-319`}},
{"-1_234.5678e-33", []string{`-1.2345678e-30`}},
{"inf", []string{`+Inf`}},
{"+inf", []string{`+Inf`}},
{"-inf", []string{`-Inf`}},
{"nan", []string{`NaN`}},
{"+nan", []string{`NaN`}},
{"-nan", []string{`NaN`}},
{"2019-06-19", []string{`2019-06-19`}},
{"08:38:54", []string{`08:38:54`}},
{"08:38:54.8765487654876", []string{`08:38:54.876548765`}},
{"2019-06-19 08:38:54", []string{`2019-06-19 08:38:54`}},
{"2019-06-19T08:38:54", []string{`2019-06-19 08:38:54`}},
{"2019-06-19T08:38:54.88888", []string{`2019-06-19 08:38:54.88888`}},
{"1979-05-27T07:32:00Z", []string{`1979-05-27T07:32:00Z`}},
{"1979-05-27T00:32:00-07:00", []string{`1979-05-27T00:32:00-07:00`}},
{"1979-05-27T00:32:00.999999-07:00", []string{`1979-05-27T00:32:00.999999-07:00`}},
{"[1,2,3]", []string{`[1, 2, 3]`}},
} {
p := &parser{}
p := newParser()
testParseHandler(t, p, p.startValue, test)
}
}