OpenSourceProjects
/
go-toml
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implemented TOML arrays as a linked list, which prevents a lot of memory copying on large arrays. Also implemented a json comparison, inspired by BurnSushi's version, which makes the tests work when keys in JSON maps are not in the expected order.

This commit is contained in:
Maurice Makaay 2019-06-29 23:11:03 +00:00
parent 7227fdcb93
commit 9680c2b844
10 changed files with 426 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
ast/ast.go
View File

@ -27,6 +27,44 @@ func NewDocument() *Document {
// Table represents a TOML table: a set of key/value pairs.
type Table map[string]*Value
// Array represents a TOML array: a list of values.
type Array struct {
First *ArrayItem
Last *ArrayItem
ItemType ValueType
Length int
}
// ArrayItem represents a single item from a TOML array.
type ArrayItem struct {
Value *Value
Next *ArrayItem
}
// NewArray initializes a new Array.
func NewArray() *Array {
return &Array{}
}
// Append add a new value to an Array. The values must all be of the same type.
// It returns an error when values of different types are added, nil otherwise.
func (a *Array) Append(value *Value) error {
item := &ArrayItem{Value: value}
if a.Length == 0 {
a.ItemType = value.Type
a.First = item
a.Last = item
} else {
if value.Type != a.ItemType {
return fmt.Errorf("type mismatch in array of %ss: found an item of type %s", a.ItemType, value.Type)
}
a.Last.Next = item
a.Last = item
}
a.Length++
return nil
}
// Key represents a TOML table key: one or more strings, where multiple strings
// can be used to represent nested tables.
type Key []string
@ -175,17 +213,17 @@ func (doc *Document) OpenArrayOfTables(key Key) error {
path := doc.formatKeyPath(key, len(key)-1)
return fmt.Errorf("invalid table array: %s value already exists at key %s", existing.Type, path)
}
// A table array exists. Add a new table to this array.
array := node[lastKeyPart]
// An array of tables exists. Add a new table to this array.
array := node[lastKeyPart].Data[0].(*Array)
subTable := make(Table)
tables := array.Data
tables = append(tables, NewValue(TypeTable, subTable))
array.Data = tables
array.Append(NewValue(TypeTable, subTable))
node = subTable
} else {
// No value exists at the defined key path. Create a new table array.
subTable := make(Table)
node[lastKeyPart] = NewValue(TypeArrayOfTables, NewValue(TypeTable, subTable))
array := NewArray()
array.Append(NewValue(TypeTable, subTable))
node[lastKeyPart] = NewValue(TypeArrayOfTables, array)
node = subTable
}
@ -213,9 +251,8 @@ func (doc *Document) makeTablePath(key Key) (Table, string, error) {
// A table was found, traverse to that table.
node = subValue.Data[0].(Table)
} else if subValue.Type == TypeArrayOfTables {
// An array of tables was found, traverse to the last table in the array.
lastValue := subValue.Data[len(subValue.Data)-1].(*Value)
lastTable := lastValue.Data[0].(Table)
// An array of tables was found, use the last table in the array.
lastTable := subValue.Data[0].(*Array).Last.Value.Data[0].(Table)
node = lastTable
} else {
path := doc.formatKeyPath(key, i)

93
ast/ast_test.go
View File

@ -1,6 +1,7 @@
package ast_test
import (
"reflect"
"testing"
"git.makaay.nl/mauricem/go-toml/ast"
@ -17,7 +18,10 @@ func Test_ConstructSlightlyComplexStructure(t *testing.T) {
p.OpenTable(ast.NewKey("key1", "key2 b"))
p.SetKeyValuePair(ast.NewKey("dieh"), ast.NewValue(ast.TypeFloat, 1.111))
p.SetKeyValuePair(ast.NewKey("duh"), ast.NewValue(ast.TypeFloat, 1.18e-12))
p.SetKeyValuePair(ast.NewKey("foo", "bar"), ast.NewValue(ast.TypeArrayOfTables, ast.NewValue(ast.TypeInteger, 1), ast.NewValue(ast.TypeInteger, 2)))
arr := ast.NewArray()
arr.Append(ast.NewValue(ast.TypeInteger, 1))
arr.Append(ast.NewValue(ast.TypeInteger, 2))
p.SetKeyValuePair(ast.NewKey("foo", "bar"), ast.NewValue(ast.TypeArray, arr))
p.OpenArrayOfTables(ast.NewKey("aaah", "table array"))
p.SetKeyValuePair(ast.NewKey("a"), ast.NewValue(ast.TypeFloat, 1.234))
p.OpenArrayOfTables(ast.NewKey("aaah", "table array"))
@ -247,3 +251,90 @@ func testAST(t *testing.T, code func() (error, *ast.Document), expectedError str
t.Fatalf("Unexpected data after parsing:\nexpected: %s\nactual: %s\n", expectedData, p.Root.String())
}
}
func Test_NewArray(t *testing.T) {
a := ast.NewArray()
if a.Type != "" {
t.Fatalf("New array unexpectedly has a Type set to %q", a.Type)
}
if a.First != nil {
t.Fatalf("New array unexpectedly has the First item set")
}
if a.Last != nil {
t.Fatalf("New array unexpectedly has the Last item set")
}
if a.Length != 0 {
t.Fatalf("New array unexpectedly has the Length set to %d", a.Length)
}
}
func Test_AppendOneValueToArray(t *testing.T) {
a := ast.NewArray()
value := ast.NewValue(ast.TypeString, "Hi!")
if err := a.Append(value); err != nil {
t.Fatalf("Unexpected error while adding string value to array: %s", err)
}
if a.Length != 1 {
t.Fatalf("Expected Array.Length 1, but got %d", a.Length)
}
if a.First != a.Last {
t.Fatalf("Array.First and Array.Last do not point to the same ArrayItem")
}
if a.First.Value != value {
t.Fatalf("Array.First.Value does not point to the appended string value")
}
if a.First.Next != nil {
t.Fatalf("Array.First unexpectedly has Next set")
}
}
func Test_AppendTwoValuesToArray(t *testing.T) {
a := ast.NewArray()
value1 := ast.NewValue(ast.TypeString, "Hi!")
value2 := ast.NewValue(ast.TypeString, "Hello!")
a.Append(value1)
if err := a.Append(value2); err != nil {
t.Fatalf("Unexpected error while adding second string value to Array: %s", err)
}
if a.Length != 2 {
t.Fatalf("Array.Length unexpected not 2, but %d", a.Length)
}
if a.First == a.Last {
t.Fatalf("Array.First and Array.Last unexpectedly point to the same ArrayItem")
}
if a.First.Next != a.Last {
t.Fatalf("Array.First.Next unexpectedly does not point to Array.Last")
}
if a.Last.Value != value2 {
t.Fatalf("Array.Last.Value does not point to the second appended string value")
}
}
func Test_AppendMultipleValuesToArray(t *testing.T) {
a := ast.NewArray()
a.Append(ast.NewValue(ast.TypeInteger, 1))
a.Append(ast.NewValue(ast.TypeInteger, 2))
a.Append(ast.NewValue(ast.TypeInteger, 3))
a.Append(ast.NewValue(ast.TypeInteger, 4))
x := make([]int, 0, a.Length)
for i := a.First; i != nil; i = i.Next {
x = append(x, i.Value.Data[0].(int))
}
if !reflect.DeepEqual(x, []int{1, 2, 3, 4}) {
t.Fatalf("Array contents do not match Array [1, 2, 3, 4]")
}
}
func Test_GivenValuesOfDifferentTypes_WhenAppendingThemToArray_AnErrorIsReturned(t *testing.T) {
a := ast.NewArray()
a.Append(ast.NewValue(ast.TypeString, "first one is string"))
err := a.Append(ast.NewValue(ast.TypeInteger, 2)) // second one an integer
if err == nil {
t.Fatalf("Unexpectedly, no error was returned while adding an integer to an array of strings")
}
expected := "type mismatch in array of strings: found an item of type integer"
if err.Error() != expected {
t.Fatalf("Unexpected error from invalid Append():\nexpected: %q\nactual: %q", expected, err.Error())
}
}

14
ast/burntsushi.go
View File

@ -41,17 +41,15 @@ func MakeSushi(value *Value) string {
// as an array of tables, so here we accomodate for that situation
// by checking for that case and render such inline array definition
// as if it were an [[array.of.tables]].
values := make([]string, len(value.Data))
isArrayOfTables := false
for i, value := range value.Data {
isArrayOfTables = value.(*Value).Type == TypeTable
values[i] = MakeSushi(value.(*Value))
arr := value.Data[0].(*Array)
values := make([]string, 0, arr.Length)
for i := arr.First; i != nil; i = i.Next {
values = append(values, MakeSushi(i.Value))
}
if isArrayOfTables {
if arr.ItemType == TypeTable {
return fmt.Sprintf("[%s]", strings.Join(values, ", "))
} else {
return fmt.Sprintf(`{"type": "array", "value": [%s]}`, strings.Join(values, ", "))
}
return fmt.Sprintf(`{"type": "array", "value": [%s]}`, strings.Join(values, ", "))
case TypeImplicitTable:
fallthrough
case TypeTable:

22
ast/string.go
View File

@ -16,29 +16,31 @@ func (t Table) String() string {
// String() produces a JSON-like (but not JSON) string representation of the value.
// This string version is mainly useful for testing and debugging purposes.
func (value Value) String() string {
data := value.Data[0]
switch value.Type {
case TypeString:
return fmt.Sprintf("%q", value.Data[0])
return fmt.Sprintf("%q", data)
case TypeOffsetDateTime:
return value.Data[0].(time.Time).Format(time.RFC3339Nano)
return data.(time.Time).Format(time.RFC3339Nano)
case TypeLocalDateTime:
return value.Data[0].(time.Time).Format("2006-01-02 15:04:05.999999999")
return data.(time.Time).Format("2006-01-02 15:04:05.999999999")
case TypeLocalDate:
return value.Data[0].(time.Time).Format("2006-01-02")
return data.(time.Time).Format("2006-01-02")
case TypeLocalTime:
return value.Data[0].(time.Time).Format("15:04:05.999999999")
return data.(time.Time).Format("15:04:05.999999999")
case TypeArrayOfTables:
fallthrough
case TypeArray:
values := make([]string, len(value.Data))
for i, value := range value.Data {
values[i] = value.(*Value).String()
a := data.(*Array)
values := make([]string, 0, a.Length)
for i := a.First; i != nil; i = i.Next {
values = append(values, i.Value.String())
}
return fmt.Sprintf("[%s]", strings.Join(values, ", "))
case TypeImplicitTable:
fallthrough
case TypeTable:
pairs := value.Data[0].(Table)
pairs := data.(Table)
keys := make([]string, len(pairs))
i := 0
for k := range pairs {
@ -52,6 +54,6 @@ func (value Value) String() string {
}
return fmt.Sprintf("{%s}", strings.Join(values, ", "))
default:
return fmt.Sprintf("%v", value.Data[0])
return fmt.Sprintf("%v", data)
}
}

5
ast/string_test.go
View File

@ -18,7 +18,10 @@ func Test_StringFormatting(t *testing.T) {
doc.SetKeyValuePair(ast.NewKey("b"), ast.NewValue(ast.TypeFloat, 2.3))
doc.SetKeyValuePair(ast.NewKey("c"), ast.NewValue(ast.TypeBool, true))
doc.SetKeyValuePair(ast.NewKey("d"), ast.NewValue(ast.TypeString, "foo"))
doc.SetKeyValuePair(ast.NewKey("e"), ast.NewValue(ast.TypeArray, ast.NewValue(ast.TypeInteger, 1), ast.NewValue(ast.TypeInteger, 2)))
arr := ast.NewArray()
arr.Append(ast.NewValue(ast.TypeInteger, 1))
arr.Append(ast.NewValue(ast.TypeInteger, 2))
doc.SetKeyValuePair(ast.NewKey("e"), ast.NewValue(ast.TypeArray, arr))
doc.SetKeyValuePair(ast.NewKey("f"), ast.NewValue(ast.TypeTable, tableData))
doc.SetKeyValuePair(ast.NewKey("g"), ast.NewValue(ast.TypeOffsetDateTime, dateTime))
doc.SetKeyValuePair(ast.NewKey("h"), ast.NewValue(ast.TypeLocalDateTime, dateTime))

BIN
cmd/burntsushi-tester/burntsushi-tester
View File

Binary file not shown.

78
parse/testfiles/valid/mmakaay/all_floats.json Normal file
View File

@ -0,0 +1,78 @@
{
"inf1" : {
"type" : "float",
"value" : "+Inf"
},
"inf2" : {
"type" : "float",
"value" : "+Inf"
},
"inf3" : {
"type" : "float",
"value" : "-Inf"
},
"nan1" : {
"type" : "float",
"value" : "NaN"
},
"nan2" : {
"type" : "float",
"value" : "NaN"
},
"nan3" : {
"type" : "float",
"value" : "NaN"
},
"negpi" : {
"type" : "float",
"value" : "-3.14"
},
"negpi2" : {
"type" : "float",
"value" : "-3.14"
},
"negpi3" : {
"type" : "float",
"value" : "-3.14"
},
"negzero" : {
"type" : "float",
"value" : "-0"
},
"pi" : {
"type" : "float",
"value" : "3.14"
},
"pi2" : {
"type" : "float",
"value" : "31.4"
},
"pi3" : {
"type" : "float",
"value" : "3.14"
},
"pospi" : {
"type" : "float",
"value" : "3.14"
},
"pospi2" : {
"type" : "float",
"value" : "3.14"
},
"pospi3" : {
"type" : "float",
"value" : "3.14"
},
"poszero" : {
"type" : "float",
"value" : "0"
},
"zero" : {
"type" : "float",
"value" : "0"
},
"zero-intpart" : {
"type" : "float",
"value" : "0.123"
}
}

19
parse/testfiles/valid/mmakaay/all_floats.toml Normal file
View File

@ -0,0 +1,19 @@
pi = 3.14
pospi = +3.14
negpi = -3.14
zero-intpart = 0.123
pi2 = 314e-1
pospi2 = +314e-2
negpi2 = -314e-2
pi3 = 31.4e-1
pospi3 = +31.4e-1
negpi3 = -31.4e-1
nan1 = nan
nan2 = nan
nan3 = nan
inf1 = +inf
inf2 = +inf
inf3 = -inf
zero = 0.0
poszero = +0.0
negzero = -0.0

172
parse/testfiles_test.go
View File

@ -5,11 +5,13 @@ import (
"fmt"
"io/ioutil"
"log"
"math"
"os"
"reflect"
"runtime"
"strconv"
"strings"
"testing"
"time"
"git.makaay.nl/mauricem/go-toml/parse"
)
@ -67,8 +69,8 @@ func Test_Valid(t *testing.T) {
fail++
continue
}
if !reflect.DeepEqual(expected, actual) {
t.Errorf("[%s] Expected result does not match the actual result", name)
if err := cmpJson("", expected, actual); err != nil {
t.Errorf("[%s] Expected result does not match the actual result: %s", name, err)
fail++
continue
}
@ -79,20 +81,20 @@ func Test_Valid(t *testing.T) {
}
type testSuite struct {
testType string
name string
dir string
tesactualType string
name string
dir string
}
func getTestSuites(testType string) []testSuite {
dir := getTestfilesDir() + "/" + testType
func getTestSuites(tesactualType string) []testSuite {
dir := getTestfilesDir() + "/" + tesactualType
entries, err := ioutil.ReadDir(dir)
if err != nil {
log.Fatalf("Cannot read directory (%s): %s", dir, err)
}
suites := make([]testSuite, len(entries))
for i, ent := range entries {
suites[i] = testSuite{testType, ent.Name(), dir + "/" + ent.Name()}
suites[i] = testSuite{tesactualType, ent.Name(), dir + "/" + ent.Name()}
}
return suites
}
@ -116,7 +118,7 @@ func getTestCasesForSuite(suite testSuite) map[string]*testCase {
for _, ent := range entries {
name := ent.Name()
id := name[0 : len(name)-5]
key := suite.testType + "/" + suite.name + "/" + id
key := suite.tesactualType + "/" + suite.name + "/" + id
c, ok := testCases[key]
if !ok {
c = &testCase{}
@ -133,3 +135,153 @@ func getTestCasesForSuite(suite testSuite) map[string]*testCase {
}
return testCases
}
// ---------------------------------------------------------------------------
// Inspired by https://github.com/BurntSushi/toml-test/blob/master/json.go
// I started out by using reflect.DeepEqual, but that solution does not
// take possibly different ordering of object keys into account.
// I do a lot less active checking on type assertions here, since this
// is only used for my local test sets and local parser code. I can live
// with a panic when I crap up my test data.
// ---------------------------------------------------------------------------
func cmpJson(key string, expected interface{}, actual interface{}) error {
switch expected := expected.(type) {
case map[string]interface{}:
return cmpJsonObjects(key, expected, actual)
case []interface{}:
return cmpJsonArrays(key, expected, actual)
default:
return fmt.Errorf("Key '%s' in expected output should be a map or a list of maps, but it's a %T.", key, expected)
}
}
func cmpJsonObjects(key string, expected map[string]interface{}, a interface{}) error {
actual, _ := a.(map[string]interface{})
// Check to make sure both or neither are values.
if isValue(expected) && !isValue(actual) {
return fmt.Errorf("Key '%s' is supposed to be a value, but the parser reports it as a table.", key)
}
if !isValue(expected) && isValue(actual) {
return fmt.Errorf("Key '%s' is supposed to be a table, but the parser reports it as a value.", key)
}
if isValue(expected) && isValue(actual) {
return cmpJsonValues(key, expected, actual)
}
// We've got two maps. Check if the keys are equivalent (order does not match).
for k, _ := range expected {
if _, ok := actual[k]; !ok {
return fmt.Errorf("Could not find key '%s.%s' in parser output.", key, k)
}
}
for k, _ := range actual {
if _, ok := expected[k]; !ok {
return fmt.Errorf("Could not find key '%s.%s' expected output.", key, k)
}
}
// Check if the values are equivalent.
for k, _ := range expected {
subKey := k
if key != "" {
subKey = key + "." + k
}
if err := cmpJson(subKey, expected[k], actual[k]); err != nil {
return err
}
}
return nil
}
func cmpJsonArrays(key string, expected, actual interface{}) error {
expectedArray, _ := expected.([]interface{})
actualArray, _ := actual.([]interface{})
if len(expectedArray) != len(actualArray) {
return fmt.Errorf("Array lengths differ for key '%s'. Expected a length of %d but got %d.",
key, len(expectedArray), len(actualArray))
}
for i := 0; i < len(expectedArray); i++ {
if err := cmpJson(key, expectedArray[i], actualArray[i]); err != nil {
return err
}
}
return nil
}
func cmpJsonValues(key string, expected, actual map[string]interface{}) error {
expectedType, _ := expected["type"].(string)
actualType, _ := actual["type"].(string)
if expectedType != actualType {
return fmt.Errorf("Type mismatch for key '%s'. Expected %s but got %s.", key, expectedType, actualType)
}
if expectedType == "array" {
return cmpJsonArrays(key, expected["value"], actual["value"])
}
// Except for floats and datetimes, other values can be compared as strings.
expectedValue, _ := expected["value"].(string)
actualValue, _ := actual["value"].(string)
switch expectedType {
case "float":
return cmpFloats(key, expectedValue, actualValue)
case "datetime":
return cmpAsDatetimes(key, expectedValue, actualValue)
default:
return cmpAsStrings(key, expectedValue, actualValue)
}
}
func cmpAsStrings(key, expected, actual string) error {
if expected != actual {
return fmt.Errorf("Values for key '%s' don't match. Expected a value of %q but got %q.", key, expected, actual)
}
return nil
}
func cmpFloats(key, expected, actual string) error {
expectedFloat, err := strconv.ParseFloat(expected, 64)
if err != nil {
return fmt.Errorf("BUG in test case. Could not read '%s' as a float value for key '%s'.", expected, key)
}
actualFloat, err := strconv.ParseFloat(actual, 64)
if err != nil {
return fmt.Errorf("Malformed parser output. Could not read '%s' as a float value for key '%s'.", actual, key)
}
if expectedFloat != actualFloat && !(math.IsNaN(expectedFloat) && math.IsNaN(actualFloat)) {
return fmt.Errorf("Values for key '%s' don't match. Expected a value of %v but got %v.", key, expectedFloat, actualFloat)
}
return nil
}
func cmpAsDatetimes(key, expected, actual string) error {
expectedTime, err := time.Parse(time.RFC3339Nano, expected)
if err != nil {
return fmt.Errorf("BUG in test case. Could not read '%s' as a datetime value for key '%s'.", expected, key)
}
actualTime, err := time.Parse(time.RFC3339Nano, actual)
if err != nil {
return fmt.Errorf("Malformed parser output. Could not read '%s' as datetime value for key '%s'.", actual, key)
}
if !expectedTime.Equal(actualTime) {
return fmt.Errorf("Values for key '%s' don't match. Expected a value of '%v' but got '%v'.", key, expectedTime, actualTime)
}
return nil
}
func isValue(m map[string]interface{}) bool {
if len(m) != 2 {
return false
}
if _, ok := m["type"]; !ok {
return false
}
if _, ok := m["value"]; !ok {
return false
}
return true
}

14
parse/value_array.go
View File

@ -43,13 +43,14 @@ func (t *parser) parseArray(p *parse.API) (*ast.Value, bool) {
return nil, false
}
a := ast.NewArray()
// Check for an empty array.
if p.Accept(arrayClose) {
return ast.NewValue(ast.TypeArray), true
return ast.NewValue(ast.TypeArray, a), true
}
// Not an empty array, parse the array values.
values := []interface{}{}
for {
// Check for a value item.
value, ok := t.parseValue(p)
@ -59,16 +60,15 @@ func (t *parser) parseArray(p *parse.API) (*ast.Value, bool) {
// Data types may not be mixed (different ways to define strings should be
// considered the same type, and so should arrays with different element types).
if len(values) > 0 && value.Type != values[0].(*ast.Value).Type {
p.Error("type mismatch in array of %ss: found an item of type %s", values[0].(*ast.Value).Type, value.Type)
// This constraint is checked by ast.Array.Append().
if err := a.Append(value); err != nil {
p.Error(err.Error())
return nil, false
}
values = append(values, value)
// Check for the end of the array.
if p.Accept(arrayClose) {
return ast.NewValue(ast.TypeArray, values...), true
return ast.NewValue(ast.TypeArray, a), true
}
// Not the end of the array? Then we should find an array separator.