Implemented the TOML number formats (integer, binary, octal, hexadecimal.

helpers_test.go

@@ -20,7 +20,9 @@ type parseTest struct {
 }
 
 func testParseHandler(t *testing.T, p *parser, handler parse.Handler, test parseTest) {
-	err := parse.New(handler)(test.input)
+	var err error
+	defer func() {
+		recovered := recover()
 		results := []string{}
 		for _, cmd := range p.commands {
 			results = append(results, cmd.String())
@@ -28,6 +30,9 @@ func testParseHandler(t *testing.T, p *parser, handler parse.Handler, test parse
 		if err != nil {
 			results = append(results, fmt.Sprintf("Error: %s", err))
 		}
+		if recovered != nil {
+			results = append(results, fmt.Sprintf("Panic: %s", recovered.(string)))
+		}
 
 		for i, e := range test.expected {
 			if i > len(results)-1 {
@@ -40,9 +45,11 @@ func testParseHandler(t *testing.T, p *parser, handler parse.Handler, test parse
 			}
 		}
 		if len(results) > len(test.expected) {
-		t.Errorf("Got more results than expected, surplus result(s):\n")
+			t.Errorf("Got more results than expected for input %q, surplus result(s):\n", test.input)
 			for i := len(test.expected); i < len(results); i++ {
 				t.Errorf("[%d] %s", i, results[i])
 			}
 		}
+	}()
+	err = parse.New(handler)(test.input)
 }

keyvaluepair.go

@@ -65,24 +65,22 @@ func (t *parser) startKeyValuePair(p *parse.API) {
 // A bare key must be non-empty, but an empty quoted key is allowed (though
 // discouraged).
 func (t *parser) startKey(p *parse.API) {
-	endFunc := func(str string) {
+	var key string
-		t.emitCommand(cKey, str)
+	var ok bool
-		p.Handle(t.endOfKeyOrDot)
-	}
-
 	switch {
 	case p.Accept(bareKey):
-		endFunc(p.Result().String())
+		key, ok = p.Result().String(), true
 	case p.Peek(a.SingleQuote):
-		if str, ok := t.parseLiteralString("key", p); ok {
+		key, ok = t.parseLiteralString("key", p)
-			endFunc(str)
-		}
 	case p.Peek(a.DoubleQuote):
-		if str, ok := t.parseBasicString("key", p); ok {
+		key, ok = t.parseBasicString("key", p)
-			endFunc(str)
-		}
 	default:
 		p.Expected("a key name")
+		return
+	}
+	if ok {
+		t.emitCommand(cKey, key)
+		p.Handle(t.endOfKeyOrDot)
 	}
 }
 
@@ -92,7 +90,7 @@ func (t *parser) startKey(p *parse.API) {
 // practice is to not use any extraneous whitespace.
 func (t *parser) endOfKeyOrDot(p *parse.API) {
 	if p.Accept(keySeparatorDot) {
-		t.emitCommand(cNewKeyLvl)
+		t.emitCommand(cKeyDot)
 		p.Handle(t.startKey)
 	}
 }

toml.go

@@ -16,9 +16,10 @@ type cmdType string
 const (
 	cComment   cmdType = "comment" // a # comment at the end of the line
 	cKey               = "key"     // set key name
-	cNewKeyLvl         = "keydot"  // new key stack level
+	cKeyDot            = "keydot"  // new key stack level
 	cAssign            = "assign"  // assign a value
 	csetStrVal         = "string"  // set a string value
+	csetIntVal         = "integer" // set an integer value
 )
 
 type parser struct {
@@ -37,7 +38,12 @@ func (cmd *cmd) String() string {
 	}
 	args := make([]string, len(cmd.args))
 	for i, arg := range cmd.args {
+		switch arg.(type) {
+		case string:
 			args[i] = fmt.Sprintf("%q", arg)
+		default:
+			args[i] = fmt.Sprintf("%v", arg)
+		}
 	}
 	return fmt.Sprintf("%s(%s)", cmd.command, strings.Join(args, ", "))
 }

value_number.go

@@ -0,0 +1,76 @@
+package parser
+
+import (
+	"strconv"
+
+	"git.makaay.nl/mauricem/go-parsekit/parse"
+)
+
+var (
+	// Integers are whole numbers. Positive numbers may be prefixed with a
+	// plus sign. Negative numbers are prefixed with a minus sign. For large
+	// numbers, you may use underscores between digits to enhance readability.
+	// Each underscore must be surrounded by at least one digit on each side.
+	// Leading zeros are not allowed.
+	integerPrefix    = a.Signed(a.DigitNotZero.Then(a.Digits.Optional()))
+	underscoreDigits = m.Drop(a.Underscore).Then(a.Digits)
+	integerSuffix    = c.ZeroOrMore(underscoreDigits)
+	integer          = integerPrefix.Then(integerSuffix)
+
+	// Integer values -0 and +0 are valid and identical to an unprefixed zero.
+	zero      = a.Rune('0')
+	plusZero  = a.Plus.Then(zero)
+	minusZero = a.Minus.Then(zero)
+
+	// Non-negative integer values may also be expressed in hexadecimal, octal,
+	// or binary. In these formats, leading + is not allowed and leading zeros
+	// are allowed (after the prefix). Hex values are case insensitive.
+	// Underscores are allowed between digits (but not between the prefix
+	// and the value).
+	// Hexadecimal with prefix `0x`.
+	hexDigits           = c.OneOrMore(a.HexDigit)
+	underscoreHexDigits = m.Drop(a.Underscore).Then(hexDigits)
+	hexadecimal         = a.Rune('x').Then(tok.Str("x", hexDigits.Then(c.ZeroOrMore(underscoreHexDigits))))
+	// Octal with prefix `0o`.
+	octalDigits           = c.OneOrMore(a.RuneRange('0', '7'))
+	underscoreOctalDigits = m.Drop(a.Underscore).Then(octalDigits)
+	octal                 = a.Rune('o').Then(tok.Str("o", octalDigits.Then(c.ZeroOrMore(underscoreOctalDigits))))
+	// Binary with prefix `0b`.
+	binaryDigits           = c.OneOrMore(a.RuneRange('0', '1'))
+	underscoreBinaryDigits = m.Drop(a.Underscore).Then(binaryDigits)
+	binary                 = a.Rune('b').Then(tok.Str("b", binaryDigits.Then(c.ZeroOrMore(underscoreBinaryDigits))))
+)
+
+func (t *parser) startInteger(p *parse.API) {
+	switch {
+	case p.Accept(zero):
+		p.Handle(t.startIntegerStartingWithZero)
+	case p.Accept(plusZero.Or(minusZero)):
+		t.emitCommand(csetIntVal, int64(0))
+	case p.Accept(tok.Int64(nil, integer)):
+		t.emitCommand(csetIntVal, p.Result().Value(0).(int64))
+	default:
+		p.Expected("an integer value")
+	}
+}
+
+func (t *parser) startIntegerStartingWithZero(p *parse.API) {
+	var value int64
+	var err error
+	switch {
+	case p.Accept(hexadecimal):
+		value, err = strconv.ParseInt(p.Result().Value(0).(string), 16, 64)
+	case p.Accept(octal):
+		value, err = strconv.ParseInt(p.Result().Value(0).(string), 8, 64)
+	case p.Accept(binary):
+		value, err = strconv.ParseInt(p.Result().Value(0).(string), 2, 64)
+	default:
+		t.emitCommand(csetIntVal, int64(0))
+		return
+	}
+	if err == nil {
+		t.emitCommand(csetIntVal, value)
+	} else {
+		p.Error("Cannot parse value 0%s: %s", p.Result().String(), err)
+	}
+}

value_number_test.go

@@ -0,0 +1,80 @@
+package parser
+
+import (
+	"testing"
+)
+
+func TestInteger(t *testing.T) {
+	for _, test := range []parseTest{
+		{``, []string{`Error: unexpected end of file (expected an integer value) at start of file`}},
+		// Decimal
+		{`0`, []string{`integer(0)`}},
+		{`+0`, []string{`integer(0)`}},
+		{`-0`, []string{`integer(0)`}},
+		{`1`, []string{`integer(1)`}},
+		{`42`, []string{`integer(42)`}},
+		{`+99`, []string{`integer(99)`}},
+		{`-17`, []string{`integer(-17)`}},
+		{`1234`, []string{`integer(1234)`}},
+		{`_`, []string{`Error: unexpected input (expected an integer value) at start of file`}},
+		{`1_`, []string{`integer(1)`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
+		{`1_000`, []string{`integer(1000)`}},
+		{`5_349_221`, []string{`integer(5349221)`}},
+		{`1_2_3_4_5`, []string{`integer(12345)`}},
+		{`9_223_372_036_854_775_807`, []string{`integer(9223372036854775807)`}},
+		{`9_223_372_036_854_775_808`, []string{
+			`Panic: Handler error: MakeInt64Token cannot handle input "9223372036854775808": ` +
+				`strconv.ParseInt: parsing "9223372036854775808": value out of range (only use a ` +
+				`type conversion token maker, when the input has been validated on beforehand)`}},
+		{`-9_223_372_036_854_775_808`, []string{`integer(-9223372036854775808)`}},
+		// TODO make the use of the same kind of handling for panics and for errors between parsekit and TOML.
+		{`-9_223_372_036_854_775_809`, []string{
+			`Panic: Handler error: MakeInt64Token cannot handle input "-9223372036854775809": ` +
+				`strconv.ParseInt: parsing "-9223372036854775809": value out of range (only use a ` +
+				`type conversion token maker, when the input has been validated on beforehand)`}},
+		// Hexadecimal
+		{`0x0`, []string{`integer(0)`}},
+		{`0x1`, []string{`integer(1)`}},
+		{`0x01`, []string{`integer(1)`}},
+		{`0x00fF`, []string{`integer(255)`}},
+		{`0xf_f`, []string{`integer(255)`}},
+		{`0x0_0_f_f`, []string{`integer(255)`}},
+		{`0xdead_beef`, []string{`integer(3735928559)`}},
+		{`0xgood_beef`, []string{`integer(0)`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
+		{`0x7FFFFFFFFFFFFFFF`, []string{`integer(9223372036854775807)`}},
+		{`0x8000000000000000`, []string{
+			`Error: Cannot parse value 0x8000000000000000: strconv.ParseInt: parsing "8000000000000000": ` +
+				`value out of range at line 1, column 19`}},
+		//Octal
+		{`0o0`, []string{`integer(0)`}},
+		{`0o1`, []string{`integer(1)`}},
+		{`0o01`, []string{`integer(1)`}},
+		{`0o10`, []string{`integer(8)`}},
+		{`0o1_6`, []string{`integer(14)`}},
+		{`0o0_0_1_1_1`, []string{`integer(73)`}},
+		{`0o9`, []string{`integer(0)`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
+		{`0o777777777777777777777`, []string{`integer(9223372036854775807)`}},
+		{`0o1000000000000000000000`, []string{
+			`Error: Cannot parse value 0o1000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000": ` +
+				`value out of range at line 1, column 25`}},
+		// Binary
+		{`0b0`, []string{`integer(0)`}},
+		{`0b1`, []string{`integer(1)`}},
+		{`0b01`, []string{`integer(1)`}},
+		{`0b10`, []string{`integer(2)`}},
+		{`0b0100`, []string{`integer(4)`}},
+		{`0b00001000`, []string{`integer(8)`}},
+		{`0b0001_0000`, []string{`integer(16)`}},
+		{`0b9`, []string{`integer(0)`, `Error: unexpected input (expected end of file) at line 1, column 2`}},
+		{`0b1_1_0_1_1`, []string{`integer(27)`}},
+		{`0b11111111_11111111`, []string{`integer(65535)`}},
+		{`0b01111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111`, []string{`integer(9223372036854775807)`}},
+		{`0b10000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000`, []string{
+			`Error: Cannot parse value 0b1000000000000000000000000000000000000000000000000000000000000000: ` +
+				`strconv.ParseInt: parsing "1000000000000000000000000000000000000000000000000000000000000000": ` +
+				`value out of range at line 1, column 74`}},
+	} {
+		p := &parser{}
+		testParseHandler(t, p, p.startInteger, test)
+	}
+}