From e8739d38eabc9b3552f822fced4229be92d34753 Mon Sep 17 00:00:00 2001 From: Maurice Makaay Date: Tue, 18 Jun 2019 15:45:33 +0000 Subject: [PATCH] Implemented the TOML number formats (integer, binary, octal, hexadecimal. --- helpers_test.go | 51 ++++++++++++++++------------ keyvaluepair.go | 26 +++++++------- toml.go | 10 ++++-- value_number.go | 76 +++++++++++++++++++++++++++++++++++++++++ value_number_test.go | 80 ++++++++++++++++++++++++++++++++++++++++++++ value_string_test.go | 2 +- 6 files changed, 206 insertions(+), 39 deletions(-) create mode 100644 value_number.go create mode 100644 value_number_test.go diff --git a/helpers_test.go b/helpers_test.go index 6d53cbb..6a3db51 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -20,29 +20,36 @@ type parseTest struct { } func testParseHandler(t *testing.T, p *parser, handler parse.Handler, test parseTest) { - err := parse.New(handler)(test.input) - results := []string{} - for _, cmd := range p.commands { - results = append(results, cmd.String()) - } - if err != nil { - results = append(results, fmt.Sprintf("Error: %s", err)) - } + var err error + defer func() { + recovered := recover() + results := []string{} + for _, cmd := range p.commands { + results = append(results, cmd.String()) + } + if err != nil { + results = append(results, fmt.Sprintf("Error: %s", err)) + } + if recovered != nil { + results = append(results, fmt.Sprintf("Panic: %s", recovered.(string))) + } - for i, e := range test.expected { - if i > len(results)-1 { - t.Errorf("No result at index %d, expected: %s", i, e) - continue + for i, e := range test.expected { + if i > len(results)-1 { + t.Errorf("No result at index %d, expected: %s", i, e) + continue + } + r := results[i] + if e != r { + t.Errorf("Unexpected result at index %d:\nexpected: %s\nactual: %s\n", i, e, r) + } } - r := results[i] - if e != r { - t.Errorf("Unexpected result at index %d:\nexpected: %s\nactual: %s\n", i, e, r) + if len(results) > len(test.expected) { + t.Errorf("Got more results than expected for input %q, surplus result(s):\n", test.input) + for i := len(test.expected); i < len(results); i++ { + t.Errorf("[%d] %s", i, results[i]) + } } - } - if len(results) > len(test.expected) { - t.Errorf("Got more results than expected, surplus result(s):\n") - for i := len(test.expected); i < len(results); i++ { - t.Errorf("[%d] %s", i, results[i]) - } - } + }() + err = parse.New(handler)(test.input) } diff --git a/keyvaluepair.go b/keyvaluepair.go index 0c9b1ab..14550ab 100644 --- a/keyvaluepair.go +++ b/keyvaluepair.go @@ -57,7 +57,7 @@ func (t *parser) startKeyValuePair(p *parse.API) { // Bare keys may only contain ASCII letters, ASCII digits, underscores, and // dashes (A-Za-z0-9_-). Note that bare keys are allowed to be composed of only -//ASCII digits, e.g. 1234, but are always interpreted as strings. +// ASCII digits, e.g. 1234, but are always interpreted as strings. // // Quoted keys follow the exact same rules as either basic strings or literal // strings and allow you to use a much broader set of key names. Best practice @@ -65,24 +65,22 @@ func (t *parser) startKeyValuePair(p *parse.API) { // A bare key must be non-empty, but an empty quoted key is allowed (though // discouraged). func (t *parser) startKey(p *parse.API) { - endFunc := func(str string) { - t.emitCommand(cKey, str) - p.Handle(t.endOfKeyOrDot) - } - + var key string + var ok bool switch { case p.Accept(bareKey): - endFunc(p.Result().String()) + key, ok = p.Result().String(), true case p.Peek(a.SingleQuote): - if str, ok := t.parseLiteralString("key", p); ok { - endFunc(str) - } + key, ok = t.parseLiteralString("key", p) case p.Peek(a.DoubleQuote): - if str, ok := t.parseBasicString("key", p); ok { - endFunc(str) - } + key, ok = t.parseBasicString("key", p) default: p.Expected("a key name") + return + } + if ok { + t.emitCommand(cKey, key) + p.Handle(t.endOfKeyOrDot) } } @@ -92,7 +90,7 @@ func (t *parser) startKey(p *parse.API) { // practice is to not use any extraneous whitespace. func (t *parser) endOfKeyOrDot(p *parse.API) { if p.Accept(keySeparatorDot) { - t.emitCommand(cNewKeyLvl) + t.emitCommand(cKeyDot) p.Handle(t.startKey) } } diff --git a/toml.go b/toml.go index e792a77..00f3f52 100644 --- a/toml.go +++ b/toml.go @@ -16,9 +16,10 @@ type cmdType string const ( cComment cmdType = "comment" // a # comment at the end of the line cKey = "key" // set key name - cNewKeyLvl = "keydot" // new key stack level + cKeyDot = "keydot" // new key stack level cAssign = "assign" // assign a value csetStrVal = "string" // set a string value + csetIntVal = "integer" // set an integer value ) type parser struct { @@ -37,7 +38,12 @@ func (cmd *cmd) String() string { } args := make([]string, len(cmd.args)) for i, arg := range cmd.args { - args[i] = fmt.Sprintf("%q", arg) + switch arg.(type) { + case string: + args[i] = fmt.Sprintf("%q", arg) + default: + args[i] = fmt.Sprintf("%v", arg) + } } return fmt.Sprintf("%s(%s)", cmd.command, strings.Join(args, ", ")) } diff --git a/value_number.go b/value_number.go new file mode 100644 index 0000000..2249274 --- /dev/null +++ b/value_number.go @@ -0,0 +1,76 @@ +package parser + +import ( + "strconv" + + "git.makaay.nl/mauricem/go-parsekit/parse" +) + +var ( + // Integers are whole numbers. Positive numbers may be prefixed with a + // plus sign. Negative numbers are prefixed with a minus sign. For large + // numbers, you may use underscores between digits to enhance readability. + // Each underscore must be surrounded by at least one digit on each side. + // Leading zeros are not allowed. + integerPrefix = a.Signed(a.DigitNotZero.Then(a.Digits.Optional())) + underscoreDigits = m.Drop(a.Underscore).Then(a.Digits) + integerSuffix = c.ZeroOrMore(underscoreDigits) + integer = integerPrefix.Then(integerSuffix) + + // Integer values -0 and +0 are valid and identical to an unprefixed zero. + zero = a.Rune('0') + plusZero = a.Plus.Then(zero) + minusZero = a.Minus.Then(zero) + + // Non-negative integer values may also be expressed in hexadecimal, octal, + // or binary. In these formats, leading + is not allowed and leading zeros + // are allowed (after the prefix). Hex values are case insensitive. + // Underscores are allowed between digits (but not between the prefix + // and the value). + // Hexadecimal with prefix `0x`. + hexDigits = c.OneOrMore(a.HexDigit) + underscoreHexDigits = m.Drop(a.Underscore).Then(hexDigits) + hexadecimal = a.Rune('x').Then(tok.Str("x", hexDigits.Then(c.ZeroOrMore(underscoreHexDigits)))) + // Octal with prefix `0o`. + octalDigits = c.OneOrMore(a.RuneRange('0', '7')) + underscoreOctalDigits = m.Drop(a.Underscore).Then(octalDigits) + octal = a.Rune('o').Then(tok.Str("o", octalDigits.Then(c.ZeroOrMore(underscoreOctalDigits)))) + // Binary with prefix `0b`. + binaryDigits = c.OneOrMore(a.RuneRange('0', '1')) + underscoreBinaryDigits = m.Drop(a.Underscore).Then(binaryDigits) + binary = a.Rune('b').Then(tok.Str("b", binaryDigits.Then(c.ZeroOrMore(underscoreBinaryDigits)))) +) + +func (t *parser) startInteger(p *parse.API) { + switch { + case p.Accept(zero): + p.Handle(t.startIntegerStartingWithZero) + case p.Accept(plusZero.Or(minusZero)): + t.emitCommand(csetIntVal, int64(0)) + case p.Accept(tok.Int64(nil, integer)): + t.emitCommand(csetIntVal, p.Result().Value(0).(int64)) + default: + p.Expected("an integer value") + } +} + +func (t *parser) startIntegerStartingWithZero(p *parse.API) { + var value int64 + var err error + switch { + case p.Accept(hexadecimal): + value, err = strconv.ParseInt(p.Result().Value(0).(string), 16, 64) + case p.Accept(octal): + value, err = strconv.ParseInt(p.Result().Value(0).(string), 8, 64) + case p.Accept(binary): + value, err = strconv.ParseInt(p.Result().Value(0).(string), 2, 64) + default: + t.emitCommand(csetIntVal, int64(0)) + return + } + if err == nil { + t.emitCommand(csetIntVal, value) + } else { + p.Error("Cannot parse value 0%s: %s", p.Result().String(), err) + } +} diff --git a/value_number_test.go b/value_number_test.go new file mode 100644 index 0000000..34ac515 --- /dev/null +++ b/value_number_test.go @@ -0,0 +1,80 @@ +package parser + +import ( + "testing" +) + +func TestInteger(t *testing.T) { + for _, test := range []parseTest{ + {``, []string{`Error: unexpected end of file (expected an integer value) at start of file`}}, + // Decimal + {`0`, []string{`integer(0)`}}, + {`+0`, []string{`integer(0)`}}, + {`-0`, []string{`integer(0)`}}, + {`1`, []string{`integer(1)`}}, + {`42`, []string{`integer(42)`}}, + {`+99`, []string{`integer(99)`}}, + {`-17`, []string{`integer(-17)`}}, + {`1234`, []string{`integer(1234)`}}, + {`_`, []string{`Error: unexpected input (expected an integer value) at start of file`}}, + {`1_`, []string{`integer(1)`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, + {`1_000`, []string{`integer(1000)`}}, + {`5_349_221`, []string{`integer(5349221)`}}, + {`1_2_3_4_5`, []string{`integer(12345)`}}, + {`9_223_372_036_854_775_807`, []string{`integer(9223372036854775807)`}}, + {`9_223_372_036_854_775_808`, []string{ + `Panic: Handler error: MakeInt64Token cannot handle input "9223372036854775808": ` + + `strconv.ParseInt: parsing "9223372036854775808": value out of range (only use a ` + + `type conversion token maker, when the input has been validated on beforehand)`}}, + {`-9_223_372_036_854_775_808`, []string{`integer(-9223372036854775808)`}}, + // TODO make the use of the same kind of handling for panics and for errors between parsekit and TOML. + {`-9_223_372_036_854_775_809`, []string{ + `Panic: Handler error: MakeInt64Token cannot handle input "-9223372036854775809": ` + + `strconv.ParseInt: parsing "-9223372036854775809": value out of range (only use a ` + + `type conversion token maker, when the input has been validated on beforehand)`}}, + // Hexadecimal + {`0x0`, []string{`integer(0)`}}, + {`0x1`, []string{`integer(1)`}}, + {`0x01`, []string{`integer(1)`}}, + {`0x00fF`, []string{`integer(255)`}}, + {`0xf_f`, []string{`integer(255)`}}, + {`0x0_0_f_f`, []string{`integer(255)`}}, + {`0xdead_beef`, []string{`integer(3735928559)`}}, + {`0xgood_beef`, []string{`integer(0)`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, + {`0x7FFFFFFFFFFFFFFF`, []string{`integer(9223372036854775807)`}}, + {`0x8000000000000000`, []string{ + `Error: Cannot parse value 0x8000000000000000: strconv.ParseInt: parsing "8000000000000000": ` + + `value out of range at line 1, column 19`}}, + //Octal + {`0o0`, []string{`integer(0)`}}, + {`0o1`, []string{`integer(1)`}}, + {`0o01`, []string{`integer(1)`}}, + {`0o10`, []string{`integer(8)`}}, + {`0o1_6`, []string{`integer(14)`}}, + {`0o0_0_1_1_1`, []string{`integer(73)`}}, + {`0o9`, []string{`integer(0)`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, + {`0o777777777777777777777`, []string{`integer(9223372036854775807)`}}, + {`0o1000000000000000000000`, []string{ + `Error: Cannot parse value 0o1000000000000000000000: strconv.ParseInt: parsing "1000000000000000000000": ` + + `value out of range at line 1, column 25`}}, + // Binary + {`0b0`, []string{`integer(0)`}}, + {`0b1`, []string{`integer(1)`}}, + {`0b01`, []string{`integer(1)`}}, + {`0b10`, []string{`integer(2)`}}, + {`0b0100`, []string{`integer(4)`}}, + {`0b00001000`, []string{`integer(8)`}}, + {`0b0001_0000`, []string{`integer(16)`}}, + {`0b9`, []string{`integer(0)`, `Error: unexpected input (expected end of file) at line 1, column 2`}}, + {`0b1_1_0_1_1`, []string{`integer(27)`}}, + {`0b11111111_11111111`, []string{`integer(65535)`}}, + {`0b01111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111`, []string{`integer(9223372036854775807)`}}, + {`0b10000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000`, []string{ + `Error: Cannot parse value 0b1000000000000000000000000000000000000000000000000000000000000000: ` + + `strconv.ParseInt: parsing "1000000000000000000000000000000000000000000000000000000000000000": ` + + `value out of range at line 1, column 74`}}, + } { + p := &parser{} + testParseHandler(t, p, p.startInteger, test) + } +} diff --git a/value_string_test.go b/value_string_test.go index e6b226f..e4c6323 100644 --- a/value_string_test.go +++ b/value_string_test.go @@ -48,7 +48,7 @@ func TestMultiLineBasicString(t *testing.T) { {"\"\"\"\n\"\"\"", []string{`string("")`}}, {"\"\"\"\r\n\r\n\"\"\"", []string{`string("\n")`}}, {`"""\"\"\"\""""`, []string{`string("\"\"\"\"")`}}, - {"\"\"\"\nThe quick brown \\\n\n\n \t fox jumps over \\\n\t the lazy dog.\\\n \"\"\"", []string{`string("The quick brown fox jumps over the lazy dog.")`}}, + {"\"\"\"\nThe quick brown \\\n\n\n \t fox jumps over \\\n\t the lazy dog.\\\n \"\"\"", []string{`string("The quick brown fox jumps over the lazy dog.")`}}, {"\"\"\"No control chars \f allowed\"\"\"", []string{`Error: invalid character in multi-line basic string: '\f' (must be escaped) at line 1, column 21`}}, {"\"\"\"Escaping control chars\\nis valid\"\"\"", []string{`string("Escaping control chars\nis valid")`}}, {"\"\"\"Invalid escaping \\is not allowed\"\"\"", []string{`Error: invalid escape sequence at line 1, column 21`}},