go-toml/lexer/lexer_test.go

package lexer_test

import (
	"fmt"
	"testing"

	"github.com/mmakaay/toml/lexer"
)

func TestInvalidUtf8Data(t *testing.T) {
	assertFailureAndCheck(t, "\xbc", []string{}, "Unexpected non-UTF8 data (expected end of file)")
}

func TestEmptyInput(t *testing.T) {
	assertSuccessAndCheck(t, "", []string{})
}
func TestWhiteSpace(t *testing.T) {
	assertSuccessAndCheck(t, " ", []string{})
	assertSuccessAndCheck(t, "\t", []string{})
	assertSuccessAndCheck(t, " \t \t ", []string{})
}
func TestWhiteSpaceAndNewlines(t *testing.T) {
	assertSuccessAndCheck(t, "\n", []string{})
	assertSuccessAndCheck(t, "\n \t\r\n", []string{})
}
func TestComments(t *testing.T) {
	assertSuccessAndCheck(t, "#", []string{`Comment("#")`})
	assertSuccessAndCheck(t, " \t \t #", []string{`Comment("#")`})
	assertSuccessAndCheck(t, " \t \t # not empty", []string{`Comment("# not empty")`})
	assertSuccessAndCheck(t, " \t \t # not empty\r\r\r\n", []string{`Comment("# not empty")`})
	assertSuccessAndCheck(t, "\n \t\r\n# AAP\r\n", []string{`Comment("# AAP")`})
	assertSuccessAndCheck(t,
		"# two lines\n# of comments\n",
		[]string{`Comment("# two lines")`, `Comment("# of comments")`})
	assertSuccessAndCheck(t,
		`# \tcomment\nwith escape-y chars`,
		[]string{`Comment("# \\tcomment\\nwith escape-y chars")`})
}

func TestBareKeyWithoutValue(t *testing.T) {
	err := "Unexpected end of file (expected an '=' value assignment)"
	assertFailureAndCheck(t, "a", []string{`Key("a")`}, err)
	assertFailureAndCheck(t, "_", []string{`Key("_")`}, err)
	assertFailureAndCheck(t, " a", []string{`Key("a")`}, err)
	assertFailureAndCheck(t, " a ", []string{`Key("a")`}, err)
	assertFailureAndCheck(t, "ab", []string{`Key("ab")`}, err)
	assertFailureAndCheck(t, "Ab", []string{`Key("Ab")`}, err)
	assertFailureAndCheck(t, "Ab1", []string{`Key("Ab1")`}, err)
	assertFailureAndCheck(t, "_Ab1", []string{`Key("_Ab1")`}, err)
	assertFailureAndCheck(t, "_-Ab1", []string{`Key("_-Ab1")`}, err)
	assertFailureAndCheck(t, "_-Ab1_this-is_GOOD987", []string{`Key("_-Ab1_this-is_GOOD987")`}, err)
}

func TestDottedKey(t *testing.T) {
	err := "Unexpected end of file (expected an '=' value assignment)"
	assertFailureAndCheck(t, "a.b", []string{`Key("a")`, `KeyDot(".")`, `Key("b")`}, err)
	assertFailureAndCheck(t, " a .\t\t b\t ", []string{`Key("a")`, `KeyDot(".")`, `Key("b")`}, err)
}

func TestKeyWithAssignmentButNoValue(t *testing.T) {
	err := "Unexpected end of file (expected a value)"
	assertFailureAndCheck(t, "  some_cool_key   =  ", []string{`Key("some_cool_key")`}, err)
}

func TestUnterminatedBasicString(t *testing.T) {
	assertFailure(t, `key="value`, "Unexpected end of file (expected basic string token)")
}

func TestBasicStringWithNewline(t *testing.T) {
	assertFailure(t, "key=\"value\nwith\nnewlines\"", "ohoh")
}

func TestEmptyBasicString(t *testing.T) {
	assertSuccessAndCheck(t, `a=""`, []string{`Key("a")`, `String("")`})
	assertSuccessAndCheck(t, `a=""#hi`, []string{`Key("a")`, `String("")`, `Comment("#hi")`})
	assertSuccessAndCheck(t, `a = ""`, []string{`Key("a")`, `String("")`})
	assertSuccessAndCheck(t, `a.b = ""`, []string{`Key("a")`, `KeyDot(".")`, `Key("b")`, `String("")`})
	assertSuccessAndCheck(t, `a=""b=""`, []string{`Key("a")`, `String("")`, `Key("b")`, `String("")`})
}
func TestBasicString(t *testing.T) {
	assertSuccessAndCheck(t, `_ = "b"`,
		[]string{
			`Key("_")`,
			`String("b")`})
	assertSuccessAndCheck(t, `thing = "A cool ʎǝʞ" # huh, it's up-side down!!`,
		[]string{
			`Key("thing")`,
			`String("A cool ʎǝʞ")`,
			`Comment("# huh, it's up-side down!!")`})
}

func TestInvalidEscapeSequence(t *testing.T) {
	assertFailure(t, `a="\x"`, `Invalid escape sequence \x in string value`)
}
func TestBasicStringEscapes(t *testing.T) {
	for in, out := range map[string]string{
		`\b`:              "\b",
		`\t`:              "\t",
		`\n`:              "\n",
		`\f`:              "\f",
		`\r`:              "\r",
		`\"`:              "\"",
		`\b\t\nhuh\f\r\"`: "\b\t\nhuh\f\r\"",
		`\u2318`:          "⌘",
		`\U0001014D`:      "𐅍",
	} {
		l := assertSuccess(t, fmt.Sprintf(`x="%s"`, in))
		if out != l[1].Value {
			t.Fatalf("Unexpected result when parsing '%s'\nexpected: %q\nactual: %q", in, out, l[1].Value)
		}
	}
}

// func TestBasicStringUnicodeEscapes(t *testing.T) {
// 	for in, out := range map[string]string{
// 		`\u`: "\b",
// 	} {
// 		l := assertSuccess(t, fmt.Sprintf(`x="%s"`, in))
// 		s := l[2]
// 		if out != s.Value {
// 			t.Fatalf("Unexpected result when parsing '%s'", in)
// 		}
// 	}
// }

func TestTwoKeyValuePairs(t *testing.T) {
	assertSuccessAndCheck(t, "a=\"Hello\" #comment1\nb=\"World!\"#comment2\r\n",
		[]string{
			`Key("a")`,
			`String("Hello")`,
			`Comment("#comment1")`,
			`Key("b")`,
			`String("World!")`,
			`Comment("#comment2")`})
}

func assertSuccessAndCheck(t *testing.T, input string, expected []string) {
	l := assertSuccess(t, input)
	assertItems(t, l, expected)
}

func assertFailureAndCheck(t *testing.T, input string, expected []string, expectedErr string) {
	l := assertFailure(t, input, expectedErr)
	assertItems(t, l, expected)
}

func assertFailure(t *testing.T, input string, expectedErr string) []lexer.Item {
	l, err := lexer.Lex(input).ToArray()
	if err == nil {
		t.Fatalf("Expected lexer error '%s', but no error occurred", expectedErr)
	}
	if err.Error() != expectedErr {
		t.Fatalf("Mismatch between expected and actual error:\nExpected: %s\nActual: %s\n", expectedErr, err)
	}
	return l
}

func assertSuccess(t *testing.T, input string) []lexer.Item {
	l, err := lexer.Lex(input).ToArray()
	if err != nil {
		t.Fatalf("Unexpected lexer error: %s", err)
	}
	return l
}

func assertItems(t *testing.T, l []lexer.Item, expected []string) {
	if len(expected) != len(l) {
		t.Fatalf("Unexpected number of lexer items: %d (expected: %d)", len(l), len(expected))
	}
	for i, e := range expected {
		if l[i].String() != e {
			t.Fatalf("Unexpected lexer item at index %d: %s (expected: %s)", i, l[i], e)
		}
	}
}