package lexer_test import ( "fmt" "testing" "github.com/mmakaay/toml/lexer" ) func TestInvalidUtf8Data(t *testing.T) { assertFailureAndCheck(t, "\xbc", []string{}, "Unexpected non-UTF8 data (expected end of file)") } func TestEmptyInput(t *testing.T) { assertSuccessAndCheck(t, "", []string{}) } func TestWhiteSpace(t *testing.T) { assertSuccessAndCheck(t, " ", []string{}) assertSuccessAndCheck(t, "\t", []string{}) assertSuccessAndCheck(t, " \t \t ", []string{}) } func TestWhiteSpaceAndNewlines(t *testing.T) { assertSuccessAndCheck(t, "\n", []string{}) assertSuccessAndCheck(t, "\n \t\r\n", []string{}) } func TestWhitespacePlusComment(t *testing.T) { assertSuccessAndCheck(t, "#", []string{`Comment("#")`}) assertSuccessAndCheck(t, " \t \t #", []string{`Comment("#")`}) assertSuccessAndCheck(t, " \t \t # not empty", []string{`Comment("# not empty")`}) assertSuccessAndCheck(t, " \t \t # not empty\r\r\r\n", []string{`Comment("# not empty")`}) assertSuccessAndCheck(t, "\n \t\r\n# AAP\r\n", []string{`Comment("# AAP")`}) assertSuccessAndCheck(t, "# two lines\n# of comments\n", []string{`Comment("# two lines")`, `Comment("# of comments")`}) } func TestBareKeyWithoutValue(t *testing.T) { err := "Unexpected end of file (expected an '=' value assignment)" assertFailureAndCheck(t, "a", []string{`Key("a")`}, err) assertFailureAndCheck(t, " a", []string{`Key("a")`}, err) assertFailureAndCheck(t, " a ", []string{`Key("a")`}, err) assertFailureAndCheck(t, "ab", []string{`Key("ab")`}, err) assertFailureAndCheck(t, "Ab", []string{`Key("Ab")`}, err) assertFailureAndCheck(t, "Ab1", []string{`Key("Ab1")`}, err) assertFailureAndCheck(t, "_Ab1", []string{`Key("_Ab1")`}, err) assertFailureAndCheck(t, "_-Ab1", []string{`Key("_-Ab1")`}, err) assertFailureAndCheck(t, "_-Ab1_this-is_GOOD987", []string{`Key("_-Ab1_this-is_GOOD987")`}, err) } func TestDottedKey(t *testing.T) { err := "Unexpected end of file (expected an '=' value assignment)" assertFailureAndCheck(t, "a.b", []string{`Key("a")`, `KeyDot(".")`, `Key("b")`}, err) assertFailureAndCheck(t, " a .\t\t b\t ", []string{`Key("a")`, `KeyDot(".")`, `Key("b")`}, err) } func TestKeyWithAssignmentButNoValue(t *testing.T) { err := "Unexpected end of file (expected a value)" assertFailureAndCheck(t, " some_cool_key = ", []string{`Key("some_cool_key")`}, err) } func TestUnterminatedBasicString(t *testing.T) { assertFailure(t, `key="value`, "Unexpected end of file (expected basic string token)") } func TestBasicStringWithNewline(t *testing.T) { assertFailure(t, "key=\"value\nwith\nnewlines\"", "ohoh") } func TestEmptyBasicString(t *testing.T) { assertSuccessAndCheck(t, `a=""`, []string{`Key("a")`, `String("")`}) assertSuccessAndCheck(t, `a=""#hi`, []string{`Key("a")`, `String("")`, `Comment("#hi")`}) assertSuccessAndCheck(t, `a = ""`, []string{`Key("a")`, `String("")`}) assertSuccessAndCheck(t, `a.b = ""`, []string{`Key("a")`, `KeyDot(".")`, `Key("b")`, `String("")`}) assertSuccessAndCheck(t, `a=""b=""`, []string{`Key("a")`, `String("")`, `Key("b")`, `String("")`}) } func TestBasicString(t *testing.T) { assertSuccessAndCheck(t, `_ = "b"`, []string{ `Key("_")`, `String("b")`}) assertSuccessAndCheck(t, `thing = "A cool ʎǝʞ" # huh, it's up-side down!!`, []string{ `Key("thing")`, `String("A cool ʎǝʞ")`, `Comment("# huh, it's up-side down!!")`}) } func TestInvalidEscapeSequence(t *testing.T) { assertFailure(t, `a="\x"`, `Invalid escape sequence \x in string value`) } func TestBasicStringEscapes(t *testing.T) { for in, out := range map[string]string{ `\b`: "\b", `\t`: "\t", `\n`: "\n", `\f`: "\f", `\r`: "\r", `\"`: "\"", `\b\t\n\f\r\"`: "\b\t\n\f\r\"", } { l := assertSuccess(t, fmt.Sprintf(`x="%s"`, in)) if out != l[1].Value { t.Fatalf("Unexpected result when parsing '%s'", in) } } } // func TestBasicStringUnicodeEscapes(t *testing.T) { // for in, out := range map[string]string{ // `\u`: "\b", // } { // l := assertSuccess(t, fmt.Sprintf(`x="%s"`, in)) // s := l[2] // if out != s.Value { // t.Fatalf("Unexpected result when parsing '%s'", in) // } // } // } func TestTwoKeyValuePairs(t *testing.T) { assertSuccessAndCheck(t, "a=\"Hello\" #comment1\nb=\"World!\"#comment2\r\n", []string{ `Key("a")`, `String("Hello")`, `Comment("#comment1")`, `Key("b")`, `String("World!")`, `Comment("#comment2")`}) } func assertSuccessAndCheck(t *testing.T, input string, expected []string) { l := assertSuccess(t, input) assertItems(t, l, expected) } func assertFailureAndCheck(t *testing.T, input string, expected []string, expectedErr string) { l := assertFailure(t, input, expectedErr) assertItems(t, l, expected) } func assertFailure(t *testing.T, input string, expectedErr string) []lexer.Item { l, err := lexer.Lex(input).ToArray() if err == nil { t.Fatalf("Expected lexer error '%s', but no error occurred", expectedErr) } if err.Error() != expectedErr { t.Fatalf("Mismatch between expected and actual error:\nExpected: %s\nActual: %s\n", expectedErr, err) } return l } func assertSuccess(t *testing.T, input string) []lexer.Item { l, err := lexer.Lex(input).ToArray() if err != nil { t.Fatalf("Unexpected lexer error: %s", err) } return l } func assertItems(t *testing.T, l []lexer.Item, expected []string) { if len(expected) != len(l) { t.Fatalf("Unexpected number of lexer items: %d (expected: %d)", len(l), len(expected)) } for i, e := range expected { if l[i].String() != e { t.Fatalf("Unexpected lexer item at index %d: %s (expected: %s)", i, l[i], e) } } }