From d32458084014331cb04d2033cf3306daa107f9e2 Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Sat, 7 Mar 2026 11:45:02 +0900 Subject: [PATCH] Fix AWK tokenizer not treating a new line character as whitespace --- src/tokenizer.go | 2 +- src/tokenizer_test.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tokenizer.go b/src/tokenizer.go index 25448f59..e1502672 100644 --- a/src/tokenizer.go +++ b/src/tokenizer.go @@ -161,7 +161,7 @@ func awkTokenizer(input string) ([]string, int) { end := 0 for idx := 0; idx < len(input); idx++ { r := input[idx] - white := r == 9 || r == 32 + white := r == 9 || r == 32 || r == 10 switch state { case awkNil: if white { diff --git a/src/tokenizer_test.go b/src/tokenizer_test.go index a471a2fc..e5a45740 100644 --- a/src/tokenizer_test.go +++ b/src/tokenizer_test.go @@ -56,9 +56,9 @@ func TestParseRange(t *testing.T) { func TestTokenize(t *testing.T) { // AWK-style - input := " abc: def: ghi " + input := " abc: \n\t def: ghi " tokens := Tokenize(input, Delimiter{}) - if tokens[0].text.ToString() != "abc: " || tokens[0].prefixLength != 2 { + if tokens[0].text.ToString() != "abc: \n\t " || tokens[0].prefixLength != 2 { t.Errorf("%s", tokens) } @@ -71,9 +71,9 @@ func TestTokenize(t *testing.T) { // With delimiter regex tokens = Tokenize(input, delimiterRegexp("\\s+")) if tokens[0].text.ToString() != " " || tokens[0].prefixLength != 0 || - tokens[1].text.ToString() != "abc: " || tokens[1].prefixLength != 2 || - tokens[2].text.ToString() != "def: " || tokens[2].prefixLength != 8 || - tokens[3].text.ToString() != "ghi " || tokens[3].prefixLength != 14 { + tokens[1].text.ToString() != "abc: \n\t " || tokens[1].prefixLength != 2 || + tokens[2].text.ToString() != "def: " || tokens[2].prefixLength != 10 || + tokens[3].text.ToString() != "ghi " || tokens[3].prefixLength != 16 { t.Errorf("%s", tokens) } }