VimPlug/fzf
1
0
Fork 0
mirror of https://github.com/junegunn/fzf.git synced 2026-05-26 02:08:50 +08:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

Fix AWK tokenizer not treating a new line character as whitespace

Browse Source
This commit is contained in:
Junegunn Choi
2026-03-07 11:45:02 +09:00
parent f9830c5a3d
commit d324580840
2 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tokenizer.go
+1 -1
View File
@@ -161,7 +161,7 @@ func awkTokenizer(input string) ([]string, int) {
end := 0 end := 0
for idx := 0; idx < len(input); idx++ { for idx := 0; idx < len(input); idx++ {
r := input[idx] r := input[idx]
white := r == 9 || r == 32 white := r == 9 || r == 32 || r == 10
switch state { switch state {
case awkNil: case awkNil:
if white { if white {
src/tokenizer_test.go
+5 -5
View File
@@ -56,9 +56,9 @@ func TestParseRange(t *testing.T) {
func TestTokenize(t *testing.T) { func TestTokenize(t *testing.T) {
// AWK-style // AWK-style
input := " abc: def: ghi " input := " abc: \n\t def: ghi "
tokens := Tokenize(input, Delimiter{}) tokens := Tokenize(input, Delimiter{})
if tokens[0].text.ToString() != "abc: " || tokens[0].prefixLength != 2 { if tokens[0].text.ToString() != "abc: \n\t " || tokens[0].prefixLength != 2 {
t.Errorf("%s", tokens) t.Errorf("%s", tokens)
} }
@@ -71,9 +71,9 @@ func TestTokenize(t *testing.T) {
// With delimiter regex // With delimiter regex
tokens = Tokenize(input, delimiterRegexp("\\s+")) tokens = Tokenize(input, delimiterRegexp("\\s+"))
if tokens[0].text.ToString() != " " || tokens[0].prefixLength != 0 || if tokens[0].text.ToString() != " " || tokens[0].prefixLength != 0 ||
tokens[1].text.ToString() != "abc: " || tokens[1].prefixLength != 2 || tokens[1].text.ToString() != "abc: \n\t " || tokens[1].prefixLength != 2 ||
tokens[2].text.ToString() != "def: " || tokens[2].prefixLength != 8 || tokens[2].text.ToString() != "def: " || tokens[2].prefixLength != 10 ||
tokens[3].text.ToString() != "ghi " || tokens[3].prefixLength != 14 { tokens[3].text.ToString() != "ghi " || tokens[3].prefixLength != 16 {
t.Errorf("%s", tokens) t.Errorf("%s", tokens)
} }
} }