VimPlug/fzf
1
0
Fork 0
mirror of https://github.com/junegunn/fzf.git synced 2025-12-08 13:44:50 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Add --normalize option to normalize latin script characters

Browse Source 
Close #790
This commit is contained in:
Junegunn Choi
2017-01-09 03:12:23 +09:00
parent 9d545f9578
commit 45793d75c2
9 changed files with 563 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
src/algo/algo.go
View File

@@ -234,9 +234,36 @@ func bonusAt(input util.Chars, idx int) int16 {
return bonusFor(charClassOf(input.Get(idx-1)), charClassOf(input.Get(idx)))
}
type Algo func(caseSensitive bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int)
func normalizeRune(r rune) rune {
if r < 0x00C0 || r > 0x2184 {
return r
}
func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
n := normalized[r]
if n > 0 {
return n
}
return r
}
func normalizeRunes(runes []rune) []rune {
ret := make([]rune, len(runes))
copy(ret, runes)
for idx, r := range runes {
if r < 0x00C0 || r > 0x2184 {
continue
}
n := normalized[r]
if n > 0 {
ret[idx] = normalized[r]
}
}
return ret
}
type Algo func(caseSensitive bool, normalize bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int)
func FuzzyMatchV2(caseSensitive bool, normalize bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
// Assume that pattern is given in lowercase if case-insensitive.
// First check if there's a match and calculate bonus for each position.
// If the input string is too long, consider finding the matching chars in
@@ -247,13 +274,17 @@ func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern []
case 0:
return Result{0, 0, 0}, posArray(withPos, M)
case 1:
return ExactMatchNaive(caseSensitive, forward, input, pattern[0:1], withPos, slab)
return ExactMatchNaive(caseSensitive, normalize, forward, input, pattern[0:1], withPos, slab)
}
// Since O(nm) algorithm can be prohibitively expensive for large input,
// we fall back to the greedy algorithm.
if slab != nil && N*M > cap(slab.I16) {
return FuzzyMatchV1(caseSensitive, forward, input, pattern, withPos, slab)
return FuzzyMatchV1(caseSensitive, normalize, forward, input, pattern, withPos, slab)
}
if normalize {
pattern = normalizeRunes(pattern)
}
// Reuse pre-allocated integer slice to avoid unnecessary sweeping of garbages
@@ -285,6 +316,10 @@ func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern []
}
}
if normalize {
char = normalizeRune(char)
}
T[idx] = char
B[idx] = bonusFor(prevClass, class)
prevClass = class
@@ -432,7 +467,7 @@ func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern []
}
// Implement the same sorting criteria as V2
func calculateScore(caseSensitive bool, text util.Chars, pattern []rune, sidx int, eidx int, withPos bool) (int, *[]int) {
func calculateScore(caseSensitive bool, normalize bool, text util.Chars, pattern []rune, sidx int, eidx int, withPos bool) (int, *[]int) {
pidx, score, inGap, consecutive, firstBonus := 0, 0, false, 0, int16(0)
pos := posArray(withPos, len(pattern))
prevClass := charNonWord
@@ -449,6 +484,10 @@ func calculateScore(caseSensitive bool, text util.Chars, pattern []rune, sidx in
char = unicode.To(unicode.LowerCase, char)
}
}
// pattern is already normalized
if normalize {
char = normalizeRune(char)
}
if char == pattern[pidx] {
if withPos {
*pos = append(*pos, idx)
@@ -488,7 +527,7 @@ func calculateScore(caseSensitive bool, text util.Chars, pattern []rune, sidx in
}
// FuzzyMatchV1 performs fuzzy-match
func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
func FuzzyMatchV1(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
if len(pattern) == 0 {
return Result{0, 0, 0}, nil
}
@@ -500,6 +539,10 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r
lenRunes := text.Length()
lenPattern := len(pattern)
if normalize {
pattern = normalizeRunes(pattern)
}
for index := 0; index < lenRunes; index++ {
char := text.Get(indexAt(index, lenRunes, forward))
// This is considerably faster than blindly applying strings.ToLower to the
@@ -514,6 +557,9 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r
char = unicode.To(unicode.LowerCase, char)
}
}
if normalize {
char = normalizeRune(char)
}
pchar := pattern[indexAt(pidx, lenPattern, forward)]
if char == pchar {
if sidx < 0 {
@@ -553,7 +599,7 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r
sidx, eidx = lenRunes-eidx, lenRunes-sidx
}
score, pos := calculateScore(caseSensitive, text, pattern, sidx, eidx, withPos)
score, pos := calculateScore(caseSensitive, normalize, text, pattern, sidx, eidx, withPos)
return Result{sidx, eidx, score}, pos
}
return Result{-1, -1, 0}, nil
@@ -568,7 +614,7 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r
// bonus point, instead of stopping immediately after finding the first match.
// The solution is much cheaper since there is only one possible alignment of
// the pattern.
func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
func ExactMatchNaive(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
if len(pattern) == 0 {
return Result{0, 0, 0}, nil
}
@@ -580,6 +626,10 @@ func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern
return Result{-1, -1, 0}, nil
}
if normalize {
pattern = normalizeRunes(pattern)
}
// For simplicity, only look at the bonus at the first character position
pidx := 0
bestPos, bonus, bestBonus := -1, int16(0), int16(-1)
@@ -593,6 +643,9 @@ func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern
char = unicode.To(unicode.LowerCase, char)
}
}
if normalize {
char = normalizeRune(char)
}
pidx_ := indexAt(pidx, lenPattern, forward)
pchar := pattern[pidx_]
if pchar == char {
@@ -624,14 +677,14 @@ func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern
sidx = lenRunes - (bestPos + 1)
eidx = lenRunes - (bestPos - lenPattern + 1)
}
score, _ := calculateScore(caseSensitive, text, pattern, sidx, eidx, false)
score, _ := calculateScore(caseSensitive, normalize, text, pattern, sidx, eidx, false)
return Result{sidx, eidx, score}, nil
}
return Result{-1, -1, 0}, nil
}
// PrefixMatch performs prefix-match
func PrefixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
func PrefixMatch(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
if len(pattern) == 0 {
return Result{0, 0, 0}, nil
}
@@ -640,22 +693,29 @@ func PrefixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []ru
return Result{-1, -1, 0}, nil
}
if normalize {
pattern = normalizeRunes(pattern)
}
for index, r := range pattern {
char := text.Get(index)
if !caseSensitive {
char = unicode.ToLower(char)
}
if normalize {
char = normalizeRune(char)
}
if char != r {
return Result{-1, -1, 0}, nil
}
}
lenPattern := len(pattern)
score, _ := calculateScore(caseSensitive, text, pattern, 0, lenPattern, false)
score, _ := calculateScore(caseSensitive, normalize, text, pattern, 0, lenPattern, false)
return Result{0, lenPattern, score}, nil
}
// SuffixMatch performs suffix-match
func SuffixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
func SuffixMatch(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
lenRunes := text.Length()
trimmedLen := lenRunes - text.TrailingWhitespaces()
if len(pattern) == 0 {
@@ -666,11 +726,18 @@ func SuffixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []ru
return Result{-1, -1, 0}, nil
}
if normalize {
pattern = normalizeRunes(pattern)
}
for index, r := range pattern {
char := text.Get(index + diff)
if !caseSensitive {
char = unicode.ToLower(char)
}
if normalize {
char = normalizeRune(char)
}
if char != r {
return Result{-1, -1, 0}, nil
}
@@ -678,21 +745,37 @@ func SuffixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []ru
lenPattern := len(pattern)
sidx := trimmedLen - lenPattern
eidx := trimmedLen
score, _ := calculateScore(caseSensitive, text, pattern, sidx, eidx, false)
score, _ := calculateScore(caseSensitive, normalize, text, pattern, sidx, eidx, false)
return Result{sidx, eidx, score}, nil
}
// EqualMatch performs equal-match
func EqualMatch(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
func EqualMatch(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
lenPattern := len(pattern)
if text.Length() != lenPattern {
return Result{-1, -1, 0}, nil
}
runesStr := text.ToString()
if !caseSensitive {
runesStr = strings.ToLower(runesStr)
match := true
if normalize {
runes := text.ToRunes()
for idx, pchar := range pattern {
char := runes[idx]
if !caseSensitive {
char = unicode.To(unicode.LowerCase, char)
}
if normalizeRune(pchar) != normalizeRune(char) {
match = false
break
}
}
} else {
runesStr := text.ToString()
if !caseSensitive {
runesStr = strings.ToLower(runesStr)
}
match = runesStr == string(pattern)
}
if runesStr == string(pattern) {
if match {
return Result{0, lenPattern, (scoreMatch+bonusBoundary)*lenPattern +
(bonusFirstCharMultiplier-1)*bonusBoundary}, nil
}