[vim] Open fzf in a tmux or Zellij floating pane by default

The window fzf was started from stays visible, and can be used while fzf
runs, which a popup inside Vim cannot offer.

Only where the pane is not modal, so tmux 3.7 or above, or Zellij 0.44 or
above with fzf 0.71.0 or above. A tmux popup from 3.3 to 3.6 cannot be left,
and below 3.3 fzf goes through the fzf-tmux script, whose options are
spelled differently. Both keep the window inside Vim. An explicit popup
layout is unaffected and still works from 3.3.

fzf checks tmux before Zellij, so tmux wins when both are set.

g:fzf_layout still wins, so anyone who set it sees no change.

plugin/fzf.vim

@@ -140,6 +140,11 @@ function! s:popup_support()
 endfunction
 
 function! s:default_layout()
+  " A floating pane leaves the window fzf was started from visible and
+  " reachable while fzf is open. A popup covers it, inside Vim or not
+  if s:floating_pane()
+    return { 'tmux': '90%,60%' }
+  endif
   return s:popup_support()
         \ ? { 'window' : { 'width': 0.9, 'height': 0.6 } }
         \ : { 'down': '~40%' }
@@ -165,6 +170,12 @@ function! fzf#install()
   if v:shell_error
     throw 'Failed to download fzf: '.script
   endif
+
+  " A new binary invalidates the chosen executable and everything derived from
+  " its version, including whether fzf opens a floating pane. fzf#install() is
+  " also the vim-plug 'do' hook, so this can run long after the first fzf call
+  let [s:versions, s:checked] = [{}, {}]
+  unlet! s:exec s:tmux s:tmux_floating
 endfunction
 
 let s:versions = {}
@@ -257,7 +268,7 @@ function! fzf#exec(...)
 endfunction
 
 " Path to the fzf-tmux script, or an empty string if it is not available. Only
-" the legacy options still need it. --tmux is handled by fzf itself.
+" the legacy options still need it. --popup is handled by fzf itself.
 function! s:fzf_tmux_script()
   if !executable(s:fzf_tmux)
     if !executable('fzf-tmux')
@@ -273,14 +284,7 @@ function! s:tmux_enabled()
     return 0
   endif
 
-  " --tmux covers Zellij as well, where the fzf-tmux script and the tmux
-  " version are irrelevant, but the binary only learned it in 0.71.0
-  if exists('$ZELLIJ')
-    return exists('s:exec')
-          \ && s:compare_versions(s:get_version(s:exec), '0.71.0') >= 0
-  endif
-
-  if !exists('$TMUX')
+  if empty($TMUX) && empty($ZELLIJ)
     return 0
   endif
 
@@ -288,7 +292,21 @@ function! s:tmux_enabled()
     return s:tmux
   endif
 
-  let s:tmux = 0
+  let [s:tmux, s:tmux_floating] = [0, 0]
+
+  " --popup covers Zellij as well, where the fzf-tmux script and the tmux
+  " version are irrelevant. fzf learned it in 0.71.0, and the floating pane
+  " options it passes need Zellij 0.44 or above. fzf checks tmux first, so
+  " this branch is Zellij without tmux. Both non-empty means tmux wins.
+  " empty(), not exists(), to match how fzf reads the two variables
+  if empty($TMUX)
+    let s:tmux =
+          \ s:compare_versions(s:get_version(s:fzf_binary()), '0.71.0') >= 0
+          \ && s:compare_versions(s:zellij_version(), '0.44') >= 0
+    let s:tmux_floating = s:tmux
+    return s:tmux
+  endif
+
   let output = system('tmux -V')
   if v:shell_error
     return s:tmux
@@ -296,9 +314,15 @@ function! s:tmux_enabled()
   " e.g. 'tmux 3.7b', 'tmux next-3.8'
   let ver = matchstr(output, '\d\+\.\d\+')
 
-  " --tmux requires tmux 3.3 or above, and needs no fzf-tmux script
+  " --popup requires tmux 3.3 or above, and needs no fzf-tmux script. The
+  " default layout wants a floating pane, which also needs fzf 0.74.0 or
+  " above. fzf opens a modal popup otherwise. The version here only skips the
+  " probe for servers too old to answer it
   if s:compare_versions(ver, '3.3') >= 0
     let s:tmux = 1
+    let s:tmux_floating = s:compare_versions(ver, '3.7') >= 0
+          \ && s:tmux_floating_pane_info()
+          \ && s:compare_versions(s:get_version(s:fzf_binary()), '0.74.0') >= 0
     return s:tmux
   endif
 
@@ -451,7 +475,10 @@ function! fzf#wrap(...)
     if !exists('g:fzf_layout') && exists('g:fzf_height')
       let opts.down = g:fzf_height
     else
-      let opts = extend(opts, s:validate_layout(get(g:, 'fzf_layout', s:default_layout())))
+      " Not get(), which would evaluate s:default_layout() and run its version
+      " checks even when g:fzf_layout makes the answer irrelevant
+      let opts = extend(opts, s:validate_layout(
+            \ exists('g:fzf_layout') ? g:fzf_layout : s:default_layout()))
     endif
   endif
 
@@ -627,6 +654,57 @@ function! s:present(dict, ...)
   return 0
 endfunction
 
+" The binary fzf#exec() would choose, without its prompting or installing.
+" Layout selection runs before fzf#exec() has resolved one
+function! s:fzf_binary()
+  if exists('s:exec')
+    return s:exec
+  endif
+  let bins = filter(['fzf', s:fzf_go], 'executable(v:val)')
+  if empty(bins)
+    return ''
+  endif
+  return len(bins) > 1 ? sort(bins, 's:compare_binary_versions')[-1] : bins[0]
+endfunction
+
+function! s:zellij_version()
+  if !exists('s:zellij_ver')
+    let output = systemlist('zellij --version')
+    let s:zellij_ver = v:shell_error || empty(output)
+          \ ? '' : matchstr(output[0], '[0-9.]\+')
+  endif
+  return s:zellij_ver
+endfunction
+
+" fzf decides on the running server, not on the version the tmux client
+" reports, so ask it the same question rather than predicting the answer.
+" See tmuxFloatingPaneInfo in src/tmux.go
+function! s:tmux_floating_pane_info()
+  " fzf does not use a floating pane when it was not started from a pane
+  if empty($TMUX_PANE)
+    return 0
+  endif
+  let out = system('tmux display-message -p -t '.fzf#shellescape($TMUX_PANE)
+        \ .' ''#{window_width} #{window_height}'' '';'' list-commands new-pane')
+  " A server that does not know the command exits normally with no output
+  if v:shell_error || out !~# 'new-pane'
+    return 0
+  endif
+  " Too small for a floating pane of the minimum size
+  return str2nr(matchstr(out, '^\s*\zs\d\+')) >= 3
+        \ && str2nr(matchstr(out, '^\s*\d\+\s\+\zs\d\+')) >= 3
+endfunction
+
+" Whether fzf will open a floating pane that can be left while fzf runs. A
+" tmux popup below 3.7 is modal, and the fzf-tmux script that older tmux
+" versions need does not accept a --popup value at all. Without a job to wait
+" on fzf, s:execute_tmux() blocks on system() and the window stays visible but
+" frozen, so the default keeps fzf inside Vim there
+function! s:floating_pane()
+  return (has('nvim') || has('job'))
+        \ && s:tmux_enabled() && get(s:, 'tmux_floating', 0)
+endfunction
+
 function! s:fzf_tmux(dict)
   let size = get(a:dict, 'tmux', '')
   if empty(size)