diff --git a/autoload/ale/virtualtext.vim b/autoload/ale/virtualtext.vim
index 3ef82680..0f2b2173 100644
--- a/autoload/ale/virtualtext.vim
+++ b/autoload/ale/virtualtext.vim
@@ -75,7 +75,7 @@ function! ale#virtualtext#GetComment(buffer) abort
 endfunction
 
 function! ale#virtualtext#Clear(buffer) abort
-    if !s:has_virt_text
+    if !s:has_virt_text || !bufexists(str2nr(a:buffer))
         return
     endif