diff --git a/conftest.py b/conftest.py
index 691f3df7..480a2510 100644
--- a/conftest.py
+++ b/conftest.py
@@ -25,11 +25,11 @@ def pytest_addoption(parser):
     parser.addoption("--jedi-debug", "-D", action='store_true',
                      help="Enables Jedi's debug output.")
 
+    parser.addoption("--warning-is-error", action='store_true',
+                     help="Warnings are treated as errors.")
+
 
 def pytest_configure(config):
-    import warnings
-    warnings.simplefilter("error")
-
     global jedi_cache_directory_orig, jedi_cache_directory_temp
     jedi_cache_directory_orig = jedi.settings.cache_directory
     jedi_cache_directory_temp = tempfile.mkdtemp(prefix='jedi-test-')
@@ -38,6 +38,10 @@ def pytest_configure(config):
     if config.option.jedi_debug:
         jedi.set_debug_function()
 
+    if config.option.warning_is_error:
+        import warnings
+        warnings.simplefilter("error")
+
 
 def pytest_unconfigure(config):
     global jedi_cache_directory_orig, jedi_cache_directory_temp