diff --git a/conftest.py b/conftest.py
new file mode 100644
index 00000000..044963bb
--- /dev/null
+++ b/conftest.py
@@ -0,0 +1 @@
+collect_ignore = ["setup.py"]
diff --git a/pytest.ini b/pytest.ini
index 4afbf164..2be72740 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,5 +1,8 @@
 [pytest]
-addopts = --assert=plain
+addopts = --assert=plain --doctest-modules
+
+# Ignore broken files in blackbox test directories
+norecursedirs = .* docs completion refactor
 
 # Activate `clean_jedi_cache` fixture for all tests.  This should be
 # fine as long as we are using `clean_jedi_cache` as a session scoped
diff --git a/tox.ini b/tox.ini
index 25014940..6b2bad01 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,9 +5,6 @@ deps =
     pytest
 commands =
     py.test []
-    # Doctests can't be run with the main tests because then py.test
-    # tries to import broken python files under test/*/.
-    py.test --doctest-modules {toxinidir}/jedi
 [testenv:py25]
 deps =
     simplejson