improved static analysis test base

jedi/api/__init__.py

@@ -585,7 +585,7 @@ class Script(object):
         return [classes.CallSignature(self._evaluator, o, call, index, key_name)
                 for o in origins if o.is_callable()]
 
-    def analysis(self):
+    def _analysis(self):
         statements = set(chain(*self._parser.module().used_names.values()))
         for stmt in statements:
             if stmt.start_pos[0] != 254:
@@ -595,10 +595,8 @@ class Script(object):
             else:
                 self._evaluator.eval_statement(stmt)
 
-        for error in sorted(self._evaluator.analysis, key=lambda x: x.line):
-            print(repr(error))
-        raise AssertionError
-        return self._evaluator.analysis
+        analysis = self._evaluator.analysis
+        return sorted(analysis, key=lambda x: x.line)
 
 
 class Interpreter(Script):

test/conftest.py

@@ -68,10 +68,10 @@ def pytest_generate_tests(metafunc):
             refactor.collect_dir_tests(base_dir, test_files))
 
     if 'static_analysis_case' in metafunc.fixturenames:
-        base_dir = os.path.dirname(__file__) + 'static_analysis'
+        base_dir = os.path.join(os.path.dirname(__file__), 'static_analysis')
         metafunc.parametrize(
             'static_analysis_case',
-            refactor.collect_dir_tests(base_dir, test_files))
+            collect_static_analysis_tests(base_dir, test_files))
 
 
 def collect_static_analysis_tests(base_dir, test_files):
@@ -88,6 +88,7 @@ class StaticAnalysisCase(object):
     The tests also start with `#!`, like the goto_definition tests.
     """
     def __init__(self, path):
+        self.skip = False
         self._path = path
         with open(path) as f:
             self._source = f.read()
@@ -101,9 +102,12 @@ class StaticAnalysisCase(object):
         return cases
 
     def run(self, compare_cb):
-        analysis = jedi.Script(self._source).analysis()
+        analysis = jedi.Script(self._source)._analysis()
         analysis = [(r.line, r.name) for r in analysis]
-        assert compare_cb(self, self.collect_comparison(), analysis)
+        assert compare_cb(self, analysis, self.collect_comparison())
+
+    def __repr__(self):
+        return "<%s: %s>" % (self.__class__.__name__, os.path.basename(self._path))
 
 
 @pytest.fixture()

test/test_integration.py

@@ -20,7 +20,7 @@ desired = %s
 """ % (case, actual, desired)
 
 
-def test_integration(case, monkeypatch, pytestconfig):
+def test_integration(case, monkeypatch):
     if case.skip is not None:
         pytest.skip(case.skip)
     repo_root = helpers.root_dir
@@ -28,9 +28,7 @@ def test_integration(case, monkeypatch, pytestconfig):
     case.run(assert_case_equal)
 
 
-def test_static_analysis(static_analysis_case, monkeypatch, pytestconfig):
-    if static_analysis_case.skip is not None:
-        pytest.skip(static_analysis_case.skip)
+def test_static_analysis(static_analysis_case):
     static_analysis_case.run(assert_case_equal)