Support --package-root with native parser (#21321)

When `--package-root` is used, `fscache` creates some fake `__init__.py`
files on-the-fly (that don't actually exist). These obviously can't be
handed by the native parser, but `fscache.exists()` returns `True` for
them, causing a crash. One possible fix is to simply use
`os.path.isfile()` everywhere. But I think `fscache` can give us a
little perf boost, so I instead add a `real_only` flag to
`fscache.exists()`.

Author: ilevkivskyi

mypy/__main__.py

@@ -6,6 +6,7 @@
 import sys
 import traceback
 
+from mypy.fscache import FileSystemCache
 from mypy.main import main, process_options
 from mypy.util import FancyFormatter
 
@@ -22,7 +23,8 @@ def console_entry() -> None:
         os.dup2(devnull, sys.stdout.fileno())
         sys.exit(2)
     except KeyboardInterrupt:
-        _, options = process_options(args=sys.argv[1:])
+        # Setting fscache prevents bogus errors when --package-root is set.
+        _, options = process_options(args=sys.argv[1:], fscache=FileSystemCache())
         if options.show_traceback:
             sys.stdout.write(traceback.format_exc())
         formatter = FancyFormatter(sys.stdout, sys.stderr, False)
@@ -36,7 +38,7 @@ def console_entry() -> None:
         try:
             import mypy.errors
 
-            _, options = process_options(args=sys.argv[1:])
+            _, options = process_options(args=sys.argv[1:], fscache=FileSystemCache())
             mypy.errors.report_internal_error(e, None, 0, None, options)
         except Exception:
             pass

mypy/build.py

@@ -1025,7 +1025,7 @@ def parse_all(self, states: list[State], post_parse: bool = True) -> None:
             if state.tree is not None:
                 # The file was already parsed.
                 continue
-            if not self.fscache.exists(state.xpath):
+            if not self.fscache.exists(state.xpath, real_only=True):
                 # New parser only supports parsing on-disk files.
                 sequential_states.append(state)
                 continue
@@ -1083,7 +1083,6 @@ def parse_parallel(self, sequential_states: list[State], parallel_states: list[S
                 elif state.source_hash is None:
                     # At least namespace packages may not have source.
                     state.get_source()
-                state.size_hint = os.path.getsize(state.xpath)
                 state.early_errors = list(self.errors.error_info_map.get(state.xpath, []))
                 state.semantic_analysis_pass1()
                 self.ast_cache[state.id] = (state.tree, state.early_errors, state.source_hash)
@@ -1271,7 +1270,7 @@ def parse_file(
 
         Raise CompileError if there is a parse error.
         """
-        file_exists = self.fscache.exists(path)
+        file_exists = self.fscache.exists(path, real_only=True)
         t0 = time.time()
         if raw_data:
             # If possible, deserialize from known binary data instead of parsing from scratch.

mypy/build_worker/worker.py

@@ -103,6 +103,7 @@ def main(argv: list[str]) -> None:
         raise
 
     fscache = FileSystemCache()
+    fscache.set_package_root(options.package_root)
     cached_read = fscache.read
     errors = Errors(options, read_source=lambda path: read_py_file(path, cached_read))
 

mypy/fscache.py

@@ -253,9 +253,14 @@ def isdir(self, path: str) -> bool:
             return False
         return stat.S_ISDIR(st.st_mode)
 
-    def exists(self, path: str) -> bool:
+    def exists(self, path: str, real_only: bool = False) -> bool:
         st = self.stat_or_none(path)
-        return st is not None
+        if st is None:
+            return False
+        if real_only:
+            dirname = os.path.dirname(path)
+            return dirname not in self.fake_package_cache
+        return True
 
     def read(self, path: str) -> bytes:
         if path in self.read_cache:

test-data/unit/cmdline.test

@@ -591,6 +591,14 @@ import d
 import b.c
 import d
 
+[case testPackageRootMultipleParallel]
+# cmd: mypy --package-root=a/ --package-root=./ a/b/c.py d.py main.py --num-workers=2
+[file a/b/c.py]
+[file d.py]
+[file main.py]
+import b.c
+import d
+
 [case testCacheMap]
 -- This just checks that a valid --cache-map triple is accepted.
 -- (Errors are too verbose to check.)