Fix race conditions in FileSystemBytecodeCache

diff --git a/CHANGES.rst b/CHANGES.rst
index da7d0a68237a459f7879637841583090bfefb6f6..96a9567cb53c97d979711a68c753b8ddb3e3e18b 100644 (file)
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -7,6 +7,7 @@ Unreleased
 
 -   Add parameters to ``Environment.overlay`` to match ``__init__``.
     :issue:`1645`
+-   Handle race condition in ``FileSystemBytecodeCache``. :issue:`1654`
 
 
 Version 3.1.1

diff --git a/src/jinja2/bccache.py b/src/jinja2/bccache.py
index 3bb61b7c34cea1ae0ea7a5ae3aa33d3628f687c9..d0ddf56ef62b03cba6b6c5f9b94d819393f09d38 100644 (file)
--- a/src/jinja2/bccache.py
+++ b/src/jinja2/bccache.py
@@ -79,7 +79,7 @@ class Bucket:
             self.reset()
             return
 
-    def write_bytecode(self, f: t.BinaryIO) -> None:
+    def write_bytecode(self, f: t.IO[bytes]) -> None:
         """Dump the bytecode into the file or file like object passed."""
         if self.code is None:
             raise TypeError("can't write empty bucket")
@@ -262,13 +262,55 @@ class FileSystemBytecodeCache(BytecodeCache):
     def load_bytecode(self, bucket: Bucket) -> None:
         filename = self._get_cache_filename(bucket)
 
-        if os.path.exists(filename):
-            with open(filename, "rb") as f:
-                bucket.load_bytecode(f)
+        # Don't test for existence before opening the file, since the
+        # file could disappear after the test before the open.
+        try:
+            f = open(filename, "rb")
+        except (FileNotFoundError, IsADirectoryError, PermissionError):
+            # PermissionError can occur on Windows when an operation is
+            # in progress, such as calling clear().
+            return
+
+        with f:
+            bucket.load_bytecode(f)
 
     def dump_bytecode(self, bucket: Bucket) -> None:
-        with open(self._get_cache_filename(bucket), "wb") as f:
-            bucket.write_bytecode(f)
+        # Write to a temporary file, then rename to the real name after
+        # writing. This avoids another process reading the file before
+        # it is fully written.
+        name = self._get_cache_filename(bucket)
+        f = tempfile.NamedTemporaryFile(
+            mode="wb",
+            dir=os.path.dirname(name),
+            prefix=os.path.basename(name),
+            suffix=".tmp",
+            delete=False,
+        )
+
+        def remove_silent() -> None:
+            try:
+                os.remove(f.name)
+            except OSError:
+                # Another process may have called clear(). On Windows,
+                # another program may be holding the file open.
+                pass
+
+        try:
+            with f:
+                bucket.write_bytecode(f)
+        except BaseException:
+            remove_silent()
+            raise
+
+        try:
+            os.replace(f.name, name)
+        except OSError:
+            # Another process may have called clear(). On Windows,
+            # another program may be holding the file open.
+            remove_silent()
+        except BaseException:
+            remove_silent()
+            raise
 
     def clear(self) -> None:
         # imported lazily here because google app-engine doesn't support