daemon: Check if build root is read-write mounted.

If there has been a connection error, iSCSI LUNs
remount themselves as read-only to prevent data-loss.

To stop starting new build jobs, we now have a check
and even try to remount the build root if we can.

diff --git a/python/pakfire/builder.py b/python/pakfire/builder.py
index 68c9fd4b45d224e5b7df526478b46a7da2705e45..cf2608c1971289ecd546d18527972f32427836ef 100644 (file)
--- a/python/pakfire/builder.py
+++ b/python/pakfire/builder.py
@@ -168,6 +168,11 @@ class BuildEnviron(object):
        def start(self):
                assert not self.pakfire.initialized, "Pakfire has already been initialized"
 
+               # Check if we can write our build directory.
+               build_mp = system.get_mountpoint(self.pakfire.path)
+               if build_mp and build_mp.is_readonly():
+                       raise RuntimeError, "Build directory is read-only: %s" % self.pakfire.path
+
                # Unshare namepsace.
                # If this fails because the kernel has no support for CLONE_NEWIPC or CLONE_NEWUTS,
                # we try to fall back to just set CLONE_NEWNS.

diff --git a/python/pakfire/daemon.py b/python/pakfire/daemon.py
index 67fdd69c2c6383650baebe78496367b4cb62d8ca..8b4f8c81d1fcebcb8c586aed1e3bd0241c85c719 100644 (file)
--- a/python/pakfire/daemon.py
+++ b/python/pakfire/daemon.py
@@ -386,6 +386,11 @@ class PakfireWorker(multiprocessing.Process):
                self.transport = transport.PakfireHubTransport(self.config)
 
                while self.__running:
+                       # Check if the build root is file.
+                       if not self.check_buildroot():
+                               time.sleep(60)
+                               continue
+
                        # Try to get a new build job.
                        job = self.get_new_build_job()
                        if not job:
@@ -428,9 +433,27 @@ class PakfireWorker(multiprocessing.Process):
                """
                self.shutdown()
 
+       def check_buildroot(self):
+               """
+                       Checks if the buildroot is fine.
+               """
+               mp = system.get_mountpoint(BUILD_ROOT)
+
+               # Check if the mountpoint is read-only.
+               if mp.is_readonly():
+                       log.warning("Build directory is read-only: %s" % BUILD_ROOT)
+
+                       # Trying to remount.
+                       try:
+                               mp.remount("rw")
+                       except:
+                               log.warning("Remounting (rw) %s has failed" % BUILD_ROOT)
+                               return False
+                       else:
+                               log.warning("Successfully remounted as rw: %s" % BUILD_ROOT)
+
        def get_new_build_job(self, timeout=600):
                log.debug("Requesting new job...")
-
                try:
                        job = self.transport.get_json("/builders/jobs/queue",
                                data={ "timeout" : timeout, }, timeout=timeout)

diff --git a/python/pakfire/system.py b/python/pakfire/system.py
index a917461fd768bce943475f07db66479f3f7e3f00..2bba8020819bac4d942e5a110a49e9c9ce8edce7 100644 (file)
--- a/python/pakfire/system.py
+++ b/python/pakfire/system.py
@@ -24,8 +24,10 @@ from __future__ import division
 import multiprocessing
 import os
 import socket
+import tempfile
 
 import distro
+import shell
 
 from i18n import _
 
@@ -346,6 +348,9 @@ class Mountpoint(object):
                # Save the amount of data that is used or freed.
                self.disk_usage = 0
 
+       def __repr__(self):
+               return "<%s %s>" % (self.__class__.__name__, self.fullpath)
+
        def __cmp__(self, other):
                return cmp(self.fullpath, other.fullpath)
 
@@ -405,6 +410,45 @@ class Mountpoint(object):
 
                self.disk_usage -= file.size
 
+       def is_readonly(self):
+               """
+                       Returns True if the mountpoint is mounted read-only.
+                       Otherwise False.
+               """
+               # Using the statvfs output does not really work, so we use
+               # a very naive approach here, were we just try to create a
+               # new file. If that works, it's writable.
+
+               try:
+                       handle, path = tempfile.mkstemp(prefix="ro-test-", dir=self.fullpath)
+               except OSError, e:
+                       # Read-only file system.
+                       if e.errno == 30:
+                               return True
+
+                       # Raise all other exceptions.
+                       raise
+               else:
+                       # Close the file and remove it.
+                       os.close(handle)
+                       os.unlink(path)
+
+               return False
+
+       def remount(self, rorw=None):
+               options = "remount"
+               if rorw in ("ro", "rw"):
+                       options = "%s,%s" % (options, rorw)
+
+               try:
+                       shellenv = shell.ShellExecuteEnvironment(
+                               ["mount", "-o", options, self.fullpath],
+                               shell=False,
+                       )
+                       shellenv.execute()
+               except ShellEnvironmentError, e:
+                       raise OSError
+
 
 if __name__ == "__main__":
        print "Hostname", system.hostname