From: Luca Boccassi Date: Thu, 16 Nov 2023 15:45:20 +0000 (+0000) Subject: loop-util: restart loop_configure and reopen FD when O_DIRECT fails X-Git-Tag: v255-rc3~45^2~1 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=f5bb0a316316dfa1de2bc5e34cbe21cc0bf8a30b;p=thirdparty%2Fsystemd.git loop-util: restart loop_configure and reopen FD when O_DIRECT fails On kernel 5.10.178, when a squashfs file is stored on an EXT4 filesystem backed by a dm-crypt volume, dissecting fails: $ SYSTEMD_LOG_LEVEL=debug systemd-dissect /var/foo/bar.raw Opened '/var/foo/bar.raw' in O_RDONLY access mode, with O_DIRECT enabled. Couldn't find any partition table to derive sector size of. loop2: Acquired exclusive lock. Could not enable direct IO mode, proceeding in buffered IO mode. Successfully acquired /dev/loop2, devno=7:2, nr=2, diskseq=87 Opened /dev/loop2 (fd=3, whole_block_devnum=7:2, diskseq=87). Name: bar.raw Size: 67.2M Sec. Size: 512 Arch.: n/a Successfully forked off '(sd-dissect)' as PID 4110. Mounting /proc/self/fd/3 (squashfs) on /tmp/dissect-Zk3K5F (MS_RDONLY|MS_NODEV "")... Failed to mount /proc/self/fd/3 (type squashfs) on /tmp/dissect-Zk3K5F (MS_RDONLY|MS_NODEV ""): Input/output error Failed to mount dissected image: Input/output error Failed to read /etc/hostname of image: No such file or directory /etc/machine-id file of image is empty. Failed to read has-init-system boolean: Input/output error (sd-dissect) failed with exit status 1. Failed to acquire image metadata: Input/output error The kernel shows I/O errors: kernel: blk_update_request: I/O error, dev loop2, sector 0 op 0x0:(READ) flags 0x800 phys_seg 1 prio class 0 kernel: SQUASHFS error: Failed to read block 0x0: -5 kernel: unable to read squashfs_super_block This is independent of a particular filesystem and can be reproduced reliably in my setup, starting from freshly formatted disks. Instead of continuing when O_DIRECT fails, start over the setup process without the flag, including opening a new FD, to make the kernel happy. --- diff --git a/src/shared/loop-util.c b/src/shared/loop-util.c index d346d014d2a..fdf33fac2ad 100644 --- a/src/shared/loop-util.c +++ b/src/shared/loop-util.c @@ -108,9 +108,16 @@ static int loop_configure_verify_direct_io(int fd, const struct loop_config *c) * check here if enabling direct IO worked, to make this easily debuggable however. * * (Should anyone really care and actually wants direct IO on old kernels: it might be worth - * enabling direct IO with iteratively larger block sizes until it eventually works.) */ + * enabling direct IO with iteratively larger block sizes until it eventually works.) + * + * On older kernels (e.g.: 5.10) when this is attempted on a file stored on a dm-crypt + * backed partition the kernel will start returning I/O errors when accessing the mounted + * loop device, so return a recognizable error that causes the operation to be started + * from scratch without the LO_FLAGS_DIRECT_IO flag. */ if (!FLAGS_SET(info.lo_flags, LO_FLAGS_DIRECT_IO)) - log_debug("Could not enable direct IO mode, proceeding in buffered IO mode."); + return log_debug_errno( + SYNTHETIC_ERRNO(ENOANO), + "Could not enable direct IO mode, retrying in buffered IO mode."); } return 0; @@ -431,7 +438,7 @@ static int loop_device_make_internal( LoopDevice **ret) { _cleanup_(loop_device_unrefp) LoopDevice *d = NULL; - _cleanup_close_ int direct_io_fd = -EBADF, control = -EBADF; + _cleanup_close_ int reopened_fd = -EBADF, control = -EBADF; _cleanup_free_ char *backing_file = NULL; struct loop_config config; int r, f_flags; @@ -479,8 +486,8 @@ static int loop_device_make_internal( * Our intention here is that LO_FLAGS_DIRECT_IO is the primary knob, and O_DIRECT derived * from that automatically. */ - direct_io_fd = fd_reopen(fd, (FLAGS_SET(loop_flags, LO_FLAGS_DIRECT_IO) ? O_DIRECT : 0)|O_CLOEXEC|O_NONBLOCK|open_flags); - if (direct_io_fd < 0) { + reopened_fd = fd_reopen(fd, (FLAGS_SET(loop_flags, LO_FLAGS_DIRECT_IO) ? O_DIRECT : 0)|O_CLOEXEC|O_NONBLOCK|open_flags); + if (reopened_fd < 0) { if (!FLAGS_SET(loop_flags, LO_FLAGS_DIRECT_IO)) return log_debug_errno(errno, "Failed to reopen file descriptor without O_DIRECT: %m"); @@ -488,7 +495,7 @@ static int loop_device_make_internal( log_debug_errno(errno, "Failed to enable O_DIRECT for backing file descriptor for loopback device. Continuing without."); loop_flags &= ~LO_FLAGS_DIRECT_IO; } else - fd = direct_io_fd; /* From now on, operate on our new O_DIRECT fd */ + fd = reopened_fd; /* From now on, operate on our new O_DIRECT fd */ } control = open("/dev/loop-control", O_RDWR|O_CLOEXEC|O_NOCTTY|O_NONBLOCK); @@ -578,8 +585,9 @@ static int loop_device_make_internal( /* -ENODEV or friends: Somebody might've gotten the same number from the kernel, used the * device, and called LOOP_CTL_REMOVE on it. Let's retry with a new number. * -EBUSY: a file descriptor is already bound to the loopback block device. - * -EUCLEAN: some left-over partition devices that were cleaned up. */ - if (!ERRNO_IS_DEVICE_ABSENT(r) && !IN_SET(r, -EBUSY, -EUCLEAN)) + * -EUCLEAN: some left-over partition devices that were cleaned up. + * -ENOANO: we tried to use LO_FLAGS_DIRECT_IO but the kernel rejected it. */ + if (!ERRNO_IS_DEVICE_ABSENT(r) && !IN_SET(r, -EBUSY, -EUCLEAN, -ENOANO)) return r; /* OK, this didn't work, let's try again a bit later, but first release the lock on the @@ -590,6 +598,23 @@ static int loop_device_make_internal( if (++n_attempts >= 64) /* Give up eventually */ return -EBUSY; + /* If we failed to enable direct IO mode, let's retry without it. We restart the process as + * on some combination of kernel version and storage filesystem, the kernel is very unhappy + * about a failed DIRECT_IO enablement and throws I/O errors. */ + if (r == -ENOANO && FLAGS_SET(config.info.lo_flags, LO_FLAGS_DIRECT_IO)) { + config.info.lo_flags &= ~LO_FLAGS_DIRECT_IO; + open_flags &= ~O_DIRECT; + + int non_direct_io_fd = fd_reopen(config.fd, O_CLOEXEC|O_NONBLOCK|open_flags); + if (non_direct_io_fd < 0) + return log_debug_errno( + non_direct_io_fd, + "Failed to reopen file descriptor without O_DIRECT: %m"); + + safe_close(reopened_fd); + fd = config.fd = /* For cleanups */ reopened_fd = non_direct_io_fd; + } + /* Wait some random time, to make collision less likely. Let's pick a random time in the * range 0ms…250ms, linearly scaled by the number of failed attempts. */ usec = random_u64_range(UINT64_C(10) * USEC_PER_MSEC +