preinit-mounts: Add check for overlay file errors

In some instances when the built image has files in /etc/ that were moved
or renamed from previous builds, a code update to this new image causes
the overlay to appear corrupted. This has also been seen when downgrading
to a level with older kernel and systemd versions, although the root
cause remain unknown.

This change 'fixes' the symptom, by re-creating the overlay if it detects
an error accessing a file.

Tested: Verified the system mounts the overlay successful on a code update
        path known to fail.

(From meta-phosphor rev: f8ed65b2b6389797294b77e5a7ebabe293ca475a)

Change-Id: Iaed73955459e92ffdac3999d0d5b661ae53c487b
Signed-off-by: Adriana Kobylak <anoo@us.ibm.com>
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>

1 files changed, 33 insertions, 3 deletions

diff --git a/meta-phosphor/recipes-phosphor/preinit-mounts/preinit-mounts/init b/meta-phosphor/recipes-phosphor/preinit-mounts/preinit-mounts/init
index 91f8038f9..818a32099 100644
--- a/meta-phosphor/recipes-phosphor/preinit-mounts/preinit-mounts/init
+++ b/meta-phosphor/recipes-phosphor/preinit-mounts/preinit-mounts/init
@@ -1,5 +1,11 @@
 #!/bin/sh
 
+mount_overlay() {
+  if ! mount overlay /etc -t overlay -o defaults,lowerdir=/etc,upperdir=/var/persist/etc,workdir=/var/persist/etc-work; then
+    mount overlay /etc -t overlay -o defaults,lowerdir=/etc:/var/persist/etc
+  fi
+}
+
 if ! mount ubi0:rwfs /var -t ubifs -o defaults; then
   if ! mount ubi0:rwfs /var -t ubifs -o defaults,ro; then
     mount tmpfs /var -t tmpfs -o defaults
@@ -12,8 +18,32 @@ rm -rf /var/persist/etc-work/*
 # rm -rf specifically skips . and .. directories; pipe all output to null to avoid the error message
 rm -rf /var/persist/etc-work/.* > /dev/null 2>&1
 
-if ! mount overlay /etc -t overlay -o defaults,lowerdir=/etc,upperdir=/var/persist/etc,workdir=/var/persist/etc-work; then
-  mount overlay /etc -t overlay -o defaults,lowerdir=/etc:/var/persist/etc
-fi
+mount_overlay
+
+# Check if there are any issues accessing the files in /etc after mounting the
+# overlay by doing an 'ls' command
+error="/var/overlay-error"
+cd /var/persist/etc/
+for i in *; do
+  # Only check regular files at the top of the directory
+  if [[ -f $i ]]; then
+    ls -i /etc/$i >/dev/null 2>${error};
+    if [[ -s ${error} ]]; then
+      # We don't have a way to print this error to the journal, delete it
+      rm -f ${error}
+      # Attempt to re-create the overlay by moving out the overlay contents and
+      # copying them back to /etc, which would create them back in the overlay
+      cd
+      umount /etc
+      rm -rf /var/persist/etc-save
+      mv /var/persist/etc /var/persist/etc-save
+      mkdir -p /var/persist/etc
+      mount_overlay
+      cp -rp /var/persist/etc-save/* /etc/
+      rm -rf /var/persist/etc-save
+      break
+    fi
+  fi
+done
 
 exec /lib/systemd/systemd