firmware-watchdog, detect virtualization

13 files changed, 490 insertions, 7 deletions

diff --git a/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog.bb b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog.bb
new file mode 100644
index 0000000000..1d17ee02e2
--- /dev/null
+++ b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog.bb
@@ -0,0 +1,59 @@
+SUMMARY = "Firmware watchdog"
+DESCRIPTION = "BMC firmware watchdog service that is used to detect BMC status \
+               after image install or update \
+              "
+
+inherit allarch
+inherit obmc-phosphor-systemd
+
+RDEPENDS:${PN} = "bash"
+
+LICENSE = "Apache-2.0"
+LIC_FILES_CHKSUM = "file://${IBSBASE}/COPYING.apache-2.0;md5=34400b68072d710fecd0a2940a0d1658"
+
+
+SYSTEMD_SERVICE:${PN} += "firmware-watchdog.service"
+SYSTEMD_ENVIRONMENT_FILE:${PN} += "obmc/firmware-watchdog/firmware-watchdog.conf"
+
+SYSTEMD_SERVICE:${PN} += "firmware-watchdog.service"
+SYSTEMD_SERVICE:${PN} += "firmware-watchdog.timer"
+SRC_URI += "file://firmware-watchdog \
+            file://logrotate    \
+            file://post-install \
+            file://post-upgrade \
+           "
+
+do_install:append(){
+    install -d ${D}${sbindir}
+    install -m 0755 ${WORKDIR}/firmware-watchdog   ${D}${sbindir}
+    install -m 0755 ${WORKDIR}/post-install        ${D}${sbindir}
+    install -m 0755 ${WORKDIR}/post-upgrade        ${D}${sbindir}
+
+    install -d ${D}${sysconfdir}/logrotate.d
+    install -m 0644 ${WORKDIR}/logrotate           ${D}${sysconfdir}/logrotate.d/firmware-watchdog
+}
+
+
+#
+#S = "${WORKDIR}"
+#SRC_URI = "file://kernel-panic-check.sh \
+#           file://kernel-panic-check.service \
+#"
+#
+#LICENSE = "Apache-2.0"
+#LIC_FILES_CHKSUM = "file://${IBSBASE}/COPYING.apache-2.0;md5=34400b68072d710fecd0a2940a0d1658"
+#RDEPENDS:${PN} += "bash logger-systemd"
+#
+#inherit systemd
+#
+#FILES:${PN} += "${systemd_system_unitdir}/kernel-panic-check.service"
+#
+#do_install() {
+#    install -d ${D}${systemd_system_unitdir}
+#    install -m 0644 ${WORKDIR}/kernel-panic-check.service ${D}${systemd_system_unitdir}
+#    install -d ${D}${bindir}
+#    install -m 0755 ${S}/kernel-panic-check.sh ${D}/${bindir}/kernel-panic-check.sh
+#}
+#
+#SYSTEMD_SERVICE:${PN} += " kernel-panic-check.service"
+#
\ No newline at end of file
diff --git a/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/firmware-watchdog b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/firmware-watchdog
new file mode 100644
index 0000000000..9e5d6b5e3b
--- /dev/null
+++ b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/firmware-watchdog
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+CONFIG_FILE=/etc/default/obmc/firmware-watchdog/firmware-watchdog.conf
+LOG_FILE=/var/log/firmware-watchdog.log
+
+. ${CONFIG_FILE}
+
+set_status() {
+  sed -i "s,\(^[ \t]*SYSTEM_STATUS=\).*,\1$1," ${CONFIG_FILE}
+}
+
+set_counter() {
+  sed -i "s,\(^[ \t]*REQUEST_COUNTER=\).*,\1$1," ${CONFIG_FILE}
+}
+
+increment_counter() {
+  counter=${REQUEST_COUNTER}
+  let 'counter +=1'
+  sed -i "s,\(^[ \t]*REQUEST_COUNTER=\).*,\1${counter}," ${CONFIG_FILE}
+}
+
+push_log() {
+  message=$1
+  echo -n "[`LANG=en LANGUAGE=en date +'%d-%b-%Y %H:%M:%S'`] system status: " >> ${LOG_FILE}
+  echo    "${message}" >> ${LOG_FILE}
+}
+
+
+if [ "${REQUEST_COUNTER}" -ge "0" -a "${REQUEST_COUNTER}" -lt "9" ] ; then
+  if `systemctl is-active --quiet bmcweb` ; then
+    #
+    # bmcweb service is active:
+    # ------------------------
+
+    #
+    # Run post-{install|upgrade} script:
+    #
+    if [ "${SYSTEM_STATUS}" = "pristine" ] ; then
+      # Firstly installed clean system:
+      push_log "${SYSTEM_STATUS}: Attempt to run post-install routine"
+      if [ -x "/usr/sbin/post-install" ] ; then
+        /usr/sbin/post-install &  2>/dev/null 1>/dev/null
+      fi
+    fi
+    if [ "${SYSTEM_STATUS}" = "upgraded" ] ; then
+      # The clean system after upgrade:
+      push_log "${SYSTEM_STATUS}: Attempt to run post-upgrade routine"
+      if [ -x "/usr/sbin/post-upgrade" ] ; then
+        /usr/sbin/post-upgrade &  2>/dev/null 1>/dev/null
+      fi
+    fi
+
+    set_status operative
+    push_log "operative"
+
+    # Detect Virtualization (testing):
+    if `dmesg | grep -q "aspeed-smc 1e620000.spi: mx25l25635e"` ; then
+      /usr/bin/busctl set-property xyz.openbmc_project.FruDevice /xyz/openbmc_project/FruDevice/CP2_5422 xyz.openbmc_project.FruDevice PRODUCT_ASSET_TAG s "Virtual BMC"
+    fi
+
+    set_counter 9
+    exit 0
+  fi
+fi
+
+if [ "${REQUEST_COUNTER}" -gt "8" ] ; then
+  if [ "${SYSTEM_STATUS}" = "pristine" -o "${SYSTEM_STATUS}" = "upgraded" ] ; then
+    #
+    # bmcweb service is dead:
+    # ----------------------
+    set_status dirty
+    push_log "dirty: because bmcweb.serice is dead"
+  fi
+fi
+
+if [ "${REQUEST_COUNTER}" -lt "9" ] ; then
+  increment_counter
+fi
+
+exit 0
diff --git a/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/firmware-watchdog.service b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/firmware-watchdog.service
new file mode 100644
index 0000000000..709fcdcce7
--- /dev/null
+++ b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/firmware-watchdog.service
@@ -0,0 +1,9 @@
+
+[Unit]
+Description=Firmware OpenBMC Watchdog Service
+
+[Service]
+EnvironmentFile=/etc/default/obmc/firmware-watchdog/firmware-watchdog.conf
+Type=oneshot
+ExecStart=/usr/sbin/firmware-watchdog
+RemainAfterExit=true
diff --git a/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/firmware-watchdog.timer b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/firmware-watchdog.timer
new file mode 100644
index 0000000000..3fe47cd911
--- /dev/null
+++ b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/firmware-watchdog.timer
@@ -0,0 +1,12 @@
+
+[Unit]
+Description=Firmware OpenBMC Watchdog Timer (every first minute)
+
+[Timer]
+Unit=firmware-watchdog.service
+OnBootSec=60sec
+OnUnitActiveSec=60sec
+AccuracySec=5sec
+
+[Install]
+WantedBy=timers.target
diff --git a/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/logrotate b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/logrotate
new file mode 100644
index 0000000000..428477c6bf
--- /dev/null
+++ b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/logrotate
@@ -0,0 +1,5 @@
+/var/log/firmware-watchdog.log {
+    missingok
+    notifempty
+    size 30k
+}
diff --git a/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/obmc/firmware-watchdog/firmware-watchdog.conf b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/obmc/firmware-watchdog/firmware-watchdog.conf
new file mode 100644
index 0000000000..c1563091b3
--- /dev/null
+++ b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/obmc/firmware-watchdog/firmware-watchdog.conf
@@ -0,0 +1,10 @@
+#
+# Available values of SYSTEM_STATUS:
+# =================================
+#
+#  pristine  - firstly installed clean system;
+#  upgraded  - the clean system after upgrade;
+#  operative - normal .
+#
+SYSTEM_STATUS=pristine
+REQUEST_COUNTER=0
diff --git a/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/post-install b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/post-install
new file mode 100644
index 0000000000..55dc4951cf
--- /dev/null
+++ b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/post-install
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+/usr/bin/bmcweb-init-certs &  2>/dev/null 1>/dev/null
+
+exit 0
diff --git a/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/post-upgrade b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/post-upgrade
new file mode 100644
index 0000000000..55dc4951cf
--- /dev/null
+++ b/meta-ibs/meta-common/recipes-ibs/firmware-watchdog/firmware-watchdog/post-upgrade
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+/usr/bin/bmcweb-init-certs &  2>/dev/null 1>/dev/null
+
+exit 0
diff --git a/meta-ibs/meta-common/recipes-ibs/images/obmc-phosphor-image.bbappend b/meta-ibs/meta-common/recipes-ibs/images/obmc-phosphor-image.bbappend
index 0e0a77c4c8..5dad389c66 100644
--- a/meta-ibs/meta-common/recipes-ibs/images/obmc-phosphor-image.bbappend
+++ b/meta-ibs/meta-common/recipes-ibs/images/obmc-phosphor-image.bbappend
@@ -7,7 +7,8 @@ IMAGE_INSTALL += " openssl-bin \
                    phosphor-post-code-manager \
                    beepcode-mgr \
                    biosconfig-manager \
-                   bmcweb-init-certs"
+                   bmcweb-init-certs \
+                   firmware-watchdog"
 
 OBMC_IMAGE_EXTRA_INSTALL += " strace"
 
diff --git a/meta-ibs/meta-common/recipes-phosphor/initrdscripts/obmc-phosphor-initfs.bbappend b/meta-ibs/meta-common/recipes-phosphor/initrdscripts/obmc-phosphor-initfs.bbappend
index 61fefda887..339740dfbd 100644
--- a/meta-ibs/meta-common/recipes-phosphor/initrdscripts/obmc-phosphor-initfs.bbappend
+++ b/meta-ibs/meta-common/recipes-phosphor/initrdscripts/obmc-phosphor-initfs.bbappend
@@ -2,4 +2,10 @@ FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"
 
 DESCRIPTION = "Pinging system watchdog before BMC update"
 
-SRC_URI += "file://obmc-shutdown.sh"
+SRC_URI += "file://obmc-shutdown.sh \
+            file://obmc-update.sh \
+           "
+
+do_install:append() {
+    echo "/etc/default/obmc/firmware-watchdog/firmware-watchdog.conf" >> ${D}/whitelist
+}
diff --git a/meta-ibs/meta-common/recipes-phosphor/initrdscripts/obmc-phosphor-initfs/obmc-shutdown.sh b/meta-ibs/meta-common/recipes-phosphor/initrdscripts/obmc-phosphor-initfs/obmc-shutdown.sh
index 204665de7e..e5eadaca2b 100644
--- a/meta-ibs/meta-common/recipes-phosphor/initrdscripts/obmc-phosphor-initfs/obmc-shutdown.sh
+++ b/meta-ibs/meta-common/recipes-phosphor/initrdscripts/obmc-phosphor-initfs/obmc-shutdown.sh
@@ -55,7 +55,7 @@ then
 			wd=
 		fi
 		$update --clean-saved-files
-		remaining=$(ls $image*)
+		remaining=$(ls $image* 2>/dev/null)
 		if test -n "$remaining"
 		then
 			echo 1>&2 "Flash update failed to flash these images:"
diff --git a/meta-ibs/meta-common/recipes-phosphor/initrdscripts/obmc-phosphor-initfs/obmc-update.sh b/meta-ibs/meta-common/recipes-phosphor/initrdscripts/obmc-phosphor-initfs/obmc-update.sh
new file mode 100644
index 0000000000..bd4cfb14d4
--- /dev/null
+++ b/meta-ibs/meta-common/recipes-phosphor/initrdscripts/obmc-phosphor-initfs/obmc-update.sh
@@ -0,0 +1,290 @@
+#!/bin/sh
+
+echo update: "$@"
+
+echoerr() {
+	echo 1>&2 "ERROR: $@"
+}
+
+cd /
+if ! test -r /proc/mounts || ! test -f /proc/mounts
+then
+	mkdir -p /proc
+	mount -t proc proc proc
+fi
+if ! test -d /sys/class
+then
+	mkdir -p /sys
+	mount -t sysfs sys sys
+fi
+if ! test -c /dev/null
+then
+	mkdir -p /dev
+	mount -t devtmpfs dev dev
+fi
+
+# mtd number N with mtd name Name can be mounted via mtdN, or mtd:Name
+# (with a mtd aware fs) or by /dev/mtdblockN (with a mtd or block fs).
+mtdismounted() {
+	m=${1##mtd}
+	if grep -s "mtdblock$m " /proc/mounts || grep -s "mtd$m " /proc/mounts
+	then
+		return 0
+	fi
+	n=$(cat /sys/class/mtd/mtd$m/name)
+	if test -n "$n" && grep -s "mtd:$n " /proc/mounts
+	then
+		return 0
+	fi
+	return 1
+}
+
+# Detect child partitions when the whole flash is to be updated.
+# Ignore mtdNro and mtdblockN names in the class subsystem directory.
+childmtds() {
+	for m in /sys/class/mtd/$1/mtd*
+	do
+		m=${m##*/}
+		if test "${m%ro}" = "${m#mtdblock}"
+		then
+			echo $m
+		fi
+	done
+}
+
+toobig() {
+	if test $(stat -L -c "%s" "$1") -gt $(cat /sys/class/mtd/"$2"/size)
+	then
+		return 0
+	fi
+	return 1
+}
+
+findmtd() {
+	m=$(grep -xl "$1" /sys/class/mtd/*/name)
+	m=${m%/name}
+	m=${m##*/}
+	echo $m
+}
+
+blkid_fs_type() {
+	# Emulate util-linux's `blkid -s TYPE -o value $1`
+	# Example busybox blkid output:
+	#    # blkid /dev/mtdblock5
+	#    /dev/mtdblock5: TYPE="squashfs"
+	# Process output to extract TYPE value "squashfs".
+	blkid $1 | sed -e 's/^.*TYPE="//' -e 's/".*$//'
+}
+
+probe_fs_type() {
+	fst=$(blkid_fs_type $1)
+	echo ${fst:=jffs2}
+}
+
+rwfs=$(findmtd rwfs)
+
+rwdev=/dev/mtdblock${rwfs#mtd}
+rwopts=rw
+rorwopts=ro${rwopts#rw}
+
+rwdir=/run/initramfs/rw
+upper=$rwdir/cow
+save=/run/save/${upper##*/}
+
+mounted=
+doflash=y
+doclean=
+dosave=y
+dorestore=y
+toram=
+checksize=y
+checkmount=y
+
+whitelist=/run/initramfs/whitelist
+image=/run/initramfs/image-
+imglist=
+
+firmware_watchdog_config=$upper/etc/default/obmc/firmware-watchdog/firmware-watchdog.conf
+
+set_fw_watchdog_status() {
+	sed -i "s,\(^[ \t]*SYSTEM_STATUS=\).*,\1$1," ${firmware_watchdog_config}
+}
+
+set_fw_watchdog_counter() {
+	sed -i "s,\(^[ \t]*REQUEST_COUNTER=\).*,\1$1," ${firmware_watchdog_config}
+}
+
+while test "$1" != "${1#-}"
+do
+	case "$1" in
+	--help)
+		cat <<HERE
+Usage: $0 [options] -- Write images in /run/initramfs to flash (/dev/mtd*)
+    --help                    Show this message
+    --no-flash                Don't attempt to write images to flash
+    --ignore-size             Don't compare image size to mtd device size
+    --ignore-mount            Don't check if destination is mounted
+    --save-files              Copy whitelisted files to save directory in RAM
+    --no-save-files           Don't copy whitelisted files to save directory
+    --copy-files              Copy files from save directory to rwfs mountpoint
+    --restore-files           Restore files from save directory to rwfs layer
+    --no-restore-files        Don't restore saved files from ram to rwfs layer
+    --clean-saved-files       Delete saved whitelisted files from RAM
+    --no-clean-saved-files    Retain saved whitelisted files in RAM
+HERE
+
+	    exit 0 ;;
+
+	--no-clean-saved-files)
+		doclean=
+		shift ;;
+	--clean-saved-files)
+		doclean=y
+		shift ;;
+	--no-save-files)
+		dosave=
+		shift ;;
+	--save-files)
+		dosave=y
+		shift ;;
+	--no-restore-files)
+		dorestore=
+		shift ;;
+	--restore-files)
+		dorestore=y
+		shift ;;
+	--no-flash)
+		doflash=
+		shift ;;
+	--ignore-size)
+		checksize=
+		shift ;;
+	--ignore-mount)
+		checkmount=
+		doflash=
+		shift ;;
+	--copy-files)
+		toram=y
+		shift ;;
+	*)
+		echoerr "Unknown option $1.  Try $0 --help."
+		exit 1 ;;
+	esac
+done
+
+if test "x$dosave" = xy
+then
+	if test ! -d $upper -a -n "$rwfs"
+	then
+		mkdir -p $rwdir
+		mount $rwdev $rwdir -t $(probe_fs_type $rwdev) -o $rwopts
+		mounted=$rwdir
+	fi
+
+	if test -f ${firmware_watchdog_config}
+	then
+		set_fw_watchdog_status  upgraded
+		set_fw_watchdog_counter 0
+	fi
+
+	while read f
+	do
+		# Entries shall start with /, no trailing /.. or embedded /../
+		if test "/${f#/}" != "$f" -o "${f%/..}" != "${f#*/../}"
+		then
+			echo 1>&2 "WARNING: Skipping bad whitelist entry $f."
+			continue
+		fi
+		if ! test -e "$upper/$f"
+		then
+			continue
+		fi
+		d="$save/$f"
+		while test "${d%/}" != "${d%/.}"
+		do
+			d="${d%/.}"
+			d="${d%/}"
+		done
+		mkdir -p "${d%/*}"
+		cp -rp "$upper/$f" "${d%/*}/"
+	done < $whitelist
+
+	if test -n "$mounted"
+	then
+		umount $mounted
+	fi
+fi
+
+imglist=$(echo $image*)
+if test "$imglist" = "$image*" -a ! -e "$imglist"
+then
+	# shell didn't expand the wildcard, so no files exist
+	echo "No images found to update."
+	imglist=
+fi
+
+for f in $imglist
+do
+	m=$(findmtd ${f#$image})
+	if test -z "$m"
+	then
+		echoerr "Unable to find mtd partition for ${f##*/}."
+		exit 1
+	fi
+	if test -n "$checksize" && toobig "$f" "$m"
+	then
+		echoerr "Image ${f##*/} too big for $m."
+		exit 1
+	fi
+	for s in $m $(childmtds $m)
+	do
+		if test -n "$checkmount" && mtdismounted $s
+		then
+			echoerr "Device $s is mounted, ${f##*/} is busy."
+			exit 1
+		fi
+	done
+done
+
+if test -n "$doflash"
+then
+	for f in $imglist
+	do
+		if test ! -s $f
+		then
+			echo "Skipping empty update of ${f#$image}."
+			rm $f
+			continue
+		fi
+		m=$(findmtd ${f#$image})
+		echo "Updating ${f#$image}..."
+		flashcp -v $f /dev/$m && rm $f
+	done
+fi
+
+if test -d $save -a "x$toram" = xy
+then
+	mkdir -p $upper
+	cp -rp $save/. $upper/
+fi
+
+if test -d $save -a "x$dorestore" = xy
+then
+	odir=$rwdir
+	rwdir=/run/rw
+	upper=$rwdir${upper#$odir}
+
+	mkdir -p $rwdir
+	mount $rwdev $rwdir -t $(probe_fs_type $rwdev) -o $rwopts
+	mkdir -p $upper
+	cp -rp $save/. $upper/
+	umount $rwdir
+	rmdir $rwdir
+fi
+
+if test "x$doclean" = xy
+then
+	rm -rf $save
+fi
+
+exit
diff --git a/meta-ibs/meta-common/recipes-phosphor/interfaces/bmcweb-init-certs/bmcweb-init-certs b/meta-ibs/meta-common/recipes-phosphor/interfaces/bmcweb-init-certs/bmcweb-init-certs
index ea0d796f7d..d376f9a5d1 100644
--- a/meta-ibs/meta-common/recipes-phosphor/interfaces/bmcweb-init-certs/bmcweb-init-certs
+++ b/meta-ibs/meta-common/recipes-phosphor/interfaces/bmcweb-init-certs/bmcweb-init-certs
@@ -23,13 +23,14 @@ rm -f ${PRIV_KEY_NAME} ${SERV_KEY_NAME}
 #
 # Create new private PEM-key:
 #
-systemctl restart phosphor-certificate-manager@bmcweb.service
+systemctl stop  phosphor-certificate-manager@bmcweb.service
+systemctl start phosphor-certificate-manager@bmcweb.service
 
 count=0
 # wait for 5 minutes until the certificate manager creates the private PEM-key
 while [ ! -f ${PRIV_KEY_NAME} ] && [ ${count} -lt 300 ]
 do
-  count=$[ $count + 1]
+  count=$[ $count + 1 ]
   sleep 1
 done
 
@@ -40,9 +41,9 @@ systemctl restart bmcweb.service
 
 count=0
 # wait for 5 minutes until the bmcweb creates the server PEM-key
-while [ ! -f ${PRIV_KEY_NAME} ] && [ ${count} -lt 300 ]
+while [ ! -f ${SERV_KEY_NAME} ] && [ ${count} -lt 300 ]
 do
-  count=$[ $count + 1]
+  count=$[ $count + 1 ]
   sleep 1
 done