From 276c8182c1e57345ff67c36cb6dbbcddb09a27d7 Mon Sep 17 00:00:00 2001 From: "Jason M. Bills" Date: Mon, 25 Nov 2019 13:23:08 -0800 Subject: Add rsyslog configuration for Redfish Log and IPMI SEL Persisting the journal to flash introduced some significant issues, so this uses an rsyslog configuration to extract any IPMI SEL and Redfish events from the journal and persist them to flash in a text file. This allows many more events to be stored without any adverse issues in system behavior. Tested: Used IPMI and Redfish to retrieve logs before and after a BMC reboot and confirmed that they are correctly persisted. (From meta-intel rev: 176b19e807890f6014157d7b1b705e96c0fc4878) Change-Id: I02fac9af333d854204d41e238ae598bad7782328 Signed-off-by: Jason M. Bills Signed-off-by: Brad Bishop --- .../rsyslog/rsyslog/rotate-event-logs.service | 9 +++ .../rsyslog/rsyslog/rotate-event-logs.timer | 8 +++ .../rsyslog/rsyslog/rsyslog-override.conf | 2 + .../recipes-extended/rsyslog/rsyslog/rsyslog.conf | 79 ++++++++++++++++++++++ .../rsyslog/rsyslog/rsyslog.logrotate | 22 ++++++ .../recipes-extended/rsyslog/rsyslog_%.bbappend | 23 +++++++ 6 files changed, 143 insertions(+) create mode 100644 meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rotate-event-logs.service create mode 100644 meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rotate-event-logs.timer create mode 100644 meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rsyslog-override.conf create mode 100644 meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rsyslog.conf create mode 100644 meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rsyslog.logrotate create mode 100644 meta-intel/meta-common/recipes-extended/rsyslog/rsyslog_%.bbappend (limited to 'meta-intel/meta-common') diff --git a/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rotate-event-logs.service b/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rotate-event-logs.service new file mode 100644 index 000000000..fdeefd417 --- /dev/null +++ b/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rotate-event-logs.service @@ -0,0 +1,9 @@ +[Unit] +Description=Rotates the event logs + +[Service] +Type=oneshot +ExecStart=/usr/sbin/logrotate /etc/logrotate.d/logrotate.rsyslog + +[Install] +WantedBy=multi-user.target diff --git a/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rotate-event-logs.timer b/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rotate-event-logs.timer new file mode 100644 index 000000000..148f8e4ae --- /dev/null +++ b/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rotate-event-logs.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Run rotate-event-logs.service every minute + +[Timer] +OnCalendar=*-*-* *:*:00 + +[Install] +WantedBy=timers.target diff --git a/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rsyslog-override.conf b/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rsyslog-override.conf new file mode 100644 index 000000000..14bcc0781 --- /dev/null +++ b/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rsyslog-override.conf @@ -0,0 +1,2 @@ +[Service] +ExecReload=/bin/kill -HUP $MAINPID diff --git a/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rsyslog.conf b/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rsyslog.conf new file mode 100644 index 000000000..46a287eef --- /dev/null +++ b/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rsyslog.conf @@ -0,0 +1,79 @@ +# if you experience problems, check +# http://www.rsyslog.com/troubleshoot for assistance + +# rsyslog v3: load input modules +# If you do not load inputs, nothing happens! +# You may need to set the module load path if modules are not found. +# +# Ported from debian's sysklogd.conf + +# Journal-style logging +# Limit to no more than 2000 entries in one minute and enable the +# journal workaround to avoid duplicate entries +module(load="imjournal" StateFile="/var/log/state" + RateLimit.Interval="60" + RateLimit.Burst="2000") + +# Template for IPMI SEL messages +# " ,,,[,,]" +template(name="IPMISELTemplate" type="list") { + property(name="timereported" dateFormat="rfc3339") + constant(value=" ") + property(name="$!IPMI_SEL_RECORD_ID") + constant(value=",") + property(name="$!IPMI_SEL_RECORD_TYPE") + constant(value=",") + property(name="$!IPMI_SEL_DATA") + constant(value=",") + property(name="$!IPMI_SEL_GENERATOR_ID") + constant(value=",") + property(name="$!IPMI_SEL_SENSOR_PATH") + constant(value=",") + property(name="$!IPMI_SEL_EVENT_DIR") + constant(value="\n") +} + +# Template for Redfish messages +# " ," +template(name="RedfishTemplate" type="list") { + property(name="timereported" dateFormat="rfc3339") + constant(value=" ") + property(name="$!REDFISH_MESSAGE_ID") + constant(value=",") + property(name="$!REDFISH_MESSAGE_ARGS") + constant(value="\n") +} + +# Template for Application Crashes +# " ," +template(name="CrashTemplate" type="list") { + property(name="timereported" dateFormat="rfc3339") + constant(value=" ") + constant(value="OpenBMC.0.1.ServiceFailure") + constant(value=",") + property(name="$!UNIT") + constant(value="\n") +} + + +# If the journal entry has the IPMI SEL MESSAGE_ID, save as IPMI SEL +# The MESSAGE_ID string is generated using journalctl and must match the +# MESSAGE_ID used in IPMI to correctly find the SEL entries. +if ($!MESSAGE_ID == "b370836ccf2f4850ac5bee185b77893a") then { + action(type="omfile" file="/var/log/ipmi_sel" template="IPMISELTemplate") +} + +# If the journal entry has a Redfish MessageId, save as a Redfish event +if ($!REDFISH_MESSAGE_ID != "") then { + action(type="omfile" file="/var/log/redfish" template="RedfishTemplate") +} + +# If the journal entry has a Exit Code, save as a Redfish event +if ($!EXIT_STATUS != "" and $!EXIT_STATUS != "0") then { + action(type="omfile" file="/var/log/redfish" template="CrashTemplate") +} + +# +# Include all config files in /etc/rsyslog.d/ +# +$IncludeConfig /etc/rsyslog.d/*.conf diff --git a/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rsyslog.logrotate b/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rsyslog.logrotate new file mode 100644 index 000000000..a6ba28d86 --- /dev/null +++ b/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog/rsyslog.logrotate @@ -0,0 +1,22 @@ +# /etc/logrotate.d/rsyslog - Ported from Debian + +# Keep up to four 64k files for ipmi_sel (256k total) +/var/log/ipmi_sel +{ + rotate 3 + size 64k + missingok + postrotate + systemctl reload rsyslog 2> /dev/null || true + endscript +} +# Keep up to four 64k files for redfish (256k total) +/var/log/redfish +{ + rotate 3 + size 64k + missingok + postrotate + systemctl reload rsyslog 2> /dev/null || true + endscript +} diff --git a/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog_%.bbappend b/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog_%.bbappend new file mode 100644 index 000000000..7e282804e --- /dev/null +++ b/meta-intel/meta-common/recipes-extended/rsyslog/rsyslog_%.bbappend @@ -0,0 +1,23 @@ +FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:" + +SRC_URI += "file://rsyslog.conf \ + file://rsyslog.logrotate \ + file://rotate-event-logs.service \ + file://rotate-event-logs.timer \ + file://rsyslog-override.conf \ +" + +FILES_${PN} += "${systemd_system_unitdir}/rsyslog.service.d/rsyslog-override.conf" + +PACKAGECONFIG_append = " imjournal" + +do_install_append() { + install -m 0644 ${WORKDIR}/rotate-event-logs.service ${D}${systemd_system_unitdir} + install -m 0644 ${WORKDIR}/rotate-event-logs.timer ${D}${systemd_system_unitdir} + install -d ${D}${systemd_system_unitdir}/rsyslog.service.d + install -m 0644 ${WORKDIR}/rsyslog-override.conf \ + ${D}${systemd_system_unitdir}/rsyslog.service.d/rsyslog-override.conf + rm ${D}${sysconfdir}/rsyslog.d/imjournal.conf +} + +SYSTEMD_SERVICE_${PN} += " rotate-event-logs.service rotate-event-logs.timer" -- cgit v1.2.3