From 581544a03715c6e123b0a541fb4aacd6b4c1ea80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= <radoslaw.piliszek@gmail.com>
Date: Wed, 29 Dec 2021 10:15:55 +0000
Subject: [PATCH] [CI] Check fluentd errors
We consider them critical and prefer to fail the jobs.
Change-Id: I3c2475522f8f37e3b608d3c321aee27277fc9dee
---
tests/check-logs.sh | 28 +++++++++++++++++++++++-----
1 file changed, 23 insertions(+), 5 deletions(-)
diff --git a/tests/check-logs.sh b/tests/check-logs.sh
index d1795a9948..a47e89e835 100755
--- a/tests/check-logs.sh
+++ b/tests/check-logs.sh
@@ -9,13 +9,19 @@ set -o pipefail
export PYTHONUNBUFFERED=1
-function check_file_for_level {
+function check_openstack_log_file_for_level {
# $1: file
# $2: log level
# Filter out false positives from logged config options.
sudo egrep " $2 " $1 | egrep -v "(logging_exception_prefix|rate_limit_except_level)"
}
+function check_fluentd_log_file_for_level {
+ # $1: file
+ # $2: log level
+ sudo egrep "\[$2\]" $1
+}
+
function filter_out_expected_critical {
# $1: file
# Filter out expected critical log messages that we do not want to fail the
@@ -34,7 +40,7 @@ function filter_out_expected_critical {
*)
# We have to provide some pass-through consumer to avoid:
# grep: write error: Broken pipe
- # from check_file_for_level
+ # from check_openstack_log_file_for_level
cat
;;
esac
@@ -48,15 +54,15 @@ for level in CRITICAL ERROR WARNING; do
any_matched=0
echo "Checking for $level log messages"
for f in $(sudo find /var/log/kolla/ -type f); do
- if check_file_for_level $f $level >/dev/null; then
+ if check_openstack_log_file_for_level $f $level >/dev/null; then
any_matched=1
if [[ $level = CRITICAL ]]; then
- if check_file_for_level $f $level | filter_out_expected_critical $f >/dev/null; then
+ if check_openstack_log_file_for_level $f $level | filter_out_expected_critical $f >/dev/null; then
any_critical=1
fi
fi
echo $f >> $all_file
- check_file_for_level $f $level >> $all_file
+ check_openstack_log_file_for_level $f $level >> $all_file
echo >> $all_file
fi
done
@@ -65,6 +71,18 @@ for level in CRITICAL ERROR WARNING; do
fi
done
+
+# check fluentd errors (we consider them critical)
+fluentd_log_file=/var/log/kolla/fluentd/fluentd.log
+fluentd_error_summary_file=/tmp/logs/kolla/fluentd-error.log
+if check_fluentd_log_file_for_level $fluentd_log_file error >/dev/null; then
+ any_critical=1
+ echo "(critical) Found some error log messages in fluentd logs. Matches in $fluentd_error_summary_file"
+ check_fluentd_log_file_for_level $fluentd_log_file error > $fluentd_error_summary_file
+ echo >> $fluentd_error_summary_file
+fi
+
+
if [[ $any_critical -eq 1 ]]; then
echo "Found critical log messages - failing job."
exit 1