diff --git a/tests/check-logs.sh b/tests/check-logs.sh index 9e4c659504..c05f480284 100755 --- a/tests/check-logs.sh +++ b/tests/check-logs.sh @@ -9,13 +9,19 @@ set -o pipefail export PYTHONUNBUFFERED=1 -function check_file_for_level { +function check_openstack_log_file_for_level { # $1: file # $2: log level # Filter out false positives from logged config options. sudo egrep " $2 " $1 | egrep -v "(logging_exception_prefix|rate_limit_except_level)" } +function check_fluentd_log_file_for_level { + # $1: file + # $2: log level + sudo egrep "\[$2\]" $1 +} + function filter_out_expected_critical { # $1: file # Filter out expected critical log messages that we do not want to fail the @@ -35,7 +41,7 @@ function filter_out_expected_critical { *) # We have to provide some pass-through consumer to avoid: # grep: write error: Broken pipe - # from check_file_for_level + # from check_openstack_log_file_for_level cat ;; esac @@ -49,15 +55,15 @@ for level in CRITICAL ERROR WARNING; do any_matched=0 echo "Checking for $level log messages" for f in $(sudo find /var/log/kolla/ -type f); do - if check_file_for_level $f $level >/dev/null; then + if check_openstack_log_file_for_level $f $level >/dev/null; then any_matched=1 if [[ $level = CRITICAL ]]; then - if check_file_for_level $f $level | filter_out_expected_critical $f >/dev/null; then + if check_openstack_log_file_for_level $f $level | filter_out_expected_critical $f >/dev/null; then any_critical=1 fi fi echo $f >> $all_file - check_file_for_level $f $level >> $all_file + check_openstack_log_file_for_level $f $level >> $all_file echo >> $all_file fi done @@ -66,6 +72,18 @@ for level in CRITICAL ERROR WARNING; do fi done + +# check fluentd errors (we consider them critical) +fluentd_log_file=/var/log/kolla/fluentd/fluentd.log +fluentd_error_summary_file=/tmp/logs/kolla/fluentd-error.log +if check_fluentd_log_file_for_level $fluentd_log_file error >/dev/null; then + any_critical=1 + echo "(critical) Found some error log messages in fluentd logs. Matches in $fluentd_error_summary_file" + check_fluentd_log_file_for_level $fluentd_log_file error > $fluentd_error_summary_file + echo >> $fluentd_error_summary_file +fi + + if [[ $any_critical -eq 1 ]]; then echo "Found critical log messages - failing job." exit 1