collector: add collect of certificates

Add explicit collect of certificates files for the platform. Delete all crt, pem and key files in collect before including those files explicitly listed in certs.include file. Use openssl command to omit all but certificate information from the files. Test Plan: PASS AIO-SX, AIO-DX+, DC with AIO-SX subcloud PASS options --skip-mask, --omit-certs, --subcloud PASS bashate PASS unit test for collect_certificates PASS collect output contains no crt, pem and key files except those listed in certs.include PASS If a file listed in certs.include does not exist on the filesystem then the absence is ignored - it is ok for a specified file not to exist. PASS file with key omits key - only certs are copied PASS manual verify of file paths (including those on DC subcloud) Closes-Bug: 2029302 Change-Id: I9fafe5fde39a1a7de9a887424f274986b13e053a Signed-off-by: Michel Thebeau <Michel.Thebeau@windriver.com>
2023-07-27 20:29:29 +00:00 · 2023-07-27 20:29:29 +00:00 · 50c17f8e57
commit 50c17f8e57
parent 36170e3457
6 changed files with 137 additions and 10 deletions
--- a/tools/collector/debian-scripts/certs.include
+++ b/tools/collector/debian-scripts/certs.include
@ -0,0 +1,42 @@
+# certs.include format:
+#
+# Ignore anything that does not start with slash
+# Replace "%%RELEASE%%" with the cluster's current release
+# If the line ends with slash treat it like a directory
+# Otherwise, treat it like a file
+
+# 1. k8s certificates:
+/etc/kubernetes/pki/
+/etc/etcd/
+/var/lib/kubelet/pki/kubelet-client-current.pem
+/var/lib/kubelet/pki/kubelet.crt
+
+# 2. DC admin endpoint certificates
+/etc/ssl/private/admin-ep-cert.pem
+/opt/platform/config/%%RELEASE%%/dc-adminep-root-ca.crt
+
+# 3. docker registry certificates
+/etc/ssl/private/registry-cert.crt
+/etc/docker/certs.d/registry.local:9001/registry-cert.crt
+/etc/docker/certs.d/registry.central:9001/registry-cert.crt
+
+# 4. openldap certificates
+/etc/ldap/certs/openldap-cert.crt
+
+# 5. GUI/REST API certificates
+/etc/ssl/private/server-cert.pem
+
+# 6. Installed ssl CA certificates
+/etc/pki/ca-trust/source/anchors/
+# The following path is hardcoded with regex in collect_certificates:
+# /opt/platform/config/%%RELEASE%%/ssl_ca/ssl_ca_[0-9]{20}
+
+# 7. ceph
+/run/ceph/mgr/restful.crt
+
+# 8. platform config
+/opt/platform/config/%%RELEASE%%/
+/opt/platform/config/%%RELEASE%%/etcd/
+/opt/platform/config/%%RELEASE%%/kubernetes/pki/
+/opt/platform/config/%%RELEASE%%/registry.central/registry-cert.crt
+/opt/platform/config/%%RELEASE%%/ca-cert.pem
--- a/tools/collector/debian-scripts/collect
+++ b/tools/collector/debian-scripts/collect
@ -382,6 +382,8 @@ function print_help()
    echo ""
    echo "collect [--skip-mask]                           ... skip masking of collect data"
    echo ""
+    echo "collect [--omit-certs]                          ... do not include certificates in the collect data"
+    echo ""
    echo "Create a collect report"
    echo ""
    echo "collect [--report | -r ]                        ... run the collect report tool on the collected bundle"
@ -408,6 +410,7 @@ CLEAN=false
 REPORT=false
 VERBOSE=false
 SKIP_MASK=false
+OMIT_CERTS=false
 INVENTORY=false
 SUBCLOUD_COLLECT=false
 SUBCLOUD_LOGIN_PROMPT="controller-"
@ -745,6 +748,10 @@ while [[ ${#} -gt 0 ]] ; do
        SKIP_MASK=true
        ;;

+        --omit-certs)
+        OMIT_CERTS=true
+        ;;
+
        -in|--inline)
        # switch to inline ; one-after-the-other (legacy) mode
        PARALLEL_COLLECT_MODE=false
@ -815,6 +822,7 @@ dlog "INVENTORY = ${INVENTORY}"
 dlog "STARTDATE = ${STARTDATE}"
 dlog "ENDDATE   = ${ENDDATE}"
 dlog "SKIPMASK  = ${SKIP_MASK}"
+dlog "OMITCERTS = ${OMIT_CERTS}"
 dlog "ALLHOSTS  = ${ALLHOSTS}"
 dlog "LISTING   = ${LISTING}"
 dlog "CLEAN     = ${CLEAN}"
@ -1813,7 +1821,7 @@ function collect_host_run()
        spawn bash -i

        set timeout ${TIMEOUT}
-        send "sudo SKIP_MASK=${SKIP_MASK} ${collect_host} ${TARNAME} ${STARTDATE_OPTION} ${STARTDATE} ${STARTTIME} ${ENDDATE_OPTION} ${ENDDATE} ${ENDTIME} ${VERBOSE} ${INVENTORY}\n"
+        send "sudo OMIT_CERTS=${OMIT_CERTS} SKIP_MASK=${SKIP_MASK} ${collect_host} ${TARNAME} ${STARTDATE_OPTION} ${STARTDATE} ${STARTTIME} ${ENDDATE_OPTION} ${ENDDATE} ${ENDTIME} ${VERBOSE} ${INVENTORY}\n"
        expect {
            "assword:" {
                send "${pw}\r"
@ -1847,7 +1855,7 @@ EOF
                expect {
                    "${host}:" {
                        set timeout ${COLLECT_HOST_TIMEOUT}
-                        send "sudo SKIP_MASK=${SKIP_MASK} ${collect_host} ${TARNAME} ${STARTDATE_OPTION} ${STARTDATE} ${STARTTIME} ${ENDDATE_OPTION} ${ENDDATE} ${ENDTIME} ${VERBOSE} ${INVENTORY}\n"
+                        send "sudo OMIT_CERTS=${OMIT_CERTS} SKIP_MASK=${SKIP_MASK} ${collect_host} ${TARNAME} ${STARTDATE_OPTION} ${STARTDATE} ${STARTTIME} ${ENDDATE_OPTION} ${ENDDATE} ${ENDTIME} ${VERBOSE} ${INVENTORY}\n"
                        expect {
                            "assword:" {
                                send "${pw}\r"
--- a/tools/collector/debian-scripts/collect_certificates
+++ b/tools/collector/debian-scripts/collect_certificates
@ -0,0 +1,70 @@
+#! /bin/bash
+#
+# Copyright (c) 2023 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+EXTRA_DIR="$1"
+CERT_DIR="${EXTRA_DIR}/certs"
+CERTS_INCLUDE="/etc/collect/certs.include"
+
+# sw_version is exported by collect_host
+RELEASE="$sw_version"
+
+# Log file is exported from collect_host
+LOGF="$COLLECT_ERROR_LOG"
+
+# Read only lines beginning with slash,
+# and replace %%RELEASE%% with the sw_version
+# Include a hardcoded search for ssl_ca in /opt/platform/config
+INCLUDE_LIST="$(
+    grep "^/" "$CERTS_INCLUDE" \
+    | sed "s;%%RELEASE%%;${RELEASE};";
+    ls "/opt/platform/config/$RELEASE/ssl_ca/ssl_ca_"* 2>/dev/null \
+    | grep "/ssl_ca_[0-9]\{20\}$" )"
+
+function read_cert {
+    local certf="$1"
+    local outf
+    local based
+
+    # Put a copy of the cert file in EXTRA_DIR, using the file's full
+    # path within that space.
+    # All listed files start with slash, per global INCLUDE_LIST.
+    outf="${CERT_DIR}${certf}"
+    based="$( dirname "$outf" )"
+
+    if [ -f "$certf" ]; then
+        # Use openssl to retrieve only certificates from the file.
+        # The output includes some extra lines like this, but which
+        # doesn't affect inspection using openssl command
+        # 0: Certificate
+        # <snip>certificate data</snip>
+        # 1: Certificate
+        # <snip>certificate data</snip>
+        # Total found: 2
+        mkdir -p "$based"
+        openssl storeutl -certs "$certf" > "${outf}" 2>>$LOGF
+    fi
+}
+
+function read_certs_path {
+    local certd="$1"
+    local crtf
+
+    # copy certificates in certd, from files ending in .crt
+    while read crtf; do
+        read_cert "$crtf"
+    done <<<"$( ls -1 "${certd}"*.crt 2>/dev/null )"
+}
+
+while read fpath; do
+    if [[ "$fpath" =~ /$ ]]; then
+        # the path is a directory
+        read_certs_path "$fpath"
+    else
+        read_cert "$fpath"
+    fi
+done <<<"$INCLUDE_LIST"
+
--- a/tools/collector/debian-scripts/collect_host
+++ b/tools/collector/debian-scripts/collect_host
@ -450,6 +450,14 @@ if [ "${SKIP_MASK}" != "true" ]; then
    log_space "after passwd masking :"
 fi

+if [ "${OMIT_CERTS}" != "true" ]; then
+    # Collect certificates from the host
+    dlog "running /usr/local/sbin/collect_certificates ${EXTRA_DIR}"
+    COLLECT_ERROR_LOG="$COLLECT_ERROR_LOG" \
+        /usr/local/sbin/collect_certificates ${EXTRA_DIR}
+    log_space "after collecting certificates :"
+fi
+
 (cd ${COLLECT_BASE_DIR} ; ${IONICE_CMD} ${NICE_CMD} ${TAR_ZIP_CMD} ${COLLECT_NAME_DIR}.tgz ${COLLECT_NAME} 2>/dev/null 1>/dev/null )

 log_space "after first tarball .:"
--- a/tools/collector/debian-scripts/collect_mask_passwords
+++ b/tools/collector/debian-scripts/collect_mask_passwords
@ -60,18 +60,15 @@ do
               s/\{default_pass, <<\".*\">>\}/\{default_pass, <<\"xxxxxx\">>\}/' $conffile
 done

-find ${COLLECT_NAME_DIR} -name server-cert.pem | xargs --no-run-if-empty rm -f
+# Remove all certificate and key files.  Certificates without secrets
+# will be added to collect by collect_certificates.
+find ${COLLECT_NAME_DIR} -name "*.pem" -o -name "*.crt" -o -name "*.key" \
+    | xargs --no-run-if-empty rm -f
+# Remove ssh config and platform secrets files
 rm -rf ${COLLECT_NAME_DIR}/var/extra/platform/config/*/ssh_config
 rm -f ${COLLECT_NAME_DIR}/var/extra/platform/puppet/*/hieradata/secure*.yaml
 rm -f ${COLLECT_NAME_DIR}/etc/puppet/cache/hieradata/secure*.yaml

-# dir /etc/kubernetes/pki was etc.excluded
-if [ -d "/etc/kubernetes/pki" ] ; then
-    # grab the public certificates if /etc/kubernetes/pki exists
-    mkdir -p ${COLLECT_NAME_DIR}/etc/kubernetes/pki
-    cp -a /etc/kubernetes/pki/*.crt ${COLLECT_NAME_DIR}/etc/kubernetes/pki 2>/dev/null 1>/dev/null
-fi
-
 # Mask user passwords in sysinv db dump
 if [ -f ${COLLECT_NAME_DIR}/var/extra/database/sysinv.db.sql.txt ]; then
    sed -i -r '/COPY i_user/, /^--/ s/^(([^\t]*\t){10})[^\t]*(\t.*)/\1xxxxxx\3/;
--- a/tools/collector/debian/deb_folder/rules
+++ b/tools/collector/debian/deb_folder/rules
@ -28,6 +28,7 @@ override_dh_auto_install:
 	install -m 755 -p collect_parms $(ROOT)/usr/local/sbin/collect_parms
 	install -m 755 -p collect_timeouts $(SYSCONFDIR)/collect/collect_timeouts
 	install -m 755 -p collect_mask_passwords $(ROOT)/usr/local/sbin/collect_mask_passwords
+	install -m 755 -p collect_certificates $(ROOT)/usr/local/sbin/collect_certificates
 	install -m 755 -p expect_done $(ROOT)/usr/local/sbin/expect_done
 	install -m 755 -p mariadb-cli.sh $(ROOT)/usr/local/sbin/mariadb-cli

@ -93,6 +94,7 @@ override_dh_auto_install:
 	install -m 755 -p etc.exclude $(SYSCONFDIR)/collect/etc.exclude
 	install -m 755 -p run.exclude $(SYSCONFDIR)/collect/run.exclude
 	install -m 755 -p varlog.exclude $(SYSCONFDIR)/collect/varlog.exclude
+	install -m 755 -p certs.include $(SYSCONFDIR)/collect/certs.include

 	ln -sf /usr/local/sbin/collect $(SBINDIR)/collect
 	ln -sf /usr/local/sbin/collect $(ROOT)/usr/local/bin/collect