From c8d4b55130bcce2dd9c0eb8cd753b363d4b98e82 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Wed, 1 Aug 2018 12:23:58 -0400 Subject: [PATCH] Relocate collector to stx-integ/tools/collector Move content from stx-utils into stx-integ or stx-update Packages will be relocated to stx-update: enable-dev-patch extras stx-integ: config-files/ io-scheduler filesystem/ filesystem-scripts grub/ grubby logging/ logmgmt tools/ collector monitor-tools tools/engtools/ hostdata-collectors parsers utilities/ build-info branding (formerly wrs-branding) platform-util Change-Id: Iefa35db5a644b8ca0f1962e6c4cbbc97213b69e9 Story: 2002801 Task: 22687 Signed-off-by: Scott Little --- centos_pkg_dirs | 1 + tools/collector/LICENSE | 202 +++ tools/collector/centos/build_srpm.data | 2 + tools/collector/centos/collector.spec | 66 + tools/collector/scripts/LICENSE | 202 +++ tools/collector/scripts/collect | 1267 +++++++++++++++++ tools/collector/scripts/collect_ceph.sh | 83 ++ tools/collector/scripts/collect_coredump.sh | 35 + tools/collector/scripts/collect_crash.sh | 30 + tools/collector/scripts/collect_date | 1064 ++++++++++++++ tools/collector/scripts/collect_host | 482 +++++++ tools/collector/scripts/collect_ima.sh | 60 + .../collector/scripts/collect_mask_passwords | 123 ++ tools/collector/scripts/collect_networking.sh | 61 + tools/collector/scripts/collect_nfv_vim.sh | 45 + tools/collector/scripts/collect_openstack.sh | 75 + tools/collector/scripts/collect_parms | 29 + tools/collector/scripts/collect_patching.sh | 45 + tools/collector/scripts/collect_psqldb.sh | 122 ++ tools/collector/scripts/collect_sm.sh | 26 + tools/collector/scripts/collect_sysinv.sh | 73 + tools/collector/scripts/collect_tc.sh | 82 ++ tools/collector/scripts/collect_utils | 223 +++ tools/collector/scripts/collect_vswitch.sh | 108 ++ tools/collector/scripts/etc.exclude | 40 + tools/collector/scripts/expect_done | 1 + tools/collector/scripts/run.exclude | 12 + 27 files changed, 4559 insertions(+) create mode 100644 tools/collector/LICENSE create mode 100644 tools/collector/centos/build_srpm.data create mode 100644 tools/collector/centos/collector.spec create mode 100644 tools/collector/scripts/LICENSE create mode 100755 tools/collector/scripts/collect create mode 100755 tools/collector/scripts/collect_ceph.sh create mode 100644 tools/collector/scripts/collect_coredump.sh create mode 100644 tools/collector/scripts/collect_crash.sh create mode 100755 tools/collector/scripts/collect_date create mode 100755 tools/collector/scripts/collect_host create mode 100755 tools/collector/scripts/collect_ima.sh create mode 100644 tools/collector/scripts/collect_mask_passwords create mode 100755 tools/collector/scripts/collect_networking.sh create mode 100644 tools/collector/scripts/collect_nfv_vim.sh create mode 100755 tools/collector/scripts/collect_openstack.sh create mode 100644 tools/collector/scripts/collect_parms create mode 100755 tools/collector/scripts/collect_patching.sh create mode 100755 tools/collector/scripts/collect_psqldb.sh create mode 100644 tools/collector/scripts/collect_sm.sh create mode 100755 tools/collector/scripts/collect_sysinv.sh create mode 100755 tools/collector/scripts/collect_tc.sh create mode 100755 tools/collector/scripts/collect_utils create mode 100644 tools/collector/scripts/collect_vswitch.sh create mode 100644 tools/collector/scripts/etc.exclude create mode 100755 tools/collector/scripts/expect_done create mode 100644 tools/collector/scripts/run.exclude diff --git a/centos_pkg_dirs b/centos_pkg_dirs index 334835346..1f75a0fa2 100644 --- a/centos_pkg_dirs +++ b/centos_pkg_dirs @@ -111,3 +111,4 @@ logging/logmgmt filesystem/filesystem-scripts utilities/branding config-files/io-scheduler +tools/collector diff --git a/tools/collector/LICENSE b/tools/collector/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/tools/collector/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/collector/centos/build_srpm.data b/tools/collector/centos/build_srpm.data new file mode 100644 index 000000000..69bf0121c --- /dev/null +++ b/tools/collector/centos/build_srpm.data @@ -0,0 +1,2 @@ +SRC_DIR="scripts" +TIS_PATCH_VER=24 diff --git a/tools/collector/centos/collector.spec b/tools/collector/centos/collector.spec new file mode 100644 index 000000000..40c51c513 --- /dev/null +++ b/tools/collector/centos/collector.spec @@ -0,0 +1,66 @@ +Summary: CGCS Platform Data Collection Scripts Package +Name: collector +Version: 1.0 +Release: %{tis_patch_ver}%{?_tis_dist} +License: Apache-2.0 +Group: base +Packager: Wind River +URL: unknown +Source0: %{name}-%{version}.tar.gz + +%description +This packages scripts that implement data and log collection that field +support can execute to gather current state and runtime history for off +platform analysis and debug. + +%prep +%setup + +%install +mkdir -p %{buildroot} + +install -d 755 -d %{buildroot}%{_sysconfdir}/collect.d +install -d 755 -d %{buildroot}%{_sysconfdir}/collect +install -d 755 -d %{buildroot}/usr/local/sbin +install -d 755 -d %{buildroot}/usr/local/bin +install -d 755 -d %{buildroot}%{_sbindir} + +install -m 755 collect %{buildroot}/usr/local/sbin/collect +install -m 755 collect_host %{buildroot}/usr/local/sbin/collect_host +install -m 755 collect_date %{buildroot}/usr/local/sbin/collect_date +install -m 755 collect_utils %{buildroot}/usr/local/sbin/collect_utils +install -m 755 collect_parms %{buildroot}/usr/local/sbin/collect_parms +install -m 755 collect_mask_passwords %{buildroot}/usr/local/sbin/collect_mask_passwords +install -m 755 expect_done %{buildroot}/usr/local/sbin/expect_done + +install -m 755 collect_sysinv.sh %{buildroot}%{_sysconfdir}/collect.d/collect_sysinv +install -m 755 collect_psqldb.sh %{buildroot}%{_sysconfdir}/collect.d/collect_psqldb +install -m 755 collect_openstack.sh %{buildroot}%{_sysconfdir}/collect.d/collect_openstack +install -m 755 collect_networking.sh %{buildroot}%{_sysconfdir}/collect.d/collect_networking +install -m 755 collect_ceph.sh %{buildroot}%{_sysconfdir}/collect.d/collect_ceph +install -m 755 collect_sm.sh %{buildroot}%{_sysconfdir}/collect.d/collect_sm +install -m 755 collect_tc.sh %{buildroot}%{_sysconfdir}/collect.d/collect_tc +install -m 755 collect_nfv_vim.sh %{buildroot}%{_sysconfdir}/collect.d/collect_nfv_vim +install -m 755 collect_vswitch.sh %{buildroot}%{_sysconfdir}/collect.d/collect_vswitch +install -m 755 collect_patching.sh %{buildroot}%{_sysconfdir}/collect.d/collect_patching +install -m 755 collect_coredump.sh %{buildroot}%{_sysconfdir}/collect.d/collect_coredump +install -m 755 collect_crash.sh %{buildroot}%{_sysconfdir}/collect.d/collect_crash +install -m 755 collect_ima.sh %{buildroot}%{_sysconfdir}/collect.d/collect_ima + +install -m 755 etc.exclude %{buildroot}%{_sysconfdir}/collect/etc.exclude +install -m 755 run.exclude %{buildroot}%{_sysconfdir}/collect/run.exclude + +ln -sf /usr/local/sbin/collect %{buildroot}/usr/local/bin/collect +ln -sf /usr/local/sbin/collect %{buildroot}%{_sbindir}/collect + +%clean +rm -rf %{buildroot} + +%files +%license LICENSE +%defattr(-,root,root,-) +%{_sysconfdir}/collect/* +%{_sysconfdir}/collect.d/* +/usr/local/sbin/* +/usr/local/bin/collect +%{_sbindir}/collect diff --git a/tools/collector/scripts/LICENSE b/tools/collector/scripts/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/tools/collector/scripts/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/collector/scripts/collect b/tools/collector/scripts/collect new file mode 100755 index 000000000..e4bb8a288 --- /dev/null +++ b/tools/collector/scripts/collect @@ -0,0 +1,1267 @@ +#! /bin/bash +######################################################################## +# +# Copyright (c) 2014-2017 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +######################################################################## +# +# Description: This script creates a tarball of logs and runtime +# configuration information for any of the following +# +# - current host ... collect +# - specified host ... collect hostname +# - group of hosts ... collect --list ... +# - all hosts ... collect --all +# +# Behavior : See print_help below. +# +# Inclusions : What is collected. +# +# - /var/log +# - /var/run (exclusions listed in /etc/collect/exclude.list) +# - area specific configuration and data -> ./var/extra +# - all databases in plain text ; except for ceilometer and keystone +# +# Additional collected info is expressed by the following runtime output. +# Generally, individual commands that display output have that output +# redirected to the appropriate info file in /scratch/var/extra +# +# wrsroot@controller-0:/scratch# sudo collect +# nodetype : controller +# Collector: /scratch +# Extra Dir: /scratch/var/extra +# Database : /scratch/database +# Tarball : /scratch/controller-0.20140318.232925.tgz +# ------------------------------------------------------------------------ +# controller-0: Process Info ......: /scratch/var/extra/process.info +# controller-0: Host Info .........: /scratch/var/extra/host.info +# controller-0: Memory Info .......: /scratch/var/extra/memory.info +# controller-0: Filesystem Info ...: /scratch/var/extra/filesystem.info +# controller-0: Bash History ......: /scratch/var/extra/history.info +# controller-0: Interrupt Info ....: /scratch/var/extra/interrupt.info +# controller-0: HA Info ...........: /scratch/var/extra/crm.info +# controller-0: CIB Admin Info ....: /scratch/var/extra/crm.xml +# controller-0: Mtce Info .........: /scratch/var/extra/mtce.info +# controller-0: Networking Info ...: /scratch/var/extra/networking.info +# controller-0: RabbitMQ Info .....: /scratch/var/extra/rabbitmq.info +# controller-0: Database Info .....: /scratch/var/extra/database.info +# controller-0: Dumping Database ..: /scratch/database/postgres.db.sql.txt +# controller-0: Dumping Database ..: /scratch/database/glance.db.sql.txt +# controller-0: Dumping Database ..: /scratch/database/nova.db.sql.txt +# controller-0: Dumping Database ..: /scratch/database/cinder.db.sql.txt +# controller-0: Dumping Database ..: /scratch/database/heat.db.sql.txt +# controller-0: Dumping Database ..: /scratch/database/neutron.db.sql.txt +# controller-0: Dumping Database ..: /scratch/database/sysinv.db.sql.txt +# controller-0: Creating Tarball ..: /scratch/controller-0.20140318.232925.tgz +# +# Tarball: /scratch/..tgz +# +# The script first collects the process, host, memory, +# filesystem, interrupt and HA information. +# It then proceeds to calls run-parts against the +# /etc/collect.d direcory which contains service level +# collectors. Additional collected can be added to that +# collect.d directory and will be called automatically. +# +# Warning: Script currently must be run as root. +# The collector scripts consider nodetype when deciding +# which commands to execute where. +# +################################################################## + + +TOOL_NAME=collect +TOOL_VER=2 +TOOL_REV=0 + +# collect must be run as wrsroot +if [ ${UID} -eq 0 ]; then + echo "Error: Cannot run collect as 'root' user" + exit 1 +fi + +# pull in common utils and environment +source /usr/local/sbin/collect_utils + +# get the host type +nodetype="" +subfunction="" +PLATFORM_CONF=/etc/platform/platform.conf +if [ -e ${PLATFORM_CONF} ] ; then + source ${PLATFORM_CONF} +fi + +ACTIVE=false +if [ "${nodetype}" == "controller" ] ; then + KEYRING_PATH="/opt/platform/.keyring" + if [ -e ${KEYRING_PATH} ] ; then + CRED=`find /opt/platform/.keyring -name .CREDENTIAL` + if [ ! -z "${CRED}" ] ; then + NOVAOPENRC="/etc/nova/openrc" + if [ -e ${NOVAOPENRC} ] ; then + ACTIVE=true + source ${NOVAOPENRC} 2>/dev/null 1>/dev/null + fi + fi + fi +fi + +function clean_up() +{ + `reset` + echo "" +} + +function control_c() +{ + echo "" + echo "... received exit signal ..." + clean_up + exit 0 +} + +# Handle exit signals +trap control_c SIGINT +trap control_c SIGTERM + + + +# static expect log level control ; +# 0 = hide expect output +# 1 = show expect outout +USER_LOG_MODE=0 + +# static execution status 'return value' +RETVAL=0 + +# limit scp bandwidth to 1MB/s +# increase limit of scp bandwidth from 1MB/s to 10MB/s +SCP_CMD="scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PreferredAuthentications=password -o PubkeyAuthentication=no -l $((10*8*1000))" +SCP_TIMEOUT="600" +SSH_CMD="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PreferredAuthentications=password -o PubkeyAuthentication=no" +NOWDATE=`date +"%Y%m%d.%H%M%S"` +COLLECT_BASE_DIR="/scratch" +collect_host="/usr/local/sbin/collect_host" +CURR_DIR=`pwd` + + +# common permission error strings +pw_error="orry, try again" +ac_error="ermission denied" + +function print_help() +{ + echo "" + echo "Titanium Cloud Log Collection Tool, version ${TOOL_VER}.${TOOL_REV}" + echo "" + echo "Usage: ${TOOL_NAME} [COMMANDS ...] {options}" + echo "" + echo "Titanium Cloud 'collect' is used by the customer support organization" + echo " to collect logs and data for off system analysis." + echo "" + echo "Running collect will collect logs to /scratch/" + echo "on the host collect is run from. Use host names to specify which hosts to collect from." + echo "" + echo "Host data collection scope can be the current host, any single specified hostname," + echo "a --list of hostnames or --all hosts in the system using a single command." + echo "" + echo "Optionally specify --start-date and/or --end-date options to limit" + echo " the date range and therefore size of the collect." + echo "" + echo "Optionally specify a --name prefix of the collected tar file." + echo "" + echo "With the command set specified, simply run collect as wrsroot and when" + echo "prompted provide the wrsroot sudo password and let collect handle the rest." + echo "" + echo "Scope Options:" + echo "" + echo " collect ... collect logs for current host" + echo " collect host1 ... collect logs for single named host" + echo " collect host1 host2 host3 ... collect logs for stacked host list" + echo " collect [--list | -l] host1 host2 host3 ... collect logs for list of named hosts" + echo " collect [--all | -a] ... collect data for all hosts" + echo "" + echo "Dated Collect:" + echo "" + echo "collect [--start-date | -s] YYYYMMDD ... collection of logs on and after this date" + echo "collect [--end-date | -e] YYYYMMDD ... collection of logs on and before this date" + echo "" + echo "Tarball Prefix:" + echo "" + echo "collect [--name | -n] {scope and date options} ... specify the name prefix of the collect tarball" + echo "" + echo "Detailed Display:" + echo "" + echo "collect [--verbose | -v] ... print details during collect" + echo "" + echo "Avoid password and security masking:" + echo "" + echo "collect [--skip-mask] ... skip masking of collect data" + echo "" + echo "Examples:" + echo "" + echo "collect ... all logs for current host" + echo "collect --all ... all logs from all hosts in the system" + echo "collect --all --start-date 20150101 ... logs dated on and after Jan 1 2015 from all hosts" + echo "collect --all --start-date 20151101 --end-date 20160201 ... logs dated between Nov 1, 2015 and Feb 1 2016 from all hosts" + echo "collect --start-date 20151101 --end-date 20160201 ... only logs dated between Nov 1, 2015 and Feb 1 2016 for current host" + echo "collect --list controller-0 compute-0 storage-0 ... all logs from specified host list" + echo "collect --list controller-0 compute-1 --end-date 20160201 ... only logs before Nov 1, 2015 for host list" + echo "collect --list controller-1 storage-0 --start-date 20160101 ... only logs after Jan 1 2016 for controller-1 and storage-0" + echo "" + exit 0 +} + +# command line arguement variables ; defaulted +DEBUG=false +CLEAN=false +VERBOSE=false +SKIP_MASK=false + +# date variables +STARTDATE="any" +STARTTIME="any" +ENDDATE="any" +ENDTIME="any" +GETSTARTDATE=false +GETENDDATE=false + +# host selection variables +LISTING=false +ALLHOSTS=false +HOSTS=1 +HOSTLIST=(${HOSTNAME}) +THISHOST=false + +COLLECT_TARNAME="" + +# clear multi option modes +function clear_variable_args() +{ + LISTING=false + GETSTARTDATE=false + GETENDDATE=false +} + +# +# Utility function to print a status message and record the last error code +# +# Assumptions: Handles specific cases of invalid password and permission errors +# by exiting so as to avoid repeated errors during multi-host +# collection. +# +# $1 - status string +# $2 - status code number +# +function print_status() +{ + local string=${1} + local code=${2} + + logger -t ${COLLECT_TAG} "${string} (reason:${code})" + + # if the status code is in the FAIL range ( less than WARNING ) then update RETVAL + if [ ${code} -lt ${WARN_WARNING} ] ; then + RETVAL=${code} + fi + + if [ ${RETVAL} -eq ${FAIL_PASSWORD} ] ; then + + echo "Invalid password ; exiting (${string})" + exit ${RETVAL} + + elif [ ${RETVAL} -eq ${FAIL_PERMISSION} ] ; then + + echo "Permission error ; exiting (${string})" + exit ${RETVAL} + + elif [ ${RETVAL} -eq ${FAIL_UNREACHABLE} ] ; then + + echo "${string} (reason:${code}:host unreachable)" + + elif [ ${RETVAL} -eq ${FAIL_PERMISSION_SKIP} -o ${RETVAL} -eq ${FAIL_PERMISSION} ] ; then + + echo "${string} (reason:${code}:permission error)" + + elif [ ${RETVAL} -eq ${FAIL_OUT_OF_SPACE} ] ; then + + echo "${string} (reason:${code}) ; need to increase available space in host ${COLLECT_BASE_DIR}" + + elif [ ${RETVAL} -eq ${FAIL_OUT_OF_SPACE_LOCAL} ] ; then + + echo "${string} (reason:${code}) ; need to increase available space in ${HOSTNAME}:${COLLECT_BASE_DIR}" + + elif [ ${RETVAL} -eq ${FAIL_INSUFFICIENT_SPACE} ] ; then + + echo "${string} (reason:${code}) ; ${HOSTNAME}:${COLLECT_BASE_DIR} usage must be below ${MIN_PERCENT_SPACE_REQUIRED}%" + + elif [ ${RETVAL} -ge ${FAIL_TIMEOUT} -a ${RETVAL} -le ${FAIL_TIMEOUT9} ] ; then + + echo "${string} (reason:${code}:operation timeout)" + + else + echo "${string} (reason:${code})" + fi +} + +# +# checks to see if the specified hostname is known +# to inventory as a valid provisioned host + +# $1 - this_hostname + +function is_valid_host() +{ + local this_hostname=${1} + + if [ "${this_hostname}" == "None" ] ; then + return ${FAIL_HOSTNAME} + elif [ "${this_hostname}" == "${HOSTNAME}" ] ; then + return $PASS + elif [ "${ACTIVE}" = true ] ; then + system host-show "${this_hostname}" 2>/dev/null 1>/dev/null + if [ ${?} -ne 0 ] ; then + return ${FAIL_HOSTNAME} + fi + else + print_status "Error: can only run collect for remote hosts on active controller" ${FAIL_INACTIVE} + exit ${FAIL_INACTIVE} + fi + return $PASS +} + + +# Parse the command line +while [[ ${#} -gt 0 ]] ; do + + key="${1}" + + case $key in + + -h|--help) + print_help + exit 0 + ;; + + -n|--name) + COLLECT_TARNAME=${2}_${NOWDATE} + clear_variable_args + shift + ;; + + -v|--verbose) + VERBOSE=true + ;; + + -c|--clean) + CLEAN=true + ;; + + -l|--list) + if [[ ${#} -lt 2 ]] ; then + print_status "Error: empty host --list" ${FAIL} + exit ${FAIL} + fi + is_valid_host "${2}" + if [ ${?} -ne 0 ] ; then + print_status "Error: empty host --list or invalid first hostname" ${FAIL} + exit ${FAIL} + fi + + HOSTLIST=(${2}) + HOSTS=1 + if [ "${2}" == "${HOSTNAME}" ] ; then + THISHOST=true + elif [ "${ACTIVE}" = false ] ; then + print_status "Error: can only run collect for remote hosts on active controller" ${FAIL_INACTIVE} + exit ${FAIL_INACTIVE} + fi + LISTING=true + GETSTARTDATE=false + GETENDDATE=false + shift + ;; + + -a|--all|all) + if [ "${ACTIVE}" = false ] ; then + print_status "Error: can only run collect for remote hosts on active controller" ${FAIL_INACTIVE} + exit ${FAIL_INACTIVE} + fi + ALLHOSTS=true + HOSTLIST=(${HOSTNAME}) + HOSTS=1 + THISHOST=true + clear_variable_args + ;; + + -s|--start-date) + STARTDATE="${2}" + LISTING=false + GETSTARTDATE=true + GETENDDATE=false + shift + ;; + + -e|--end-date) + ENDDATE="${2}" + LISTING=false + GETSTARTDATE=false + GETENDDATE=true + shift + ;; + + -d|--debug) + DEBUG=true + USER_LOG_MODE=1 + clear_variable_args + ;; + + --skip-mask) + SKIP_MASK=true + shift + ;; + + *) + if [ "${LISTING}" = true ] ; then + is_valid_host ${key} + if [ ${?} -eq 0 ] ; then + HOSTS=$((${HOSTS} + 1)) + HOSTLIST=( "${HOSTLIST[@]}" ${key} ) + if [ "${key}" == "${HOSTNAME}" ] ; then + THISHOST=true + fi + else + # make the invalid hostname a warning only. + # if we got here then at least the first hostname was valid + print_status "Warning: cannot collect data from unknown host '${key}'" ${WARN_HOSTNAME} + fi + elif [ "${GETSTARTDATE}" = true ] ; then + dlog "accepting but ignoring legacy starttime specification" + elif [ "${GETENDDATE}" = true ] ; then + dlog "accepting but ignoring legacy endtime specification" + else + is_valid_host ${key} + RETVAL=${?} + if [ ${RETVAL} -eq 0 ] ; then + HOSTLIST=${key} + HOSTS=1 + LISTING=true + if [ "${key}" == "${HOSTNAME}" ] ; then + THISHOST=true + fi + else + print_status "Error: cannot collect data from unknown host '${key}'" ${RETVAL} + exit ${RETVAL} + fi + fi + GETSTARTDATE=false + GETENDDATE=false + ;; + esac + shift # past argument or value +done + +if [ ${RETVAL} -ne 0 ]; then + echo "command line parse error (${RETVAL})" + print_help + exit ${RETVAL} +fi + + +# +# request root password and use it for +# all the expect driven requests below +# +read -s -p "[sudo] password for ${USER}:" pw +echo "" + +# Although bash 'read' will handle sanitizing the password +# input for the purposes of storing it in ${pw}, expect +# will need certain special characters to be backslash +# delimited +pw=${pw/\\/\\\\} # replace '\' with '\\' +pw=${pw/\]/\\\]} # replace ']' with '\]' +pw=${pw/\[/\\\[} # replace '[' with '\[' +pw=${pw/$/\\$} # replace '$' with '\$' +pw=${pw/\"/\\\"} # replace '"' with '\"' + +# +# if the user specified the '--all' option then override +# the current list and add them all from inventory. +# +if [ "${ALLHOSTS}" = true ] ; then + + for foreign_host in $(system host-list | grep '[0-9]' | cut -d '|' -f 3 | tr -d ' ' | grep -v ${HOSTNAME}); do + if [ "${foreign_host}" != "None" ] ; then + HOSTS=$((${HOSTS} + 1)) + HOSTLIST=( "${HOSTLIST[@]}" ${foreign_host}) + dlog "Host:${HOSTS}: ${foreign_host}" + fi + done + +elif [ ${HOSTS} == 0 ] ; then + + HOSTLIST=${HOSTNAME} + THISHOST=true + COLLECT_TARNAME="${HOSTNAME}_${NOWDATE}" + +fi + +# Print Summary +if [ "${DEBUG}" == true ] ; then + + echo "HOSTLIST = <${HOSTLIST[@]}>" + echo "HOSTS = ${HOSTS}" + echo "ALLHOSTS = ${ALLHOSTS}" + echo "STARTDATE= ${STARTDATE}" + echo "ENDDATE = ${ENDDATE}" + + for hosts in "${HOSTLIST[@]}" ; do + echo "Host:${hosts}" + done + +elif [ ${HOSTS} -eq 0 ] ; then + + print_status "Error: no hosts specified" "${FAIL}" + exit ${FAIL} + +elif [ "${CLEAN}" == false ] ; then + + ilog "collecting data from ${HOSTS} host(s): ${HOSTLIST[@]}" + +else + + ilog "cleaning scratch space on ${HOSTLIST[@]}" + +fi + +# +# removes contents of the local /scratch directory +# +# $1 - host +# $2 - specified directory (always $COLLECT_BASE_DIR) +# +function clean_scratch_dir_local () +{ + local this_hostname=${1} + local directory=${2} + +/usr/bin/expect << EOF + log_user ${USER_LOG_MODE} + spawn bash -i + set timeout 60 + expect -re $ + send -- "sudo rm -rf ${directory}/*_????????.??????* ; cat ${cmd_done_file}\n" + expect { + "assword:" { send "${pw}\r" ; exp_continue } + "${cmd_done_sig}" { exit ${PASS} } + "annot remove" { exit ${FAIL_CLEANUP} } + "${pw_error}" { exit ${FAIL_PASSWORD} } + "${ac_error}" { exit ${FAIL_PERMISSION} } + timeout { exit ${FAIL_TIMEOUT} } + } +EOF + local rc=${?} + if [ ${rc} -ne ${PASS} ] ; then + print_status "Error: clean_scratch_dir_local ${this_hostname} failed" ${rc} + fi + return ${rc} +} + +# +# cleans the contents of the specified hosts's scratch dir +# +# $1 - this hostname +# $2 - specified directory (always $COLLECT_BASE_DIR) +# +function clean_scratch_dir_remote() +{ + local this_hostname=${1} + local directory=${2} + +/usr/bin/expect << EOF + log_user ${USER_LOG_MODE} + spawn bash -i + expect -re $ + set timeout 60 + send "${SSH_CMD} wrsroot@${this_hostname}\n" + expect { + "assword:" { + send "${pw}\r" + expect { + "${this_hostname}" { + set timeout 30 + expect -re $ + send "sudo rm -rf ${directory}/*_????????.??????* ; cat ${cmd_done_file}\n" + expect { + "assword:" { send -- "${pw}\r" ; exp_continue } + "${cmd_done_sig}" { exit ${PASS} } + "${cmd_done_file}: No such file or directory" { exit ${PASS} } + "annot remove" { exit ${FAIL_CLEANUP} } + "${pw_error}" { exit ${FAIL_PASSWORD} } + "${ac_error}" { exit ${FAIL_PERMISSION}} + timeout { exit ${FAIL_TIMEOUT3} } + } + } + timeout { exit ${FAIL_TIMEOUT1} } + } + } + "(yes/no)?" { + send "yes\r" + exp_continue + } + "No route to host" { + exit ${FAIL_UNREACHABLE} + } + "Could not resolve hostname" { + exit ${FAIL_UNREACHABLE} + } + timeout { exit ${FAIL_TIMEOUT} } + } +EOF + local rc=${?} + if [ ${rc} -ne ${PASS} ] ; then + print_status "Error: clean_scratch_dir_remote ${this_hostname} failed" ${rc} + fi + return ${rc} +} + +# +# deletes a remote directory or file +# +# $1 - this hostname +# $2 - dir or file with full path +# +function delete_remote_dir_or_file() +{ + local this_hostname=${1} + local dir_or_file=${2} + +/usr/bin/expect << EOF + log_user ${USER_LOG_MODE} + spawn bash -i + expect -re $ + set timeout 60 + send "${SSH_CMD} wrsroot@${this_hostname}\n" + expect { + "assword:" { + send "${pw}\r" + expect { + "${this_hostname}:" { + set timeout 10 + expect -re $ + send "sudo rm -rf ${dir_or_file} ; cat ${cmd_done_file}\n" + expect { + "assword:" { send -- "${pw}\r" ; exp_continue } + "${cmd_done_sig}" { exit ${PASS} } + "${cmd_done_file}: No such file or directory" { exit ${PASS} } + "annot remove" { exit ${FAIL_CLEANUP} } + "${pw_error}" { exit ${FAIL_PASSWORD} } + "${ac_error}" { exit ${FAIL_PERMISSION}} + timeout { exit ${FAIL_TIMEOUT3} } + } + } + timeout { exit ${FAIL_TIMEOUT1} } + } + } + "(yes/no)?" { + send "yes\r" + exp_continue + } + "No route to host" { + exit ${FAIL_UNREACHABLE} + } + "Could not resolve hostname" { + exit ${FAIL_UNREACHABLE} + } + timeout { exit ${FAIL_TIMEOUT} } + } +EOF + local rc=${?} + if [ ${rc} -ne ${PASS} ] ; then + print_status "Error: delete_remote_dir_or_file ${this_hostname} failed" ${rc} + fi + return ${rc} +} + +HOST_COLLECT_ERROR_LOG="/tmp/host_collect_error.log" +# +# Fetch a file from a remote host using the global pw +# $1 - this hostname +# $2 - remote source path/filename +# $3 - local path destination +# +function get_file_from_host() +{ + local this_hostname=${1} + local remote_src=${2} + local local_dest=${3} + + remove_file_local ${HOST_COLLECT_ERROR_LOG} + +/usr/bin/expect << EOF + log_user ${USER_LOG_MODE} + spawn bash -i + set timeout ${SCP_TIMEOUT} + expect -re $ + send "${SCP_CMD} wrsroot@${this_hostname}:${remote_src} ${local_dest} 2>>${HOST_COLLECT_ERROR_LOG}\n" + expect { + "assword:" { + send "${pw}\r" + expect { + "100%" { exit ${PASS} } + "${pw_error}" { exit ${FAIL_PASSWORD} } + "${ac_error}" { exit ${FAIL_PERMISSION}} + timeout { exit ${FAIL_TIMEOUT1} } + } + } + "No route to host" { + exit ${FAIL_UNREACHABLE} + } + "Could not resolve hostname" { + exit ${FAIL_UNREACHABLE} + } + timeout { exit ${FAIL_TIMEOUT} } + } +EOF + local rc=${?} + if [ ${rc} -ne ${PASS} ] ; then + print_status "failed to get_file_from ${this_hostname}" ${rc} + else + # Look for "No space left on device" error + grep -q "${FAIL_OUT_OF_SPACE_STR}" ${HOST_COLLECT_ERROR_LOG} + if [ "$?" == "0" ] ; then + rc=${FAIL_OUT_OF_SPACE} + fi + fi + + remove_file_local ${HOST_COLLECT_ERROR_LOG} + + return ${rc} +} + +# +# Create the local dated collect dir where all +# the tarballs for this collect will get put. +# +# Permissions are set to make it easy to copy +# tarballs from remote host into +# +# $1 - the fill dir +# +function create_collect_dir_local() +{ + local dir=${1} + +/usr/bin/expect << EOF + log_user ${USER_LOG_MODE} + spawn bash -i + set timeout 10 + expect -re $ + send "sudo mkdir -m 775 -p ${dir} ; cat ${cmd_done_file}\n" + expect { + "assword:" { + send "${pw}\r" + expect { + "${cmd_done_sig}" { exit ${PASS} } + "${pw_error}" { exit ${FAIL_PASSWORD} } + "${ac_error}" { exit ${FAIL_PERMISSION}} + timeout { exit ${FAIL_TIMEOUT1} } + } + } + "${cmd_done_sig}" { exit ${PASS} } + "${ac_error}" { exit ${FAIL_PERMISSION}} + timeout { exit ${FAIL_TIMEOUT} } + } +EOF + local rc=${?} + if [ ${rc} -ne ${PASS} ] ; then + print_status "failed to create_collect_dir_local for ${dir}" ${rc} + fi + return ${rc} +} + +# +# Delete the specified file using sudo +# +# $1 - the file to be delete with full path specified +# +function remove_file_local() +{ + local local_file=${1} + local rc=${PASS} + + if [ -e ${local_file} ] ; then + +/usr/bin/expect << EOF + log_user ${USER_LOG_MODE} + spawn bash -i + set timeout 10 + expect -re $ + send -- "sudo rm -f ${local_file} ; cat ${cmd_done_file}\n" + expect { + "assword:" { send -- "${pw}\r" ; exp_continue } + "${cmd_done_sig}" { exit ${PASS} } + "annot remove" { exit ${FAIL_CLEANUP} } + "${pw_error}" { exit ${FAIL_PASSWORD} } + "${ac_error}" { exit ${FAIL_PERMISSION} } + timeout { exit ${FAIL_TIMEOUT} } + } +EOF + local rc=${?} + if [ ${rc} -ne ${PASS} ] ; then + print_status "failed to remove_file_local ${local_file}" ${rc} + fi + fi + return ${rc} +} + +# +# Delete the specified file using sudo +# +# $1 - the directory to be removed with full path specified +# +function remove_dir_local() +{ + local dir=${1} + +/usr/bin/expect << EOF + log_user ${USER_LOG_MODE} + spawn bash -i + set timeout 10 + expect -re $ + send -- "sudo rm -rf ${dir} ; cat ${cmd_done_file}\n" + expect { + "assword:" { send -- "${pw}\r" ; exp_continue } + "${cmd_done_sig}" { exit ${PASS} } + "annot remove" { exit ${FAIL_CLEANUP} } + "${pw_error}" { exit ${FAIL_PASSWORD} } + "${ac_error}" { exit ${FAIL_PERMISSION} } + timeout { exit ${FAIL_TIMEOUT} } + } +EOF + local rc=${?} + if [ ${rc} -ne ${PASS} ] ; then + print_status "failed to remove_dir_local ${dir}" ${rc} + fi + return ${rc} +} + +# +# Move a file and change permissions using sudo +# +# $1 - src path/file +# $2 - dest path/file +# +function move_file_local() +{ + local src=${1} + local dst=${2} + +/usr/bin/expect << EOF + log_user ${USER_LOG_MODE} + spawn bash -i + set timeout 10 + expect -re $ + send -- "sudo mv ${src} ${dst} ; cat ${cmd_done_file}\n" + expect { + "assword:" { send -- "${pw}\r" ; exp_continue } + "${cmd_done_sig}" { exit ${PASS} } + "annot remove" { exit ${FAIL_CLEANUP} } + "${pw_error}" { exit ${FAIL_PASSWORD} } + "${ac_error}" { exit ${FAIL_PERMISSION} } + timeout { exit ${FAIL_TIMEOUT} } + } +EOF + local rc=${?} + if [ ${rc} -ne ${PASS} ] ; then + print_status "failed to move_file_local ${src} to ${dst}" ${rc} + fi + return ${rc} +} + +# Append the echoed collect done with collect duration and file size +# ... done (HH:MM:SS xxM) +function echo_stats() +{ + local secs=${1} + local file=${2} + + echo -n " ($(date -d@${secs} -u +%H:%M:%S)" + if [ -e ${file} ] ; then + size=$(du -h ${file} | cut -f 1 2>/dev/null) + if [ $? -eq 0 ] ; then + printf " %5s)\n" "${size}" + return + fi + fi + echo ")" +} + + +# Handle clean command +if [ "${CLEAN}" == true ] ; then + for host in "${HOSTLIST[@]}" ; do + if [ "${host}" != " " ] ; then + + if [ "${host}" == "None" ] ; then + continue + elif [ "${host}" == "" ] ; then + continue + fi + + echo -n "cleaning ${host}:${COLLECT_BASE_DIR} ... " + if [ "${host}" == "${HOSTNAME}" ] ; then + clean_scratch_dir_local ${host} ${COLLECT_BASE_DIR} + if [ ${?} -eq ${PASS} ] ; then + echo "done" + fi + else + clean_scratch_dir_remote ${host} ${COLLECT_BASE_DIR} + if [ ${?} -eq ${PASS} ] ; then + echo "done" + fi + fi + logger -t ${COLLECT_TAG} "user cleaned ${host}:${COLLECT_BASE_DIR} content" + fi + done + exit 0 +fi + + +if [ ! -z ${COLLECT_TARNAME} ] ; then + + # User specified tarname + COLLECT_NAME=${COLLECT_TARNAME} + COLLECT_DIR="${COLLECT_BASE_DIR}/${COLLECT_NAME}" + TARBALL_NAME="${COLLECT_DIR}.tar" + named="user-named" + +elif [ "${ALLHOSTS}" = true ] ; then + + # All hosts bundle + COLLECT_NAME="ALL_NODES_${NOWDATE}" + COLLECT_DIR="${COLLECT_BASE_DIR}/${COLLECT_NAME}" + TARBALL_NAME="${COLLECT_DIR}.tar" + named="all-nodes" + + +elif [ ${HOSTS} -eq 1 ] ; then + + # Single host bundle + COLLECT_NAME="${HOSTLIST[0]}_${NOWDATE}" + COLLECT_DIR="${COLLECT_BASE_DIR}/${COLLECT_NAME}" + TARBALL_NAME="${COLLECT_DIR}.tar" + named="single-node" + +else + + # Otherwise its a multi host bundle + COLLECT_NAME="SELECT_NODES_${NOWDATE}" + COLLECT_DIR="${COLLECT_BASE_DIR}/${COLLECT_NAME}" + TARBALL_NAME="${COLLECT_DIR}.tar" + named="selected-node" + +fi + +# +# Create the local collect directory where +# the tarball(s) will temporarily stored +# +create_collect_dir_local "${COLLECT_DIR}" + +declare COLLECT_START_TIME=${SECONDS} + +declare -i longest_hostname=0 +for host in "${HOSTLIST[@]}" ; do + len=${#host} + if [ $len -gt ${longest_hostname} ] ; then + longest_hostname=$len + fi +done + +# +# Loop over all the targetted hosts and +# 1. run collect +# 2. copy the tarball to $COLLECT_DIR +# +for host in "${HOSTLIST[@]}" ; do + if [ "${host}" != " " ] ; then + + if [ "${host}" == "None" ] ; then + continue + elif [ "${host}" == "" ] ; then + continue + fi + + HOST_START_TIME=${SECONDS} + + TARNAME="${host}_${NOWDATE}" + + # line up the hostr namaes + echo -n "collecting" + len=${#host} + for ((i=len;i>${COLLECT_ERROR_LOG} ; cat ${cmd_done_file})\n" + expect { + "assword:" { + send "${pw}\r" + expect { + "${cmd_done_sig}" { exit ${PASS} } + "${pw_error}" { exit ${FAIL_PASSWORD} } + "${ac_error}" { exit ${FAIL_PERMISSION} } + timeout { exit ${FAIL_TIMEOUT1} } + } + } + timeout { exit ${FAIL_TIMEOUT} } + } +EOF + RETVAL=${?} + if [ ${RETVAL} -ne ${PASS} ] ; then + collect_errors ${HOSTNAME} + print_status "failed to create ${TARBALL_NAME}" ${RETVAL} + else + collect_errors ${HOSTNAME} + RETVAL=$? + if [ ${RETVAL} -eq ${PASS} ] ; then + secs=$((SECONDS-COLLECT_START_TIME)) + echo -n "done" + echo_stats $secs "${TARBALL_NAME}" + logger -t ${COLLECT_TAG} "created ${named} tarball ${TARBALL_NAME}" + else + echo "removing incomplete collect: ${TARBALL_NAME}" + remove_file_local "${TARBALL_NAME}" + fi + fi + remove_file_local ${COLLECT_ERROR_LOG} + remove_dir_local "${COLLECT_DIR}" + +# return to callers dir +cd ${CURR_DIR} + +exit ${RETVAL} diff --git a/tools/collector/scripts/collect_ceph.sh b/tools/collector/scripts/collect_ceph.sh new file mode 100755 index 000000000..0898d8db8 --- /dev/null +++ b/tools/collector/scripts/collect_ceph.sh @@ -0,0 +1,83 @@ +#! /bin/bash +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# Loads Up Utilities and Commands Variables +source /usr/local/sbin/collect_parms +source /usr/local/sbin/collect_utils + +SERVICE="ceph" +LOGFILE="${extradir}/ceph.info" +echo "${hostname}: Ceph Info .........: ${LOGFILE}" + +function is_service_active() +{ + active=`sm-query service management-ip | grep "enabled-active"` + if [ -z "$active" ] ; then + return 0 + else + return 1 + fi +} + +function exit_if_timeout() +{ + if [ "$?" = "124" ] ; then + echo "Exiting due to ceph command timeout" >> ${LOGFILE} + exit 0 + fi +} + +############################################################################### +# Only Controller +############################################################################### +if [ "$nodetype" = "controller" ] ; then + + # Using timeout with all ceph commands because commands can hang for + # minutes if the ceph cluster is down. If ceph is not configured, the + # commands return immediately. + + delimiter ${LOGFILE} "ceph status" + timeout 30 ceph status >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + exit_if_timeout + + delimiter ${LOGFILE} "ceph mon dump" + timeout 30 ceph mon dump >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + exit_if_timeout + + delimiter ${LOGFILE} "ceph osd dump" + timeout 30 ceph osd dump >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + exit_if_timeout + + delimiter ${LOGFILE} "ceph osd tree" + timeout 30 ceph osd tree >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + exit_if_timeout + + delimiter ${LOGFILE} "ceph osd crush dump" + timeout 30 ceph osd crush dump >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + exit_if_timeout + + is_service_active + if [ "$?" = "0" ] ; then + exit 0 + fi + + delimiter ${LOGFILE} "ceph df" + timeout 30 ceph df >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + exit_if_timeout + + delimiter ${LOGFILE} "ceph osd df tree" + timeout 30 ceph osd df tree >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + exit_if_timeout + + delimiter ${LOGFILE} "ceph health detail" + timeout 30 ceph health detail >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + exit_if_timeout + +fi + +exit 0 diff --git a/tools/collector/scripts/collect_coredump.sh b/tools/collector/scripts/collect_coredump.sh new file mode 100644 index 000000000..7614909f4 --- /dev/null +++ b/tools/collector/scripts/collect_coredump.sh @@ -0,0 +1,35 @@ +#! /bin/bash +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# Loads Up Utilities and Commands Variables + +source /usr/local/sbin/collect_parms +source /usr/local/sbin/collect_utils + +SERVICE="coredump" +LOGFILE="${extradir}/${SERVICE}.info" + + +COREDUMPDIR="/var/lib/systemd/coredump" + +echo "${hostname}: Core Dump Info ....: ${LOGFILE}" + +files=`ls ${COREDUMPDIR} | wc -l` +if [ "${files}" == "0" ] ; then + echo "No core dumps" >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} +else + COMMAND="ls -lrtd ${COREDUMPDIR}/*" + delimiter ${LOGFILE} "${COMMAND}" + ${COMMAND} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + COMMAND="md5sum ${COREDUMPDIR}/*" + delimiter ${LOGFILE} "${COMMAND}" + ${COMMAND} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} +fi + +exit 0 diff --git a/tools/collector/scripts/collect_crash.sh b/tools/collector/scripts/collect_crash.sh new file mode 100644 index 000000000..fc8c7982c --- /dev/null +++ b/tools/collector/scripts/collect_crash.sh @@ -0,0 +1,30 @@ +#! /bin/bash +# +# Copyright (c) 2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# Loads Up Utilities and Commands Variables + +source /usr/local/sbin/collect_parms +source /usr/local/sbin/collect_utils + +SERVICE="crash" +LOGFILE="${extradir}/${SERVICE}.info" + + +CRASHDIR="/var/crash" + +echo "${hostname}: Kernel Crash Info .: ${LOGFILE}" + +COMMAND="find ${CRASHDIR}" +delimiter ${LOGFILE} "${COMMAND}" +${COMMAND} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + +COMMAND="rsync -a --include=*.txt --include=*/ --exclude=* ${CRASHDIR} ${basedir}/var/" +delimiter ${LOGFILE} "${COMMAND}" +${COMMAND} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + +exit 0 diff --git a/tools/collector/scripts/collect_date b/tools/collector/scripts/collect_date new file mode 100755 index 000000000..22c62fb1b --- /dev/null +++ b/tools/collector/scripts/collect_date @@ -0,0 +1,1064 @@ +#!/bin/bash +####################################################################### +# +# Copyright (c) 2017 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +######################################################################## +# +# This file is a new member of the Titanium Cloud "Log Collect Utility". +# This file enhances date restricted collect in response. +# +# This file is invoked by collect_host when a date restricted +# collect using --start-date and/or --end-date options are used. +# +# This new data restricted collect service applies to /var/log and its +# subdirectories only. This service determines if a log file is to be +# included in dated collect by looking at the logs at the head and tail +# of the files and subdirectories in /var/log. Those dates are then +# compared to the user specified date range. if a file is determined to +# contain logs within that date range then that file is included in the +# collect log. A valid log date prefix is "YYYY-MM-DD". +# +# Unfortunately, not all log files contain the correct date placement and +# format. This feature has implemented special case handling for many but not +# not all of such cases. To avoid accidental exclusion of a key file, this +# feature will by default include log files whose log date content could +# not be determined if its file date is after the specified start date. +# +# Note: local convension , example ${head_date} vs ${HEAD_DATE} +# +# Lower case date variables contain integer values while +# Upper case date variables contain formatted string values of same. +# +# Calling sequence: +# +# /usr/local/sbin/collect_date +# /usr/local/sbin/collect_date 20170701 20170901 /tmp/file.list true +# +######################################################################## + +# +# Import commands, variables and convenience functions available to +# all collectors ; common and user defined. +# +source /usr/local/sbin/collect_utils + +# where to find the logs +declare -r baselogdir="/var/log" + +# include / exclude labels +declare -r INCLUDE_FILE="inc" +declare -r EXCLUDE_FILE="exc" + +# a global reason string that is only valid +# in the context of the file beeing looked at. +declare __this_reason="" + +# setup defaults +INC_FILE_LIST="/var/run/collect_include.list" +EXC_FILE_LIST="/var/run/collect_exclude.list" +NOD_FILE_LIST="/var/run/collect_nodate.list" + +BOT_DATE="2000-01-01" # beginning of time date +bot_date=730013 # beginning of time date as integer + +EOT_DATE="9999-12-31" # end of time date +eot_date=3649810 # end of time date as integer + +# manage debug mode +DEBUG="${4}" +set_debug_mode "${DEBUG}" +echo "Debug Mode: ${DEBUG}" + +dlog "collect_date args: ${1} ${2} ${3} ${4} ${5}" + +############################################################################# +# +# 'track' is the main accounting procedure that manages file inclusions and +# exclusions as well as the metrics around all the parsed files. +# +# It also reports accounting mismatch logs, if they occur (should not) +# and the file that started the mismatch (to assist in debug). +# +# $1 - filename +# $2 - label +# +############################################################################# + +# accounting defaults +declare -i file_count=0 +declare -i inc_file_count=0 +declare -i exc_file_count=0 +declare -i empty_file_count=0 + +function track() +{ + local fn="${1}" + local label="${2}" + + if [ -z "${fn}" ] ; then + elog "Ignoring call with empty filename" + return + + elif [ "${label}" == "totals" ] ; then + ((file_count++)) + return + + elif [ "${label}" == "empty" ] ; then + ((empty_file_count++)) + return + + elif [ "${label}" == "${INCLUDE_FILE}" ] ; then + manage_file "${fn}" "${label}" "${__this_reason}" + ((inc_file_count++)) + + elif [ "${label}" == "${EXCLUDE_FILE}" ] ; then + manage_file "${fn}" "${label}" "${__this_reason}" + ((exc_file_count++)) + + else + elog "Unknown label '${label}'" + + fi + + sum=$((inc_file_count + exc_file_count)) + if [ ${file_count} -ne ${sum} ] ; then + wlog "MISMATCH: ${file_count} != ${inc_file_count} + ${exc_file_count} - ${fn}" + fi +} + +############################################################################ +# +# 'summary' is an accounting display procedure used to show the +# accounting results of the total number of files processed, +# number of empty files and most importanly the number if +# included or excluded files. +# +############################################################################ + +function summary() +{ + dlog "Summary:" + dlog "Total Files: ${file_count}" + dlog "Empty Files: ${empty_file_count}" + dlog "Added Files: ${inc_file_count}" + dlog "Omitd Files: ${exc_file_count}" +} + +############################################################################# +# +# 'date_to_int' converts a standard formatted YYYY-MM-DD string date +# to an integer and stores it in __this_integer_date variable +# to be used in context on demand. +# +############################################################################# + +# short lived global integer date value updated by date_to_int utility +declare -i __this_integer_date="" + +function date_to_int() +{ + local yy="${1:0:4}" + local mm="${1:5:2}" + local dd="${1:8:2}" + + # handle leading zeros in month and day + if [ "${mm:0:1}" == "0" ] ; then + mm=${mm:1:1} + fi + if [ "${dd:0:1}" == "0" ] ; then + dd=${dd:1:1} + fi + + # 365 days in a year, 31 days in a month, 1 day in a day + __this_integer_date=$((yy*365 + mm*31 + dd)) +} + +############################################################################ +# +# 'create_list_file' removes old/stale list file and creates a new empty +# one with correct permissions. +# +############################################################################ + +function create_list_file() +{ + local fn="${1}" + if [ -e "${fn}" ] ; then + rm -f "${fn}" + fi + touch "${fn}" + chmod 644 "${fn}" +} + +######################################################################## +# +# Handle the incoming 'start' and 'end' date format defensively. +# +# If the date is with no dashes as it would come in from the user's +# date specification then set it up like the standard date delimited with '-' +# i.e. 20171002 is updated to 2017-10-02. +# +# If verified to be in the standard format just copy in. +# +# Otherwise assume the start date is from the beginning of time or +# end date is the end of time. + +# load up the start date string and integer representation +if [ -z "${1}" ] ; then + START_DATE="${BOT_DATE}" +elif [[ "${1}" =~ [0-9]{4}[0-9]{2}[0-9]{2} ]] ; then + START_DATE="${1:0:4}-${1:4:2}-${1:6:2}" +elif [[ "${1}" =~ [0-9]{4}-[0-9]{2}-[0-9]{2} ]] ; then + START_DATE="${1}" +else + START_DATE="${BOT_DATE}" +fi + +# Convert the correct or corrected 'start' date to an integer value +date_to_int "${START_DATE}" +start_date=${__this_integer_date} + + +# load up the end date string and integer representation +if [ -z "${2}" ] ; then + END_DATE="${EOT_DATE}" +elif [[ "${2}" =~ [0-9]{4}[0-9]{2}[0-9]{2} ]] ; then + END_DATE="${2:0:4}-${2:4:2}-${2:6:2}" +elif [[ "${2}" =~ [0-9]{4}-[0-9]{2}-[0-9]{2} ]] ; then + END_DATE="${2}" +else + END_DATE="${EOT_DATE}" +fi + +# Convert the correct or corrected 'end' date to an integer value +date_to_int "${END_DATE}" +end_date=${__this_integer_date} + +# Handle user error of specifying an end date that is before the start date +if [ ${start_date} -gt ${end_date} ] ; then + wlog "invalid date range ; end date (${END_DATE}:${end_date}) is before start (${START_DATE}:${start_date})" + wlog "correcting to defaults: from ${START_DATE} to ${END_DATE}" + START_DATE="${BOT_DATE}" + END_DATE="${EOT_DATE}" + start_date=${bot_date} + end_date="${eot_date}" +fi + +ilog "collecting log files containing logs dated ${START_DATE} to ${END_DATE} (inclusive)" + + +if [ "${3}" == "" ] ; then + elog "dated collect include file list name not specified ... exiting" + exit 1 +else + VAR_LOG_INCLUDE_LIST=${3} +fi + +create_list_file "${VAR_LOG_INCLUDE_LIST}" +create_list_file "${INC_FILE_LIST}" +create_list_file "${EXC_FILE_LIST}" +create_list_file "${NOD_FILE_LIST}" + +# Declare and init the include and exclude debug lists. +inclist=("") +exclist=("") + +############################################################################# +# +# 'filedatelist' is a list of files that are known to not contain dated logs. +# Instead these files are included unless its file date is +# older that the specified start date. +# +############################################################################# + +filedatelist=("") +filedatelist+=("/var/log/wtmp") +filedatelist+=("/var/log/dmesg") +filedatelist+=("/var/log/dmesg.old") +filedatelist+=("/var/log/sm-trap.log") +filedatelist+=("/var/log/sm-customer.log") +filedatelist+=("/var/log/sm-customer.alarm") +filedatelist+=("/var/log/sm-shutdown.log") +filedatelist+=("/var/log/nfv-vim-events.log") +filedatelist+=("/var/log/fm-customer.log") +filedatelist+=("/var/log/fm-alarm.log") +filedatelist+=("/var/log/lighttpd-access.log") +filedatelist+=("/var/log/audit/audit.log") +filedatelist+=("/var/log/rabbitmq/shutdown_log") +filedatelist+=("/var/log/rabbitmq/startup_log") +filedatelist+=("/var/log/rabbitmq/wait_log") +filedatelist+=("/var/log/rabbitmq/rabbit@localhost.log") +filedatelist+=("/var/log/nfv-vim-alarms.log") +filedatelist+=("/var/log/vswitch.cmds.log") + +# This is a list of files to always include +autoaddlist=("") +autoaddlist+=("/var/log/collect.log") + +######################################################################### +# +# 'is_in_range' returns true if the specified log file data range +# is within the bounded date range specified by the caller. +# Otherwise a false is returned. +# +# ${1} is HEAD_DATE and is the date of the first log of the file in contect +# ${2} is TAIL_DATE and is the date of the last log in the file in context +# +# expected date format is ... YYYY-MM-DD +# +# Calling Sequence is ... is_in_range HEAD_DATE TAIL_DATE +# +# There are several cases that aer handled ; +# see case comment inline below. +# +######################################################################### + +function is_in_range() +{ + local HEAD_DATE="${1}" + local TAIL_DATE="${2}" + if [[ ${HEAD_DATE} =~ [0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then + + # Convert the date to an integer value + # to make the compare easier and faster + date_to_int "${HEAD_DATE}" + head_date=${__this_integer_date} + + if [[ ${TAIL_DATE} =~ [0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then + + # Convert the date to an integer value + # to make the compare easier and faster + date_to_int "${TAIL_DATE}" + tail_date=${__this_integer_date} + + in_range=false + + # The last log is before the start date or the first log is after the end date + # if [[ "${TAIL_DATE}" < "${START_DATE}" || "${HEAD_DATE}" > "${END_DATE}" ]] ; then + if [ ${tail_date} -lt ${start_date} -o ${head_date} -gt ${end_date} ] ; then + __this_reason+=":case 0" + in_range=false + + # Case 1: the head after the start but before the end date + # .... S ... head ... E .... + elif [ ${head_date} -ge ${start_date} -a ${head_date} -le ${end_date} ] ; then + __this_reason+=":case 1" + in_range=true + + # Case 2: the tail after the start but before the end date + # .... S ... tail ... E .... + elif [ ${tail_date} -ge ${start_date} -a ${tail_date} -le ${end_date} ] ; then + __this_reason+=":case 2" + in_range=true + + # Case 3: log file date range spans the start and end dates + # head S ... ... E tail + elif [ ${head_date} -le ${start_date} -a ${tail_date} -ge ${end_date} ] ; then + __this_reason+=":case 3" + in_range=true + + else + __this_reason+=":default" + fi + else + __this_reason+=":invalid-tail-date" + # so the tail date is unknown. + # include this file as long as the head date is before end date + if [ ${head_date} -lt ${end_date} ] ; then + in_range=true + else + in_range=false + fi + fi + + if [ "${in_range}" = true ] ; then + __this_reason+=":in-range ${HEAD_DATE} to ${TAIL_DATE}" + true + else + __this_reason+=":out-of-range ${HEAD_DATE} to ${TAIL_DATE}" + false + fi + return + fi + + __this_reason+=":date-format-error ${HEAD_DATE} to ${TAIL_DATE}" + true + return +} + +########################################################################### +# +# Name : want_this_file +# +# Description: This utility first compares the filename to known exception +# cases and handles them accordingly. Exception cases do look +# for the date but with different methods. Once the date info +# is or is not found then the choice to or not to include it +# follows same general logic as others below. +# +# If not an exception case then it determines the file type +# and performs any preprocessing required. i.e. uncompressing +# the file and switching the filename to the uncompressed name. +# Data files or other unknown file types are automatically +# included without further data query by immediately returning +# true. +# +# With an expected supported filename in hand this utility will +# extract the date-only (time not included) portion, the first +# 10 characters of the first and last logs and determin if this +# logfile has logs that fall withing the specified date range. +# +# Returns : If there is no valid date found then true is returned. +# If file contains in range logs then true is returned. +# if file does not contain in range logs then false is returned. +# +# Parameters : $1 is the full pathed log file name. +# +# $1 - the filename of the file to check the date for +# +########################################################################### + +function want_this_file() +{ + local inc=true + local LOGFILE="${1}" + local filetype=$(file "${LOGFILE}") + local HEAD_DATE="" + local TAIL_DATE="" + + for add in "${autoaddlist[@]}" + do + if [ "${add}" == "${LOGFILE}" ] ; then + __this_reason+="autoadd" + true + return + fi + done + + ########################################################################## + # Exception Case: known free formatted log files. + ########################################################################## + # + # Some log files are known to not contain properly dated logs. + # Such files may just contian free format strings of information. + # + # A list of such files is in hard coded in filedatelist. + # TODO: consider making this a file that is loaded. + # + # Check to see if this is an auto add file + # Only exlude such files if its last modified date is before start date. + # + ########################################################################## + for add in "${filedatelist[@]}" + do + if [ "${add}" == "${LOGFILE}" ] ; then + __this_reason+="filedate" + + # Don't include empty files that are in the hard coded filedatelist + filetype=$(file "${LOGFILE}") + if [ ! -z "${filetype}" ] ; then + case ${filetype} in + *empty*) + __this_reason="empty" + track "${LOGFILE}" "empty" + false + return + ;; + *) + ;; + esac + fi + + # get last modified date + FILE_DATE=$(stat -c %y "${LOGFILE}" | cut -b 1-10) + date_to_int "${FILE_DATE}" + if [ ${__this_integer_date} -ge ${start_date} ] ; then + __this_reason+=":in-range ${FILE_DATE}" + true + else + __this_reason+=":out-of-range ${FILE_DATE}" + false + fi + return + fi + done + + # O.K. if we get here then this filename is not in the static list + if [ ! -z "${filetype}" ] ; then + + case ${filetype} in + + *directory*) + # Skip over a directory only path. + # No worries, the files in that directory will be handled. + __this_reason+="directory" + false + return + ;; + + *ASCII*|*text*|*compressed*) + + if [[ ${filetype} == *"compressed"* ]] ; then + fileext=${LOGFILE##*.} + case "${fileext}" in + gz) + tmpfile=$(mktemp) + #__this_reason+="gzipped" + zcat "${LOGFILE}" | head -5 > "$tmpfile" + zcat "${LOGFILE}" | tail -5 >> "$tmpfile" + + # save the current compressed log filename + # so that it can be restored after the + # recursion call below + LOGFILE_save="${LOGFILE}" + want_this_file "$tmpfile" + rc=${?} + LOGFILE="${LOGFILE_save}" + + # cleanup ; get rid of the temp file + rm -f "$tmpfile" 2>/dev/null + if [ ${rc} -eq 0 ] ; then + true + else + false + fi + return + ;; + tgz) + __this_reason+="tarball" + true + return + ;; + *) + __this_reason+="compress:[${fileext}]" + true + return + ;; + esac + fi + + # Read the first log in the file + HEAD_DATE=$(head -1 "${LOGFILE}") + + ############################################################## + # Minor Exception Case: empty/short first log + ############################################################## + # + # handle one empty or short first line by fetching second log + # + ############################################################## + + if [ ${#HEAD_DATE} -lt 10 ] ; then + HEAD_DATE=$(head -2 "${LOGFILE}" | sed -n '2p' | cut -b 1-11) + fi + + + ############################################################## + # Typical Case: YYYY-MM-DD + ############################################################## + # + # check for most typical date format. + # + ############################################################## + + if [[ ${HEAD_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then + __this_reason+="typical" + TAIL_DATE=$(tail -1 "${LOGFILE}" | cut -b 1-11) + if [[ ${TAIL_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then + + # a call to 'is_in_range' returns false (1) if this + # file's logs are all out of range date + is_in_range "${HEAD_DATE:0:10}" "${TAIL_DATE:0:10}" + if [ $? -eq 0 ] ; then + true + else + false + fi + return + + else + + ####################################################### + # Exception Case: Unrecognized date format in last log + ####################################################### + # + # try the second last line. This case is typical in + # cron.log in 15.12 MAIL logs which send a purious ')' + # as a second log. Also if the log file has auto blank + # lines between logs leaving a blank line as the last + # log. + # + # this exception ties the second last log instead. + # + ####################################################### + TAIL_DATE=$(tail -2 "${LOGFILE}" | sed -n '1p' | cut -b 1-11) + if [[ ${TAIL_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then + + is_in_range "${HEAD_DATE:0:10}" "${TAIL_DATE:0:10}" + if [ $? -eq 0 ] ; then + true + else + false + fi + return + + else + # default to true if the dates could not be parsed + __this_reason+=":invalid-tail-date" + + date_to_int "${HEAD_DATE}" + head_date=${__this_integer_date} + + # so the tail date is unknown. + # include this file as long as the head date is before end date + if [ ${head_date} -lt ${end_date} ] ; then + true + else + false + fi + return + fi + fi + + else + + ########################################################### + # Exception Case 1: logs date prefix starts with '[' + ########################################################### + # + # logdate starts with a '[' ... [2017-10-02 + # + # In this case we just recognize it and increment past it + # and then assume the last log will have the same format + # + ########################################################### + + if [ "${HEAD_DATE:0:1}" == "[" ] ; then + __this_reason+="exception1" + HEAD_DATE=${HEAD_DATE:1:11} + if [[ ${HEAD_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then + + TAIL_DATE=$(tail -1 "${LOGFILE}" | cut -b 2-11) + if [[ ${TAIL_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then + __this_reason+=".1" + is_in_range "${HEAD_DATE:0:10}" "${TAIL_DATE:0:10}" + if [ $? -eq 0 ] ; then + true + else + false + fi + return + else + TAIL_DATE=$(tail -1 "${LOGFILE}" | cut -b 1-10) + if [[ ${TAIL_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then + __this_reason+=".2" + is_in_range "${HEAD_DATE:0:10}" "${TAIL_DATE:0:10}" + if [ $? -eq 0 ] ; then + true + else + false + fi + return + + else + + if [ "${TAIL_DATE:0:1}" == "[" ] ; then + __this_reason+=".3" + TAIL_DATE=${TAIL_DATE:1:11} + if [[ ${TAIL_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then + is_in_range "${HEAD_DATE}" "${TAIL_DATE}" + if [ $? -eq 0 ] ; then + true + else + false + fi + return + else + __this_reason+=":invalid-tail-date" + true + return + fi + else + __this_reason+=":tail-date-not-found" + is_in_range "${HEAD_DATE}" "${EOT_DATE}" + if [ $? -eq 0 ] ; then + true + else + false + fi + return + fi + fi + fi + else + # /var/log/dmesg is typical of this case + # no log date and many logs start with [ uptime] + __this_reason+=":invalid-head-date" + true + return + fi + + ########################################################### + # Exception Case 2: journel.log handling + ########################################################### + # + # first log in file contains start and stop date + # + # "-- Logs begin at Thu 2017-07-06 12:28:35 UTC, end at Thu 2017-07-06 12:33:31 UTC. --" + # ^^^^^^^^^^ ^^^^^^^^^^ + # + # This exception case gets the head and tail log date from + # this first log. + ########################################################### + + elif [ "${HEAD_DATE:0:13}" == "-- Logs begin" ] ; then + __this_reason+="exception2" + + # need to get more of the line + HEAD_DATE=$(head -1 "${LOGFILE}") + + is_in_range "${HEAD_DATE:21:10}" "${HEAD_DATE:57:10}" + if [ $? -eq 0 ] ; then + true + else + false + fi + return + + ########################################################### + # Exception Case 3: journel.log handling + ########################################################### + # + # some logs like openstack.log have some logs that are + # prefixed by keystone:log. This case handles that + # + ########################################################### + elif [ "${HEAD_DATE:0:13}" == "keystone:log " ] ; then + __this_reason+="exception3" + + # need to get more of the line + HEAD_DATE="${HEAD_DATE:13:10}" + TAIL_DATE=$(tail -1 "${LOGFILE}") + + if [ "${TAIL_DATE:0:13}" == "keystone:log " ] ; then + TAIL_DATE="${TAIL_DATE:13:10}" + else + TAIL_DATE="${TAIL_DATE:0:10}" + fi + + is_in_range "${HEAD_DATE}" "${TAIL_DATE}" + if [ $? -eq 0 ] ; then + true + else + false + fi + return + + else + + ####################################################### + # Exception Case 4: horizon.log + ####################################################### + # + # Search the first and last 30 logs for a valid date. + # This should handle seeing a traceback at the head or + # tail of the log file. + # + ####################################################### + __this_reason+="exception4" + temp_head=$(head -30 "${LOGFILE}") + for ((loop_head=1;loop_head<31;loop_head++)) + do + HEAD_DATE=$(echo "${temp_head}" | sed -n "${loop_head}"p | cut -b 1-10) + if [[ ${HEAD_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then + temp_tail=$(tail -30 "${LOGFILE}") + for ((loop_tail=1;loop_tail<31;loop_tail++)) + do + TAIL_DATE=$(echo "${temp_tail}" | sed -n ${loop_tail}p | cut -b 1-10) + if [[ ${TAIL_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then + + is_in_range "${HEAD_DATE}" "${TAIL_DATE}" + if [ $? -eq 0 ] ; then + true + else + false + fi + return + + fi + done + + # default to including it if no date at + # the end of the file is found + true + return + fi + done + + ###################################################### + # Exception Case 5: + ###################################################### + # + # Otherwise the file has no date or the date + # format is unrecognized so just include the file + # regardless of its date. + # + ###################################################### + __this_reason="nodate" + true + return + fi + fi + ;; + + *archive*) + + # Archive files like .tar are not extracted. + # Instead it is only collected if its last modified date is + # after the start date + + __this_reason+="archive" + FILE_DATE=$(stat -c %y "${LOGFILE}" | cut -b 1-10) + date_to_int "${FILE_DATE}" + if [ ${__this_integer_date} -ge ${start_date} ] ; then + __this_reason+=":in-range ${FILE_DATE}" + true + else + __this_reason+=":out-of-range ${FILE_DATE}" + false + fi + return + ;; + + *empty*) + __this_reason="empty" + track "${LOGFILE}" "empty" + false + return + ;; + + *data*) + __this_reason="data" + true + return + ;; + + *executable*) + __this_reason="executable" + true + return + ;; + + # very short file (no magic) + *"very short file"*) + __this_reason="small" + true + return + ;; + + *link*) + __this_reason="link" + false + return + ;; + + *swap*) + + __this_reason="swap" + false + return + ;; + + *fifo*) + + __this_reason="fifo" + false + return + ;; + + *socket*) + + __this_reason="socket" + false + return + ;; + + *) + __this_reason="other" + true + return + ;; + esac + else + __this_reason="unknown" + wlog "Adding ${logfile} ; unknown filetype" + true + return + fi + + # catch all default + true + return +} + +############################################################################# +# +# 'manage_file' adds the specified file to either the 'include' or exclude' +# reason lists. In the include case the most important part of +# this function appends the filename to the file specified by +# "VAR_LOG_INCLUDE_LIST" which is the file that collect_host +# uses to know what files in /var/log need to be included in +# the collect tarball. +# +############################################################################# + +function manage_file() +{ + local filename="${1}" + local action="${2}" + local reason="${3}" + + if [ "${action}" == "${EXCLUDE_FILE}" ] ; then + echo "${filename} excluded (${reason})" >> "${EXC_FILE_LIST}" + else + echo "${filename} included (${reason})" >> "${INC_FILE_LIST}" + + # add the file to the list of files to be collected + echo "${filename}" >> ${VAR_LOG_INCLUDE_LIST} + fi + + dlog "${action}: ${filename} (${reason})" +} + +############################################################################# +# +# 'handle_response' adds or excludes the specified file based on +# arguement $2 being 0 - true - include or +# !0 - false - exclude +# +# $1 - file +# $2 - include control ( true or false ) +# +############################################################################# + +function handle_response() +{ + local logfile="${1}" + local include="${2}" + + if [ "${include}" -eq 0 ] ; then + inclist=("${inclist[@]}" ${logfile}) + track "${logfile}" "${INCLUDE_FILE}" + + else + exclist=("${exclist[@]}" ${logfile}) + track "${logfile}" "${EXCLUDE_FILE}" + fi + + # record any that have been tagged as 'nodate' as + # candidate for special handling. + if [[ "${__this_reason}" == *"nodate"* ]] ; then + echo "${logfile}" >> "${NOD_FILE_LIST}" + fi +} + +########################################################################### +########################################################################### +# +# Lets start looking at the files now ... +# +# Get all the files in /var/log base dir (not the subdirectories) +# +########################################################################### +########################################################################### + +# get a list of the files in "baselogdir" ; aka /var/log +# will look at the sub directories later. +dirlist+=$(find ${baselogdir} -mindepth 1 -maxdepth 1 -type f) + +# +# Debug: +# +# To debug handling a specific file as a filelist override. +# This clears the list in favor of the specific file specified as +# argument 8 on the command line. +# +if [ "${5}" != "" ] ; then + dlog "Overriding dirlist with specified file:${5}" + dirlist=("${5}") +fi + +# echo "${baselogdir} filelist: ... ${dirlist}..." +for logfile in ${dirlist} +do + # echo "File: ${logfile}" + __this_reason="" + track "${logfile}" "totals" + want_this_file "${logfile}" + handle_response "${logfile}" "${?}" +done + +########################################################################### +# Get all the files in baselogdir subdirectories # +########################################################################### + +subdirlist=$(find ${baselogdir} -mindepth 1 -maxdepth 20 -type d) + +# +# Debug: +# +# To debug handling a specific file that is in a /var/log subdirectory as a +# filelist override. +# +if [ "${5}" != "" ] ; then + dlog "Overriding subdirlist with specified file:${5}" + subdirlist=("") +fi + +# echo "${baselogdir} subdirlist ${subdirlist}..." +for logdir in ${subdirlist} +do + __this_reason="" + + # this find must find more than just its own dir + # so we compare to greater than one + if [ $(find "${logdir}" | wc -l) -gt 1 ]; then + for logfile in ${logdir}/* + do + __this_reason="" + track "$logfile" "totals" + want_this_file "$logfile" + handle_response "$logfile" "$?" + done + else + __this_reason="empty" + manage_file "${logdir}" "${EXCLUDE_FILE}" "empty directory" + fi +done + + +dlog "Include List: ${INC_FILE_LIST}" +for inc in "${inclist[@]}" +do + if [ ${#inc} -gt 2 ] ; then + dlog "including ${inc}" + # echo "${inc}" >> "${INC_FILE_LIST}.summary" + fi +done + + +dlog "Exclude List: ${EXC_FILE_LIST}" +for exc in "${exclist[@]}" +do + if [ ${#exc} -gt 2 ] ; then + dlog "excluding ${exc}" + # echo "${exc}" >> "${EXC_FILE_LIST}.summary" + fi +done + +summary + +exit 0 diff --git a/tools/collector/scripts/collect_host b/tools/collector/scripts/collect_host new file mode 100755 index 000000000..873b191ca --- /dev/null +++ b/tools/collector/scripts/collect_host @@ -0,0 +1,482 @@ +#! /bin/bash +######################################################################## +# +# Copyright (c) 2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +######################################################################## + +# +# Import commands, variables and convenience functions available to +# all collectors ; common and user defined. +# +source /usr/local/sbin/collect_utils + +KEYRING_PATH="/opt/platform/.keyring" +if [ -e ${KEYRING_PATH} ] ; then + CRED=`find /opt/platform/.keyring -name .CREDENTIAL` + if [ ! -z "${CRED}" ] ; then + NOVAOPENRC="/etc/nova/openrc" + if [ -e ${NOVAOPENRC} ] ; then + source ${NOVAOPENRC} 2>/dev/null 1>/dev/null + fi + fi +fi + +# +# parse input parameters +# +COLLECT_NAME="${1}" +DEBUG=${8} +set_debug_mode ${DEBUG} + +# Calling parms +# +# 1 = collect name +# 2 = start date option +# 3 = start date +# 4 = "any" (ignored - no longer used ; kept to support upgrades/downgrades) +# 5 = end date option +# 6 = end date +# 7 = "any" (ignored - no longer used ; kept to support upgrades/downgrades) +# 8 = debug mode +logger -t ${COLLECT_TAG} "${0} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8}" + +# parse out the start data/time data if it is present +STARTDATE_RANGE=false +STARTDATE="any" +if [ "${2}" == "${STARTDATE_OPTION}" ] ; then + if [ "${3}" != "any" -a ${#3} -gt 7 ] ; then + STARTDATE_RANGE=true + STARTDATE="${3}" + fi +fi + +# parse out the end date/time if it is present +ENDDATE_RANGE=false +ENDDATE="any" +if [ "${5}" == "${ENDDATE_OPTION}" ] ; then + if [ "${6}" != "any" -a ${#6} -gt 7 ] ; then + ENDDATE_RANGE=true + ENDDATE="${6}" + fi +fi + +COLLECT_BASE_DIR="/scratch" +EXTRA="var/extra" +hostname="${HOSTNAME}" +COLLECT_NAME_DIR="${COLLECT_BASE_DIR}/${COLLECT_NAME}" +EXTRA_DIR="${COLLECT_NAME_DIR}/${EXTRA}" +TARBALL="${COLLECT_NAME_DIR}.tgz" +COLLECT_PATH="/etc/collect.d" +RUN_EXCLUDE="/etc/collect/run.exclude" +ETC_EXCLUDE="/etc/collect/etc.exclude" +COLLECT_INCLUDE="/var/run /etc /root" +FLIGHT_RECORDER_PATH="var/lib/sm/" +FLIGHT_RECORDER_FILE="sm.eru.v1" +VAR_LOG_INCLUDE_LIST="/tmp/${COLLECT_NAME}.lst" +COLLECT_DIR_PCENT_CMD="df --output=pcent ${COLLECT_BASE_DIR}" +COLLECT_DIR_USAGE_CMD="df -h ${COLLECT_BASE_DIR}" +COLLECT_DATE="/usr/local/sbin/collect_date" + +function log_space() +{ + local msg=${1} + + space="`${COLLECT_DIR_USAGE_CMD}`" + space1=`echo "${space}" | grep -v Filesystem` + ilog "${COLLECT_BASE_DIR} ${msg} ${space1}" +} + +function space_precheck() +{ + space="`${COLLECT_DIR_PCENT_CMD}`" + space1=`echo "${space}" | grep -v Use` + size=`echo ${space1} | cut -f 1 -d '%'` + if [ ${size} -ge 0 -a ${size} -le 100 ] ; then + ilog "${COLLECT_BASE_DIR} is $size% full" + if [ ${size} -ge ${MIN_PERCENT_SPACE_REQUIRED} ] ; then + wlog "${HOSTNAME}:${COLLECT_BASE_DIR} does not have enough available space in to perform collect" + wlog "${HOSTNAME}:${COLLECT_BASE_DIR} must be below ${MIN_PERCENT_SPACE_REQUIRED}% to perform collect" + wlog "Increase available space in ${HOSTNAME}:${COLLECT_BASE_DIR} and retry operation." + echo "${FAIL_INSUFFICIENT_SPACE_STR}" + exit ${FAIL_INSUFFICIENT_SPACE} + fi + else + wlog "unable to parse available space from '${COLLECT_DIR_PCENT_CMD}' output" + fi +} + +space_precheck + +CURR_DIR=`pwd` +mkdir -p ${COLLECT_NAME_DIR} +cd ${COLLECT_NAME_DIR} + +# create dump target extra-stuff directory +mkdir -p ${EXTRA_DIR} + +RETVAL=0 + +# Remove any previous collect error log. +# Start this collect with an empty file. +# +# stderr is directed to this log during the collect process. +# By searching this log after collect_host is run we can find +# errors that occured during collect. +# The only real error that we care about right now is the +# +# "No space left on device" error +# +rm -f ${COLLECT_ERROR_LOG} +touch ${COLLECT_ERROR_LOG} +chmod 644 ${COLLECT_ERROR_LOG} +echo "`date '+%F %T'` :${COLLECT_NAME_DIR}" > ${COLLECT_ERROR_LOG} + +ilog "creating local collect tarball ${COLLECT_NAME_DIR}.tgz" + +################################################################################ +# Run collect scripts to check system status +################################################################################ +function collect_parts() +{ + if [ -d ${COLLECT_PATH} ]; then + for i in ${COLLECT_PATH}/*; do + if [ -f $i ]; then + $i ${COLLECT_NAME_DIR} ${EXTRA_DIR} ${hostname} + fi + done + fi +} + + +function collect_extra() +{ + # dump process lists + LOGFILE="${EXTRA_DIR}/process.info" + echo "${hostname}: Process Info ......: ${LOGFILE}" + + delimiter ${LOGFILE} "ps -e -H -o ..." + ${PROCESS_DETAIL_CMD} >> ${LOGFILE} + + # Collect process and thread info (tree view) + delimiter ${LOGFILE} "pstree --arguments --ascii --long --show-pids" + pstree --arguments --ascii --long --show-pids >> ${LOGFILE} + + # Collect process, thread and scheduling info (compute subfunction only) + # (also gets process 'affinity' which is useful on computes; + which ps-sched.sh >/dev/null 2>&1 + if [ $? -eq 0 ]; then + delimiter ${LOGFILE} "ps-sched.sh" + ps-sched.sh >> ${LOGFILE} + fi + + # Collect process, thread and scheduling, and elapsed time + # This has everything that ps-sched.sh does, except for cpu affinity mask, + # adds: stime,etime,time,wchan,tty). + delimiter ${LOGFILE} "ps -eL -o pid,lwp,ppid,state,class,nice,rtprio,priority,psr,stime,etime,time,wchan:16,tty,comm,command" + ps -eL -o pid,lwp,ppid,state,class,nice,rtprio,priority,psr,stime,etime,time,wchan:16,tty,comm,command >> ${LOGFILE} + + # Various host attributes + LOGFILE="${EXTRA_DIR}/host.info" + echo "${hostname}: Host Info .........: ${LOGFILE}" + + # CGCS build info + delimiter ${LOGFILE} "${BUILD_INFO_CMD}" + ${BUILD_INFO_CMD} >> ${LOGFILE} + + delimiter ${LOGFILE} "uptime" + uptime >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "cat /proc/cmdline" + cat /proc/cmdline >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "cat /proc/version" + cat /proc/version >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "cat /proc/cpuinfo" + cat /proc/cpuinfo >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "ip addr show" + ip addr show >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "lspci -nn" + lspci -nn >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "find /sys/kernel/iommu_groups/ -type l" + find /sys/kernel/iommu_groups/ -type l >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + # networking totals + delimiter ${LOGFILE} "cat /proc/net/dev" + cat /proc/net/dev >> ${LOGFILE} + + delimiter ${LOGFILE} "dmidecode" + dmidecode >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + # summary of scheduler tunable settings + delimiter ${LOGFILE} "cat /proc/sched_debug | head -15" + cat /proc/sched_debug | head -15 >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + if [ "${SKIP_MASK}" = "true" ]; then + delimiter ${LOGFILE} "facter (excluding ssh info)" + facter | grep -iv '^ssh' >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + else + delimiter ${LOGFILE} "facter" + facter >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + fi + + if [[ "$nodetype" == "compute" || "$subfunction" == *"compute"* ]] ; then + delimiter ${LOGFILE} "topology" + topology >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + fi + + LOGFILE="${EXTRA_DIR}/memory.info" + echo "${hostname}: Memory Info .......: ${LOGFILE}" + + delimiter ${LOGFILE} "cat /proc/meminfo" + cat /proc/meminfo >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "cat /sys/devices/system/node/node?/meminfo" + cat /sys/devices/system/node/node?/meminfo >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "cat /proc/slabinfo" + log_slabinfo ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "ps -e -o ppid,pid,nlwp,rss:10,vsz:10,cmd --sort=-rss" + ps -e -o ppid,pid,nlwp,rss:10,vsz:10,cmd --sort=-rss >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + # mounted hugepages + delimiter ${LOGFILE} "lsof | grep /mnt/huge" + lsof | awk '($3 !~ /^[0-9]+$/ && /\/mnt\/huge/) || NR==1 {print $0;}' >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + # hugepages numa mapping + delimiter ${LOGFILE} "grep huge /proc/*/numa_maps" + grep -e " huge " /proc/*/numa_maps >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + # rootfs and tmpfs usage + delimiter ${LOGFILE} "df -h -H -T --local -t rootfs -t tmpfs" + df -h -H -T --local -t rootfs -t tmpfs >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + LOGFILE="${EXTRA_DIR}/filesystem.info" + echo "${hostname}: Filesystem Info ...: ${LOGFILE}" + + # disk inodes usage + delimiter ${LOGFILE} "df -h -H -T --local -t rootfs -t tmpfs" + df -h -H -T --local -t rootfs -t tmpfs >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + # disk space usage + delimiter ${LOGFILE} "df -h -H -T --local -t ext2 -t ext3 -t ext4 -t xfs --total" + df -h -H -T --local -t ext2 -t ext3 -t ext4 -t xfs --total >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + # disk inodes usage + delimiter ${LOGFILE} "df -h -H -T --local -i -t ext2 -t ext3 -t ext4 -t xfs --total" + df -h -H -T --local -i -t ext2 -t ext3 -t ext4 -t xfs --total >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + # disks by-path values + delimiter ${LOGFILE} "ls -lR /dev/disk" + ls -lR /dev/disk >> ${LOGFILE} + + # disk summary (requires sudo/root) + delimiter ${LOGFILE} "fdisk -l" + fdisk -l >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "cat /proc/scsi/scsi" + cat /proc/scsi/scsi >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + # Controller specific stuff + if [ "$nodetype" = "controller" ] ; then + + delimiter ${LOGFILE} "cat /proc/drbd" + cat /proc/drbd >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "/sbin/drbdadm dump" + /sbin/drbdadm dump >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + fi + + # LVM summary + delimiter ${LOGFILE} "/usr/sbin/vgs --version ; /usr/sbin/pvs --version ; /usr/sbin/lvs --version" + /usr/sbin/vgs --version >> ${LOGFILE} + /usr/sbin/pvs --version >> ${LOGFILE} + /usr/sbin/lvs --version >> ${LOGFILE} + + delimiter ${LOGFILE} "/usr/sbin/vgs --all --options all" + /usr/sbin/vgs --all --options all >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "/usr/sbin/pvs --all --options all" + /usr/sbin/pvs --all --options all >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "/usr/sbin/lvs --all --options all" + /usr/sbin/lvs --all --options all >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + # iSCSI Information + LOGFILE="${EXTRA_DIR}/iscsi.info" + echo "${hostname}: iSCSI Information ......: ${LOGFILE}" + + if [ "$nodetype" = "controller" ] ; then + # Controller- LIO exported initiators summary + delimiter ${LOGFILE} "targetcli ls" + targetcli ls >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + # Controller - LIO sessions + delimiter ${LOGFILE} "targetcli sessions detail" + targetcli sessions detail >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + elif [[ "$nodetype" == "compute" || "$subfunction" == *"compute"* ]] ; then + # Compute - iSCSI initiator information + collect_dir=${EXTRA_DIR}/iscsi_initiator_info + mkdir -p ${collect_dir} + cp -rf /run/iscsi-cache/nodes/* ${collect_dir} + find ${collect_dir} -type d -exec chmod 750 {} \; + + # Compute - iSCSI initiator active sessions + delimiter ${LOGFILE} "iscsiadm -m session" + iscsiadm -m session >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + # Compute - iSCSI udev created nodes + delimiter ${LOGFILE} "ls -la /dev/disk/by-path | grep \"iqn\"" + ls -la /dev/disk/by-path | grep "iqn" >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + fi + + LOGFILE="${EXTRA_DIR}/history.info" + echo "${hostname}: Bash History ......: ${LOGFILE}" + + # history + delimiter ${LOGFILE} "cat /home/wrsroot/.bash_history" + cat /home/wrsroot/.bash_history >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + LOGFILE="${EXTRA_DIR}/interrupt.info" + echo "${hostname}: Interrupt Info ....: ${LOGFILE}" + + # interrupts + delimiter ${LOGFILE} "cat /proc/interrupts" + cat /proc/interrupts >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "cat /proc/softirqs" + cat /proc/softirqs >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + # Controller specific stuff + if [ "$nodetype" = "controller" ] ; then + netstat -pan > ${EXTRA_DIR}/netstat.info + fi + + LOGFILE="${EXTRA_DIR}/blockdev.info" + echo "${hostname}: Block Devices Info : ${LOGFILE}" + + # Collect block devices - show all sda and cinder devices, and size + delimiter ${LOGFILE} "lsblk" + lsblk >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + # Collect block device topology - show devices and which io-scheduler + delimiter ${LOGFILE} "lsblk --topology" + lsblk --topology >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + # Collect SCSI devices - show devices and cinder attaches, etc + delimiter ${LOGFILE} "lsblk --scsi" + lsblk --scsi >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} +} + +log_space "before collect ......:" + +collect_extra +collect_parts + +# +# handle collect collect-after and collect-range and then +# in elif clause collect-before +# +VAR_LOG="/var/log" +if [ -e /www/var/log ]; then + VAR_LOG="$VAR_LOG /www/var/log" +fi + +rm -f ${VAR_LOG_INCLUDE_LIST} + +if [ "${STARTDATE_RANGE}" == true ] ; then + if [ "${ENDDATE_RANGE}" == false ] ; then + ilog "collecting $VAR_LOG files containing logs after ${STARTDATE}" + ${COLLECT_DATE} ${STARTDATE} ${ENDDATE} ${VAR_LOG_INCLUDE_LIST} ${DEBUG} "" + else + ilog "collecting $VAR_LOG files containing logs between ${STARTDATE} and ${ENDDATE}" + ${COLLECT_DATE} ${STARTDATE} ${ENDDATE} ${VAR_LOG_INCLUDE_LIST} ${DEBUG} "" + fi +elif [ "${ENDDATE_RANGE}" == true ] ; then + STARTDATE="20130101" + ilog "collecting $VAR_LOG files containing logs before ${ENDDATE}" + ${COLLECT_DATE} ${STARTDATE} ${ENDDATE} ${VAR_LOG_INCLUDE_LIST} ${DEBUG} "" +else + ilog "collecting all of $VAR_LOG" + find $VAR_LOG ! -empty > ${VAR_LOG_INCLUDE_LIST} +fi + +# Add VM console.log +for i in /etc/nova/instances/*/console.log; do + if [ -e "$i" ]; then + tmp=`dirname $i` + mkdir -p ${COLLECT_NAME_DIR}/$tmp + cp $i ${COLLECT_NAME_DIR}/$tmp + fi +done + +log_space "before first tar ....:" + +(cd ${COLLECT_NAME_DIR} ; ${IONICE_CMD} ${NICE_CMD} ${TAR_CMD} ${COLLECT_NAME_DIR}/${COLLECT_NAME}.tar -T ${VAR_LOG_INCLUDE_LIST} -X ${RUN_EXCLUDE} -X ${ETC_EXCLUDE} ${COLLECT_INCLUDE} 2>>${COLLECT_ERROR_LOG} 1>>${COLLECT_ERROR_LOG} ) + +log_space "after first tar .....:" + +(cd ${COLLECT_NAME_DIR} ; ${IONICE_CMD} ${NICE_CMD} ${UNTAR_CMD} ${COLLECT_NAME_DIR}/${COLLECT_NAME}.tar 2>>${COLLECT_ERROR_LOG} 1>>${COLLECT_ERROR_LOG} ) + +log_space "after first untar ...:" + +rm -f ${COLLECT_NAME_DIR}/${COLLECT_NAME}.tar + +log_space "after delete tar ....:" + +if [ "${SKIP_MASK}" != "true" ]; then + # Run password masking before final tar + dlog "running /usr/local/sbin/collect_mask_passwords ${COLLECT_NAME_DIR} ${EXTRA_DIR}" + /usr/local/sbin/collect_mask_passwords ${COLLECT_NAME_DIR} ${EXTRA_DIR} + log_space "after passwd masking :" +fi + +(cd ${COLLECT_BASE_DIR} ; ${IONICE_CMD} ${NICE_CMD} ${TAR_ZIP_CMD} ${COLLECT_NAME_DIR}.tgz ${COLLECT_NAME} 2>/dev/null 1>/dev/null ) + +log_space "after first tarball .:" + +mkdir -p ${COLLECT_NAME_DIR}/${FLIGHT_RECORDER_PATH} + +(cd /${FLIGHT_RECORDER_PATH} ; ${TAR_ZIP_CMD} ${COLLECT_NAME_DIR}/${FLIGHT_RECORDER_PATH}/${FLIGHT_RECORDER_FILE}.tgz ./${FLIGHT_RECORDER_FILE} 2>>${COLLECT_ERROR_LOG} 1>>${COLLECT_ERROR_LOG}) + +# Pull in an updated user.log which contains the most recent collect logs +# ... be sure to exclude any out of space logs +tail -30 /var/log/user.log | grep "COLLECT:" | grep -v "${FAIL_OUT_OF_SPACE_STR}" >> ${COLLECT_ERROR_LOG} +cp -a ${COLLECT_LOG} ${COLLECT_LOG}.last +cp -a ${COLLECT_ERROR_LOG} ${COLLECT_LOG} +cp -a ${COLLECT_LOG} ${COLLECT_NAME_DIR}/var/log + +log_space "with flight data ....:" + +(cd ${COLLECT_BASE_DIR} ; ${IONICE_CMD} ${NICE_CMD} ${TAR_ZIP_CMD} ${COLLECT_NAME_DIR}.tgz ${COLLECT_NAME} 2>>${COLLECT_ERROR_LOG} 1>>${COLLECT_ERROR_LOG} ) + +log_space "after collect .......:" + +rm -rf ${COLLECT_NAME_DIR} +rm -f ${VAR_LOG_INCLUDE_LIST} + +log_space "after cleanup .......:" + +# Check for collect errors +# Only out of space error is enough to fail this hosts's collect +collect_errors ${HOSTNAME} +RC=${?} + +rm -f ${COLLECT_ERROR_LOG} + +if [ ${RC} -ne 0 ] ; then + rm -f ${COLLECT_NAME_DIR}.tgz + ilog "${FAIL_OUT_OF_SPACE_STR} ${COLLECT_BASE_DIR}" +else + ilog "collect of ${COLLECT_NAME_DIR}.tgz succeeded" + echo "${collect_done}" +fi diff --git a/tools/collector/scripts/collect_ima.sh b/tools/collector/scripts/collect_ima.sh new file mode 100755 index 000000000..dd35bfa63 --- /dev/null +++ b/tools/collector/scripts/collect_ima.sh @@ -0,0 +1,60 @@ +#! /bin/bash +# +# Copyright (c) 2017 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# Loads Up Utilities and Commands Variables +source /usr/local/sbin/collect_parms +source /usr/local/sbin/collect_utils + +function is_extended_profile() +{ + if [ ! -n "${security_profile}" ] || [ "${security_profile}" != "extended" ]; then + return 0 + else + return 1 + fi +} + +SERVICE="ima" +LOGFILE="${extradir}/${SERVICE}.info" + +############################################################################### +# All Node Types +############################################################################### + +is_extended_profile +if [ "$?" = "0" ] ; then + exit 0 +fi + +echo "${hostname}: IMA Info ..........: ${LOGFILE}" + +delimiter ${LOGFILE} "IMA Kernel Modules" +lsmod | grep ima >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + +delimiter ${LOGFILE} "Auditd status" +service auditd status >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} +ps -aux | grep audit >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + +mkdir -p ${extradir}/integrity 2>>${COLLECT_ERROR_LOG} + +delimiter ${LOGFILE} "IMA Runtime Measurement and Violations cache" +if [ -d "/sys/kernel/security/ima" ]; then + ls /sys/kernel/security/ima >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + cp -rf /sys/kernel/security/ima ${extradir}/integrity 2>>${COLLECT_ERROR_LOG} +else + echo "ERROR: IMA Securityfs directory does not exist!" >> ${LOGFILE} +fi + +cp -rf /etc/modprobe.d/ima.conf ${extradir}/integrity 2>>${COLLECT_ERROR_LOG} +cp -rf /etc/modprobe.d/integrity.conf ${extradir}/integrity 2>>${COLLECT_ERROR_LOG} +cp -rf /etc/ima.policy ${extradir}/integrity 2>>${COLLECT_ERROR_LOG} + +# make sure all these collected files are world readible +chmod -R 755 ${extradir}/integrity + +exit 0 diff --git a/tools/collector/scripts/collect_mask_passwords b/tools/collector/scripts/collect_mask_passwords new file mode 100644 index 000000000..b7f0e2461 --- /dev/null +++ b/tools/collector/scripts/collect_mask_passwords @@ -0,0 +1,123 @@ +#! /bin/bash +# +# Copyright (c) 2017 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +COLLECT_NAME_DIR=$1 +EXTRA_DIR=$2 + +# Strip the passwords from assorted config files +for conffile in \ + ${COLLECT_NAME_DIR}/etc/aodh/aodh.conf \ + ${COLLECT_NAME_DIR}/etc/ceilometer/ceilometer.conf \ + ${COLLECT_NAME_DIR}/etc/cinder/cinder.conf \ + ${COLLECT_NAME_DIR}/etc/fm.conf \ + ${COLLECT_NAME_DIR}/etc/glance/glance-api.conf \ + ${COLLECT_NAME_DIR}/etc/glance/glance-registry.conf \ + ${COLLECT_NAME_DIR}/etc/heat/heat.conf \ + ${COLLECT_NAME_DIR}/etc/ironic/ironic.conf \ + ${COLLECT_NAME_DIR}/etc/keystone/keystone.conf \ + ${COLLECT_NAME_DIR}/etc/magnum/magnum.conf \ + ${COLLECT_NAME_DIR}/etc/murano/murano.conf \ + ${COLLECT_NAME_DIR}/etc/neutron/metadata_agent.ini \ + ${COLLECT_NAME_DIR}/etc/neutron/neutron.conf \ + ${COLLECT_NAME_DIR}/etc/nfv/nfv_plugins/nfvi_plugins/config.ini \ + ${COLLECT_NAME_DIR}/etc/nova/nova.conf \ + ${COLLECT_NAME_DIR}/etc/nslcd.conf \ + ${COLLECT_NAME_DIR}/etc/openldap/slapd.conf.backup \ + ${COLLECT_NAME_DIR}/etc/openstack-dashboard/local_settings \ + ${COLLECT_NAME_DIR}/etc/panko/panko.conf \ + ${COLLECT_NAME_DIR}/etc/patching/patching.conf \ + ${COLLECT_NAME_DIR}/etc/proxy/nova-api-proxy.conf \ + ${COLLECT_NAME_DIR}/etc/rabbitmq/murano-rabbitmq.config \ + ${COLLECT_NAME_DIR}/etc/rabbitmq/rabbitmq.config \ + ${COLLECT_NAME_DIR}/etc/sysinv/api-paste.ini \ + ${COLLECT_NAME_DIR}/etc/sysinv/sysinv.conf \ + ${COLLECT_NAME_DIR}/var/extra/platform/sysinv/*/sysinv.conf.default \ + ${COLLECT_NAME_DIR}/etc/mtc.ini + +do + if [ ! -f $conffile ]; then + continue + fi + + sed -i -r 's/^(admin_password) *=.*/\1 = xxxxxx/; + s/^(auth_encryption_key) *=.*/\1 = xxxxxx/; + s/^(bindpw) .*/\1 xxxxxx/; + s/^(rootpw) .*/\1 xxxxxx/; + s/^(connection) *=.*/\1 = xxxxxx/; + s/^( *credentials) *=.*/\1 = xxxxxx/; + s/^(metadata_proxy_shared_secret) *=.*/\1 = xxxxxx/; + s/^(password) *=.*/\1 = xxxxxx/; + s/^(rabbit_password) *=.*/\1 = xxxxxx/; + s/^(sql_connection) *=.*/\1 = xxxxxx/; + s/^(stack_domain_admin_password) *=.*/\1 = xxxxxx/; + s/^(transport_url) *=.*/\1 = xxxxxx/; + s/^(SECRET_KEY) *=.*/\1 = xxxxxx/; + s/^(keystone_auth_pw) *=.*/\1 = xxxxxx/; + s/\{default_pass, <<\".*\">>\}/\{default_pass, <<\"xxxxxx\">>\}/' $conffile +done + +find ${COLLECT_NAME_DIR} -name server-cert.pem | xargs --no-run-if-empty rm -f +rm -rf ${COLLECT_NAME_DIR}/var/extra/platform/config/*/ssh_config +rm -f ${COLLECT_NAME_DIR}/var/extra/platform/.keyring/*/python_keyring/crypted_pass.cfg +rm -f ${COLLECT_NAME_DIR}/var/extra/platform/puppet/*/hieradata/secure*.yaml + +# Mask user passwords in sysinv db dump +if [ -f ${COLLECT_NAME_DIR}/var/extra/database/sysinv.db.sql.txt ]; then + sed -i -r '/COPY i_user/, /^--/ s/^(([^\t]*\t){10})[^\t]*(\t.*)/\1xxxxxx\3/; + /COPY i_community/, /^--/ s/^(([^\t]*\t){5})[^\t]*(\t.*)/\1xxxxxx\3/; + /COPY i_trap_destination/, /^--/ s/^(([^\t]*\t){6})[^\t]*(\t.*)/\1xxxxxx\3/; + s/(identity\t[^\t]*\tpassword\t)[^\t]*/\1xxxxxx/' \ + ${COLLECT_NAME_DIR}/var/extra/database/sysinv.db.sql.txt +fi + +# Mask passwords in host profiles +grep -rl '\"name\": \"password\"' ${COLLECT_NAME_DIR}/var/extra/platform/sysinv/ \ + | xargs --no-run-if-empty perl -i -e ' + $prev=""; + while (<>) + { + if (/\"name\": \"password\"/) + { + $prev =~ s/\"value\": \".*\"/\"value\": \"xxxxxx\"/; + } + print $prev; + $prev=$_; + } + print $prev;' + +# Cleanup snmp +sed -i -r 's/(rocommunity[^ ]*).*/\1 xxxxxx/' ${COLLECT_NAME_DIR}/var/extra/platform/config/*/snmp/* +sed -i -r 's/(trap2sink *[^ ]*).*/\1 xxxxxx/' ${COLLECT_NAME_DIR}/var/extra/platform/config/*/snmp/* + +# Mask passwords in bash.log and history logs +USER_HISTORY_FILES=$(find ${COLLECT_NAME_DIR} -type f -name .bash_history 2>/dev/null) +sed -i -r 's/(snmp-comm-(delete|show)) *((\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*) *){1,}/\1 xxxxxx/; + s/(snmp.*) *(--community|-c) *(\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*)/\1 \2 xxxxxx/; + s/(password)=(\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*)/\1=xxxxxx/; + s/(openstack.*) *(--password) *(\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*)/\1 \2 xxxxxx/; + s/(ldapmodifyuser.*userPassword *)(\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*)/\1 xxxxxx/' \ + ${USER_HISTORY_FILES} \ + ${COLLECT_NAME_DIR}/var/extra/history.info \ + ${COLLECT_NAME_DIR}/var/log/bash.log \ + ${COLLECT_NAME_DIR}/var/log/auth.log \ + ${COLLECT_NAME_DIR}/var/log/ldapscripts.log + +for f in ${COLLECT_NAME_DIR}/var/log/bash.log.*.gz \ + ${COLLECT_NAME_DIR}/var/log/auth.log.*.gz \ + ${COLLECT_NAME_DIR}/var/log/ldapscripts.log.*.gz +do + zgrep -q 'snmp|password' $f || continue + gunzip $f + unzipped=${f%%.gz} + sed -i -r 's/(snmp-comm-(delete|show)) *((\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*) *){1,}/\1 xxxxxx/; + s/(snmp.*) *(--community|-c) *(\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*)/\1 \2 xxxxxx/; + s/(password)=(\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*)/\1=xxxxxx/; + s/(openstack.*) *(--password) *(\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*)/\1 \2 xxxxxx/; + s/(ldapmodifyuser.*userPassword *)(\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*)/\1 xxxxxx/' $unzipped + gzip $unzipped +done + diff --git a/tools/collector/scripts/collect_networking.sh b/tools/collector/scripts/collect_networking.sh new file mode 100755 index 000000000..d60663216 --- /dev/null +++ b/tools/collector/scripts/collect_networking.sh @@ -0,0 +1,61 @@ +#! /bin/bash +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# Loads Up Utilities and Commands Variables + +source /usr/local/sbin/collect_parms +source /usr/local/sbin/collect_utils + +SERVICE="networking" +LOGFILE="${extradir}/${SERVICE}.info" +echo "${hostname}: Networking Info ...: ${LOGFILE}" + +############################################################################### +# All nodes +############################################################################### +delimiter ${LOGFILE} "ip -s link" +ip -s link >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + +delimiter ${LOGFILE} "ip -s addr" +ip -s addr >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + +delimiter ${LOGFILE} "ip -s neigh" +ip -s neigh >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + +delimiter ${LOGFILE} "ip rule" +ip rule >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + +delimiter ${LOGFILE} "ip route" +ip route >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + +delimiter ${LOGFILE} "iptables -L -v -x -n" +iptables -L -v -x -n >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + +delimiter ${LOGFILE} "iptables -L -v -x -n -t nat" +iptables -L -v -x -n -t nat >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + +delimiter ${LOGFILE} "iptables -L -v -x -n -t mangle" +iptables -L -v -x -n -t mangle >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + +############################################################################### +# Only Compute +############################################################################### +if [[ "$nodetype" = "compute" || "$subfunction" == *"compute"* ]] ; then + NAMESPACES=($(ip netns)) + for NS in ${NAMESPACES[@]}; do + delimiter ${LOGFILE} "${NS}" + ip netns exec ${NS} ip -s link + ip netns exec ${NS} ip -s addr + ip netns exec ${NS} ip -s neigh + ip netns exec ${NS} ip route + ip netns exec ${NS} ip rule + done >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} +fi + +exit 0 diff --git a/tools/collector/scripts/collect_nfv_vim.sh b/tools/collector/scripts/collect_nfv_vim.sh new file mode 100644 index 000000000..4ccd309de --- /dev/null +++ b/tools/collector/scripts/collect_nfv_vim.sh @@ -0,0 +1,45 @@ +#! /bin/bash +# +# Copyright (c) 2013-2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Loads Up Utilities and Commands Variables +source /usr/local/sbin/collect_parms +source /usr/local/sbin/collect_utils + +LOGFILE="${extradir}/nfv-vim.info" +echo "${hostname}: NFV-Vim Info ......: ${LOGFILE}" + +function is_service_active() +{ + active=`sm-query service vim | grep "enabled-active"` + if [ -z "$active" ] ; then + return 0 + else + return 1 + fi +} + +############################################################################### +# Only Controller +############################################################################### + +if [ "$nodetype" = "controller" ] ; then + is_service_active + if [ "$?" = "0" ] ; then + exit 0 + fi + + # Assumes that database_dir is unique in /etc/nfv/vim/config.ini + DATABASE_DIR=$(awk -F "=" '/database_dir/ {print $2}' /etc/nfv/vim/config.ini) + + SQLITE_DUMP="/usr/bin/sqlite3 ${DATABASE_DIR}/vim_db_v1 .dump" + + delimiter ${LOGFILE} "dump database" + timeout 30 ${SQLITE_DUMP} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} +fi + +exit 0 + diff --git a/tools/collector/scripts/collect_openstack.sh b/tools/collector/scripts/collect_openstack.sh new file mode 100755 index 000000000..743dd5d56 --- /dev/null +++ b/tools/collector/scripts/collect_openstack.sh @@ -0,0 +1,75 @@ +#! /bin/bash +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# Loads Up Utilities and Commands Variables +source /usr/local/sbin/collect_parms +source /usr/local/sbin/collect_utils + +function is_service_active() +{ + active=`sm-query service rabbit-fs | grep "enabled-active"` + if [ -z "$active" ] ; then + return 0 + else + return 1 + fi +} + +SERVICE="openstack" +LOGFILE="${extradir}/${SERVICE}.info" +echo "${hostname}: Openstack Info ....: ${LOGFILE}" + +############################################################################### +# Only Controller +############################################################################### +if [ "$nodetype" = "controller" ] ; then + + is_service_active + if [ "$?" = "0" ] ; then + exit 0 + fi + +delimiter ${LOGFILE} "openstack project list" +openstack project list >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + +delimiter ${LOGFILE} "openstack user list" +openstack user list >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + +MQ_STATUS="rabbitmqctl status" +delimiter ${LOGFILE} "${MQ_STATUS} | grep -e '{memory' -A30" +${MQ_STATUS} 2>/dev/null | grep -e '{memory' -A30 >> ${LOGFILE} + +delimiter ${LOGFILE} "RabbitMQ Queue Info" +num_queues=$(rabbitmqctl list_queues | wc -l); ((num_queues-=2)) +num_bindings=$(rabbitmqctl list_bindings | wc -l); ((num_bindings-=2)) +num_exchanges=$(rabbitmqctl list_exchanges | wc -l); ((num_exchanges-=2)) +num_connections=$(rabbitmqctl list_connections | wc -l); ((num_connections-=2)) +num_channels=$(rabbitmqctl list_channels | wc -l); ((num_channels-=2)) +arr=($(rabbitmqctl list_queues messages consumers memory | \ + awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}')) +messages=${arr[0]}; consumers=${arr[1]}; memory=${arr[2]} +printf "%6s %8s %9s %11s %8s %8s %9s %10s\n" "queues" "bindings" "exchanges" "connections" "channels" "messages" "consumers" "memory" >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} +printf "%6d %8d %9d %11d %8d %8d %9d %10d\n" $num_queues $num_bindings $num_exchanges $num_connections $num_channels $messages $consumers $memory >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + if [ -e /opt/cgcs/ceilometer/pipeline.yaml ] ; then + cp /opt/cgcs/ceilometer/pipeline.yaml ${extradir}/ceilometer_pipeline.yaml + fi +fi + + + +############################################################################### +# collect does not retrieve /etc/keystone dir +# Additional logic included to copy /etc/keystone directory +############################################################################### + +mkdir -p ${extradir}/../../etc/ +cp -R /etc/keystone/ ${extradir}/../../etc +chmod -R 755 ${extradir}/../../etc/keystone + +exit 0 diff --git a/tools/collector/scripts/collect_parms b/tools/collector/scripts/collect_parms new file mode 100644 index 000000000..66001504f --- /dev/null +++ b/tools/collector/scripts/collect_parms @@ -0,0 +1,29 @@ +#! /bin/bash +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +#echo "defaults: $1-$2-$3-$4" + +if [ -z ${1} ] ; then + basedir=/scratch +else + basedir=$1 +fi + +if [ -z ${2} ] ; then + extradir=$basedir/var/extra +else + extradir=$2 +fi + +if [ -z ${3} ] ; then + hostname=$HOSTNAME +else + hostname=$3 +fi + +mkdir -p ${extradir} diff --git a/tools/collector/scripts/collect_patching.sh b/tools/collector/scripts/collect_patching.sh new file mode 100755 index 000000000..3d696d259 --- /dev/null +++ b/tools/collector/scripts/collect_patching.sh @@ -0,0 +1,45 @@ +#! /bin/bash +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# Loads Up Utilities and Commands Variables +source /usr/local/sbin/collect_parms +source /usr/local/sbin/collect_utils + +SERVICE="patching" +LOGFILE="${extradir}/${SERVICE}.info" +echo "${hostname}: Patching Info .....: ${LOGFILE}" + +############################################################################### +# All nodes +############################################################################### +delimiter ${LOGFILE} "smart channel --show" +smart channel --show 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE} + +############################################################################### +# Only Controller +############################################################################### +if [ "$nodetype" = "controller" ] ; then + + delimiter ${LOGFILE} "sw-patch query" + sw-patch query 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE} + + delimiter ${LOGFILE} "sw-patch query-hosts" + sw-patch query-hosts 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE} + + delimiter ${LOGFILE} "sw-patch query-hosts --debug" + sw-patch query-hosts --debug 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE} + + delimiter ${LOGFILE} "find /opt/patching" + find /opt/patching 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE} + + delimiter ${LOGFILE} "find /www/pages/updates" + find /www/pages/updates 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE} + +fi + +exit 0 diff --git a/tools/collector/scripts/collect_psqldb.sh b/tools/collector/scripts/collect_psqldb.sh new file mode 100755 index 000000000..410aefa5c --- /dev/null +++ b/tools/collector/scripts/collect_psqldb.sh @@ -0,0 +1,122 @@ +#! /bin/bash +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# Loads Up Utilities and Commands Variables +source /usr/local/sbin/collect_parms +source /usr/local/sbin/collect_utils + +# postgres database commands +PSQL_CMD="sudo -u postgres psql --pset pager=off -q" +PG_DUMP_CMD="sudo -u postgres pg_dump" + +SERVICE="database" +DB_DIR="${extradir}/database" +LOGFILE="${extradir}/database.info" +echo "${hostname}: Database Info .....: ${LOGFILE}" + +function is_service_active() +{ + active=`sm-query service postgres | grep "enabled-active"` + if [ -z "$active" ] ; then + return 0 + else + return 1 + fi +} + +############################################################################### +# All node types +############################################################################### +mkdir -p ${DB_DIR} + +function log_database() +{ + db_list=( $(${PSQL_CMD} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") ) + for db in "${db_list[@]}" + do + echo "postgres database: ${db}" + ${PSQL_CMD} -d ${db} -c " + SELECT + table_schema, + table_name, + pg_size_pretty(table_size) AS table_size, + pg_size_pretty(indexes_size) AS indexes_size, + pg_size_pretty(total_size) AS total_size, + live_tuples, + dead_tuples + FROM ( + SELECT + table_schema, + table_name, + pg_table_size(table_name) AS table_size, + pg_indexes_size(table_name) AS indexes_size, + pg_total_relation_size(table_name) AS total_size, + pg_stat_get_live_tuples(table_name::regclass) AS live_tuples, + pg_stat_get_dead_tuples(table_name::regclass) AS dead_tuples + FROM ( + SELECT + table_schema, + table_name + FROM information_schema.tables + WHERE table_schema='public' + AND table_type='BASE TABLE' + ) AS all_tables + ORDER BY total_size DESC + ) AS pretty_sizes; + " + done >> ${1} +} + + + +DB_EXT=db.sql.txt +function database_dump() +{ + mkdir -p ${DB_DIR} + db_list=( $(${PSQL_CMD} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") ) + for DB in "${db_list[@]}" + do + if [ "$DB" != "keystone" -a "$DB" != "ceilometer" ] ; then + echo "${hostname}: Dumping Database ..: ${DB_DIR}/$DB.$DB_EXT" + (cd ${DB_DIR} ; sudo -u postgres pg_dump $DB > $DB.$DB_EXT) + fi + done +} + +############################################################################### +# Only Controller +############################################################################### + +if [ "$nodetype" = "controller" ] ; then + is_service_active + if [ "$?" = "0" ] ; then + exit 0 + fi + + # postgres DB sizes + delimiter ${LOGFILE} "formatted ${PSQL_CMD} -c" + ${PSQL_CMD} -c " + SELECT + pg_database.datname, + pg_database_size(pg_database.datname), + pg_size_pretty(pg_database_size(pg_database.datname)) + FROM pg_database + ORDER BY pg_database_size DESC; + " >> ${LOGFILE} + + # Number of postgres connections + delimiter ${LOGFILE} "ps -C postgres -o cmd=" + ps -C postgres -o cmd= >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "call to log_database" + log_database ${LOGFILE} + + database_dump +fi + +exit 0 diff --git a/tools/collector/scripts/collect_sm.sh b/tools/collector/scripts/collect_sm.sh new file mode 100644 index 000000000..5f0f3c9ba --- /dev/null +++ b/tools/collector/scripts/collect_sm.sh @@ -0,0 +1,26 @@ +#! /bin/bash +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# Loads Up Utilities and Commands Variables +source /usr/local/sbin/collect_parms +source /usr/local/sbin/collect_utils + +SERVICE="sm" +LOGFILE="${extradir}/sm.info" +echo "${hostname}: Service Management : ${LOGFILE}" + +############################################################################### +# Only Controller +############################################################################### + +if [ "$nodetype" = "controller" ] ; then + kill -SIGUSR1 $(>${COLLECT_ERROR_LOG} >> ${LOGFILE} +fi + +exit 0 diff --git a/tools/collector/scripts/collect_sysinv.sh b/tools/collector/scripts/collect_sysinv.sh new file mode 100755 index 000000000..37ba26af5 --- /dev/null +++ b/tools/collector/scripts/collect_sysinv.sh @@ -0,0 +1,73 @@ +#! /bin/bash +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# Loads Up Utilities and Commands Variables +source /usr/local/sbin/collect_parms +source /usr/local/sbin/collect_utils + +SERVICE="inventory" +LOGFILE="${extradir}/${SERVICE}.info" +RPMLOG="${extradir}/rpm.info" + +function is_service_active() +{ + active=`sm-query service management-ip | grep "enabled-active"` + if [ -z "$active" ] ; then + return 0 + else + return 1 + fi +} + +############################################################################### +# Only Controller +############################################################################### +if [ "$nodetype" = "controller" ] ; then + + echo "${hostname}: Software Config ...: ${RPMLOG}" + # These go into the SERVICE.info file + delimiter ${RPMLOG} "rpm -qa" + rpm -qa >> ${RPMLOG} + + is_service_active + if [ "$?" = "0" ] ; then + exit 0 + fi + + echo "${hostname}: System Inventory ..: ${LOGFILE}" + + # These go into the SERVICE.info file + delimiter ${LOGFILE} "system host-list" + system host-list 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE} + + delimiter ${LOGFILE} "system service-list" + system service-list 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE} + + delimiter ${LOGFILE} "nova service-list" + nova service-list 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE} + + delimiter ${LOGFILE} "neutron host-list" + neutron host-list 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE} + + delimiter ${LOGFILE} "system host-port-list controller-0" + system host-port-list controller-0 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE} + + delimiter ${LOGFILE} "system host-port-list controller-1" + system host-port-list controller-1 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE} + + delimiter ${LOGFILE} "Dump all Instances" + nova list --fields name,status,OS-EXT-SRV-ATTR:host --all-tenant 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE} + + delimiter ${LOGFILE} "vm-topology" + timeout 60 vm-topology --show all 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE} + + cp -a /opt/platform ${extradir} +fi + + +exit 0 diff --git a/tools/collector/scripts/collect_tc.sh b/tools/collector/scripts/collect_tc.sh new file mode 100755 index 000000000..160b970e6 --- /dev/null +++ b/tools/collector/scripts/collect_tc.sh @@ -0,0 +1,82 @@ +#! /bin/bash +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# Loads Up Utilities and Commands Variables +source /usr/local/sbin/collect_parms +source /usr/local/sbin/collect_utils + +SERVICE="tc" +LOGFILE="${extradir}/tc.info" +echo "${hostname}: Traffic Controls . : ${LOGFILE}" + +############################################################################### +# Interface Info +############################################################################### +delimiter ${LOGFILE} "cat /etc/network/interfaces" +if [ -f /etc/network/interfaces ]; then + cat /etc/network/interfaces >> ${LOGFILE} +else + echo "/etc/network/interfaces NOT FOUND" >> ${LOGFILE} +fi + +delimiter ${LOGFILE} "ip link" +ip link >> ${LOGFILE} + +for i in $(ip link | grep mtu | grep eth |awk '{print $2}' | sed 's#:##g'); +do + delimiter ${LOGFILE} "ethtool ${i}" + ethtool ${i} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "cat /sys/class/net/${i}/speed" + cat /sys/class/net/${i}/speed >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "ethtool -S ${i}" + ethtool -S ${i} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} +done + +############################################################################### +# TC Configuration Script (/usr/local/bin/cgcs_tc_setup.sh) +############################################################################### +delimiter ${LOGFILE} "cat /usr/local/bin/cgcs_tc_setup.sh" +if [ -f /usr/local/bin/cgcs_tc_setup.sh ]; then + cat /usr/local/bin/cgcs_tc_setup.sh >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} +else + echo "/usr/local/bin/cgcs_tc_setup.sh NOT FOUND" >> ${LOGFILE} +fi + +############################################################################### +# TC Configuration +############################################################################### +delimiter ${LOGFILE} "tc qdisc show" +tc qdisc show >> ${LOGFILE} + +for i in $(ip link | grep htb | awk '{print $2}' | sed 's#:##g'); +do + delimiter ${LOGFILE} "tc class show dev ${i}" + tc class show dev ${i} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "tc filter show dev ${i}" + tc filter show dev ${i} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} +done + +############################################################################### +# TC Statistics +############################################################################### +delimiter ${LOGFILE} "tc -s qdisc show" +tc -s qdisc show >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + +for i in $(ip link | grep htb | awk '{print $2}' | sed 's#:##g'); +do + delimiter ${LOGFILE} "tc -s class show dev ${i}" + tc -s class show dev ${i} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + + delimiter ${LOGFILE} "tc -s filter show dev ${i}" + tc -s filter show dev ${i} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} +done + +exit 0 diff --git a/tools/collector/scripts/collect_utils b/tools/collector/scripts/collect_utils new file mode 100755 index 000000000..0e91013ff --- /dev/null +++ b/tools/collector/scripts/collect_utils @@ -0,0 +1,223 @@ +#! /bin/bash +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +########################################################################################## + +DEBUG=false + +# Fail Codes +PASS=0 +FAIL=1 +RETRY=2 + +FAIL_NODETYPE=3 + +FAIL_TIMEOUT=10 +FAIL_TIMEOUT1=11 +FAIL_TIMEOUT2=12 +FAIL_TIMEOUT3=13 +FAIL_TIMEOUT4=14 +FAIL_TIMEOUT5=15 +FAIL_TIMEOUT6=16 +FAIL_TIMEOUT7=17 +FAIL_TIMEOUT8=18 +FAIL_TIMEOUT9=19 + +FAIL_PASSWORD=30 +FAIL_PERMISSION=31 +FAIL_CLEANUP=32 +FAIL_UNREACHABLE=33 +FAIL_HOSTNAME=34 +FAIL_INACTIVE=35 +FAIL_PERMISSION_SKIP=36 +FAIL_OUT_OF_SPACE=37 +FAIL_INSUFFICIENT_SPACE=38 +FAIL_OUT_OF_SPACE_LOCAL=39 +FAIL_CREATE=39 + +# Warnings are above 200 +WARN_WARNING=200 +WARN_HOSTNAME=201 + +# Failure Strings +FAIL_OUT_OF_SPACE_STR="No space left on device" +FAIL_INSUFFICIENT_SPACE_STR="Not enough space on device" + +# The minimum amount of % free space on /scratch to allow collect to proceed +MIN_PERCENT_SPACE_REQUIRED=75 + +# Log file path/names +COLLECT_LOG=/var/log/collect.log +COLLECT_ERROR_LOG=/tmp/collect_error.log + +# Load up the nova openrc file if its possible +KEYRING_PATH="/opt/platform/.keyring" +if [ -e ${KEYRING_PATH} ] ; then + CRED=`find /opt/platform/.keyring -name .CREDENTIAL` + if [ ! -z "${CRED}" ] ; then + NOVAOPENRC="/etc/nova/openrc" + if [ -e ${NOVAOPENRC} ] ; then + ACTIVE=true + source ${NOVAOPENRC} 2>/dev/null 1>/dev/null + fi + fi +fi + +# get the node and subfunction types +nodetype="" +subfunction="" +PLATFORM_CONF=/etc/platform/platform.conf +if [ -e ${PLATFORM_CONF} ] ; then + source ${PLATFORM_CONF} +fi + +if [ "${nodetype}" != "controller" -a "${nodetype}" != "compute" -a "${nodetype}" != "storage" ] ; then + logger -t ${COLLECT_TAG} "could not identify nodetype ($nodetype)" + exit $FAIL_NODETYPE +fi + +NODETYPE=$nodetype +SUBFUNCTION=$subfunction + +# Setup an expect command completion file. +# This is used to force serialization of expect +# sequences and highlight command completion +collect_done="collect done" +cmd_done_sig="expect done" +cmd_done_file="/usr/local/sbin/expect_done" + +# Compression Commands +TAR_ZIP_CMD="tar -cvzf" +TAR_UZIP_CMD="tar -xvzf" +TAR_CMD="tar -cvhf" +UNTAR_CMD="tar -xvf" +ZIP_CMD="gzip" +NICE_CMD="/usr/bin/nice -n19" +IONICE_CMD="/usr/bin/ionice -c2 -n7" +COLLECT_TAG="COLLECT" + +STARTDATE_OPTION="--start-date" +ENDDATE_OPTION="--end-date" + + +PROCESS_DETAIL_CMD="ps -e -H -o ruser,tid,pid,ppid,flags,stat,policy,rtprio,nice,priority,rss:10,vsz:10,sz:10,psr,stime,tty,cputime,wchan:14,cmd" +BUILD_INFO_CMD="cat /etc/build.info" + +################################################################################ +# Log Debug, Info or Error log message to syslog +################################################################################ +function log +{ + logger -t ${COLLECT_TAG} $@ +} + +function ilog +{ + echo "$@" + logger -t ${COLLECT_TAG} $@ + #logger -p local3.info -t ${COLLECT_TAG} $@ +} + +function elog +{ + echo "Error: $@" + logger -t ${COLLECT_TAG} $@ +} + +function wlog +{ + echo "Warning: $@" + logger -t ${COLLECT_TAG} $@ +} + +function set_debug_mode() +{ + DEBUG=${1} +} + +function dlog() +{ + if [ "$DEBUG" == true ] ; then + logger -t ${COLLECT_TAG} $@ + echo "Debug: $@" + fi +} + + +function delimiter() +{ + echo "--------------------------------------------------------------------" >> ${1} 2>>${COLLECT_ERROR_LOG} + echo "`date` : ${myhostname} : ${2}" >> ${1} 2>>${COLLECT_ERROR_LOG} + echo "--------------------------------------------------------------------" >> ${1} 2>>${COLLECT_ERROR_LOG} +} + +function log_slabinfo() +{ + PAGE_SIZE=$(getconf PAGE_SIZE) + cat /proc/slabinfo | awk -v page_size_B=${PAGE_SIZE} ' + BEGIN {page_KiB = page_size_B/1024; TOT_KiB = 0;} + (NF == 17) { + gsub(/[<>]/, ""); + printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n", + $2, $3, $4, $5, $6, $7, $8, $10, $11, $12, $13, $15, $16, $17, "KiB"); + } + (NF == 16) { + num_objs=$3; obj_per_slab=$5; pages_per_slab=$6; + KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0; + TOT_KiB += KiB; + printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n", + $1, $2, $3, $4, $5, $6, $7, $9, $10, $11, $12, $14, $15, $16, KiB); + } + END { + printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n", + "TOTAL", "-", "-", "-", "-", "-", ":", "-", "-", "-", ":", "-", "-", "-", TOT_KiB); + } + ' >> ${1} 2>>${COLLECT_ERROR_LOG} +} +########################################################################### +# +# Name : collect_errors +# +# Description: search COLLECT_ERROR_LOG for "No space left on device" logs +# Return 0 if no such logs are found. +# Return 1 if such logs are found +# +# Assumptions: Caller should assume a non-zero return as an indication of +# a corrupt or incomplete collect log +# +# Create logs and screen echos that record the error for the user. +# +# May look for other errors in the future +# +########################################################################### + +function collect_errors() +{ + local host=${1} + local RC=0 + + # Look for "No space left on device" error + grep -q "${FAIL_OUT_OF_SPACE_STR}" ${COLLECT_ERROR_LOG} + + if [ "$?" == "0" ] ; then + + string="failed to collect from ${host} (reason:${FAIL_OUT_OF_SPACE}:${FAIL_OUT_OF_SPACE_STR})" + + # /var/log/user.log it + logger -t ${COLLECT_TAG} "${string}" + + # logs that show up in the foreground + echo "${string}" + echo "Increase available space in ${host}:${COLLECT_BASE_DIR} and retry operation." + + # return error code + RC=1 + + fi + + return ${RC} +} diff --git a/tools/collector/scripts/collect_vswitch.sh b/tools/collector/scripts/collect_vswitch.sh new file mode 100644 index 000000000..e9bd2b884 --- /dev/null +++ b/tools/collector/scripts/collect_vswitch.sh @@ -0,0 +1,108 @@ +#! /bin/bash +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# Loads Up Utilities and Commands Variables + +source /usr/local/sbin/collect_parms +source /usr/local/sbin/collect_utils + +SERVICE="vswitch" +LOGFILE="${extradir}/${SERVICE}.info" + +LIST_COMMANDS=( + # keep items sorted alphabetically + "address-list" + "dvr-mac-list" + "dvr-subnet-list" + "engine-list" + "engine-queue-list" + "filter-bindings-list" + "filter-rule-list" + "flow-group-list" + "flow-rule-list" + "igmp-group-list" + "igmp-interface-list" + "interface-list" + "lacp-interface-list" + "lacp-neighbour-list" + "lldp-agent-list" + "lldp-neighbour-list" + "mld-group-list" + "mld-interface-list" + "nat-list" + "neighbour-list" + "network-list" + "network-table-list" + "openflow-controller-list" + "openflow-errors-list" + "ovsdb-manager-list" + "ovsdb-monitor-list" + "port-list" + "route-list" + "router-list" + "router-list" + "snat-list" + "stream-list" + "vxlan-endpoint-list" + "vxlan-ip-endpoint-list" + "vxlan-peer-list") + +STATS_COMMANDS=( + # keep below items sorted alphabetically + "arp-stats-list" + "dvr-stats-list" + "engine-stats-list" + "filter-stats-list" + "flow-cache-stats-list" + "flow-event-stats-list" + "flow-switch-stats-list" + "flow-table-stats-list" + "icmp-stats-list" + "igmp-stats-list" + "interface-stats-list" + "ip-stats-list" + "ip-stats-list-errors" + "lacp-stats-list" + "lldp-stats-list" + "memory-stats-list" + "mld-stats-list" + "nat-stats-list" + "ndp-stats-list" + "network-stats-list" + "openflow-stats-list" + "port-queue-stats-list" + "port-rate-list" + "port-stats-list" + "snat-stats-list" + "udp-stats-list" + "vxlan-endpoint-stats-list") + +############################################################################### +# Only Compute Nodes +############################################################################### +if [[ "$nodetype" == "compute" || "$subfunction" == *"compute"* ]] ; then + + echo "${hostname}: AVS Info ..........: ${LOGFILE}" + + for COMMAND in ${LIST_COMMANDS[@]}; do + delimiter ${LOGFILE} "vshell ${COMMAND}" + vshell ${COMMAND} --expanded >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + done + + for COMMAND in ${STATS_COMMANDS[@]}; do + delimiter ${LOGFILE} "vshell ${COMMAND}" + vshell ${COMMAND} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + done + + if [[ "$sdn_enabled" == "yes" ]] ; then + delimiter ${LOGFILE} "ovsdb-client dump" + ovsdb-client dump >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} + fi +fi + +exit 0 diff --git a/tools/collector/scripts/etc.exclude b/tools/collector/scripts/etc.exclude new file mode 100644 index 000000000..d3f3827d8 --- /dev/null +++ b/tools/collector/scripts/etc.exclude @@ -0,0 +1,40 @@ +/etc/cinder/volumes +/etc/nova/instances +/etc/postgresql +/etc/alternatives +/etc/terminfo +/etc/tempest +/etc/security +/etc/yum +/etc/collect +/etc/collect.d +/etc/logrotate.d +/etc/logrotate* +/etc/keystone +/etc/pam.d +/etc/environment +/etc/sudoers.d +/etc/sudoers +/etc/passwd +/etc/passwd- +/etc/shadow +/etc/shadow- +/etc/gshadow +/etc/gshadow- +/etc/group +/etc/group- +/etc/ssh +/etc/X11 +/etc/bluetooth +/etc/chatscripts +/etc/cron* +/etc/rc5.d +/etc/rc4.d +/etc/rc1.d +/etc/rc2.d +/etc/bash_completion.d +/etc/pm +/etc/systemd/system/*.mount +/etc/systemd/system/ctrl-alt-del.target +/etc/ssl +/etc/mtc/tmp diff --git a/tools/collector/scripts/expect_done b/tools/collector/scripts/expect_done new file mode 100755 index 000000000..a846adb7e --- /dev/null +++ b/tools/collector/scripts/expect_done @@ -0,0 +1 @@ +expect done diff --git a/tools/collector/scripts/run.exclude b/tools/collector/scripts/run.exclude new file mode 100644 index 000000000..b1c1794cb --- /dev/null +++ b/tools/collector/scripts/run.exclude @@ -0,0 +1,12 @@ +/var/run/sanlock/sanlock.sock +/var/run/tgtd.ipc_abstract_namespace.0 +/var/run/wdmd/wdmd.sock +/var/run/acpid.socket +/var/run/rpcbind.sock +/var/run/libvirt/libvirt-sock-ro +/var/run/libvirt/libvirt-sock +/var/run/dbus/system_bus_socket +/var/run/named-chroot +/var/run/avahi-daemon +/var/run/neutron/metadata_proxy +/var/run/.vswitch