From b57da6e88845863ced1d264ca92f54b6f44143b3 Mon Sep 17 00:00:00 2001 From: Sean McGinnis Date: Fri, 8 May 2020 09:33:57 -0500 Subject: [PATCH] Add tool to generate a health report for all reqs This adds a tool script that loops through all of our requirements and output basic information for each one along with any things we might be concerned about with each one. For now it just checks a few things, but we could extend this issue checking as we find other things to check on. Also refactors the unused package tool to put some common things into a shared function library used by both scripts. Change-Id: I140c3a5fb71a8fb82ccb696bade5be13bb9b0ba0 Signed-off-by: Sean McGinnis --- tools/functions | 45 +++++++++++++++++++++ tools/get-health-report.sh | 76 +++++++++++++++++++++++++++++++++++ tools/list-unused-packages.sh | 30 ++------------ 3 files changed, 125 insertions(+), 26 deletions(-) create mode 100755 tools/functions create mode 100755 tools/get-health-report.sh diff --git a/tools/functions b/tools/functions new file mode 100755 index 0000000000..f4e5b329f4 --- /dev/null +++ b/tools/functions @@ -0,0 +1,45 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Shared functions for shell scripts + +function enable_venv () { + BASE="${1}" + if [[ -z "${VIRTUAL_ENV}" ]]; then + if [[ ! -d ${BASE}/.tox/venv ]]; then + (cd ${BASE} && tox -e venv --notest > /dev/null) + fi + source ${BASE}/.tox/venv/bin/activate + fi +} + +# Search for requirements used in openstack/ repos +function search_reqs () { + beagle search --ignore-case --file '(.*requirement.*|setup.cfg)' "${1}" | \ + grep "openstack/" | \ + # Sometimes we get false positives from a package name being a + # substring within another package. This filter isn't working right + # though. This just means we might miss a package that isn't being + # used. + # grep "${1}[ |\!|>]" | \ + grep -v "openstack.requirements" +} + +# Get a list of all package names by filtering out comments, blank lines, and +# any package modifiers like version constraints. +function get_tracked_requirements () { + reqs=$(sed 's/[!|>|<|=|;].*//g' global-requirements.txt | + sed 's/ .*//g' | + sed '/^#/d' | + sed '/^$/d' | + sort | uniq) +} diff --git a/tools/get-health-report.sh b/tools/get-health-report.sh new file mode 100755 index 0000000000..66ccf21ea9 --- /dev/null +++ b/tools/get-health-report.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Checks all of our tracked packages for any issues + +TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BASEDIR=$(dirname ${TOOLSDIR}) + +source ${TOOLSDIR}/functions + +# Make sure we are using our venv +enable_venv "${BASEDIR}" + +update= +if [[ "$#" -eq 1 ]]; then + update="${1}" +fi + +# Save off our current timestamp for use later +current=$(date +%s) + +# Loop through each package to get details and check for issues +get_tracked_requirements +for req in $reqs; do + count=$(search_reqs ${req} | + grep -v " openstack/${req} " | + wc -l) + + metadata=$(curl -s -L "https://pypi.org/pypi/$req/json") + summary=$(echo "${metadata}" | jq -r '.info.summary') + last_release=$(echo "${metadata}" | jq -r '.info.version') + release_date=$(echo "${metadata}" | jq -r ".releases.\"${last_release}\" | .[0].upload_time") + + # Print basic package information + echo "${req}" + if [[ "${summary}" != "" ]]; then + echo " Summary: ${summary}" + fi + echo " Used by repos: ${count}" + echo " Last release: ${last_release}" + echo " Release date: ${release_date}" + + # Check for various things to warn about + package_name=$(echo "${metadata}" | jq -r '.info.name') + if [[ "${req}" != "${package_name}" ]]; then + echo " WARNING: In g-r as ${req} but actual name is ${package_name}" + fi + + py3=$(echo "${metadata}" | \ + jq -r '.info.classifiers | .[]' | \ + grep "Programming Language :: Python :: 3") + if [[ -z ${py3} ]]; then + echo " WARNING: No python 3 classifier in metadata" + fi + + release=$(date -d $release_date +%s) + seconds_since_release=$((current-release)) + years_since_release=$((seconds_since_release/60/60/24/365)) + message=$(echo "It's been ${years_since_release} years since last release") + if [[ ${years_since_release} -gt 4 ]]; then + echo " !!WARNING!! ${message}" + elif [[ ${years_since_release} -gt 2 ]]; then + echo " WARNING ${message}" + fi +done diff --git a/tools/list-unused-packages.sh b/tools/list-unused-packages.sh index 51681b0c73..1150525a3d 100755 --- a/tools/list-unused-packages.sh +++ b/tools/list-unused-packages.sh @@ -17,41 +17,19 @@ TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" BASEDIR=$(dirname ${TOOLSDIR}) +source ${TOOLSDIR}/functions + # Make sure we are using our venv -if [[ -z "${VIRTUAL_ENV}" ]]; then - if [[ ! -d ${BASEDIR}/.tox/venv ]]; then - (cd ${BASEDIR} && tox -e venv --notest > /dev/null) - fi - source ${BASEDIR}/.tox/venv/bin/activate -fi +enable_venv "${BASEDIR}" update= if [[ "$#" -eq 1 ]]; then update="${1}" fi -search_reqs () -{ - beagle search --ignore-case --file '(.*requirement.*|setup.cfg)' "${1}" | \ - grep "openstack/" | \ - # Sometimes we get false positives from a package name being a - # substring within another package. This filter isn't working right - # though. This just means we might miss a package that isn't being - # used. - # grep "${1}[ |\!|>]" | \ - grep -v "openstack.requirements" -} - -# Get a list of all package names by filtering out comments, blank lines, and -# any package modifiers like version constraints. -reqs=$(sed 's/[!|>|<|=|;].*//g' global-requirements.txt | - sed 's/ .*//g' | - sed '/^#/d' | - sed '/^$/d' | - sort | uniq) - # Loop through each package and check for its presence in any repo's # requirements files other than mentions in its own repo +get_tracked_requirements for req in $reqs; do count=$(search_reqs ${req} | grep -v " openstack/${req} " |