From f5bba0834b27f387e54443f26d6efc18732eb4c4 Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Fri, 6 Sep 2013 21:19:49 +0200 Subject: [PATCH] Handle removed files in tools/validate.py Previously if a file was removed, no checks were done that the file was not referenced anywhere. Now you get e.g.: Checking for removed files Removed file: doc/src/docbkx/common/section_keystone_cli_users_tenants_roles.xml Removed file: doc/src/docbkx/openstack-user-admin/src/section_cli_set_block_storage_quotas.xml Removed file: doc/src/docbkx/openstack-user-admin/src/section_cli_set_compute_quotas.xml Removed file: doc/src/docbkx/openstack-user-admin/src/section_keystone_cli_manage_projects_users.xml Removed file: doc/src/docbkx/openstack-user/src/section_cli_help.xml Removed file: doc/src/docbkx/openstack-user/src/section_cli_install.xml Removed file: doc/src/docbkx/openstack-user/src/section_cli_openrc.xml Removed file: doc/src/docbkx/openstack-user/src/section_cli_overview.xml Removed file: doc/src/docbkx/openstack-user/src/section_cli_version.xml File section_compute-configure-quotas.xml has an xi:include on deleted file ../openstack-user-admin/src/section_cli_set_compute_quotas.xml And validate.py exists with error code to mark this as failure. Also, limit number of parallel build jobs to 4 to reduce Jenkins load, hope we see less org.apache.fop.fonts.FontTriplet. Related-Bug: # 1221721 Change-Id: I3e47217958840f61347f71b159414928f266d6d5 --- tools/validate.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/tools/validate.py b/tools/validate.py index 005c28ec3a..969bea0561 100755 --- a/tools/validate.py +++ b/tools/validate.py @@ -146,6 +146,52 @@ def get_modified_files(): return modified_files +def check_deleted_files(rootdir, file_exceptions): + """ Check whether files got deleted and verify that no other file references them. + + """ + print("\nChecking for removed files") + modified_files = get_modified_files() + deleted_files = [] + any_removed = False + for f in modified_files: + full = os.path.abspath(f) + if not os.path.exists(full): + print(" Removed file: %s" % f) + deleted_files.append(full) + any_removed = True + + if any_removed: + # Figure out whether this file was included anywhere + missing_reference = False + + for root, dirs, files in os.walk(rootdir): + # Don't descend into 'target' subdirectories + try: + ind = dirs.index('target') + del dirs[ind] + except ValueError: + pass + + os.chdir(root) + + for f in files: + if (f.endswith('.xml') and + f != 'pom.xml' and + f not in file_exceptions): + path = os.path.abspath(os.path.join(root, f)) + doc = etree.parse(path) + ns = {"xi": "http://www.w3.org/2001/XInclude"} + for node in doc.xpath('//xi:include', namespaces=ns): + href = node.get('href') + if (href.endswith('.xml') and + os.path.abspath(href) in deleted_files): + print(" File %s has an xi:include on deleted file %s " % (f, href)) + missing_reference = True + if missing_reference: + sys.exit(1) + + def validate_individual_files(rootdir, exceptions, force): schema = get_schema() @@ -268,7 +314,13 @@ def build_affected_books(rootdir, book_exceptions, file_exceptions, force): else: print("No books are affected by modified files. Building all books.") - pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()) + maxjobs = multiprocessing.cpu_count() + # Jenkins fails sometimes with errors if too many jobs run, artificially + # limit to 4 for now. + # See https://bugs.launchpad.net/openstack-manuals/+bug/1221721 + if maxjobs > 4: + maxjobs = 4 + pool = multiprocessing.Pool(maxjobs) print("Queuing the following books for building:") for book in books: print(" %s" % os.path.basename(book)) @@ -294,6 +346,7 @@ def main(rootdir, force): if force: print("Validation of all files and build of all books will be forced.") + check_deleted_files(rootdir, FILE_EXCEPTIONS) validate_individual_files(rootdir, FILE_EXCEPTIONS, force) build_affected_books(rootdir, BOOK_EXCEPTIONS, FILE_EXCEPTIONS, force)