zuul-jobs/roles/build-releasenotes/tasks/main.yaml
melanie witt b892aad548 Run sphinx-build in parallel for releasenotes
Since the recent releases and upper-constraints bumps for
openstackdocstheme 2.1.0 and reno 3.1.0, both sphinx extensions now
support safe parallel reading.

This adds the '-j auto' option to the sphinx-build command for building
release notes to parallelize the build over multiple CPUs.

This effort was inspired by the nova build-openstack-releasenotes job
which typically takes 52m to 1h to run. The job timeout is 1h, so we
are occasionally seeing TIMED_OUT job failures.

With parallel sphinx-build, the job run time is reduced down to 15m.

Change-Id: If3c615119b1a72ce294e22685ddc2dcd6b63f4a0
2020-05-15 00:35:29 +00:00

132 lines
4.6 KiB
YAML

# TODO(mordred) Put the translations logic into a sphinx plugin?
- name: Check if translations exist for release notes
stat:
path: "{{ zuul_work_dir }}/releasenotes/source/locale"
get_checksum: false
get_mime: false
get_md5: false
register: translations
- name: Prepare release note translations
shell:
chdir: '{{ zuul_work_dir }}'
executable: /bin/bash
cmd: |
set -e
set -x
DOCNAME=releasenotes
DIRECTORY=releasenotes
source {{ zuul_work_virtualenv }}/bin/activate
# Mapping of language codes to language names
declare -A LANG_NAME=(
["cs"]="Czech"
["de"]="German"
["en_AU"]="English (Australian)"
["en_GB"]="English (United Kingdom)"
["es"]="Spanish"
["fr"]="French"
["id"]="Indonesian"
["it"]="Italian"
["ja"]="Japanese"
["ko_KR"]="Korean (South Korea)"
["ne"]="Nepali"
["pt_BR"]="Portuguese (Brazil)"
["ru"]="Russian"
["tr_TR"]="Turkish (Turkey)"
["zh_CN"]="Chinese (China)"
)
# Check that locale_dirs is really set, otherwise translations
# will not work.
if ! grep -q -E '^locale_dirs *=' $DIRECTORY/source/conf.py; then
echo "Translations exist and locale_dirs missing in source/conf.py"
exit 1
fi
REFERENCES=`mktemp`
trap "rm -f -- '$REFERENCES'" EXIT
# Extract translations
sphinx-build -j auto -b gettext \
-d ${DIRECTORY}/build/doctrees.gettext \
${DIRECTORY}/source/ \
${DIRECTORY}/source/locale/
# Add links for translations to index file
cat <<EOF >> ${REFERENCES}
Translated Release Notes
========================
EOF
# Check all language translation resources
for locale in `find ${DIRECTORY}/source/locale/ -maxdepth 1 -type d` ; do
# Skip if it is not a valid language translation resource.
if [ ! -e ${locale}/LC_MESSAGES/${DOCNAME}.po ]; then
continue
fi
language=$(basename $locale)
echo "Building $language translation"
# Prepare all translation resources
for pot in ${DIRECTORY}/source/locale/*.pot ; do
# Get filename
resname=$(basename ${pot} .pot)
# Merge all translation resources. Note this is done the same
# way as done in common_translation_update.sh where we merge
# all strings together in a single file.
msgmerge --silent -o \
${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po \
${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po \
${pot}
# Compile all translation resources
msgfmt -o \
${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.mo \
${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po
done
# Build translated document
sphinx-build -j auto -b html -D language=${language} \
-d "${DIRECTORY}/build/doctrees.${language}" \
${DIRECTORY}/source/ ${DIRECTORY}/build/html/${language}
# Reference translated document from index file
if [ ${LANG_NAME["${language}"]+_} ] ; then
name=${LANG_NAME["${language}"]}
name+=" (${language})"
echo "* \`$name <${language}/index.html>\`__" >> ${REFERENCES}
else
echo "* \`${language} <${language}/index.html>\`__" >> ${REFERENCES}
fi
# Remove newly created files
git clean -f -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.po
git clean -f -x -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.mo
# revert changes to po file
git reset -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
git checkout -- ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
done
# Now append our references to the index file. We cannot do this
# earlier since the sphinx commands will read this file.
cat ${REFERENCES} >> ${DIRECTORY}/source/index.rst
# Remove newly created pot files
rm -f ${DIRECTORY}/source/locale/*.pot
when: translations.stat.exists
- name: Run releasenotes sphinx build
shell:
executable: /bin/bash
chdir: '{{ zuul_work_dir }}'
cmd: |
{{ zuul_work_virtualenv }}/bin/sphinx-build -a -E -W -j auto \
-d releasenotes/build/doctrees \
-b html releasenotes/source releasenotes/build/html