0091670583
1. Locked the WTForms to 2.2.1 to address the import issue with wtforms.widgets.HTMLString. WTForms 2.3.0 was released on April 21/2020. This release causes shipyard gate fail with import error for wtforms.widgets.HTMLString. 2. Deleted psycopg2==2.7.7, which is installed as a dependency of apache-airflow extra package postgres, and resoted the newer release psycopg2-binary==2.8.4, to be used instead. Change-Id: I303a2c94ec409e97af1192ae892b8148fcdbb8d5
146 lines
5.0 KiB
Docker
146 lines
5.0 KiB
Docker
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Docker image to run Airflow on Kubernetes
|
|
ARG FROM=opensuse/leap:15.1
|
|
FROM ${FROM}
|
|
|
|
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
|
|
org.opencontainers.image.url='https://airshipit.org' \
|
|
org.opencontainers.image.documentation='https://airship-shipyard.readthedocs.org' \
|
|
org.opencontainers.image.source='https://opendev.org/airship/shipyard' \
|
|
org.opencontainers.image.vendor='The Airship Authors' \
|
|
org.opencontainers.image.licenses='Apache-2.0'
|
|
|
|
# Do not prompt user for choices on installation/configuration of packages
|
|
# Set port 8080 for Airflow Web
|
|
# Set port 5555 for Airflow Flower
|
|
# Set port 8793 for Airflow Worker
|
|
ENV container docker
|
|
ENV WEB_PORT 8080
|
|
ENV FLOWER_PORT 5555
|
|
ENV WORKER_PORT 8793
|
|
ENV SLUGIFY_USES_TEXT_UNIDECODE yes
|
|
|
|
# Expose port for applications
|
|
EXPOSE $WEB_PORT
|
|
EXPOSE $FLOWER_PORT
|
|
EXPOSE $WORKER_PORT
|
|
|
|
# Set ARG for usage during build
|
|
ARG AIRFLOW_HOME=/usr/local/airflow
|
|
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.5"
|
|
ARG ctx_base=src/bin
|
|
|
|
# Kubectl version
|
|
ARG KUBECTL_VERSION=1.16.2
|
|
|
|
# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
|
|
ENV AIRFLOW_HOME=${AIRFLOW_HOME}
|
|
|
|
RUN set -ex && \
|
|
zypper -q update -y ;\
|
|
zypper --non-interactive install --no-recommends \
|
|
ca-certificates \
|
|
curl \
|
|
git-core \
|
|
gcc-c++ \
|
|
libffi-devel \
|
|
libopenssl-devel \
|
|
libpqxx-devel \
|
|
gcc-locale \
|
|
netcat-openbsd \
|
|
netcfg \
|
|
which \
|
|
python3 \
|
|
python3-setuptools \
|
|
python3-pip \
|
|
python3-devel \
|
|
python3-python-dateutil \
|
|
make
|
|
|
|
RUN pip3 install -U pip \
|
|
&& zypper clean -a \
|
|
&& rm -rf \
|
|
/tmp/* \
|
|
/var/tmp/* \
|
|
/var/log/* \
|
|
/usr/share/man \
|
|
/usr/share/doc \
|
|
/usr/share/doc-base
|
|
|
|
# Explicitly need to create usergroup with same name in suse
|
|
RUN useradd -U -ms /bin/bash -d ${AIRFLOW_HOME} airflow
|
|
|
|
# Things that change mostly infrequently
|
|
RUN curl -L -o /usr/local/bin/kubectl \
|
|
https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl \
|
|
&& chmod +x /usr/local/bin/kubectl
|
|
|
|
# Dependency requirements
|
|
# Note - removing snakebite (python 2 vs. 3). See:
|
|
# https://github.com/puckel/docker-airflow/issues/77
|
|
COPY images/airflow/requirements.txt /tmp/
|
|
RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \
|
|
&& pip3 uninstall -y snakebite || true \
|
|
&& pip3 uninstall -y psycopg2 || true \
|
|
&& pip3 install --no-cache-dir --force-reinstall $(pip freeze | grep psycopg2-binary) || true
|
|
|
|
# Copy scripts used in the container:
|
|
COPY images/airflow/script/*.sh ${AIRFLOW_HOME}/
|
|
|
|
# Copy configuration (e.g. logging config for Airflow):
|
|
COPY images/airflow/config/*.py ${AIRFLOW_HOME}/config/
|
|
|
|
# Change permissions
|
|
RUN chown -R airflow:airflow ${AIRFLOW_HOME}
|
|
|
|
# Setting the version explicitly for PBR
|
|
ENV PBR_VERSION 0.1a1
|
|
|
|
# Shipyard
|
|
#
|
|
# Shipyard provides core functionality used by the Airflow plugins/operators
|
|
# Since Shipyard and Airflow are built together as images, this should prevent
|
|
# stale or out-of-date code between these parts.
|
|
# Shipyard requirements, source and installation
|
|
COPY ${ctx_base}/shipyard_airflow/requirements.txt /tmp/api_requirements.txt
|
|
RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir \
|
|
&& pip3 install $AIRFLOW_SRC --no-cache-dir
|
|
|
|
COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
|
|
RUN cd /tmp/shipyard \
|
|
&& python3 setup.py install
|
|
|
|
# Note: The value for the dags and plugins directories that are sourced
|
|
# from the values.yaml of the Shipyard Helm chart need to align with these
|
|
# directories. If they do not, airflow will not find the intended dags and
|
|
# plugins.
|
|
#
|
|
# Note: In the case of building images using the provided Makefile, a test is
|
|
# run against the built-in dags provided with Airflow. Since there is no Helm
|
|
# chart to reconfigure the airflow.cfg with these directories, these dags and
|
|
# plugins are not known to Airflow during the image test.
|
|
#
|
|
# Copy the plugins and dags that will be used by this Airflow image:
|
|
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins ${AIRFLOW_HOME}/plugins/
|
|
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags ${AIRFLOW_HOME}/dags/
|
|
|
|
# Set work directory
|
|
USER airflow
|
|
WORKDIR ${AIRFLOW_HOME}
|
|
|
|
# Execute entrypoint
|
|
ENTRYPOINT ["./entrypoint.sh"]
|