diff --git a/.gitignore b/.gitignore index 930384cf..5901f123 100644 --- a/.gitignore +++ b/.gitignore @@ -133,3 +133,4 @@ latest src/bin/shipyard_airflow/shipyard_airflow/config src/bin/shipyard_airflow/shipyard_airflow/webserver_config.py airflow-runtime +shipyard.yaml diff --git a/charts/shipyard/Chart.yaml b/charts/shipyard/Chart.yaml index b270725f..22418a08 100644 --- a/charts/shipyard/Chart.yaml +++ b/charts/shipyard/Chart.yaml @@ -16,7 +16,8 @@ apiVersion: v1 description: A Helm chart for Shipyard and Airflow name: shipyard -version: 0.2.0 +version: 0.2.1 +appVersion: 2.6.2 keywords: - shipyard - airflow diff --git a/charts/shipyard/templates/bin/_apache.sh.tpl b/charts/shipyard/templates/bin/_apache.sh.tpl new file mode 100644 index 00000000..3d59868e --- /dev/null +++ b/charts/shipyard/templates/bin/_apache.sh.tpl @@ -0,0 +1,44 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ev + +COMMAND="${@:-start}" + +function start () { + + if [ -f /etc/apache2/envvars ]; then + # Loading Apache2 ENV variables + source /etc/httpd/apache2/envvars + fi + # Apache gets grumpy about PID files pre-existing + rm -f /etc/httpd/logs/httpd.pid + + if [ -f /usr/local/apache2/conf/.htpasswd ]; then + htpasswd -b /usr/local/apache2/conf/.htpasswd "$AIRFLOW_USERNAME" "$AIRFLOW_PASSWORD" + else + htpasswd -cb /usr/local/apache2/conf/.htpasswd "$AIRFLOW_USERNAME" "$AIRFLOW_PASSWORD" + fi + + #Launch Apache on Foreground + exec httpd -DFOREGROUND +} + +function stop () { + apachectl -k graceful-stop +} + +$COMMAND diff --git a/charts/shipyard/templates/configmap-airflow-bin.yaml b/charts/shipyard/templates/configmap-airflow-bin.yaml index 0a7ab784..8d8b5a09 100644 --- a/charts/shipyard/templates/configmap-airflow-bin.yaml +++ b/charts/shipyard/templates/configmap-airflow-bin.yaml @@ -21,6 +21,8 @@ kind: ConfigMap metadata: name: airflow-bin data: + apache.sh: | +{{ tuple "bin/_apache.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} rabbit-init.sh: |+ {{- include "helm-toolkit.scripts.rabbit_init" . | indent 4 }} airflow-db-init.sh: |+ diff --git a/charts/shipyard/templates/configmap-airflow-etc.yaml b/charts/shipyard/templates/configmap-airflow-etc.yaml index 33796ef2..2469355b 100644 --- a/charts/shipyard/templates/configmap-airflow-etc.yaml +++ b/charts/shipyard/templates/configmap-airflow-etc.yaml @@ -46,3 +46,16 @@ metadata: data: airflow.cfg: |+ {{ include "helm-toolkit.utils.to_oslo_conf" .Values.conf.airflow | indent 4 }} + webserver_config.py: |+ + """Default configuration for the Airflow webserver.""" + from __future__ import annotations + import os + from airflow.www.fab_security.manager import AUTH_DB + basedir = os.path.abspath(os.path.dirname(__file__)) + WTF_CSRF_ENABLED = True + WTF_CSRF_TIME_LIMIT = None + AUTH_TYPE = AUTH_DB + AUTH_ROLE_ADMIN = 'Viewer' + AUTH_ROLE_PUBLIC = 'Viewer' + + diff --git a/charts/shipyard/templates/deployment-airflow-webserver-ui.yaml b/charts/shipyard/templates/deployment-airflow-webserver-ui.yaml new file mode 100644 index 00000000..5037bd8f --- /dev/null +++ b/charts/shipyard/templates/deployment-airflow-webserver-ui.yaml @@ -0,0 +1,153 @@ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.manifests.deployment_airflow_webserver_ui }} +{{- $envAll := . }} +{{- $labels := tuple $envAll "airflow_webserver_ui" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" -}} +{{- $serviceAccountName := "airflow-webserver-ui" }} +{{- $airflowProxyPort := tuple "airflow_webserver_ui" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- $airflowUserSecret := .Values.secrets.airflow_webserver_ui.admin }} +{{ tuple $envAll "airflow_server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +{{- $mounts_airflow_webserver_ui := .Values.pod.mounts.airflow_webserver_ui.airflow_webserver_ui }} +{{- $mounts_airflow_webserver_ui_init := .Values.pod.mounts.airflow_webserver_ui.init_container }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: airflow-webserver-ui + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + replicas: {{ .Values.pod.replicas.airflow_webserver_ui.airflow_webserver_ui }} + selector: + matchLabels: +{{ $labels | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ $labels | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-airflow-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-airflow-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "airflow-webserver-ui" "containerNames" (list "init" "airflow-webserver-ui") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "airflow_webserver_ui" "airflow_webserver_ui" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} +{{ dict "envAll" $envAll "application" "airflow_webserver_ui" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + nodeSelector: + {{ .Values.labels.airflow.node_selector_key }}: {{ .Values.labels.airflow.node_selector_value }} + restartPolicy: Always + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }} + initContainers: +{{ tuple $envAll "airflow_server" $mounts_airflow_webserver_ui_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: apache-proxy +{{ tuple $envAll "apache_proxy" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.apache_proxy | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "dashboard" "container" "apache_proxy" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/apache.sh + - start + ports: + - name: http + containerPort: {{ $airflowProxyPort }} + readinessProbe: + tcpSocket: + port: {{ $airflowProxyPort }} + initialDelaySeconds: 20 + periodSeconds: 30 + env: + - name: AIRFLOW_USERNAME + valueFrom: + secretKeyRef: + name: {{ $airflowUserSecret }} + key: AIRFLOW_USERNAME + - name: AIRFLOW_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $airflowUserSecret }} + key: AIRFLOW_PASSWORD + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: airflow-bin + mountPath: /tmp/apache.sh + subPath: apache.sh + readOnly: true + - name: apache-etc + mountPath: /usr/local/apache2/conf/httpd.conf + subPath: httpd.conf + readOnly: true + - name: airflow-webserver-ui + image: {{ .Values.images.tags.airflow }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.airflow_webserver_ui.airflow_webserver_ui | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "airflow-webserver-ui" "container" "airflow_webserver_ui" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: AIRFLOW_CONN_AIRFLOWS_OWN_DB + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql_airflow_db.user }} + key: AIRFLOW_DATABASE_URI + args: ["webserver"] + volumeMounts: + - name: airflow-etc + mountPath: {{ .Values.conf.airflow_webserver_config_file.path }} + subPath: webserver_config.py + readOnly: true + - name: airflow-etc + mountPath: {{ .Values.conf.airflow_config_file.path }} + subPath: airflow.cfg + readOnly: true + - name: airflow-usr + mountPath: {{ .Values.conf.airflow_unittests_file.path }} + subPath: unittests.cfg + readOnly: true + - name: shipyard-etc + mountPath: /usr/local/airflow/plugins/shipyard.conf + subPath: shipyard.conf + readOnly: true + - name: airflow-logs + mountPath: {{ .Values.conf.airflow.logging.base_log_folder }} +{{ if $mounts_airflow_webserver_ui.volumeMounts }}{{ toYaml $mounts_airflow_webserver_ui.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: airflow-etc + configMap: + name: airflow-etc + defaultMode: 0444 + - name: apache-etc + secret: + secretName: apache-etc + defaultMode: 0444 + - name: airflow-usr + configMap: + name: airflow-usr + defaultMode: 0444 + - name: shipyard-etc + configMap: + name: shipyard-etc + defaultMode: 0444 + - name: airflow-bin + configMap: + name: airflow-bin + defaultMode: 0555 + - name: airflow-logs + emptyDir: {} +{{ if $mounts_airflow_webserver_ui.volumes }}{{ toYaml $mounts_airflow_webserver_ui.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/charts/shipyard/templates/ingress-airflow-webserver-ui.yaml b/charts/shipyard/templates/ingress-airflow-webserver-ui.yaml new file mode 100644 index 00000000..b93df8ac --- /dev/null +++ b/charts/shipyard/templates/ingress-airflow-webserver-ui.yaml @@ -0,0 +1,21 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. +Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.ingress_shipyard_api .Values.network.airflow_webserver_ui.ingress.public }} +{{- $ingressOpts := dict "envAll" . "backendService" "airflow_webserver_ui" "backendServiceType" "airflow_webserver_ui" "backendPort" "api" -}} +{{- $ingressOpts | include "helm-toolkit.manifests.ingress" -}} +{{- end }} diff --git a/charts/shipyard/templates/secret-airflow-creds.yaml b/charts/shipyard/templates/secret-airflow-creds.yaml new file mode 100644 index 00000000..d78cac6c --- /dev/null +++ b/charts/shipyard/templates/secret-airflow-creds.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_airflow_webserver_ui }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.airflow_webserver_ui.admin }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + AIRFLOW_USERNAME: {{ .Values.endpoints.airflow_webserver_ui.auth.admin.username | b64enc }} + AIRFLOW_PASSWORD: {{ .Values.endpoints.airflow_webserver_ui.auth.admin.password | b64enc }} + BIND_DN: {{ .Values.endpoints.ldap.auth.admin.bind | b64enc }} + BIND_PASSWORD: {{ .Values.endpoints.ldap.auth.admin.password | b64enc }} +{{- end }} diff --git a/charts/shipyard/templates/secret-airflow-webserver-ui-ingress-tls.yaml b/charts/shipyard/templates/secret-airflow-webserver-ui-ingress-tls.yaml new file mode 100644 index 00000000..6dfb5d11 --- /dev/null +++ b/charts/shipyard/templates/secret-airflow-webserver-ui-ingress-tls.yaml @@ -0,0 +1,19 @@ +{{/* +Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_airflow_webserver_ui_ingress_tls }} +{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendService" "airflow_webserver_ui" "backendServiceType" "airflow_webserver_ui" ) }} +{{- end }} diff --git a/charts/shipyard/templates/secret-apache-etc.yaml b/charts/shipyard/templates/secret-apache-etc.yaml new file mode 100644 index 00000000..1d2dbcb2 --- /dev/null +++ b/charts/shipyard/templates/secret-apache-etc.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_apache_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: apache-etc +type: Opaque +data: + # NOTE(portdirect): this must be last, to work round helm ~2.7 bug. +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.httpd "key" "httpd.conf" "format" "Secret") | indent 2 }} +{{- end }} diff --git a/charts/shipyard/templates/secret-ingress-tls.yaml b/charts/shipyard/templates/secret-ingress-tls.yaml index 8d9ca4a6..c222c57b 100644 --- a/charts/shipyard/templates/secret-ingress-tls.yaml +++ b/charts/shipyard/templates/secret-ingress-tls.yaml @@ -16,4 +16,6 @@ limitations under the License. {{- if .Values.manifests.secret_ingress_tls }} {{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendService" "shipyard" "backendServiceType" "shipyard" ) }} +--- +{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendService" "airflow_webserver_ui" "backendServiceType" "airflow_webserver_ui" ) }} {{- end }} diff --git a/charts/shipyard/templates/service-airflow-webserver-ui.yaml b/charts/shipyard/templates/service-airflow-webserver-ui.yaml new file mode 100644 index 00000000..0ee34dd2 --- /dev/null +++ b/charts/shipyard/templates/service-airflow-webserver-ui.yaml @@ -0,0 +1,44 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. +Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_airflow_webserver_ui }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "airflow_webserver_ui" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + {{ if .Values.network.shipyard.enable_node_port }} + - name: api + protocol: TCP + nodePort: {{ .Values.network.airflow_webserver_ui.node_port }} + port: {{ tuple "airflow_webserver_ui" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + targetPort: {{ tuple "airflow_webserver_ui" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ else }} + - name: api + protocol: TCP + port: {{ tuple "airflow_webserver_ui" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + targetPort: {{ tuple "airflow_webserver_ui" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ end }} + selector: +{{ tuple $envAll "airflow_webserver_ui" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.network.airflow_webserver_ui.enable_node_port }} + type: NodePort + {{ end }} +{{- end }} diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index 32907f28..6c6d79dd 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -21,6 +21,9 @@ labels: shipyard: node_selector_key: ucp-control-plane node_selector_value: enabled + airflow_webserver_ui: + node_selector_key: ucp-control-plane + node_selector_value: enabled airflow: node_selector_key: ucp-control-plane node_selector_value: enabled @@ -31,12 +34,13 @@ labels: images: tags: airflow: quay.io/airshipit/airflow:latest-ubuntu_focal + apache_proxy: docker.io/library/httpd:2.4 shipyard: quay.io/airshipit/shipyard:latest-ubuntu_focal - dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 - shipyard_db_init: docker.io/postgres:14.5 - shipyard_db_auxiliary: docker.io/postgres:14.5 + dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 + shipyard_db_init: docker.io/postgres:14.8 + shipyard_db_auxiliary: docker.io/postgres:14.8 shipyard_db_sync: quay.io/airshipit/shipyard:latest-ubuntu_focal - airflow_db_init: docker.io/postgres:14.5 + airflow_db_init: docker.io/postgres:14.8 rabbit_init: docker.io/rabbitmq:3.7-management airflow_db_sync: quay.io/airshipit/airflow:latest-ubuntu_focal ks_user: docker.io/openstackhelm/heat:ocata @@ -74,6 +78,22 @@ network: name: airflow-worker port: 8793 enable_node_port: false + airflow_webserver_ui: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/proxy-read-timeout: "600" + nginx.ingress.kubernetes.io/proxy-body-size: "10m" + nginx.ingress.kubernetes.io/configuration-snippet: | + more_clear_headers "Server"; + more_set_headers "X-Content-Type-Options: 'nosniff'"; + more_set_headers "X-Frame-Options: 'deny'"; + node_port: 31902 + enable_node_port: false dependencies: static: @@ -208,6 +228,32 @@ endpoints: # tls: # crt: null # key: null + airflow_webserver_ui: + name: airflow-webserver-ui + hosts: + default: airflow-int + public: airflow-api + port: + api: + default: 80 + public: 80 + auth: + admin: + username: airflow + password: password + path: + default: / + scheme: + default: http + host_fqdn_override: + default: null + # NOTE(bryan-strassner): this chart supports TLS for fqdn over-ridden public + # endpoints using the following format: + # public: + # host: null + # tls: + # crt: null + # key: null airflow_worker: name: airflow-worker hosts: @@ -307,6 +353,22 @@ endpoints: port: memcache: default: 11211 + ldap: + hosts: + default: ldap + auth: + admin: + bind: "cn=admin,dc=cluster,dc=local" + password: password + host_fqdn_override: + default: null + path: + default: "/ou=People,dc=cluster,dc=local" + scheme: + default: ldap + port: + ldap: + default: 389 secrets: identity: @@ -321,12 +383,118 @@ secrets: postgresql_airflow_db: admin: airflow-db-admin user: airflow-db-user + airflow_webserver_ui: + admin: airflow-admin-creds tls: shipyard: shipyard: public: shipyard-tls-public + airflow_webserver_ui: + airflow_webserver_ui: + public: airflow-tls-public + conf: + httpd: | + ServerRoot "/usr/local/apache2" + + Listen 80 + + LoadModule mpm_event_module modules/mod_mpm_event.so + LoadModule authn_file_module modules/mod_authn_file.so + LoadModule authn_core_module modules/mod_authn_core.so + LoadModule authz_host_module modules/mod_authz_host.so + LoadModule authz_groupfile_module modules/mod_authz_groupfile.so + LoadModule authz_user_module modules/mod_authz_user.so + LoadModule authz_core_module modules/mod_authz_core.so + LoadModule access_compat_module modules/mod_access_compat.so + LoadModule auth_basic_module modules/mod_auth_basic.so + LoadModule ldap_module modules/mod_ldap.so + LoadModule authnz_ldap_module modules/mod_authnz_ldap.so + LoadModule reqtimeout_module modules/mod_reqtimeout.so + LoadModule filter_module modules/mod_filter.so + LoadModule proxy_html_module modules/mod_proxy_html.so + LoadModule log_config_module modules/mod_log_config.so + LoadModule env_module modules/mod_env.so + LoadModule headers_module modules/mod_headers.so + LoadModule setenvif_module modules/mod_setenvif.so + LoadModule version_module modules/mod_version.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_connect_module modules/mod_proxy_connect.so + LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule proxy_balancer_module modules/mod_proxy_balancer.so + LoadModule remoteip_module modules/mod_remoteip.so + LoadModule slotmem_shm_module modules/mod_slotmem_shm.so + LoadModule slotmem_plain_module modules/mod_slotmem_plain.so + LoadModule unixd_module modules/mod_unixd.so + LoadModule status_module modules/mod_status.so + LoadModule autoindex_module modules/mod_autoindex.so + + + User daemon + Group daemon + + + + AllowOverride none + Require all denied + + + + Require all denied + + + ErrorLog /dev/stderr + + LogLevel warn + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout common + CustomLog /dev/stdout combined + CustomLog /dev/stdout proxy env=forwarded + + + + AllowOverride None + Options None + Require all granted + + + + RequestHeader unset Proxy early + + + + Include conf/extra/proxy-html.conf + + + + RemoteIPHeader X-Original-Forwarded-For + + ProxyPreserveHost On + ProxyPass / http://localhost:8080/ + ProxyPassReverse / http://localhost:8080/ + + + AuthName "Airflow" + AuthType Basic + AuthBasicProvider file ldap + AuthUserFile /usr/local/apache2/conf/.htpasswd + AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }} + AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }} + AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} + Require valid-user + + uwsgi: # for more info about these default overrides please read # https://uwsgi-docs.readthedocs.io/en/latest/ThingsToKnow.html @@ -461,6 +629,8 @@ conf: deployment_version_commit: Skip airflow_config_file: path: /usr/local/airflow/airflow.cfg + airflow_webserver_config_file: + path: /usr/local/airflow/webserver_config.py airflow_unittests_file: path: /usr/local/airflow/unittests.cfg airflow: @@ -762,6 +932,10 @@ pod: airflow-scheduler: init: runtime/default airflow-scheduler: runtime/default + airflow-webserver-ui: + init: runtime/default + apache-proxy: runtime/default + airflow-webserver-ui: runtime/default shipyard-db-auxiliary: init: runtime/default shipyard-db-auxiliary: runtime/default @@ -797,6 +971,15 @@ pod: airflow_scheduler: readOnlyRootFilesystem: true allowPrivilegeEscalation: false + airflow_webserver_ui: + container: + apache_proxy: + runAsUser: 0 + readOnlyRootFilesystem: false + airflow_webserver_ui: + runAsUser: 1000 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false airflow_worker: pod: runAsUser: 1000 @@ -856,6 +1039,9 @@ pod: init_container: null airflow_worker: airflow_scheduler: + airflow_webserver_ui: + airflow_webserver_ui: + init_container: null shipyard: init_container: null shipyard: @@ -880,6 +1066,8 @@ pod: airflow: worker: 2 scheduler: 2 + airflow_webserver_ui: + airflow_webserver_ui: 1 probes: airflow_worker: airflow_worker: @@ -939,6 +1127,18 @@ pod: requests: memory: "128Mi" cpu: "100m" + apache_proxy: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + airflow_webserver_ui: + airflow_webserver_ui: + limits: + memory: "128Mi" + cpu: "100m" shipyard_api: limits: memory: "128Mi" @@ -1040,9 +1240,11 @@ manifests: # TODO: Set this to false only if a new deployment, or if the worker pod is # running the scheduler deployment_airflow_scheduler: true + deployment_airflow_webserver_ui: true deployment_shipyard: true statefulset_airflow_worker: true ingress_shipyard_api: true + ingress_airflow_webserver_ui: true job_shipyard_db_init: true job_shipyard_db_auxiliary: true job_shipyard_db_sync: true @@ -1052,6 +1254,8 @@ manifests: job_ks_endpoints: true job_ks_service: true job_ks_user: true + secret_apache_etc: true + secret_airflow_webserver_ui: true secret_airflow_db: true secret_shipyard_db: true secret_ingress_tls: true @@ -1059,6 +1263,8 @@ manifests: secret_rabbitmq: true service_shipyard: true service_shipyard_ingress: true + service_airflow_webserver_ui: true + service_airflow_webserver_ui_ingress: true service_airflow_worker: true service_discovery_airflow_worker: true test_shipyard_api: true diff --git a/tools/gate/playbooks/airskiff-deploy.yaml b/tools/gate/playbooks/airskiff-deploy.yaml index f89455c4..d4d885da 100644 --- a/tools/gate/playbooks/airskiff-deploy.yaml +++ b/tools/gate/playbooks/airskiff-deploy.yaml @@ -22,6 +22,7 @@ - ensure-pip - ensure-docker + tasks: - name: Install Packaging python module for tools/airship