diff --git a/helm-toolkit/templates/snippets/_kubernetes_apparmor_configmap.tpl b/helm-toolkit/templates/snippets/_kubernetes_apparmor_configmap.tpl new file mode 100644 index 000000000..8119883da --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_apparmor_configmap.tpl @@ -0,0 +1,70 @@ +{{/* +Copyright 2017-2018 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders a configmap used for loading custom AppArmor profiles. +values: | + pod: + mandatory_access_control: + type: apparmor + configmap_apparmor: true + apparmor_profiles: |- + my_apparmor-v1.profile: |- + #include + profile my-apparmor-v1 flags=(attach_disconnected,mediate_deleted) { + + } +usage: | + {{ dict "envAll" . "component" "myComponent" | include "helm-toolkit.snippets.kubernetes_apparmor_configmap" }} +return: | +apiVersion: v1 +kind: ConfigMap +metadata: + name: releaseName-myComponent-apparmor + namespace: myNamespace +data: + my_apparmor-v1.profile: |- + #include + profile my-apparmor-v1 flags=(attach_disconnected,mediate_deleted) { + + } +*/}} +{{- define "helm-toolkit.snippets.kubernetes_apparmor_configmap" -}} +{{- $envAll := index . "envAll" -}} +{{- $component := index . "component" -}} +{{- if hasKey $envAll.Values.pod "mandatory_access_control" -}} +{{- if hasKey $envAll.Values.pod.mandatory_access_control "type" -}} +{{- if eq $envAll.Values.pod.mandatory_access_control.type "apparmor" -}} +{{- if hasKey $envAll.Values.pod.mandatory_access_control "configmap_apparmor" -}} +{{- if $envAll.Values.pod.mandatory_access_control.configmap_apparmor }} +{{- $mapName := printf "%s-%s-%s" $envAll.Release.Name $component "apparmor" -}} +{{- if $envAll.Values.conf.apparmor_profiles }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $mapName }} + namespace: {{ $envAll.Release.Namespace }} +data: +{{ $envAll.Values.conf.apparmor_profiles | toYaml | indent 2 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/snippets/_kubernetes_apparmor_loader_init_container.tpl b/helm-toolkit/templates/snippets/_kubernetes_apparmor_loader_init_container.tpl new file mode 100644 index 000000000..7fcf482ff --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_apparmor_loader_init_container.tpl @@ -0,0 +1,77 @@ +{{/* +Copyright 2017-2018 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders the init container used for apparmor loading. +values: | + images: + tags: + apparmor_loader: my-repo.io/apparmor-loader:1.0.0 + pod: + mandatory_access_control: + type: apparmor + configmap_apparmor: true + apparmor-loader: unconfined +usage: | + {{ dict "envAll" . | include "helm-toolkit.snippets.kubernetes_apparmor_loader_init_container" }} +return: | + - name: apparmor-loader + image: my-repo.io/apparmor-loader:1.0.0 + args: + - /profiles + securityContext: + privileged: true + volumeMounts: + - name: sys + mountPath: /sys + readOnly: true + - name: includes + mountPath: /etc/apparmor.d + readOnly: true + - name: profiles + mountPath: /profiles + readOnly: true +*/}} +{{- define "helm-toolkit.snippets.kubernetes_apparmor_loader_init_container" -}} +{{- $envAll := index . "envAll" -}} +{{- if hasKey $envAll.Values.pod "mandatory_access_control" -}} +{{- if hasKey $envAll.Values.pod.mandatory_access_control "type" -}} +{{- if hasKey $envAll.Values.pod.mandatory_access_control "configmap_apparmor" -}} +{{- if eq $envAll.Values.pod.mandatory_access_control.type "apparmor" -}} +{{- if $envAll.Values.pod.mandatory_access_control.configmap_apparmor }} +- name: apparmor-loader + image: {{ $envAll.Values.images.tags.apparmor_loader }} + args: + - /profiles + securityContext: + privileged: true + volumeMounts: + - name: sys + mountPath: /sys + readOnly: true + - name: includes + mountPath: /etc/apparmor.d + readOnly: true + - name: profiles + mountPath: /profiles + readOnly: true +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/snippets/_kubernetes_apparmor_volumes.tpl b/helm-toolkit/templates/snippets/_kubernetes_apparmor_volumes.tpl new file mode 100644 index 000000000..75b1ce8f5 --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_apparmor_volumes.tpl @@ -0,0 +1,70 @@ +{{/* +Copyright 2017-2018 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders the volumes used by the apparmor loader. +values: | + pod: + mandatory_access_control: + type: apparmor + configmap_apparmor: true +inputs: | + envAll: "Environment or Context." + component: "Name of the component used for the name of configMap." + requireSys: "Boolean. True if it needs the hostpath /sys in volumes." +usage: | + {{ dict "envAll" . "component" "keystone" "requireSys" true | include "helm-toolkit.snippets.kubernetes_apparmor_volumes" }} +return: | +- name: sys + hostPath: + path: /sys +- name: includes + hostPath: + path: /etc/apparmor.d +- name: profiles + configMap: + name: RELEASENAME-keystone-apparmor + defaultMode: 0555 +*/}} +{{- define "helm-toolkit.snippets.kubernetes_apparmor_volumes" -}} +{{- $envAll := index . "envAll" -}} +{{- $component := index . "component" -}} +{{- $requireSys := index . "requireSys" | default false -}} +{{- $configName := printf "%s-%s-%s" $envAll.Release.Name $component "apparmor" -}} +{{- if hasKey $envAll.Values.pod "mandatory_access_control" -}} +{{- if hasKey $envAll.Values.pod.mandatory_access_control "type" -}} +{{- if hasKey $envAll.Values.pod.mandatory_access_control "configmap_apparmor" -}} +{{- if eq $envAll.Values.pod.mandatory_access_control.type "apparmor" -}} +{{- if $envAll.Values.pod.mandatory_access_control.configmap_apparmor }} +{{- if $requireSys }} +- name: sys + hostPath: + path: /sys +{{- end }} +- name: includes + hostPath: + path: /etc/apparmor.d +- name: profiles + configMap: + name: {{ $configName | quote }} + defaultMode: 0555 +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/memcached/templates/configmap-apparmor.yaml b/memcached/templates/configmap-apparmor.yaml new file mode 100644 index 000000000..6c28dcae1 --- /dev/null +++ b/memcached/templates/configmap-apparmor.yaml @@ -0,0 +1,17 @@ +{{/* +Copyright 2017-2018 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- dict "envAll" . "component" "memcached" | include "helm-toolkit.snippets.kubernetes_apparmor_configmap" }} diff --git a/memcached/templates/deployment.yaml b/memcached/templates/deployment.yaml index 0d55a0f4b..931da801f 100644 --- a/memcached/templates/deployment.yaml +++ b/memcached/templates/deployment.yaml @@ -39,6 +39,7 @@ spec: template: metadata: annotations: +{{- dict "envAll" $envAll "podName" "memcached" "containerNames" (list "apparmor-loader" "memcached") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} labels: {{ tuple $envAll "memcached" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} @@ -53,6 +54,7 @@ spec: terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.memcached.timeout | default "30" }} initContainers: {{ tuple $envAll "memcached" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ dict "envAll" $envAll | include "helm-toolkit.snippets.kubernetes_apparmor_loader_init_container" | indent 8 }} containers: - name: memcached {{ tuple $envAll "memcached" | include "helm-toolkit.snippets.image" | indent 10 }} @@ -81,4 +83,5 @@ spec: configMap: name: {{ $configMapBinName | quote }} defaultMode: 0555 +{{ dict "envAll" $envAll "component" "memcached" "requireSys" true | include "helm-toolkit.snippets.kubernetes_apparmor_volumes" | indent 8 }} {{- end }} diff --git a/tools/deployment/apparmor/000-install-packages.sh b/tools/deployment/apparmor/000-install-packages.sh new file mode 120000 index 000000000..d702c4899 --- /dev/null +++ b/tools/deployment/apparmor/000-install-packages.sh @@ -0,0 +1 @@ +../common/000-install-packages.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/001-setup-apparmor-profiles.sh b/tools/deployment/apparmor/001-setup-apparmor-profiles.sh new file mode 120000 index 000000000..543e2fc9d --- /dev/null +++ b/tools/deployment/apparmor/001-setup-apparmor-profiles.sh @@ -0,0 +1 @@ +../common/001-setup-apparmor-profiles.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/005-deploy-k8s.sh b/tools/deployment/apparmor/005-deploy-k8s.sh new file mode 120000 index 000000000..257a39f7a --- /dev/null +++ b/tools/deployment/apparmor/005-deploy-k8s.sh @@ -0,0 +1 @@ +../common/005-deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/040-memcached.sh b/tools/deployment/apparmor/040-memcached.sh new file mode 100755 index 000000000..a09144b26 --- /dev/null +++ b/tools/deployment/apparmor/040-memcached.sh @@ -0,0 +1,140 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +namespace="osh-infra" + +# NOTE: Lint and package chart +make memcached + +tee /tmp/memcached.yaml < + profile my-apparmor-v1 flags=(attach_disconnected,mediate_deleted) { + #include + network inet tcp, + network inet udp, + network inet icmp, + deny network raw, + deny network packet, + file, + umount, + deny /bin/** wl, + deny /boot/** wl, + deny /dev/** wl, + deny /etc/** wl, + deny /home/** wl, + deny /lib/** wl, + deny /lib64/** wl, + deny /media/** wl, + deny /mnt/** wl, + deny /opt/** wl, + deny /proc/** wl, + deny /root/** wl, + deny /sbin/** wl, + deny /srv/** wl, + deny /tmp/** wl, + deny /sys/** wl, + deny /usr/** wl, + audit /** w, + /var/run/nginx.pid w, + /usr/sbin/nginx ix, + deny /bin/dash mrwklx, + deny /bin/sh mrwklx, + deny /usr/bin/top mrwklx, + capability chown, + capability dac_override, + capability setuid, + capability setgid, + capability net_bind_service, + deny @{PROC}/{*,**^[0-9*],sys/kernel/shm*} wkx, + deny @{PROC}/sysrq-trigger rwklx, + deny @{PROC}/mem rwklx, + deny @{PROC}/kmem rwklx, + deny @{PROC}/kcore rwklx, + deny mount, + deny /sys/[^f]*/** wklx, + deny /sys/f[^s]*/** wklx, + deny /sys/fs/[^c]*/** wklx, + deny /sys/fs/c[^g]*/** wklx, + deny /sys/fs/cg[^r]*/** wklx, + deny /sys/firmware/efi/efivars/** rwklx, + deny /sys/kernel/security/** rwklx, + } +EOF + +# NOTE: Deploy command +: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} +helm upgrade --install memcached ./memcached \ + --namespace=$namespace \ + --set pod.replicas.server=1 \ + --values=/tmp/memcached.yaml \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_MEMCACHED} + +# NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh $namespace + +# NOTE: Validate Deployment info +helm status memcached + +# Run a test. Note: the simple "cat /proc/1/attr/current" verification method +# will not work, as memcached has multiple processes running, so we have to +# find out which one is the memcached application process. +pod=$(kubectl -n $namespace get pod | grep memcached | awk '{print $1}') +unsorted_process_file="/tmp/unsorted_proc_list" +sorted_process_file="/tmp/proc_list" +expected_profile="my-apparmor-v1 (enforce)" + +# Grab the processes (numbered directories) from the /proc directory, +# and then sort them. Highest proc number indicates most recent process. +kubectl -n $namespace exec $pod -- ls -1 /proc | grep -e "^[0-9]*$" > $unsorted_process_file +sort --numeric-sort $unsorted_process_file > $sorted_process_file + +# The last/latest process in the list will actually be the "ls" command above, +# which isn't running any more, so remove it. +sed -i '$ d' $sorted_process_file + +while IFS='' read -r process || [[ -n "$process" ]]; do + echo "Process ID: $process" + proc_name=`kubectl -n $namespace exec $pod -- cat /proc/$process/status | grep "Name:" | awk -F' ' '{print $2}'` + echo "Process Name: $proc_name" + profile=`kubectl -n $namespace exec $pod -- cat /proc/$process/attr/current` + echo "Profile running: $profile" + if test "$profile" != "$expected_profile" + then + if test "$proc_name" == "pause" + then + echo "Root process (pause) can run docker-default, it's ok." + else + echo "$profile is the WRONG PROFILE!!" + return 1 + fi + fi +done < $sorted_process_file diff --git a/tools/deployment/common/001-setup-apparmor-profiles.sh b/tools/deployment/common/001-setup-apparmor-profiles.sh new file mode 100755 index 000000000..42a8e666b --- /dev/null +++ b/tools/deployment/common/001-setup-apparmor-profiles.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2017-2018 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +# Ensure that apparmor is installed and enabled +sudo -H -E apt-get install -y apparmor +sudo systemctl enable apparmor && sudo systemctl start apparmor +sudo systemctl status apparmor.service diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 9ade7f505..75269da61 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -194,6 +194,21 @@ - ./tools/deployment/network-policy/901-test-networkpolicy.sh +- job: + name: openstack-helm-infra-apparmor + parent: openstack-helm-infra-functional + timeout: 7200 + pre-run: playbooks/osh-infra-upgrade-host.yaml + run: playbooks/osh-infra-gate-runner.yaml + post-run: playbooks/osh-infra-collect-logs.yaml + nodeset: openstack-helm-single-node + vars: + gate_scripts: + - ./tools/deployment/apparmor/000-install-packages.sh + - ./tools/deployment/apparmor/001-setup-apparmor-profiles.sh + - ./tools/deployment/apparmor/005-deploy-k8s.sh + - ./tools/deployment/apparmor/040-memcached.sh + - job: name: openstack-helm-infra-openstack-support parent: openstack-helm-infra-functional diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index a236be245..453c0cb04 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -29,7 +29,8 @@ # override functionality - openstack-helm-infra-airship-divingbell: voting: false - - openstack-helm-infra-aio-podsecuritypolicy: + - openstack-helm-infra-aio-podsecuritypolicy + - openstack-helm-infra-apparmor: voting: false gate: jobs: @@ -38,6 +39,7 @@ - openstack-helm-infra-aio-monitoring - openstack-helm-infra-openstack-support - openstack-helm-infra-kubernetes-keystone-auth + - openstack-helm-infra-aio-podsecuritypolicy periodic: jobs: - openstack-helm-infra-tenant-ceph