Gate: Simplify and speed up pod and k8s object log collection
This PS simplifys and accelerates the collection of pod container logs and k8s object info in the zuul gates, which can currently take 40 minutes for the 5 node OSH gate to complete. Change-Id: Ie9b23174fade3df4a87f2b771ea654e2081b4f4e
This commit is contained in:
portdirect
parent
ac53413b47
commit
217385a0dd
tools/gate/playbooks
describe-kubernetes-objects/tasks
describe-kubernetes-resources/tasks
main.yamlutil-common-cluster-describe.yamlutil-common-namespace-describe.yamlutil-namespace-describe.yaml
gather-pod-logs/tasks
osh-infra-collect-logs.yaml
108
tools/gate/playbooks/describe-kubernetes-objects/tasks/main.yaml
Normal file
108
tools/gate/playbooks/describe-kubernetes-objects/tasks/main.yaml
Normal file
@ -0,0 +1,108 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: "creating directory for cluster scoped objects"
|
||||
file:
|
||||
path: "{{ logs_dir }}/objects/cluster"
|
||||
state: directory
|
||||
|
||||
- name: "Gathering descriptions for cluster scoped objects"
|
||||
shell: |-
|
||||
set -e
|
||||
export OBJECT_TYPE=node,clusterrole,clusterrolebinding,storageclass
|
||||
export PARALLELISM_FACTOR=2
|
||||
|
||||
function list_objects () {
|
||||
printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P1 -n1 bash -c 'echo "$@"' _ {}
|
||||
}
|
||||
export -f list_objects
|
||||
|
||||
function name_objects () {
|
||||
export OBJECT=$1
|
||||
kubectl get ${OBJECT} -o name | xargs -L1 -I {} -P1 -n1 bash -c 'echo "${OBJECT} ${1#*/}"' _ {}
|
||||
}
|
||||
export -f name_objects
|
||||
|
||||
function get_objects () {
|
||||
input=($1)
|
||||
export OBJECT=${input[0]}
|
||||
export NAME=${input[1]#*/}
|
||||
echo "${OBJECT}/${NAME}"
|
||||
DIR="{{ logs_dir }}/objects/cluster/${OBJECT}"
|
||||
mkdir -p ${DIR}
|
||||
kubectl get ${OBJECT} ${NAME} -o yaml > "${DIR}/${NAME}.yaml"
|
||||
kubectl describe ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt"
|
||||
}
|
||||
export -f get_objects
|
||||
|
||||
list_objects | \
|
||||
xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \
|
||||
xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {}
|
||||
args:
|
||||
executable: /bin/bash
|
||||
ignore_errors: True
|
||||
|
||||
- name: "creating directory for namespace scoped objects"
|
||||
file:
|
||||
path: "{{ logs_dir }}/objects/namespaced"
|
||||
state: directory
|
||||
|
||||
- name: "Gathering descriptions for namespace scoped objects"
|
||||
shell: |-
|
||||
set -e
|
||||
export OBJECT_TYPE=configmaps,cronjobs,daemonsets,deployment,endpoints,ingresses,jobs,networkpolicies,pods,podsecuritypolicies,persistentvolumeclaims,rolebindings,roles,secrets,serviceaccounts,services,statefulsets
|
||||
export PARALLELISM_FACTOR=2
|
||||
function get_namespaces () {
|
||||
kubectl get namespaces -o name | awk -F '/' '{ print $NF }'
|
||||
}
|
||||
|
||||
function list_namespaced_objects () {
|
||||
export NAMESPACE=$1
|
||||
printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} $@"' _ {}
|
||||
}
|
||||
export -f list_namespaced_objects
|
||||
|
||||
function name_objects () {
|
||||
input=($1)
|
||||
export NAMESPACE=${input[0]}
|
||||
export OBJECT=${input[1]}
|
||||
kubectl get -n ${NAMESPACE} ${OBJECT} -o name | xargs -L1 -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} ${OBJECT} $@"' _ {}
|
||||
}
|
||||
export -f name_objects
|
||||
|
||||
function get_objects () {
|
||||
input=($1)
|
||||
export NAMESPACE=${input[0]}
|
||||
export OBJECT=${input[1]}
|
||||
export NAME=${input[2]#*/}
|
||||
echo "${NAMESPACE}/${OBJECT}/${NAME}"
|
||||
DIR="{{ logs_dir }}/objects/namespaced/${NAMESPACE}/${OBJECT}"
|
||||
mkdir -p ${DIR}
|
||||
kubectl get -n ${NAMESPACE} ${OBJECT} ${NAME} -o yaml > "${DIR}/${NAME}.yaml"
|
||||
kubectl describe -n ${NAMESPACE} ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt"
|
||||
}
|
||||
export -f get_objects
|
||||
|
||||
get_namespaces | \
|
||||
xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'list_namespaced_objects "$@"' _ {} | \
|
||||
xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \
|
||||
xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {}
|
||||
args:
|
||||
executable: /bin/bash
|
||||
ignore_errors: True
|
||||
|
||||
- name: "Downloads logs to executor"
|
||||
synchronize:
|
||||
src: "{{ logs_dir }}/objects"
|
||||
dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
|
||||
mode: pull
|
||||
ignore_errors: yes
|
||||
@ -1,47 +0,0 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: "Gather all deployed namespaces"
|
||||
shell: |-
|
||||
set -e
|
||||
kubectl get namespaces -o json | jq -r '.items[].metadata.name'
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: namespaces
|
||||
|
||||
- name: "Gathering descriptions for namespaced objects"
|
||||
include: util-namespace-describe.yaml
|
||||
vars:
|
||||
namespace: "{{ namespace }}"
|
||||
loop_control:
|
||||
loop_var: namespace
|
||||
with_items: "{{ namespaces.stdout_lines }}"
|
||||
|
||||
|
||||
- name: "Gathering descriptions for cluster scoped objects"
|
||||
include: util-common-cluster-describe.yaml
|
||||
vars:
|
||||
cluster_object: "{{ cluster_object }}"
|
||||
loop_control:
|
||||
loop_var: cluster_object
|
||||
with_items:
|
||||
- node
|
||||
- clusterrole
|
||||
- clusterrolebinding
|
||||
- storageclass
|
||||
|
||||
- name: "Downloads logs to executor"
|
||||
synchronize:
|
||||
src: "{{ logs_dir }}/resources"
|
||||
dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
|
||||
mode: pull
|
||||
ignore_errors: yes
|
||||
@ -1,37 +0,0 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Kubectl describe cluster scoped objects common block
|
||||
vars:
|
||||
cluster_object: null
|
||||
|
||||
block:
|
||||
- name: "creating directory for {{ cluster_object }} descriptions"
|
||||
file: path="{{ logs_dir }}/resources/{{ cluster_object }}" state=directory
|
||||
|
||||
- name: "gathering names of {{ cluster_object }}s currently deployed"
|
||||
shell: |-
|
||||
set -e
|
||||
kubectl get {{ cluster_object }} -o json | jq -r '.items[].metadata.name'
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: resource_names
|
||||
|
||||
- name: "getting descriptions of {{ cluster_object }}s deployed"
|
||||
shell: |-
|
||||
set -e
|
||||
kubectl describe {{ cluster_object }} {{ object_name }} > {{ logs_dir }}/resources/{{ cluster_object }}/{{ object_name }}.yaml
|
||||
args:
|
||||
executable: /bin/bash
|
||||
loop_control:
|
||||
loop_var: object_name
|
||||
with_items: "{{ resource_names.stdout_lines }}"
|
||||
@ -1,41 +0,0 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Kubectl describe namespaced object common block
|
||||
vars:
|
||||
api_object: null
|
||||
object_namespace: null
|
||||
|
||||
block:
|
||||
- name: "creating directory for {{ api_object }} descriptions in {{ object_namespace }} namespace"
|
||||
file:
|
||||
path: "{{ logs_dir }}/resources/{{ object_namespace }}/{{ api_object }}"
|
||||
state: directory
|
||||
|
||||
- name: "gathering names of {{ api_object }}s currently deployed in {{ object_namespace }} namespace"
|
||||
shell: |-
|
||||
set -e
|
||||
kubectl get {{ api_object }} --namespace={{ object_namespace }} -o json | jq -r '.items[].metadata.name'
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: namespaced_resource_names
|
||||
|
||||
- name: "getting descriptions of {{ api_object }}s deployed in {{ object_namespace }} namespace"
|
||||
when: namespaced_resource_names
|
||||
shell: |-
|
||||
set -e
|
||||
kubectl describe {{ api_object }} {{ resource_name }} --namespace={{ object_namespace }} > {{ logs_dir }}/resources/{{ object_namespace }}/{{ api_object }}/{{ resource_name }}.yaml
|
||||
args:
|
||||
executable: /bin/bash
|
||||
loop_control:
|
||||
loop_var: resource_name
|
||||
with_items: "{{ namespaced_resource_names.stdout_lines }}"
|
||||
@ -1,43 +0,0 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Kubectl describe all namespaced objects common block
|
||||
vars:
|
||||
api_objects:
|
||||
- configmaps
|
||||
- cronjobs
|
||||
- daemonsets
|
||||
- deployment
|
||||
- endpoints
|
||||
- ingresses
|
||||
- jobs
|
||||
- networkpolicies
|
||||
- pods
|
||||
- podsecuritypolicies
|
||||
- persistentvolumeclaims
|
||||
- rolebindings
|
||||
- roles
|
||||
- secrets
|
||||
- serviceaccounts
|
||||
- services
|
||||
- statefulsets
|
||||
namespace: null
|
||||
|
||||
block:
|
||||
- name: "Describe all {{ api_object }} objects in {{ namespace }} namespace"
|
||||
vars:
|
||||
object_namespace: "{{ namespace }}"
|
||||
api_object: "{{ api_object }}"
|
||||
loop_control:
|
||||
loop_var: api_object
|
||||
include: util-common-namespace-describe.yaml
|
||||
with_items: "{{ api_objects }}"
|
||||
@ -15,20 +15,36 @@
|
||||
path: "{{ logs_dir }}/pod-logs"
|
||||
state: directory
|
||||
|
||||
- name: "retrieve all deployed namespaces"
|
||||
- name: "retrieve all container logs"
|
||||
shell: |-
|
||||
set -e
|
||||
kubectl get namespaces -o json | jq -r '.items[].metadata.name'
|
||||
PARALLELISM_FACTOR=2
|
||||
function get_namespaces () {
|
||||
kubectl get namespaces -o name | awk -F '/' '{ print $NF }'
|
||||
}
|
||||
function get_pods () {
|
||||
NAMESPACE=$1
|
||||
kubectl get pods -n ${NAMESPACE} -o name --show-all | awk -F '/' '{ print $NF }' | xargs -L1 -P 1 -I {} echo ${NAMESPACE} {}
|
||||
}
|
||||
export -f get_pods
|
||||
function get_pod_logs () {
|
||||
NAMESPACE=${1% *}
|
||||
POD=${1#* }
|
||||
INIT_CONTAINERS=$(kubectl get pod $POD -n ${NAMESPACE} -o json | jq -r '.spec.initContainers[]?.name')
|
||||
CONTAINERS=$(kubectl get pod $POD -n ${NAMESPACE} -o json | jq -r '.spec.containers[].name')
|
||||
for CONTAINER in ${INIT_CONTAINERS} ${CONTAINERS}; do
|
||||
echo "${NAMESPACE}/${POD}/${CONTAINER}"
|
||||
mkdir -p "{{ logs_dir }}/pod-logs/${NAMESPACE}/${POD}"
|
||||
kubectl logs ${POD} -n ${NAMESPACE} -c ${CONTAINER} > "{{ logs_dir }}/pod-logs/${NAMESPACE}/${POD}/${CONTAINER}.txt"
|
||||
done
|
||||
}
|
||||
export -f get_pod_logs
|
||||
get_namespaces | \
|
||||
xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pods "$@"' _ {} | \
|
||||
xargs -r -n 2 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pod_logs "$@"' _ {}
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: namespaces
|
||||
|
||||
- include: util-container-logs.yaml
|
||||
vars:
|
||||
namespace: "{{ namespace }}"
|
||||
loop_control:
|
||||
loop_var: namespace
|
||||
with_items: "{{ namespaces.stdout_lines }}"
|
||||
ignore_errors: True
|
||||
|
||||
- name: "Downloads logs to executor"
|
||||
synchronize:
|
||||
|
||||
@ -1,56 +0,0 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Gather pod container logs common block
|
||||
vars:
|
||||
pod: null
|
||||
|
||||
block:
|
||||
- name: "Gather list of init containers in {{ pod }} pod in {{ namespace }} namespace"
|
||||
shell: |-
|
||||
set -e
|
||||
kubectl get pod {{ pod }} -n {{ namespace }} -o json | jq -r '.spec.initContainers[].name'
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: init_container_names
|
||||
ignore_errors: True
|
||||
|
||||
- name: "Gather logs from all init containers in pod {{ pod }}"
|
||||
shell: |-
|
||||
set -e
|
||||
kubectl logs {{ pod }} -n {{ namespace }} -c {{ init_container }} >> {{ logs_dir }}/pod-logs/{{ namespace }}-{{ pod }}-{{ init_container }}.txt
|
||||
args:
|
||||
executable: /bin/bash
|
||||
loop_control:
|
||||
loop_var: init_container
|
||||
with_items: "{{ init_container_names.stdout_lines }}"
|
||||
ignore_errors: True
|
||||
|
||||
- name: "Gather list of containers in {{ pod }} pod in {{ namespace }} namespace"
|
||||
shell: |-
|
||||
set -e
|
||||
kubectl get pod {{ pod }} -n {{ namespace }} -o json | jq -r '.spec.containers[].name'
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: container_names
|
||||
ignore_errors: True
|
||||
|
||||
- name: "Gather logs from all containers in pod {{ pod }}"
|
||||
shell: |-
|
||||
set -e
|
||||
kubectl logs {{ pod }} -n {{ namespace }} -c {{ container }} >> {{ logs_dir }}/pod-logs/{{ namespace }}-{{ pod }}-{{ container }}.txt
|
||||
args:
|
||||
executable: /bin/bash
|
||||
loop_control:
|
||||
loop_var: container
|
||||
with_items: "{{ container_names.stdout_lines }}"
|
||||
ignore_errors: True
|
||||
@ -1,44 +0,0 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Gather container logs common block
|
||||
vars:
|
||||
namespace: null
|
||||
|
||||
block:
|
||||
- name: "Gather list of pods in {{ namespace }} namespace"
|
||||
shell: |-
|
||||
set -e
|
||||
kubectl get pods -n {{ namespace }} -o json | jq -r '.items[].metadata.name'
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: pod_names
|
||||
ignore_errors: True
|
||||
|
||||
- include: util-common-gather-logs.yaml
|
||||
vars:
|
||||
pod: "{{ pod_name }}"
|
||||
loop_control:
|
||||
loop_var: pod_name
|
||||
with_items: "{{ pod_names.stdout_lines }}"
|
||||
@ -31,9 +31,9 @@
|
||||
work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
|
||||
logs_dir: "/tmp/logs"
|
||||
roles:
|
||||
- describe-kubernetes-resources
|
||||
- describe-kubernetes-objects
|
||||
tags:
|
||||
- describe-kubernetes-resources
|
||||
- describe-kubernetes-objects
|
||||
|
||||
- hosts: primary
|
||||
vars_files:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user