diff --git a/ceph-mon/templates/bin/utils/_checkObjectReplication.py.tpl b/ceph-mon/templates/bin/utils/_checkObjectReplication.py.tpl new file mode 100755 index 000000000..0b87c7f2d --- /dev/null +++ b/ceph-mon/templates/bin/utils/_checkObjectReplication.py.tpl @@ -0,0 +1,30 @@ +#!/usr/bin/python2 + +import subprocess +import json +import sys +import collections + +if (int(len(sys.argv)) == 1): + print "Please provide pool name to test , example: checkObjectReplication.py " + sys.exit(1) +else: + poolName = sys.argv[1] + cmdRep = 'ceph osd map' + ' ' + str(poolName) + ' ' + 'testreplication -f json-pretty' + objectRep = subprocess.check_output(cmdRep, shell=True) + repOut = json.loads(objectRep) + osdNumbers = repOut['up'] + print "Test object got replicated on these osds:" + " " + str(osdNumbers) + + osdHosts= [] + for osd in osdNumbers: + cmdFind = 'ceph osd find' + ' ' + str(osd) + osdFind = subprocess.check_output(cmdFind , shell=True) + osdHost = json.loads(osdFind) + osdHostLocation = osdHost['crush_location'] + osdHosts.append(osdHostLocation['host']) + + print "Test object got replicated on these hosts:" + " " + str(osdHosts) + + print "Hosts hosting multiple copies of a placement groups are:" + str([item for item, count in collections.Counter(osdHosts).items() if count > 1]) + sys.exit(0) diff --git a/ceph-mon/templates/configmap-bin.yaml b/ceph-mon/templates/configmap-bin.yaml index 64ffd5515..5203f25ae 100644 --- a/ceph-mon/templates/configmap-bin.yaml +++ b/ceph-mon/templates/configmap-bin.yaml @@ -59,5 +59,7 @@ data: utils-checkPGs.sh: | {{ tuple "bin/utils/_checkPGs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + utils-checkObjectReplication.py: | +{{ tuple "bin/utils/_checkObjectReplication.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/ceph-mon/templates/daemonset-mon.yaml b/ceph-mon/templates/daemonset-mon.yaml index dfd38ff11..f6ebf6aeb 100644 --- a/ceph-mon/templates/daemonset-mon.yaml +++ b/ceph-mon/templates/daemonset-mon.yaml @@ -182,6 +182,10 @@ spec: mountPath: /tmp/utils-checkPGs.sh subPath: utils-checkPGs.sh readOnly: true + - name: ceph-mon-bin + mountPath: /tmp/checkObjectReplication.py + subPath: utils-checkObjectReplication.py + readOnly: true - name: ceph-mon-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf diff --git a/doc/source/testing/ceph-resiliency/index.rst b/doc/source/testing/ceph-resiliency/index.rst index c93958e87..3dc3ccd93 100644 --- a/doc/source/testing/ceph-resiliency/index.rst +++ b/doc/source/testing/ceph-resiliency/index.rst @@ -7,3 +7,4 @@ Ceph Resiliency README failure-domain + validate-object-replication diff --git a/doc/source/testing/ceph-resiliency/validate-object-replication.rst b/doc/source/testing/ceph-resiliency/validate-object-replication.rst new file mode 100644 index 000000000..db61ca919 --- /dev/null +++ b/doc/source/testing/ceph-resiliency/validate-object-replication.rst @@ -0,0 +1,65 @@ +=========================================== +Ceph - Test object replication across hosts +=========================================== + +This document captures steps to validate object replcation is happening across +hosts or not . + + +Setup: +====== +- Follow OSH single node or multinode guide to bring up OSH envronment. + + + +Step 1: Setup the OSH environment and check ceph cluster health +================================================================= + +.. note:: + Make sure we have healthy ceph cluster running + +``Ceph status:`` + +.. code-block:: console + + ubuntu@mnode1:/opt/openstack-helm$ kubectl exec -n ceph ceph-mon-5qn68 -- ceph -s + cluster: + id: 54d9af7e-da6d-4980-9075-96bb145db65c + health: HEALTH_OK + + services: + mon: 3 daemons, quorum mnode1,mnode2,mnode3 + mgr: mnode2(active), standbys: mnode3 + mds: cephfs-1/1/1 up {0=mds-ceph-mds-6f66956547-c25cx=up:active}, 1 up:standby + osd: 3 osds: 3 up, 3 in + rgw: 2 daemons active + + data: + pools: 19 pools, 101 pgs + objects: 354 objects, 260 MB + usage: 77807 MB used, 70106 MB / 144 GB avail + pgs: 101 active+clean + + io: + client: 48769 B/s wr, 0 op/s rd, 12 op/s wr + + +- Ceph cluster is in HEALTH_OK state with 3 MONs and 3 OSDs. + + +Step 2: Run validation script +============================= + +.. note:: + Exec into ceph mon pod and execute the validation script by giving pool name as + first arugment, as shown below rbd is the pool name . + +.. code-block:: console + + ubuntu@mnode1:/opt/openstack-helm$ /tmp/checkObjectReplication.py rbd + Test object got replicated on these osds: [1, 0, 2] + Test object got replicated on these hosts: [u'mnode1', u'mnode2', u'mnode3'] + Hosts hosting multiple copies of a placement groups are:[] + +- If there are any objects replicated on same host then we will see them in the last + line of the script output