From bf760189f290dc1533c9c6fd5a7965a41e5e811e Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 26 Apr 2019 10:58:31 +0100
Subject: [PATCH] Improve detection of base path

On Ubuntu sys.prefix is set to '/usr' even though pip will install
packages to '/usr/local' when not using a virtualenv. This change fixes
the detection in this instance. Non standard install locations are not
currently supported.

Change-Id: I214e11e7d099d1b39041fdc6b91002e1929d9c00
Story: 2005510
Task: 30620
---
 doc/source/configuration/kayobe.rst           | 22 ++++++++++++
 kayobe/tests/unit/test_utils.py               |  7 ++++
 kayobe/utils.py                               | 35 ++++++++++++++++---
 ...-detection-on-ubuntu-dfa8e4e7a1d3ea27.yaml | 10 ++++++
 4 files changed, 70 insertions(+), 4 deletions(-)
 create mode 100644 releasenotes/notes/fix-data-files-path-detection-on-ubuntu-dfa8e4e7a1d3ea27.yaml

diff --git a/doc/source/configuration/kayobe.rst b/doc/source/configuration/kayobe.rst
index 19276d6d3..82788e513 100644
--- a/doc/source/configuration/kayobe.rst
+++ b/doc/source/configuration/kayobe.rst
@@ -116,3 +116,25 @@ configuration files may be encrypted.  Since encryption can make working with
 Kayobe difficult, it is recommended to follow `best practice
 <http://docs.ansible.com/ansible/playbooks_best_practices.html#best-practices-for-variables-and-vaults>`_,
 adding a layer of indirection and using encryption only where necessary.
+
+Location of data files
+----------------------
+
+Kayobe needs to know where to find any files not contained within its python package;
+this includes its Ansible playbooks and any other files it needs for runtime operation.
+These files are known collectively as 'data files'.
+
+Kayobe will attempt to detect the location of its data files automatically. However, if
+you have installed kayobe to a non-standard location this auto-detection may fail.
+It is possible to manually override the path using the environment variable:
+``KAYOBE_DATA_FILES_PATH``. This should be set to a path with the following structure::
+
+    requirements.yml
+    ansible/
+        roles/
+            ...
+        ...
+
+Where ``ansible`` is the ``ansible`` directory from the source checkout and ``...``
+is an elided representation of any files and subdirectories contained within
+that directory.
diff --git a/kayobe/tests/unit/test_utils.py b/kayobe/tests/unit/test_utils.py
index d46838e20..02659987d 100644
--- a/kayobe/tests/unit/test_utils.py
+++ b/kayobe/tests/unit/test_utils.py
@@ -12,6 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import os
 import subprocess
 import unittest
 
@@ -132,3 +133,9 @@ key2: value2
         value = "string to escape"
         expected = "{{'c3RyaW5nIHRvIGVzY2FwZQ==' | b64decode }}"
         self.assertEqual(expected, utils.escape_jinja(value))
+
+    def test_detect_install_prefix(self):
+        path = "/tmp/test/local/lib/python2.7/dist-packages"
+        expected = os.path.normpath("/tmp/test/local/")
+        result = utils._detect_install_prefix(path)
+        self.assertEqual(expected, os.path.normpath(result))
diff --git a/kayobe/utils.py b/kayobe/utils.py
index b5f45b717..60df582f3 100644
--- a/kayobe/utils.py
+++ b/kayobe/utils.py
@@ -14,6 +14,7 @@
 
 import base64
 import glob
+import itertools
 import logging
 import os
 import six
@@ -25,13 +26,33 @@ import yaml
 
 LOG = logging.getLogger(__name__)
 
-_BASE_PATH = os.path.join(sys.prefix, "share", "kayobe")
-
 
 def get_data_files_path(*relative_path):
     """Given a relative path to a data file, return the absolute path"""
     # Detect editable pip install / python setup.py develop and use a path
     # relative to the source directory
+    return os.path.join(_get_base_path(), *relative_path)
+
+
+def _detect_install_prefix(path):
+    script_path = os.path.realpath(path)
+    script_path = os.path.normpath(script_path)
+    components = script_path.split(os.sep)
+    # use heuristic: anything before 'lib' in path is the prefix
+    if 'lib' not in components:
+        return None
+    prefix = itertools.takewhile(
+        lambda x: x != "lib",
+        components
+    )
+    prefix_path = os.sep.join(prefix)
+    return prefix_path
+
+
+def _get_base_path():
+    override = os.environ.get("KAYOBE_DATA_FILES_PATH")
+    if override:
+        return os.path.join(override)
     egg_glob = os.path.join(
         sys.prefix, 'lib*', 'python*', '*-packages', 'kayobe.egg-link'
     )
@@ -39,8 +60,14 @@ def get_data_files_path(*relative_path):
     if egg_link:
         with open(egg_link[0], "r") as f:
             realpath = f.readline().strip()
-        return os.path.join(realpath, *relative_path)
-    return os.path.join(_BASE_PATH, *relative_path)
+        return os.path.join(realpath)
+
+    prefix = _detect_install_prefix(__file__)
+    if prefix:
+        return os.path.join(prefix, "share", "kayobe")
+
+    # Assume uninstalled
+    return os.path.join(os.path.realpath(__file__), "..")
 
 
 def yum_install(packages):
diff --git a/releasenotes/notes/fix-data-files-path-detection-on-ubuntu-dfa8e4e7a1d3ea27.yaml b/releasenotes/notes/fix-data-files-path-detection-on-ubuntu-dfa8e4e7a1d3ea27.yaml
new file mode 100644
index 000000000..7bd870679
--- /dev/null
+++ b/releasenotes/notes/fix-data-files-path-detection-on-ubuntu-dfa8e4e7a1d3ea27.yaml
@@ -0,0 +1,10 @@
+---
+fixes:
+  - |
+    Fixes the detection of kayobe's data files on Ubuntu when not installing into a
+    virtualenv, see `Story 2005510 <https://storyboard.openstack.org/#!/story/2005510>`_
+    for details.
+features:
+  - |
+    Added the ``KAYOBE_DATA_FILES_PATH`` environment variable to override the
+    auto-detection of the data files path. See the `documentation <https://docs.openstack.org/kayobe/latest/configuration/kayobe.html#location-of-data-files>`_ for more details.