Add datasource support

We now support the ability to create a datasource using yaml files.

Change-Id: I1db38ac25bc309398924c15635ea5dee4eaf264c
Signed-off-by: Paul Belanger <pabelanger@redhat.com>

doc/source/api.rst

@@ -7,6 +7,10 @@ API Reference
     :members:
     :undoc-members:
 
+.. automodule:: grafana_dashboards.grafana.datasource
+    :members:
+    :undoc-members:
+
 .. automodule:: grafana_dashboards.grafana.dashboard
     :members:
     :undoc-members:

grafana_dashboards/builder.py

@@ -33,13 +33,14 @@ class Builder(object):
             key=config.get('grafana', 'apikey'))
         self.parser = YamlParser()
 
-    def delete_dashboard(self, path):
+    def delete(self, path):
         self.load_files(path)
+        datasources = self.parser.data.get('datasource', {})
+        LOG.info('Number of datasources to be deleted: %d', len(datasources))
+        self._delete_datasource(datasources)
         dashboards = self.parser.data.get('dashboard', {})
-        for name in dashboards:
-            LOG.debug('Deleting grafana dashboard %s', name)
-            self.grafana.dashboard.delete(name)
-            self.cache.set(name, '')
+        LOG.info('Number of dashboards to be deleted: %d', len(dashboards))
+        self._delete_dashboard(dashboards)
 
     def load_files(self, path):
         files_to_process = []
@@ -54,14 +55,49 @@ class Builder(object):
         for fn in files_to_process:
             self.parser.parse(fn)
 
-    def update_dashboard(self, path):
+    def update(self, path):
         self.load_files(path)
+        datasources = self.parser.data.get('datasource', {})
+        LOG.info('Number of datasources to be updated: %d', len(datasources))
+        self._update_datasource(datasources)
         dashboards = self.parser.data.get('dashboard', {})
-        LOG.info('Number of dashboards generated: %d', len(dashboards))
-        for name in dashboards:
+        LOG.info('Number of dashboards to be updated: %d', len(dashboards))
+        self._update_dashboard(dashboards)
+
+    def _delete_dashboard(self, data):
+        for name in data:
+            LOG.debug('Deleting grafana dashboard %s', name)
+            self.grafana.dashboard.delete(name)
+            self.cache.set(name, '')
+
+    def _delete_datasource(self, data):
+        for name in data:
+            LOG.debug('Deleting grafana datasource %s', name)
+            datasource_id = self.grafana.datasource.is_datasource(name)
+            if datasource_id:
+                self.grafana.datasource.delete(datasource_id)
+                self.cache.set(name, '')
+
+    def _update_dashboard(self, data):
+        for name in data:
             data, md5 = self.parser.get_dashboard(name)
             if self.cache.has_changed(name, md5):
                 self.grafana.dashboard.create(name, data, overwrite=True)
                 self.cache.set(name, md5)
             else:
                 LOG.debug("'%s' has not changed" % name)
+
+    def _update_datasource(self, data):
+        for name in data:
+            data, md5 = self.parser.get_datasource(name)
+            if self.cache.has_changed(name, md5):
+                # Check for existing datasource so we can find the
+                # datasource_id.
+                datasource_id = self.grafana.datasource.is_datasource(name)
+                if datasource_id:
+                    self.grafana.datasource.update(datasource_id, data)
+                else:
+                    self.grafana.datasource.create(name, data)
+                self.cache.set(name, md5)
+            else:
+                LOG.debug("'%s' has not changed" % name)

grafana_dashboards/cmd.py

@@ -27,9 +27,9 @@ LOG = logging.getLogger(__name__)
 class Client(object):
 
     def delete(self):
-        LOG.info('Deleting dashboards in %s', self.args.path)
+        LOG.info('Deleting schema in %s', self.args.path)
         builder = Builder(self.config)
-        builder.delete_dashboard(self.args.path)
+        builder.delete(self.args.path)
 
     def main(self):
         self.parse_arguments()
@@ -90,12 +90,12 @@ class Client(object):
             logging.basicConfig(level=logging.INFO)
 
     def update(self):
-        LOG.info('Updating dashboards in %s', self.args.path)
+        LOG.info('Updating schema in %s', self.args.path)
         builder = Builder(self.config)
-        builder.update_dashboard(self.args.path)
+        builder.update(self.args.path)
 
     def validate(self):
-        LOG.info('Validating dashboards in %s', self.args.path)
+        LOG.info('Validating schema in %s', self.args.path)
         # NOTE(pabelanger): Disable caching support by default, in an effort
         # to improve performance.
         self.config.set('cache', 'enabled', 'false')

grafana_dashboards/grafana/__init__.py

@@ -12,14 +12,10 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-try:
-    from urllib.parse import urljoin
-except ImportError:
-    from urlparse import urljoin
-
 import requests
 
 from grafana_dashboards.grafana.dashboard import Dashboard
+from grafana_dashboards.grafana.datasource import Datasource
 
 
 class Grafana(object):
@@ -36,7 +32,6 @@ class Grafana(object):
         self.server = url
         self.auth = None
 
-        base_url = urljoin(self.server, 'api/dashboards/db/')
         session = requests.Session()
         session.headers.update({
             'Content-Type': 'application/json',
@@ -47,4 +42,5 @@ class Grafana(object):
             self.auth = {'Authorization': 'Bearer %s' % key}
             session.headers.update(self.auth)
 
-        self.dashboard = Dashboard(base_url, session)
+        self.dashboard = Dashboard(self.server, session)
+        self.datasource = Datasource(self.server, session)

grafana_dashboards/grafana/dashboard.py

@@ -14,18 +14,15 @@
 
 import json
 
-try:
-    from urllib.parse import urljoin
-except ImportError:
-    from urlparse import urljoin
-
 from requests import exceptions
 
+from grafana_dashboards.grafana import utils
+
 
 class Dashboard(object):
 
     def __init__(self, url, session):
-        self.url = url
+        self.url = utils.urljoin(url, 'api/dashboards/db/')
         self.session = session
 
     def create(self, name, data, overwrite=False):
@@ -64,7 +61,7 @@ class Dashboard(object):
         :raises Exception: if dashboard failed to delete
 
         """
-        url = urljoin(self.url, name)
+        url = utils.urljoin(self.url, name)
         self.session.delete(url)
         if self.is_dashboard(name):
             raise Exception('dashboard[%s] failed to delete' % name)
@@ -78,7 +75,7 @@ class Dashboard(object):
         :rtype: dict or None
 
         """
-        url = urljoin(self.url, name)
+        url = utils.urljoin(self.url, name)
         try:
             res = self.session.get(url)
             res.raise_for_status()

grafana_dashboards/grafana/datasource.py

@@ -0,0 +1,127 @@
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+
+from requests import exceptions
+
+from grafana_dashboards.grafana import utils
+
+
+class Datasource(object):
+
+    def __init__(self, url, session):
+        self.url = utils.urljoin(url, 'api/datasources/')
+        self.session = session
+
+    def create(self, name, data):
+        """Create a new datasource
+
+        :param name: URL friendly title of the datasource
+        :type name: str
+        :param data: Datasource model
+        :type data: dict
+
+        :raises Exception: if datasource already exists
+
+        """
+        if self.is_datasource(name):
+            raise Exception('datasource[%s] already exists' % name)
+
+        res = self.session.post(
+            self.url, data=json.dumps(data))
+
+        res.raise_for_status()
+        return res.json()
+
+    def delete(self, datasource_id):
+        """Delete a datasource
+
+        :param datasource_id: Id number of datasource
+        :type datasource_id: int
+
+        :raises Exception: if datasource failed to delete
+
+        """
+        url = utils.urljoin(self.url, str(datasource_id))
+        self.session.delete(url)
+        if self.get(datasource_id):
+            raise Exception('datasource[%s] failed to delete' % datasource_id)
+
+    def get(self, datasource_id):
+        """Get a datasource
+
+        :param datasource_id: Id number of datasource
+        :type datasource_id: int
+
+        :rtype: dict or None
+
+        """
+        url = utils.urljoin(self.url, str(datasource_id))
+        try:
+            res = self.session.get(url)
+            res.raise_for_status()
+        except exceptions.HTTPError:
+            return None
+
+        return res.json()
+
+    def get_all(self):
+        """List all datasource
+
+        :rtype: dict
+
+        """
+        res = self.session.get(self.url)
+        res.raise_for_status()
+
+        return res.json()
+
+    def is_datasource(self, name):
+        """Check if a datasource exists
+
+        :param name: URL friendly title of the dashboard
+        :type name: str
+
+        :returns: if datasource exists return id number.
+        :rtype: int
+
+        """
+        datasources = self.get_all()
+        for datasource in datasources:
+            if datasource['name'].lower() == name.lower():
+                return datasource['id']
+        return 0
+
+    def update(self, datasource_id, data):
+        """Update an existing datasource
+
+        :param datasource_id: URL friendly title of the dashboard
+        :type datasource_id: int
+        :param data: Datasource model
+        :type data: dict
+        :param overwrite: Overwrite existing dashboard with newer version or
+                          with the same dashboard title
+        :type overwrite: bool
+
+        :raises Exception: if datasource already exists
+
+        """
+        url = utils.urljoin(self.url, str(datasource_id))
+
+        res = self.session.put(
+            url, data=json.dumps(data))
+
+        res.raise_for_status()
+        return res.json()

grafana_dashboards/grafana/utils.py

@@ -0,0 +1,18 @@
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+try:
+    from urllib.parse import urljoin  # noqa
+except ImportError:
+    from urlparse import urljoin  # noqa

grafana_dashboards/parser.py

@@ -20,7 +20,9 @@ import yaml
 
 from slugify import slugify
 
-from grafana_dashboards.schema.dashboard import Dashboard
+from grafana_dashboards.schema import Schema
+
+LOG = logging.getLogger(__name__)
 
 LOG = logging.getLogger(__name__)
 
@@ -32,15 +34,18 @@ class YamlParser(object):
 
     def get_dashboard(self, slug):
         data = self.data.get('dashboard', {}).get(slug, None)
-        md5 = None
-        if data:
-            # Sort json keys to help our md5 hash are constant.
-            content = json.dumps(data, sort_keys=True)
-            md5 = hashlib.md5(content.encode('utf-8')).hexdigest()
+        md5 = self._generate_md5(data)
         LOG.debug('Dashboard %s: %s' % (slug, md5))
 
         return data, md5
 
+    def get_datasource(self, slug):
+        data = self.data.get('datasource', {}).get(slug, None)
+        md5 = self._generate_md5(data)
+        LOG.debug('Datasource %s: %s' % (slug, md5))
+
+        return data, md5
+
     def parse(self, fn):
         with io.open(fn, 'r', encoding='utf-8') as fp:
             self.parse_fp(fp)
@@ -51,15 +56,26 @@ class YamlParser(object):
         for item in result.items():
             group = self.data.get(item[0], {})
             # Create slug to make it easier to find dashboards.
-            title = item[1]['title']
-            slug = slugify(title)
+            if item[0] == 'dashboard':
+                name = item[1]['title']
+            else:
+                name = item[1]['name']
+            slug = slugify(name)
             if slug in group:
                 raise Exception(
-                    "Duplicate dashboard found in '{0}: '{1}' "
-                    "already defined".format(fp.name, title))
+                    "Duplicate {0} found in '{1}: '{2}' "
+                    "already defined".format(item[0], fp.name, name))
             group[slug] = item[1]
             self.data[item[0]] = group
 
     def validate(self, data):
-        schema = Dashboard()
+        schema = Schema()
         return schema.validate(data)
+
+    def _generate_md5(self, data):
+        md5 = None
+        if data:
+            # Sort json keys to help our md5 hash are constant.
+            content = json.dumps(data, sort_keys=True)
+            md5 = hashlib.md5(content.encode('utf-8')).hexdigest()
+        return md5

grafana_dashboards/schema/__init__.py

@@ -0,0 +1,32 @@
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import voluptuous as v
+
+from grafana_dashboards.schema.dashboard import Dashboard
+from grafana_dashboards.schema.datasource import Datasource
+
+
+class Schema(object):
+
+    def validate(self, data):
+        dashboard = Dashboard().get_schema()
+        datasource = Datasource().get_schema()
+
+        schema = v.Schema({
+            v.Optional('dashboard'): dashboard,
+            v.Optional('datasource'): datasource,
+        })
+
+        return schema(data)

grafana_dashboards/schema/dashboard.py

@@ -19,15 +19,11 @@ from grafana_dashboards.schema.row import Row
 
 class Dashboard(object):
 
-    def validate(self, data):
+    def get_schema(self):
         dashboard = {
             v.Required('title'): v.All(str, v.Length(min=1)),
             v.Optional('id'): int,
         }
         rows = Row().get_schema()
         dashboard.update(rows.schema)
-        schema = v.Schema({
-            v.Required('dashboard'): dashboard,
-        })
-
-        return schema(data)
+        return dashboard

grafana_dashboards/schema/datasource.py

@@ -0,0 +1,29 @@
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import voluptuous as v
+
+
+class Datasource(object):
+
+    def get_schema(self):
+        datasource = {
+            v.Required('access', default='direct'): v.Any('direct', 'proxy'),
+            v.Required('isDefault', default=False): v.All(bool),
+            v.Required('name'): v.All(str, v.Length(min=1)),
+            v.Required('type', default='graphite'): v.Any('graphite'),
+            v.Required('url'): v.All(str, v.Length(min=1)),
+            v.Optional('orgId'): int,
+        }
+        return datasource

tests/fixtures/builder/datasource-0001.yaml

@@ -0,0 +1,3 @@
+datasource:
+  name: Default
+  url: http://graphite.example.org:8080

tests/fixtures/builder/datasource-0002.yaml

@@ -0,0 +1,3 @@
+datasource:
+  name: Default
+  url: http://graphite.example.net:8080

tests/grafana/test_datasource.py

@@ -0,0 +1,99 @@
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import requests_mock
+from testtools import TestCase
+
+from grafana_dashboards.grafana import Grafana
+
+DATASOURCE001 = {
+    "id": 1,
+    "orgId": 1,
+    "name": "foobar",
+    "type": "graphite",
+    "access": "direct",
+    "url": "http://example.org:8080",
+    "password": "",
+    "user": "",
+    "database": "",
+    "basicAuth": False,
+    "basicAuthUser": "",
+    "basicAuthPassword": "",
+    "isDefault": True,
+    "jsonData": None,
+}
+
+DATASOURCE_NOT_FOUND = {
+    "message": "Failed to query datasources"
+}
+
+
+class TestCaseDatasource(TestCase):
+
+    def setUp(self):
+        super(TestCaseDatasource, self).setUp()
+        self.url = 'http://localhost'
+        self.grafana = Grafana(self.url)
+
+    @requests_mock.Mocker()
+    def test_create_new(self, mock_requests):
+        mock_requests.post('/api/datasources/', json=DATASOURCE001)
+        mock_requests.get('/api/datasources/', json=[])
+        res = self.grafana.datasource.create('foobar', DATASOURCE001)
+        self.assertEqual(res, DATASOURCE001)
+
+    @requests_mock.Mocker()
+    def test_get_not_found(self, mock_requests):
+        mock_requests.get(
+            '/api/datasources/1', json=DATASOURCE_NOT_FOUND,
+            status_code=404)
+        res = self.grafana.datasource.get(1)
+        self.assertEqual(res, None)
+
+    @requests_mock.Mocker()
+    def test_get_success(self, mock_requests):
+        mock_requests.get('/api/datasources/1', json=DATASOURCE001)
+        res = self.grafana.datasource.get(1)
+        self.assertEqual(res, DATASOURCE001)
+
+    @requests_mock.Mocker()
+    def test_get_all(self, mock_requests):
+        mock_requests.get(
+            '/api/datasources/', json=[DATASOURCE001])
+        res = self.grafana.datasource.get_all()
+        self.assertEqual(res, [DATASOURCE001])
+
+    @requests_mock.Mocker()
+    def test_get_all_empty(self, mock_requests):
+        mock_requests.get('/api/datasources/', json=[])
+        res = self.grafana.datasource.get_all()
+        self.assertEqual(res, [])
+
+    @requests_mock.Mocker()
+    def test_is_datasource_empty(self, mock_requests):
+        mock_requests.get('/api/datasources/', json=[])
+        res = self.grafana.datasource.is_datasource('foobar')
+        self.assertFalse(res)
+
+    @requests_mock.Mocker()
+    def test_is_datasource_false(self, mock_requests):
+        mock_requests.get('/api/datasources/', json=[DATASOURCE001])
+        res = self.grafana.datasource.is_datasource('new')
+        self.assertFalse(res)
+
+    @requests_mock.Mocker()
+    def test_is_datasource_true(self, mock_requests):
+        mock_requests.get('/api/datasources/', json=[DATASOURCE001])
+        res = self.grafana.datasource.is_datasource('foobar')
+        self.assertTrue(res)

tests/schema/fixtures/datasource-0001.json

@@ -0,0 +1,11 @@
+{
+    "datasource": {
+        "new-datasource": {
+            "access": "direct",
+            "isDefault": false,
+            "name": "New datasource",
+            "type": "graphite",
+            "url": "http://example.org"
+        }
+    }
+}

tests/schema/fixtures/datasource-0001.yaml

@@ -0,0 +1,3 @@
+datasource:
+  name: New datasource
+  url: http://example.org

tests/schema/test_schema.py

@@ -21,6 +21,6 @@ from tests.base import get_scenarios
 from tests.schema.base import TestCase as BaseTestCase
 
 
-class TestCaseSchemaDashboard(TestWithScenarios, TestCase, BaseTestCase):
+class TestCaseSchema(TestWithScenarios, TestCase, BaseTestCase):
     fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
     scenarios = get_scenarios(fixtures_path)

tests/test_builder.py

@@ -36,7 +36,7 @@ class TestCaseBuilder(TestCase):
         # Create a new builder to avoid duplicate dashboards.
         builder2 = builder.Builder(self.config)
         # Delete same dashboard, ensure we delete it from grafana.
-        builder2.delete_dashboard(path)
+        builder2.delete(path)
         self.assertEqual(mock_grafana.call_count, 1)
 
     def test_grafana_defaults(self):
@@ -54,11 +54,56 @@ class TestCaseBuilder(TestCase):
         # Create a new builder to avoid duplicate dashboards.
         builder2 = builder.Builder(self.config)
         # Update again with same dashboard, ensure we don't update grafana.
-        builder2.update_dashboard(path)
+        builder2.update(path)
         self.assertEqual(mock_grafana.call_count, 0)
 
+    @mock.patch('grafana_dashboards.grafana.Datasource.create')
+    def test_create_datasource(self, mock_grafana):
+        path = os.path.join(
+            os.path.dirname(__file__), 'fixtures/builder/datasource-0001.yaml')
+
+        # Create a datasource.
+        self._create_datasource(path)
+        # Create a new builder to avoid duplicate datasources.
+        builder2 = builder.Builder(self.config)
+        # Update again with same datasource, ensure we don't update grafana.
+        builder2.update(path)
+        self.assertEqual(mock_grafana.call_count, 0)
+
+    @mock.patch(
+        'grafana_dashboards.grafana.Datasource.is_datasource',
+        return_value=True)
+    @mock.patch('grafana_dashboards.grafana.Datasource.update')
+    def test_update_datasource(self, mock_is_datasource, mock_update):
+        path = os.path.join(
+            os.path.dirname(__file__), 'fixtures/builder/datasource-0001.yaml')
+
+        # Create a datasource.
+        self._create_datasource(path)
+        # Create a new builder to avoid duplicate datasources.
+        builder2 = builder.Builder(self.config)
+
+        # Same datasource name, different content.
+        path = os.path.join(
+            os.path.dirname(__file__), 'fixtures/builder/datasource-0002.yaml')
+
+        # Update again with same datasource, ensure we update grafana.
+        builder2.update(path)
+        self.assertEqual(mock_is_datasource.call_count, 1)
+        self.assertEqual(mock_update.call_count, 1)
+
     @mock.patch('grafana_dashboards.grafana.Dashboard.create')
-    def _update_dashboard(self, path, mock_grafana):
-        self.builder.update_dashboard(path)
+    def _update_dashboard(self, path, mock_create):
+        self.builder.update(path)
         # Cache is empty, so we should update grafana.
-        self.assertEqual(mock_grafana.call_count, 1)
+        self.assertEqual(mock_create.call_count, 1)
+
+    @mock.patch(
+        'grafana_dashboards.grafana.Datasource.is_datasource',
+        return_value=False)
+    @mock.patch('grafana_dashboards.grafana.Datasource.create')
+    def _create_datasource(self, path, mock_is_datasource, mock_create):
+        self.builder.update(path)
+        # Cache is empty, so we should update grafana.
+        self.assertEqual(mock_is_datasource.call_count, 1)
+        self.assertEqual(mock_create.call_count, 1)