Fix for invalid literal ValueError parsing ipv6 url(s)

Switch to using network_utils for splitting the URL. The code
in oslo-incubator supports ipv6 urls

HEAD of oslo-incubator is bb52a3fc49f033b9f36238231ca56e754a78cf4b

Updated openstack-common.conf to pick up the new dependency from
oslo-incubator

Change-Id: Ifa3dec384e85942a191260d17e8141030d31ff84
Closes-Bug: #1298137

novaclient/client.py

@@ -35,6 +35,7 @@ from six.moves.urllib import parse
 
 from novaclient import exceptions
 from novaclient.openstack.common.gettextutils import _
+from novaclient.openstack.common import network_utils
 from novaclient import service_catalog
 from novaclient import utils
 
@@ -355,7 +356,7 @@ class HTTPClient(object):
                                              extract_token=False)
 
     def authenticate(self):
-        magic_tuple = parse.urlsplit(self.auth_url)
+        magic_tuple = network_utils.urlsplit(self.auth_url)
         scheme, netloc, path, query, frag = magic_tuple
         port = magic_tuple.port
         if port is None:

novaclient/openstack/common/network_utils.py

@@ -0,0 +1,108 @@
+# Copyright 2012 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Network-related utilities and helper functions.
+"""
+
+# TODO(jd) Use six.moves once
+# https://bitbucket.org/gutworth/six/pull-request/28
+# is merged
+try:
+    import urllib.parse
+    SplitResult = urllib.parse.SplitResult
+except ImportError:
+    import urlparse
+    SplitResult = urlparse.SplitResult
+
+from six.moves.urllib import parse
+
+
+def parse_host_port(address, default_port=None):
+    """Interpret a string as a host:port pair.
+
+    An IPv6 address MUST be escaped if accompanied by a port,
+    because otherwise ambiguity ensues: 2001:db8:85a3::8a2e:370:7334
+    means both [2001:db8:85a3::8a2e:370:7334] and
+    [2001:db8:85a3::8a2e:370]:7334.
+
+    >>> parse_host_port('server01:80')
+    ('server01', 80)
+    >>> parse_host_port('server01')
+    ('server01', None)
+    >>> parse_host_port('server01', default_port=1234)
+    ('server01', 1234)
+    >>> parse_host_port('[::1]:80')
+    ('::1', 80)
+    >>> parse_host_port('[::1]')
+    ('::1', None)
+    >>> parse_host_port('[::1]', default_port=1234)
+    ('::1', 1234)
+    >>> parse_host_port('2001:db8:85a3::8a2e:370:7334', default_port=1234)
+    ('2001:db8:85a3::8a2e:370:7334', 1234)
+
+    """
+    if address[0] == '[':
+        # Escaped ipv6
+        _host, _port = address[1:].split(']')
+        host = _host
+        if ':' in _port:
+            port = _port.split(':')[1]
+        else:
+            port = default_port
+    else:
+        if address.count(':') == 1:
+            host, port = address.split(':')
+        else:
+            # 0 means ipv4, >1 means ipv6.
+            # We prohibit unescaped ipv6 addresses with port.
+            host = address
+            port = default_port
+
+    return (host, None if port is None else int(port))
+
+
+class ModifiedSplitResult(SplitResult):
+    """Split results class for urlsplit."""
+
+    # NOTE(dims): The functions below are needed for Python 2.6.x.
+    # We can remove these when we drop support for 2.6.x.
+    @property
+    def hostname(self):
+        netloc = self.netloc.split('@', 1)[-1]
+        host, port = parse_host_port(netloc)
+        return host
+
+    @property
+    def port(self):
+        netloc = self.netloc.split('@', 1)[-1]
+        host, port = parse_host_port(netloc)
+        return port
+
+
+def urlsplit(url, scheme='', allow_fragments=True):
+    """Parse a URL using urlparse.urlsplit(), splitting query and fragments.
+    This function papers over Python issue9374 when needed.
+
+    The parameters are the same as urlparse.urlsplit.
+    """
+    scheme, netloc, path, query, fragment = parse.urlsplit(
+        url, scheme, allow_fragments)
+    if allow_fragments and '#' in path:
+        path, fragment = path.split('#', 1)
+    if '?' in path:
+        path, query = path.split('?', 1)
+    return ModifiedSplitResult(scheme, netloc,
+                               path, query, fragment)

novaclient/tests/test_auth_plugins.py

@@ -96,7 +96,7 @@ class DeprecatedAuthPluginTest(utils.TestCase):
         def test_auth_call():
             plugin = auth_plugin.DeprecatedAuthPlugin("fake")
             cs = client.Client("username", "password", "project_id",
-                               "auth_url/v2.0", auth_system="fake",
+                               utils.AUTH_URL_V2, auth_system="fake",
                                auth_plugin=plugin)
             cs.client.authenticate()
 
@@ -126,7 +126,7 @@ class DeprecatedAuthPluginTest(utils.TestCase):
             auth_plugin.discover_auth_systems()
             plugin = auth_plugin.DeprecatedAuthPlugin("notexists")
             cs = client.Client("username", "password", "project_id",
-                               "auth_url/v2.0", auth_system="notexists",
+                               utils.AUTH_URL_V2, auth_system="notexists",
                                auth_plugin=plugin)
             self.assertRaises(exceptions.AuthSystemNotFound,
                               cs.client.authenticate)
@@ -217,7 +217,7 @@ class AuthPluginTest(utils.TestCase):
         auth_plugin.discover_auth_systems()
         plugin = auth_plugin.load_plugin("fake")
         cs = client.Client("username", "password", "project_id",
-                           "auth_url/v2.0", auth_system="fake",
+                           utils.AUTH_URL_V2, auth_system="fake",
                            auth_plugin=plugin)
         cs.client.authenticate()
 

novaclient/tests/test_http.py

@@ -47,7 +47,8 @@ unknown_error_mock_request = mock.Mock(return_value=unknown_error_response)
 
 def get_client():
     cl = client.HTTPClient("username", "password",
-                           "project_id", "auth_test")
+                           "project_id",
+                           utils.AUTH_URL_V2)
     return cl
 
 

novaclient/tests/utils.py

@@ -17,6 +17,10 @@ import fixtures
 import requests
 import testtools
 
+AUTH_URL = "http://localhost:5002/auth_url"
+AUTH_URL_V1 = "http://localhost:5002/auth_url/v1.0"
+AUTH_URL_V2 = "http://localhost:5002/auth_url/v2.0"
+
 
 class TestCase(testtools.TestCase):
     TEST_REQUEST_BASE = {

novaclient/tests/v1_1/test_auth.py

@@ -25,7 +25,7 @@ from novaclient.v1_1 import client
 class AuthenticateAgainstKeystoneTests(utils.TestCase):
     def test_authenticate_success(self):
         cs = client.Client("username", "password", "project_id",
-                           "auth_url/v2.0", service_type='compute')
+                           utils.AUTH_URL_V2, service_type='compute')
         resp = {
             "access": {
                 "token": {
@@ -94,7 +94,7 @@ class AuthenticateAgainstKeystoneTests(utils.TestCase):
 
     def test_authenticate_failure(self):
         cs = client.Client("username", "password", "project_id",
-                           "auth_url/v2.0")
+                           utils.AUTH_URL_V2)
         resp = {"unauthorized": {"message": "Unauthorized", "code": "401"}}
         auth_response = utils.TestResponse({
             "status_code": 401,
@@ -111,7 +111,7 @@ class AuthenticateAgainstKeystoneTests(utils.TestCase):
 
     def test_v1_auth_redirect(self):
         cs = client.Client("username", "password", "project_id",
-                           "auth_url/v1.0", service_type='compute')
+                           utils.AUTH_URL_V1, service_type='compute')
         dict_correct_response = {
             "access": {
                 "token": {
@@ -199,7 +199,7 @@ class AuthenticateAgainstKeystoneTests(utils.TestCase):
 
     def test_v2_auth_redirect(self):
         cs = client.Client("username", "password", "project_id",
-                           "auth_url/v2.0", service_type='compute')
+                           utils.AUTH_URL_V2, service_type='compute')
         dict_correct_response = {
             "access": {
                 "token": {
@@ -287,7 +287,7 @@ class AuthenticateAgainstKeystoneTests(utils.TestCase):
 
     def test_ambiguous_endpoints(self):
         cs = client.Client("username", "password", "project_id",
-                           "auth_url/v2.0", service_type='compute')
+                           utils.AUTH_URL_V2, service_type='compute')
         resp = {
             "access": {
                 "token": {
@@ -340,7 +340,7 @@ class AuthenticateAgainstKeystoneTests(utils.TestCase):
 
     def test_authenticate_with_token_success(self):
         cs = client.Client("username", None, "project_id",
-                           "auth_url/v2.0", service_type='compute')
+                           utils.AUTH_URL_V2, service_type='compute')
         cs.client.auth_token = "FAKE_ID"
         resp = {
             "access": {
@@ -405,7 +405,7 @@ class AuthenticateAgainstKeystoneTests(utils.TestCase):
             self.assertEqual(cs.client.auth_token, token_id)
 
     def test_authenticate_with_token_failure(self):
-        cs = client.Client("username", None, "project_id", "auth_url/v2.0")
+        cs = client.Client("username", None, "project_id", utils.AUTH_URL_V2)
         cs.client.auth_token = "FAKE_ID"
         resp = {"unauthorized": {"message": "Unauthorized", "code": "401"}}
         auth_response = utils.TestResponse({
@@ -421,7 +421,8 @@ class AuthenticateAgainstKeystoneTests(utils.TestCase):
 
 class AuthenticationTests(utils.TestCase):
     def test_authenticate_success(self):
-        cs = client.Client("username", "password", "project_id", "auth_url")
+        cs = client.Client("username", "password",
+                           "project_id", utils.AUTH_URL)
         management_url = 'https://localhost/v1.1/443470'
         auth_response = utils.TestResponse({
             'status_code': 204,
@@ -456,7 +457,8 @@ class AuthenticationTests(utils.TestCase):
         test_auth_call()
 
     def test_authenticate_failure(self):
-        cs = client.Client("username", "password", "project_id", "auth_url")
+        cs = client.Client("username", "password",
+                           "project_id", utils.AUTH_URL)
         auth_response = utils.TestResponse({'status_code': 401})
         mock_request = mock.Mock(return_value=(auth_response))
 
@@ -467,7 +469,8 @@ class AuthenticationTests(utils.TestCase):
         test_auth_call()
 
     def test_auth_automatic(self):
-        cs = client.Client("username", "password", "project_id", "auth_url")
+        cs = client.Client("username", "password",
+                           "project_id", utils.AUTH_URL)
         http_client = cs.client
         http_client.management_url = ''
         mock_request = mock.Mock(return_value=(None, None))
@@ -482,7 +485,8 @@ class AuthenticationTests(utils.TestCase):
         test_auth_call()
 
     def test_auth_manual(self):
-        cs = client.Client("username", "password", "project_id", "auth_url")
+        cs = client.Client("username", "password",
+                           "project_id", utils.AUTH_URL)
 
         @mock.patch.object(cs.client, 'authenticate')
         def test_auth_call(m):

openstack-common.conf

@@ -9,6 +9,7 @@ module=uuidutils
 module=apiclient
 module=importutils
 module=cliutils
+module=network_utils
 
 # The base module to hold the copy of openstack.common
 base=novaclient