Encode unicode from JSON before using it as a string.

Right now this code fails when used with a JSON
decoder that always produces unicode. This isn't
usually the case with CPython, where simplejson
is used most of the time, however with the stdlib
JSON library (as used on PyPy), this code fails.

Change-Id: Ib2343243f40194d5b2784551a807c7f58970a6e9

swift/common/middleware/staticweb.py

@@ -125,13 +125,17 @@ from swift.common.swob import Response, HTTPMovedPermanently, HTTPNotFound
 from swift.proxy.controllers.base import get_container_info
 
 
+def ensure_utf8_bytes(value):
+    if isinstance(value, unicode):
+        value = value.encode('utf-8')
+    return value
+
+
 def quote(value, safe='/'):
     """
     Patched version of urllib.quote that encodes utf-8 strings before quoting
     """
-    if isinstance(value, unicode):
+    return urllib_quote(ensure_utf8_bytes(value), safe)
-        value = value.encode('utf-8')
-    return urllib_quote(value, safe)
 
 
 class _StaticWebContext(WSGIContext):
@@ -272,10 +276,7 @@ class _StaticWebContext(WSGIContext):
                     '   </tr>\n'
         for item in listing:
             if 'subdir' in item:
-                if isinstance(item['subdir'], unicode):
+                subdir = ensure_utf8_bytes(item['subdir'])
-                    subdir = item['subdir'].encode('utf-8')
-                else:
-                    subdir = item['subdir']
                 if prefix:
                     subdir = subdir[len(prefix):]
                 body += '   <tr class="item subdir">\n' \
@@ -286,23 +287,22 @@ class _StaticWebContext(WSGIContext):
                         (quote(subdir), cgi.escape(subdir))
         for item in listing:
             if 'name' in item:
-                if isinstance(item['name'], unicode):
+                name = ensure_utf8_bytes(item['name'])
-                    name = item['name'].encode('utf-8')
-                else:
-                    name = item['name']
                 if prefix:
                     name = name[len(prefix):]
+                content_type = ensure_utf8_bytes(item['content_type'])
+                bytes = ensure_utf8_bytes(human_readable(item['bytes']))
+                last_modified = (cgi.escape(item['last_modified']).
+                                 split('.')[0].replace('T', ' '))
                 body += '   <tr class="item %s">\n' \
                         '    <td class="colname"><a href="%s">%s</a></td>\n' \
                         '    <td class="colsize">%s</td>\n' \
                         '    <td class="coldate">%s</td>\n' \
                         '   </tr>\n' % \
                         (' '.join('type-' + cgi.escape(t.lower(), quote=True)
-                                  for t in item['content_type'].split('/')),
+                                  for t in content_type.split('/')),
                          quote(name), cgi.escape(name),
-                         human_readable(item['bytes']),
+                         bytes, ensure_utf8_bytes(last_modified))
-                         cgi.escape(item['last_modified']).split('.')[0].
-                            replace('T', ' '))
         body += '  </table>\n' \
                 ' </body>\n' \
                 '</html>\n'

test/unit/common/middleware/test_staticweb.py

@@ -17,8 +17,11 @@ try:
     import simplejson as json
 except ImportError:
     import json
+import json as stdlib_json
 import unittest
 
+import mock
+
 from swift.common.swob import Request, Response
 from swift.common.middleware import staticweb
 
@@ -659,12 +662,30 @@ class TestStaticWeb(unittest.TestCase):
                 self.test_staticweb)
         self.assertEquals(resp.status_int, 200)
 
+
     def test_container12unredirectedrequest(self):
         resp = Request.blank('/v1/a/c12/').get_response(
                 self.test_staticweb)
         self.assertEquals(resp.status_int, 200)
         self.assert_('index file' in resp.body)
 
+    def test_container_unicode_stdlib_json(self):
+        with mock.patch('swift.common.middleware.staticweb.json', new=stdlib_json):
+            resp = Request.blank(
+                '/v1/a/c10/').get_response(self.test_staticweb)
+            self.assertEquals(resp.status_int, 200)
+            self.assert_('Listing of /v1/a/c10/' in resp.body)
+            resp = Request.blank(
+                '/v1/a/c10/\xe2\x98\x83/').get_response(self.test_staticweb)
+            self.assertEquals(resp.status_int, 200)
+            self.assert_('Listing of /v1/a/c10/\xe2\x98\x83/' in resp.body)
+            resp = Request.blank(
+                '/v1/a/c10/\xe2\x98\x83/\xe2\x98\x83/'
+            ).get_response(self.test_staticweb)
+            self.assertEquals(resp.status_int, 200)
+            self.assert_(
+                'Listing of /v1/a/c10/\xe2\x98\x83/\xe2\x98\x83/' in resp.body)
+
     def test_subrequest_once_if_possible(self):
         resp = Request.blank(
                 '/v1/a/c4/one.txt').get_response(self.test_staticweb)