Support tests for Apache

Add support for functional tests that work with Apache web front end

Change-Id: I72358a12016eeccc842d834461dbebaa188aa117
Implements: blueprint wsgi-application-interface

doc/source/apache_deployment_guide.rst

@@ -0,0 +1,178 @@
+=======================
+Apache Deployment Guide
+=======================
+
+----------------------------
+Web Front End Considerations
+----------------------------
+
+Swift can be configured to work both using an integral web front-end
+and using a full-fledged Web Server such as the Apache2 (HTTPD) web server.
+The integral web front-end is a wsgi mini "Web Server" which opens
+up its own socket and serves http requests directly.
+The incoming requests accepted by the integral web front-end are then forwarded
+to a wsgi application (the core swift) for further handling, possibly
+via wsgi middleware sub-components.
+
+client<---->'integral web front-end'<---->middleware<---->'core swift'
+
+To gain full advantage of Apache2, Swift can alternatively be
+configured to work as a request processor of the Apache2 server.
+This alternative deployment scenario uses mod_wsgi of Apache2
+to forward requests to the swift wsgi application and middleware.
+
+client<---->'Apache2 with mod_wsgi'<----->middleware<---->'core swift'
+
+The integral web front-end offers simplicity and requires
+minimal configuration. It is also the web front-end most commonly used
+with Swift.
+Additionlly, the integral web front-end includes support for
+receiving chunked transfer encoding from a client,
+presently not supported by Apache2 in the operation mode described here.
+
+The use of Apache2 offers new ways to extend Swift and integrate it with
+existing authentication, administration and control systems.
+A single Apache2 server can serve as the web front end of any number of swift
+servers residing on a swift node.
+For example when a storage node offers account, container and object services,
+a single Apache2 server can serve as the web front end of all three services.
+
+The apache variant described here was tested as part of an IBM research work.
+It was found that following tuning, the Apache2 offer generally equivalent
+performance to that offered by the integral web front-end.
+Alternative to Apache2, other web servers may be used, but were never tested.
+
+-------------
+Apache2 Setup
+-------------
+Both Apache2 and mod-wsgi needs to be installed on the system.
+Ubuntu comes with Apache2 installed. Install mod-wsgi using:: 
+
+    sudo apt-get install libapache2-mod-wsgi
+
+First, change the User and Group IDs of Apache2 to be those used by Swift.
+For example in /etc/apache2/envvars use::
+
+    export APACHE_RUN_USER=swift
+    export APACHE_RUN_GROUP=swift
+
+Create a directory for the Apache2 wsgi files::
+
+    sudo mkdir /var/www/swift
+
+Create a file for each service under /var/www/swift.
+
+For a proxy service create /var/www/swift/proxy-server.wsgi::
+
+    from swift.common.wsgi import init_request_processor
+    application, conf, logger, log_name = \
+        init_request_processor('/etc/swift/proxy-server.conf','proxy-server')
+
+For an account service create /var/www/swift/account-server.wsgi::
+
+    from swift.common.wsgi import init_request_processor
+    application, conf, logger, log_name = \
+        init_request_processor('/etc/swift/account-server.conf',
+                               'account-server')
+
+For an container service create /var/www/swift/container-server.wsgi::
+
+    from swift.common.wsgi import init_request_processor
+    application, conf, logger, log_name = \
+        init_request_processor('/etc/swift/container-server.conf',
+                              'container-server')
+
+For an object service create /var/www/swift/object-server.wsgi::
+
+    from swift.common.wsgi import init_request_processor
+    application, conf, logger, log_name = \
+        init_request_processor('/etc/swift/object-server.conf',
+                               'object-server')
+
+Create a /etc/apache2/conf.d/swift_wsgi.conf configuration file that will
+define a port and Virtual Host per each local service.
+For example an Apache2 serving as a web front end of a proxy service::
+
+    #Proxy
+    NameVirtualHost *:8080
+    Listen 8080
+    <VirtualHost *:8080>
+        ServerName proxy-server
+        LimitRequestBody 5368709122
+        WSGIDaemonProcess proxy-server processes=5 threads=1
+        WSGIProcessGroup proxy-server
+        WSGIScriptAlias / /var/www/swift/proxy-server.wsgi
+        LimitRequestFields 200
+        ErrorLog /var/log/apache2/proxy-server
+        LogLevel debug
+        CustomLog /var/log/apache2/proxy.log combined
+    </VirtualHost>
+
+Notice that when using Apache the limit on the maximal object size should
+be imposed by Apache using the LimitRequestBody rather by the swift proxy.
+Note also that the LimitRequestBody should indicate the same value
+as indicated by max_file_size located in both 
+/etc/swift/swift.conf and in /etc/swift/test.conf.
+The Swift default value for max_file_size (when not present) is 5368709122.
+For example an Apache2 serving as a web front end of a storage node::
+
+    #Object Service
+    NameVirtualHost *:6000
+    Listen 6000
+    <VirtualHost *:6000>
+        ServerName object-server
+        WSGIDaemonProcess object-server processes=5 threads=1
+        WSGIProcessGroup object-server
+        WSGIScriptAlias / /var/www/swift/object-server.wsgi
+        LimitRequestFields 200
+        ErrorLog /var/log/apache2/object-server
+        LogLevel debug
+        CustomLog /var/log/apache2/access.log combined
+    </VirtualHost>
+
+    #Container Service
+    NameVirtualHost *:6001
+    Listen 6001
+    <VirtualHost *:6001>
+        ServerName container-server
+        WSGIDaemonProcess container-server processes=5 threads=1
+        WSGIProcessGroup container-server
+        WSGIScriptAlias / /var/www/swift/container-server.wsgi
+        LimitRequestFields 200
+        ErrorLog /var/log/apache2/container-server
+        LogLevel debug
+        CustomLog /var/log/apache2/access.log combined
+    </VirtualHost>
+
+    #Account Service
+    NameVirtualHost *:6002
+    Listen 6002
+    <VirtualHost *:6002>
+        ServerName account-server
+        WSGIDaemonProcess account-server processes=5 threads=1
+        WSGIProcessGroup account-server
+        WSGIScriptAlias / /var/www/swift/account-server.wsgi
+        LimitRequestFields 200
+        ErrorLog /var/log/apache2/account-server
+        LogLevel debug
+        CustomLog /var/log/apache2/access.log combined
+    </VirtualHost>
+
+Next stop the Apache2 and start it again (apache2ctl restart is not enough)::
+
+    apache2ctl stop
+    apache2ctl start
+
+Edit the tests config file and add::
+
+    web_front_end = apache2
+    normalized_urls = True
+
+Also check to see that the file includes max_file_size of the same value as
+used for the LimitRequestBody in the apache config file above.
+
+We are done.
+You may run functional tests to test - e.g.::
+
+    cd ~swift/swift
+    ./.functests

doc/source/deployment_guide.rst

@@ -51,6 +51,15 @@ Load balancing and network design is left as an exercise to the reader,
 but this is a very important part of the cluster, so time should be spent
 designing the network for a Swift cluster.
 
+
+---------------------
+Web Front End Options
+---------------------
+
+Swift comes with an integral web front end. However, it can also be deployed
+as a request processor of an Apache2 using mod_wsgi as described in 
+:doc:`Apache Deployment Guide <apache_deployment_guide>`.
+
 .. _ring-preparing:
 
 ------------------

test/functional/tests.py

@@ -67,6 +67,8 @@ for k in default_constraints:
         # tests.
         config[k] = '%s constraint is not defined' % k
 
+web_front_end = config.get('web_front_end', 'integral')
+normalized_urls = config.get('normalized_urls', False)
 
 def load_constraint(name):
     c = config[name]
@@ -220,7 +222,10 @@ class TestAccount(Base):
 
     def testInvalidPath(self):
         was_url = self.env.account.conn.storage_url
-        self.env.account.conn.storage_url = "/%s" % was_url
+        if (normalized_urls):
+            self.env.account.conn.storage_url = '/'
+        else:
+            self.env.account.conn.storage_url = "/%s" % was_url
         self.env.account.conn.make_request('GET')
         try:
             self.assert_status(404)
@@ -724,6 +729,7 @@ class TestContainerPathsEnv:
             'dir1/subdir+with{whatever/file D',
         ]
 
+        stored_files = set()
         for f in cls.files:
             file = cls.container.file(f)
             if f.endswith('/'):
@@ -731,6 +737,16 @@ class TestContainerPathsEnv:
             else:
                 file.write_random(cls.file_size, hdrs={'Content-Type':
                                   'application/directory'})
+            if (normalized_urls):
+                nfile = '/'.join(filter(None, f.split('/')))
+                if (f[-1] == '/'):
+                    nfile += '/'
+                stored_files.add(nfile)
+            else:
+                stored_files.add(f)
+        cls.stored_files = sorted(stored_files)
+
+        
 
 
 class TestContainerPaths(Base):
@@ -754,7 +770,7 @@ class TestContainerPaths(Base):
                     found_files.append(file)
 
         recurse_path('')
-        for file in self.env.files:
+        for file in self.env.stored_files:
             if file.startswith('/'):
                 self.assert_(file not in found_dirs)
                 self.assert_(file not in found_files)
@@ -764,10 +780,11 @@ class TestContainerPaths(Base):
             else:
                 self.assert_(file in found_files)
                 self.assert_(file not in found_dirs)
+
         found_files = []
         found_dirs = []
         recurse_path('/')
-        for file in self.env.files:
+        for file in self.env.stored_files:
             if not file.startswith('/'):
                 self.assert_(file not in found_dirs)
                 self.assert_(file not in found_files)
@@ -785,7 +802,7 @@ class TestContainerPaths(Base):
             if isinstance(files[0], dict):
                 files = [str(x['name']) for x in files]
 
-            self.assertEquals(files, sorted(self.env.files))
+            self.assertEquals(files, self.env.stored_files)
 
         for format in ('json', 'xml'):
             for file in self.env.container.files(parms={'format': format}):
@@ -799,18 +816,20 @@ class TestContainerPaths(Base):
         def assert_listing(path, list):
             files = self.env.container.files(parms={'path': path})
             self.assertEquals(sorted(list, cmp=locale.strcoll), files)
-
+        if not normalized_urls:
-        assert_listing('/', ['/dir1/', '/dir2/', '/file1', '/file A'])
+            assert_listing('/', ['/dir1/', '/dir2/', '/file1', '/file A'])
-        assert_listing('/dir1',
+            assert_listing('/dir1',
-                       ['/dir1/file2', '/dir1/subdir1/', '/dir1/subdir2/'])
+                           ['/dir1/file2', '/dir1/subdir1/', '/dir1/subdir2/'])
-        assert_listing('/dir1/',
+            assert_listing('/dir1/',
-                       ['/dir1/file2', '/dir1/subdir1/', '/dir1/subdir2/'])
+                           ['/dir1/file2', '/dir1/subdir1/', '/dir1/subdir2/'])
-        assert_listing('/dir1/subdir1',
+            assert_listing('/dir1/subdir1',
-                       ['/dir1/subdir1/subsubdir2/', '/dir1/subdir1/file2',
+                           ['/dir1/subdir1/subsubdir2/', '/dir1/subdir1/file2',
-                        '/dir1/subdir1/file3', '/dir1/subdir1/file4',
+                            '/dir1/subdir1/file3', '/dir1/subdir1/file4',
-                        '/dir1/subdir1/subsubdir1/'])
+                            '/dir1/subdir1/subsubdir1/'])
-        assert_listing('/dir1/subdir2', [])
+            assert_listing('/dir1/subdir2', [])
-        assert_listing('', ['file1', 'dir1/', 'dir2/'])
+            assert_listing('', ['file1', 'dir1/', 'dir2/'])
+        else:
+            assert_listing('', ['file1', 'dir1/', 'dir2/', 'file A'])
         assert_listing('dir1', ['dir1/file2', 'dir1/subdir1/',
                                 'dir1/subdir2/', 'dir1/subdir with spaces/',
                                 'dir1/subdir+with{whatever/'])
@@ -1486,6 +1505,8 @@ class TestFile(Base):
         self.assertEquals(etag, header_etag)
 
     def testChunkedPut(self):
+        if (web_front_end == 'apache2'):
+            raise SkipTest()
         data = File.random_data(10000)
         etag = File.compute_md5sum(data)
 

test/functionalnosetests/swift_testing.py

@@ -17,16 +17,17 @@ import errno
 import os
 import socket
 import sys
-from httplib import HTTPException
 from time import sleep
 from nose import SkipTest
 from ConfigParser import MissingSectionHeaderError
 
 from test import get_config
 
-from swiftclient import get_auth, http_connection
+from swiftclient import get_auth, http_connection, HTTPException
 
 conf = get_config('func_test')
+web_front_end = conf.get('web_front_end', 'integral')
+normalized_urls = conf.get('normalized_urls', False)
 
 # If no conf was read, we will fall back to old school env vars
 swift_test_auth = os.environ.get('SWIFT_TEST_AUTH')

test/functionalnosetests/test_container.py

@@ -24,7 +24,7 @@ from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \
     MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH
 
 from swift_testing import check_response, retry, skip, skip2, skip3, \
-                          swift_test_user
+                          swift_test_user, web_front_end
 
 
 class TestContainer(unittest.TestCase):
@@ -561,8 +561,11 @@ class TestContainer(unittest.TestCase):
                          {'X-Auth-Token': token})
             return check_response(conn)
         resp = retry(put)
-        self.assertEquals(resp.read(), 'Invalid UTF8 or contains NULL')
+        if (web_front_end == 'apache2'):
-        self.assertEquals(resp.status, 412)
+            self.assertEquals(resp.status, 404)
+        else:
+            self.assertEquals(resp.read(), 'Invalid UTF8 or contains NULL')
+            self.assertEquals(resp.status, 412)
 
 
 if __name__ == '__main__':

test/functionalnosetests/test_object.py

@@ -22,7 +22,9 @@ from uuid import uuid4
 from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \
     MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH
 
-from swift_testing import check_response, retry, skip, skip3, swift_test_user
+from swift_testing import check_response, retry, skip, skip3, \
+    swift_test_user, web_front_end
+from test import get_config
 
 
 class TestObject(unittest.TestCase):
@@ -587,8 +589,11 @@ class TestObject(unittest.TestCase):
                 self.container), 'test', {'X-Auth-Token': token})
             return check_response(conn)
         resp = retry(put)
-        self.assertEquals(resp.read(), 'Invalid UTF8 or contains NULL')
+        if (web_front_end == 'apache2'):
-        self.assertEquals(resp.status, 412)
+            self.assertEquals(resp.status, 404)
+        else:
+            self.assertEquals(resp.read(), 'Invalid UTF8 or contains NULL')
+            self.assertEquals(resp.status, 412)
 
 
 if __name__ == '__main__':