Use the same python version to run storlet agents

This patch refactors package installations in storlets devstack
plugin, so that we can run storlets agents with the same python
version as the one we have in host to run storlets middleware.

This patch also bumps up openjdk version from 8 to 11, and python3
version from 3.5 to 3.6, so that we use the latest versin provided
in distro.

Change-Id: I6cb87976ff4fa6caaa58cf555685f24d95f214a8

devstack/plugin.sh

@@ -75,25 +75,25 @@ TMP_REGISTRY_PREFIX=/tmp/registry
 # Functions
 # ---------
 
-_storlets_swift_start() {
+function _storlets_swift_start {
     swift-init --run-dir=${SWIFT_DATA_DIR}/run all start || true
 }
 
-_storlets_swift_stop() {
+function _storlets_swift_stop {
     swift-init --run-dir=${SWIFT_DATA_DIR}/run all stop || true
 }
 
-_storlets_swift_restart() {
+function _storlets_swift_restart {
     swift-init --run-dir=${SWIFT_DATA_DIR}/run all restart || true
 }
 
-_export_os_vars() {
+function _export_os_vars {
     export OS_IDENTITY_API_VERSION=3
     export OS_AUTH_URL="http://$KEYSTONE_IP/identity/v3"
     export OS_REGION_NAME=RegionOne
 }
 
-_export_keystone_os_vars() {
+function _export_keystone_os_vars {
     _export_os_vars
     export OS_USERNAME=$ADMIN_USER
     export OS_USER_DOMAIN_ID=$STORLETS_DEFAULT_USER_DOMAIN_ID
@@ -102,7 +102,7 @@ _export_keystone_os_vars() {
     export OS_PROJECT_DOMAIN_ID=$STORLETS_DEFAULT_PROJECT_DOMAIN_ID
 }
 
-_export_swift_os_vars() {
+function _export_swift_os_vars {
     _export_os_vars
     export OS_USERNAME=$SWIFT_DEFAULT_USER
     export OS_USER_DOMAIN_ID=$STORLETS_DEFAULT_USER_DOMAIN_ID
@@ -111,7 +111,7 @@ _export_swift_os_vars() {
     export OS_PROJECT_DOMAIN_ID=$STORLETS_DEFAULT_PROJECT_DOMAIN_ID
 }
 
-configure_swift_and_keystone_for_storlets() {
+function configure_swift_and_keystone_for_storlets {
     # Add project and users to Keystone
     _export_keystone_os_vars
     project_test_created=$(openstack project list | grep -w $SWIFT_DEFAULT_PROJECT | wc -l)
@@ -149,7 +149,7 @@ configure_swift_and_keystone_for_storlets() {
     swift post $STORLETS_LOG_CONTAIER_NAME
 }
 
-_install_docker() {
+function _install_docker {
     # TODO: Add other dirstors.
     # This one is geared towards Ubuntu
     # See other projects that install docker
@@ -190,47 +190,50 @@ _install_docker() {
 fi
 }
 
-prepare_storlets_install() {
+function prepare_storlets_install {
     sudo mkdir -p "$STORLETS_DOCKER_DEVICE"/docker
     sudo chmod 777 $STORLETS_DOCKER_DEVICE
     _install_docker
-    sudo add-apt-repository -y ppa:openjdk-r/ppa
+
-    sudo apt-get update
+    if is_ubuntu; then
-    sudo apt-get install -y openjdk-8-jdk-headless
+      install_package openjdk-11-jdk-headless ant
-    sudo apt-get install -y ant
+    else
-    sudo apt-get install -y python
+      die $LINENO "Unsupported distro"
-    sudo apt-get install -y python-setuptools
+    fi
-    sudo apt-get install -y python3.5
+
-    sudo apt-get install -y python3-setuptools
+    if python3_enabled; then
+      install_python3
+    fi
 }
 
-_generate_jre_dockerfile() {
+function _generate_jre_dockerfile {
-    cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/${STORLETS_DOCKER_BASE_IMG_NAME}_jre8/Dockerfile
+    local PYTHON_PACKAGES='python2.7 python-dev python3.6 python3.6-dev'
+    if python3_enabled; then
+        PYTHON_PACKAGES="python2.7 python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev"
+    fi
+
+    cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/${STORLETS_DOCKER_BASE_IMG_NAME}_jre11/Dockerfile
 FROM $STORLETS_DOCKER_BASE_IMG
 MAINTAINER root
 
 RUN apt-get update && \
-    apt-get install python -y && \
+    apt-get install ${PYTHON_PACKAGES} openjdk-11-jre-headless -y && \
-    apt-get install python3.5 -y && \
-    apt-get install git -y && \
-    apt-get update && \
-    apt-get install openjdk-8-jre-headless -y && \
     apt-get clean
 EOF
 }
 
-create_base_jre_image() {
+function create_base_jre_image {
     echo "Create base jre image"
     docker pull $STORLETS_DOCKER_BASE_IMG
-    mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8
+    mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11
     _generate_jre_dockerfile
-    cd ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8
+    cd ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11
-    docker build -q -t ${STORLETS_DOCKER_BASE_IMG_NAME}_jre8 .
+    docker build -q -t ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11 .
     cd -
 }
 
-_generate_logback_xml() {
+function _generate_logback_xml {
-    cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8_storlets/logback.xml
+    cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11_storlets/logback.xml
 <configuration>
   <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
     <file>/tmp/SDaemon.log</file>
@@ -257,9 +260,9 @@ _generate_logback_xml() {
 EOF
 }
 
-_generate_jre_storlet_dockerfile() {
+function _generate_jre_storlet_dockerfile {
-    cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8_storlets/Dockerfile
+    cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11_storlets/Dockerfile
-FROM ${STORLETS_DOCKER_BASE_IMG_NAME}_jre8
+FROM ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11
 MAINTAINER root
 RUN [ "groupadd", "-g", "$STORLETS_DOCKER_SWIFT_GROUP_ID", "swift" ]
 RUN [ "useradd", "-u" , "$STORLETS_DOCKER_SWIFT_USER_ID", "-g", "$STORLETS_DOCKER_SWIFT_GROUP_ID", "swift" ]
@@ -275,25 +278,29 @@ ENTRYPOINT ["/usr/local/libexec/storlets/init_container.sh"]
 EOF
 }
 
-create_storlet_engine_image() {
+function create_storlet_engine_image {
     echo "Create Storlet engine image"
-    mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8_storlets
+    mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11_storlets
     _generate_logback_xml
     _generate_jre_storlet_dockerfile
-    cd ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8_storlets
+    cd ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11_storlets
-    docker build -q -t ${STORLETS_DOCKER_BASE_IMG_NAME}_jre8_storlets .
+    docker build -q -t ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11_storlets .
     cd -
 }
 
-install_storlets_code() {
+function install_storlets_code {
     echo "Installing storlets"
     cd $REPO_DIR
     sudo ./install_libs.sh
-    sudo pip install -r requirements.txt
+    pip_install .
-    sudo python setup.py install
+
-    sudo pip3 install -r requirements.txt
+    # Also install code to library directory so that we can import them
-    sudo python3 setup.py install
+    # from docker container.
-    sudo chown -R ${STORLETS_SWIFT_RUNTIME_USER} storlets.egg-info*
+    sudo mkdir -p -m 755 /usr/local/lib/storlets/python
+    pip_install . -t /usr/local/lib/storlets/python --no-compile
+    for bin_file in storlets-daemon storlets-daemon-factory ; do
+        sudo cp `which ${bin_file}` /usr/local/libexec/storlets/
+    done
 
     sudo mkdir -p $STORLETS_DOCKER_DEVICE/scripts
     sudo chown "$STORLETS_SWIFT_RUNTIME_USER":"$STORLETS_SWIFT_RUNTIME_GROUP" "$STORLETS_DOCKER_DEVICE"/scripts
@@ -305,7 +312,7 @@ install_storlets_code() {
     cd -
 }
 
-_generate_swift_middleware_conf() {
+function _generate_swift_middleware_conf {
     cat <<EOF > /tmp/swift_middleware_conf
 [proxy-confs]
 proxy_server_conf_file = /etc/swift/proxy-server.conf
@@ -327,7 +334,7 @@ storlet_proxy_execution = $STORLETS_PROXY_EXECUTION_ONLY
 EOF
 }
 
-_generate_storlet-docker-gateway() {
+function _generate_storlet-docker-gateway {
     cat <<EOF > /tmp/storlet-docker-gateway.conf
 [DEFAULT]
 storlet_logcontainer = $STORLETS_LOG_CONTAIER_NAME
@@ -341,14 +348,14 @@ storlet_timeout = $STORLETS_RUNTIME_TIMEOUT
 EOF
 }
 
-_generate_default_tenant_dockerfile() {
+function _generate_default_tenant_dockerfile {
     cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/"$SWIFT_DEFAULT_PROJECT_ID"/Dockerfile
-FROM ${STORLETS_DOCKER_BASE_IMG_NAME}_jre8_storlets
+FROM ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11_storlets
 MAINTAINER root
 EOF
 }
 
-create_default_tenant_image() {
+function create_default_tenant_image {
     SWIFT_DEFAULT_PROJECT_ID=`openstack project list | grep -w $SWIFT_DEFAULT_PROJECT | awk '{ print $2 }'`
     mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/$SWIFT_DEFAULT_PROJECT_ID
     _generate_default_tenant_dockerfile
@@ -357,7 +364,7 @@ create_default_tenant_image() {
     cd -
 }
 
-create_test_config_file() {
+function create_test_config_file {
     testfile=${REPO_DIR}/test.conf
     iniset ${testfile} general keystone_default_domain $STORLETS_DEFAULT_PROJECT_DOMAIN_ID
     iniset ${testfile} general keystone_public_url $KEYSTONE_PUBLIC_URL
@@ -369,8 +376,7 @@ create_test_config_file() {
     iniset ${testfile} general region
 }
 
-
+function install_storlets {
-install_storlets() {
     echo "Install storlets dependencies"
     prepare_storlets_install
 
@@ -392,7 +398,7 @@ install_storlets() {
     _storlets_swift_restart
 }
 
-uninstall_storlets() {
+function uninstall_storlets {
     sudo service docker stop
     sudo sed -r 's#^.*DOCKER_OPTS=.*$#DOCKER_OPTS="--debug --storage-opt dm.override_udev_sync_check=true"#' /etc/default/docker
 

doc/source/engine_dev_installation.rst

@@ -119,10 +119,7 @@ We need the following for the Java parts
 
 ::
 
-    sudo add-apt-repository ppa:openjdk-r/ppa
+    sudo apt-get install openjdk-11-jdk ant
-    sudo apt-get update
-    sudo apt-get install openjdk-8-jdk
-    sudo apt-get install ant
 
 We need the following for Docker
 
@@ -160,9 +157,9 @@ Step 2: Create a Docker image with Java
 
 ::
 
-    mkdir -p $HOME/docker_repos/ubuntu_18.04_jre8
+    mkdir -p $HOME/docker_repos/ubuntu_18.04_jre11
 
-Create the file: $HOME/docker_repos/ubuntu_18.04_jre8/Dockerfile
+Create the file: $HOME/docker_repos/ubuntu_18.04_jre11/Dockerfile
 with the following content:
 
 ::
@@ -175,17 +172,15 @@ with the following content:
     RUN apt-get update && \
     apt-get install python && \
     apt-get install software-properties-common && \
-    add-apt-repository ppa:openjdk-r/ppa && \
+    apt-get install openjdk-11-jre-headless ant  && \
-    apt-get update && \
-    apt-get install openjdk-8-jre && \
     apt-get clean
 
 Build the image
 
 ::
 
-    cd $HOME/docker_repos/ubuntu_18.04_jre8
+    cd $HOME/docker_repos/ubuntu_18.04_jre11
-    sudo docker build -q -t ubuntu_18.04_jre8 .
+    sudo docker build -q -t ubuntu_18.04_jre11 .
     cd -
 
 
@@ -193,16 +188,16 @@ Step 3: Augment the above created image with the storlets stuff
 
 ::
 
-    mkdir -p $HOME/docker_repos/ubuntu_18.04_jre8_storlets
+    mkdir -p $HOME/docker_repos/ubuntu_18.04_jre11_storlets
     cp $HOME/storlets/install/storlets/roles/docker_storlet_engine_image/files/logback.xml .
     cd -
 
-Create the file: $HOME/docker_repos/ubuntu_18.04_jre8_storlets/Dockerfile
+Create the file: $HOME/docker_repos/ubuntu_18.04_jre11_storlets/Dockerfile
 with the following content:
 
 ::
 
-    FROM ubuntu_18.04_jre8
+    FROM ubuntu_18.04_jre11
 
     MAINTAINER root
 
@@ -222,8 +217,8 @@ Build the image
 
 ::
 
-    cd $HOME/docker_repos/ubuntu_18.04_jre8_storlets
+    cd $HOME/docker_repos/ubuntu_18.04_jre11_storlets
-    sudo docker build -q -t ubuntu_18.04_jre8_storlets .
+    sudo docker build -q -t ubuntu_18.04_jre11_storlets .
     cd -
 
 Step 4: Create a tenant specific image. The engine looks for images
@@ -242,12 +237,12 @@ The response from the above contains the account line, e.g.:
 
 The account id is the number following the 'AUTH\_' prefix.
 
-Next create the file $HOME/docker_repos/ubuntu_18.04_jre8_storlets_<account id>/Dockerfile
+Next create the file $HOME/docker_repos/ubuntu_18.04_jre11_storlets_<account id>/Dockerfile
 with the following content:
 
 ::
 
-    FROM ubuntu_18.04_jre8_storlets
+    FROM ubuntu_18.04_jre11_storlets
     MAINTAINER root
 
 
@@ -255,7 +250,7 @@ Build the image
 
 ::
 
-    cd $HOME/docker_repos/ubuntu_18.04_jre8_storlets_<account id>
+    cd $HOME/docker_repos/ubuntu_18.04_jre11_storlets_<account id>
     sudo docker build -q -t <account id> .
     cd -
 

install_libs.sh

@@ -23,14 +23,5 @@ ant install
 # Install container side scripts
 SRC=bin
 DST=/usr/local/libexec/storlets
-mkdir -p $DST
+mkdir -p -m 755 $DST
-chmod 755 $DST
 cp "$SRC/init_container.sh" $DST
-cp "$SRC/storlets-daemon" $DST
-cp "$SRC/storlets-daemon-factory" $DST
-
-# Install python library to be mouted by containers
-DST=/usr/local/lib/storlets/python
-mkdir -p $DST
-chmod 755 $DST
-pip install . -t $DST --no-compile

playbooks/storlets-functional/pre.yaml

@@ -8,3 +8,5 @@
           ./s2aio.sh install
         executable: /bin/bash
         chdir: 'src/{{ zuul.project.canonical_name }}'
+      environment:
+        USE_PYTHON3: "False"

s2aio.sh

@@ -37,6 +37,7 @@ _prepare_devstack_env() {
         cp devstack/localrc.sample $DEVSTACK_DIR/localrc
     fi
 
+    source $DEVSTACK_DIR/stackrc
     source $DEVSTACK_DIR/functions
     source $DEVSTACK_DIR/functions-common
     source $DEVSTACK_DIR/lib/swift

setup.cfg

@@ -26,10 +26,6 @@ skip_changelog = True
 [files]
 packages =
     storlets
-scripts =
-    bin/storlets-daemon
-    bin/storlets-daemon-factory
-    bin/sbus
 
 [entry_points]
 paste.filter_factory =
@@ -39,6 +35,11 @@ storlets.gateways =
     stub = storlets.gateway.gateways.stub:StorletGatewayStub
     docker = storlets.gateway.gateways.docker:StorletGatewayDocker
 
+console_scripts =
+    sbus = storlets.sbus.cli:main
+    storlets-daemon = storlets.agent.daemon.server:main
+    storlets-daemon-factory = storlets.agent.daemon_factory.server:main
+
 [build_sphinx]
 source-dir = doc/source
 build-dir = doc/build

src/java/SBus/build.xml

@@ -28,20 +28,15 @@
             srcdir="src/main"
             destdir="bin"
             classpath="../dependencies/json_simple-1.1.jar"
-            includeantruntime="false"/>
+            includeantruntime="false"
+            nativeheaderdir='.'/>
     </target>
 
-    <target name="h" depends="java">
+    <target name="so" depends="java">
-        <javah destdir="." force="yes" classpath="bin">
-            <class name="org.openstack.storlet.sbus.SBusJNI" />
-        </javah>
-    </target>
-
-    <target name="so" depends="h">
         <exec dir="." executable="gcc">
             <arg line="-shared -o bin/libjsbus.so -fPIC" />
-            <arg line="-I/usr/lib/jvm/java-8-openjdk-amd64/include/" />
+            <arg line="-I/usr/lib/jvm/java-11-openjdk-amd64/include/" />
-            <arg line="-I/usr/lib/jvm/java-8-openjdk-amd64/include/linux/" />
+            <arg line="-I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" />
             <arg line="-I../../c/sbus/" />
             <arg line="SBusJNI.c ../../c/sbus/sbus.c" />
         </exec>

src/java/SBus/src/main/org/openstack/storlet/sbus/ServerSBusInDatagram.java

@@ -114,15 +114,15 @@ public class ServerSBusInDatagram {
         }
 
         String strMD = msg.getMetadata();
-                this.metadata = (HashMap<String, HashMap<String, String>>[])new HashMap[getNFiles()];
+        this.metadata = (HashMap<String, HashMap<String, String>>[])new HashMap[getNFiles()];
         JSONArray jsonarray = (JSONArray)(new JSONParser().parse(strMD));
-        Iterator it = jsonarray.iterator();
+        Iterator<JSONObject> it = jsonarray.iterator();
         int i=0;
         while (it.hasNext()) {
             this.metadata[i] = new HashMap<String, HashMap<String, String>>();
             HashMap<String, String> storletsMetadata = new HashMap<String, String>();
             HashMap<String, String> storageMetadata = new HashMap<String, String>();
-            JSONObject jsonobject = (JSONObject)it.next();
+            JSONObject jsonobject = it.next();
             if (jsonobject.containsKey("storage")) {
                 populateMetadata(storageMetadata, (JSONObject)jsonobject.get("storage"));
             }

storlets/agent/common/server.py

@@ -108,7 +108,7 @@ class SBusServer(object):
             handler = self.get_handler(command)
             resp = handler(dtg)
         except ValueError as err:
-            self.logger.error(err.message)
+            self.logger.exception('Failed to handle request')
             resp = CommandFailure(str(err))
         except CommandResponse as err:
             resp = err
@@ -140,7 +140,7 @@ class SBusServer(object):
         :param resp: CommandResponse instance
         """
         try:
-            outfile.write(resp.report_message)
+            outfile.write(resp.report_message.encode('utf-8'))
         except IOError:
             self.logger.exception('Unable to return response to client')
 

storlets/agent/common/utils.py

@@ -15,8 +15,9 @@
 import logging
 from logging.handlers import SysLogHandler
 
+# TODO(takashi): Make the following parameters configurable
 DEFAULT_PY2 = 2.7
-DEFAULT_PY3 = 3.5
+DEFAULT_PY3 = 3.6
 
 
 def get_logger(logger_name, log_level, container_id):

storlets/agent/daemon/server.py

@@ -14,12 +14,12 @@
 # limitations under the License.
 import argparse
 import errno
+import importlib
 import os
 import pwd
+import signal
 import sys
 import uuid
-import signal
-import importlib
 from storlets.sbus import SBus
 from storlets.agent.common.server import command_handler, EXIT_FAILURE, \
     CommandSuccess, CommandFailure, SBusServer
@@ -254,20 +254,22 @@ def main():
     # Initialize logger
     logger = get_logger("storlets-daemon", opts.log_level, opts.container_id)
     logger.debug("Storlet Daemon started")
-    SBus.start_logger("DEBUG", container_id=opts.container_id)
 
-    # Impersonate the swift user
-    pw = pwd.getpwnam('swift')
-    os.setresgid(pw.pw_gid, pw.pw_gid, pw.pw_gid)
-    os.setresuid(pw.pw_uid, pw.pw_uid, pw.pw_uid)
-
-    # create an instance of storlet daemon
     try:
+        SBus.start_logger("DEBUG", container_id=opts.container_id)
+
+        # Impersonate the swift user
+        pw = pwd.getpwnam('swift')
+        os.setresgid(pw.pw_gid, pw.pw_gid, pw.pw_gid)
+        os.setresuid(pw.pw_uid, pw.pw_uid, pw.pw_uid)
+
+        # create an instance of storlet daemon
         daemon = StorletDaemon(opts.storlet_name, opts.sbus_path,
                                logger, opts.pool_size)
-    except Exception as err:
-        logger.error(err.message)
-        return EXIT_FAILURE
 
-    # Start the main loop
+        # Start the main loop
-    return daemon.main_loop()
+        sys.exit(daemon.main_loop())
+
+    except Exception:
+        logger.error('Unhandled exception')
+        sys.exit(EXIT_FAILURE)

storlets/agent/daemon_factory/server.py

@@ -18,14 +18,15 @@ import os
 import pwd
 import signal
 import subprocess
+import sys
 import time
 
 from storlets.sbus import SBus
 from storlets.sbus.client import SBusClient
 from storlets.sbus.client.exceptions import SBusClientException, \
     SBusClientSendError
-from storlets.agent.common.server import command_handler, CommandSuccess, \
+from storlets.agent.common.server import command_handler, EXIT_FAILURE, \
-    CommandFailure, SBusServer
+    CommandSuccess, CommandFailure, SBusServer
 from storlets.agent.common.utils import get_logger, DEFAULT_PY2, DEFAULT_PY3
 
 
@@ -107,6 +108,8 @@ class StorletDaemonFactory(SBusServer):
         if int(float(daemon_language_version)) == 3:
             daemon_language_version = DEFAULT_PY3
         else:
+            # TODO(takashi): Switch default python version to 3 once we drop
+            #                python2 support.
             daemon_language_version = DEFAULT_PY2
 
         python_interpreter = '/usr/bin/python%s' % daemon_language_version
@@ -521,15 +524,22 @@ def main():
     # Initialize logger
     logger = get_logger("daemon-factory", opts.log_level, opts.container_id)
     logger.debug("Daemon factory started")
-    SBus.start_logger("DEBUG", container_id=opts.container_id)
 
-    # Impersonate the swift user
+    try:
-    pw = pwd.getpwnam('swift')
+        SBus.start_logger("DEBUG", container_id=opts.container_id)
-    os.setresgid(pw.pw_gid, pw.pw_gid, pw.pw_gid)
-    os.setresuid(pw.pw_uid, pw.pw_uid, pw.pw_uid)
 
-    # create an instance of daemon_factory
+        # Impersonate the swift user
-    factory = StorletDaemonFactory(opts.sbus_path, logger, opts.container_id)
+        pw = pwd.getpwnam('swift')
+        os.setresgid(pw.pw_gid, pw.pw_gid, pw.pw_gid)
+        os.setresuid(pw.pw_uid, pw.pw_uid, pw.pw_uid)
 
-    # Start the main loop
+        # create an instance of daemon_factory
-    return factory.main_loop()
+        factory = StorletDaemonFactory(opts.sbus_path, logger,
+                                       opts.container_id)
+
+        # Start the main loop
+        sys.exit(factory.main_loop())
+
+    except Exception:
+        logger.eception('Unhandled exception')
+        sys.exit(EXIT_FAILURE)

storlets/gateway/gateways/docker/runtime.py

@@ -232,7 +232,8 @@ class RunTimeSandbox(object):
         self.logger = logger
 
         self.default_docker_image_name = \
-            conf.get('default_docker_image_name', 'ubuntu_18.04_jre8_storlets')
+            conf.get('default_docker_image_name',
+                     'ubuntu_18.04_jre11_storlets')
 
     def ping(self):
         """

storlets/sbus/client/cli.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os
-from sys import exit
+import sys
 from storlets.sbus.client import SBusClient
 from storlets.sbus.client.exceptions import SBusClientException
 
@@ -21,19 +21,20 @@ EXIT_SUCCESS = 0
 EXIT_ERROR = 1
 
 
-def main(argv):
+def main():
-    # TODO(takashi): Add more detailed help message
+    argv = sys.argv
 
+    # TODO(takashi): Add more detailed help message
     if len(argv) < 3:
         print('sbus <command> <pipe_path>')
-        exit(EXIT_ERROR)
+        sys.exit(EXIT_ERROR)
 
     command = argv[1]
     pipe_path = argv[2]
 
     if not os.path.exists(pipe_path):
         print('ERROR: Pipe file %s does not exist' % pipe_path)
-        exit(EXIT_ERROR)
+        sys.exit(EXIT_ERROR)
 
     client = SBusClient(pipe_path)
     try:
@@ -45,19 +46,19 @@ def main(argv):
         resp = handler()
     except (AttributeError, NotImplementedError):
         print('ERROR: Command %s is not supported' % command)
-        exit(EXIT_ERROR)
+        sys.exit(EXIT_ERROR)
     except SBusClientException as err:
         print('ERROR: Failed to send sbus command %s to %s: %s'
               % (command, pipe_path, err))
-        exit(EXIT_ERROR)
+        sys.exit(EXIT_ERROR)
     except Exception as err:
         print('ERROR: Unknown error: %s' % err)
-        exit(EXIT_ERROR)
+        sys.exit(EXIT_ERROR)
 
     print('Response: %s: %s' % (resp.status, resp.message))
     if resp.status:
         print('OK')
-        exit(EXIT_SUCCESS)
+        sys.exit(EXIT_SUCCESS)
     else:
         print('ERROR: Got error response')
-        exit(EXIT_ERROR)
+        sys.exit(EXIT_ERROR)

storlets/sbus/sbus.py

@@ -62,7 +62,8 @@ class SBus(object):
         sbus_back_ = CDLL(SBus.SBUS_SO_NAME)
 
         sbus_back_.sbus_start_logger.argtypes = [c_char_p, c_char_p]
-        sbus_back_.sbus_start_logger(str_log_level, container_id)
+        sbus_back_.sbus_start_logger(str_log_level.encode("utf-8"),
+                                     container_id.encode("utf-8"))
 
     @staticmethod
     def stop_logger():

storlets/swift_middleware/handlers/proxy.py

@@ -195,9 +195,9 @@ class StorletProxyHandler(StorletBaseHandler):
         acl_string = '.r:%s' % self._build_acl_string(user_name, storlet_name)
         try:
             clean_acl('X-Container-Read', acl_string)
-        except ValueError as e:
+        except ValueError as err:
             msg = ('storlet ACL update request has invalid values %s'
-                   % e.message)
+                   % str(err))
             raise HTTPBadRequest(msg.encode('utf8'))
 
         # Make sure the resulting acl permits a single entity

tests/functional/java/test_thumbnail_storlet.py

@@ -61,7 +61,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
 
         headers = c.head_object(self.url, self.token,
                                 self.container, 'gen_thumb_on_put.jpg')
-        self.assertEqual('49032', headers['content-length'])
+        self.assertLess(int(headers['content-length']), 1087318)
         self.assertEqual('thumbnail', headers['x-object-meta-name'])
 
     def invoke_storlet_on_copy_from(self):
@@ -88,7 +88,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
 
         headers = c.head_object(self.url, self.token,
                                 self.container, 'gen_thumb_on_copy.jpg')
-        self.assertEqual('49032', headers['content-length'])
+        self.assertLess(int(headers['content-length']), 1087318)
         self.assertEqual('thumbnail', headers['x-object-meta-name'])
         self.assertTrue('x-object-meta-x-timestamp' not in headers)
         self.assertTrue('x-timestamp' in headers)
@@ -109,7 +109,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
 
         headers = c.head_object(self.url, self.token,
                                 self.container, 'gen_thumb_on_copy_.jpg')
-        self.assertEqual('49032', headers['content-length'])
+        self.assertLess(int(headers['content-length']), 1087318)
         self.assertEqual('thumbnail', headers['x-object-meta-name'])
         self.assertTrue('x-object-meta-x-timestamp' not in headers)
         self.assertTrue('x-timestamp' in headers)