openstack/storlets
Code Issues Proposed changes

Merge "Prevent undesired retry when starting container"

Browse Source
This commit is contained in:
Zuul 2019-07-03 06:41:34 +00:00 committed by Gerrit Code Review
commit e6f7869974
3 changed files with 93 additions and 34 deletions
scripts
restart_docker_container.c
storlets/gateway/gateways/docker
runtime.py
tests/unit/gateway/gateways/docker
test_runtime.py

15
scripts/restart_docker_container.c

@ -56,12 +56,12 @@ int main(int argc, char **argv) {
snprintf(mount_dir5,(size_t)512, "%s", argv[7]);
int ret;
setresuid(0,0,0);
setresgid(0,0,0);
sprintf(command,"/usr/bin/docker stop -t 1 %s",container_name);
setresuid(0, 0, 0);
setresgid(0, 0, 0);
sprintf(command, "/usr/bin/docker stop -t 1 %s", container_name);
ret = system(command);
sprintf(command,"/usr/bin/docker rm %s",container_name);
sprintf(command, "/usr/bin/docker rm %s", container_name);
ret = system(command);
sprintf(command,
@ -74,7 +74,8 @@ int main(int argc, char **argv) {
mount_dir5,
container_image);
ret = system(command);
if (ret)
return(EXIT_FAILURE);
return(EXIT_SUCCESS);
if(ret){
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}

52
storlets/gateway/gateways/docker/runtime.py

@ -314,7 +314,7 @@ class RunTimeSandbox(object):
Restarts the scope's sandbox using the specified docker image
:param docker_image_name: name of the docker image to start
:returns: returned value of restart_docker_container
:raises StorletRuntimeException: when failed to restart the container
"""
if self.docker_repo:
docker_image_name = '%s/%s' % (self.docker_repo,
@ -343,9 +343,17 @@ class RunTimeSandbox(object):
storlet_mount, storlet_python_lib_mount,
storlet_native_lib_mount, storlet_native_bin_mount]
self.logger.debug('About to start container %s' % cmd)
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = proc.communicate()
return subprocess.call(cmd)
if stdout:
self.logger.debug('STDOUT: %s' % stdout.replace('\n', '#012'))
if stderr:
self.logger.error('STDERR: %s' % stderr.replace('\n', '#012'))
if proc.returncode:
raise StorletRuntimeException('Failed to restart docker container')
def restart(self):
"""
@ -355,20 +363,24 @@ class RunTimeSandbox(object):
self.paths.create_host_pipe_prefix()
docker_image_name = self.scope
if self._restart(docker_image_name) == 0:
try:
self._restart(docker_image_name)
self.wait()
return
except StorletTimeout:
raise
except StorletRuntimeException:
# We were unable to start docker container from the tenant image.
# Let us try to start docker container from default image.
self.logger.exception("Failed to start docker container from "
"tenant image %s" % docker_image_name)
# We were unable to start docker container from the tenant image.
# Let us try to start docker container from default image.
self.logger.info("Failed to start docker container from tenant image "
"%s" % docker_image_name)
self.logger.info("Trying to start docker container from default image")
# TODO(eranr): move the default tenant image name to a config var
docker_image_name = 'ubuntu_16.04_jre8_storlets'
# TODO(eranr): move the default tenant image name to a config var
docker_image_name = 'ubuntu_16.04_jre8_storlets'
self._restart(docker_image_name)
self.wait()
self.logger.info("Trying to start docker container from default "
"image: %s" % docker_image_name)
self._restart(docker_image_name)
self.wait()
def start_storlet_daemon(
self, spath, storlet_id, language, language_version=None):
@ -485,11 +497,7 @@ class RunTimeSandbox(object):
# Best we can do is execute the container.
self.logger.debug('Failed to check Storlet daemon status, '
'restart Docker container')
try:
self.restart()
except StorletTimeout:
raise StorletRuntimeException('Docker container is '
'not responsive')
self.restart()
storlet_daemon_status = 0
if (cache_updated is True and storlet_daemon_status == 1):
@ -499,11 +507,7 @@ class RunTimeSandbox(object):
'is running. Stopping daemon')
res = self.stop_storlet_daemon(sreq.storlet_main)
if res != 1:
try:
self.restart()
except StorletTimeout:
raise StorletRuntimeException('Docker container is '
'not responsive')
self.restart()
else:
self.logger.debug('Deamon stopped')
storlet_daemon_status = 0

60
tests/unit/gateway/gateways/docker/test_runtime.py

@ -323,29 +323,83 @@ class TestRunTimeSandbox(unittest.TestCase):
# TODO(takashi): should test timeout case
def test_restart(self):
class FakeProc(object):
def __init__(self, stdout, stderr, code):
self.stdout = stdout
self.stderr = stderr
self.returncode = code
def communicate(self):
return (self.stdout, self.stderr)
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'RunTimePaths.create_host_pipe_prefix'), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'subprocess.call'):
'subprocess.Popen') as popen:
_wait = self.sbox.wait
def dummy_wait_success(*args, **kwargs):
return 1
self.sbox.wait = dummy_wait_success
# Test that popen is called successfully
popen.return_value = FakeProc('Try to restart\nOK', '', 0)
self.sbox.restart()
self.assertEqual(1, popen.call_count)
self.sbox.wait = _wait
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'RunTimePaths.create_host_pipe_prefix'), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'subprocess.call'):
'subprocess.Popen') as popen:
_wait = self.sbox.wait
def dummy_wait_success(*args, **kwargs):
return 1
self.sbox.wait = dummy_wait_success
# Test double failure to restart the container
# for both the tenant image and generic image
popen.return_value = FakeProc('Try to restart', 'Some error', 1)
with self.assertRaises(StorletRuntimeException):
self.sbox.restart()
self.assertEqual(2, popen.call_count)
self.sbox.wait = _wait
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'RunTimePaths.create_host_pipe_prefix'), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'subprocess.Popen') as popen:
_wait = self.sbox.wait
def dummy_wait_success(*args, **kwargs):
return 1
self.sbox.wait = dummy_wait_success
# Test failure to restart the container for the tenant image
# success for the generic image
popen.side_effect = [FakeProc('Try to restart', 'Some error', 1),
FakeProc('Try to restart\nOK', '', 0)]
self.sbox.restart()
self.assertEqual(2, popen.call_count)
self.sbox.wait = _wait
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'RunTimePaths.create_host_pipe_prefix'), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'subprocess.Popen') as popen:
_wait = self.sbox.wait
def dummy_wait_failure(*args, **kwargs):
raise StorletRuntimeException()
raise StorletTimeout()
self.sbox.wait = dummy_wait_failure
popen.return_value = FakeProc('OK', '', 0)
with self.assertRaises(StorletRuntimeException):
self.sbox.restart()
self.sbox.wait = _wait