openstack/neutron
Code Issues Proposed changes

Migrate to oslo.concurrency

Browse Source 
Namespaces are not used anymore, so imports are for oslo_concurrency,
not oslo.concurrency.

lockutils direct call is not supported anymore, so using shipped
lockutils-wrapper instead.

Use ExternalLockFixture to set lock_path for unit testing.

Updated cache and cache/_backends module from oslo-incubator to use
oslo.concurrency and not incubator version of lockutils module.

Dropped lockutils incubator module that is now moved to oslo.concurrency. Not
dropping fixture module that includes lockutils fixture since other fixtures
are still used in the tree.

Closes-Bug: #1387092
Change-Id: I2ba2295f8f5d55f272a9d23555940586b25b5a1c
This commit is contained in:
Ihar Hrachyshka 2014-12-05 18:17:15 +01:00
parent 150f5fbdca
commit 35d838ff60
22 changed files with 35 additions and 346 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
neutron/agent/linux/external_process.py
View File

@ -16,11 +16,11 @@ import collections
import eventlet
from oslo.config import cfg
from oslo_concurrency import lockutils
from neutron.agent.linux import ip_lib
from neutron.agent.linux import utils
from neutron.i18n import _LE
from neutron.openstack.common import lockutils
from neutron.openstack.common import log as logging
LOG = logging.getLogger(__name__)

2
neutron/agent/linux/iptables_manager.py
View File

@ -24,13 +24,13 @@ import sys
from oslo.config import cfg
from oslo.utils import excutils
from oslo_concurrency import lockutils
from neutron.agent.common import config
from neutron.agent.linux import iptables_comments as ic
from neutron.agent.linux import utils as linux_utils
from neutron.common import utils
from neutron.i18n import _LE, _LW
from neutron.openstack.common import lockutils
from neutron.openstack.common import log as logging
LOG = logging.getLogger(__name__)

2
neutron/common/utils.py
View File

@ -32,9 +32,9 @@ import uuid
from eventlet.green import subprocess
from oslo.config import cfg
from oslo.utils import excutils
from oslo_concurrency import lockutils
from neutron.common import constants as q_const
from neutron.openstack.common import lockutils
from neutron.openstack.common import log as logging

2
neutron/openstack/common/cache/_backends/memory.py vendored
View File

@ -15,9 +15,9 @@
import collections
from oslo.utils import timeutils
from oslo_concurrency import lockutils
from neutron.openstack.common.cache import backends
from neutron.openstack.common import lockutils
class MemoryBackend(backends.BaseCache):

14
neutron/openstack/common/cache/cache.py vendored
View File

@ -20,6 +20,20 @@ Supported configuration options:
`key_namespace`: Namespace under which keys will be created.
"""
########################################################################
#
# THIS MODULE IS DEPRECATED
#
# Please refer to
# https://etherpad.openstack.org/p/kilo-neutron-library-proposals for
# the discussion leading to this deprecation.
#
# We recommend helping with the new neutron.cache library being created
# as a wrapper for dogpile.
#
########################################################################
from six.moves.urllib import parse
from stevedore import driver

326
neutron/openstack/common/lockutils.py
View File

@ -1,326 +0,0 @@
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import contextlib
import errno
import functools
import logging
import os
import shutil
import subprocess
import sys
import tempfile
import threading
import time
import weakref
from oslo.config import cfg
from neutron.openstack.common import fileutils
from neutron.openstack.common._i18n import _, _LE, _LI
LOG = logging.getLogger(__name__)
util_opts = [
cfg.BoolOpt('disable_process_locking', default=False,
help='Enables or disables inter-process locks.'),
cfg.StrOpt('lock_path',
default=os.environ.get("NEUTRON_LOCK_PATH"),
help='Directory to use for lock files.')
]
CONF = cfg.CONF
CONF.register_opts(util_opts)
def set_defaults(lock_path):
cfg.set_defaults(util_opts, lock_path=lock_path)
class _FileLock(object):
"""Lock implementation which allows multiple locks, working around
issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does
not require any cleanup. Since the lock is always held on a file
descriptor rather than outside of the process, the lock gets dropped
automatically if the process crashes, even if __exit__ is not executed.
There are no guarantees regarding usage by multiple green threads in a
single process here. This lock works only between processes. Exclusive
access between local threads should be achieved using the semaphores
in the @synchronized decorator.
Note these locks are released when the descriptor is closed, so it's not
safe to close the file descriptor while another green thread holds the
lock. Just opening and closing the lock file can break synchronisation,
so lock files must be accessed only using this abstraction.
"""
def __init__(self, name):
self.lockfile = None
self.fname = name
def acquire(self):
basedir = os.path.dirname(self.fname)
if not os.path.exists(basedir):
fileutils.ensure_tree(basedir)
LOG.info(_LI('Created lock path: %s'), basedir)
self.lockfile = open(self.fname, 'w')
while True:
try:
# Using non-blocking locks since green threads are not
# patched to deal with blocking locking calls.
# Also upon reading the MSDN docs for locking(), it seems
# to have a laughable 10 attempts "blocking" mechanism.
self.trylock()
LOG.debug('Got file lock "%s"', self.fname)
return True
except IOError as e:
if e.errno in (errno.EACCES, errno.EAGAIN):
# external locks synchronise things like iptables
# updates - give it some time to prevent busy spinning
time.sleep(0.01)
else:
raise threading.ThreadError(_("Unable to acquire lock on"
" `%(filename)s` due to"
" %(exception)s") %
{'filename': self.fname,
'exception': e})
def __enter__(self):
self.acquire()
return self
def release(self):
try:
self.unlock()
self.lockfile.close()
LOG.debug('Released file lock "%s"', self.fname)
except IOError:
LOG.exception(_LE("Could not release the acquired lock `%s`"),
self.fname)
def __exit__(self, exc_type, exc_val, exc_tb):
self.release()
def exists(self):
return os.path.exists(self.fname)
def trylock(self):
raise NotImplementedError()
def unlock(self):
raise NotImplementedError()
class _WindowsLock(_FileLock):
def trylock(self):
msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1)
def unlock(self):
msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1)
class _FcntlLock(_FileLock):
def trylock(self):
fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
def unlock(self):
fcntl.lockf(self.lockfile, fcntl.LOCK_UN)
if os.name == 'nt':
import msvcrt
InterProcessLock = _WindowsLock
else:
import fcntl
InterProcessLock = _FcntlLock
_semaphores = weakref.WeakValueDictionary()
_semaphores_lock = threading.Lock()
def _get_lock_path(name, lock_file_prefix, lock_path=None):
# NOTE(mikal): the lock name cannot contain directory
# separators
name = name.replace(os.sep, '_')
if lock_file_prefix:
sep = '' if lock_file_prefix.endswith('-') else '-'
name = '%s%s%s' % (lock_file_prefix, sep, name)
local_lock_path = lock_path or CONF.lock_path
if not local_lock_path:
raise cfg.RequiredOptError('lock_path')
return os.path.join(local_lock_path, name)
def external_lock(name, lock_file_prefix=None, lock_path=None):
LOG.debug('Attempting to grab external lock "%(lock)s"',
{'lock': name})
lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path)
return InterProcessLock(lock_file_path)
def remove_external_lock_file(name, lock_file_prefix=None):
"""Remove an external lock file when it's not used anymore
This will be helpful when we have a lot of lock files
"""
with internal_lock(name):
lock_file_path = _get_lock_path(name, lock_file_prefix)
try:
os.remove(lock_file_path)
except OSError:
LOG.info(_LI('Failed to remove file %(file)s'),
{'file': lock_file_path})
def internal_lock(name):
with _semaphores_lock:
try:
sem = _semaphores[name]
LOG.debug('Using existing semaphore "%s"', name)
except KeyError:
sem = threading.Semaphore()
_semaphores[name] = sem
LOG.debug('Created new semaphore "%s"', name)
return sem
@contextlib.contextmanager
def lock(name, lock_file_prefix=None, external=False, lock_path=None):
"""Context based lock
This function yields a `threading.Semaphore` instance (if we don't use
eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is
True, in which case, it'll yield an InterProcessLock instance.
:param lock_file_prefix: The lock_file_prefix argument is used to provide
lock files on disk with a meaningful prefix.
:param external: The external keyword argument denotes whether this lock
should work across multiple processes. This means that if two different
workers both run a method decorated with @synchronized('mylock',
external=True), only one of them will execute at a time.
"""
int_lock = internal_lock(name)
with int_lock:
LOG.debug('Acquired semaphore "%(lock)s"', {'lock': name})
try:
if external and not CONF.disable_process_locking:
ext_lock = external_lock(name, lock_file_prefix, lock_path)
with ext_lock:
yield ext_lock
else:
yield int_lock
finally:
LOG.debug('Releasing semaphore "%(lock)s"', {'lock': name})
def synchronized(name, lock_file_prefix=None, external=False, lock_path=None):
"""Synchronization decorator.
Decorating a method like so::
@synchronized('mylock')
def foo(self, *args):
...
ensures that only one thread will execute the foo method at a time.
Different methods can share the same lock::
@synchronized('mylock')
def foo(self, *args):
...
@synchronized('mylock')
def bar(self, *args):
...
This way only one of either foo or bar can be executing at a time.
"""
def wrap(f):
@functools.wraps(f)
def inner(*args, **kwargs):
try:
with lock(name, lock_file_prefix, external, lock_path):
LOG.debug('Got semaphore / lock "%(function)s"',
{'function': f.__name__})
return f(*args, **kwargs)
finally:
LOG.debug('Semaphore / lock released "%(function)s"',
{'function': f.__name__})
return inner
return wrap
def synchronized_with_prefix(lock_file_prefix):
"""Partial object generator for the synchronization decorator.
Redefine @synchronized in each project like so::
(in nova/utils.py)
from nova.openstack.common import lockutils
synchronized = lockutils.synchronized_with_prefix('nova-')
(in nova/foo.py)
from nova import utils
@utils.synchronized('mylock')
def bar(self, *args):
...
The lock_file_prefix argument is used to provide lock files on disk with a
meaningful prefix.
"""
return functools.partial(synchronized, lock_file_prefix=lock_file_prefix)
def main(argv):
"""Create a dir for locks and pass it to command from arguments
If you run this:
python -m openstack.common.lockutils python setup.py testr <etc>
a temporary directory will be created for all your locks and passed to all
your tests in an environment variable. The temporary dir will be deleted
afterwards and the return value will be preserved.
"""
lock_dir = tempfile.mkdtemp()
os.environ["NEUTRON_LOCK_PATH"] = lock_dir
try:
ret_val = subprocess.call(argv[1:])
finally:
shutil.rmtree(lock_dir, ignore_errors=True)
return ret_val
if __name__ == '__main__':
sys.exit(main(sys.argv))

2
neutron/plugins/cisco/cfg_agent/cfg_agent.py
View File

@ -22,6 +22,7 @@ from oslo.config import cfg
from oslo import messaging
from oslo.utils import importutils
from oslo.utils import timeutils
from oslo_concurrency import lockutils
from neutron.agent.common import config
from neutron.agent.linux import external_process
@ -33,7 +34,6 @@ from neutron.common import topics
from neutron import context as n_context
from neutron.i18n import _LE, _LI, _LW
from neutron import manager
from neutron.openstack.common import lockutils
from neutron.openstack.common import log as logging
from neutron.openstack.common import loopingcall
from neutron.openstack.common import periodic_task

2
neutron/plugins/cisco/db/l3/l3_router_appliance_db.py
View File

@ -15,6 +15,7 @@
import copy
from oslo.config import cfg
from oslo_concurrency import lockutils
from sqlalchemy.orm import exc
from sqlalchemy.orm import joinedload
from sqlalchemy.sql import expression as expr
@ -31,7 +32,6 @@ from neutron.db import models_v2
from neutron.db import portbindings_db as p_binding
from neutron.extensions import providernet as pr_net
from neutron.i18n import _LE, _LI
from neutron.openstack.common import lockutils
from neutron.openstack.common import log as logging
from neutron.openstack.common import loopingcall
from neutron.plugins.cisco.common import cisco_constants as c_const

2
neutron/plugins/ml2/drivers/cisco/apic/apic_topology.py
View File

@ -22,6 +22,7 @@ eventlet.monkey_patch()
from oslo.config import cfg
from oslo import messaging
from oslo_concurrency import lockutils
from neutron.agent.common import config
from neutron.agent.linux import ip_lib
@ -32,7 +33,6 @@ from neutron.common import utils as neutron_utils
from neutron.db import agents_db
from neutron.i18n import _LE, _LI
from neutron import manager
from neutron.openstack.common import lockutils
from neutron.openstack.common import log as logging
from neutron.openstack.common import periodic_task
from neutron.openstack.common import service as svc

2
neutron/plugins/ml2/drivers/cisco/apic/mechanism_apic.py
View File

@ -17,9 +17,9 @@ from apicapi import apic_manager
from keystoneclient.v2_0 import client as keyclient
import netaddr
from oslo.config import cfg
from oslo_concurrency import lockutils
from neutron.common import constants as n_constants
from neutron.openstack.common import lockutils
from neutron.openstack.common import log
from neutron.plugins.common import constants
from neutron.plugins.ml2 import driver_api as api

2
neutron/plugins/ml2/plugin.py
View File

@ -21,6 +21,7 @@ from oslo.db import exception as os_db_exception
from oslo.serialization import jsonutils
from oslo.utils import excutils
from oslo.utils import importutils
from oslo_concurrency import lockutils
from sqlalchemy import exc as sql_exc
from sqlalchemy.orm import exc as sa_exc
@ -55,7 +56,6 @@ from neutron.extensions import portbindings
from neutron.extensions import providernet as provider
from neutron.i18n import _LE, _LI, _LW
from neutron import manager
from neutron.openstack.common import lockutils
from neutron.openstack.common import log
from neutron.openstack.common import uuidutils
from neutron.plugins.common import constants as service_constants

2
neutron/plugins/nuage/plugin.py
View File

@ -19,6 +19,7 @@ import netaddr
from oslo.config import cfg
from oslo.utils import excutils
from oslo.utils import importutils
from oslo_concurrency import lockutils
from sqlalchemy.orm import exc
from neutron.api import extensions as neutron_extensions
@ -38,7 +39,6 @@ from neutron.extensions import l3
from neutron.extensions import portbindings
from neutron.extensions import providernet as pnet
from neutron.extensions import securitygroup as ext_sg
from neutron.openstack.common import lockutils
from neutron.openstack.common import log as logging
from neutron.openstack.common import loopingcall
from neutron.plugins.nuage.common import config

2
neutron/plugins/nuage/syncmanager.py
View File

@ -15,6 +15,7 @@
from oslo.config import cfg
from oslo.utils import importutils
from oslo_concurrency import lockutils
import sqlalchemy.orm.exc as db_exc
from neutron import context as ncontext
@ -22,7 +23,6 @@ from neutron.db import db_base_plugin_v2
from neutron.db import extraroute_db
from neutron.db import securitygroups_db
from neutron.i18n import _LE, _LI, _LW
from neutron.openstack.common import lockutils
from neutron.openstack.common import log
from neutron.plugins.nuage.common import config
from neutron.plugins.nuage import nuagedb

2
neutron/plugins/vmware/plugins/base.py
View File

@ -18,6 +18,7 @@ import uuid
from oslo.config import cfg
from oslo.db import exception as db_exc
from oslo.utils import excutils
from oslo_concurrency import lockutils
from sqlalchemy import exc as sql_exc
from sqlalchemy.orm import exc as sa_exc
import webob.exc
@ -52,7 +53,6 @@ from neutron.extensions import portsecurity as psec
from neutron.extensions import providernet as pnet
from neutron.extensions import securitygroup as ext_sg
from neutron.i18n import _LE, _LI, _LW
from neutron.openstack.common import lockutils
from neutron.openstack.common import log as logging
from neutron.plugins.common import constants as plugin_const
from neutron.plugins import vmware

3
neutron/services/advanced_service.py
View File

@ -13,7 +13,8 @@
# License for the specific language governing permissions and limitations
# under the License.
from neutron.openstack.common import lockutils
from oslo_concurrency import lockutils
from neutron.openstack.common import log as logging
LOG = logging.getLogger(__name__)

3
neutron/tests/base.py
View File

@ -29,6 +29,7 @@ import os.path
import fixtures
from oslo.config import cfg
from oslo.messaging import conffixture as messaging_conffixture
from oslo_concurrency.fixture import lockutils
from neutron.common import config
from neutron.common import rpc as n_rpc
@ -85,6 +86,8 @@ class BaseTestCase(sub_base.SubBaseTestCase):
nuke_handlers=capture_logs,
))
self.useFixture(lockutils.ExternalLockFixture())
self.temp_dir = self.useFixture(fixtures.TempDir()).path
cfg.CONF.set_override('state_path', self.temp_dir)

3
neutron/tests/unit/test_security_groups_rpc.py
View File

@ -2515,9 +2515,6 @@ class TestSecurityGroupAgentWithIptables(base.BaseTestCase):
super(TestSecurityGroupAgentWithIptables, self).setUp()
config.register_root_helper(cfg.CONF)
config.register_iptables_opts(cfg.CONF)
cfg.CONF.set_override(
'lock_path',
'$state_path/lock')
set_firewall_driver(self.FIREWALL_DRIVER)
cfg.CONF.set_override('enable_ipset', False, group='SECURITYGROUP')
cfg.CONF.set_override('comment_iptables_rules', False, group='AGENT')

1
openstack-common.conf
View File

@ -5,7 +5,6 @@ module=eventlet_backdoor
module=fileutils
module=install_venv_common
module=local
module=lockutils
module=log
module=loopingcall
module=middleware

1
requirements.txt
View File

@ -22,6 +22,7 @@ python-keystoneclient>=0.11.1
alembic>=0.7.1
six>=1.7.0
stevedore>=1.1.0 # Apache-2.0
oslo.concurrency>=0.3.0,!=0.4.0
oslo.config>=1.6.0 # Apache-2.0
oslo.context>=0.1.0 # Apache-2.0
oslo.db>=1.3.0 # Apache-2.0

2
run_tests.sh
View File

@ -196,7 +196,7 @@ function run_pep8_changed {
}
TESTRTESTS="python -m neutron.openstack.common.lockutils python setup.py testr"
TESTRTESTS="python setup.py testr"
if [ $never_venv -eq 0 ]
then

2
tools/pretty_tox.sh
View File

@ -3,4 +3,4 @@
TESTRARGS=$1
exec 3>&1
status=$(exec 4>&1 >&3; ( python -m neutron.openstack.common.lockutils python setup.py testr --slowest --testr-args="--subunit $TESTRARGS"; echo $? >&4 ) | subunit-trace -f) && exit $status
status=$(exec 4>&1 >&3; ( python setup.py testr --slowest --testr-args="--subunit $TESTRARGS"; echo $? >&4 ) | subunit-trace -f) && exit $status

2
tox.ini
View File

@ -61,7 +61,7 @@ commands = python ./tools/check_i18n.py ./neutron ./tools/i18n_cfg.py
[testenv:cover]
commands =
python -m neutron.openstack.common.lockutils python setup.py testr --coverage --testr-args='{posargs}'
python setup.py testr --coverage --testr-args='{posargs}'
[testenv:venv]
commands = {posargs}