Fix DBDeadlock during clean up

Nodes old status clean-up has a top-level DB transaction that conflicts
with the node.fsm_event(timeout), eventually causing a deadlock.

Bumping the node.fsm_event(timeout) out from the top-level DB
transaction fixes the issue.

Also remove redundant attributes and options db records clean up and an
unneeded db flush on commit.

Closes-Bug: 1658065
Change-Id: I21a2fd051d0ee05ffa66464d75c912a0968612d6
This commit is contained in:
Anton Arefiev 2017-01-19 19:00:16 +02:00 committed by Milan Kováčik
parent 0d2bc1a47c
commit 31cd5f40cd
2 changed files with 16 additions and 18 deletions

View File

@ -165,7 +165,6 @@ class NodeInfo(object):
self._set_version_id(uuidutils.generate_uuid(), session)
row = self._row(session)
row.update(fields)
row.save(session)
def commit(self):
"""Commit current node status into the database."""
@ -857,24 +856,19 @@ def clean_up():
db.model_query(db.Node.uuid, session=session).filter(
db.Node.started_at < threshold,
db.Node.finished_at.is_(None)).all()]
if not uuids:
return []
if not uuids:
return []
LOG.error(_LE('Introspection for nodes %s has timed out'), uuids)
for u in uuids:
node_info = get_node(u, locked=True)
try:
if node_info.finished_at or node_info.started_at > threshold:
continue
node_info.fsm_event(istate.Events.timeout)
node_info.finished(error='Introspection timeout')
db.model_query(db.Attribute, session=session).filter_by(
uuid=u).delete()
db.model_query(db.Option, session=session).filter_by(
uuid=u).delete()
finally:
node_info.release_lock()
LOG.error(_LE('Introspection for nodes %s has timed out'), uuids)
for u in uuids:
node_info = get_node(u, locked=True)
try:
if node_info.finished_at or node_info.started_at > threshold:
continue
node_info.fsm_event(istate.Events.timeout)
node_info.finished(error='Introspection timeout')
finally:
node_info.release_lock()
return uuids

View File

@ -0,0 +1,4 @@
---
fixes:
- |
Periodic clean up fails with DBDeadlock if introspection timeout.