Fix DBDeadlock during clean up
Nodes old status clean-up has a top-level DB transaction that conflicts with the node.fsm_event(timeout), eventually causing a deadlock. Bumping the node.fsm_event(timeout) out from the top-level DB transaction fixes the issue. Also remove redundant attributes and options db records clean up and an unneeded db flush on commit. Closes-Bug: 1658065 Change-Id: I21a2fd051d0ee05ffa66464d75c912a0968612d6
This commit is contained in:
parent
0d2bc1a47c
commit
31cd5f40cd
@ -165,7 +165,6 @@ class NodeInfo(object):
|
||||
self._set_version_id(uuidutils.generate_uuid(), session)
|
||||
row = self._row(session)
|
||||
row.update(fields)
|
||||
row.save(session)
|
||||
|
||||
def commit(self):
|
||||
"""Commit current node status into the database."""
|
||||
@ -857,24 +856,19 @@ def clean_up():
|
||||
db.model_query(db.Node.uuid, session=session).filter(
|
||||
db.Node.started_at < threshold,
|
||||
db.Node.finished_at.is_(None)).all()]
|
||||
if not uuids:
|
||||
return []
|
||||
if not uuids:
|
||||
return []
|
||||
|
||||
LOG.error(_LE('Introspection for nodes %s has timed out'), uuids)
|
||||
for u in uuids:
|
||||
node_info = get_node(u, locked=True)
|
||||
try:
|
||||
if node_info.finished_at or node_info.started_at > threshold:
|
||||
continue
|
||||
node_info.fsm_event(istate.Events.timeout)
|
||||
node_info.finished(error='Introspection timeout')
|
||||
|
||||
db.model_query(db.Attribute, session=session).filter_by(
|
||||
uuid=u).delete()
|
||||
db.model_query(db.Option, session=session).filter_by(
|
||||
uuid=u).delete()
|
||||
finally:
|
||||
node_info.release_lock()
|
||||
LOG.error(_LE('Introspection for nodes %s has timed out'), uuids)
|
||||
for u in uuids:
|
||||
node_info = get_node(u, locked=True)
|
||||
try:
|
||||
if node_info.finished_at or node_info.started_at > threshold:
|
||||
continue
|
||||
node_info.fsm_event(istate.Events.timeout)
|
||||
node_info.finished(error='Introspection timeout')
|
||||
finally:
|
||||
node_info.release_lock()
|
||||
|
||||
return uuids
|
||||
|
||||
|
@ -0,0 +1,4 @@
|
||||
---
|
||||
fixes:
|
||||
- |
|
||||
Periodic clean up fails with DBDeadlock if introspection timeout.
|
Loading…
Reference in New Issue
Block a user