[Prometheus] Relax disk IO constraints

Relax the timing constrains for disk IO to accommodate rotating disks;
a "measured IO" might be the result of a small number of physical IOs,
allow for enough time for a small number of disk rotations (this isn't
perfect but seems to be about right in testing under load).

Change-Id: Ifb067a2218528e5918d2f4b2ba169b6e739084e0
This commit is contained in:
Chris Wedgwood 2019-01-28 23:10:02 +00:00
parent 4fb6ee6e35
commit d7808468fc

View File

@ -1610,7 +1610,7 @@ conf:
description: '{{$labels.alias}} has time difference of more than 2 seconds compared to NTP server: {{ $value }}'
summary: '{{$labels.alias}}: time is skewed by : {{$value}} seconds'
- alert: node_disk_read_latency
expr: (rate(node_disk_read_time_ms[5m]) / rate(node_disk_reads_completed[5m])) > 10
expr: (rate(node_disk_read_time_ms[5m]) / rate(node_disk_reads_completed[5m])) > 40
for: 5m
labels:
severity: page
@ -1618,7 +1618,7 @@ conf:
description: '{{$labels.device}} has a high read latency of {{ $value }}'
summary: 'High read latency observed for device {{ $labels.device }}'
- alert: node_disk_write_latency
expr: (rate(node_disk_write_time_ms[5m]) / rate(node_disk_writes_completed[5m])) > 10
expr: (rate(node_disk_write_time_ms[5m]) / rate(node_disk_writes_completed[5m])) > 40
for: 5m
labels:
severity: page