Merge "Import osa-gate-profile"

osa-gate-profile/branchprofile.php

@@ -0,0 +1,103 @@
+#!/usr/bin/php
+<?php
+/*
+  Copyright 2016, Logan Vig <logan2211@gmail.com>
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+$fetch_results = 10000;
+$search_days = 14;
+
+$search = '{"query":{"filtered":{"filter":{"bool":{"must":[{"range":{"@timestamp":{"from":1463241312439,"to":1463846112439}}},{"fquery":{"query":{"query_string":{"query":"project:(\"openstack\/openstack-ansible\")"}},"_cache":true}},{"fquery":{"query":{"query_string":{"query":"build_name:(\"gate-openstack-ansible-dsvm-commit\")"}},"_cache":true}},{"fquery":{"query":{"query_string":{"query":"message:(\"- Operation: [\u00a0openstack-ansible --forks\")"}},"_cache":true}}]}}}},"size":10000,"sort":[{"@timestamp":{"order":"desc","ignore_unmapped":true}},{"@timestamp":{"order":"desc","ignore_unmapped":true}}]}';
+$search = json_decode($search);
+$ts =& $search->query->filtered->filter->bool->must[0]->range->{'@timestamp'};
+$ts->from = strtotime("-$search_days days")*1000;
+$ts->to = time()*1000;
+
+//remove timestamp limit from search
+//$nothing = array_shift($search->query->filtered->filter->bool->must);
+
+$search->size = $fetch_results;
+$search = json_encode($search);
+
+$date = time();
+$stats = array();
+$total_samples = 0;
+while ($total_samples < $fetch_results && $date > strtotime("-$search_days days")) {
+  $strdate = date('Y.m.d', $date);
+  $date -= 86400;
+
+  echo "Fetching results for $strdate\n";
+  $url = "http://logstash.openstack.org/elasticsearch/logstash-$strdate/_search";
+  $result = json_decode(fetch_results($search, $url));
+  if (empty($result) || isset($result->error) || $result->status == 404) {
+    echo "Error fetching results for $strdate.. skipping\n";
+    continue;
+  }
+
+  echo "Analyzing ".count($result->hits->hits)." samples\n";
+  $total_samples += process_results($result, $stats);
+}
+
+foreach ($stats as &$branch) {
+  foreach ($branch as &$nodepool) {
+    foreach ($nodepool as &$playbook) {
+      $avg = array_sum($playbook) / count($playbook);
+      $playbook['samples'] = count($playbook);
+      $playbook['average'] = $avg;
+    }
+  }
+}
+
+foreach ($stats as $bname => $branch) {
+  echo "Stats for openstack-ansible/$bname\n";
+  foreach ($branch as $npname => $nodepool) {
+    echo "Node pool $npname:\n";
+    uasort($nodepool, 'avg_cmp');
+    foreach ($nodepool as $pbname => $playbook) {
+      echo "\t$pbname: ".ceil($playbook['average'])." avg, ({$playbook['samples']} samples)\n";
+    }
+  }
+}
+
+function fetch_results($search, $url) {
+  $ch = curl_init($url);
+  curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
+  curl_setopt($ch, CURLOPT_POST, true);
+  curl_setopt($ch, CURLOPT_POSTFIELDS, $search);
+  curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+  curl_setopt($ch, CURLOPT_HTTPHEADER, array(
+      'Content-Type: application/json',
+      'Content-Length: ' . strlen($search))
+  );
+  return curl_exec($ch);
+}
+
+function process_results($result, &$stats) {
+  $regex = '/Operation: \[ openstack-ansible --forks [0-9]+\s+(?<play_name>[^\.]+).yml \]\s+(?<play_seconds>[0-9]+)\s+seconds/';
+  foreach ($result->hits->hits as $r) {
+    $rs = $r->{'_source'};
+    if (!preg_match($regex, $rs->message, $m)) continue;
+    $stats[$rs->build_branch][$rs->node_provider][$m['play_name']][] = $m['play_seconds'];
+  }
+  return count($result->hits->hits);
+}
+
+function avg_cmp($a, $b) {
+  if ($a['average'] == $b['average']) {
+    return 0;
+  }
+  return ($a['average'] > $b['average']) ? -1 : 1;
+}
+?>

osa-gate-profile/fetchlogs.php

@@ -0,0 +1,107 @@
+#!/usr/bin/php
+<?php
+/*
+  Copyright 2016, Logan Vig <logan2211@gmail.com>
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+ * Argument required is json output from logstash.openstack.org's table of metrics. There is no an export option so hack this together by changing the paging size
+ * and watch the network traffic in chrome, then save it to a file.
+ * Once the file exists, feed it to this script as an argument and it will go through and download all of the log_url items to a directory called 'dump'
+ * You should probably mkdir dump/ in the same directory as this script first.
+ */
+
+date_default_timezone_set('US/Central');
+
+$fetch_results = 500;
+$search_days = 30;
+
+$search = '{"query":{"filtered":{"filter":{"bool":{"must":[{"range":{"@timestamp":{"from":1463241312439,"to":1463846112439}}},{"fquery":{"query":{"query_string":{"query":"project:(\"openstack\/openstack-ansible\")"}},"_cache":true}},{"fquery":{"query":{"query_string":{"query":"build_name:(/gate-openstack-ansible-openstack-ansible-aio-ubuntu-(trusty|xenial-nv)/)"}},"_cache":true}},{"fquery":{"query":{"query_string":{"query":"message:(\"gate-check-commit.sh\")"}},"_cache":true}}]}}}},"size":'.$fetch_results.',"sort":[{"@timestamp":{"order":"desc","ignore_unmapped":true}},{"@timestamp":{"order":"desc","ignore_unmapped":true}}]}';
+$search = json_decode($search);
+$ts =& $search->query->filtered->filter->bool->must[0]->range->{'@timestamp'};
+$ts->from = strtotime("-$search_days days")*1000;
+$ts->to = time()*1000;
+
+$search->size = $fetch_results;
+$search = json_encode($search);
+
+$date = time();
+$stats = array();
+$total_samples = 0;
+while ($total_samples < $fetch_results && $date > strtotime("-$search_days days")) {
+  $strdate = date('Y.m.d', $date);
+  $date -= 86400;
+
+  echo "Fetching results for $strdate\n";
+  $url = "http://logstash.openstack.org/elasticsearch/logstash-$strdate/_search";
+  $result = json_decode(fetch_results($search, $url));
+  if (empty($result) || isset($result->error) || $result->status == 404) {
+    echo "Error fetching results for $strdate.. skipping\n";
+    continue;
+  }
+
+  echo "Fetching ".count($result->hits->hits)." samples\n";
+  foreach($result->hits->hits as $h) {
+    $outfile = $h->{'_source'}->{'build_change'}.'-'.$h->{'_source'}->{'build_patchset'}.'.html';
+    if (file_exists('dump/'.$outfile)) {
+      echo "Already downloaded $outfile..skipping\n";
+      continue;
+    }
+    $url = $h->{'_source'}->{'log_url'};
+    $wget = "wget -O dump/{$outfile} {$url}";
+    echo "running $wget\n";
+    echo "Fetching $url to $outfile\n";
+    $exec = fetch_log($url, $outfile);
+    if ($exec['exit_code'] != 0) {
+      $url = $url.'.gz';
+      echo "Fetch failed. Trying $url\n";
+      $exec = fetch_log($url, $outfile);
+      if ($exec['return_code'] != 0) echo "Fetch failed retry. Skipping file.\n";
+      else echo "Fetch succeeded retry\n";
+    }
+    else echo "Fetch succeeded\n";
+  }
+}
+
+function fetch_log($url, $outfile) {
+  $wget = "wget -O dump/{$outfile} {$url}";
+  exec($wget, $shell_output, $code);
+  return array('output' => $shell_output, 'exit_code' => $code);
+}
+
+function fetch_results($search, $url) {
+  $ch = curl_init($url);
+  //curl_setopt($ch, CURLOPT_VERBOSE, true);
+  //curl_setopt($ch, CURLOPT_HEADER, true);
+  curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
+  curl_setopt($ch, CURLOPT_POST, true);
+  curl_setopt($ch, CURLOPT_POSTFIELDS, $search);
+  curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+  curl_setopt($ch, CURLOPT_HTTPHEADER, array(
+      'Content-Type: application/json',
+      'Content-Length: ' . strlen($search))
+  );
+  $result = curl_exec($ch);
+  return $result;
+}
+
+function process_results($result, &$stats) {
+  $regex = '/Operation: \[ openstack-ansible --forks [0-9]+\s+(?<play_name>[^\.]+).yml \]\s+(?<play_seconds>[0-9]+)\s+seconds/';
+  foreach ($result->hits->hits as $r) {
+    $rs = $r->{'_source'};
+    if (!preg_match($regex, $rs->message, $m)) continue;
+    $stats[$rs->build_branch][$rs->node_provider][$m['play_name']][] = $m['play_seconds'];
+  }
+  return count($result->hits->hits);
+}
+?>

osa-gate-profile/generatereports.php

@@ -0,0 +1,39 @@
+#!/usr/bin/php
+<?php
+/*
+  Copyright 2016, Logan Vig <logan2211@gmail.com>
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+$f = json_decode(file_get_contents($argv[1]), true);
+$profiles = [];
+$sorted = array();
+foreach ($f as $taskname => $nodeproviders) {
+  echo "$taskname:\n";
+  foreach ($nodeproviders as $p => $n) {
+    $avg = round(array_sum($n) / count($n), 2);
+    $samples = count($n);
+    echo "\t$p ($samples samples) - $avg\n";
+    $sorted[$p][$taskname] = $avg;
+  }
+}
+
+foreach ($sorted as $k => $v) {
+  echo $k."\n";
+  arsort($v);
+  foreach ($v as $tasks => $avgs) {
+    echo "\t$tasks = $avgs\n";
+  }
+}
+?>

osa-gate-profile/parselogs.php

@@ -0,0 +1,66 @@
+#!/usr/bin/php
+<?php
+/*
+  Copyright 2016, Logan Vig <logan2211@gmail.com>
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+ * Parse all of the console logs saved to dump/*
+ * This will try to regex match the first few lines which identify the nodepool
+ * Then it will search through the console log for PLAY RECAP style output which contains execution times
+ * for all of the ansible tasks run. Once it is done it will output a json object of metrics.
+ * Example output at http://cdn.pasteraw.com/5efufwesy1nmxnl6qis9z1zr1hbce0
+ */
+$dir = new DirectoryIterator(dirname(__FILE__).'/dump/');
+$profile = array();
+$profiles=0;
+
+pcntl_signal(SIGHUP,  function($signo) {
+  global $profile;
+  echo json_encode($profile)."\n";
+});
+pcntl_signal(SIGTERM,  function($signo) {
+  print_exit();
+});
+pcntl_signal(SIGINT,  function($signo) {
+  print_exit();
+});
+
+foreach ($dir as $f) {
+  if (!$f->isFile()) continue;
+  $c = file_get_contents('dump/'.$f);
+  if (!preg_match('/Building remotely on (?:<a .*>)?([^<\s]+)-[0-9]+(?:<\/a>)?/', $c, $m)) {
+    file_put_contents("php://stderr", "Failed to find node type of $f\n");
+    continue;
+  }
+  $nodetype = $m[1];
+  foreach (explode("\n", $c) as $l) {
+    if (preg_match('/(?:[^|]+)\|\s+(.*)\s+[-]+\s+([0-9\.s]+)s\s*$/', $l, $m)) {
+      $role_task = $m[1];
+      $seconds = (float)$m[2];
+      $profile[$role_task][$nodetype][] = $seconds;
+    }
+  }
+  $profiles++;
+  pcntl_signal_dispatch();
+}
+
+print_exit();
+
+function print_exit() {
+  global $profile, $profiles;
+  file_put_contents("php://stderr", "\nExiting with $profiles profiles completed.\n");
+  exit(json_encode($profile));
+}
+
+?>

osa-gate-profile/readme.rst

@@ -0,0 +1,27 @@
+Generate metrics from openstack-ansible gate check console logs
+###############################################################
+:date: 2016-10-10
+:tags: openstack, ansible
+:category: \*openstack, \*nix
+
+
+About this repository
+---------------------
+
+These scripts will query logstash.openstack.org to find a set of OSA gate check
+console logs, download them, and perform task timing analytics.
+
+- step1: fetchlogs.php
+- step2: parselogs.php
+- step3: generatereports.php
+
+Example run
+-----------
+
+.. code-block:: bash
+
+    mkdir dump
+    php fetchlogs.php
+    php parselogs.php > intermediary.json
+    php generatereports.php intermediary.json
+    rm -rf intermediary.json