process-relvals-exceptions-es

#!/usr/bin/env python3
from argparse import ArgumentParser
from subprocess import getstatusoutput
from operator import itemgetter
from string import maketrans
import json
import os
import re

# Given an IB, it queries the information from elasticsearch looking for the different exceptions 
# that were found in the relvals. 
# It then creates the corresponding json files with the results that it finds. 

GET_ALL_INFO_EXCEPTIONS_QUERY_FILE="es-queries/get-wf-arch-exception-from-ib.json"
JSON_OUT_FILE_TEMPLATE="data/relvals/{arch}/{date}/{release_queue}_EXCEPTIONS.json"
JSON_RELVALS_FILE_TEMPLATE="data/relvals/{arch}/{date}/{release_queue}.json"

#
# makes a query to get information from elasticsearch
#
def query_es( query ):
  cmd = "curl -s -XPOST http://{hostname}/{index}/_search -d '{query}'".format(
         hostname=ES_HOSTNAME,
         index=ES_INDEX,
         query=query )
  err, out = getstatusoutput(cmd)
  if err:
    print( "Error while querying elasticsearch" )
    print( out )
    exit(0)

  try:
    response = json.loads( out )
  except Exception as e:
    print ('Error loading elasticsearch response')
    print( 'Reason: ' + str(e) )
    return {}
  return response

#
# Queries elasticsearch to get the workflow_id, architecture and exception of
# the documents that have an exception 
#
def get_all_info_exceptions( ib ):
  query = open( GET_ALL_INFO_EXCEPTIONS_QUERY_FILE, 'r' ).read().replace( 'IB_NAME', ib )
  print( "Querying exceptions for %s" % ib )
  response = query_es( query )
  if response[ 'hits' ][ 'total' ] == 0:
    print 'No exceptions found for ', ib, ' I cannot continue'
    exit( 0 )
  all_info_exceptions = [ x[ 'fields' ] for x in response[ 'hits' ][ 'hits' ] ]
  return all_info_exceptions
 


#
# organizes the results of the query in a estructure that simplyfies the generation
# of the json file
# The structure is as follows:
# results = {
#           "<arch1>":{ "<exceptionA>":[ "<wf1>", "<wf2>, ..., "<wfN>" ],
#                       "<exceptionB>":[ "<wf1>", "<wf2>, ..., "<wfN>" ]
#                     },
#           "<arch2>":{ "<exceptionB>":[ "<wf1>", "<wf2>, ..., "<wfN>" ],
#                       "<exceptionC>":[ "<wf1>", "<wf2>, ..., "<wfN>" ]
#                     } 
# }
#
def organize_exceptions( all_info_exceptions ):
  results = {}

  for document in all_info_exceptions:
    workflow_id = str( round( document[ 'workflow_id' ][0], 2 ) )
    arch = document[ 'architecture' ][0]
    exception = document[ 'exception' ][0]

    if not results.get( arch ):
      results[ arch ] = {}

    if not results[ arch ].get( exception ):
      results[ arch ][ exception ] = []

    results[ arch ][ exception ].append( workflow_id )

  return results

#---------------------------------------------------------------
# Start
#--------------------------------------------------------------
if __name__ == "__main__":
  parser = ArgumentParser()
  parser.add_argument("input")
  parser.add_argument("--es_hostname", type=str, help="elasticsearch hostname", required=True)
  parser.add_argument("--es_rv_index", type=str, help="elasticsearch relvals index", required=True)
  args = parser.parse_args()

  ES_HOSTNAME = args.es_hostname
  ES_INDEX = args.es_rv_index

  if ( not args.input ):
    print ("Please specify an IB to query")
    exit(1)

  ib = args.input

  all_info_exceptions = get_all_info_exceptions( ib )

  results = organize_exceptions( all_info_exceptions )

  # create the json file
  ib_rq = re.match("(.*)_2[0-9]*-[01][0-9]-[0-3][0-9]-[0-9]*$", ib).group(1)
  ib_date = re.match(".*(2[0-9]*-[01][0-9]-[0-3][0-9]-[0-9]*)$", ib).group(1)

  for arch in results.keys():
    out_file_path = JSON_OUT_FILE_TEMPLATE.format( arch=arch, date=ib_date, release_queue=ib_rq )
    dir_file_path = out_file_path[:out_file_path.rfind("/")]
    if not os.path.exists( dir_file_path ):
      os.makedirs( dir_file_path ) 
 
    print ('Generating exceptions file ', out_file_path)
    out_file = open( out_file_path, 'w' )
    json.dump( results[ arch ], out_file, sort_keys=True, indent=4, separators=(',', ': ') )
    out_file.close()