From 01c40bf873e510337f896ff97030f21fbb6c4986 Mon Sep 17 00:00:00 2001 From: Dmitry Romanov Date: Sat, 29 Jun 2024 13:05:08 -0400 Subject: [PATCH] Cleanup and update scripts --- scripts/find_ccdb_usages/find_ccdb_usages.py | 36 +-- .../halld_convert_from_svn/convert_script.py | 303 ------------------ .../install_web_site/ccdb_web_installer.py | 111 ------- scripts/install_web_site/install_fedora.py | 37 --- scripts/install_web_site/install_ifarml64.py | 37 --- scripts/mysql_query_parser/parse.go | 2 - scripts/users_create/group_parse.pl | 15 - scripts/users_create/users_create.py | 39 +-- 8 files changed, 34 insertions(+), 546 deletions(-) delete mode 100644 scripts/halld_convert_from_svn/convert_script.py delete mode 100644 scripts/install_web_site/ccdb_web_installer.py delete mode 100644 scripts/install_web_site/install_fedora.py delete mode 100644 scripts/install_web_site/install_ifarml64.py delete mode 100755 scripts/users_create/group_parse.pl diff --git a/scripts/find_ccdb_usages/find_ccdb_usages.py b/scripts/find_ccdb_usages/find_ccdb_usages.py index 80c3cac1..7150ce40 100644 --- a/scripts/find_ccdb_usages/find_ccdb_usages.py +++ b/scripts/find_ccdb_usages/find_ccdb_usages.py @@ -5,24 +5,24 @@ import re def search_calib_function(line): - #preg = "->Get.*\\\((?P^[0-9]*)(?P[0-9a-zA-Z_]+)*(=(?P.*))*" - #if self.no_columns_quantity: - # preg = "(?P[0-9a-zA-Z_]+)*(=(?P.*))*" - try: - preg = '^.*>Get(Calib)*\\((?P[\w/"]+),\s*(?P[0-9a-zA-Z_]+)(,.*)*\\)' #\\\((?P^.+),(?P[0-9a-zA-Z_]+)\\\)' - - m = re.match(preg, line) - - if not m: - return None - #print line - result={} - - if m.group("namepath") and m.group("variable"): + #preg = "->Get.*\\\((?P^[0-9]*)(?P[0-9a-zA-Z_]+)*(=(?P.*))*" + #if self.no_columns_quantity: + # preg = "(?P[0-9a-zA-Z_]+)*(=(?P.*))*" + try: + preg = '^.*>Get(Calib)*\\((?P[\w/"]+),\s*(?P[0-9a-zA-Z_]+)(,.*)*\\)' #\\\((?P^.+),(?P[0-9a-zA-Z_]+)\\\)' + + m = re.match(preg, line) + + if not m: + return None + #print line + result={} + + if m.group("namepath") and m.group("variable"): return (m.group("namepath"), m.group("variable")) - except Exception as ex: - print ex + except Exception as ex: + print(ex) return None @@ -48,7 +48,7 @@ def check_name(name): ]) report = "" -print "SEARCHING {} files in {} directories".format(len(files_to_process), len(dirs_to_search)) +print("SEARCHING {} files in {} directories".format(len(files_to_process), len(dirs_to_search))) for file in files_to_process: try: line_num = 0 @@ -86,4 +86,4 @@ def check_name(name): except: pass -print report \ No newline at end of file +print(report) \ No newline at end of file diff --git a/scripts/halld_convert_from_svn/convert_script.py b/scripts/halld_convert_from_svn/convert_script.py deleted file mode 100644 index 3efa18f6..00000000 --- a/scripts/halld_convert_from_svn/convert_script.py +++ /dev/null @@ -1,303 +0,0 @@ -from xml.dom.minidom import parse, parseString -from os import environ, path -from xml.dom import minidom -import xml.dom.minidom -import os -import os.path -import sys -import ccdb -import subprocess - -dom = xml.dom.minidom.getDOMImplementation(); -#dom = xml.dom.dom() -print """ -============================ -SVN CALIB to CCDB convertion -============================ - -This script aims to convers SVN file based GluEx calibration constats set to CCDB database - -Three environment variables must be set: -CCDB_HOME - ccdb package home dir -JANA_CALIB_URL - Jana callib url in form of file://path/to/data -JANA_CALIB_RULES - path to xml files with conversion rules - -run this script with '--rehearsal' flag to see what will be done. -run this script with '--execute' flag will execute ccdb commands -""" - - -#-------------------------- -# CONFIGURE CCDB PACKAGE -#-------------------------- - -#ccdb pakage path -if not "CCDB_HOME" in os.environ: - print "CCDB_HOME must be set" - exit(1) - -ccdb_dir = os.environ['CCDB_HOME'] - -# configure ccdb package to use -try: - import ccdb -except: - ccdb_python_dir = os.path.join(ccdb_dir, 'python') - sys.path.append(ccdb_python_dir) - try: - import ccdb - except: - print "Cannot load ccdb python library. Check it is in path" - exit(1) - -#configure ccdbcmd -ccdbcmd_opts = "" - - -#-------------------------------------------- -# *** CONFIGURE SVN CALIBRATION PACKAGE *** -#-------------------------------------------- - -if not "JANA_CALIB_URL" in os.environ: - print "JANA_CALIB_URL must be set" - exit(1) - -#this is directory with calib data -calib_dir = os.environ['JANA_CALIB_URL'] -calib_dir = calib_dir[7:] #skip 'file://' part - -#------------------------------------- -# *** CONFIGURE CONVERSION RULES *** -#------------------------------------- - -if not "JANA_CALIB_RULES" in os.environ: - print "JANA_CALIB_RULES must be set" - exit(1) - -#this is 'rules dir' the directory where xml files with data table definition are located -rules_dir = os.environ['JANA_CALIB_RULES'] -rules_xml_dir = os.path.join(rules_dir, 'xml') - - - - -#----------------------------- -# *** PARSE XML DIRS *** -#----------------------------- -dirs_xmldoc = minidom.parse(os.path.join(rules_xml_dir, 'directory.xml')) -xml_dirs_names = [f.attributes["name"].value for f in dirs_xmldoc.getElementsByTagName('directory')] - -#----------------------------- -# *** PARSE ARGS *** -#----------------------------- -import argparse - -parser = argparse.ArgumentParser(description='Converts existing svn HallD collibrations to CCDB callibrations.') -parser.add_argument('-r','--rehearsal', action="store_true", default=False, help='Run script to view what will be done. Script will run everything, but instead of invoking ccdb commands, they will be printed on screen') -parser.add_argument('-e','--execute', action="store_true", default=False, help='Run script and execute ccdb commands') -parser.add_argument('-v','--verbose', action="store_true", default=False, help='Print additional info') -parser.add_argument('-c','--colorless', action="store_true", default=True, help='Use colors for output') - -result = parser.parse_args() -#(execute_commands, is_reharsal) = result - -execute_ccdb_commands = result.execute -is_reharsal = result.rehearsal -is_verbose = result.verbose -if result.colorless: ccdbcmd_opts = ccdbcmd_opts+" --no-color" -#----------------------------- -# *** PRINT CONFIGURATION *** -#----------------------------- -print "CCDB path: " + ccdb_dir -print "CCDB options: " + ccdbcmd_opts -print "SVN calib: " + calib_dir -print "Converse rules: " + rules_dir -print "Cnv. rules xml: " + rules_xml_dir -print " execute_commands " + repr(execute_ccdb_commands) -print " is_reharsal " + repr(is_reharsal) -print " is verbose " + repr(is_verbose) - - -#-----------------------------------------------------" -# process_file -#-----------------------------------------------------" -def process_file(home_dir, rule_file_name, ccdb_parent_path): - - #parse xml file - rule_file_path = os.path.join(home_dir, rule_file_name); - xmldoc = minidom.parse(rule_file_path) - - #>oO - print " Processing file " + rule_file_path - print " ***********************************************" - - #get type tables - xml_tables = xmldoc.getElementsByTagName('type') - - #iterate type tables - for xml_table in xml_tables: - - #parameters - table_name = xml_table.attributes['name'].value - nrows = int(xml_table.attributes['nrow'].value) - is_name_value_format = bool(int(xml_table.attributes['namevalue'].value)) - - #comments - comments = '' - xml_comments = xml_table.getElementsByTagName('comment') - if len(xml_comments): - comments = " ".join(t.nodeValue for t in xml_comments[0].childNodes if t.nodeType == t.TEXT_NODE) - comments.replace("\r\n", "\\n") - comments.replace("\n","\\n") - comments.replace('"',"'") - - #comments = comments.replace("\\n",os.linesep) - - #print out what we've got - print " Process table: " + table_name - print " Comments: " - print " " + comments[0:50] - print " Rows Number: " + str(nrows) - print " Is namevalue: " + repr(is_name_value_format) - - - #iterate columns, create columns command - columns_create_command = '' - xml_columns = xml_table.getElementsByTagName('column') - if not is_verbose: print " Columns : " + repr(len(xml_columns)) - else: print " Columns: " - for xml_column in xml_columns: - column_name = xml_column.attributes['name'].value - column_type = xml_column.attributes['type'].value - if(is_verbose): print " {:<35} = {}".format(column_name, column_type) - columns_create_command+=' "{}={}"'.format(column_name, column_type) - - #create table command - table_path = (ccdb_parent_path + "/" + table_name).replace("//","/") - create_table_command = 'ccdb ' + ccdbcmd_opts +' mktbl --no-quantity {0} -r {1} {2} "#{3}"' - create_table_command = create_table_command.format(table_path, nrows, columns_create_command, "") - - print " Create command" - print " " + create_table_command -# print " " + create_table_command[0:50]+" ... " - if(execute_ccdb_commands): - (code, response) = get_status_output(create_table_command) - print code, response - if code!=0 or "error" in response or "failed" in response: exit("Conversion aborted") - - print - print " Filling data " - print " =============================================" - - - #now fill it with data - - data_file_path = os.path.join(ccdb_parent_path, table_name) - data_file_path = (calib_dir + "/" + data_file_path).replace("//","/") - print " Data file is: " + data_file_path - - #read dom - dom = ccdb.TextFileDOM() - if(is_name_value_format): dom = ccdb.read_namevalue_text_file(data_file_path) - else: dom = ccdb.read_ccdb_text_file(data_file_path) - - #print verbose info - if is_verbose: - print dom.column_names - print dom.rows - for i in range(min(len(dom.column_names),50)): - print "{:>35} {}".format(dom.column_names[i], dom.rows[0][i]) - - add_command = "ccdb " + ccdbcmd_opts + " add --c-comments " - if(is_name_value_format) : add_command += "--name-value " - add_command += table_path +" -v default -r 0- " + data_file_path - print add_command - - if(execute_ccdb_commands): - (code, response) = get_status_output(add_command) - print code, response - if code!=0 or "error" in response or "failed" in response: exit("Conversion aborted") - - print " =============================================" - print " Finished with file " - - - -#-----------------------------------------------------" -# process_directories -#-----------------------------------------------------" -def process_directories(processing_dir): - """ Create all directories according to rules_xml_dir""" - - print - print "----------------------------------------------------------------------------------------------------------" - print "Entered directory " + processing_dir - print "----------------------------------------------------------------------------------------------------------" - print - - #get ccdb directory that corresponds to this directory - ccdb_dir = processing_dir[len(rules_xml_dir):].replace("\\","/") - print processing_dir[len(rules_xml_dir):] - print " CCDB parent path that corresponds to this directory: " - print " " + ccdb_dir - print - - #get all xml files in the directory - sub_files = [f - for f in os.listdir(processing_dir) - if os.path.isfile(os.path.join(processing_dir, f)) and f.endswith(".xml") - ] - print " Found " + repr(len(sub_files)) + " files" - - #iterate and process xml files - for filename in sub_files: - process_file(processing_dir, filename, ccdb_dir) - - - print " Scanning for subdirectories" - sub_dirs = [dir - for dir in os.listdir(processing_dir) - if os.path.isdir(os.path.join(processing_dir, dir)) and not dir == '.svn' - ] - - if not len(sub_dirs): return - - for directory in sub_dirs: - print " found subdirectory " + directory - create_directory(directory, ccdb_dir) - process_directories(os.path.join(processing_dir, directory)) - - - #print "Xml dir names" - #for xml_name in xml_dirs_names: - # print xml_name - #create_directory(dir, '/') - print "===================================================================================================" - print "CONVERSION IS SUCCESSFULL" - -#---------------------------------- -def create_directory(dir_name, parent_path): - path = (parent_path + "/" + dir_name).replace("//","/") - print "creating ccdb directory: " + path - command = "ccdb " + ccdbcmd_opts + " mkdir " + path - print command - if(execute_ccdb_commands): - (code, response) = get_status_output(command) - print code, response - if code!=0 or "error" in response or "failed" in response: exit("Conversion aborted") - print "here" - - -def get_status_output(cmd): - pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, universal_newlines=True) - (output, errout) = pipe.communicate(None) - #assert not errout - status = pipe.returncode - return (status, output) - - -if execute_ccdb_commands or is_reharsal: - - process_directories(rules_xml_dir); - - diff --git a/scripts/install_web_site/ccdb_web_installer.py b/scripts/install_web_site/ccdb_web_installer.py deleted file mode 100644 index 09717b6a..00000000 --- a/scripts/install_web_site/ccdb_web_installer.py +++ /dev/null @@ -1,111 +0,0 @@ -import sys -import os - - -class CcdbWebInstaller: - """class that produce installation of ccdb site to target""" - - #ccdb default directory $CCDB_HOME by default - ccdb_home = "" - - #Site source directory $CCDB_HOME/web by default - web_source_path = "" - - #destination server directory where .html and .php files should be - dest_html_dir = "" - - #destination server cgi-bin directory path - dest_cgi_dir = "" - - #config file for php part - php_config_file = "" - - #default name for config file for php part - default_php_config_file = "config.php" - - #config file for cgi part - cgi_config_file = "" - - #default name for config file for cgi part - default_cgi_config_file = "ccdb.inp" - - #libccdb.so library full path - library_file_name = "libccdb.so" - - #copy libccdb.so to destination cgi-bin or not - # the copying is required if libccdb.so is not in LD_LIBRARY PATH - # of the apache server - copy_library = True - - -#------------------------------------ -# Initialization -#------------------------------------ - def __init__(self): - if "CCDB_HOME" in os.environ: - self.ccdb_home = os.environ["CCDB_HOME"] - self.web_source_path = os.path.join(self.ccdb_home, "web") - else: - print "CCDB_HOME is not set!" - -#------------------------------------ -# Instalation -#------------------------------------ - def install(self): - - #*** COPY SITE FILES *** - print "1. Copy html folder" - - all_source = os.path.join(self.web_source_path, "ccdb", "*") - all_cgi = os.path.join(self.web_source_path, "cgi-bin", "*") - - print all_source - os.system("cp -rf " + all_source + " " + self.dest_html_dir) - os.system("cp -rf " + all_cgi + " " + self.dest_cgi_dir) - - #*** COPY LIBRARY - if self.copy_library: - print "1.2 Copy libccdb.so" - full_lib_path = os.path.join(self.ccdb_home,"lib", self.library_file_name) - os.system("cp -rf " + full_lib_path + " " + self.dest_cgi_dir) - - #*** USE RIGHT CONFIG FILES *** - print "2. Apply right configs" - - if self.cgi_config_file != self.default_cgi_config_file: - orig_name = os.path.join(self.dest_cgi_dir, self.cgi_config_file) - dest_name = os.path.join(self.dest_cgi_dir, self.default_cgi_config_file) - os.system("mv " + orig_name + " " + dest_name) - - if self.php_config_file != self.default_php_config_file: - orig_name = os.path.join(self.dest_html_dir, self.php_config_file) - dest_name = os.path.join(self.dest_html_dir, self.default_php_config_file) - os.system("mv " + orig_name + " " + dest_name) - - #*** CLEANUP **** - print "3. Cleanup" - cgi_pattern = os.path.join(self.dest_cgi_dir, self.default_cgi_config_file) - php_pattern = os.path.join(self.dest_html_dir, self.default_php_config_file) - - cgi_pattern = cgi_pattern[:-4] + ".*" + cgi_pattern[-4:] - php_pattern = php_pattern[:-4] + ".*" + php_pattern[-4:] - print cgi_pattern - print php_pattern - os.system("rm -f " + cgi_pattern) - os.system("rm -f " + php_pattern) - - -#------------------------------------ -# Process information -#------------------------------------ - def print_self(self): - print "Server html directory:" - print self.dest_html_dir - print - print "Server cgi-bin directory:" - print self.dest_html_dir - print - print "Site source directory:" - print self.web_source_path - print - diff --git a/scripts/install_web_site/install_fedora.py b/scripts/install_web_site/install_fedora.py deleted file mode 100644 index 71d886fd..00000000 --- a/scripts/install_web_site/install_fedora.py +++ /dev/null @@ -1,37 +0,0 @@ -# It is assumed that by default the site is lying in /var/www/ -# and cgi directory is in 'site/cgi-bin' and html is is 'site/html' - -import ccdb_web_installer -import os - - -if __name__ == "__main__": - installer = ccdb_web_installer.CcdbWebInstaller() - - #directory where all .php, .html and other contet files are - installer.dest_html_dir = "/var/www/html" - - - #directory where cgi part is - installer.dest_cgi_dir = "/var/www/cgi-bin" - - - #config file for php part - #should be config..php - installer.php_config_file = "config.fedora.php" - - - #config file for cgi part - #should be ccdb..inp - installer.cgi_config_file = "ccdb.fedora.inp" - - - # the copying libccdb.so to destination cgi-bin is - # required if libccdb.so is not in LD_LIBRARY PATH - # of the apache server - installer.copy_library = True - - #INSTALLATION - installer.print_self() - installer.install() - diff --git a/scripts/install_web_site/install_ifarml64.py b/scripts/install_web_site/install_ifarml64.py deleted file mode 100644 index 8e42fbec..00000000 --- a/scripts/install_web_site/install_ifarml64.py +++ /dev/null @@ -1,37 +0,0 @@ -# It is assumed that by default the site is lying in /var/www/ -# and cgi directory is in 'site/cgi-bin' and html is is 'site/html' - -import ccdb_web_installer -import os - - -if __name__ == "__main__": - installer = ccdb_web_installer.CcdbWebInstaller() - - #directory where all .php, .html and other contet files are - installer.dest_html_dir = "/group/halld/www/halldweb1/html/ccdb/" - - - #directory where cgi part is - installer.dest_cgi_dir = "/group/halld/www/halldweb1/htbin/ccdb" - - - #config file for php part - #should be config..php - installer.php_config_file = "config.ifarml64.php" - - - #config file for cgi part - #should be ccdb..inp - installer.cgi_config_file = "ccdb.ifarml64.inp" - - - # the copying libccdb.so to destination cgi-bin is - # required if libccdb.so is not in LD_LIBRARY PATH - # of the apache server - installer.copy_library = True - - #INSTALLATION - installer.print_self() - installer.install() - diff --git a/scripts/mysql_query_parser/parse.go b/scripts/mysql_query_parser/parse.go index 372d30c4..b011990a 100644 --- a/scripts/mysql_query_parser/parse.go +++ b/scripts/mysql_query_parser/parse.go @@ -35,10 +35,8 @@ func main() { fmt.Println(line) fmt.Printf("%#v\n", runMinRegexp.FindStringSubmatch(line)) fmt.Printf("%#v\n", variationRegexp.FindStringSubmatch(line)) - } - lineNum++ if lineNum > 1000 { break diff --git a/scripts/users_create/group_parse.pl b/scripts/users_create/group_parse.pl deleted file mode 100755 index aa3720e8..00000000 --- a/scripts/users_create/group_parse.pl +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env perl -$groupid = 267; -open (YPCAT,"getent group |"); -$group_count = 0; -while () { - if (/:$groupid:/) { - $group_count++; - if ($group_count > 1) {print ',';} - chomp; - @t = split/:$groupid:/; - print $t[1]; - } -} -print "\n"; -exit; diff --git a/scripts/users_create/users_create.py b/scripts/users_create/users_create.py index d5174aed..aaf547b7 100644 --- a/scripts/users_create/users_create.py +++ b/scripts/users_create/users_create.py @@ -7,8 +7,7 @@ import select import argparse -def print_help(): - print(""" +script_help = """ This utility can add users to ccdb database by user names. The idea behind this utility is to recreate users list from cronjob. @@ -16,18 +15,20 @@ def print_help(): | python users_create.py --recreate mysql://ccdb_user@localhost/ccdb Flags: - --recreate - delete all users before create users. - User deletion doesn't affect logs. The script does not delete system users: "anonymous", "test_user". + --recreate - delete all users before create users. + User deletion doesn't affect logs. The script does not delete system users: "anonymous", "test_user" + + --groupid - POSIX Group ID to get users from. E.g. on ifarm halld is 267 + Examples: #recreate user list from halld group - ypmatch halld group | python users_create.py --recreate mysql://ccdb_user@localhost/ccdb + python3 users_create.py --recreate --groupid=267 mysql://ccdb_user@localhost/ccdb - #just add 3 users - echo "anna,bob,smith' | python users_create.py mysql://ccdb_user@localhost/ccdb + python users_create.py mysql://ccdb_user@localhost/ccdb The scripts fails if no '--recreate' flag is given and a user exists. - """) + """ def get_provider(con_str): @@ -53,10 +54,8 @@ def delete_users(provider): print("Users deleted {}".format(deleted_count)) -def get_names(): +def get_names(groupid): import subprocess - - groupid = 267 group_count = 0 output = [] @@ -78,6 +77,7 @@ def get_names(): print(''.join(output)) return output + def get_user_names_from_ypmatch(ypstr): """ ypmatch string is like "halld:*:267:marki,davidl,...,jrsteven". This function trim it and returns username as list @@ -117,22 +117,20 @@ def create_users(provider, user_names): def parse_arguments(): - parser = argparse.ArgumentParser(description="Utility to add users to the ccdb database from cronjobs.") + parser = argparse.ArgumentParser(description="Utility to add users to the ccdb database from cronjobs.", add_help=script_help) parser.add_argument('connection_string', type=str, help='Connection string for the database, e.g., mysql://ccdb_user@localhost/ccdb') parser.add_argument('--recreate', action='store_true', help='Delete all users before creating new ones') + parser.add_argument('--groupid', default=267, help='POSIX Group ID to get usernames from') args = parser.parse_args() - return args.connection_string, args.recreate + return args.connection_string, args.recreate, args.groupid def main(): # Parse command line arguments - connection_str, recreate = parse_arguments() - - connection_str = "" - recreate = False + connection_str, recreate, groupid = parse_arguments() # Check stdin for usernames - names = get_names() + names = get_names(groupid) if not names: print("No user names found in stdin (pipe). Script continues to delete somebody if --recreate flag is given...") exit(1) @@ -144,11 +142,6 @@ def main(): else: connection_str = token - # check we have a connection string - if not connection_str: - print("Error! No connection string given!") - print_help() - sys.exit(1) print("Connecting to '" + connection_str + "'") provider = get_provider(connection_str)