From 5c81c2f72a4dd5385536afea7af87f185697474d Mon Sep 17 00:00:00 2001 From: habibayassin Date: Sat, 8 Jun 2024 23:33:11 +0000 Subject: [PATCH 1/7] refactor deltadebug Signed-off-by: habibayassin --- etc/deltaDebug.py | 218 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 211 insertions(+), 7 deletions(-) diff --git a/etc/deltaDebug.py b/etc/deltaDebug.py index a0694d40ab9..a03a097abf7 100644 --- a/etc/deltaDebug.py +++ b/etc/deltaDebug.py @@ -19,8 +19,10 @@ ################################ import odb +import re from openroad import Design import os +import glob import sys import signal import subprocess @@ -72,12 +74,17 @@ help= 'Exit early on unrelated errors to speed things up, but risks exiting on false negatives.' ) +parser.add_argument('--lib_path', + type=str, + help='Path to the library files (.lib)') +parser.add_argument('--lef_path', + type=str, + help='Path to the macro files (.lef)') parser.add_argument( '--dump_def', action='store_true', help='Determines whether to dumb def at each step in addition to the odb') - class cutLevel(enum.Enum): Nets = 0 Insts = 1 @@ -94,6 +101,11 @@ def __init__(self, opt): base_db_name = os.path.basename(opt.base_db_path) self.base_db_file = opt.base_db_path + self.lib_directory = opt.lib_path + self.lef_directory = opt.lef_path + self.reduced_lib = False + self.reduced_lef = False + self.error_string = opt.error_string self.use_stdout = opt.use_stdout self.exit_early_on_error = opt.exit_early_on_error @@ -114,10 +126,14 @@ def __init__(self, opt): base_db_directory, f"deltaDebug_base_temp_{base_db_name}") # The name of the result file after running deltaDebug + self.deltaDebug_result_def_file = os.path.join( + base_db_directory, f"deltaDebug_base_result_def_{base_db_name}") + + # # The name of the result file after running deltaDebug self.deltaDebug_result_base_file = os.path.join( base_db_directory, f"deltaDebug_base_result_{base_db_name}") - # This determines whether design def shall be dumped or not + # # This determines whether design def shall be dumped or not self.dump_def = opt.dump_def if (self.dump_def != 0): self.base_def_file = self.base_db_file[:-3] + "def" @@ -196,6 +212,7 @@ def debug(self): # Change deltaDebug resultant base_db file name to a representative name if os.path.exists(self.temp_base_db_file): + self.write_final_def() os.rename(self.temp_base_db_file, self.deltaDebug_result_base_file) # Restoring the original base_db file @@ -203,7 +220,8 @@ def debug(self): os.rename(self.original_base_db_file, self.base_db_file) print("___________________________________") - print(f"Resultant file is {self.deltaDebug_result_base_file}") + print(f"Resultant odb file is {self.deltaDebug_result_base_file}") + print(f"Resultant def file is {self.deltaDebug_result_def_file}") print("Delta Debugging Done!") # A function that do a cut in the db, writes the base db to disk @@ -213,6 +231,12 @@ def perform_step(self, cut_index=-1): self.base_db = Design.createDetachedDb() self.base_db = odb.read_db(self.base_db, self.temp_base_db_file) + # reduce .lib and .lef files + if (not self.reduced_lib): + self.reduce_lib_files() + if (not self.reduced_lef): + self.reduce_lef_files() + # Cut the block with the given step index. # if cut index of -1 is provided it means # that no cut will be made. @@ -223,8 +247,7 @@ def perform_step(self, cut_index=-1): odb.write_db(self.base_db, self.base_db_file) if (self.dump_def != 0): print("Writing def file") - odb.write_def(self.base_db.getChip().getBlock(), - self.base_def_file) + self.write_dump_def(self.base_def_file) cuts = self.get_cuts() if cut_index != -1 else None @@ -406,13 +429,194 @@ def remove_unused_masters(self): if (self.dump_def != 0): print("Writing def file") - odb.write_def(self.base_db.getChip().getBlock(), - self.temp_base_db_file[:-3] + "def") + self.write_dump_def(self.temp_base_db_file[:-3] + "def") if (self.base_db is not None): self.base_db.destroy(self.base_db) self.base_db = None + def reduce_lib_files(self): + print("Attempt to reduce lib files in", self.lib_directory) + if not os.path.exists(self.lib_directory): + return + for lib_file in glob.glob(os.path.join( self.lib_directory, "*.lib")): + used_cells = self.get_used_cells() + with open(lib_file, 'r') as f: + lines = f.readlines() + + with open(lib_file, 'w') as f: + write_lines = False + for line in lines: + if any(cell in line for cell in used_cells): + write_lines = True + if 'cell (' in line and not any(cell in line for cell in used_cells): + write_lines = False + if write_lines: + f.write(line) + self.reduced_lib = True + + def reduce_lef_files(self): + print("Attempt to reduce lef files in", self.lef_directory) + if not os.path.exists(self.lef_directory): + return + + for lef_file in glob.glob(os.path.join( self.lef_directory, "*.lef")): + with open(lef_file, 'r') as infile: + lines = infile.readlines() + + in_layer_block = False + essential_lines = [] + + for line in lines: + if re.match(r'\s*LAYER\s+\w+', line): + in_layer_block = True + essential_lines.append(line) + elif in_layer_block and re.match(r'\s*END\s+\w+', line): + essential_lines.append(line) + in_layer_block = False + elif in_layer_block: + essential_lines.append(line) + + with open(lef_file, 'w') as outfile: + outfile.writelines(essential_lines) + + self.reduced_lef = True + + def get_used_cells(self): + block = self.base_db.getChip().getBlock() + used_cells = set() + for inst in block.getInsts(): + master = inst.getMaster() + if master: + used_cells.add(master.getName()) + return used_cells + + def get_used_macros(self): + block = self.base_db.getChip().getBlock() + used_macros = set() + for inst in block.getInsts(): + master = inst.getMaster() + if master and master.isMacro(): + used_macros.add(master.getName()) + return used_macros + + def write_dump_def(self, output_file): + if self.base_db is None: + raise ValueError("Database is not loaded.") + + block = self.base_db.getChip().getBlock() + if block is None: + raise ValueError("Block is not present in the database.") + odb.write_def(block, output_file) + mangled_file = "mangled_" + output_file + self.mangle_def_file(output_file, mangled_file) + + def write_final_def(self): + self.base_db = odb.read_db(self.base_db, self.temp_base_db_file) + if self.base_db is None: + raise ValueError("Database is not loaded.") + + block = self.base_db.getChip().getBlock() + if block is None: + raise ValueError("Block is not present in the database.") + + odb.write_def(block, self.deltaDebug_result_def_file) + mangled_file = "mangled_" + self.deltaDebug_result_def_file + self.mangle_def_file(self.deltaDebug_result_def_file, mangled_file) + + if (self.base_db is not None): + self.base_db.destroy(self.base_db) + self.base_db = None + + + def mangle_def_file(self, input_def_file, output_def_file): + # if self.base_db is None: + # raise ValueError("Database is not loaded.") + + # block = self.base_db.getChip().getBlock() + # if block is None: + # raise ValueError("Block is not present in the database.") + + # with open(input_def_file, 'r') as infile: + # def_contents = infile.readlines() + + # net_pattern = re.compile(r'^-\s+(\S+)') + # port_pattern = re.compile(r'^\+\s+PORT\s+(\S+)') + # net_count = 1 + # port_count = 1 + # net_mapping = {} + # port_mapping = {} + + # mangled_def_contents = [] + + # for line in def_contents: + # net_match = net_pattern.match(line) + # if net_match: + # original_net_name = net_match.group(1) + # if original_net_name not in net_mapping: + # net_mapping[original_net_name] = f"net{net_count}" + # net_count += 1 + # mangled_line = line.replace(original_net_name, net_mapping[original_net_name]) + # mangled_def_contents.append(mangled_line) + # else: + # port_match = port_pattern.match(line) + # if port_match: + # original_port_name = port_match.group(1) + # if original_port_name not in port_mapping: + # port_mapping[original_port_name] = f"port{port_count}" + # port_count += 1 + # mangled_line = line.replace(original_port_name, port_mapping[original_port_name]) + # mangled_def_contents.append(mangled_line) + # else: + # mangled_def_contents.append(line) + + # with open(output_def_file, 'w') as outfile: + # outfile.writelines(mangled_def_contents) + + # print(f"Mangled .def file written to {output_def_file}") + # Patterns to identify different elements in the DEF file + patterns = { + 'nets': r'(-\s+\S+\s+\(.*?\)\s+\+\s+USE\s+\S+\s*;)', + 'components': r'(-\s+\S+\s+\S+\s+\+\s+PLACED\s+\(\s+\d+\s+\d+\s+\)\s+\S\s*;)' + } + + # Counters for renaming + net_count = 1 + element_count = 1 + + def rename_nets(text): + nonlocal net_count + def repl(match): + nonlocal net_count + new_name = f"net{net_count}" + net_count += 1 + return f"- {new_name} " + return re.sub(r'-\s+\S+', repl, text) + + def rename_elements(text): + nonlocal element_count + def repl(match): + nonlocal element_count + new_name = f"element{element_count}" + element_count += 1 + return f"- {new_name} " + return re.sub(r'-\s+\S+', repl, text) + + # Read the DEF file + with open(input_def_file, 'r') as file: + content = file.read() + + # Rename nets + content = rename_nets(content) + + # Rename elements + content = rename_elements(content) + + # Write the mangled content to the output file + with open(output_def_file, 'w') as file: + file.write(content) + + print(f"Mangled DEF file has been written to {output_def_file}") if __name__ == '__main__': opt = parser.parse_args() From e729cca378f6a6343ebc611fe97bb11a8bb1cd52 Mon Sep 17 00:00:00 2001 From: habibayassin Date: Sat, 8 Jun 2024 23:47:26 +0000 Subject: [PATCH 2/7] fix def file path Signed-off-by: habibayassin --- etc/deltaDebug.py | 63 ++++++----------------------------------------- 1 file changed, 7 insertions(+), 56 deletions(-) diff --git a/etc/deltaDebug.py b/etc/deltaDebug.py index a03a097abf7..89dd0375eb6 100644 --- a/etc/deltaDebug.py +++ b/etc/deltaDebug.py @@ -127,7 +127,7 @@ def __init__(self, opt): # The name of the result file after running deltaDebug self.deltaDebug_result_def_file = os.path.join( - base_db_directory, f"deltaDebug_base_result_def_{base_db_name}") + base_db_directory, f"deltaDebug_base_result_def.def") # # The name of the result file after running deltaDebug self.deltaDebug_result_base_file = os.path.join( @@ -508,8 +508,7 @@ def write_dump_def(self, output_file): if block is None: raise ValueError("Block is not present in the database.") odb.write_def(block, output_file) - mangled_file = "mangled_" + output_file - self.mangle_def_file(output_file, mangled_file) + self.mangle_def_file(output_file) def write_final_def(self): self.base_db = odb.read_db(self.base_db, self.temp_base_db_file) @@ -521,66 +520,19 @@ def write_final_def(self): raise ValueError("Block is not present in the database.") odb.write_def(block, self.deltaDebug_result_def_file) - mangled_file = "mangled_" + self.deltaDebug_result_def_file - self.mangle_def_file(self.deltaDebug_result_def_file, mangled_file) + self.mangle_def_file(self.deltaDebug_result_def_file) if (self.base_db is not None): self.base_db.destroy(self.base_db) self.base_db = None - def mangle_def_file(self, input_def_file, output_def_file): - # if self.base_db is None: - # raise ValueError("Database is not loaded.") - - # block = self.base_db.getChip().getBlock() - # if block is None: - # raise ValueError("Block is not present in the database.") - - # with open(input_def_file, 'r') as infile: - # def_contents = infile.readlines() - - # net_pattern = re.compile(r'^-\s+(\S+)') - # port_pattern = re.compile(r'^\+\s+PORT\s+(\S+)') - # net_count = 1 - # port_count = 1 - # net_mapping = {} - # port_mapping = {} - - # mangled_def_contents = [] - - # for line in def_contents: - # net_match = net_pattern.match(line) - # if net_match: - # original_net_name = net_match.group(1) - # if original_net_name not in net_mapping: - # net_mapping[original_net_name] = f"net{net_count}" - # net_count += 1 - # mangled_line = line.replace(original_net_name, net_mapping[original_net_name]) - # mangled_def_contents.append(mangled_line) - # else: - # port_match = port_pattern.match(line) - # if port_match: - # original_port_name = port_match.group(1) - # if original_port_name not in port_mapping: - # port_mapping[original_port_name] = f"port{port_count}" - # port_count += 1 - # mangled_line = line.replace(original_port_name, port_mapping[original_port_name]) - # mangled_def_contents.append(mangled_line) - # else: - # mangled_def_contents.append(line) - - # with open(output_def_file, 'w') as outfile: - # outfile.writelines(mangled_def_contents) - - # print(f"Mangled .def file written to {output_def_file}") - # Patterns to identify different elements in the DEF file + def mangle_def_file(self, input_def_file): patterns = { 'nets': r'(-\s+\S+\s+\(.*?\)\s+\+\s+USE\s+\S+\s*;)', 'components': r'(-\s+\S+\s+\S+\s+\+\s+PLACED\s+\(\s+\d+\s+\d+\s+\)\s+\S\s*;)' } - # Counters for renaming net_count = 1 element_count = 1 @@ -602,17 +554,16 @@ def repl(match): return f"- {new_name} " return re.sub(r'-\s+\S+', repl, text) - # Read the DEF file with open(input_def_file, 'r') as file: content = file.read() - # Rename nets content = rename_nets(content) - # Rename elements content = rename_elements(content) + + base_name = os.path.splitext(input_def_file)[0] + output_def_file = f"{base_name}_mangled.def" - # Write the mangled content to the output file with open(output_def_file, 'w') as file: file.write(content) From b5a57cbe375995d7daa3d652a20197e76d452f71 Mon Sep 17 00:00:00 2001 From: habibayassin Date: Mon, 10 Jun 2024 22:45:48 +0000 Subject: [PATCH 3/7] replace all instances Signed-off-by: habibayassin --- etc/deltaDebug.py | 52 +++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/etc/deltaDebug.py b/etc/deltaDebug.py index 89dd0375eb6..9939595d504 100644 --- a/etc/deltaDebug.py +++ b/etc/deltaDebug.py @@ -529,45 +529,61 @@ def write_final_def(self): def mangle_def_file(self, input_def_file): patterns = { - 'nets': r'(-\s+\S+\s+\(.*?\)\s+\+\s+USE\s+\S+\s*;)', - 'components': r'(-\s+\S+\s+\S+\s+\+\s+PLACED\s+\(\s+\d+\s+\d+\s+\)\s+\S\s*;)' + 'nets': r'(-\s+(\S+)\s+\(.*?\)\s+\+\s+USE\s+\S+\s*;)', + 'components': r'(-\s+(\S+)\s+\S+\s+\+\s+PLACED\s+\(\s+\d+\s+\d+\s+\)\s+\S\s*;)' } - + net_count = 1 element_count = 1 - + net_mapping = {} + element_mapping = {} + def rename_nets(text): nonlocal net_count def repl(match): nonlocal net_count - new_name = f"net{net_count}" - net_count += 1 - return f"- {new_name} " - return re.sub(r'-\s+\S+', repl, text) - + original_name = match.group(2) + if original_name not in net_mapping: + new_name = f"net{net_count}" + net_mapping[original_name] = new_name + net_count += 1 + return match.group(1).replace(original_name, net_mapping[original_name]) + return re.sub(patterns['nets'], repl, text) + def rename_elements(text): nonlocal element_count def repl(match): nonlocal element_count - new_name = f"element{element_count}" - element_count += 1 - return f"- {new_name} " - return re.sub(r'-\s+\S+', repl, text) - + original_name = match.group(2) + if original_name not in element_mapping: + new_name = f"element{element_count}" + element_mapping[original_name] = new_name + element_count += 1 + return match.group(1).replace(original_name, element_mapping[original_name]) + return re.sub(patterns['components'], repl, text) + with open(input_def_file, 'r') as file: content = file.read() - + content = rename_nets(content) - content = rename_elements(content) + def replace_all(text, mapping): + for original, new in mapping.items(): + text = re.sub(rf'\b{re.escape(original)}\b', new, text) + return text + + content = replace_all(content, net_mapping) + content = replace_all(content, element_mapping) + base_name = os.path.splitext(input_def_file)[0] output_def_file = f"{base_name}_mangled.def" - + with open(output_def_file, 'w') as file: file.write(content) - + print(f"Mangled DEF file has been written to {output_def_file}") + if __name__ == '__main__': opt = parser.parse_args() From 77a9e90b42e53daa31b30301821351d215ab6f0a Mon Sep 17 00:00:00 2001 From: habibayassin Date: Mon, 1 Jul 2024 19:43:06 +0000 Subject: [PATCH 4/7] format and cleanup Signed-off-by: habibayassin --- etc/deltaDebug.py | 258 +++++++++++++++++++++++++--------------------- 1 file changed, 138 insertions(+), 120 deletions(-) diff --git a/etc/deltaDebug.py b/etc/deltaDebug.py index 9939595d504..43529b831ab 100644 --- a/etc/deltaDebug.py +++ b/etc/deltaDebug.py @@ -37,53 +37,57 @@ persistence_range = [1, 2, 3, 4, 5, 6] cut_multiple = range(1, 128) -parser = argparse.ArgumentParser('Arguments for delta debugging') -parser.add_argument('--base_db_path', - type=str, - help='Path to the db file to perform the step on') -parser.add_argument('--error_string', - type=str, - help='The output that indicates target error has occurred') -parser.add_argument('--step', - type=str, - help='Command used to perform step on the input odb file') +parser = argparse.ArgumentParser("Arguments for delta debugging") parser.add_argument( - '--timeout', + "--base_db_path", type=str, help="Path to the db file to perform the step on" +) +parser.add_argument( + "--error_string", + type=str, + help="The output that indicates target error has occurred", +) +parser.add_argument( + "--step", type=str, help="Command used to perform step on the input odb file" +) +parser.add_argument( + "--timeout", type=int, default=None, - help='Specify initial timeout in seconds, default is to measure it') -parser.add_argument('--multiplier', - type=int, - default=cut_multiple[0], - choices=cut_multiple, - help='Multiply number of cuts with this number') -parser.add_argument('--persistence', - type=int, - default=persistence_range[0], - choices=persistence_range, - help='Indicates maximum input fragmentation; ' - 'fragments = 2^persistence; value in ' + - ', '.join(map(str, persistence_range))) + help="Specify initial timeout in seconds, default is to measure it", +) +parser.add_argument( + "--multiplier", + type=int, + default=cut_multiple[0], + choices=cut_multiple, + help="Multiply number of cuts with this number", +) parser.add_argument( - '--use_stdout', - action='store_true', - help='Enables reading the error string from standard output') + "--persistence", + type=int, + default=persistence_range[0], + choices=persistence_range, + help="Indicates maximum input fragmentation; " + "fragments = 2^persistence; value in " + ", ".join(map(str, persistence_range)), +) parser.add_argument( - '--exit_early_on_error', - action='store_true', - help= - 'Exit early on unrelated errors to speed things up, but risks exiting on false negatives.' + "--use_stdout", + action="store_true", + help="Enables reading the error string from standard output", ) -parser.add_argument('--lib_path', - type=str, - help='Path to the library files (.lib)') -parser.add_argument('--lef_path', - type=str, - help='Path to the macro files (.lef)') parser.add_argument( - '--dump_def', - action='store_true', - help='Determines whether to dumb def at each step in addition to the odb') + "--exit_early_on_error", + action="store_true", + help="Exit early on unrelated errors to speed things up, but risks exiting on false negatives.", +) +parser.add_argument("--lib_path", type=str, help="Path to the library files (.lib)") +parser.add_argument("--lef_path", type=str, help="Path to the macro files (.lef)") +parser.add_argument( + "--dump_def", + action="store_true", + help="Determines whether to dumb def at each step in addition to the odb", +) + class cutLevel(enum.Enum): Nets = 0 @@ -91,11 +95,11 @@ class cutLevel(enum.Enum): class deltaDebugger: - def __init__(self, opt): if not os.path.exists(opt.base_db_path): - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), - opt.base_db_path) + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), opt.base_db_path + ) base_db_directory = os.path.dirname(opt.base_db_path) base_db_name = os.path.basename(opt.base_db_path) @@ -119,23 +123,27 @@ def __init__(self, opt): # Temporary file names to hold the original base_db file across the run self.original_base_db_file = os.path.join( - base_db_directory, f"deltaDebug_base_original_{base_db_name}") + base_db_directory, f"deltaDebug_base_original_{base_db_name}" + ) # Temporary file used to hold current base_db to ensure its integrity across cuts self.temp_base_db_file = os.path.join( - base_db_directory, f"deltaDebug_base_temp_{base_db_name}") + base_db_directory, f"deltaDebug_base_temp_{base_db_name}" + ) # The name of the result file after running deltaDebug self.deltaDebug_result_def_file = os.path.join( - base_db_directory, f"deltaDebug_base_result_def.def") + base_db_directory, f"deltaDebug_base_result_def.def" + ) - # # The name of the result file after running deltaDebug + # The name of the result file after running deltaDebug self.deltaDebug_result_base_file = os.path.join( - base_db_directory, f"deltaDebug_base_result_{base_db_name}") + base_db_directory, f"deltaDebug_base_result_{base_db_name}" + ) - # # This determines whether design def shall be dumped or not + # This determines whether design def shall be dumped or not self.dump_def = opt.dump_def - if (self.dump_def != 0): + if self.dump_def != 0: self.base_def_file = self.base_db_file[:-3] + "def" # A variable to hold the base_db @@ -172,7 +180,7 @@ def debug(self): sys.exit(1) for self.cut_level in (cutLevel.Insts, cutLevel.Nets): - while (True): + while True: err = None self.n = 2 # Initial Number of cuts @@ -184,7 +192,7 @@ def debug(self): while j == 0 or j < cuts: current_err, cuts = self.perform_step(cut_index=j) self.step_count += 1 - if (current_err is not None): + if current_err is not None: # Found the target error with the cut DB # # This is a suitable level of detail to look @@ -194,7 +202,7 @@ def debug(self): self.prepare_new_step() j += 1 - if (error_in_range is None): + if error_in_range is None: # Increase the granularity of the cut in case target # error not found self.n *= 2 @@ -232,20 +240,20 @@ def perform_step(self, cut_index=-1): self.base_db = odb.read_db(self.base_db, self.temp_base_db_file) # reduce .lib and .lef files - if (not self.reduced_lib): + if not self.reduced_lib: self.reduce_lib_files() - if (not self.reduced_lef): + if not self.reduced_lef: self.reduce_lef_files() # Cut the block with the given step index. # if cut index of -1 is provided it means # that no cut will be made. - if (cut_index != -1): + if cut_index != -1: self.cut_block(index=cut_index) # Write DB odb.write_db(self.base_db, self.base_db_file) - if (self.dump_def != 0): + if self.dump_def != 0: print("Writing def file") self.write_dump_def(self.base_def_file) @@ -253,7 +261,7 @@ def perform_step(self, cut_index=-1): # Destroy the DB in memory to avoid being out-of-memory when # the step code is running - if (self.base_db is not None): + if self.base_db is not None: self.base_db.destroy(self.base_db) self.base_db = None @@ -265,7 +273,7 @@ def perform_step(self, cut_index=-1): # Handling timeout so as not to run the code for time # that is more than the original buggy code or a # buggy cut. - if (error_string is not None): + if error_string is not None: self.timeout = max(120, 1.2 * (end_time - start_time)) print(f"Error Code found: {error_string}") @@ -273,21 +281,25 @@ def perform_step(self, cut_index=-1): def run_command(self, command): poll_obj = select.poll() - if (self.use_stdout == 0): - process = subprocess.Popen(command, - shell=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE, - encoding='utf-8', - preexec_fn=os.setsid) + if self.use_stdout == 0: + process = subprocess.Popen( + command, + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + encoding="utf-8", + preexec_fn=os.setsid, + ) poll_obj.register(process.stderr, select.POLLIN) else: - process = subprocess.Popen(command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - encoding='utf-8', - preexec_fn=os.setsid) + process = subprocess.Popen( + command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + preexec_fn=os.setsid, + ) poll_obj.register(process.stdout, select.POLLIN) start_time = time.time() @@ -301,32 +313,32 @@ def run_command(self, command): pass def poll(self, process, poll_obj, start_time): - output = '' + output = "" error_string = None # None for any error code other than self.error_string while True: # polling on the output of the process with a timeout of 1 second # to avoid busywaiting if poll_obj.poll(1): - if (self.use_stdout == 0): + if self.use_stdout == 0: output = process.stderr.readline() else: output = process.stdout.readline() - if (output.find(self.error_string) != -1): + if output.find(self.error_string) != -1: # found the error code that we are searching for. error_string = self.error_string break - elif (self.exit_early_on_error and output.find("ERROR") != -1): + elif self.exit_early_on_error and output.find("ERROR") != -1: # Found different error (bad cut) so we can just # terminate early and ignore this cut. break curr_time = time.time() - if ((curr_time - start_time) > self.timeout): + if (curr_time - start_time) > self.timeout: print(f"Step {self.step_count} timed out!", flush=True) break - if (process.poll() is not None): + if process.poll() is not None: break return error_string @@ -336,7 +348,7 @@ def poll(self, process, poll_obj, start_time): # cutting on it. def prepare_new_step(self): # Delete the old temporary db file - if (os.path.exists(self.temp_base_db_file)): + if os.path.exists(self.temp_base_db_file): os.remove(self.temp_base_db_file) # Rename the new base db file to the temp name to keep it from overwriting across the two steps cut if os.path.exists(self.base_db_file): @@ -373,27 +385,26 @@ def get_cuts(self): # whether to cut Insts or Nets. def cut_block(self, index=0): message = [f"Step {self.step_count}"] - if (self.cut_level == cutLevel.Insts): # Insts cut level + if self.cut_level == cutLevel.Insts: # Insts cut level elms = self.get_insts() message += ["Insts level debugging"] - elif (self.cut_level == cutLevel.Nets): # Nets cut level + elif self.cut_level == cutLevel.Nets: # Nets cut level elms = self.get_nets() message += ["Nets level debugging"] - message += [ - f"Insts {len(self.get_insts())}", f"Nets {len(self.get_nets())}" - ] + message += [f"Insts {len(self.get_insts())}", f"Nets {len(self.get_nets())}"] num_elms = len(elms) - assert (num_elms > 0) + assert num_elms > 0 cuts = self.get_cuts() start = num_elms * index // cuts end = num_elms * (index + 1) // cuts - cut_position_string = '#' * cuts - cut_position_string = (cut_position_string[:index] + 'C' + - cut_position_string[index + 1:]) + cut_position_string = "#" * cuts + cut_position_string = ( + cut_position_string[:index] + "C" + cut_position_string[index + 1 :] + ) message += [f"cut elements {end-start}"] message += [f"timeout {ceil(self.timeout/60.0)} minutes"] self.cut_elements(start, end) @@ -404,9 +415,9 @@ def cut_block(self, index=0): def cut_elements(self, start, end): block = self.base_db.getChip().getBlock() - if (self.cut_level == cutLevel.Insts): # Insts cut level + if self.cut_level == cutLevel.Insts: # Insts cut level elms = block.getInsts() - elif (self.cut_level == cutLevel.Nets): # Nets cut level + elif self.cut_level == cutLevel.Nets: # Nets cut level elms = block.getNets() for i in range(start, end): @@ -427,11 +438,11 @@ def remove_unused_masters(self): print(f"Removed {unused} masters.") odb.write_db(self.base_db, self.temp_base_db_file) - if (self.dump_def != 0): + if self.dump_def != 0: print("Writing def file") self.write_dump_def(self.temp_base_db_file[:-3] + "def") - if (self.base_db is not None): + if self.base_db is not None: self.base_db.destroy(self.base_db) self.base_db = None @@ -439,17 +450,19 @@ def reduce_lib_files(self): print("Attempt to reduce lib files in", self.lib_directory) if not os.path.exists(self.lib_directory): return - for lib_file in glob.glob(os.path.join( self.lib_directory, "*.lib")): + for lib_file in glob.glob(os.path.join(self.lib_directory, "*.lib")): used_cells = self.get_used_cells() - with open(lib_file, 'r') as f: + with open(lib_file, "r") as f: lines = f.readlines() - with open(lib_file, 'w') as f: + with open(lib_file, "w") as f: write_lines = False for line in lines: if any(cell in line for cell in used_cells): write_lines = True - if 'cell (' in line and not any(cell in line for cell in used_cells): + if "cell (" in line and not any( + cell in line for cell in used_cells + ): write_lines = False if write_lines: f.write(line) @@ -459,27 +472,27 @@ def reduce_lef_files(self): print("Attempt to reduce lef files in", self.lef_directory) if not os.path.exists(self.lef_directory): return - - for lef_file in glob.glob(os.path.join( self.lef_directory, "*.lef")): - with open(lef_file, 'r') as infile: + + for lef_file in glob.glob(os.path.join(self.lef_directory, "*.lef")): + with open(lef_file, "r") as infile: lines = infile.readlines() - + in_layer_block = False essential_lines = [] - + for line in lines: - if re.match(r'\s*LAYER\s+\w+', line): + if re.match(r"\s*LAYER\s+\w+", line): in_layer_block = True essential_lines.append(line) - elif in_layer_block and re.match(r'\s*END\s+\w+', line): + elif in_layer_block and re.match(r"\s*END\s+\w+", line): essential_lines.append(line) in_layer_block = False elif in_layer_block: essential_lines.append(line) - - with open(lef_file, 'w') as outfile: + + with open(lef_file, "w") as outfile: outfile.writelines(essential_lines) - + self.reduced_lef = True def get_used_cells(self): @@ -498,7 +511,7 @@ def get_used_macros(self): master = inst.getMaster() if master and master.isMacro(): used_macros.add(master.getName()) - return used_macros + return used_macros def write_dump_def(self, output_file): if self.base_db is None: @@ -514,7 +527,7 @@ def write_final_def(self): self.base_db = odb.read_db(self.base_db, self.temp_base_db_file) if self.base_db is None: raise ValueError("Database is not loaded.") - + block = self.base_db.getChip().getBlock() if block is None: raise ValueError("Block is not present in the database.") @@ -522,15 +535,14 @@ def write_final_def(self): odb.write_def(block, self.deltaDebug_result_def_file) self.mangle_def_file(self.deltaDebug_result_def_file) - if (self.base_db is not None): + if self.base_db is not None: self.base_db.destroy(self.base_db) self.base_db = None - def mangle_def_file(self, input_def_file): patterns = { - 'nets': r'(-\s+(\S+)\s+\(.*?\)\s+\+\s+USE\s+\S+\s*;)', - 'components': r'(-\s+(\S+)\s+\S+\s+\+\s+PLACED\s+\(\s+\d+\s+\d+\s+\)\s+\S\s*;)' + "nets": r"(-\s+(\S+)\s+\(.*?\)\s+\+\s+USE\s+\S+\s*;)", + "components": r"(-\s+(\S+)\s+\S+\s+\+\s+PLACED\s+\(\s+\d+\s+\d+\s+\)\s+\S\s*;)", } net_count = 1 @@ -540,6 +552,7 @@ def mangle_def_file(self, input_def_file): def rename_nets(text): nonlocal net_count + def repl(match): nonlocal net_count original_name = match.group(2) @@ -548,10 +561,12 @@ def repl(match): net_mapping[original_name] = new_name net_count += 1 return match.group(1).replace(original_name, net_mapping[original_name]) - return re.sub(patterns['nets'], repl, text) + + return re.sub(patterns["nets"], repl, text) def rename_elements(text): nonlocal element_count + def repl(match): nonlocal element_count original_name = match.group(2) @@ -559,10 +574,13 @@ def repl(match): new_name = f"element{element_count}" element_mapping[original_name] = new_name element_count += 1 - return match.group(1).replace(original_name, element_mapping[original_name]) - return re.sub(patterns['components'], repl, text) + return match.group(1).replace( + original_name, element_mapping[original_name] + ) - with open(input_def_file, 'r') as file: + return re.sub(patterns["components"], repl, text) + + with open(input_def_file, "r") as file: content = file.read() content = rename_nets(content) @@ -570,7 +588,7 @@ def repl(match): def replace_all(text, mapping): for original, new in mapping.items(): - text = re.sub(rf'\b{re.escape(original)}\b', new, text) + text = re.sub(rf"\b{re.escape(original)}\b", new, text) return text content = replace_all(content, net_mapping) @@ -579,13 +597,13 @@ def replace_all(text, mapping): base_name = os.path.splitext(input_def_file)[0] output_def_file = f"{base_name}_mangled.def" - with open(output_def_file, 'w') as file: + with open(output_def_file, "w") as file: file.write(content) print(f"Mangled DEF file has been written to {output_def_file}") - -if __name__ == '__main__': + +if __name__ == "__main__": opt = parser.parse_args() debugger = deltaDebugger(opt) debugger.debug() From a9ccf9636f2c6d0e2e6f1e9d9902118756d52d84 Mon Sep 17 00:00:00 2001 From: habibayassin Date: Sun, 28 Jul 2024 23:10:45 +0000 Subject: [PATCH 5/7] prev refactor Signed-off-by: habibayassin --- etc/deltaDebug.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/etc/deltaDebug.py b/etc/deltaDebug.py index 43529b831ab..0bbc842f08c 100644 --- a/etc/deltaDebug.py +++ b/etc/deltaDebug.py @@ -80,8 +80,6 @@ action="store_true", help="Exit early on unrelated errors to speed things up, but risks exiting on false negatives.", ) -parser.add_argument("--lib_path", type=str, help="Path to the library files (.lib)") -parser.add_argument("--lef_path", type=str, help="Path to the macro files (.lef)") parser.add_argument( "--dump_def", action="store_true", @@ -105,8 +103,8 @@ def __init__(self, opt): base_db_name = os.path.basename(opt.base_db_path) self.base_db_file = opt.base_db_path - self.lib_directory = opt.lib_path - self.lef_directory = opt.lef_path + self.lib_directory , self.lef_directory = parse_vars_file(parse_vars_file_name(opt.step)) + self.reduced_lib = False self.reduced_lef = False @@ -602,6 +600,32 @@ def replace_all(text, mapping): print(f"Mangled DEF file has been written to {output_def_file}") +def parse_vars_file_name(step_arg): + match = re.search(r'run-me-(.*)-(.*)-base.sh', step_arg) + vars_file = "" + if match: + design = match.group(1) + platform = match.group(2) + + if design is None or platform is None: + print("Invalid step argument format. Expected format: run-me---base.sh") + sys.exit(1) + + vars_file = f"vars-{design}-{platform}-base.sh" + return vars_file + +def parse_vars_file(filename): + lib_files = None + tech_lef = None + + with open(filename, 'r') as file: + for line in file: + if match := re.match(r'^\s*LIB_FILES\s*=\s*(.*)$', line): + lib_files = match.group(1).strip() + elif match := re.match(r'^\s*TECH_LEF\s*=\s*(.*)$', line): + tech_lef = match.group(1).strip() + + return lib_files, tech_lef if __name__ == "__main__": opt = parser.parse_args() From 7cb43a70b97ed964c5c46dab96edd2e2a5f7c5d3 Mon Sep 17 00:00:00 2001 From: habibayassin Date: Sun, 28 Jul 2024 23:24:28 +0000 Subject: [PATCH 6/7] take args dynamically Signed-off-by: habibayassin --- etc/deltaDebug.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/deltaDebug.py b/etc/deltaDebug.py index 0bbc842f08c..30f0d9b7649 100644 --- a/etc/deltaDebug.py +++ b/etc/deltaDebug.py @@ -249,6 +249,10 @@ def perform_step(self, cut_index=-1): if cut_index != -1: self.cut_block(index=cut_index) + # reduce .lib and .lef files + self.reduce_lib_file() + self.reduce_lef_file() + # Write DB odb.write_db(self.base_db, self.base_db_file) if self.dump_def != 0: From 2a8b7cfb2a37eb68d094a1a6d2f04b0b224a21ba Mon Sep 17 00:00:00 2001 From: habibayassin Date: Sun, 28 Jul 2024 23:46:33 +0000 Subject: [PATCH 7/7] parse vars Signed-off-by: habibayassin --- etc/deltaDebug.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/etc/deltaDebug.py b/etc/deltaDebug.py index 30f0d9b7649..6e7a3fce257 100644 --- a/etc/deltaDebug.py +++ b/etc/deltaDebug.py @@ -249,10 +249,6 @@ def perform_step(self, cut_index=-1): if cut_index != -1: self.cut_block(index=cut_index) - # reduce .lib and .lef files - self.reduce_lib_file() - self.reduce_lef_file() - # Write DB odb.write_db(self.base_db, self.base_db_file) if self.dump_def != 0: @@ -604,6 +600,11 @@ def replace_all(text, mapping): print(f"Mangled DEF file has been written to {output_def_file}") +def extract_lib_files(lib_files): + lib_list = lib_files.split() + filtered_libs = [lib for lib in lib_list if lib.endswith('.lib')] + return filtered_libs[0] + def parse_vars_file_name(step_arg): match = re.search(r'run-me-(.*)-(.*)-base.sh', step_arg) vars_file = "" @@ -612,10 +613,11 @@ def parse_vars_file_name(step_arg): platform = match.group(2) if design is None or platform is None: - print("Invalid step argument format. Expected format: run-me---base.sh") - sys.exit(1) - + print("Invalid step argument format. Expected format: run-me---base.sh") + sys.exit(1) + vars_file = f"vars-{design}-{platform}-base.sh" + print("vars file name", vars_file) return vars_file def parse_vars_file(filename): @@ -624,12 +626,13 @@ def parse_vars_file(filename): with open(filename, 'r') as file: for line in file: - if match := re.match(r'^\s*LIB_FILES\s*=\s*(.*)$', line): + if match := re.match(r'^\s*export\s+LIB_FILES\s*=\s*"([^"]*)"\s*$', line): lib_files = match.group(1).strip() - elif match := re.match(r'^\s*TECH_LEF\s*=\s*(.*)$', line): + elif match := re.match(r'^\s*export\s+TECH_LEF\s*=\s*"([^"]*)"\s*$', line): tech_lef = match.group(1).strip() - - return lib_files, tech_lef + print("lib_files", extract_lib_files(lib_files)) + print("tech lef", tech_lef) + return extract_lib_files(lib_files), tech_lef if __name__ == "__main__": opt = parser.parse_args()