From dc72a93125d6df45683f6a78d1707991a91eaf2d Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Mon, 9 Dec 2024 16:52:31 -0800 Subject: [PATCH 01/25] MNT: outfit this repository to use pre-commit and ruff --- .gitignore | 5 +++++ .pre-commit-config.yaml | 34 ++++++++++++++++++++++++++++++++++ ruff.toml | 9 +++++++++ 3 files changed, 48 insertions(+) create mode 100644 .pre-commit-config.yaml create mode 100644 ruff.toml diff --git a/.gitignore b/.gitignore index 214d6e46..6e38a090 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,6 @@ realpath +__pycache__ +core.* +tests/common/* +tests/examples/* +tests/expected/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..6046dbe7 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,34 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks + +exclude: | + (?x)^( + tests/common/.*| + tests/examples/.*| + tests/expected/.*| + )$ + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: no-commit-to-branch + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-ast + - id: check-case-conflict + - id: check-json + - id: check-merge-conflict + - id: check-symlinks + - id: check-xml + - id: check-yaml + - id: debug-statements + +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.1 + hooks: + # Run the linter. + - id: ruff + args: [ --fix ] + # Run the formatter. + - id: ruff-format diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..025ef914 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,9 @@ +exclude = [ + ".git", +] + +line-length = 88 + +[lint] +select = ["C", "E", "F", "W", "B", "I"] +ignore = ["C901"] From e1c1cae1e53128a2450cf5cc8354465560a788ff Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Mon, 9 Dec 2024 16:53:11 -0800 Subject: [PATCH 02/25] MNT: convert expand, expand.py to python 3 ENH: refactor into main func to make testing easier STY: ruff --- expand | 7 +- expand.py | 705 +++++++++++++++++++++++++++++------------------------- 2 files changed, 375 insertions(+), 337 deletions(-) diff --git a/expand b/expand index b315dc59..b370633d 100755 --- a/expand +++ b/expand @@ -1,7 +1,2 @@ #!/bin/bash -export PSPKG_RELEASE=controls-0.0.6 -export PATH=/usr/local/bin:/bin:/usr/bin -unset LD_LIBRARY_PATH -unset PYTHONPATH -source $PSPKG_ROOT/etc/set_env.sh -$0.py "$@" +/cds/group/pcds/pyps/conda/py39/envs/pcds-5.9.1/bin/python $0.py "$@" diff --git a/expand.py b/expand.py index 66ace3cd..cbcbf4d8 100755 --- a/expand.py +++ b/expand.py @@ -1,118 +1,127 @@ #!/usr/bin/env python -import os -import sys -import re -import string import ast +import io import operator -import StringIO +import os +import re +import sys expand_path = [] # Predefine some regular expressions! -w = re.compile("^[ \t]*([^ \t=]+)") -wq = re.compile('^[ \t]*"([^"]*)"') -wqq = re.compile("^[ \t]*'([^']*)'") -assign = re.compile("^[ \t]*=") -sp = re.compile("^[ \t]*([A-Za-z_][A-Za-z0-9_]*)[ \t]+(.+?)[ \t]*$") -spq = re.compile('^[ \t]*([A-Za-z_][A-Za-z0-9_]*)[ \t]+"([^"]*)"[ \t]*$') -spqq = re.compile("^[ \t]*([A-Za-z_][A-Za-z0-9_]*)[ \t]+'([^']*)'[ \t]*$") -eq = re.compile("^[ \t]*([A-Za-z_][A-Za-z0-9_]*)[ \t]*=[ \t]*(.*?)[ \t]*$") -eqq = re.compile('^[ \t]*([A-Za-z_][A-Za-z0-9_]*)[ \t]*=[ \t]*"([^"]*)"[ \t]*$') -eqqq = re.compile("^[ \t]*([A-Za-z_][A-Za-z0-9_]*)[ \t]*=[ \t]*'([^']*)'[ \t]*$") -inst = re.compile("^[ \t]*(([A-Za-z_][A-Za-z0-9_]*):[ \t]*)?([A-Za-z_][A-Za-z0-9_]*)\((.*)\)[ \t]*$") -inst2 = re.compile("^[ \t]*INSTANCE[ \t]+([A-Za-z_][A-Za-z0-9_]*)[ \t]*([A-Za-z0-9_]*)[ \t]*$") -prminst = re.compile("^([A-Za-z_][A-Za-z0-9_]*)(,)") -prmidx = re.compile("^([A-Za-z_][A-Za-z0-9_]*?)([0-9_]+)(,)") -prmeq = re.compile("^([A-Za-z_][A-Za-z0-9_]*)=([^,]*)(,)") -prmeqq = re.compile('^([A-Za-z_][A-Za-z0-9_]*)="([^"]*)"(,)') -prmeqqq = re.compile("^([A-Za-z_][A-Za-z0-9_]*)='([^']*)'(,)") -inc = re.compile("^\$\$INCLUDE\((.*)\)") -idxre = re.compile("^INDEX([0-9]*)") -doubledollar = re.compile("^(.*?)\$\$") -keyword = re.compile("^(ROOT|SUBSTR|UP|LOOP|IF|INCLUDE|TRANSLATE|COUNT|NAME)\(|^(ASSIGN|CALC|IFCALC)\{") -parens = re.compile("^\(([^)]*?)\)") -brackets = re.compile("^\{([^}]*?)\}") -trargs = re.compile('^\(([^,]*?),"([^"]*?)","([^"]*?)"\)') -dbargs = re.compile('^\(([^,)]*?),([^,)]*?)\)') -ifargs = re.compile('^\(([^,)]*?),([^,)]*?),([^,)]*?)\)') -word = re.compile("^([A-Za-z0-9_]*)") - -operators = {ast.Add: operator.add, - ast.Sub: operator.sub, - ast.Mult: operator.mul, - ast.Mod: operator.mod, - ast.Div: operator.truediv, - ast.Pow: operator.pow, - ast.LShift : operator.lshift, - ast.RShift: operator.rshift, - ast.BitOr: operator.or_, - ast.BitAnd : operator.and_, - ast.BitXor: operator.xor, - ast.USub: operator.neg, - ast.Invert: operator.not_ +w = re.compile(r"^[ \t]*([^ \t=]+)") +wq = re.compile(r'^[ \t]*"([^"]*)"') +wqq = re.compile(r"^[ \t]*'([^']*)'") +assign = re.compile(r"^[ \t]*=") +sp = re.compile(r"^[ \t]*([A-Za-z_][A-Za-z0-9_]*)[ \t]+(.+?)[ \t]*$") +spq = re.compile(r'^[ \t]*([A-Za-z_][A-Za-z0-9_]*)[ \t]+"([^"]*)"[ \t]*$') +spqq = re.compile(r"^[ \t]*([A-Za-z_][A-Za-z0-9_]*)[ \t]+'([^']*)'[ \t]*$") +eq = re.compile(r"^[ \t]*([A-Za-z_][A-Za-z0-9_]*)[ \t]*=[ \t]*(.*?)[ \t]*$") +eqq = re.compile(r'^[ \t]*([A-Za-z_][A-Za-z0-9_]*)[ \t]*=[ \t]*"([^"]*)"[ \t]*$') +eqqq = re.compile(r"^[ \t]*([A-Za-z_][A-Za-z0-9_]*)[ \t]*=[ \t]*'([^']*)'[ \t]*$") +inst = re.compile( + r"^[ \t]*(([A-Za-z_][A-Za-z0-9_]*):[ \t]*)?([A-Za-z_][A-Za-z0-9_]*)\((.*)\)[ \t]*$" +) +inst2 = re.compile( + r"^[ \t]*INSTANCE[ \t]+([A-Za-z_][A-Za-z0-9_]*)[ \t]*([A-Za-z0-9_]*)[ \t]*$" +) +prminst = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*)(,)") +prmidx = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*?)([0-9_]+)(,)") +prmeq = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*)=([^,]*)(,)") +prmeqq = re.compile(r'^([A-Za-z_][A-Za-z0-9_]*)="([^"]*)"(,)') +prmeqqq = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*)='([^']*)'(,)") +inc = re.compile(r"^\$\$INCLUDE\((.*)\)") +idxre = re.compile(r"^INDEX([0-9]*)") +doubledollar = re.compile(r"^(.*?)\$\$") +keyword = re.compile( + r"^(ROOT|SUBSTR|UP|LOOP|IF|INCLUDE|TRANSLATE|COUNT|NAME)\(|^(ASSIGN|CALC|IFCALC)\{" +) +parens = re.compile(r"^\(([^)]*?)\)") +brackets = re.compile(r"^\{([^}]*?)\}") +trargs = re.compile(r'^\(([^,]*?),"([^"]*?)","([^"]*?)"\)') +dbargs = re.compile(r"^\(([^,)]*?),([^,)]*?)\)") +ifargs = re.compile(r"^\(([^,)]*?),([^,)]*?),([^,)]*?)\)") +word = re.compile(r"^([A-Za-z0-9_]*)") + +operators = { + ast.Add: operator.add, + ast.Sub: operator.sub, + ast.Mult: operator.mul, + ast.Mod: operator.mod, + ast.Div: operator.truediv, + ast.Pow: operator.pow, + ast.LShift: operator.lshift, + ast.RShift: operator.rshift, + ast.BitOr: operator.or_, + ast.BitAnd: operator.and_, + ast.BitXor: operator.xor, + ast.USub: operator.neg, + ast.Invert: operator.not_, } + def myopen(file): - if file == '-': + if file == "-": return sys.stdin try: fp = open(file) return fp - except: + except Exception: pass - if file[0] == '/': + if file[0] == "/": return None for f in expand_path: fn = f + "/" + file try: fp = open(fn) return fp - except: + except Exception: pass return None -class config(): + +class config: """ - This is the class that handles the configuration namespace. + This is the class that handles the configuration namespace. - It includes functions to read configuration files, add new - instances and definitions, and evaluate numeric expressions - in the current environment. + It includes functions to read configuration files, add new + instances and definitions, and evaluate numeric expressions + in the current environment. - Member variables: - ddict - The variable dictionary mapping from names to values. - idict - The instance dictionary mapping from instance type - names to lists of instances. - assigns - A stack of lists of variables that have been $$ASSIGNED. + Member variables: + ddict - The variable dictionary mapping from names to values. + idict - The instance dictionary mapping from instance type + names to lists of instances. + assigns - A stack of lists of variables that have been $$ASSIGNED. - Methods: - read_config(filename, extra_input) - Read in the configuration - in filename, prefixing it with the list of extra_input. + Methods: + read_config(filename, extra_input) - Read in the configuration + in filename, prefixing it with the list of extra_input. - assign(varname, value) - $$ASSIGN the varname to the specified - value, adding it to the top assigns list as well. + assign(varname, value) - $$ASSIGN the varname to the specified + value, adding it to the top assigns list as well. - eval_expr(expr) - Evaluate the specified expression text in the - current context. + eval_expr(expr) - Evaluate the specified expression text in the + current context. """ + def __init__(self): self.path = os.getcwd() - self.dirname = self.path.split('/')[-1] + self.dirname = self.path.split("/")[-1] self.ddict = {} self.idict = {} - self.assigns = [set([])] + self.assigns = [set()] def create_instance(self, iname, id, idict, ndict): try: allinst = idict[iname] - except: + except Exception: allinst = [] idict[iname] = [] n = str(len(allinst)) - if id != None: + if id is not None: ndict[id] = (iname, int(n)) dd = {} dd["INDEX"] = n @@ -125,62 +134,62 @@ def assign(self, dname, value): self.ddict[dname] = value self.assigns[-1].add(dname) - def process_config_line(self, l, d): - l = l.strip() - m = inst.search(l) - if m != None: - return True # Skip instantiations for now! - m = inst2.search(l) - if m != None: # First new-style instantiation --> we're done here! + def process_config_line(self, L, d): + L = L.strip() + m = inst.search(L) + if m is not None: + return True # Skip instantiations for now! + m = inst2.search(L) + if m is not None: # First new-style instantiation --> we're done here! return False # Search for a one-line assignment of some form! - m = eqqq.search(l) - if m == None: - m = eqq.search(l) - if m == None: - m = eq.search(l) - if m == None: - m = spqq.search(l) - if m == None: - m = spq.search(l) - if m == None: - m = sp.search(l) - if m != None: + m = eqqq.search(L) + if m is None: + m = eqq.search(L) + if m is None: + m = eq.search(L) + if m is None: + m = spqq.search(L) + if m is None: + m = spq.search(L) + if m is None: + m = sp.search(L) + if m is not None: var = m.group(1) val = m.group(2) - d[var] = val; + d[var] = val return True - m = inc.search(l) - if m != None: + m = inc.search(L) + if m is not None: fn = m.group(1) try: fn = d[fn] - except: + except Exception: pass try: - output = StringIO.StringIO() + output = io.StringIO() expand(d, [fn], output) fn = output.getvalue().strip() output.close() - except: + except Exception: pass try: - newlines=myopen(fn).readlines() - except: + newlines = myopen(fn).readlines() + except Exception: d["_failed_include"].append(fn) return True for ll in newlines: self.process_config_line(ll, d) return True - if l != "" and l[0] != '#': - print "Skipping unknown line: %s" % l + if L != "" and L[0] != "#": + print("Skipping unknown line: %s" % L) return True def read_config(self, file, extra): fp = myopen(file) if not fp: - raise IOError, "File %s not found!" % ( file ) - lines = [l + "\n" for l in extra] + fp.readlines() + raise IOError("File %s not found!" % (file)) + lines = [L + "\n" for L in extra] + fp.readlines() fp.close() origlines = lines @@ -193,43 +202,45 @@ def read_config(self, file, extra): while len(fi) != 0 and cnt < 5: lines = origlines cnt += 1 - output = StringIO.StringIO() + output = io.StringIO() expand(self, lines, output, True) # Leave failed $$INCLUDE in the output! value = output.getvalue() output.close() lines = value.split("\n") - d = {"DIRNAME": self.dirname, "PATH" : self.path, "_failed_include" : []} - for l in lines: - if not self.process_config_line(l, d): + d = {"DIRNAME": self.dirname, "PATH": self.path, "_failed_include": []} + for L in lines: + if not self.process_config_line(L, d): break - fi = d['_failed_include'] - del d['_failed_include'] + fi = d["_failed_include"] + del d["_failed_include"] self.ddict = d cnt += 1 # Now that we have the aliases, reprocess the config! - + lines = origlines - output = StringIO.StringIO() + output = io.StringIO() expand(self, lines, output) value = output.getvalue() output.close() lines = value.split("\n") - + i = {} d = {"DIRNAME": self.dirname, "PATH": self.path} nd = {} newstyle = False - ininst = False - - for l in lines: - l = l.strip() - m = inst2.search(l) - if m != None: + ininst = False + iname = "" + dd = {} + + for L in lines: + L = L.strip() + m = inst2.search(L) + if m is not None: newstyle = True if newstyle: - if m != None: + if m is not None: if ininst: self.finish_instance(iname, i, dd) ininst = True @@ -237,139 +248,141 @@ def read_config(self, file, extra): id = m.group(2) dd, n = self.create_instance(iname, id, i, nd) else: - loc = 0 # Look for parameters! + loc = 0 # Look for parameters! first = None haveeq = False - while l[loc:] != '': - m = assign.search(l[loc:]) - if m != None: + while L[loc:] != "": + m = assign.search(L[loc:]) + if m is not None: loc += m.end() if haveeq: - print "Double equal sign in |%s|" % l + print("Double equal sign in |%s|" % L) haveeq = True - continue # Just ignore it! + continue # Just ignore it! - m = wqq.search(l[loc:]) - if m != None: + m = wqq.search(L[loc:]) + if m is not None: loc += m.end() else: - m = wq.search(l[loc:]) - if m != None: + m = wq.search(L[loc:]) + if m is not None: loc += m.end() + 1 else: - m = w.search(l[loc:]) - if m != None: + m = w.search(L[loc:]) + if m is not None: loc += m.end() + 1 else: - break # How does this even happen?!? + break # How does this even happen?!? val = m.group(1) - if first != None: + if first is not None: dd[first] = val d[iname + first + n] = val first = None else: # Could this be an instance parameter? - useinst = '' - usenum = 0 + useinst = "" + usenum = 0 try: t = nd[val] useinst = t[0] usenum = t[1] - except: - m = prmidx.search(val+",") - if m != None: + except Exception: + m = prmidx.search(val + ",") + if m is not None: useinst = m.group(1) usenum = int(m.group(2)) try: used = i[useinst][usenum] - for k in used.keys(): + for k in list(used.keys()): var = useinst + k val = used[k] dd[var] = val - except: + except Exception: first = val haveeq = False continue - m = inst.search(l) - if m != None: + m = inst.search(L) + if m is not None: id = m.group(2) iname = m.group(3) params = m.group(4) + "," dd, n = self.create_instance(iname, id, i, nd) - while (params != ""): + while params != "": m = prmeqqq.search(params) - if m == None: + if m is None: m = prmeqq.search(params) - if m == None: + if m is None: m = prmeq.search(params) - if m != None: + if m is not None: # Parameter of the form VAR=VAL. Global dictionary will also # get inameVARn=VAL. var = m.group(1) val = m.group(2) dd[var] = val d[iname + var + n] = val - params = params[m.end(3):len(params)] + params = params[m.end(3) : len(params)] else: m = prminst.search(params) - if m != None: + if m is not None: # This is an instance parameter. It is either old-style, # INSTn, or an arbitrary name. Check the name dict first! try: t = nd[m.group(1)] useinst = t[0] usenum = t[1] - params = params[m.end(2):len(params)] - except: + params = params[m.end(2) : len(params)] + except Exception: m = prmidx.search(params) - if m == None: - print "Unknown parameter in line %s" % params + if m is None: + print("Unknown parameter in line %s" % params) params = "" continue useinst = m.group(1) usenum = int(m.group(2)) - params = params[m.end(3):len(params)] + params = params[m.end(3) : len(params)] # Find the instance, and add all of its named parameters # VAL with the name INSTVAL. used = i[useinst][usenum] - for k in used.keys(): + for k in list(used.keys()): var = useinst + k val = used[k] dd[var] = val else: - print "Unknown parameter in line %s" % params + print("Unknown parameter in line %s" % params) params = "" self.finish_instance(iname, i, dd) continue # Search for a one-line assignment of some form! - m = eqqq.search(l) - if m == None: - m = eqq.search(l) - if m == None: - m = eq.search(l) - if m == None: - m = spqq.search(l) - if m == None: - m = spq.search(l) - if m == None: - m = sp.search(l) - if m != None: + m = eqqq.search(L) + if m is None: + m = eqq.search(L) + if m is None: + m = eq.search(L) + if m is None: + m = spqq.search(L) + if m is None: + m = spq.search(L) + if m is None: + m = sp.search(L) + if m is not None: var = m.group(1) val = m.group(2) - d[var] = val; + d[var] = val continue - if l != "" and l[0] != '#': - print "Skipping unknown line: %s" % l + if L != "" and L[0] != "#": + print("Skipping unknown line: %s" % L) if ininst: self.finish_instance(iname, i, dd) - for (k,v) in nd.items(): - d[k + ":TYPE"] = v[0] + for k, v in list(nd.items()): + d[k + ":TYPE"] = v[0] d[k + ":INDEX"] = v[1] self.idict = i self.ddict = d def eval_expr(self, expr): - return self.eval_(ast.parse(expr).body[0].value) # Module(body=[Expr(value=...)]) + return self.eval_( + ast.parse(expr).body[0].value + ) # Module(body=[Expr(value=...)]) def eval_(self, node): if isinstance(node, ast.Num): @@ -377,13 +390,13 @@ def eval_(self, node): elif isinstance(node, ast.Name): try: n = self.ddict[node.id] - if n[:2] == '0x': + if n[:2] == "0x": x = int(self.ddict[node.id], 16) - elif n[0] == '0': + elif n[0] == "0": x = int(self.ddict[node.id], 8) else: x = int(self.ddict[node.id], 10) - except: + except Exception: x = 0 return x elif isinstance(node, ast.operator): @@ -400,13 +413,14 @@ def eval_(self, node): else: raise TypeError(node) + # Find the endre in the lines starting at index i, offset l. # However, lb and rb are regular expressions that denote a region to be skipped. # Note that endre might be equal to rb! # Return a tuple: (newlines, newi, newloc), or None if it wasn't found. -def searchforend(lines, endre, lb, rb, i, l): +def searchforend(lines, endre, lb, rb, i, L): j = i - loc = l + loc = L nest = 0 newlines = [] while j < len(lines): @@ -414,7 +428,7 @@ def searchforend(lines, endre, lb, rb, i, l): # Looking at lines[j][loc:]! # lbm = lb.search(lines[j][loc:]) - if lbm != None: + if lbm is not None: lbp = lbm.end(2) else: lbp = 100000 @@ -422,7 +436,7 @@ def searchforend(lines, endre, lb, rb, i, l): endm = rb.search(lines[j][loc:]) else: endm = endre.search(lines[j][loc:]) - if endm != None: + if endm is not None: endp = endm.end(2) else: endp = 100000 @@ -432,16 +446,16 @@ def searchforend(lines, endre, lb, rb, i, l): newlines.append(lines[j][loc:]) j += 1 loc = 0 - continue; + continue if lbp < endp: # Found a new start! nest = nest + 1 pos = loc loc += lbp - if pos == 0 and lines[j][loc] == '\n': + if pos == 0 and lines[j][loc] == "\n": loc += 1 newlines.append(lines[j][pos:loc]) - continue; + continue else: # Found the end, either rb or endre! if nest != 0: @@ -449,33 +463,34 @@ def searchforend(lines, endre, lb, rb, i, l): nest = nest - 1 pos = loc loc += endp - if pos == 0 and lines[j][loc] == '\n': + if pos == 0 and lines[j][loc] == "\n": loc += 1 newlines.append(lines[j][pos:loc]) - continue; + continue # We're really done. Strip off what we were looking for. newlines.append(endm.group(1)) pos = loc loc += endp if pos == 0 and lines[j][loc:].strip() == "": # If the $$ directive is the entire line, don't add a newline! - loc = 0; + loc = 0 j += 1 return (newlines, j, loc) return None + def rename_index(d): """ - When entering a new loop, rename the existing INDEX/INDEXn variables to - be INDEX1/INDEXn+1. + When entering a new loop, rename the existing INDEX/INDEXn variables to + be INDEX1/INDEXn+1. """ new = [] val = [] - for k in d.keys(): + for k in list(d.keys()): m = idxre.search(k) - if m != None: + if m is not None: arg = m.group(1) - if arg == '': + if arg == "": new.append("INDEX1") else: new.append("INDEX%d" % (int(arg) + 1)) @@ -485,6 +500,7 @@ def rename_index(d): d[new[i]] = val[i] return d + # # Let's shorten translation strings by accepting ranges with '-'. We'll treat # '-' special at the beginning or end of the string and just let them be. @@ -494,40 +510,41 @@ def enumstring(s): out = m.group(1) body = m.group(2) while body != "": - if len(body) > 1 and body[1] == '-': + if len(body) > 1 and body[1] == "-": first = ord(body[0]) last = ord(body[2]) if first <= last: - out += "".join(map(chr, range(first, last+1))) + out += "".join(map(chr, list(range(first, last + 1)))) else: - out += "".join(map(chr, range(first, last-1, -1))) + out += "".join(map(chr, list(range(first, last - 1, -1)))) body = body[3:] pass else: - out += body[0]; + out += body[0] body = body[1:] out += m.group(3) return out + def expand(cfg, lines, f, isfirst=False): """ - expand is where the magic happens. + expand is where the magic happens. - cfg is a config object specifying the current variable values and instances. + cfg is a config object specifying the current variable values and instances. - lines is a list of strings to be expanded. + lines is a list of strings to be expanded. - f is an output file (or StringIO) to be written. + f is an output file (or StringIO) to be written. - isfirst is a flag indicating that we are actually processing the config file, - and so $$INCLUDE might fail until we evaluate enough variables to properly - expand the filename. + isfirst is a flag indicating that we are actually processing the config file, + and so $$INCLUDE might fail until we evaluate enough variables to properly + expand the filename. """ i = 0 loc = 0 while i < len(lines): m = doubledollar.search(lines[i][loc:]) - if m == None: + if m is None: # Line without a $$. f.write("%s" % lines[i][loc:]) i += 1 @@ -536,90 +553,90 @@ def expand(cfg, lines, f, isfirst=False): # Write the first part f.write(m.group(1)) - pos = loc + m.end(1) # save where we found this! - loc = pos + 2 # skip the '$$'! + pos = loc + m.end(1) # save where we found this! + loc = pos + 2 # skip the '$$'! m = keyword.search(lines[i][loc:]) - if m != None: + if m is not None: kw = m.group(1) - if kw == None: + if kw is None: kw = m.group(2) - loc += m.end(2) # Leave on the '{'! + loc += m.end(2) # Leave on the '{'! else: - loc += m.end(1) # Leave on the '('! - + loc += m.end(1) # Leave on the '('! + if kw == "TRANSLATE": argm = trargs.search(lines[i][loc:]) - if argm != None: - loc += argm.end(3)+2 + if argm is not None: + loc += argm.end(3) + 2 elif kw == "CALC" or kw == "IFCALC" or kw == "ASSIGN": argm = brackets.search(lines[i][loc:]) - if argm != None: - loc += argm.end(1)+1 + if argm is not None: + loc += argm.end(1) + 1 elif kw == "IF": argm = ifargs.search(lines[i][loc:]) - if argm != None: - kw = "TIF" # Triple IF! - loc += argm.end(3)+1 + if argm is not None: + kw = "TIF" # Triple IF! + loc += argm.end(3) + 1 else: argm = dbargs.search(lines[i][loc:]) - if argm != None: + if argm is not None: kw = "DIF" - loc += argm.end(2)+1 + loc += argm.end(2) + 1 else: argm = parens.search(lines[i][loc:]) - if argm != None: - loc += argm.end(1)+1 + if argm is not None: + loc += argm.end(1) + 1 if pos == 0 and lines[i][loc:].strip() == "": # If the $$ directive is the entire line, don't add a newline! - loc = 0; + loc = 0 i += 1 elif kw == "SUBSTR": argm = ifargs.search(lines[i][loc:]) - if argm != None: - loc += argm.end(3)+1 + if argm is not None: + loc += argm.end(3) + 1 else: argm = dbargs.search(lines[i][loc:]) - if argm != None: - loc += argm.end(2)+1 + if argm is not None: + loc += argm.end(2) + 1 kw = "TAIL" else: argm = parens.search(lines[i][loc:]) - if argm != None: - loc += argm.end(1)+1 + if argm is not None: + loc += argm.end(1) + 1 if pos == 0 and lines[i][loc:].strip() == "": # If the $$ directive is the entire line, don't add a newline! - loc = 0; + loc = 0 i += 1 - - if argm != None: + + if argm is not None: if kw == "LOOP": iname = argm.group(1) - startloop = re.compile("(.*?)\$\$LOOP\(" + iname + "(\))") - endloop = re.compile("(.*?)\$\$ENDLOOP\(" + iname + "(\))") + startloop = re.compile(r"(.*?)\$\$LOOP\(" + iname + r"(\))") + endloop = re.compile(r"(.*?)\$\$ENDLOOP\(" + iname + r"(\))") t = searchforend(lines, endloop, startloop, endloop, i, loc) - if t == None: - print "Cannot find $$ENDLOOP(%s)?" % iname + if t is None: + print("Cannot find $$ENDLOOP(%s)?" % iname) sys.exit(1) if iname[0] >= "0" and iname[0] <= "9": try: cnt = int(iname) - except: + except Exception: cnt = 0 ilist = [{"INDEX": str(n)} for n in range(cnt)] - elif iname in cfg.idict.keys(): + elif iname in list(cfg.idict.keys()): try: ilist = cfg.idict[iname] - except: + except Exception: ilist = [] else: try: cnt = int(cfg.ddict[iname]) - except: + except Exception: cnt = 0 ilist = [{"INDEX": str(n)} for n in range(cnt)] olddict = cfg.ddict - cfg.assigns.append(set([])) # Push a new assignment context. + cfg.assigns.append(set()) # Push a new assignment context. for inst in ilist: cfg.ddict = rename_index(olddict.copy()) cfg.ddict.update(inst) @@ -628,21 +645,23 @@ def expand(cfg, lines, f, isfirst=False): # We need to pull these back into olddict! for dname in cfg.assigns[-1]: olddict[dname] = cfg.ddict[dname] - cfg.assigns[-1] = set([]) - cfg.assigns = cfg.assigns[:-1] # Pop the assignment context for the loop. + cfg.assigns[-1] = set() + cfg.assigns = cfg.assigns[ + :-1 + ] # Pop the assignment context for the loop. cfg.ddict = olddict i = t[1] loc = t[2] elif kw == "IF" or kw == "DIF" or kw == "IFCALC": if kw == "IFCALC": iname = "CALC" - output = StringIO.StringIO() + output = io.StringIO() expand(cfg, [argm.group(1)], output, isfirst) value = output.getvalue() output.close() try: testv = cfg.eval_expr(value) - except: + except Exception: testv = 0 else: iname = argm.group(1) @@ -653,51 +672,55 @@ def expand(cfg, lines, f, isfirst=False): dif = False try: if kw == "IFCALC": - ifre = re.compile("(.*?)\$\$IFCALC\{([^}]*?)\}") + ifre = re.compile(r"(.*?)\$\$IFCALC\{([^}]*?)\}") else: - ifre = re.compile("(.*?)\$\$IF\(" + iname + "(\))") - endre = re.compile("(.*?)\$\$ENDIF\(" + iname + "(\))") - elsere = re.compile("(.*?)\$\$ELSE\(" + iname + "(\))") - except: - print "Invalid $$IF name: %s" % iname + ifre = re.compile(r"(.*?)\$\$IF\(" + iname + r"(\))") + endre = re.compile(r"(.*?)\$\$ENDIF\(" + iname + r"(\))") + elsere = re.compile(r"(.*?)\$\$ELSE\(" + iname + r"(\))") + except Exception: + print("Invalid $$IF name: %s" % iname) sys.exit(1) t = searchforend(lines, endre, ifre, endre, i, loc) - if t == None: - print "Cannot find $$ENDIF(%s)?" % iname + if t is None: + print("Cannot find $$ENDIF(%s)?" % iname) sys.exit(1) elset = searchforend(t[0], elsere, ifre, endre, 0, 0) if kw != "IFCALC": try: v = cfg.ddict[iname] - except: + except Exception: v = "" - testv = 1 if ((dif and v == eqval) or ((not dif) and v != "")) else 0 + testv = ( + 1 + if ((dif and v == eqval) or ((not dif) and v != "")) + else 0 + ) if testv != 0: # True, do the if! - if elset != None: + if elset is not None: newlines = elset[0] else: newlines = t[0] expand(cfg, newlines, f, isfirst) else: # False, do the else! - if elset != None: - newlines = t[0][elset[1]:] - newlines[0] = newlines[0][elset[2]:] + if elset is not None: + newlines = t[0][elset[1] :] + newlines[0] = newlines[0][elset[2] :] expand(cfg, newlines, f, isfirst) i = t[1] loc = t[2] elif kw == "TIF": iname = argm.group(1) if "$$" in iname: - output = StringIO.StringIO() + output = io.StringIO() expand(cfg, [iname], output, isfirst) iname = output.getvalue() output.close() newlines = [] try: v = cfg.ddict[iname] - except: + except Exception: v = "" if v != "": # True, do the if! @@ -709,38 +732,40 @@ def expand(cfg, lines, f, isfirst=False): elif kw == "INCLUDE": try: fn = cfg.ddict[argm.group(1)] - except: + except Exception: fn = argm.group(1) try: - output = StringIO.StringIO() + output = io.StringIO() expand(cfg, [fn], output, isfirst) fn = output.getvalue().strip() output.close() - except: + except Exception: pass try: - newlines=myopen(fn).readlines() + newlines = myopen(fn).readlines() expand(cfg, newlines, f, isfirst) - except: + except Exception: if isfirst: f.write("$$INCLUDE(%s)\n" % argm.group(1)) else: - print "Cannot open file %s!\n" % fn + print("Cannot open file %s!\n" % fn) elif kw == "COUNT": try: cnt = str(len(cfg.idict[argm.group(1)])) - except: + except Exception: cnt = "0" f.write(cnt) elif kw == "ASSIGN": args = argm.group(1).split(",") - output = StringIO.StringIO() + output = io.StringIO() expand(cfg, [args[1]], output, isfirst) value = output.getvalue() output.close() - v = cfg.eval_expr(value) # Yeah, if this isn't valid, just let it crash! + v = cfg.eval_expr( + value + ) # Yeah, if this isn't valid, just let it crash! cfg.assign(args[0], str(v)) - if loc < len(lines[i]) and lines[i][loc] == '\n': + if loc < len(lines[i]) and lines[i][loc] == "\n": loc = loc + 1 elif kw == "CALC": # Either $$CALC{expr} or $$CALC{expr,format}. @@ -749,9 +774,9 @@ def expand(cfg, lines, f, isfirst=False): # We'll basically say if the first string has a '(', # then just assume the first kind. args = argm.group(1).split(",") - if '(' in args[0]: + if "(" in args[0]: args = [argm.group(1)] - output = StringIO.StringIO() + output = io.StringIO() expand(cfg, [args[0]], output, isfirst) value = output.getvalue() output.close() @@ -761,152 +786,170 @@ def expand(cfg, lines, f, isfirst=False): fmt = "%d" try: v = cfg.eval_expr(value) - except: + except Exception: v = 0 f.write(fmt % (v)) elif kw == "UP": try: fn = cfg.ddict[argm.group(1)] - except: + except Exception: fn = argm.group(1) try: - f.write(fn[:fn.rindex('/')]) - except: + f.write(fn[: fn.rindex("/")]) + except Exception: pass elif kw == "ROOT": try: fn = cfg.ddict[argm.group(1)] - except: + except Exception: fn = argm.group(1) - if '.' in fn: - fn = fn[:fn.index('.')] + if "." in fn: + fn = fn[: fn.index(".")] f.write(fn) elif kw == "SUBSTR": - output = StringIO.StringIO() + output = io.StringIO() expand(cfg, [argm.group(1)], output, isfirst) value = output.getvalue() output.close() - start = argm.group(2) + start = argm.group(2) try: - start = cfg.ddict[start] - except: + start = cfg.ddict[start] + except Exception: pass start = int(start) finish = argm.group(3) try: finish = cfg.ddict[finish] - except: + except Exception: pass finish = int(finish) f.write(value[start:finish]) elif kw == "TAIL": - output = StringIO.StringIO() + output = io.StringIO() expand(cfg, [argm.group(1)], output, isfirst) value = output.getvalue() output.close() - start = argm.group(2) + start = argm.group(2) try: - start = cfg.ddict[start] - except: + start = cfg.ddict[start] + except Exception: pass start = int(start) f.write(value[start:]) elif kw == "NAME": - s = argm.group(1).split(',') + s = argm.group(1).split(",") if len(s) != 2: - print "Malformed $$NAME(%s) doesn't have two arguments!" % argm.group(1) + print( + "Malformed $$NAME(%s) doesn't have two arguments!" + % argm.group(1) + ) sys.exit(1) try: s[0] = cfg.ddict[s[0]] - except: + except Exception: pass try: - s = cfg.ddict[s[0] + ":TYPE"] + s[1] + str(cfg.ddict[s[0] + ":INDEX"]) - except: - print "Can't find $$NAME(%s)?" % argm.group(1) + s = ( + cfg.ddict[s[0] + ":TYPE"] + + s[1] + + str(cfg.ddict[s[0] + ":INDEX"]) + ) + except Exception: + print("Can't find $$NAME(%s)?" % argm.group(1)) sys.exit(1) try: val = cfg.ddict[s] f.write(val) - except: + except Exception: pass - else: # Must be "TRANSLATE" + else: # Must be "TRANSLATE" try: - val = cfg.ddict[argm.group(1)].translate(string.maketrans(enumstring(argm.group(2)), - enumstring(argm.group(3)))) + val = cfg.ddict[argm.group(1)].translate( + str.maketrans( + enumstring(argm.group(2)), enumstring(argm.group(3)) + ) + ) f.write(val) - except: + except Exception: pass else: - print "Malformed $$%s statement?" % kw + print("Malformed $$%s statement?" % kw) sys.exit(1) continue - + # Just a variable reference! if lines[i][loc] == "(": m = parens.search(lines[i][loc:]) else: m = word.search(lines[i][loc:]) - if m != None: + if m is not None: try: val = cfg.ddict[m.group(1)] f.write(val) - except: + except Exception: pass - if lines[i][loc] == '(': + if lines[i][loc] == "(": loc += m.end(1) + 1 else: loc += m.end(1) else: - print "Can't find variable name?!?" + print("Can't find variable name?!?") -if __name__ == '__main__': + +def main() -> int: + global expand_path + global extra xp = os.getenv("EXPAND_PATH") - if xp == None: + if xp is None: expand_path = [".."] else: expand_path = xp.split(":") + [".."] - av = sys.argv[1:] # Drop expand.py - if av[0] == '-c': - configfile = av[1] # -c CONFIG + av = sys.argv[1:] # Drop expand.py + if av[0] == "-c": + configfile = av[1] # -c CONFIG av = av[2:] name = os.path.basename(configfile) if name[-4:] == ".cfg": name = name[:-4] extra = "CONFIG=" + name else: - configfile = "config" + configfile = "config" extra = "CONFIG=" - if len(av) == 0 or av[0] == '-h': - print "Usage: expand.py [ -c CONFIG ] TEMPLATE OUTFILE [ ADDITIONAL_STATEMENTS ]" - print " or: expand.py [ -c CONFIG ] NAME" - sys.exit(1) + if len(av) == 0 or av[0] == "-h": + print( + "Usage: expand.py [ -c CONFIG ] TEMPLATE OUTFILE [ ADDITIONAL_STATEMENTS ]" + ) + print(" or: expand.py [ -c CONFIG ] NAME") + return 1 try: if len(av) == 1: - cfg=config() + cfg = config() cfg.read_config(configfile, []) - lines=['$$' + av[0] + '\n'] + lines = ["$$" + av[0] + "\n"] expand(cfg, lines, sys.stdout) - sys.exit(0) - cfg=config() + return 0 + cfg = config() cfg.read_config(configfile, av[2:]) try: - tplFile=myopen(av[0]) + tplFile = myopen(av[0]) if not tplFile: - print "Unable to open template file:", av[0] - sys.exit(1) - except IOError, e: - print e - sys.exit(1) - lines=tplFile.readlines() - if av[1] == '-': + print("Unable to open template file:", av[0]) + return 1 + except IOError as e: + print(e) + return 1 + lines = tplFile.readlines() + if av[1] == "-": fp = sys.stdout else: - fp = open(av[1], 'w') + fp = open(av[1], "w") expand(cfg, lines, fp) fp.close() - sys.exit(0) - except IOError, e: - print e - sys.exit(1) + return 0 + except IOError as e: + print(e) + return 1 + +if __name__ == "__main__": + sys.exit(main()) From aa32183794c43eda059487f59d143c8fa6962d1d Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Mon, 9 Dec 2024 16:54:40 -0800 Subject: [PATCH 03/25] TST: add substantial end-to-end testing and one unit test --- tests/__init__.py | 0 tests/conftest.py | 14 ++ tests/generate_artifacts.py | 344 ++++++++++++++++++++++++++++++++++++ tests/test_expand.py | 216 ++++++++++++++++++++++ tests/test_keywords.py | 35 ++++ 5 files changed, 609 insertions(+) create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/generate_artifacts.py create mode 100644 tests/test_expand.py create mode 100644 tests/test_keywords.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..5b782a88 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,14 @@ +import contextlib +import sys + + +@contextlib.contextmanager +def cli_args(args): + """ + Context manager for running a block of code with a specific set of + command-line arguments. + """ + prev_args = sys.argv + sys.argv = args + yield + sys.argv = prev_args diff --git a/tests/generate_artifacts.py b/tests/generate_artifacts.py new file mode 100644 index 00000000..8ffc229f --- /dev/null +++ b/tests/generate_artifacts.py @@ -0,0 +1,344 @@ +""" +This is a script to generate the contents of the common, examples, and expected folders. + +Some of this information is not trivially available to a web consumer, so I'm +opting to generate it via copying from deployed filesystem locations. + +This would allow us to run unit tests that reference real configurations +on the GitHub actions CI. +""" + +from __future__ import annotations + +import logging +import pathlib +import shutil +import subprocess +import typing + +logger = logging.getLogger(__name__) + +# Add an ioc name to BANNED_IOCs to skip all IOCs that would otherwise +# be sorted into these names. +# Only ban IOCs that are malformed, +# failing the test with a file not found error and such +# or e.g. IOC name duplication that creates ambiguity. +BANNED_IOC_TYPES = [ + "edtCam", # not used any more and lots of IOC name duplications come from it + "gigEcam", # annoying typo breaks some things (should have been gigECam) + "leviton", # renamed to pdu_snmp, also similar name typo hell + "Leviton", # renamed to pdu_snmp, also similar name typo hell + "Levitons", # renamed to pdu_snmp, also similar name typo hell +] +# Special cases: only for IOCs whose latest versions have build errors! +BANNED_IOC_NAMES = [ + # These scope IOCs must have had an NFS issue during the build. + # The example files just say "this path doesn't exist!" (but it does) + "ioc-cxi-scope-portable01", + "ioc-cxi-scope-portable02", + # These IOCs' releases were hand-edited... + "ioc-las-ftl-mcs2-01", + "ioc-las-ftl-mcs2-02", + "ioc-mfx-hera-smc100", + "ioc-tmo-mcs2-01", +] +# Focus on the targets of expand.py from RULES_EXPAND +# Avoid other potentially large files +VALID_EXPAND_GLOBS = [ + "Makefile", + "st.cmd", + "edm-*.cmd", + "pydm-*.cmd", + "launchgui-*.cmd", + "syncts-*.cmd", + "*.sh", + # Missing here is iocname.*, handle later when we know iocname +] +# Save time in the recursive search for templated iocs +SKIP_SEARCH_DIRS = [ + ".git", + ".svn", + "build", +] + + +def generate_examples( + ioc_deploy_path: pathlib.Path, examples_path: pathlib.Path +) -> None: + """ + Generate the examples filder. + + Fills examples_path with the latest versions of .cfg files from + under the ioc_deploy_path tree. + """ + if not ioc_deploy_path.is_dir() or not examples_path.is_dir(): + raise ValueError( + f"Expected {ioc_deploy_path} and {examples_path} to be directories." + ) + # Should be e.g. "/cds/group/pcds/epics/ioc/xpp/gigECam/R2.0.7" + for template_ioc in iter_latest_template_iocs(ioc_deploy_path=ioc_deploy_path): + if template_ioc.parent.name in BANNED_IOC_TYPES: + continue + for cfg_path in template_ioc.glob("*.cfg"): + if cfg_path.stem in BANNED_IOC_NAMES: + continue + try: + # Should be e.g. "/cds/group/pcds/epics/ioc/common/gigECam/R5.0.4" + release_path = get_release_path(cfg_file=cfg_path) + except InvalidReleaseError: + logger.info(f"{cfg_path} does not have a valid release.") + continue + variant = release_path.parent.name + if variant in BANNED_IOC_TYPES: + continue + examples_target = examples_path / variant + examples_target.mkdir(exist_ok=True) + log_copy(src=cfg_path, dst=examples_target) + + chmod_uplusw(path=examples_path) + + +def chmod_uplusw(path: str | pathlib.Path) -> subprocess.CompletedProcess: + return subprocess.run(["chmod", "-R", "u+w", str(path)]) + + +def log_copy(src: pathlib.Path, dst: pathlib.Path): + """ + Copy file from src to dst, log the copy, error on copies outside of tests dir. + """ + tests_dir = pathlib.Path(__file__).parent + check_parent = dst + while check_parent not in (tests_dir, check_parent.parent): + check_parent = check_parent.parent + if check_parent.parent == check_parent: + raise ValueError( + f"Cannot copy to outside the tests folder. Tried to copy to {dst}" + ) + logger.info(f"shutil.copy(src={src}, dst={dst})") + try: + shutil.copy(src=src, dst=dst) + except OSError as exc: + logger.warning(f"File copy failed! {exc}") + + +def iter_latest_template_iocs( + ioc_deploy_path: pathlib.Path, +) -> typing.Iterator[pathlib.Path]: + if not ioc_deploy_path.is_dir(): + return + try: + latest_version = pick_latest_version(ioc_path=ioc_deploy_path) + except RuntimeError: + ... + else: + if is_template_ioc(ioc_path=latest_version): + return (yield latest_version) + else: + return + for subpath in ioc_deploy_path.iterdir(): + if subpath.name in SKIP_SEARCH_DIRS: + continue + yield from iter_latest_template_iocs(ioc_deploy_path=subpath) + + +def pick_latest_version(ioc_path: pathlib.Path) -> pathlib.Path: + """ + Given a directory with version-named folders, return the latest version. + """ + latest = (0, 0, 0) + latest_path = None + for version_path in ioc_path.iterdir(): + try: + version = get_version_tuple(version_str=version_path.name) + except ValueError: + continue + if len(version) < 2: + logger.info( + f"In {ioc_path} found {version_path.name} " + "which has fewer than 2 elements." + ) + continue + if version > latest: + latest = version + latest_path = version_path + if latest_path is None: + raise RuntimeError(f"No version directories in {ioc_path}") + return latest_path + + +def get_version_tuple(version_str: str) -> tuple[int, int, int]: + """ + Convert a version string like R2.0.0 to a tuple for easy comparisons. + """ + orig_ver_str = version_str + # Remove leading v, V, r, R + while version_str and version_str[0].isalpha(): + version_str = version_str[1:] + if not version_str: + raise ValueError(f"{orig_ver_str} is not a valid version.") + try: + return tuple(int(ver) for ver in version_str.split(".")) + except ValueError as exc: + raise ValueError(f"{orig_ver_str} is not a valid version.") from exc + + +def is_template_ioc(ioc_path: pathlib.Path) -> bool: + """ + Returns True if the ioc deployed at ioc_path is a template ioc (with .cfg files) + """ + return bool(list(ioc_path.glob("*.cfg"))) and (ioc_path / "build").is_dir() + + +def get_release_path(cfg_file: pathlib.Path) -> pathlib.Path: + """ + Get the path to the release folder that cfg_file will use to build. + + This will raise if there is no release path or if the release path is not tagged. + """ + release_dir = None + with open(cfg_file, "r") as fd: + for line in fd: + if line.startswith("RELEASE") and "=" in line: + release_dir = line.split("=")[1].strip() + break + # Should be e.g. "/cds/group/pcds/epics/ioc/common/gigECam/R5.0.4" + if release_dir is None: + raise InvalidReleaseError + release_path = pathlib.Path(release_dir) + try: + get_version_tuple(release_path.name) + except ValueError as exc: + raise InvalidReleaseError from exc + return release_path + + +class InvalidReleaseError(RuntimeError): ... + + +def generate_common(examples_path: pathlib.Path, common_path: pathlib.Path) -> None: + """ + Given config files in examples_path, generate common_path with necessary templates. + + The contents of common_path will be e.g. + common/gigECam/R3.0.0/st.cmd + + It will not be the full IOC, just the contents of iocBoot/templates. + This will include every template referenced by the config files in example_path. + """ + if not examples_path.is_dir() or not common_path.is_dir(): + raise ValueError( + f"Expected {examples_path} and {common_path} to be directories." + ) + for cfg_file in examples_path.glob("**/*.cfg"): + # Should be e.g. "/cds/group/pcds/epics/ioc/common/gigECam/R5.0.4" + release_path = get_release_path(cfg_file=cfg_file) + variant = release_path.parent.name + if variant in BANNED_IOC_TYPES: + continue + version = release_path.name + templates = release_path / "iocBoot" / "templates" + this_cfg_common_dir = common_path / variant / version + if this_cfg_common_dir.exists(): + logger.debug(f"{this_cfg_common_dir} already exists, skipping.") + continue + this_cfg_common_dir.parent.mkdir(exist_ok=True) + this_cfg_common_dir.mkdir() + for file_path in templates.iterdir(): + if file_path.is_file(): + log_copy(src=file_path, dst=this_cfg_common_dir) + + chmod_uplusw(path=common_path) + + +def generate_expected( + ioc_deploy_path: pathlib.Path, + examples_path: pathlib.Path, + expected_path: pathlib.Path, +) -> None: + """ + Given the ioc_deploy_path, generate expected_path with the real template results. + + The contents of expected_path will be e.g. + expected/gigECam/iocname/st.cmd + + It will not be the full IOC, just the contents of build/iocBoot/iocname + """ + if ( + not ioc_deploy_path.is_dir() + or not examples_path.is_dir() + or not expected_path.is_dir() + ): + raise ValueError( + f"Expected {ioc_deploy_path}, {examples_path}, " + f"and {expected_path} to be directories." + ) + + # template_ioc is something like "/cds/group/pcds/epics/ioc/xpp/gigECam/R2.0.4" + for template_ioc in iter_latest_template_iocs(ioc_deploy_path=ioc_deploy_path): + variant = template_ioc.parent.name + if variant in BANNED_IOC_TYPES: + continue + + built_iocs_subfolder = template_ioc / "build" / "iocBoot" + for built_ioc in built_iocs_subfolder.iterdir(): + if not built_ioc.is_dir(): + continue + if not list(examples_path.glob(f"**/{built_ioc.name}.cfg")): + logger.info(f"{built_ioc.name}.cfg not in {examples_path}, skipping") + continue + + iocname = built_ioc.name + + expected_path.mkdir(exist_ok=True) + (expected_path / variant).mkdir(exist_ok=True) + + expected_ioc_target = expected_path / variant / iocname + expected_ioc_target.mkdir(exist_ok=True) + + for glob_pattern in VALID_EXPAND_GLOBS + [f"{iocname}.*"]: + for built_file in built_ioc.glob(glob_pattern): + log_copy(src=built_file, dst=expected_ioc_target) + + chmod_uplusw(path=expected_path) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.WARNING) + + ioc_deploy_path = pathlib.Path("/cds/group/pcds/epics/ioc") + examples_path = pathlib.Path(__file__).parent / "examples" + common_path = pathlib.Path(__file__).parent / "common" + expected_path = pathlib.Path(__file__).parent / "expected" + + areas = [ + "cxi", + "det", + "kfe", + "las", + "lfe", + "mec", + "mfx", + "rix", + "tmo", + "txi", + "ued", + "xcs", + "xpp", + "xrt", + ] + + for area in areas: + generate_examples( + ioc_deploy_path=ioc_deploy_path / area, + examples_path=examples_path, + ) + generate_common( + examples_path=examples_path, + common_path=common_path, + ) + for area in areas: + generate_expected( + ioc_deploy_path=ioc_deploy_path / area, + examples_path=examples_path, + expected_path=expected_path, + ) diff --git a/tests/test_expand.py b/tests/test_expand.py new file mode 100644 index 00000000..ba883fc8 --- /dev/null +++ b/tests/test_expand.py @@ -0,0 +1,216 @@ +import os +import pathlib +import re + +import pytest + +from expand import main + +from .conftest import cli_args + + +def get_release_dir(config_file: pathlib.Path) -> str: + release_dir = None + with open(config_file, "r") as fd: + for line in fd: + if line.startswith("RELEASE") and "=" in line: + release_dir = line.split("=")[1].strip() + break + + # release_dir should be something like /reg/g/pcds/epics/ioc/common/ims/R6.7.0 + if not isinstance(release_dir, str): + raise RuntimeError(f"No release dir found for {config_file}!") + return release_dir + + +def get_template_dir_from_release_dir(release_dir: str) -> pathlib.Path: + release_dir = release_dir.removesuffix("/") + long_path, version_str = os.path.split(release_dir) + long_path, common_name = os.path.split(long_path) + + # something like .../ioc-template-macros/tests/common/ims/R6.7.0 + return pathlib.Path(__file__).parent / "common" / common_name / version_str + + +examples = pathlib.Path(__file__).parent / "examples" +configs = {pth.name: pth for pth in examples.glob("**/*.cfg")} +template_globs = [ + "Makefile", + "st.cmd", + "edm-ioc.cmd", + "pydm-ioc.cmd", + "launchgui-ioc.cmd", + "syncts-ioc.cmd", + "ioc.*", + "*.sh", +] + +variants = [] + + +def init_variants(): + variants.clear() + for cfg_name, config_file in configs.items(): + this_cfg_templates = [] + release_dir = get_release_dir(config_file=config_file) + template_dir = get_template_dir_from_release_dir(release_dir=release_dir) + for pattern in template_globs: + if list(template_dir.glob(pattern=pattern)): + this_cfg_templates.append(pattern) + if not this_cfg_templates: + raise RuntimeError( + f"Did not find any templates for {cfg_name} ({config_file})" + ) + for template in this_cfg_templates: + variants.append((cfg_name, template)) + + +init_variants() + + +@pytest.mark.parametrize( + "cfg_name,template", + variants, +) +def test_expand_full(tmp_path: pathlib.Path, cfg_name: str, template: str): + """ + Check that each config file can be used with expand.py. + + Command-line direct testing of expand.py is something like: + expand -c config_file template_file output_file + + The Makefiles automate this process for real IOC builds. + This looks something like: + + @$(EXPAND) -c $(1).cfg $$(<) $$@ IOCNAME=$(1) TOP=$(BUILD_TOP_ABS) IOCTOP=$(IOC_APPL_TOP) + + Where: + - $(EXPAND) gets replaced with our "expand" script (which calls expand.py) + - $(1).cfg gets filled in with the config file + - $$(<) gets filled in with the first prerequisite, which ends up being a + template file + - $$@ gets filled in with the target filename + - The rest get passed in as "additional arguments" (undocumented), + but all this does is treat the config file as if it starts with those + declarations on separate lines. + Effectively, this adds additional variable definitions from RULES_EXPAND: + + - IOCNAME gets set to the name of the config file, without .cfg + - TOP gets set to the absolute path to the "build" directory. + The "build" directory is e.g. + /cds/group/pcds/epics/ioc/xpp/gigECam/R2.0.7/build + - IOCTOP ultimately gets set to whatever the RELEASE line in the config file is. + + There is an additional undocumented "expand -c config_file name" syntax + that initially appears unused, but it's actually used to generate a + shell script named IOC_APPL_TOP that sets IOC_APPL_TOP to the RELEASE line. + Bizarre behavior. TODO add a unit test for that case. + + For the test here, we want to generate files one by one + and check that they are correct (no regressions) + """ # noqa: E501 + config_file = configs[cfg_name] + # Something like /cds/group/pcds/epics/ioc/common/gigECam/R5.0.5 + release_dir = get_release_dir(config_file=config_file) + # Something like ../ioc-template-macros/tests/common/gigECam/R5.0.5 + template_dir = get_template_dir_from_release_dir(release_dir=release_dir) + version_str = template_dir.name + common_name = template_dir.parent.name + + template_files = list(template_dir.glob(template)) + if not template_files: + raise RuntimeError( + "Test collection error, should not check " + f"{template} for {common_name}/{version_str}." + ) + + iocname = config_file.stem + build_dir = str(tmp_path) + for template_file in template_files: + if "ioc" in template: + target_file = tmp_path / template_file.name.replace("ioc", iocname) + else: + target_file = tmp_path / template_file.name + with cli_args( + [ + "expand", + "-c", + str(config_file), + str(template_file), + str(target_file), + f"IOCNAME={iocname}", + f"TOP={build_dir}", + f"IOCTOP={release_dir}", + ] + ): + main() + + assert target_file.exists() + with open(target_file, "r") as fd: + output_lines = fd.read().splitlines() + expected_file = ( + pathlib.Path(__file__).parent + / "expected" + / common_name + / iocname + / target_file.name + ) + if not expected_file.exists(): + # Oh no, maybe it's somewhere nearby! IOCs get renamed sometimes... + glob_paths = list( + (pathlib.Path(__file__).parent / "expected").glob( + f"**/{iocname}/{target_file.name}" + ) + ) + if not glob_paths: + # Argh + raise RuntimeError(f"Cannot find {expected_file} in test.") + elif len(glob_paths) == 1: + # Phew, we found it! + expected_file = glob_paths[0] + else: + # We found... two or more??? + raise RuntimeError( + f"Found more than one alternate candidate for {expected_file} " + f"in test: {glob_paths}" + ) + with open(expected_file, "r") as fd: + expected_lines = fd.read().splitlines() + failure_info = ( + f"Regression in using config file {config_file} " + f"to expand template file {template_file} " + f"to make {target_file.name}. " + f"Expected to match {expected_file}." + ) + assert len(output_lines) == len(expected_lines), failure_info + working_dir = os.getcwd() + for output, expected in zip(output_lines, expected_lines): + if build_dir in output: + # Special case 1: our pytest build dir is not the real build dir + assert full_match_ignoring_test_artifact( + text=output, test_artifact=build_dir, expected=expected + ) + elif working_dir in output: + # Special case 2: our cwd is not the same cwd as the original make + assert full_match_ignoring_test_artifact( + text=output, test_artifact=working_dir, expected=expected + ) + else: + assert output == expected, failure_info + + +def full_match_ignoring_test_artifact( + text: str, test_artifact: str, expected: str +) -> bool: + """ + Return True if text and expected are equal, except for instances of test_artifact. + + This is useful when we don't have the original paths at build time, + so we expect that the test suite output only differs from the original output + in a small number of places. + """ + text = text.replace(test_artifact, ".*") + for special_char in "()$": + text = text.replace(special_char, f"\\{special_char}") + text = f"^{text}$" + return re.fullmatch(text, expected) diff --git a/tests/test_keywords.py b/tests/test_keywords.py new file mode 100644 index 00000000..eecf3734 --- /dev/null +++ b/tests/test_keywords.py @@ -0,0 +1,35 @@ +""" +Unit tests for individual supported keywords in expand.py +""" + +import pathlib + +from expand import main + +from .conftest import cli_args + + +def test_translate(tmp_path: pathlib.Path): + config = "TRIG=2" + template = 'AFTER.B.COMES.$$TRANSLATE(TRIG,"0123456789AB","ABCDEFGHIJKL")' + expected = "AFTER.B.COMES.C" + + config_path = tmp_path / "translate.cfg" + template_path = tmp_path / "template.txt" + output_path = tmp_path / "output.cfg" + + with open(config_path, "w") as fd: + fd.write(config) + + with open(template_path, "w") as fd: + fd.write(template) + + with cli_args( + ["expand", "-c", str(config_path), str(template_path), str(output_path)] + ): + main() + + with open(output_path, "r") as fd: + result = fd.read() + + assert result == expected From e2af9bcc5ea4c2bf1f9e21a9bd6b8e7961e3823f Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Mon, 9 Dec 2024 16:55:02 -0800 Subject: [PATCH 04/25] STY: pre-commit run --all-files --- NOTES | 14 +++++++------- RULES_EXPAND | 1 - realpath.c | 2 -- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/NOTES b/NOTES index a3820ac6..12e6df3e 100644 --- a/NOTES +++ b/NOTES @@ -36,8 +36,8 @@ All of the macro commands in the template files begin with "$$". These can be: - Process the contents of the FILENAME. $$CALC{EXPRESSION} or $$CALC{EXPRESSION,FORMAT} - NOTE THE BRACKETS!!! This allows arbitrary arithmetic. The EXPRESSION - is expanded, and then evaluated as a mathematical expression. Any - undefined atom is assumed to be zero. Atoms within the expression do + is expanded, and then evaluated as a mathematical expression. Any + undefined atom is assumed to be zero. Atoms within the expression do not need to be prefixed by $$, but maybe. (They should be if within a $$LOOP inside the expression.) The result is output as a decimal number, or using the given format if one was given. @@ -75,7 +75,7 @@ For example, if we have: TEST:E(X=Z) A(B=C,D=F,E0) A(B=Q,D=R,TEST) - + Then we have defined global symbols $$EX0 = Y, $$EX1 = Z, $$AB0 = C, $$AD0 = F, $$AB1 = Q and $$AD1 = R. If we $$LOOP(A), then within the body of the loop, we will have $$B = C, $$D = F, and $$EX = Y when $$INDEX is 0, and $$B = Q, @@ -116,9 +116,9 @@ To clarify this, a config file such as: ENGINEER=Michael Browne (mcbrowne) LOCATION=MEC:R64A:24 IOC_PV=IOC:MEC:IMB02 - + EVR(NAME=MEC:XT2:EVR:01,TYPE=PMC) - + IPIMB(NAME=MEC:XT2:IPM:02,PORT=/dev/ttyPS3,BLDID=23,EVR0,TRIG=0) IPIMB(NAME=MEC:XT2:PIM:02,PORT=/dev/ttyPS2,BLDID=42,EVR0,TRIG=0) IPIMB(NAME=MEC:XT2:IPM:03,PORT=/dev/ttyPS1,BLDID=24,EVR0,TRIG=0) @@ -130,11 +130,11 @@ could be written: ENGINEER Michael Browne (mcbrowne) LOCATION MEC:R64A:24 IOC_PV IOC:MEC:IMB02 - + INSTANCE EVR NAME MEC:XT2:EVR:01 TYPE PMC - + INSTANCE IPIMB NAME MEC:XT2:IPM:02 PORT /dev/ttyPS3 diff --git a/RULES_EXPAND b/RULES_EXPAND index dc1afafe..e970ec21 100644 --- a/RULES_EXPAND +++ b/RULES_EXPAND @@ -182,4 +182,3 @@ expandclean: -rm -rf $(BUILD_TOP)/iocBoot -rm -rf $(BUILD_TOP)/Makefile -rm -rf $(BUILD_TOP)/IOC_APPL_TOP - diff --git a/realpath.c b/realpath.c index 090a8fd2..1fc21bf0 100644 --- a/realpath.c +++ b/realpath.c @@ -13,5 +13,3 @@ int main(int argc, char **argv) printf("\n"); return 0; } - - From 9d6e4bc59ba032193f1108c86c381a2641bab3d6 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Mon, 9 Dec 2024 16:56:09 -0800 Subject: [PATCH 05/25] MNT: placeholder files so git keeps these empty directories --- tests/common/.gitkeep | 0 tests/examples/.gitkeep | 0 tests/expected/.gitkeep | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/common/.gitkeep create mode 100644 tests/examples/.gitkeep create mode 100644 tests/expected/.gitkeep diff --git a/tests/common/.gitkeep b/tests/common/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/examples/.gitkeep b/tests/examples/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/expected/.gitkeep b/tests/expected/.gitkeep new file mode 100644 index 00000000..e69de29b From 12b67cd3c404463cc5c21e81b0397490448a86a0 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Mon, 9 Dec 2024 18:04:43 -0800 Subject: [PATCH 06/25] TST: include children subfolder when looking for example templates --- tests/generate_artifacts.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/generate_artifacts.py b/tests/generate_artifacts.py index 8ffc229f..9504346e 100644 --- a/tests/generate_artifacts.py +++ b/tests/generate_artifacts.py @@ -76,9 +76,8 @@ def generate_examples( f"Expected {ioc_deploy_path} and {examples_path} to be directories." ) # Should be e.g. "/cds/group/pcds/epics/ioc/xpp/gigECam/R2.0.7" + # or e.g. "/cds/group/pcds/epics/ioc/common/bk-1697/R1.0.1/children" for template_ioc in iter_latest_template_iocs(ioc_deploy_path=ioc_deploy_path): - if template_ioc.parent.name in BANNED_IOC_TYPES: - continue for cfg_path in template_ioc.glob("*.cfg"): if cfg_path.stem in BANNED_IOC_NAMES: continue @@ -131,8 +130,13 @@ def iter_latest_template_iocs( except RuntimeError: ... else: + ioc_type = latest_version.parent.name + if ioc_type in BANNED_IOC_TYPES: + return if is_template_ioc(ioc_path=latest_version): return (yield latest_version) + elif is_template_ioc(ioc_path=latest_version / "children"): + return (yield latest_version / "children") else: return for subpath in ioc_deploy_path.iterdir(): @@ -274,10 +278,12 @@ def generate_expected( ) # template_ioc is something like "/cds/group/pcds/epics/ioc/xpp/gigECam/R2.0.4" + # or, it can also be "/cds/group/pcds/epics/ioc/common/bk-1697/R1.0.1/children" for template_ioc in iter_latest_template_iocs(ioc_deploy_path=ioc_deploy_path): - variant = template_ioc.parent.name - if variant in BANNED_IOC_TYPES: - continue + if template_ioc.name == "children": + variant = template_ioc.parent.parent.name + else: + variant = template_ioc.parent.name built_iocs_subfolder = template_ioc / "build" / "iocBoot" for built_ioc in built_iocs_subfolder.iterdir(): From e08efacee5510a569817a382d8686534e96b0c8a Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Mon, 9 Dec 2024 18:52:51 -0800 Subject: [PATCH 07/25] TST: special handling for UP macro in test gen, include common area --- tests/generate_artifacts.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/generate_artifacts.py b/tests/generate_artifacts.py index 9504346e..2d4c647e 100644 --- a/tests/generate_artifacts.py +++ b/tests/generate_artifacts.py @@ -29,6 +29,9 @@ "leviton", # renamed to pdu_snmp, also similar name typo hell "Leviton", # renamed to pdu_snmp, also similar name typo hell "Levitons", # renamed to pdu_snmp, also similar name typo hell + "optics-notepad", # renamed to optics-pitch-notepad + "topas", # latest release broken (failed build), not deployed in iocmanager + "tricatt", # typo (tricam) and failed build ] # Special cases: only for IOCs whose latest versions have build errors! BANNED_IOC_NAMES = [ @@ -93,6 +96,16 @@ def generate_examples( examples_target = examples_path / variant examples_target.mkdir(exist_ok=True) log_copy(src=cfg_path, dst=examples_target) + # We need to edit the file in place if it has $$UP(PATH) as its release + # Since the original path had the context for the absolute release path + new_file = examples_target / cfg_path.name + with open(new_file, "r") as fd: + text = fd.read() + if "$$UP(PATH)" in text: + new_text = text.replace("$$UP(PATH)", str(release_path)) + chmod_uplusw(path=new_file) + with open(new_file, "w") as fd: + fd.write(new_text) chmod_uplusw(path=examples_path) @@ -174,6 +187,9 @@ def get_version_tuple(version_str: str) -> tuple[int, int, int]: """ Convert a version string like R2.0.0 to a tuple for easy comparisons. """ + # Avoid cases like ek9000 which otherwise parse to version 9000 + if "." not in version_str: + raise ValueError(f"{version_str} is not a valid version.") orig_ver_str = version_str # Remove leading v, V, r, R while version_str and version_str[0].isalpha(): @@ -208,7 +224,11 @@ def get_release_path(cfg_file: pathlib.Path) -> pathlib.Path: # Should be e.g. "/cds/group/pcds/epics/ioc/common/gigECam/R5.0.4" if release_dir is None: raise InvalidReleaseError - release_path = pathlib.Path(release_dir) + # Special case: cfg file in children folder refers to parent dir via macro + if release_dir == "$$UP(PATH)": + release_path = cfg_file.parent.parent + else: + release_path = pathlib.Path(release_dir) try: get_version_tuple(release_path.name) except ValueError as exc: @@ -317,6 +337,7 @@ def generate_expected( expected_path = pathlib.Path(__file__).parent / "expected" areas = [ + "common", "cxi", "det", "kfe", From b1e37dc3edb693ab86a9e16fc6038e0e75d9e279 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Mon, 9 Dec 2024 18:58:33 -0800 Subject: [PATCH 08/25] TST: normalize /reg/g/ to /cds/group for comparisons --- tests/test_expand.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_expand.py b/tests/test_expand.py index ba883fc8..a859513f 100644 --- a/tests/test_expand.py +++ b/tests/test_expand.py @@ -185,6 +185,9 @@ def test_expand_full(tmp_path: pathlib.Path, cfg_name: str, template: str): assert len(output_lines) == len(expected_lines), failure_info working_dir = os.getcwd() for output, expected in zip(output_lines, expected_lines): + # Preprocessing: /reg/g/ -> /cds/group/ for fair comparison + output = normalize_reg(output) + expected = normalize_reg(expected) if build_dir in output: # Special case 1: our pytest build dir is not the real build dir assert full_match_ignoring_test_artifact( @@ -214,3 +217,10 @@ def full_match_ignoring_test_artifact( text = text.replace(special_char, f"\\{special_char}") text = f"^{text}$" return re.fullmatch(text, expected) + + +def normalize_reg(text: str) -> str: + """ + /reg/g/ -> /cds/group + """ + return text.replace("/reg/g/", "/cds/group/") From ab41504225459f372e1589afdaff442139e3abb7 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 10:23:47 -0800 Subject: [PATCH 09/25] TST: ban more disruptive iocs --- tests/generate_artifacts.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/generate_artifacts.py b/tests/generate_artifacts.py index 2d4c647e..c7319a5d 100644 --- a/tests/generate_artifacts.py +++ b/tests/generate_artifacts.py @@ -25,11 +25,14 @@ # or e.g. IOC name duplication that creates ambiguity. BANNED_IOC_TYPES = [ "edtCam", # not used any more and lots of IOC name duplications come from it + "hpi_6012", # most recent build has a missing file so we can't check against it "gigEcam", # annoying typo breaks some things (should have been gigECam) "leviton", # renamed to pdu_snmp, also similar name typo hell "Leviton", # renamed to pdu_snmp, also similar name typo hell "Levitons", # renamed to pdu_snmp, also similar name typo hell "optics-notepad", # renamed to optics-pitch-notepad + "pnccd", # very old, not used, has super old path names that are a distraction + "RohdeSchwartzNGPS", # pnccd support IOC with same issues as pnccd ioc "topas", # latest release broken (failed build), not deployed in iocmanager "tricatt", # typo (tricam) and failed build ] @@ -40,10 +43,12 @@ "ioc-cxi-scope-portable01", "ioc-cxi-scope-portable02", # These IOCs' releases were hand-edited... + "ioc-cxi-setra", "ioc-las-ftl-mcs2-01", "ioc-las-ftl-mcs2-02", "ioc-mfx-hera-smc100", "ioc-tmo-mcs2-01", + "ioc-xpp-ensemble-01", ] # Focus on the targets of expand.py from RULES_EXPAND # Avoid other potentially large files From 2f2d150bd0e34489fd3ecd837aab472cc439a475 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 10:41:58 -0800 Subject: [PATCH 10/25] TST: add a test for uppath with otherwise would be skipped --- tests/conftest.py | 15 +++++++++++++++ tests/test_expand.py | 3 ++- tests/test_keywords.py | 41 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5b782a88..34cc9299 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,8 @@ +from __future__ import annotations + import contextlib +import os +import pathlib import sys @@ -12,3 +16,14 @@ def cli_args(args): sys.argv = args yield sys.argv = prev_args + + +@contextlib.contextmanager +def pushd(directory: str | pathlib.Path): + """ + Context manager for changing to a specific directory for a code block. + """ + cwd = os.getcwd() + os.chdir(directory) + yield + os.chdir(cwd) diff --git a/tests/test_expand.py b/tests/test_expand.py index a859513f..916a277a 100644 --- a/tests/test_expand.py +++ b/tests/test_expand.py @@ -104,7 +104,8 @@ def test_expand_full(tmp_path: pathlib.Path, cfg_name: str, template: str): There is an additional undocumented "expand -c config_file name" syntax that initially appears unused, but it's actually used to generate a shell script named IOC_APPL_TOP that sets IOC_APPL_TOP to the RELEASE line. - Bizarre behavior. TODO add a unit test for that case. + Bizarre behavior. The overall goal for this is apparently to locate the + RELEASE directory in a preprocessing step. This will be tested separately. For the test here, we want to generate files one by one and check that they are correct (no regressions) diff --git a/tests/test_keywords.py b/tests/test_keywords.py index eecf3734..3e19c709 100644 --- a/tests/test_keywords.py +++ b/tests/test_keywords.py @@ -4,12 +4,17 @@ import pathlib +import pytest + from expand import main -from .conftest import cli_args +from .conftest import cli_args, pushd def test_translate(tmp_path: pathlib.Path): + """ + Unit test included here because the TRANSLATE keyword broke and needed to be fixed. + """ config = "TRIG=2" template = 'AFTER.B.COMES.$$TRANSLATE(TRIG,"0123456789AB","ABCDEFGHIJKL")' expected = "AFTER.B.COMES.C" @@ -33,3 +38,37 @@ def test_translate(tmp_path: pathlib.Path): result = fd.read() assert result == expected + + +def test_standard_up_path(tmp_path: pathlib.Path, capsys: pytest.CaptureFixture): + """ + Unit test included because testing the $$UP(PATH) construct is skipped otherwise. + + This is because test_expand is only testing how we fill templates, not how we + find templates. + + This string is included in a config file (not a template) in order to reference + the encapsulating IOC directory. + + It is used in a preprocessing step to find where the templates directory is. + + The expected behavior is for this string to be replaced by the directory above + the user's working directory, and then returned to us as in RULES_EXPAND: + + IOC_APPL_TOP = $$(shell $(EXPAND) -c $(1).cfg RELEASE) + + This specific combination is tested because it is used extensively in + common IOCs with a children folder. + """ + config = "RELEASE=$$UP(PATH)" + + with open(tmp_path / "up-path-test.cfg", "w") as fd: + fd.write(config) + + with pushd(tmp_path): + with cli_args(["expand", "-c", "up-path-test.cfg", "RELEASE"]): + capsys.readouterr() + main() + outerr = capsys.readouterr() + + assert outerr.out.strip() == f"{tmp_path.parent}" From c7b8b232b0d06729fa94ece647fccbec49ea969e Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 10:57:15 -0800 Subject: [PATCH 11/25] TST: direct test for basic preprocessing feature --- tests/test_expand.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/test_expand.py b/tests/test_expand.py index 916a277a..1d291303 100644 --- a/tests/test_expand.py +++ b/tests/test_expand.py @@ -225,3 +225,47 @@ def normalize_reg(text: str) -> str: /reg/g/ -> /cds/group """ return text.replace("/reg/g/", "/cds/group/") + + +config_vars = { + "RELEASE": "/some/release/path", + "ENGINEER": "Mr. Beckhoff", + "LOCATION": "Hammer space", + "ASDF": "asdfasdf", +} + + +@pytest.mark.parametrize( + "config_var", + list(config_vars), +) +def test_expand_preprocessing( + tmp_path: pathlib.Path, capsys: pytest.CaptureFixture, config_var: str +): + """ + Before we fill a template, we can inspect the .cfg file using expand.py. + + This has some pretty involved specifications in the source code that + involve including macros inside the config file itself. + + I'll ignore all of this and test the most basic thing, which is actually + used in RULES_EXPAND: checking variables values from the cfg file. + + This is typically used to get the RELEASE path: + + IOC_APPL_TOP = $$(shell $(EXPAND) -c $(1).cfg RELEASE) + + Often this is combined with the "UP" macro: + see test_keywords::test_standard_up_path where we test this macro. + """ + cfg_file = tmp_path / "test-expand-processing.cfg" + with open(cfg_file, "w") as fd: + for key, value in config_vars.items(): + fd.write(f"{key}={value}\n") + + with cli_args(["expand", "-c", str(cfg_file), config_var]): + capsys.readouterr() + main() + outerr = capsys.readouterr() + + assert outerr.out.strip() == config_vars[config_var] From 082c52b113ef21d01d29832420bb729a3b8e9077 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 12:07:52 -0800 Subject: [PATCH 12/25] TST/WIP: first pass at testing RULES_EXPAND --- .../children/ioc-tst-unittest1.cfg | 4 ++ .../children/ioc-tst-unittest2.cfg | 4 ++ .../children/ioc-tst-unittest3.cfg | 4 ++ .../iocBoot/templates/Makefile | 5 ++ .../iocBoot/templates/edm-ioc.cmd | 3 + .../iocBoot/templates/ioc.sub-arch | 3 + .../iocBoot/templates/ioc.sub-req | 3 + .../iocBoot/templates/launchgui-ioc.cmd | 3 + .../iocBoot/templates/pydm-ioc.cmd | 3 + .../iocBoot/templates/some_script.sh | 3 + .../ioc-tst-unittest/iocBoot/templates/st.cmd | 3 + .../iocBoot/templates/syncts-ioc.cmd | 3 + tests/test_rules_expand.py | 57 +++++++++++++++++++ 13 files changed, 98 insertions(+) create mode 100644 tests/ioc-tst-unittest/children/ioc-tst-unittest1.cfg create mode 100644 tests/ioc-tst-unittest/children/ioc-tst-unittest2.cfg create mode 100644 tests/ioc-tst-unittest/children/ioc-tst-unittest3.cfg create mode 100644 tests/ioc-tst-unittest/iocBoot/templates/Makefile create mode 100644 tests/ioc-tst-unittest/iocBoot/templates/edm-ioc.cmd create mode 100644 tests/ioc-tst-unittest/iocBoot/templates/ioc.sub-arch create mode 100644 tests/ioc-tst-unittest/iocBoot/templates/ioc.sub-req create mode 100644 tests/ioc-tst-unittest/iocBoot/templates/launchgui-ioc.cmd create mode 100644 tests/ioc-tst-unittest/iocBoot/templates/pydm-ioc.cmd create mode 100644 tests/ioc-tst-unittest/iocBoot/templates/some_script.sh create mode 100644 tests/ioc-tst-unittest/iocBoot/templates/st.cmd create mode 100644 tests/ioc-tst-unittest/iocBoot/templates/syncts-ioc.cmd create mode 100644 tests/test_rules_expand.py diff --git a/tests/ioc-tst-unittest/children/ioc-tst-unittest1.cfg b/tests/ioc-tst-unittest/children/ioc-tst-unittest1.cfg new file mode 100644 index 00000000..c11558ce --- /dev/null +++ b/tests/ioc-tst-unittest/children/ioc-tst-unittest1.cfg @@ -0,0 +1,4 @@ +RELEASE = $$UP(PATH) +ENGINEER = "Unit test" +LOCATION = "pytest" +PREFIX = "IOC:TST:UNITTEST1" diff --git a/tests/ioc-tst-unittest/children/ioc-tst-unittest2.cfg b/tests/ioc-tst-unittest/children/ioc-tst-unittest2.cfg new file mode 100644 index 00000000..c09792c3 --- /dev/null +++ b/tests/ioc-tst-unittest/children/ioc-tst-unittest2.cfg @@ -0,0 +1,4 @@ +RELEASE = $$UP(PATH) +ENGINEER = "Unit test" +LOCATION = "pytest" +PREFIX = "IOC:TST:UNITTEST2" diff --git a/tests/ioc-tst-unittest/children/ioc-tst-unittest3.cfg b/tests/ioc-tst-unittest/children/ioc-tst-unittest3.cfg new file mode 100644 index 00000000..892f20ad --- /dev/null +++ b/tests/ioc-tst-unittest/children/ioc-tst-unittest3.cfg @@ -0,0 +1,4 @@ +RELEASE = $$UP(PATH) +ENGINEER = "Unit test" +LOCATION = "pytest" +PREFIX = "IOC:TST:UNITTEST3" diff --git a/tests/ioc-tst-unittest/iocBoot/templates/Makefile b/tests/ioc-tst-unittest/iocBoot/templates/Makefile new file mode 100644 index 00000000..f2b20e28 --- /dev/null +++ b/tests/ioc-tst-unittest/iocBoot/templates/Makefile @@ -0,0 +1,5 @@ +all: + touch we_ran_make.txt + +clean: + rm we_ran_make.txt diff --git a/tests/ioc-tst-unittest/iocBoot/templates/edm-ioc.cmd b/tests/ioc-tst-unittest/iocBoot/templates/edm-ioc.cmd new file mode 100644 index 00000000..f7d292d4 --- /dev/null +++ b/tests/ioc-tst-unittest/iocBoot/templates/edm-ioc.cmd @@ -0,0 +1,3 @@ +$$ENGINEER +$$LOCATION +$$PREFIX diff --git a/tests/ioc-tst-unittest/iocBoot/templates/ioc.sub-arch b/tests/ioc-tst-unittest/iocBoot/templates/ioc.sub-arch new file mode 100644 index 00000000..f7d292d4 --- /dev/null +++ b/tests/ioc-tst-unittest/iocBoot/templates/ioc.sub-arch @@ -0,0 +1,3 @@ +$$ENGINEER +$$LOCATION +$$PREFIX diff --git a/tests/ioc-tst-unittest/iocBoot/templates/ioc.sub-req b/tests/ioc-tst-unittest/iocBoot/templates/ioc.sub-req new file mode 100644 index 00000000..f7d292d4 --- /dev/null +++ b/tests/ioc-tst-unittest/iocBoot/templates/ioc.sub-req @@ -0,0 +1,3 @@ +$$ENGINEER +$$LOCATION +$$PREFIX diff --git a/tests/ioc-tst-unittest/iocBoot/templates/launchgui-ioc.cmd b/tests/ioc-tst-unittest/iocBoot/templates/launchgui-ioc.cmd new file mode 100644 index 00000000..f7d292d4 --- /dev/null +++ b/tests/ioc-tst-unittest/iocBoot/templates/launchgui-ioc.cmd @@ -0,0 +1,3 @@ +$$ENGINEER +$$LOCATION +$$PREFIX diff --git a/tests/ioc-tst-unittest/iocBoot/templates/pydm-ioc.cmd b/tests/ioc-tst-unittest/iocBoot/templates/pydm-ioc.cmd new file mode 100644 index 00000000..f7d292d4 --- /dev/null +++ b/tests/ioc-tst-unittest/iocBoot/templates/pydm-ioc.cmd @@ -0,0 +1,3 @@ +$$ENGINEER +$$LOCATION +$$PREFIX diff --git a/tests/ioc-tst-unittest/iocBoot/templates/some_script.sh b/tests/ioc-tst-unittest/iocBoot/templates/some_script.sh new file mode 100644 index 00000000..f7d292d4 --- /dev/null +++ b/tests/ioc-tst-unittest/iocBoot/templates/some_script.sh @@ -0,0 +1,3 @@ +$$ENGINEER +$$LOCATION +$$PREFIX diff --git a/tests/ioc-tst-unittest/iocBoot/templates/st.cmd b/tests/ioc-tst-unittest/iocBoot/templates/st.cmd new file mode 100644 index 00000000..f7d292d4 --- /dev/null +++ b/tests/ioc-tst-unittest/iocBoot/templates/st.cmd @@ -0,0 +1,3 @@ +$$ENGINEER +$$LOCATION +$$PREFIX diff --git a/tests/ioc-tst-unittest/iocBoot/templates/syncts-ioc.cmd b/tests/ioc-tst-unittest/iocBoot/templates/syncts-ioc.cmd new file mode 100644 index 00000000..f7d292d4 --- /dev/null +++ b/tests/ioc-tst-unittest/iocBoot/templates/syncts-ioc.cmd @@ -0,0 +1,3 @@ +$$ENGINEER +$$LOCATION +$$PREFIX diff --git a/tests/test_rules_expand.py b/tests/test_rules_expand.py new file mode 100644 index 00000000..1f8b8ebe --- /dev/null +++ b/tests/test_rules_expand.py @@ -0,0 +1,57 @@ +import pathlib +import shutil +import subprocess + +import pytest + +from .conftest import pushd + + +@pytest.mark.parametrize( + "target,expected", + [ + ("ioc-tst-unittest1", [1]), + ("ioc-tst-unittest2", [2]), + ("ioc-tst-unittest3", [3]), + ("all", [1, 2, 3]), + ], +) +def test_rules_expand(tmp_path: pathlib.Path, target: str, expected: list[int]): + """ + Build one or more instance of ioc-tst-unittest. + + This uses whichever RULES_EXPAND is in the local clone. + The IOCs are built in the pytest temp folder. + """ + # Copy the entire source test into the temp folder + ioc_source = pathlib.Path(__file__).parent / "ioc-tst-unittest" + shutil.copytree(ioc_source, tmp_path / "ioc-tst-unittest") + # Create the children Makefile + rules_expand = pathlib.Path(__file__).parent.parent / "RULES_EXPAND" + children_dir = tmp_path / "ioc-tst-unittest" / "children" + with open(children_dir / "Makefile", "w") as fd: + fd.write("IOC_CFG += $(wildcard *.cfg)\n") + fd.write(f"include {rules_expand}\n") + with pushd(children_dir): + subprocess.run(["make", target], check=True) + for num in expected: + ioc_name = f"ioc-tst-unittest{num}" + ioc_bld_path = children_dir / "build" / "iocBoot" / ioc_name + for filename in [ + f"edm-{ioc_name}.cmd", + f"{ioc_name}.sub-arch", + f"{ioc_name}.sub-req", + f"launchgui-{ioc_name}.cmd", + f"pydm-{ioc_name}.cmd", + "some_script.sh", + "st.cmd", + f"syncts-{ioc_name}.cmd", + ]: + bld_path = ioc_bld_path / filename + with open(bld_path, "r") as fd: + text = fd.read().splitlines() + assert text[0] == "Unit test" + assert text[1] == "pytest" + assert text[2] == f"IOC:TST:UNITTEST{num}" + assert (ioc_bld_path / "Makefile").exists() + assert (ioc_bld_path / "we_ran_make.txt").exists() From 1a1af5d944a4525dd95d19733bafe81f02ae2833 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 13:23:59 -0800 Subject: [PATCH 13/25] TST: add some dummy makefiles that are used in the RULES_EXPAND builds to make subdirs without full epics build system --- tests/ioc-tst-unittest/Makefile | 7 +++++++ tests/ioc-tst-unittest/iocBoot/Makefile | 7 +++++++ 2 files changed, 14 insertions(+) create mode 100644 tests/ioc-tst-unittest/Makefile create mode 100644 tests/ioc-tst-unittest/iocBoot/Makefile diff --git a/tests/ioc-tst-unittest/Makefile b/tests/ioc-tst-unittest/Makefile new file mode 100644 index 00000000..a8c0ca5e --- /dev/null +++ b/tests/ioc-tst-unittest/Makefile @@ -0,0 +1,7 @@ +SUBDIRS := $(wildcard */.) + +all: $(SUBDIRS) +$(SUBDIRS): + $(MAKE) -C $@ + +.PHONY: all $(SUBDIRS) diff --git a/tests/ioc-tst-unittest/iocBoot/Makefile b/tests/ioc-tst-unittest/iocBoot/Makefile new file mode 100644 index 00000000..a8c0ca5e --- /dev/null +++ b/tests/ioc-tst-unittest/iocBoot/Makefile @@ -0,0 +1,7 @@ +SUBDIRS := $(wildcard */.) + +all: $(SUBDIRS) +$(SUBDIRS): + $(MAKE) -C $@ + +.PHONY: all $(SUBDIRS) From 8a7878a27ce7f182c9d9755395274265ac108c33 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 13:28:11 -0800 Subject: [PATCH 14/25] TST: correct target to check was default/nothing --- tests/test_rules_expand.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_rules_expand.py b/tests/test_rules_expand.py index 1f8b8ebe..5b40ea98 100644 --- a/tests/test_rules_expand.py +++ b/tests/test_rules_expand.py @@ -13,7 +13,7 @@ ("ioc-tst-unittest1", [1]), ("ioc-tst-unittest2", [2]), ("ioc-tst-unittest3", [3]), - ("all", [1, 2, 3]), + ("", [1, 2, 3]), ], ) def test_rules_expand(tmp_path: pathlib.Path, target: str, expected: list[int]): @@ -33,7 +33,10 @@ def test_rules_expand(tmp_path: pathlib.Path, target: str, expected: list[int]): fd.write("IOC_CFG += $(wildcard *.cfg)\n") fd.write(f"include {rules_expand}\n") with pushd(children_dir): - subprocess.run(["make", target], check=True) + args = ["make"] + if target: + args.append(target) + subprocess.run(args, check=True) for num in expected: ioc_name = f"ioc-tst-unittest{num}" ioc_bld_path = children_dir / "build" / "iocBoot" / ioc_name From e1fb84c120c95a9b8c539d32ba9a95ea777a7db9 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 13:31:33 -0800 Subject: [PATCH 15/25] TST: these makefiles need literal tabs, apparently --- tests/ioc-tst-unittest/Makefile | 2 +- tests/ioc-tst-unittest/iocBoot/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ioc-tst-unittest/Makefile b/tests/ioc-tst-unittest/Makefile index a8c0ca5e..07a149a3 100644 --- a/tests/ioc-tst-unittest/Makefile +++ b/tests/ioc-tst-unittest/Makefile @@ -2,6 +2,6 @@ SUBDIRS := $(wildcard */.) all: $(SUBDIRS) $(SUBDIRS): - $(MAKE) -C $@ + $(MAKE) -C $@ .PHONY: all $(SUBDIRS) diff --git a/tests/ioc-tst-unittest/iocBoot/Makefile b/tests/ioc-tst-unittest/iocBoot/Makefile index a8c0ca5e..07a149a3 100644 --- a/tests/ioc-tst-unittest/iocBoot/Makefile +++ b/tests/ioc-tst-unittest/iocBoot/Makefile @@ -2,6 +2,6 @@ SUBDIRS := $(wildcard */.) all: $(SUBDIRS) $(SUBDIRS): - $(MAKE) -C $@ + $(MAKE) -C $@ .PHONY: all $(SUBDIRS) From 76225f0651f975db787adb1d2a065adb4d76578e Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 14:02:16 -0800 Subject: [PATCH 16/25] TST: check for IOC_APPL_TOP too --- tests/test_rules_expand.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_rules_expand.py b/tests/test_rules_expand.py index 5b40ea98..b002181e 100644 --- a/tests/test_rules_expand.py +++ b/tests/test_rules_expand.py @@ -53,8 +53,15 @@ def test_rules_expand(tmp_path: pathlib.Path, target: str, expected: list[int]): bld_path = ioc_bld_path / filename with open(bld_path, "r") as fd: text = fd.read().splitlines() + # Did each of the templateable files get templated? assert text[0] == "Unit test" assert text[1] == "pytest" assert text[2] == f"IOC:TST:UNITTEST{num}" + # Did IOC_APPL_TOP get created with the correct contents? + with open(ioc_bld_path / "IOC_APPL_TOP", "r") as fd: + text = fd.read().strip() + assert text == f"IOC_APPL_TOP={children_dir.parent}" + # Did the Makefile get copied over? assert (ioc_bld_path / "Makefile").exists() + # Did the inner Makefile get run? assert (ioc_bld_path / "we_ran_make.txt").exists() From 26c9ef8db170b2d983e647f6b6877c4e71334bfb Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 14:02:29 -0800 Subject: [PATCH 17/25] TST: find a minimal working dummy Makefile --- tests/ioc-tst-unittest/Makefile | 5 ++++- tests/ioc-tst-unittest/iocBoot/Makefile | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/ioc-tst-unittest/Makefile b/tests/ioc-tst-unittest/Makefile index 07a149a3..aa4b32c8 100644 --- a/tests/ioc-tst-unittest/Makefile +++ b/tests/ioc-tst-unittest/Makefile @@ -1,6 +1,9 @@ SUBDIRS := $(wildcard */.) +SUBDIRS := $(filter-out archive/. autosave/.,$(SUBDIRS)) -all: $(SUBDIRS) +all: install + +install: $(SUBDIRS) $(SUBDIRS): $(MAKE) -C $@ diff --git a/tests/ioc-tst-unittest/iocBoot/Makefile b/tests/ioc-tst-unittest/iocBoot/Makefile index 07a149a3..aa4b32c8 100644 --- a/tests/ioc-tst-unittest/iocBoot/Makefile +++ b/tests/ioc-tst-unittest/iocBoot/Makefile @@ -1,6 +1,9 @@ SUBDIRS := $(wildcard */.) +SUBDIRS := $(filter-out archive/. autosave/.,$(SUBDIRS)) -all: $(SUBDIRS) +all: install + +install: $(SUBDIRS) $(SUBDIRS): $(MAKE) -C $@ From f925916a6a6fe12911ed6e45d9ad7e523801b276 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 14:06:47 -0800 Subject: [PATCH 18/25] TST: nitpick the unit test names a bit --- tests/test_rules_expand.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_rules_expand.py b/tests/test_rules_expand.py index b002181e..d42dd838 100644 --- a/tests/test_rules_expand.py +++ b/tests/test_rules_expand.py @@ -10,13 +10,13 @@ @pytest.mark.parametrize( "target,expected", [ - ("ioc-tst-unittest1", [1]), - ("ioc-tst-unittest2", [2]), - ("ioc-tst-unittest3", [3]), - ("", [1, 2, 3]), + ("ioc-tst-unittest1", "1"), + ("ioc-tst-unittest2", "2"), + ("ioc-tst-unittest3", "3"), + ("all", "123"), ], ) -def test_rules_expand(tmp_path: pathlib.Path, target: str, expected: list[int]): +def test_rules_expand(tmp_path: pathlib.Path, target: str, expected: str): """ Build one or more instance of ioc-tst-unittest. @@ -34,7 +34,7 @@ def test_rules_expand(tmp_path: pathlib.Path, target: str, expected: list[int]): fd.write(f"include {rules_expand}\n") with pushd(children_dir): args = ["make"] - if target: + if target != "all": args.append(target) subprocess.run(args, check=True) for num in expected: From 927e333f7ec762e1f2a7008cb461a8419f45ad8a Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 14:09:54 -0800 Subject: [PATCH 19/25] DOC: missing docstrings --- tests/generate_artifacts.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/generate_artifacts.py b/tests/generate_artifacts.py index c7319a5d..7de92186 100644 --- a/tests/generate_artifacts.py +++ b/tests/generate_artifacts.py @@ -116,6 +116,16 @@ def generate_examples( def chmod_uplusw(path: str | pathlib.Path) -> subprocess.CompletedProcess: + """ + Make everything at path user-writable (recursively). + + The python standard library has only annoying ways to do this. + The cli chmod tool is much more convenient. + + This is used to make sure we can write to our own directories, since + the source files are often write-protected, and therefore are + still write-protected after we copy them. + """ return subprocess.run(["chmod", "-R", "u+w", str(path)]) @@ -141,6 +151,13 @@ def log_copy(src: pathlib.Path, dst: pathlib.Path): def iter_latest_template_iocs( ioc_deploy_path: pathlib.Path, ) -> typing.Iterator[pathlib.Path]: + """ + Yield latest versioned directories containing .cfg files. + + The path either ends in a version number, e.g. R1.0.0, + or it ends in a version number followed by children, + e.g. R1.0.0/children. + """ if not ioc_deploy_path.is_dir(): return try: From dbea19be278e0e363f20c3a583e6bebdc2d7040b Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 14:20:45 -0800 Subject: [PATCH 20/25] CI: add basic ci, very minimal to not require repackaging yet --- .github/workflows/pytest.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/pytest.yml diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 00000000..ca2297ba --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,24 @@ +name: pytest + +on: + push: + pull_request: + release: + types: + - created + +jobs: + test: + name: "Python 3.9: (pip)" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + - name: pip + run: pip install pytest + - name: pytest + run: pytest -v From cb3d0ba172a1eb53f5851a5b97795e179607d410 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 14:25:26 -0800 Subject: [PATCH 21/25] FIX: allow expand to run on non-cds systems using backup --- expand | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/expand b/expand index b370633d..33b2bae1 100755 --- a/expand +++ b/expand @@ -1,2 +1,8 @@ #!/bin/bash -/cds/group/pcds/pyps/conda/py39/envs/pcds-5.9.1/bin/python $0.py "$@" +PYTHON=/cds/group/pcds/pyps/conda/py39/envs/pcds-5.9.1/bin/python +if [ ! -x "${PYTHON}" ]; then + # Fallback for other filesystems, should be OK + # Mostly for CI + PYTHON=python3 +fi +$PYTHON $0.py "$@" From 298d32879198a5c9bd6e2b4bffff8ad828e07bbc Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 14:31:25 -0800 Subject: [PATCH 22/25] DOC: add a note about why I've done CI in a nonstandard way --- .github/workflows/pytest.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index ca2297ba..456caf96 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,3 +1,7 @@ +# Custom simplest pytest action +# Normally we should use pcds-ci-helpers, but that expects an installable package. +# In the future this package could be refactored to be installable, +# And then we'd switch this out for the standard ci suite name: pytest on: From 6f4339ae70eae72649a7e0f0bdf487619a558339 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 15:10:12 -0800 Subject: [PATCH 23/25] DOC: convert NOTES to README.md and include some of my learning --- NOTES => README.md | 51 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) rename NOTES => README.md (76%) diff --git a/NOTES b/README.md similarity index 76% rename from NOTES rename to README.md index 12e6df3e..eea72786 100644 --- a/NOTES +++ b/README.md @@ -1,3 +1,54 @@ +## ioc-template-macros +`ioc-template-macros` is a repo that provides the following tools for building templated IOCs: +- `RULES_EXPAND`: a file to include in a templated IOC `Makefile` such that when we `make` that IOC it uses this repo to expand the templates. A typical templated IOC `Makefile` is something like: +``` +# SLAC PCDS Makefile for building templated IOC instances +IOC_CFG += $(wildcard *.cfg) +include /reg/g/pcds/controls/macro/RULES_EXPAND +``` +- `expand`: a shell script that sets up a Python environment to run `expand.py` +- `expand.py`: a shell script that reads from config files and uses the data within them to expand IOC templates. + +## API + +The `expand` script has two supported forms. There are others but they are not well understood by this readme writer: + +``` +expand -c CONFIG_FILE KEYWORD +expand -c CONFIG_FILE TEMPLATE_FILE OUTPUT_FILE +``` + +The first form finds the value of a keyword from the config file and sends it to stdout. + +For example, if your config file is: + +``` +RELEASE = /some/path +ENGINEER = somebody +``` + +And you run: + +``` +expand -d path_to_that_file.cfg ENGINEER +``` + +Then "somebody" would be sent to stdout (with a trailing newline). +This is used in RULES_EXPAND to get the RELEASE path. + +Note that you can use macros in your cfg file too, and they will be expanded here. +Some special environment variables such as PATH are also supported. + +The second form uses the config file and the template file to create an output file. +Values from the config file will be referred to in template and will ultimately be used +to create a fully-formed output. + +Typically, templates come from common IOCs and +config files come from hutch-specific IOCs that reference the common IOC. + + +## Template Macro Language + All of the macro commands in the template files begin with "$$". These can be: $$VAR or $$(VAR) - VAR is a variable name that is evaluated and inserted. From 598bff34ca9b637b8c3bce707be141eccabd520b Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 10 Dec 2024 15:23:08 -0800 Subject: [PATCH 24/25] DOC: md reformatting --- README.md | 121 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 68 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index eea72786..7958b961 100644 --- a/README.md +++ b/README.md @@ -30,18 +30,19 @@ ENGINEER = somebody And you run: ``` -expand -d path_to_that_file.cfg ENGINEER +expand -c path_to_that_file.cfg ENGINEER ``` Then "somebody" would be sent to stdout (with a trailing newline). -This is used in RULES_EXPAND to get the RELEASE path. +This is used in `RULES_EXPAND` to get the `RELEASE` path. Note that you can use macros in your cfg file too, and they will be expanded here. -Some special environment variables such as PATH are also supported. +Some special environment variables such as `PATH` are also supported, +which during a standard `make` stores the directory containing the config file. The second form uses the config file and the template file to create an output file. -Values from the config file will be referred to in template and will ultimately be used -to create a fully-formed output. +Values from the config file will be referred to by the template +and will ultimately be used to create a fully-formed output. Typically, templates come from common IOCs and config files come from hutch-specific IOCs that reference the common IOC. @@ -49,88 +50,91 @@ config files come from hutch-specific IOCs that reference the common IOC. ## Template Macro Language -All of the macro commands in the template files begin with "$$". These can be: - $$VAR or $$(VAR) - - VAR is a variable name that is evaluated and inserted. - $$DIRNAME +All of the macro commands in the template files begin with "$$". + +These can be: +- `$$VAR` or `$$(VAR)` + - `VAR` is a variable name that is evaluated and inserted. +- `$$DIRNAME` - A special variable name that is the current directory name. (Not the whole path!) - $$COUNT(INST) - - How many instantiations of INST are there. - $$LOOP(INST)loop-body$$ENDLOOP(INST) - - INST is the name of an instantiation from the config file. The +- `$$COUNT(INST)` + - How many instantiations of `INST` are there. +- `$$LOOP(INST)`loop-body`$$ENDLOOP(INST)` + - `INST` is the name of an instantiation from the config file. The macro processor loops over all of the named instantiation, and inserts a copy of the loop-body with the variable replacements using the parameters of the particular instance. - $$LOOP(N)loop-body$$ENDLOOP(N) - - N is an integer. This will expand the loop-body N times, with $$INDEX - ranging from 0 to N-1. - $$TRANSLATE(VAR, "STR1", "STR2") - - VAR is a variable name that is evaluated and then has all of the - characters in STR1 replaced by the corresponding characters in STR2. +- `$$LOOP(N)`loop-body`$$ENDLOOP(N)` + - `N` is an integer. This will expand the loop-body `N` times, with `$$INDEX` + ranging from `0` to `N-1`. +- `$$TRANSLATE(VAR, "STR1", "STR2")` + - `VAR` is a variable name that is evaluated and then has all of the + characters in `STR1` replaced by the corresponding characters in `STR2`. The double-quotes are manditory! - $$IF(VAR) - if-body - $$ELSE(VAR) - else-body - $$ENDIF(VAR) - - A simple conditional. The $$ELSE and else-body are optional. If the - value of VAR is not "", expand the if-body, otherwise expand the else-body. - $$IF(VAR,if-body,else-body) - - An abbreviated form of the $$IF, useful for expressions and other cases +- `$$IF(VAR)`if-body`$$ELSE(VAR)`else-body`$$ENDIF(VAR)` + - A simple conditional. The `$$ELSE` and else-body are optional. If the + value of `VAR` is not "", expand the if-body, otherwise expand the else-body. +- `$$IF(VAR,if-body,else-body)` + - An abbreviated form of the `$$IF`, useful for expressions and other cases where neither body includes a comma. - $$IF(VAR,VAL) - $$ELSE(VAR) - $$ENDIF(VAR) - - Test if VAR is equal to the given value. - $$INCLUDE(FILENAME) - - Process the contents of the FILENAME. - $$CALC{EXPRESSION} or $$CALC{EXPRESSION,FORMAT} - - NOTE THE BRACKETS!!! This allows arbitrary arithmetic. The EXPRESSION +- `$$IF(VAR,VAL)`if-body`$$ELSE(VAR)`else-body`$$ENDIF(VAR)` + - Test if `VAR` is equal to the given value. +- `$$INCLUDE(FILENAME)` + - Process the contents of the `FILENAME`. +- `$$CALC{EXPRESSION}` or `$$CALC{EXPRESSION,FORMAT}` + - NOTE THE BRACKETS!!! This allows arbitrary arithmetic. The `EXPRESSION` is expanded, and then evaluated as a mathematical expression. Any undefined atom is assumed to be zero. Atoms within the expression do not need to be prefixed by $$, but maybe. (They should be if within a $$LOOP inside the expression.) The result is output as a decimal number, or using the given format if one was given. +## Config Files + Configuration files are rather fussy, in that whitespace can only appear within value strings and comments. If a line starts with '#', it is a comment that runs until the end of the line. The configuration file has two types of statements. Variable assignments have the forms: +``` VAR=VALUE VAR="VALUE" VAR='VALUE' - -All of these define $$VAR to have the value VALUE. +``` +All of these define `$$VAR` to have the value `VALUE`. Instantiations have the form: +``` [ INAME: ] INST(PARAMLIST) -where INAME is the instance name, and the PARAMLIST is a comma-separated +``` +where `INAME` is the instance name, and the `PARAMLIST` is a comma-separated list of entries of the forms: +``` VAR=VALUE VAR="VALUE" VAR='VALUE' OTHERINSTn OTHERINAME - +``` Instantiations are numbered, with the instantiation number assigned to a -special variable $$INDEX. The first three types of parameters all create -global symbols of the form $$INSTVARn with value VALUE. The last two forms +special variable `$$INDEX`. The first three types of parameters all create +global symbols of the form `$$INSTVARn` with value `VALUE`. The last two forms indicate that this instantiation uses a particular instantiation of some other instantiation. For example, if we have: +``` E(X=Y) TEST:E(X=Z) A(B=C,D=F,E0) A(B=Q,D=R,TEST) - -Then we have defined global symbols $$EX0 = Y, $$EX1 = Z, $$AB0 = C, $$AD0 = F, -$$AB1 = Q and $$AD1 = R. If we $$LOOP(A), then within the body of the loop, -we will have $$B = C, $$D = F, and $$EX = Y when $$INDEX is 0, and $$B = Q, -$$D = R and $$EX = Z when $$INDEX is 1. +``` +Then we have defined global symbols `$$EX0 = Y`, `$$EX1 = Z`, `$$AB0 = C`, `$$AD0 = F`, +`$$AB1 = Q` and `$$AD1 = R`. If we `$$LOOP(A)`, then within the body of the loop, +we will have `$$B = C`, `$$D = F`, and `$$EX = Y` when `$$INDEX` is 0, and `$$B = Q`, +`$$D = R` and `$$EX = Z` when `$$INDEX` is 1. Limited expansion is done while reading the config file as well. The file is expanded once with no variable definitions, and then processed to get a set @@ -139,29 +143,37 @@ expansion is parsed to get the actual definions and instantiations used to process the input file. NOTE: THE DEFINITIONS USED ARE THE *FINAL* DEFINITIONS IN THE CONFIG FILE!!! ------------------------------------------------------------------------------- +## New Style Configs + New style: config files start with a set of definitions: +``` VAR=VALUE VAR="VALUE" VAR='VALUE' VAR VALUE VAR "VALUE" VAR 'VALUE' -In the last three, any amount of whitespace can occur after VAR but before VALUE, -but whitespace in VALUE is preserved. +``` +In the last three, any amount of whitespace can occur after `VAR` but before `VALUE`, +but whitespace in `VALUE` is preserved. Then, we have instances: either as before or begun with +``` INSTANCE xxx [ yyy ] -where xxx is the type of the instance and yyy is an optional name for this +``` +where `xxx` is the type of the instance and `yyy` is an optional name for this instance. In this case, the following lines consist of definitions as above, with the exception that lines of the form: +``` VAR VALUE -cannot have any whitespace in the VALUE. Multiple assignments can be on a +``` +cannot have any whitespace in the `VALUE`. Multiple assignments can be on a single line. -The instance ends with either EOF or a new INSTANCE. +The instance ends with either EOF or a new `INSTANCE`. To clarify this, a config file such as: +``` RELEASE=/reg/g/pcds/package/epics/3.14/ioc/common/ipimb/R2.0.17 ARCH=linux-x86 ENGINEER=Michael Browne (mcbrowne) @@ -174,8 +186,10 @@ To clarify this, a config file such as: IPIMB(NAME=MEC:XT2:PIM:02,PORT=/dev/ttyPS2,BLDID=42,EVR0,TRIG=0) IPIMB(NAME=MEC:XT2:IPM:03,PORT=/dev/ttyPS1,BLDID=24,EVR0,TRIG=0) IPIMB(NAME=MEC:XT2:PIM:03,PORT=/dev/ttyPS0,BLDID=43,EVR0,TRIG=0) +``` could be written: +``` RELEASE /reg/g/pcds/package/epics/3.14/ioc/common/ipimb/R2.0.17 ARCH linux-x86 ENGINEER Michael Browne (mcbrowne) @@ -209,3 +223,4 @@ could be written: PORT /dev/ttyPS0 BLDID 43 EVR0 TRIG=0 +``` From 13dbb5a24d6f664e43edbc0f4cac1bd3e21b5969 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Wed, 11 Dec 2024 10:04:14 -0800 Subject: [PATCH 25/25] DOC: fix minor errors and formatting improvements in readme --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7958b961..89776a86 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,22 @@ ## ioc-template-macros `ioc-template-macros` is a repo that provides the following tools for building templated IOCs: -- `RULES_EXPAND`: a file to include in a templated IOC `Makefile` such that when we `make` that IOC it uses this repo to expand the templates. A typical templated IOC `Makefile` is something like: + +- `expand`: a shell script that sets up a Python environment to run `expand.py` +- `expand.py`: a python script that reads from config files and uses the data within them to expand IOC templates. +- `RULES_EXPAND`: a file to include in a templated IOC `Makefile` such that when we `make` that IOC it uses this repo to expand the templates. + +A typical templated IOC `Makefile` is something like: ``` # SLAC PCDS Makefile for building templated IOC instances IOC_CFG += $(wildcard *.cfg) include /reg/g/pcds/controls/macro/RULES_EXPAND ``` -- `expand`: a shell script that sets up a Python environment to run `expand.py` -- `expand.py`: a shell script that reads from config files and uses the data within them to expand IOC templates. ## API -The `expand` script has two supported forms. There are others but they are not well understood by this readme writer: +The `expand` script has two supported forms. There are others but they are not used in RULES_EXPAND. +The two forms look like: ``` expand -c CONFIG_FILE KEYWORD expand -c CONFIG_FILE TEMPLATE_FILE OUTPUT_FILE