From 9922170f69736d3beac66474a2031f8cbf4885c6 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 29 Oct 2024 17:50:57 -0400 Subject: [PATCH 01/39] #221 Add `force_checkout` and `force_compile` click options --- fre/make/createCheckout.py | 3 +- fre/make/createCompile.py | 2 +- fre/make/fremake.py | 58 +++++++++++++++++++++++++------------- fre/make/runFremake.py | 2 +- 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/fre/make/createCheckout.py b/fre/make/createCheckout.py index 595c3054..efc1f1aa 100644 --- a/fre/make/createCheckout.py +++ b/fre/make/createCheckout.py @@ -9,7 +9,7 @@ import fre.yamltools.combine_yamls as cy @click.command() -def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): +def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose,force_checkout): # Define variables yml = yamlfile name = yamlfile.split(".")[0] @@ -83,6 +83,7 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v else: sys.exit() else: + print("\nCheckout script PREVIOUSLY created in "+ srcDir + "/checkout.sh \n") if run == True: os.chmod(srcDir+"/checkout.sh", 0o744) diff --git a/fre/make/createCompile.py b/fre/make/createCompile.py index 90fc8127..284db2af 100644 --- a/fre/make/createCompile.py +++ b/fre/make/createCompile.py @@ -10,7 +10,7 @@ import fre.yamltools.combine_yamls as cy @click.command() -def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): +def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_compile): # Define variables yml = yamlfile name = yamlfile.split(".")[0] diff --git a/fre/make/fremake.py b/fre/make/fremake.py index 54946349..ef668c7e 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -5,29 +5,34 @@ from .createMakefile import makefile_create from .runFremake import fremake_run -yamlfile_opt_help = """Experiment yaml compile FILE -""" +yamlfile_opt_help = """Experiment yaml compile FILE""" experiment_opt_help = """Name of experiment""" platform_opt_help = """Hardware and software FRE platform space separated list of STRING(s). -This sets platform-specific data and instructions -""" +This sets platform-specific data and instructions""" target_opt_help = """a space separated list of STRING(s) that defines compilation settings and linkage directives for experiments. Predefined targets refer to groups of directives that exist in the mkmf template file (referenced in buildDocker.py). Possible predefined targets include 'prod', 'openmp', 'repro', 'debug, 'hdf5'; however 'prod', 'repro', and 'debug' are mutually exclusive -(cannot not use more than one of these in the target list). Any number of targets can be used. -""" -parallel_opt_help = """Number of concurrent model compiles (default 1) -""" +(cannot not use more than one of these in the target list). Any number of targets can be used.""" +parallel_opt_help = """Number of concurrent model compiles (default 1)""" jobs_opt_help = """Number of jobs to run simultaneously. Used for make -jJOBS and git clone -recursive --jobs=JOBS -""" +recursive --jobs=JOBS""" no_parallel_checkout_opt_help = """Use this option if you do not want a parallel checkout. -The default is to have parallel checkouts. -""" -verbose_opt_help = """Get verbose messages (repeat the option to increase verbosity level) -""" - +The default is to have parallel checkouts.""" +verbose_opt_help = """Get verbose messages (repeat the option to increase verbosity level)""" +force_checkout_opt_help = """Force checkout in case the source directory exists +Get a fresh checkout to the source directory. +An existing source directory is normally reused if possible. +However it might be an issue if current checkout instructions do not follow +changes in the experiment suite configuration file. +The option --force-checkout allows to get a fresh checkout according +to the current configuration file.""" +force_compile_opt_help = """Force compile in case the executable directory exists +Compile a fresh executable. +An existing executable directory is normally reused if possible. +It's an error if current compile instructions don't match the experiment suite configuration +file UNLESS the option --force-compile is used.This option allows to recreate the compile +script according to the current configuration file.""" @click.group(help=click.style(" - access fre make subcommands", fg=(210,73,57))) @@ -70,8 +75,16 @@ def make_cli(): "--verbose", is_flag = True, help = verbose_opt_help) +@click.option("-f", + "--force-checkout", + is_flag = True, + help = force_checkout_opt_help) +@click.option("-F", + "--force-compile", + is_flag=True, + help = force_compile_opt_help) @click.pass_context -def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, verbose): +def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, verbose, force_checkout, force_compile): """ - Perform all fremake functions to run checkout and compile model""" context.forward(fremake_run) @@ -111,8 +124,12 @@ def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel "--verbose", is_flag = True, help = verbose_opt_help) +@click.option("-f", + "--force-checkout", + is_flag = True, + help = force_checkout_opt_help) @click.pass_context -def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): +def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose,force_checkout): """ - Write the checkout script """ context.forward(checkout_create) @@ -175,8 +192,12 @@ def create_makefile(context,yamlfile,platform,target): "--verbose", is_flag = True, help = verbose_opt_help) +@click.option("-F", + "--force-compile", + is_flag=True, + help = force_compile_opt_help) @click.pass_context -def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbose): +def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbose,force_compile): """ - Write the compile script """ context.forward(compile_create) @@ -204,6 +225,5 @@ def create_dockerfile(context,yamlfile,platform,target,execute): """ - Write the dockerfile """ context.forward(dockerfile_create) - if __name__ == "__main__": make_cli() diff --git a/fre/make/runFremake.py b/fre/make/runFremake.py index 934a07fd..b911dbc0 100644 --- a/fre/make/runFremake.py +++ b/fre/make/runFremake.py @@ -18,7 +18,7 @@ makefilefre, buildDocker, buildBaremetal ) @click.command() -def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose): +def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose,force_checkout,foce_compile): ''' run fremake via click''' yml = yamlfile name = yamlfile.split(".")[0] From 6157b7b5fcbceb329de12ca5f0743a81715cb9aa Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 29 Oct 2024 18:16:19 -0400 Subject: [PATCH 02/39] #221 Add force-checkout functionality in createCheckout tool --- fre/make/createCheckout.py | 34 ++++++++++++++++++++++++++------ fre/make/gfdlfremake/checkout.py | 3 --- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/fre/make/createCheckout.py b/fre/make/createCheckout.py index efc1f1aa..c72e91a4 100644 --- a/fre/make/createCheckout.py +++ b/fre/make/createCheckout.py @@ -4,6 +4,7 @@ import subprocess import logging import sys +import shutil import click from .gfdlfremake import varsfre, yamlfre, checkout, targetfre import fre.yamltools.combine_yamls as cy @@ -83,8 +84,20 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v else: sys.exit() else: + if force_checkout: + print("Re-creating the checkout script...\n") + # Remove previous checkout + shutil.rmtree(srcDir) + # Create checkout script + freCheckout = checkout.checkout("checkout.sh",srcDir) + freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc) + freCheckout.finish(pc) + # Make checkout script executable + os.chmod(srcDir+"/"+checkoutScriptName, 0o744) + print(" Checkout script created in "+ srcDir + "/checkout.sh \n") + else: + print("\nCheckout script PREVIOUSLY created in "+ srcDir + "/checkout.sh \n") - print("\nCheckout script PREVIOUSLY created in "+ srcDir + "/checkout.sh \n") if run == True: os.chmod(srcDir+"/checkout.sh", 0o744) try: @@ -100,11 +113,20 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v image="ecpe4s/noaa-intel-prototype:2023.09.25" bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/exec" tmpDir = "tmp/"+platformName - freCheckout = checkout.checkoutForContainer("checkout.sh", srcDir, tmpDir) - freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc) - freCheckout.finish(pc) - click.echo("\nCheckout script created at " + tmpDir + "/checkout.sh" + "\n") - + if not os.path.exists(tmpDir+"/checkout.sh"): + freCheckout = checkout.checkoutForContainer("checkout.sh", srcDir, tmpDir) + freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc) + freCheckout.finish(pc) + click.echo("\nCheckout script created at " + tmpDir + "/checkout.sh" + "\n") + else: + if force_checkout: + print("Re-creating checkout script...") + freCheckout = checkout.checkoutForContainer("checkout.sh", srcDir, tmpDir) + freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc) + freCheckout.finish(pc) + print(" Checkout script created in "+ tmpDir + "/checkout.sh" + "\n") + else: + print("\nCheckout script PREVIOUSLY created in "+ tmpDir + "/checkout.sh" + "\n") if __name__ == "__main__": checkout_create() diff --git a/fre/make/gfdlfremake/checkout.py b/fre/make/gfdlfremake/checkout.py index 4e4c905c..9afd0ef9 100644 --- a/fre/make/gfdlfremake/checkout.py +++ b/fre/make/gfdlfremake/checkout.py @@ -108,9 +108,6 @@ def finish (self,pc): else: self.checkoutScript.close() - # Make checkout script executable - os.chmod(self.src+"/"+self.fname, 0o744) - ## TODO: batch script building def run (self): """ From ab6bc128164878c6ad0edb63c2aa53ab379f344b Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 29 Oct 2024 18:27:42 -0400 Subject: [PATCH 03/39] #221 Add `force_compile` functionalty in createCompile tool --- fre/make/createCompile.py | 54 ++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/fre/make/createCompile.py b/fre/make/createCompile.py index 284db2af..bae1ca42 100644 --- a/fre/make/createCompile.py +++ b/fre/make/createCompile.py @@ -60,34 +60,52 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_ raise ValueError (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml")) (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName) - ## Make the bldDir based on the modelRoot, the platform, and the target + ## Make the bldDir based on the modelRoot, the platform, and the target srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src" ## Check for type of build if iscontainer == False: baremetalRun = True bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/" + platformName + "-" + target.gettargetName() + "/exec" os.system("mkdir -p " + bldDir) - ## Create a list of compile scripts to run in parallel - fremakeBuild = buildBaremetal.buildBaremetal(exp = fremakeYaml["experiment"], - mkTemplatePath = mkTemplate, - srcDir = srcDir, - bldDir = bldDir, - target = target, - modules = modules, - modulesInit = modulesInit, - jobs = jobs) - for c in fremakeYaml['src']: - fremakeBuild.writeBuildComponents(c) - fremakeBuild.writeScript() - fremakeBuildList.append(fremakeBuild) - click.echo("\nCompile script created at " + bldDir + "/compile.sh" + "\n") + if not os.path.exists(bldDir+"/compile.sh"): + ## Create a list of compile scripts to run in parallel + fremakeBuild = buildBaremetal.buildBaremetal(exp = fremakeYaml["experiment"], + mkTemplatePath = mkTemplate, + srcDir = srcDir, + bldDir = bldDir, + target = target, + modules = modules, + modulesInit = modulesInit, + jobs = jobs) + for c in fremakeYaml['src']: + fremakeBuild.writeBuildComponents(c) + fremakeBuild.writeScript() + fremakeBuildList.append(fremakeBuild) + print("\nCompile script created in " + bldDir + "/compile.sh" + "\n") + else: + if force_compile: + print("Re-creating the compile script...") + # Re-create compile script + fremakeBuild = buildBaremetal.buildBaremetal(exp = fremakeYaml["experiment"], + mkTemplatePath = mkTemplate, + srcDir = srcDir, + bldDir = bldDir, + target = target, + modules = modules, + modulesInit = modulesInit, + jobs = jobs) + for c in fremakeYaml['src']: + fremakeBuild.writeBuildComponents(c) + fremakeBuild.writeScript() + fremakeBuildList.append(fremakeBuild) + print(" Compile script created in " + bldDir + "/compile.sh" + "\n") + else: + print("\nCompile script PREVIOUSLY created in " + bldDir + "/compile.sh" + "\n") + if run: - #print("ITS GONNA RUN") if baremetalRun: pool = Pool(processes=nparallel) # Create a multiprocessing Pool pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) # process data_inputs iterable with pool -# else: -# fremakeBuild.run() else: sys.exit() From 6a13ab59fac28e68c6821f8a9cc811bf8c8b8d07 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 31 Oct 2024 15:39:56 -0400 Subject: [PATCH 04/39] #221 Add `force_checkout` and `force_compile` arguments --- fre/make/createCheckout.py | 4 ++-- fre/make/createCompile.py | 4 ++-- fre/make/runFremake.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fre/make/createCheckout.py b/fre/make/createCheckout.py index aa284689..73f98769 100644 --- a/fre/make/createCheckout.py +++ b/fre/make/createCheckout.py @@ -129,12 +129,12 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v print("\nCheckout script PREVIOUSLY created in "+ tmpDir + "/checkout.sh" + "\n") @click.command() -def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): +def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose,force_checkout): ''' Decorator for calling checkout_create - allows the decorated version of the function to be separate from the undecorated version ''' - return checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose) + return checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose,force_checkout) if __name__ == "__main__": checkout_create() diff --git a/fre/make/createCompile.py b/fre/make/createCompile.py index d67f6914..cb93b5a0 100644 --- a/fre/make/createCompile.py +++ b/fre/make/createCompile.py @@ -9,7 +9,7 @@ from .gfdlfremake import varsfre, yamlfre, targetfre, buildBaremetal import fre.yamltools.combine_yamls as cy -def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): +def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_compile): # Define variables yml = yamlfile name = yamlfile.split(".")[0] @@ -109,7 +109,7 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): sys.exit() @click.command() -def _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): +def _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_compile): ''' Decorator for calling compile_create - allows the decorated version of the function to be separate from the undecorated version diff --git a/fre/make/runFremake.py b/fre/make/runFremake.py index 3c1fa7d4..f96348c0 100644 --- a/fre/make/runFremake.py +++ b/fre/make/runFremake.py @@ -200,12 +200,12 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) @click.command() -def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose): +def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose,force_checkout,force_compile): ''' Decorator for calling fremake_run - allows the decorated version of the function to be separate from the undecorated version ''' - return fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose) + return fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose,force_checkout,force_compile) if __name__ == "__main__": fremake_run() From a8b5bd5969b5565238dcd30e02848a317391f565 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 27 Nov 2024 11:15:59 -0500 Subject: [PATCH 05/39] #221 Update use of click options --- fre/make/createCheckout.py | 8 +++++--- fre/make/createCompile.py | 7 ++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/fre/make/createCheckout.py b/fre/make/createCheckout.py index 73f98769..6bda101a 100644 --- a/fre/make/createCheckout.py +++ b/fre/make/createCheckout.py @@ -13,7 +13,6 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v # Define variables yml = yamlfile name = yamlfile.split(".")[0] - run = execute jobs = str(jobs) pcheck = no_parallel_checkout @@ -80,7 +79,7 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v print("\nCheckout script created in "+ srcDir + "/checkout.sh \n") # Run the checkout script - if run is True: + if execute: freCheckout.run() else: sys.exit() @@ -99,7 +98,7 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v else: print("\nCheckout script PREVIOUSLY created in "+ srcDir + "/checkout.sh \n") - if run == True: + if execute: try: subprocess.run(args=[srcDir+"/checkout.sh"], check=True) except: @@ -120,6 +119,9 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v click.echo("\nCheckout script created at " + tmpDir + "/checkout.sh" + "\n") else: if force_checkout: + # Remove the checkout script + os.remove(tmpDir+"/checkout.sh") + # Create the checkout script print("Re-creating checkout script...") freCheckout = checkout.checkoutForContainer("checkout.sh", srcDir, tmpDir) freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc) diff --git a/fre/make/createCompile.py b/fre/make/createCompile.py index cb93b5a0..53e1b2e7 100644 --- a/fre/make/createCompile.py +++ b/fre/make/createCompile.py @@ -15,7 +15,6 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_ name = yamlfile.split(".")[0] nparallel = parallel jobs = str(jobs) - run = execute if verbose: logging.basicCOnfig(level=logging.INFO) @@ -83,8 +82,10 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_ print("\nCompile script created in " + bldDir + "/compile.sh" + "\n") else: if force_compile: - print("Re-creating the compile script...") + # Remove compile script + os.remove(bldDir + "/compile.sh") # Re-create compile script + print("Re-creating the compile script...") fremakeBuild = buildBaremetal.buildBaremetal(exp = fremakeYaml["experiment"], mkTemplatePath = mkTemplate, srcDir = srcDir, @@ -101,7 +102,7 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_ else: print("\nCompile script PREVIOUSLY created in " + bldDir + "/compile.sh" + "\n") - if run: + if execute: if baremetalRun: pool = Pool(processes=nparallel) # Create a multiprocessing Pool pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) # process data_inputs iterable with pool From 5263a9fe325baf6a51947ad9d1e929f2a8aebeb8 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 4 Dec 2024 11:42:15 -0500 Subject: [PATCH 06/39] #221 Update click option descriptions and click option functinality --- fre/make/fremake.py | 71 +++++++++----- fre/make/runFremake.py | 93 ++++++++++++++----- .../compilation/test_fre_make_run_fremake.py | 2 +- 3 files changed, 121 insertions(+), 45 deletions(-) diff --git a/fre/make/fremake.py b/fre/make/fremake.py index cebb8621..bcf4e9bf 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -10,30 +10,19 @@ platform_opt_help = """Hardware and software FRE platform space separated list of STRING(s). This sets platform-specific data and instructions""" target_opt_help = """a space separated list of STRING(s) that defines compilation settings and -linkage directives for experiments. Predefined targets refer to groups of directives that exist in -the mkmf template file (referenced in buildDocker.py). Possible predefined targets include 'prod', -'openmp', 'repro', 'debug, 'hdf5'; however 'prod', 'repro', and 'debug' are mutually exclusive -(cannot not use more than one of these in the target list). Any number of targets can be used.""" +linkage directives for experiments.""" +#Predefined targets refer to groups of directives that exist in +#the mkmf template file (referenced in buildDocker.py). Possible predefined targets include 'prod', +#'openmp', 'repro', 'debug, 'hdf5'; however 'prod', 'repro', and 'debug' are mutually exclusive +#(cannot not use more than one of these in the target list). Any number of targets can be used.""" parallel_opt_help = """Number of concurrent model compiles (default 1)""" jobs_opt_help = """Number of jobs to run simultaneously. Used for make -jJOBS and git clone recursive --jobs=JOBS""" no_parallel_checkout_opt_help = """Use this option if you do not want a parallel checkout. The default is to have parallel checkouts.""" verbose_opt_help = """Get verbose messages (repeat the option to increase verbosity level)""" -force_checkout_opt_help = """Force checkout in case the source directory exists -Get a fresh checkout to the source directory. -An existing source directory is normally reused if possible. -However it might be an issue if current checkout instructions do not follow -changes in the experiment suite configuration file. -The option --force-checkout allows to get a fresh checkout according -to the current configuration file.""" -force_compile_opt_help = """Force compile in case the executable directory exists -Compile a fresh executable. -An existing executable directory is normally reused if possible. -It's an error if current compile instructions don't match the experiment suite configuration -file UNLESS the option --force-compile is used.This option allows to recreate the compile -script according to the current configuration file.""" - +force_checkout_opt_help = """Force checkout in case the source directory exists.""" +force_compile_opt_help = """Force compile in case the executable directory exists.""" @click.group(help=click.style(" - access fre make subcommands", fg=(210,73,57))) def make_cli(): @@ -90,7 +79,24 @@ def make_cli(): help = force_compile_opt_help) @click.pass_context def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, execute, verbose, force_checkout, force_compile): - """ - Perform all fremake functions to run checkout and compile model""" + """ + - Perform all fremake functions to run checkout and compile model\n + - For --target use: Predefined targets refer to groups of directives that exist in the mkmf template file.\n + Possible predefined targets include 'prod','openmp', 'repro', 'debug, 'hdf5'; +however 'prod', 'repro', and 'debug' are mutually exclusive (cannot not use more than one +of these in the target list). Any number of targets can be used.\n + - The -npc option is REQUIRED for container builds\n + - Use --force-checkout to get a fresh checkout to the source directory.\n + An existing source directory is normally reused if possible. +However it might be an issue if current checkout instructions do not follow changes in the +experiment suite configuration file. The option --force-checkout allows to get a fresh checkout +according to the current configuration file.\n + - Use `--force-compile` to compile a fresh executable.\n + An existing executable directory is normally reused if possible. It's an error if +current compile instructions don't match the experiment suite configuration file UNLESS the +option --force-compile is used. This option allows the user to recreate the compile script +according to the current configuration file. + """ context.forward(runFremake._fremake_run) #### @@ -135,7 +141,19 @@ def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel help = force_checkout_opt_help) @click.pass_context def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose,force_checkout): - """ - Write the checkout script """ + """ + - Write the checkout script\n + - For --target use: Predefined targets refer to groups of directives that exist in the mkmf template file.\n + Possible predefined targets include 'prod', 'openmp', 'repro', 'debug, 'hdf5'; +however 'prod', 'repro', and 'debug' are mutually exclusive (cannot not use more than one + of these in the target list). Any number of targets can be used.\n + - The -npc option is REQUIRED for container builds\n + - Use --force-checkout to get a fresh checkout to the source directory.\n + An existing source directory is normally reused if possible. +However it might be an issue if current checkout instructions do not follow changes in the +experiment suite configuration file. The option --force-checkout allows to get a fresh checkout +according to the current configuration file.\n + """ context.forward(createCheckout._checkout_create) ##### @@ -203,7 +221,18 @@ def create_makefile(context,yamlfile,platform,target): help = force_compile_opt_help) @click.pass_context def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbose,force_compile): - """ - Write the compile script """ + """ + - Write the compile script\n + - For --target use: Predefined targets refer to groups of directives that exist in the mkmf template file.\n + Possible predefined targets include 'prod','openmp', 'repro', 'debug, 'hdf5'; +however 'prod', 'repro', and 'debug' are mutually exclusive (cannot not use more than one +of these in the target list). Any number of targets can be used.\n + - Use `--force-compile` to compile a fresh executable.\n + An existing executable directory is normally reused if possible. It's an error if +current compile instructions don't match the experiment suite configuration file UNLESS the +option --force-compile is used. This option allows the user to recreate the compile script +according to the current configuration file. + """ context.forward(createCompile._compile_create) @make_cli.command diff --git a/fre/make/runFremake.py b/fre/make/runFremake.py index 5e29e34a..f1c87be7 100644 --- a/fre/make/runFremake.py +++ b/fre/make/runFremake.py @@ -11,6 +11,7 @@ from pathlib import Path import click import subprocess +import shutil import fre.yamltools.combine_yamls as cy from .gfdlfremake import ( targetfre, varsfre, yamlfre, checkout, @@ -78,7 +79,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec RUNenv ) = modelYaml.platforms.getPlatformFromName(platformName) ## Create the checkout script - if not iscontainer: + if iscontainer is False: ## Create the source directory for the platform srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src" if not os.path.exists(srcDir): @@ -91,6 +92,28 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec print("\nCheckout script created at "+ srcDir + "/checkout.sh \n") ## TODO: Options for running on login cluster? freCheckout.run() + else: + if force_checkout: + print("Re-creating the checkout script...\n") + # Remove previous checkout + shutil.rmtree(srcDir) + # Create checkout script + freCheckout = checkout.checkout("checkout.sh",srcDir) + freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc) + freCheckout.finish(pc) + # Make checkout script executable + os.chmod(srcDir+"/"+checkoutScriptName, 0o744) + print(" Checkout script created in "+ srcDir + "/checkout.sh \n") + # Run the checkout script + freCheckout.run() + else: + print("\nCheckout script PREVIOUSLY created in "+ srcDir + "/checkout.sh \n") + try: + subprocess.run(args=[srcDir+"/checkout.sh"], check=True) + except: + print("\nThere was an error with the checkout script "+srcDir+"/checkout.sh.", + "\nTry removing test folder: " + modelRoot +"\n") + raise fremakeBuildList = [] ## Loop through platforms and targets @@ -110,7 +133,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src" ## Check for type of build - if not iscontainer: + if iscontainer is False: baremetalRun = True ## Make the build directory based on the modelRoot, the platform, and the target bldDir = f'{modelRoot}/{fremakeYaml["experiment"]}/' + \ @@ -131,25 +154,52 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec print("\nMakefile created at " + bldDir + "/Makefile" + "\n") freMakefile.writeMakefile() - ## Create a list of compile scripts to run in parallel - fremakeBuild = buildBaremetal.buildBaremetal(exp = fremakeYaml["experiment"], - mkTemplatePath = mkTemplate, - srcDir = srcDir, - bldDir = bldDir, - target = target, - modules = modules, - modulesInit = modulesInit, - jobs = jobs) - - for c in fremakeYaml['src']: - fremakeBuild.writeBuildComponents(c) - fremakeBuild.writeScript() - fremakeBuildList.append(fremakeBuild) - ## Run the build if --execute option given, otherwise print out compile script path - if execute: - fremakeBuild.run() - else: + if not os.path.exists(bldDir+"/compile.sh"): + ## Create a list of compile scripts to run in parallel + fremakeBuild = buildBaremetal.buildBaremetal(exp = fremakeYaml["experiment"], + mkTemplatePath = mkTemplate, + srcDir = srcDir, + bldDir = bldDir, + target = target, + modules = modules, + modulesInit = modulesInit, + jobs = jobs) + + for c in fremakeYaml['src']: + fremakeBuild.writeBuildComponents(c) + fremakeBuild.writeScript() + fremakeBuildList.append(fremakeBuild) +# ## Run the build if --execute option given, otherwise print out compile script path +# if execute: +# fremakeBuild.run() +# else: +# print("Compile script created at "+ bldDir+"/compile.sh\n\n") print("Compile script created at "+ bldDir+"/compile.sh\n\n") + else: + if force_compile: + # Remove compile script + os.remove(bldDir + "/compile.sh") + # Re-create compile script + print("Re-creating the compile script...") + fremakeBuild = buildBaremetal.buildBaremetal(exp = fremakeYaml["experiment"], + mkTemplatePath = mkTemplate, + srcDir = srcDir, + bldDir = bldDir, + target = target, + modules = modules, + modulesInit = modulesInit, + jobs = jobs) + for c in fremakeYaml['src']: + fremakeBuild.writeBuildComponents(c) + fremakeBuild.writeScript() + fremakeBuildList.append(fremakeBuild) + print(" Compile script created in " + bldDir + "/compile.sh" + "\n") +# if execute: +# fremakeBuild.run() +# else: +# print("Compile script created at "+ bldDir+"/compile.sh\n\n") + else: + print("\nCompile script PREVIOUSLY created in " + bldDir + "/compile.sh" + "\n") else: ###################### container stuff below ####################################### ## Run the checkout script @@ -201,9 +251,6 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec if execute: subprocess.run(args=[dockerBuild.userScriptPath], check=True) - #freCheckout.cleanup() - #buildDockerfile(fremakeYaml,image) - if baremetalRun: if __name__ == '__main__': if execute: diff --git a/fre/make/tests/compilation/test_fre_make_run_fremake.py b/fre/make/tests/compilation/test_fre_make_run_fremake.py index c693a7a5..183eae7b 100644 --- a/fre/make/tests/compilation/test_fre_make_run_fremake.py +++ b/fre/make/tests/compilation/test_fre_make_run_fremake.py @@ -21,6 +21,6 @@ @pytest.mark.skip(reason='failing: fix in development, see PR 275') def test_fre_make_run_fremake_null_model_serial_compile(): ''' run fre make with run-fremake subcommand and build the null model experiment with gnu''' - runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, True, False) + runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, True, False, False, False) assert Path(f"{HOME_DIR}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists() From b3bb1183d6022c047bcedab98f45aeee38fc47d3 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 4 Dec 2024 12:49:20 -0500 Subject: [PATCH 07/39] #221 Fix click options passed --- fre/make/create_checkout_script.py | 4 ++-- fre/make/create_compile_script.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py index 374def8b..92df4cd1 100644 --- a/fre/make/create_checkout_script.py +++ b/fre/make/create_checkout_script.py @@ -132,12 +132,12 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v print("\nCheckout script PREVIOUSLY created in "+ tmp_dir + "/checkout.sh" + "\n") @click.command() -def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): +def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose,force_checkout): ''' Decorator for calling checkout_create - allows the decorated version of the function to be separate from the undecorated version ''' - return checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose) + return checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose,force_checkout) if __name__ == "__main__": checkout_create() diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index 53e1b2e7..8b7aca6e 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -115,7 +115,7 @@ def _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force Decorator for calling compile_create - allows the decorated version of the function to be separate from the undecorated version ''' - return compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose) + return compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_compile) if __name__ == "__main__": compile_create() From 295b04d85737ae972f06d57acd75d78dbc5acf3d Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 4 Dec 2024 12:49:39 -0500 Subject: [PATCH 08/39] #221 Update fre make unit test to use null_example --- fre/tests/test_fre_make_cli.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/fre/tests/test_fre_make_cli.py b/fre/tests/test_fre_make_cli.py index 279760b3..e3c5c82b 100644 --- a/fre/tests/test_fre_make_cli.py +++ b/fre/tests/test_fre_make_cli.py @@ -23,10 +23,10 @@ def test_cli_fre_make_opt_dne(): assert result.exit_code == 2 def test_cli_fre_make_create_checkout_baremetal(): - ''' fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t debug''' + ''' fre make create-checkout -y null_model.yaml -p ncrc5.intel23 -t debug''' # Set paths and click options test_dir = Path("fre/tests") - yamlfile = Path("fre/make/tests/AM5_example/") + yamlfile = Path("fre/make/tests/null_example") platform = "ncrc5.intel23" target = "debug" @@ -35,26 +35,30 @@ def test_cli_fre_make_create_checkout_baremetal(): Path(out_path).mkdir(parents=True,exist_ok=True) # Set HOME for modelRoot location (output location) in fre make + def_HOME = str(os.environ["HOME"]) os.environ["HOME"]=str(Path(out_path)) # run create-checkout - result = runner.invoke(fre.fre, args=["make", "create-checkout", "-y", f"{yamlfile}/am5.yaml", "-p", platform, "-t", target]) + result = runner.invoke(fre.fre, args=["make", "create-checkout", "-y", f"{yamlfile}/null_model.yaml", "-p", platform, "-t", target]) + + # Reset HOME + os.environ["HOME"] = def_HOME # Check for successful command, creation of checkout script, and that script is executable (os.access - checks is file has specific access mode, os.X_OK - checks executable permission) assert all ([result.exit_code == 0, - Path(f"{out_path}/fremake_canopy/test/am5/src/checkout.sh").exists(), - os.access(Path(f"{out_path}/fremake_canopy/test/am5/src/checkout.sh"), os.X_OK)]) + Path(f"{out_path}/fremake_canopy/test/null_model_full/src/checkout.sh").exists(), + os.access(Path(f"{out_path}/fremake_canopy/test/null_model_full/src/checkout.sh"), os.X_OK)]) def test_cli_fre_make_create_checkout_container(): - ''' fre make create-checkout -y am5.yaml -p hpcme.2023 -t debug''' + ''' fre make create-checkout -y null_model.yaml -p hpcme.2023 -t debug''' # Set paths and click options test_dir = Path("fre/tests") - yamlfile = Path("fre/make/tests/AM5_example/") + yamlfile = Path("fre/make/tests/null_example") platform = "hpcme.2023" target = "debug" # run create-checkout - result = runner.invoke(fre.fre, args=["make", "create-checkout", "-y", f"{yamlfile}/am5.yaml", "-p", platform, "-t", target]) + result = runner.invoke(fre.fre, args=["make", "create-checkout", "-y", f"{yamlfile}/null_model.yaml", "-p", platform, "-t", target]) # Check for successful command, creation of checkout script, and that script is executable (os.access - checks is file has specific access mode, os.X_OK - checks executable permission) assert all ([result.exit_code == 0, From d947e072d3a1cf227ca6137ac6229f89e504f5de Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 4 Dec 2024 14:11:40 -0500 Subject: [PATCH 09/39] #221 Re-create compile script if `--force-checkout` was used --- fre/make/run_fremake_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index f1c87be7..1d3e84e9 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -176,7 +176,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec # print("Compile script created at "+ bldDir+"/compile.sh\n\n") print("Compile script created at "+ bldDir+"/compile.sh\n\n") else: - if force_compile: + if force_compile or force_checkout: # Remove compile script os.remove(bldDir + "/compile.sh") # Re-create compile script From 10ef4fe47f7016155d097c5ca216d4bef0fab535 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 4 Dec 2024 17:47:30 -0500 Subject: [PATCH 10/39] #221 Add `-force-dockerfile` to re-create dockerfile --- fre/make/fremake.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fre/make/fremake.py b/fre/make/fremake.py index af1334aa..4ec97093 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -28,6 +28,8 @@ """ FORCE_COMPILE_OPT_HELP = """Force compile in case the executable directory exists. """ +FORCE_DOCKERFILE_OPT_HELP = """Force dockerfile creation in case dockerfile exists. +""" @click.group(help=click.style(" - access fre make subcommands", fg=(210,73,57))) def make_cli(): @@ -82,8 +84,12 @@ def make_cli(): "--force-compile", is_flag=True, help = FORCE_COMPILE_OPT_HELP) +@click.option("-FD", + "--force-dockerfile", + is_flag=True, + help = FORCE_DOCKERFILE_OPT_HELP) @click.pass_context -def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, execute, verbose, force_checkout, force_compile): +def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, execute, verbose, force_checkout, force_compile, force_dockerfile): # pylint: disable=unused-argument """ - Perform all fremake functions to run checkout and compile model\n From 65f8a0d67616b353cc84675a4604b2f210e48c2d Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 4 Dec 2024 17:48:17 -0500 Subject: [PATCH 11/39] #221 Create functions to remove duplication --- fre/make/create_checkout_script.py | 62 ++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py index 92df4cd1..565f7a1e 100644 --- a/fre/make/create_checkout_script.py +++ b/fre/make/create_checkout_script.py @@ -11,7 +11,35 @@ import fre.yamltools.combine_yamls as cy from .gfdlfremake import varsfre, yamlfre, checkout, targetfre +def baremetal_checkout_write_steps(model_yaml,src_dir,jobs,pc): + """ + Go through steps to write the checkout script for bare-metal build + """ + fre_checkout = checkout.checkout("checkout.sh",src_dir) + fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc) + fre_checkout.finish(pc) + + # Make checkout script executable + os.chmod(src_dir+"/checkout.sh", 0o744) + print(" Checkout script created in "+ src_dir + "/checkout.sh \n") + + return fre_checkout + +def container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc): + """ + Go through steps to write the checkout script for container + """ + fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir) + fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc) + fre_checkout.finish(pc) + print(" Checkout script created at " + tmp_dir + "/checkout.sh" + "\n") + + return fre_checkout + def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose,force_checkout): + """ + Call gfdlfremake/checkout.py to create the checkout script + """ # Define variables yml = yamlfile name = yamlfile.split(".")[0] @@ -73,12 +101,8 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v os.system("mkdir -p " + src_dir) # if the checkout script does not exist, it is created if not os.path.exists(src_dir+"/checkout.sh"): - fre_checkout = checkout.checkout("checkout.sh",src_dir) - fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc) - fre_checkout.finish(pc) - # Make checkout script executable - os.chmod(src_dir+"/checkout.sh", 0o744) - print("\nCheckout script created in "+ src_dir + "/checkout.sh \n") + print("Creating checkout script...") + fre_checkout = baremetal_checkout_write_steps(model_yaml,src_dir,jobs,pc) # Run the checkout script if execute: @@ -87,16 +111,13 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v sys.exit() else: if force_checkout: - print("Re-creating the checkout script...\n") - # Remove previous checkout + # Remove previous checkout + print("\nRemoving previously checkout script and checked out source code") shutil.rmtree(src_dir) + # Create checkout script - fre_checkout = checkout.checkout("checkout.sh",src_dir) - fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc) - fre_checkout.finish(pc) - # Make checkout script executable - os.chmod(src_dir+"/"+checkout_script_name, 0o744) - print(" Checkout script created in "+ src_dir + "/checkout.sh \n") + print("Re-creating the checkout script...\n") + fre_checkout = baremetal_checkout_write_steps(model_yaml,src_dir,jobs,pc) else: print("\nCheckout script PREVIOUSLY created in "+ src_dir + "/checkout.sh \n") @@ -115,19 +136,18 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v bld_dir = model_root + "/" + fremake_yaml["experiment"] + "/exec" tmp_dir = "tmp/"+platform_name if not os.path.exists(tmp_dir+"/checkout.sh"): - fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir) - fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc) - fre_checkout.finish(pc) - print("\nCheckout script created at " + tmp_dir + "/checkout.sh" + "\n") + # Create the checkout script + print("Creating checkout script...") + container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc) else: if force_checkout: # Remove the checkout script + print("\nRemoving previously made checkout script") os.remove(tmp_dir+"/checkout.sh") + # Create the checkout script print("Re-creating checkout script...") - fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir) - fre_checkout.finish(pc) - print(" Checkout script created in "+ tmp_dir + "/checkout.sh" + "\n") + container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc) else: print("\nCheckout script PREVIOUSLY created in "+ tmp_dir + "/checkout.sh" + "\n") From 2f31d4f80dcfb18c34606b3e65b5d133deddce55 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 4 Dec 2024 17:48:54 -0500 Subject: [PATCH 12/39] #221 Create function to remove duplication in compile creation - code was repeated if `--force-compile` was used --- fre/make/create_compile_script.py | 90 ++++++++++++++++++------------- 1 file changed, 52 insertions(+), 38 deletions(-) diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index 8b7aca6e..9084d0e8 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -9,6 +9,26 @@ from .gfdlfremake import varsfre, yamlfre, targetfre, buildBaremetal import fre.yamltools.combine_yamls as cy +def compile_script_write_steps(yaml_obj,build_list,mkTemplate,src_dir,bld_dir,target,modules,modulesInit,jobs): + """ + Go through steps to create the compile script + """ + ## Create a list of compile scripts to run in parallel + fremakeBuild = buildBaremetal.buildBaremetal(exp = yaml_obj["experiment"], + mkTemplatePath = mkTemplate, + srcDir = src_dir, + bldDir = bld_dir, + target = target, + modules = modules, + modulesInit = modulesInit, + jobs = jobs) + for c in yaml_obj['src']: + fremakeBuild.writeBuildComponents(c) + fremakeBuild.writeScript() + build_list.append(fremakeBuild) + + print(" Compile script created in " + bld_dir + "/compile.sh" + "\n") + def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_compile): # Define variables yml = yamlfile @@ -21,8 +41,8 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_ else: logging.basicConfig(level=logging.ERROR) - srcDir="src" - checkoutScriptName = "checkout.sh" + src_dir="src" + checkout_script_name = "checkout.sh" baremetalRun = False # This is needed if there are no bare metal runs ## Split and store the platforms and targets in a list @@ -37,10 +57,10 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_ full_combined = cy.combined_compile_existcheck(combined,yml,platform,target) ## Get the variables in the model yaml - freVars = varsfre.frevars(full_combined) + fre_vars = varsfre.frevars(full_combined) ## Open the yaml file and parse as fremakeYaml - modelYaml = yamlfre.freyaml(full_combined,freVars) + modelYaml = yamlfre.freyaml(full_combined,fre_vars) fremakeYaml = modelYaml.getCompileYaml() ## Error checking the targets @@ -59,48 +79,42 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_ (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName) ## Make the bldDir based on the modelRoot, the platform, and the target - srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src" + src_dir = modelRoot + "/" + fremakeYaml["experiment"] + "/src" ## Check for type of build if iscontainer is False: baremetalRun = True - bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/" + platformName + "-" + target.gettargetName() + "/exec" - os.system("mkdir -p " + bldDir) - if not os.path.exists(bldDir+"/compile.sh"): - ## Create a list of compile scripts to run in parallel - fremakeBuild = buildBaremetal.buildBaremetal(exp = fremakeYaml["experiment"], - mkTemplatePath = mkTemplate, - srcDir = srcDir, - bldDir = bldDir, - target = target, - modules = modules, - modulesInit = modulesInit, - jobs = jobs) - for c in fremakeYaml['src']: - fremakeBuild.writeBuildComponents(c) - fremakeBuild.writeScript() - fremakeBuildList.append(fremakeBuild) - print("\nCompile script created in " + bldDir + "/compile.sh" + "\n") + bld_dir = modelRoot + "/" + fremakeYaml["experiment"] + "/" + platformName + "-" + target.gettargetName() + "/exec" + os.system("mkdir -p " + bld_dir) + if not os.path.exists(bld_dir+"/compile.sh"): + print("\nCreating the compile script...") + compile_script_write_steps(yaml_obj = fremakeYaml, + build_list = fremakeBuildList, + #exp_name = fremakeYaml["experiment"], + mkTemplate = mkTemplate, + src_dir = src_dir, + bld_dir = bld_dir, + target = target, + modules = modules, + modulesInit = modulesInit, + jobs = jobs) else: if force_compile: # Remove compile script - os.remove(bldDir + "/compile.sh") + os.remove(bld_dir + "/compile.sh") # Re-create compile script - print("Re-creating the compile script...") - fremakeBuild = buildBaremetal.buildBaremetal(exp = fremakeYaml["experiment"], - mkTemplatePath = mkTemplate, - srcDir = srcDir, - bldDir = bldDir, - target = target, - modules = modules, - modulesInit = modulesInit, - jobs = jobs) - for c in fremakeYaml['src']: - fremakeBuild.writeBuildComponents(c) - fremakeBuild.writeScript() - fremakeBuildList.append(fremakeBuild) - print(" Compile script created in " + bldDir + "/compile.sh" + "\n") + print("\nRe-creating the compile script...") + compile_script_write_steps(yaml_obj = fremakeYaml, + build_list = fremakeBuildList, + #exp_name = fremakeYaml["experiment"], + mkTemplate = mkTemplate, + src_dir = src_dir, + bld_dir = bld_dir, + target = target, + modules = modules, + modulesInit = modulesInit, + jobs = jobs) else: - print("\nCompile script PREVIOUSLY created in " + bldDir + "/compile.sh" + "\n") + print("\nCompile script PREVIOUSLY created in " + bld_dir + "/compile.sh" + "\n") if execute: if baremetalRun: From f32a3f612b63861ea4b617524144ed963ee2508b Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 4 Dec 2024 17:50:48 -0500 Subject: [PATCH 13/39] #221 Create functions to remove duplication - code was repeated if `--force-checkout`, `--force-compile` or `--force-dockerfile` was used --- fre/make/run_fremake_script.py | 298 +++++++++++++++++++++------------ 1 file changed, 192 insertions(+), 106 deletions(-) diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index 1d3e84e9..dbdd88b9 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -17,7 +17,75 @@ targetfre, varsfre, yamlfre, checkout, makefilefre, buildDocker, buildBaremetal ) -def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose,force_checkout,force_compile): +def baremetal_checkout_write_steps(model_yaml,src_dir,jobs,pc): + """ + Go through steps to write the checkout script for bare-metal build + """ + fre_checkout = checkout.checkout("checkout.sh",src_dir) + fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc) + fre_checkout.finish(pc) + + # Make checkout script executable + os.chmod(src_dir+"/checkout.sh", 0o744) + print(" Checkout script created here: "+ src_dir + "/checkout.sh \n") + + return fre_checkout + +def container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc): + """ + Go through steps to write the checkout script for container + """ + fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir) + fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc) + fre_checkout.finish(pc) + print(" Checkout script created here: " + tmp_dir + "/checkout.sh" + "\n") + + return fre_checkout + +def compile_script_write_steps(yaml_obj,build_list,mkTemplate,src_dir,bld_dir,target,modules,modulesInit,jobs): + """ + Go through steps to create compile script + """ + ## Create a list of compile scripts to run in parallel + fremakeBuild = buildBaremetal.buildBaremetal(exp = yaml_obj["experiment"], + mkTemplatePath = mkTemplate, + srcDir = src_dir, + bldDir = bld_dir, + target = target, + modules = modules, + modulesInit = modulesInit, + jobs = jobs) + for c in yaml_obj['src']: + fremakeBuild.writeBuildComponents(c) + fremakeBuild.writeScript() + build_list.append(fremakeBuild) + + print(" Compile script created here: " + bld_dir + "/compile.sh" + "\n") + +def dockerfile_write_steps(yaml_obj,makefile_obj,img,run_env,target,td,cr,cb,cd): + """ + """ + dockerBuild = buildDocker.container(base = img, + exp = yaml_obj["experiment"], + libs = yaml_obj["container_addlibs"], + RUNenv = run_env, + target = target) + + dockerBuild.writeDockerfileCheckout("checkout.sh", td+"/checkout.sh") + dockerBuild.writeDockerfileMakefile(makefile_obj.getTmpDir() + "/Makefile", + makefile_obj.getTmpDir() + "/linkline.sh") + + for c in yaml_obj['src']: + dockerBuild.writeDockerfileMkmf(c) + + dockerBuild.writeRunscript(run_env,cr,td+"/execrunscript.sh") + print(f" Dockerfile created here: {cd}") + + # Create build script for container + dockerBuild.createBuildScript(cb, cr) + print(f" Container build script created here: {dockerBuild.userScriptPath}\n") + +def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose,force_checkout,force_compile,force_dockerfile): ''' run fremake via click''' yml = yamlfile name = yamlfile.split(".")[0] @@ -36,7 +104,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec logging.basicConfig(level=logging.ERROR) #### Main - srcDir="src" + #srcDir="src" checkoutScriptName = "checkout.sh" baremetalRun = False # This is needed if there are no bare metal runs @@ -52,11 +120,11 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec full_combined = cy.combined_compile_existcheck(combined,yml,platform,target) ## Get the variables in the model yaml - freVars = varsfre.frevars(full_combined) + fre_vars = varsfre.frevars(full_combined) ## Open the yaml file and parse as fremakeYaml - modelYaml = yamlfre.freyaml(full_combined,freVars) - fremakeYaml = modelYaml.getCompileYaml() + model_yaml = yamlfre.freyaml(full_combined,fre_vars) + fremake_yaml = model_yaml.getCompileYaml() ## Error checking the targets for targetName in tlist: @@ -67,185 +135,203 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec ## This should be done separately and serially because bare metal platforms should all be using ## the same source code. for platformName in plist: - if modelYaml.platforms.hasPlatform(platformName): + if model_yaml.platforms.hasPlatform(platformName): pass else: raise ValueError(f'{platformName} does not exist in ' - f'{modelYaml.combined.get("compile").get("platformYaml")}') + f'{model_yaml.combined.get("compile").get("platformYaml")}') ( compiler, modules, modulesInit, fc, cc, modelRoot, iscontainer, mkTemplate, containerBuild, ContainerRun, - RUNenv ) = modelYaml.platforms.getPlatformFromName(platformName) + RUNenv ) = model_yaml.platforms.getPlatformFromName(platformName) ## Create the checkout script if iscontainer is False: ## Create the source directory for the platform - srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src" - if not os.path.exists(srcDir): - os.system("mkdir -p " + srcDir) - if not os.path.exists(srcDir+"/checkout.sh"): - freCheckout = checkout.checkout("checkout.sh",srcDir) - freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc) - freCheckout.finish(pc) - os.chmod(srcDir+"/checkout.sh", 0o744) - print("\nCheckout script created at "+ srcDir + "/checkout.sh \n") + src_dir = modelRoot + "/" + fremake_yaml["experiment"] + "/src" + if not os.path.exists(src_dir): + os.system("mkdir -p " + src_dir) + if not os.path.exists(src_dir+"/checkout.sh"): + print("Creating checkout script...") + freCheckout = baremetal_checkout_write_steps(model_yaml,src_dir,jobs,pc) ## TODO: Options for running on login cluster? freCheckout.run() else: if force_checkout: - print("Re-creating the checkout script...\n") # Remove previous checkout - shutil.rmtree(srcDir) + print("\nRemoving previously checkout script and checked out source code") + shutil.rmtree(src_dir) + # Create checkout script - freCheckout = checkout.checkout("checkout.sh",srcDir) - freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc) - freCheckout.finish(pc) - # Make checkout script executable - os.chmod(srcDir+"/"+checkoutScriptName, 0o744) - print(" Checkout script created in "+ srcDir + "/checkout.sh \n") + print("Re-creating the checkout script...") + freCheckout = baremetal_checkout_write_steps(model_yaml,src_dir,jobs,pc) # Run the checkout script freCheckout.run() else: - print("\nCheckout script PREVIOUSLY created in "+ srcDir + "/checkout.sh \n") - try: - subprocess.run(args=[srcDir+"/checkout.sh"], check=True) - except: - print("\nThere was an error with the checkout script "+srcDir+"/checkout.sh.", - "\nTry removing test folder: " + modelRoot +"\n") - raise + print("\nCheckout script PREVIOUSLY created and run here: "+ src_dir + "/checkout.sh") +# try: +# subprocess.run(args=[src_dir+"/checkout.sh"], check=True) +# except: +# print("\nThere was an error with the checkout script "+src_dir+"/checkout.sh.", +# "\nTry:\n", +# " - removing test folder: " + modelRoot + "\n", +# " - add --force-checkout option\n") +# raise fremakeBuildList = [] ## Loop through platforms and targets for platformName in plist: for targetName in tlist: target = targetfre.fretarget(targetName) - if modelYaml.platforms.hasPlatform(platformName): + if model_yaml.platforms.hasPlatform(platformName): pass else: - raise ValueError (platformName + " does not exist in " + modelYaml.platformsfile) + raise ValueError (platformName + " does not exist in " + model_yaml.platformsfile) ( compiler, modules, modulesInit, fc, cc, modelRoot, iscontainer, mkTemplate, containerBuild, containerRun, - RUNenv ) = modelYaml.platforms.getPlatformFromName(platformName) + RUNenv ) = model_yaml.platforms.getPlatformFromName(platformName) ## Make the source directory based on the modelRoot and platform - srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src" + src_dir = modelRoot + "/" + fremake_yaml["experiment"] + "/src" ## Check for type of build if iscontainer is False: baremetalRun = True ## Make the build directory based on the modelRoot, the platform, and the target - bldDir = f'{modelRoot}/{fremakeYaml["experiment"]}/' + \ + bld_dir = f'{modelRoot}/{fremake_yaml["experiment"]}/' + \ f'{platformName}-{target.gettargetName()}/exec' - os.system("mkdir -p " + bldDir) + os.system("mkdir -p " + bld_dir) ## Create the Makefile - freMakefile = makefilefre.makefile(exp = fremakeYaml["experiment"], - libs = fremakeYaml["baremetal_linkerflags"], - srcDir = srcDir, - bldDir = bldDir, + freMakefile = makefilefre.makefile(exp = fremake_yaml["experiment"], + libs = fremake_yaml["baremetal_linkerflags"], + srcDir = src_dir, + bldDir = bld_dir, mkTemplatePath = mkTemplate) # Loop through components, send component name/requires/overrides for Makefile - for c in fremakeYaml['src']: + for c in fremake_yaml['src']: freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides']) - print("\nMakefile created at " + bldDir + "/Makefile" + "\n") + print("Makefile created here: " + bld_dir + "/Makefile")# + "\n") freMakefile.writeMakefile() - if not os.path.exists(bldDir+"/compile.sh"): + if not os.path.exists(bld_dir+"/compile.sh"): ## Create a list of compile scripts to run in parallel - fremakeBuild = buildBaremetal.buildBaremetal(exp = fremakeYaml["experiment"], - mkTemplatePath = mkTemplate, - srcDir = srcDir, - bldDir = bldDir, - target = target, - modules = modules, - modulesInit = modulesInit, - jobs = jobs) - - for c in fremakeYaml['src']: - fremakeBuild.writeBuildComponents(c) - fremakeBuild.writeScript() - fremakeBuildList.append(fremakeBuild) + print("\nCreating the compile script...") + compile_script_write_steps(yaml_obj = fremake_yaml, + build_list = fremakeBuildList, + #exp_name = fremake_yaml["experiment"], + mkTemplate = mkTemplate, + src_dir = src_dir, + bld_dir = bld_dir, + target = target, + modules = modules, + modulesInit = modulesInit, + jobs = jobs) + # ## Run the build if --execute option given, otherwise print out compile script path # if execute: # fremakeBuild.run() # else: # print("Compile script created at "+ bldDir+"/compile.sh\n\n") - print("Compile script created at "+ bldDir+"/compile.sh\n\n") else: if force_compile or force_checkout: # Remove compile script - os.remove(bldDir + "/compile.sh") + os.remove(bld_dir + "/compile.sh") # Re-create compile script - print("Re-creating the compile script...") - fremakeBuild = buildBaremetal.buildBaremetal(exp = fremakeYaml["experiment"], - mkTemplatePath = mkTemplate, - srcDir = srcDir, - bldDir = bldDir, - target = target, - modules = modules, - modulesInit = modulesInit, - jobs = jobs) - for c in fremakeYaml['src']: - fremakeBuild.writeBuildComponents(c) - fremakeBuild.writeScript() - fremakeBuildList.append(fremakeBuild) - print(" Compile script created in " + bldDir + "/compile.sh" + "\n") + print("\nRe-creating the compile script...") + compile_script_write_steps(yaml_obj = fremake_yaml, + build_list = fremakeBuildList, + #exp_name = fremake_yaml["experiment"], + mkTemplate = mkTemplate, + src_dir = src_dir, + bld_dir = bld_dir, + target = target, + modules = modules, + modulesInit = modulesInit, + jobs = jobs) # if execute: # fremakeBuild.run() # else: # print("Compile script created at "+ bldDir+"/compile.sh\n\n") else: - print("\nCompile script PREVIOUSLY created in " + bldDir + "/compile.sh" + "\n") + print("Compile script PREVIOUSLY created here: " + bld_dir + "/compile.sh" + "\n") else: ###################### container stuff below ####################################### ## Run the checkout script # image="hpc-me-intel:2021.1.1" image="ecpe4s/noaa-intel-prototype:2023.09.25" - bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/exec" - tmpDir = "tmp/"+platformName + bld_dir = modelRoot + "/" + fremake_yaml["experiment"] + "/exec" + tmp_dir = "tmp/"+platformName ## Create the checkout script - freCheckout = checkout.checkoutForContainer("checkout.sh", srcDir, tmpDir) - freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc) - freCheckout.finish(pc) + if not os.path.exists(tmp_dir+"/checkout.sh"): + # Create the checkout script + print("Creating checkout script...") + container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc) + else: + if force_checkout: + # Remove the checkout script + print("\nRemoving previously made checkout script") + os.remove(tmp_dir+"/checkout.sh") + + # Create the checkout script + print("Re-creating checkout script...") + container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc) + else: + print("\nCheckout script PREVIOUSLY created and run here: "+ tmp_dir + "/checkout.sh") ## Create the makefile ### Should this even be a separate class from "makefile" in makefilefre? ~ ejs - freMakefile = makefilefre.makefileContainer(exp = fremakeYaml["experiment"], - libs = fremakeYaml["container_addlibs"], - srcDir = srcDir, - bldDir = bldDir, + freMakefile = makefilefre.makefileContainer(exp = fremake_yaml["experiment"], + libs = fremake_yaml["container_addlibs"], + srcDir = src_dir, + bldDir = bld_dir, mkTemplatePath = mkTemplate, - tmpDir = tmpDir) + tmpDir = tmp_dir) # Loop through components and send the component name and requires for the Makefile - for c in fremakeYaml['src']: + for c in fremake_yaml['src']: freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides']) freMakefile.writeMakefile() + print("Makefile created here: " + tmp_dir + "/Makefile")# + "\n") ## Build the dockerfile - dockerBuild = buildDocker.container(base = image, - exp = fremakeYaml["experiment"], - libs = fremakeYaml["container_addlibs"], - RUNenv = RUNenv, - target = target) - - dockerBuild.writeDockerfileCheckout("checkout.sh", tmpDir+"/checkout.sh") - dockerBuild.writeDockerfileMakefile(freMakefile.getTmpDir() + "/Makefile", - freMakefile.getTmpDir() + "/linkline.sh") - - for c in fremakeYaml['src']: - dockerBuild.writeDockerfileMkmf(c) - - dockerBuild.writeRunscript(RUNenv,containerRun,tmpDir+"/execrunscript.sh") - - # Create build script for container - dockerBuild.createBuildScript(containerBuild, containerRun) - print("Container build script created at "+dockerBuild.userScriptPath+"\n\n") + #if is doesn't exist, write + curr_dir = os.getcwd() + if not os.path.exists(f"{curr_dir}/Dockerfile"): + dockerfile_write_steps(yaml_obj = fremake_yaml, + makefile_obj = freMakefile, + img = image, + run_env = RUNenv, + target = target, + td = tmp_dir, + cr = containerRun, + cb = containerBuild, + cd = curr_dir) + else: + if force_dockerfile or force_checkout: + # Remove the dockerfile + print("\nRemoving previously made dockerfile") + os.remove(curr_dir+"/Dockerfile") + + # Create the checkout script + print("Re-creating Dockerfile...") + dockerfile_write_steps(yaml_obj = fremake_yaml, + makefile_obj = freMakefile, + img = image, + run_env = RUNenv, + target = target, + td = tmp_dir, + cr = containerRun, + cb = containerBuild, + cd = curr_dir) + else: + print(f"Dockerfile PREVIOUSLY created here: {curr_dir}/Dockerfile") + print(f"Container build script created here: {curr_dir}/createContainer.sh\n") # Execute if flag is given if execute: @@ -260,12 +346,12 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) @click.command() -def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose,force_checkout,force_compile): +def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose,force_checkout,force_compile,force_dockerfile): ''' Decorator for calling fremake_run - allows the decorated version of the function to be separate from the undecorated version ''' - return fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose,force_checkout,force_compile) + return fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose,force_checkout,force_compile,force_dockerfile) if __name__ == "__main__": fremake_run() From fbce70176e3f854aaf2b8b487d26de0fd7c2e24e Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 4 Dec 2024 18:17:22 -0500 Subject: [PATCH 14/39] #221 Add `--force-dockerfile` option --- fre/make/fremake.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fre/make/fremake.py b/fre/make/fremake.py index 4ec97093..9c580148 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -270,8 +270,12 @@ def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbos @click.option("--execute", is_flag = True, help = "Build Dockerfile that has been generated by create-docker.") +@click.option("-FD", + "--force-dockerfile", + is_flag=True, + help = FORCE_DOCKERFILE_OPT_HELP) @click.pass_context -def create_dockerfile(context,yamlfile,platform,target,execute): +def create_dockerfile(context,yamlfile,platform,target,execute,force_dockerfile): # pylint: disable=unused-argument """ - Write the dockerfile """ context.forward(create_docker_script._dockerfile_create) From 6a99f4d1d231d5af638ab3af873e202d5189026f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 4 Dec 2024 18:17:54 -0500 Subject: [PATCH 15/39] #221 Create function to remove duplication - code would be duplicated if `--force-dockerfile` option is used --- fre/make/create_docker_script.py | 85 ++++++++++++++++++++++---------- 1 file changed, 59 insertions(+), 26 deletions(-) diff --git a/fre/make/create_docker_script.py b/fre/make/create_docker_script.py index 5731991e..671e22b8 100644 --- a/fre/make/create_docker_script.py +++ b/fre/make/create_docker_script.py @@ -5,11 +5,32 @@ import subprocess from pathlib import Path import click -#from .gfdlfremake import varsfre, targetfre, makefilefre, platformfre, yamlfre, buildDocker from .gfdlfremake import varsfre, targetfre, yamlfre, buildDocker import fre.yamltools.combine_yamls as cy -def dockerfile_create(yamlfile,platform,target,execute): +def dockerfile_write_steps(yaml_obj,img,run_env,target,td,cr,cb,cd): + """ + """ + dockerBuild = buildDocker.container(base = img, + exp = yaml_obj["experiment"], + libs = yaml_obj["container_addlibs"], + RUNenv = run_env, + target = target) + + dockerBuild.writeDockerfileCheckout("checkout.sh", td+"/checkout.sh") + dockerBuild.writeDockerfileMakefile(td+"/Makefile", td+"/linkline.sh") + + for c in yaml_obj['src']: + dockerBuild.writeDockerfileMkmf(c) + + dockerBuild.writeRunscript(run_env,cr,td+"/execrunscript.sh") + print(f" Dockerfile created here: {cd}") + + # Create build script for container + dockerBuild.createBuildScript(cb, cr) + print(f" Container build script created here: {dockerBuild.userScriptPath}\n") + +def dockerfile_create(yamlfile,platform,target,execute,force_dockerfile): srcDir="src" checkoutScriptName = "checkout.sh" baremetalRun = False # This is needed if there are no bare metal runs @@ -57,37 +78,49 @@ def dockerfile_create(yamlfile,platform,target,execute): bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/exec" tmpDir = "tmp/"+platformName - dockerBuild = buildDocker.container(base = image, - exp = fremakeYaml["experiment"], - libs = fremakeYaml["container_addlibs"], - RUNenv = RUNenv, - target = targetObject) - dockerBuild.writeDockerfileCheckout("checkout.sh", tmpDir+"/checkout.sh") - dockerBuild.writeDockerfileMakefile(tmpDir+"/Makefile", tmpDir+"/linkline.sh") - - for c in fremakeYaml['src']: - dockerBuild.writeDockerfileMkmf(c) - - dockerBuild.writeRunscript(RUNenv,containerRun,tmpDir+"/execrunscript.sh") - currDir = os.getcwd() - click.echo("\ntmpDir created in " + currDir + "/tmp") - click.echo("Dockerfile created in " + currDir +"\n") - - # create build script for container - dockerBuild.createBuildScript(containerBuild, containerRun) - print("Container build script created at "+dockerBuild.userScriptPath+"\n\n") - - # run the script if option is given - if run: + curr_dir = os.getcwd() + if not os.path.exists(f"{curr_dir}/Dockerfile"): + dockerfile_write_steps(yaml_obj = fremakeYaml, + #makefile_obj = freMakefile, + img = image, + run_env = RUNenv, + target = targetObject, + td = tmpDir, + cr = containerRun, + cb = containerBuild, + cd = curr_dir) + else: + if force_dockerfile: + # Remove the dockerfile + print("\nRemoving previously made dockerfile") + os.remove(curr_dir+"/Dockerfile") + + # Create the checkout script + print("Re-creating Dockerfile...") + dockerfile_write_steps(yaml_obj = fremakeYaml, +# makefile_obj = freMakefile, + img = image, + run_env = RUNenv, + target = targetObject, + td = tmpDir, + cr = containerRun, + cb = containerBuild, + cd = curr_dir) + else: + print(f"Dockerfile PREVIOUSLY created here: {curr_dir}/Dockerfile") + print(f"Container build script created here: {curr_dir}/createContainer.sh\n") + + # Execute if flag is given + if execute: subprocess.run(args=[dockerBuild.userScriptPath], check=True) @click.command() -def _dockerfile_create(yamlfile,platform,target,execute): +def _dockerfile_create(yamlfile,platform,target,execute,force_dockerfile): ''' Decorator for calling dockerfile_create - allows the decorated version of the function to be separate from the undecorated version ''' - return dockerfile_create(yamlfile,platform,target,execute) + return dockerfile_create(yamlfile,platform,target,execute,force_dockerfile) if __name__ == "__main__": dockerfile_create() From a6dd49397588de107cd5522c07360cee4af65d59 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 5 Dec 2024 16:53:36 -0500 Subject: [PATCH 16/39] #221 Fix print statement --- fre/make/create_checkout_script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py index 565f7a1e..f42a4105 100644 --- a/fre/make/create_checkout_script.py +++ b/fre/make/create_checkout_script.py @@ -101,7 +101,7 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v os.system("mkdir -p " + src_dir) # if the checkout script does not exist, it is created if not os.path.exists(src_dir+"/checkout.sh"): - print("Creating checkout script...") + print("\nCreating checkout script...") fre_checkout = baremetal_checkout_write_steps(model_yaml,src_dir,jobs,pc) # Run the checkout script @@ -116,7 +116,7 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v shutil.rmtree(src_dir) # Create checkout script - print("Re-creating the checkout script...\n") + print("Re-creating the checkout script...") fre_checkout = baremetal_checkout_write_steps(model_yaml,src_dir,jobs,pc) else: print("\nCheckout script PREVIOUSLY created in "+ src_dir + "/checkout.sh \n") From 6426b321850939968415da1912deb791073ceb97 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 5 Dec 2024 16:54:50 -0500 Subject: [PATCH 17/39] #221 Add error statement if container doesn't build --- fre/make/create_docker_script.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fre/make/create_docker_script.py b/fre/make/create_docker_script.py index 671e22b8..228239a5 100644 --- a/fre/make/create_docker_script.py +++ b/fre/make/create_docker_script.py @@ -112,7 +112,11 @@ def dockerfile_create(yamlfile,platform,target,execute,force_dockerfile): # Execute if flag is given if execute: - subprocess.run(args=[dockerBuild.userScriptPath], check=True) + try: + subprocess.run(args=[f"{curr_dir}/createContainer.sh"], check=True) + except: + print(f"There was an error in runnning the container build script: {curr_dir}/createContainer.sh") + raise @click.command() def _dockerfile_create(yamlfile,platform,target,execute,force_dockerfile): From 6ae45d596219b3fa6a2742cb4f721261a4cd63be Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 5 Dec 2024 16:56:06 -0500 Subject: [PATCH 18/39] #221 Update print statements and change `click.echo` to `print` for consistency --- fre/make/create_makefile_script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/make/create_makefile_script.py b/fre/make/create_makefile_script.py index eaf340dd..0e9cc737 100644 --- a/fre/make/create_makefile_script.py +++ b/fre/make/create_makefile_script.py @@ -59,7 +59,7 @@ def makefile_create(yamlfile,platform,target): for c in fremakeYaml['src']: freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides']) freMakefile.writeMakefile() - click.echo("\nMakefile created at " + bldDir + "/Makefile" + "\n") + print("\nMakefile created here: " + bldDir + "/Makefile" + "\n") else: image="ecpe4s/noaa-intel-prototype:2023.09.25" bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/exec" @@ -75,7 +75,7 @@ def makefile_create(yamlfile,platform,target): for c in fremakeYaml['src']: freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides']) freMakefile.writeMakefile() - click.echo("\nMakefile created at " + bldDir + "/Makefile" + "\n") + print("\nMakefile created here: ./" + tmpDir + "/Makefile" + "\n") @click.command() def _makefile_create(yamlfile,platform,target): From 8162994163c49454950a7c2c63cb29d4dca38fbf Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 5 Dec 2024 16:57:26 -0500 Subject: [PATCH 19/39] #221 Accidentally removed `--execute` options - add back in --- fre/make/create_compile_script.py | 56 ++++++++++++++++++------------- fre/make/run_fremake_script.py | 38 ++++++++++----------- 2 files changed, 50 insertions(+), 44 deletions(-) diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index 9084d0e8..43640c0e 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -6,10 +6,11 @@ from pathlib import Path from multiprocessing.dummy import Pool import click +import subprocess from .gfdlfremake import varsfre, yamlfre, targetfre, buildBaremetal import fre.yamltools.combine_yamls as cy -def compile_script_write_steps(yaml_obj,build_list,mkTemplate,src_dir,bld_dir,target,modules,modulesInit,jobs): +def compile_script_write_steps(yaml_obj,mkTemplate,src_dir,bld_dir,target,modules,modulesInit,jobs): """ Go through steps to create the compile script """ @@ -25,9 +26,9 @@ def compile_script_write_steps(yaml_obj,build_list,mkTemplate,src_dir,bld_dir,ta for c in yaml_obj['src']: fremakeBuild.writeBuildComponents(c) fremakeBuild.writeScript() - build_list.append(fremakeBuild) print(" Compile script created in " + bld_dir + "/compile.sh" + "\n") + return fremakeBuild def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_compile): # Define variables @@ -87,38 +88,45 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_ os.system("mkdir -p " + bld_dir) if not os.path.exists(bld_dir+"/compile.sh"): print("\nCreating the compile script...") - compile_script_write_steps(yaml_obj = fremakeYaml, - build_list = fremakeBuildList, - #exp_name = fremakeYaml["experiment"], - mkTemplate = mkTemplate, - src_dir = src_dir, - bld_dir = bld_dir, - target = target, - modules = modules, - modulesInit = modulesInit, - jobs = jobs) + fremakeBuild = compile_script_write_steps(yaml_obj = fremakeYaml, + mkTemplate = mkTemplate, + src_dir = src_dir, + bld_dir = bld_dir, + target = target, + modules = modules, + modulesInit = modulesInit, + jobs = jobs) + fremakeBuildList.append(fremakeBuild) + if execute: + print("Running the compile script\n") + fremakeBuild.run() else: if force_compile: # Remove compile script os.remove(bld_dir + "/compile.sh") # Re-create compile script print("\nRe-creating the compile script...") - compile_script_write_steps(yaml_obj = fremakeYaml, - build_list = fremakeBuildList, - #exp_name = fremakeYaml["experiment"], - mkTemplate = mkTemplate, - src_dir = src_dir, - bld_dir = bld_dir, - target = target, - modules = modules, - modulesInit = modulesInit, - jobs = jobs) + fremakeBuild = compile_script_write_steps(yaml_obj = fremakeYaml, + mkTemplate = mkTemplate, + src_dir = src_dir, + bld_dir = bld_dir, + target = target, + modules = modules, + modulesInit = modulesInit, + jobs = jobs) + fremakeBuildList.append(fremakeBuild) + if execute: + print("Running the compile script\n") + fremakeBuild.run() else: - print("\nCompile script PREVIOUSLY created in " + bld_dir + "/compile.sh" + "\n") + print("Compile script PREVIOUSLY created here: " + bld_dir + "/compile.sh" + "\n") + if execute: + subprocess.run(args=[bld_dir+"/compile.sh"], check=True) + ##TO-DO --> THIS COULD CAUSE PROBLEMS IF USER FORGOT TO DO FORCE-COMPILE AFTER A CHANGE --> IT'LL JUST RUN PREVIOUS ONE. I have the message about running previous compile script, but is it better to just do --force-compile (even after no change?) if execute: if baremetalRun: - pool = Pool(processes=nparallel) # Create a multiprocessing Pool + pool = Pool(processes=nparallel) # Create a multiprocessing Pool pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) # process data_inputs iterable with pool else: sys.exit() diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index dbdd88b9..46a3d844 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -38,11 +38,11 @@ def container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc): fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir) fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc) fre_checkout.finish(pc) - print(" Checkout script created here: " + tmp_dir + "/checkout.sh" + "\n") + print(" Checkout script created here: " + tmp_dir + "/checkout.sh") return fre_checkout -def compile_script_write_steps(yaml_obj,build_list,mkTemplate,src_dir,bld_dir,target,modules,modulesInit,jobs): +def compile_script_write_steps(yaml_obj,mkTemplate,src_dir,bld_dir,target,modules,modulesInit,jobs): """ Go through steps to create compile script """ @@ -58,12 +58,13 @@ def compile_script_write_steps(yaml_obj,build_list,mkTemplate,src_dir,bld_dir,ta for c in yaml_obj['src']: fremakeBuild.writeBuildComponents(c) fremakeBuild.writeScript() - build_list.append(fremakeBuild) - print(" Compile script created here: " + bld_dir + "/compile.sh" + "\n") + return fremakeBuild + def dockerfile_write_steps(yaml_obj,makefile_obj,img,run_env,target,td,cr,cb,cd): """ + Go through steps to create Dockerfile and container build script. """ dockerBuild = buildDocker.container(base = img, exp = yaml_obj["experiment"], @@ -221,9 +222,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec if not os.path.exists(bld_dir+"/compile.sh"): ## Create a list of compile scripts to run in parallel print("\nCreating the compile script...") - compile_script_write_steps(yaml_obj = fremake_yaml, - build_list = fremakeBuildList, - #exp_name = fremake_yaml["experiment"], + fremakeBuild = compile_script_write_steps(yaml_obj = fremake_yaml, mkTemplate = mkTemplate, src_dir = src_dir, bld_dir = bld_dir, @@ -231,21 +230,17 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec modules = modules, modulesInit = modulesInit, jobs = jobs) - -# ## Run the build if --execute option given, otherwise print out compile script path -# if execute: -# fremakeBuild.run() -# else: -# print("Compile script created at "+ bldDir+"/compile.sh\n\n") + fremakeBuildList.append(fremakeBuild) + ## Run the build if --execute option given, otherwise print out compile script path + if execute: + fremakeBuild.run() else: if force_compile or force_checkout: # Remove compile script os.remove(bld_dir + "/compile.sh") # Re-create compile script print("\nRe-creating the compile script...") - compile_script_write_steps(yaml_obj = fremake_yaml, - build_list = fremakeBuildList, - #exp_name = fremake_yaml["experiment"], + fremakeBuild = compile_script_write_steps(yaml_obj = fremake_yaml, mkTemplate = mkTemplate, src_dir = src_dir, bld_dir = bld_dir, @@ -253,12 +248,14 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec modules = modules, modulesInit = modulesInit, jobs = jobs) -# if execute: -# fremakeBuild.run() -# else: -# print("Compile script created at "+ bldDir+"/compile.sh\n\n") + fremakeBuildList.append(fremakeBuild) + if execute: + fremakeBuild.run() else: print("Compile script PREVIOUSLY created here: " + bld_dir + "/compile.sh" + "\n") + if execute: + subprocess.run(args=[bld_dir+"/compile.sh"], check=True) + ##TO-DO: log file here else: ###################### container stuff below ####################################### ## Run the checkout script @@ -303,6 +300,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec #if is doesn't exist, write curr_dir = os.getcwd() if not os.path.exists(f"{curr_dir}/Dockerfile"): + print("Creating Dockerfile and build script...") dockerfile_write_steps(yaml_obj = fremake_yaml, makefile_obj = freMakefile, img = image, From a4894af692644d813147faee79e19707fd016071 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 26 Dec 2024 13:04:02 -0500 Subject: [PATCH 20/39] #221 Add arguments for `force-checkout`,`force-compile`, and `force-dockerfile` --- fre/make/tests/test_run_fremake.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/fre/make/tests/test_run_fremake.py b/fre/make/tests/test_run_fremake.py index 4447acf6..d17e676b 100644 --- a/fre/make/tests/test_run_fremake.py +++ b/fre/make/tests/test_run_fremake.py @@ -17,7 +17,7 @@ YAMLDIR = "fre/make/tests/null_example" YAMLFILE = "null_model.yaml" YAMLPATH = f"{YAMLDIR}/{YAMLFILE}" -PLATFORM = [ "ci.gnu" ] +PLATFORM = ["ci.gnu"] CONTAINER_PLATFORM = ["hpcme.2023"] TARGET = ["debug"] BADOPT = ["foo"] @@ -51,24 +51,24 @@ def test_platformyaml_exists(): @pytest.mark.xfail() def test_bad_platform_option(): ''' test run-fremake with a invalid platform option''' - run_fremake_script.fremake_run(YAMLPATH, BADOPT, TARGET, False, 1, False, False, VERBOSE) + run_fremake_script.fremake_run(YAMLPATH, BADOPT, TARGET, False, 1, False, False, VERBOSE, False, False, False) @pytest.mark.xfail() def test_bad_target_option(): ''' test run-fremake with a invalid target option''' - run_fremake_script.fremake_run(YAMLPATH, PLATFORM, BADOPT, False, 1, False, False, VERBOSE) + run_fremake_script.fremake_run(YAMLPATH, PLATFORM, BADOPT, False, 1, False, False, VERBOSE, False, False, False) @pytest.mark.xfail() def test_bad_yamlpath_option(): ''' test run-fremake with a invalid target option''' - run_fremake_script.fremake_run(BADOPT[0], PLATFORM, TARGET, False, 1, False, False, VERBOSE) + run_fremake_script.fremake_run(BADOPT[0], PLATFORM, TARGET, False, 1, False, False, VERBOSE, False, False, False) # tests script/makefile creation without executing (serial compile) # first test runs the run-fremake command, subsequent tests check for creation of scripts def test_run_fremake_serial(): ''' run fre make with run-fremake subcommand and build the null model experiment with gnu''' os.environ["TEST_BUILD_DIR"] = SERIAL_TEST_PATH - run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, False, 1, False, False, VERBOSE) + run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, False, 1, False, False, VERBOSE, False, False, False) def test_run_fremake_compile_script_creation_serial(): ''' check for compile script creation from previous test ''' @@ -86,7 +86,7 @@ def test_run_fremake_makefile_creation_serial(): def test_run_fremake_multijob(): ''' run fre make with run-fremake subcommand and build the null model experiment with gnu''' os.environ["TEST_BUILD_DIR"] = MULTIJOB_TEST_PATH - run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, True, 4, True, False, VERBOSE) + run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, True, 4, True, False, VERBOSE, False, False, False) def test_run_fremake_compile_script_creation_multijob(): ''' check for compile script creation from previous test ''' @@ -103,7 +103,7 @@ def test_run_fremake_makefile_creation_multijob(): # tests container build script/makefile/dockerfile creation def test_run_fremake_container(): '''run run-fremake with options for containerized build''' - run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, False, 1, True, False, VERBOSE) + run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, False, 1, True, False, VERBOSE, False, False, False) def test_run_fremake_build_script_creation_container(): ''' checks container build script creation from previous test ''' @@ -126,16 +126,15 @@ def test_run_fremake_run_script_creation_container(): assert Path(f"tmp/{CONTAINER_PLATFORM[0]}/execrunscript.sh").exists() # tests for builds with multiple targets - def test_run_fremake_bad_target(): ''' checks invalid target returns an error ''' os.environ["TEST_BUILD_DIR"] = MULTITARGET_TEST_PATH - result = runner.invoke(fre.fre, args=["make", "run-fremake", "-y", YAMLPATH, "-p", PLATFORM[0], "-t", "prod-repro"]) + result = runner.invoke(fre.fre, args=["make", "run-fremake", "-y", YAMLPATH, "-p", PLATFORM, "-t", "prod-repro"]) assert result.exit_code == 1 def test_run_fremake_multiple_targets(): ''' passes all valid targets for a build ''' - result = runner.invoke(fre.fre, args=["make", "run-fremake", "-y", YAMLPATH, "-p", PLATFORM[0], "-t", \ + result = runner.invoke(fre.fre, args=["make", "run-fremake", "-y", YAMLPATH, "-p", PLATFORM, "-t", \ "debug", "-t", "prod", "-t", "repro", "-t", "debug-openmp", "-t",\ "prod-openmp", "-t", "repro-openmp"]) assert result.exit_code == 0 From 18f49ea8a94c72bb3632c2bdeaaa092c9fa24df1 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 26 Dec 2024 14:06:00 -0500 Subject: [PATCH 21/39] #221 Fix arguments and snake case --- fre/make/run_fremake_script.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index 28492343..0fc06742 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -46,12 +46,12 @@ def compile_script_write_steps(yaml_obj,mkTemplate,src_dir,bld_dir,target,module """ ## Create a list of compile scripts to run in parallel fremakeBuild = buildBaremetal.buildBaremetal(exp = yaml_obj["experiment"], - mkTemplatePath = platform["mkTemplate"], + mkTemplatePath = mkTemplate, srcDir = src_dir, bldDir = bld_dir, target = target, - modules = platform["modules"], - modulesInit = platform["modulesInit"], + modules = modules, + modulesInit = modulesInit, jobs = jobs) for c in yaml_obj['src']: fremakeBuild.writeBuildComponents(c) @@ -60,7 +60,7 @@ def compile_script_write_steps(yaml_obj,mkTemplate,src_dir,bld_dir,target,module return fremakeBuild -def dockerfile_write_steps(yaml_obj,makefile_obj,img,run_env,target,td,cr,cb,cd): +def dockerfile_write_steps(yaml_obj,makefile_obj,img,run_env,target,mkTemplate,td,cr,cb,cd): """ Go through steps to create Dockerfile and container build script. """ @@ -68,7 +68,8 @@ def dockerfile_write_steps(yaml_obj,makefile_obj,img,run_env,target,td,cr,cb,cd) exp = yaml_obj["experiment"], libs = yaml_obj["container_addlibs"], RUNenv = run_env, - target = target) + target = target, + mkTemplate = mkTemplate) dockerBuild.writeDockerfileCheckout("checkout.sh", td+"/checkout.sh") dockerBuild.writeDockerfileMakefile(makefile_obj.getTmpDir() + "/Makefile", @@ -140,7 +141,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec raise ValueError(f'{platformName} does not exist in ' f'{model_yaml.combined.get("compile").get("platformYaml")}') - platform = modelYaml.platforms.getPlatformFromName(platformName) + platform = model_yaml.platforms.getPlatformFromName(platformName) ## Create the checkout script if not platform["container"]: @@ -185,7 +186,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec else: raise ValueError (platformName + " does not exist in " + model_yaml.platformsfile) - platform = modelYaml.platforms.getPlatformFromName(platformName) + # platform = model_yaml.platforms.getPlatformFromName(platformName) ## Make the source directory based on the modelRoot and platform src_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/src" @@ -252,10 +253,10 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec ###################### container stuff below ####################################### ## Run the checkout script # image="hpc-me-intel:2021.1.1" - image=modelYaml.platforms.getContainerImage(platformName) - srcDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/src" - bldDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/exec" - tmpDir = "tmp/"+platformName + image=model_yaml.platforms.getContainerImage(platformName) + src_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/src" + bld_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/exec" + tmp_dir = "tmp/"+platformName ## Create the checkout script if not os.path.exists(tmp_dir+"/checkout.sh"): @@ -299,6 +300,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec img = image, run_env = platform["RUNenv"], target = target, + mkTemplate = platform["mkTemplate"], td = tmp_dir, cr = platform["containerRun"], cb = platform["containerBuild"], @@ -317,6 +319,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec run_env = platform["RUNenv"], target = target, td = tmp_dir, + mkTemplate = platform["mkTemplate"], cr = platform["containerRun"], cb = platform["containerBuild"], cd = curr_dir) From 6e35a7af385fdb4b05ff32485fc7af2a16f41cda Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 26 Dec 2024 14:06:37 -0500 Subject: [PATCH 22/39] #221 Update multi-target test --- fre/make/tests/test_run_fremake.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fre/make/tests/test_run_fremake.py b/fre/make/tests/test_run_fremake.py index d17e676b..2fc21c19 100644 --- a/fre/make/tests/test_run_fremake.py +++ b/fre/make/tests/test_run_fremake.py @@ -134,22 +134,28 @@ def test_run_fremake_bad_target(): def test_run_fremake_multiple_targets(): ''' passes all valid targets for a build ''' - result = runner.invoke(fre.fre, args=["make", "run-fremake", "-y", YAMLPATH, "-p", PLATFORM, "-t", \ - "debug", "-t", "prod", "-t", "repro", "-t", "debug-openmp", "-t",\ - "prod-openmp", "-t", "repro-openmp"]) - assert result.exit_code == 0 + os.environ["TEST_BUILD_DIR"] = MULTITARGET_TEST_PATH + run_fremake_script.fremake_run(YAMLPATH, PLATFORM, targets, True, 4, True, False, VERBOSE, False, False, False) +# result = runner.invoke(fre.fre, args=["make", "run-fremake", "-y", YAMLPATH, "-p", PLATFORM, "-t", \ +# "debug", "-t", "prod", "-t", "repro", "-t", "debug-openmp", "-t",\ +# "prod-openmp", "-t", "repro-openmp"]) +# assert result.exit_code == 0 +# assert (Path(f"{MULTITARGET_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{t}/exec/compile.sh").exists() def test_run_fremake_compile_script_creation_multitarget(): ''' check compile scripts for all targets exist from previous test''' + os.environ["TEST_BUILD_DIR"] = MULTITARGET_TEST_PATH for t in targets: assert Path(f"{MULTITARGET_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{t}/exec/compile.sh").exists() def test_run_fremake_checkout_script_creation_multitarget(): ''' check for checkout script creation for mulit-target build''' ''' check checkout script exists from previous test''' + os.environ["TEST_BUILD_DIR"] = MULTITARGET_TEST_PATH assert Path(f"{MULTITARGET_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/src/checkout.sh").exists() def test_run_fremake_makefile_creation_multitarget(): ''' check for makefile creation from previous test ''' + os.environ["TEST_BUILD_DIR"] = MULTITARGET_TEST_PATH for t in targets: assert Path(f"{MULTITARGET_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{t}/exec/Makefile").exists() From b3a2482931cdf391367e95db64a41a0710818cae Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 26 Dec 2024 14:23:40 -0500 Subject: [PATCH 23/39] #221 Add options for `force-checkout`, `force-compile`, and `force-dockerfile` --- fre/make/tests/compilation/test_run_fremake_builds.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/make/tests/compilation/test_run_fremake_builds.py b/fre/make/tests/compilation/test_run_fremake_builds.py index 9fe6169e..9ed4dca2 100644 --- a/fre/make/tests/compilation/test_run_fremake_builds.py +++ b/fre/make/tests/compilation/test_run_fremake_builds.py @@ -34,20 +34,20 @@ def test_run_fremake_serial_compile(): ''' run fre make with run-fremake subcommand and build the null model experiment with gnu''' os.environ["TEST_BUILD_DIR"] = SERIAL_TEST_PATH - run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, False, 1, False, True, VERBOSE) + run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, False, 1, False, True, VERBOSE, False, False, False) assert Path(f"{SERIAL_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists() # same test with a parallel build def test_run_fremake_multijob_compile(): ''' test run-fremake parallel compile with gnu''' os.environ["TEST_BUILD_DIR"] = MULTIJOB_TEST_PATH - run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, True, 4, False, True, VERBOSE) + run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, True, 4, False, True, VERBOSE, False, False, False) assert Path(f"{MULTIJOB_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists() # containerized build @pytest.mark.skip(reason="podman fails to pull image base on CI runner") def test_run_fremake_container_build(): ''' checks image creation for the container build''' - run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, False, 1, True, True, VERBOSE) + run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, False, 1, True, True, VERBOSE, False, False, False) assert Path("null_model_full-debug.sif").exists() From d1d2c1d630637a57783fca71da95682133ff96c7 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 7 Jan 2025 17:20:21 -0500 Subject: [PATCH 24/39] #221 Update create_compile_script.py --- fre/make/create_compile_script.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index bafbfa2a..2cd96d2b 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -16,12 +16,12 @@ def compile_script_write_steps(yaml_obj,mkTemplate,src_dir,bld_dir,target,module """ ## Create a list of compile scripts to run in parallel fremakeBuild = buildBaremetal.buildBaremetal(exp = yaml_obj["experiment"], - mkTemplatePath = platform["mkTemplate"], + mkTemplatePath = mkTemplate, srcDir = src_dir, bldDir = bld_dir, target = target, - modules = platform["modules"], - modulesInit = platform["modulesInit"], + modules = modules, + modulesInit = modulesInit, jobs = jobs) for c in yaml_obj['src']: fremakeBuild.writeBuildComponents(c) @@ -84,8 +84,8 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_ ## Check for type of build if platform["container"] is False: baremetalRun = True - bldDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/" + platformName + "-" + target.gettargetName() + "/exec" - os.system("mkdir -p " + bldDir) + bld_dir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/" + platformName + "-" + target.gettargetName() + "/exec" + os.system("mkdir -p " + bld_dir) if not os.path.exists(bld_dir+"/compile.sh"): print("\nCreating the compile script...") fremakeBuild = compile_script_write_steps(yaml_obj = fremakeYaml, From 2a45bf18bbade9eee94dfcdbe7b3236004fb13a7 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 15 Jan 2025 14:33:11 -0500 Subject: [PATCH 25/39] #221 Add tests for `force-checkout`, `force-compile`, and `force-dockerfile` --- fre/make/create_checkout_script.py | 2 +- fre/make/run_fremake_script.py | 2 +- fre/make/tests/test_run_fremake.py | 68 +++++++++++++++++++++++++----- 3 files changed, 60 insertions(+), 12 deletions(-) diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py index b86b8a08..25ab977d 100644 --- a/fre/make/create_checkout_script.py +++ b/fre/make/create_checkout_script.py @@ -145,7 +145,7 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v os.remove(tmp_dir+"/checkout.sh") # Create the checkout script - print("Re-creating checkout script...") + print("Re-creating the checkout script...") container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc) else: print("\nCheckout script PREVIOUSLY created in "+ tmp_dir + "/checkout.sh" + "\n") diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index 76029c4f..fb258972 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -269,7 +269,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec os.remove(tmp_dir+"/checkout.sh") # Create the checkout script - print("Re-creating checkout script...") + print("Re-creating the checkout script...") container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc) else: print("\nCheckout script PREVIOUSLY created and run here: "+ tmp_dir + "/checkout.sh") diff --git a/fre/make/tests/test_run_fremake.py b/fre/make/tests/test_run_fremake.py index 2fc21c19..831736b9 100644 --- a/fre/make/tests/test_run_fremake.py +++ b/fre/make/tests/test_run_fremake.py @@ -3,11 +3,8 @@ import os from shutil import rmtree from pathlib import Path - from click.testing import CliRunner - import pytest - from fre import fre from fre.make import run_fremake_script @@ -82,6 +79,30 @@ def test_run_fremake_makefile_creation_serial(): ''' check for makefile creation from previous test ''' assert Path(f"{SERIAL_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/Makefile").exists() +def test_run_fremake_serial_force_checkout(capfd): + '''run run-fremake with options for serial build with force-checkout''' + run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, False, 1, False, False, VERBOSE, True, False, False) + + #Capture output + out,err=capfd.readouterr() + if "Re-creating the checkout script" in out and "Re-creating the compile script" in out: + assert Path(f"{SERIAL_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/src/checkout.sh").exists() + assert Path(f"{SERIAL_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/Makefile").exists() + assert Path(f"{SERIAL_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/compile.sh").exists() + else: + assert False + +def test_run_fremake_serial_force_compile(capfd): + '''run run-fremake with options for serial build with force-compile''' + run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, False, 1, False, False, VERBOSE, False, True, False) + + #Capture output + out,err=capfd.readouterr() + if "Re-creating the compile script" in out: + assert Path(f"{SERIAL_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/compile.sh").exists() + else: + assert False + # same tests with multijob compile and non-parallel-checkout options enabled def test_run_fremake_multijob(): ''' run fre make with run-fremake subcommand and build the null model experiment with gnu''' @@ -125,6 +146,36 @@ def test_run_fremake_run_script_creation_container(): ''' checks (internal) container run script creation from previous test ''' assert Path(f"tmp/{CONTAINER_PLATFORM[0]}/execrunscript.sh").exists() +def test_run_fremake_container_force_checkout(capfd): + '''run run-fremake with options for containerized build with force-checkout option''' + run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, False, 1, True, False, VERBOSE, True, False, False) + + #Capture output + out,err=capfd.readouterr() + print(out) + + if "Re-creating the checkout script" in out: + assert Path(f"tmp/{CONTAINER_PLATFORM[0]}/checkout.sh").exists() + assert Path(f"tmp/{CONTAINER_PLATFORM[0]}/Makefile").exists() + assert Path("Dockerfile").exists() + assert Path("createContainer.sh").exists() + assert Path(f"tmp/{CONTAINER_PLATFORM[0]}/execrunscript.sh").exists() + else: + assert False + +def test_run_fremake_container_force_dockerfile(capfd): + '''run run-fremake with options for containerized build with force-dockerfile option''' + run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, False, 1, True, False, VERBOSE, False, False, True) + + #Capture output + out,err=capfd.readouterr() + if "Re-creating Dockerfile" in out: + assert Path("Dockerfile").exists() + assert Path("createContainer.sh").exists() + assert Path(f"tmp/{CONTAINER_PLATFORM[0]}/execrunscript.sh").exists() + else: + assert False + # tests for builds with multiple targets def test_run_fremake_bad_target(): ''' checks invalid target returns an error ''' @@ -134,13 +185,10 @@ def test_run_fremake_bad_target(): def test_run_fremake_multiple_targets(): ''' passes all valid targets for a build ''' - os.environ["TEST_BUILD_DIR"] = MULTITARGET_TEST_PATH - run_fremake_script.fremake_run(YAMLPATH, PLATFORM, targets, True, 4, True, False, VERBOSE, False, False, False) -# result = runner.invoke(fre.fre, args=["make", "run-fremake", "-y", YAMLPATH, "-p", PLATFORM, "-t", \ -# "debug", "-t", "prod", "-t", "repro", "-t", "debug-openmp", "-t",\ -# "prod-openmp", "-t", "repro-openmp"]) -# assert result.exit_code == 0 -# assert (Path(f"{MULTITARGET_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{t}/exec/compile.sh").exists() + result = runner.invoke(fre.fre, args=["make", "run-fremake", "-y", YAMLPATH, "-p", PLATFORM[0], "-t", \ + "debug", "-t", "prod", "-t", "repro", "-t", "debug-openmp", "-t",\ + "prod-openmp", "-t", "repro-openmp"]) + assert result.exit_code == 0 def test_run_fremake_compile_script_creation_multitarget(): ''' check compile scripts for all targets exist from previous test''' From caa86e884329774eb2278311c3e3324ed28e2b4c Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 21 Jan 2025 11:57:44 -0500 Subject: [PATCH 26/39] #221 Correct camel/snake case - these changes will be done in another issue --- fre/make/run_fremake_script.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index 348637f0..ba691d15 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -182,27 +182,27 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec for platformName in plist: for targetName in tlist: target = targetfre.fretarget(targetName) - if model_yaml.platforms.hasPlatform(platformName): + if modelYaml.platforms.hasPlatform(platformName): pass else: - raise ValueError (platformName + " does not exist in " + model_yaml.platformsfile) + raise ValueError (platformName + " does not exist in " + modelYaml.platformsfile) platform = modelYaml.platforms.getPlatformFromName(platformName) ## Make the source directory based on the modelRoot and platform - src_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/src" + src_dir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/src" ## Check for type of build if not platform["container"]: baremetalRun = True ## Make the build directory based on the modelRoot, the platform, and the target - bld_dir = f'{platform["modelRoot"]}/{fremake_yaml["experiment"]}/' + \ + bld_dir = f'{platform["modelRoot"]}/{fremakeYaml["experiment"]}/' + \ f'{platformName}-{target.gettargetName()}/exec' os.system("mkdir -p " + bld_dir) ## Create the Makefile - freMakefile = makefilefre.makefile(exp = fremake_yaml["experiment"], - libs = fremake_yaml["baremetal_linkerflags"], + freMakefile = makefilefre.makefile(exp = fremakeYaml["experiment"], + libs = fremakeYaml["baremetal_linkerflags"], srcDir = src_dir, bldDir = bld_dir, mkTemplatePath = platform["mkTemplate"]) @@ -216,7 +216,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec if not os.path.exists(bld_dir+"/compile.sh"): ## Create a list of compile scripts to run in parallel print("\nCreating the compile script...") - fremakeBuild = compile_script_write_steps(yaml_obj = fremake_yaml, + fremakeBuild = compile_script_write_steps(yaml_obj = fremakeYaml, mkTemplate = platform["mkTemplate"], src_dir = src_dir, bld_dir = bld_dir, @@ -234,7 +234,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec os.remove(bld_dir + "/compile.sh") # Re-create compile script print("\nRe-creating the compile script...") - fremakeBuild = compile_script_write_steps(yaml_obj = fremake_yaml, + fremakeBuild = compile_script_write_steps(yaml_obj = fremakeYaml, mkTemplate = platform["mkTemplate"], src_dir = src_dir, bld_dir = bld_dir, @@ -254,16 +254,16 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec ###################### container stuff below ####################################### ## Run the checkout script # image="hpc-me-intel:2021.1.1" - image=model_yaml.platforms.getContainerImage(platformName) - src_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/src" - bld_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/exec" + image=modelYaml.platforms.getContainerImage(platformName) + src_dir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/src" + bld_dir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/exec" tmp_dir = "tmp/"+platformName ## Create the checkout script if not os.path.exists(tmp_dir+"/checkout.sh"): # Create the checkout script print("Creating checkout script...") - container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc) + container_checkout_write_steps(modelYaml,src_dir,tmp_dir,jobs,pc) else: if force_checkout: # Remove the checkout script @@ -272,14 +272,14 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec # Create the checkout script print("Re-creating the checkout script...") - container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc) + container_checkout_write_steps(modelYaml,src_dir,tmp_dir,jobs,pc) else: print("\nCheckout script PREVIOUSLY created and run here: "+ tmp_dir + "/checkout.sh") ## Create the makefile ### Should this even be a separate class from "makefile" in makefilefre? ~ ejs - freMakefile = makefilefre.makefileContainer(exp = fremake_yaml["experiment"], - libs = fremake_yaml["container_addlibs"], + freMakefile = makefilefre.makefileContainer(exp = fremakeYaml["experiment"], + libs = fremakeYaml["container_addlibs"], srcDir = src_dir, bldDir = bld_dir, mkTemplatePath = platform["mkTemplate"], @@ -296,7 +296,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec curr_dir = os.getcwd() if not os.path.exists(f"{curr_dir}/Dockerfile"): print("Creating Dockerfile and build script...") - dockerfile_write_steps(yaml_obj = fremake_yaml, + dockerfile_write_steps(yaml_obj = fremakeYaml, makefile_obj = freMakefile, img = image, run_env = platform["RUNenv"], @@ -314,7 +314,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec # Create the checkout script print("Re-creating Dockerfile...") - dockerfile_write_steps(yaml_obj = fremake_yaml, + dockerfile_write_steps(yaml_obj = fremakeYaml, makefile_obj = freMakefile, img = image, run_env = platform["RUNenv"], From ce64065fe08dafddec541f3ef5856f8318630b25 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 21 Jan 2025 14:48:01 -0500 Subject: [PATCH 27/39] #221 Re-create combined yaml if `force-checkout` defined --- fre/make/create_checkout_script.py | 15 +++++++++++---- fre/make/create_compile_script.py | 1 + fre/make/create_docker_script.py | 1 + fre/make/run_fremake_script.py | 15 +++++++++------ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py index 20cb2161..9295cba7 100644 --- a/fre/make/create_checkout_script.py +++ b/fre/make/create_checkout_script.py @@ -7,6 +7,7 @@ import logging import sys import shutil +from pathlib import Path import fre.yamltools.combine_yamls as cy from .gfdlfremake import varsfre, yamlfre, checkout, targetfre @@ -64,10 +65,16 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v plist = platform tlist = target - # Combine model, compile, and platform yamls - # Default behavior - combine yamls / rewrite combined yaml - comb = cy.init_compile_yaml(yml,platform,target) - full_combined = cy.get_combined_compileyaml(comb) + # If force-checkout defined: re-combine model, compile, and platform yamls + if force_checkout: + print("Re-combine yaml files") + comb = cy.init_compile_yaml(yml,platform,target) + full_combined = cy.get_combined_compileyaml(comb) + else: + ## If combined yaml exists, note message of its existence + ## If combined yaml does not exist, combine model, compile, and platform yamls + combined = Path(f"combined-{name}.yaml") + full_combined = cy.combined_compile_existcheck(combined,yml,platform,target) ## Get the variables in the model yaml fre_vars = varsfre.frevars(full_combined) diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index 1aa18621..d61126b5 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -33,6 +33,7 @@ def compile_script_write_steps(yaml_obj,mkTemplate,src_dir,bld_dir,target,module def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_compile): """ + Create the compile script """ # Define variables yml = yamlfile diff --git a/fre/make/create_docker_script.py b/fre/make/create_docker_script.py index 1dea95df..f312655c 100644 --- a/fre/make/create_docker_script.py +++ b/fre/make/create_docker_script.py @@ -34,6 +34,7 @@ def dockerfile_write_steps(yaml_obj,img,run_env,target,td,cr,cb,cd): def dockerfile_create(yamlfile,platform,target,execute,force_dockerfile): """ + Create the Dockerfile """ srcDir="src" checkoutScriptName = "checkout.sh" diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index ba691d15..8a2bee66 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -112,12 +112,15 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec plist = platform tlist = target - # Combined compile yaml file - combined = Path(f"combined-{name}.yaml") - - ## If combined yaml exists, note message of its existence - ## If combined yaml does not exist, combine model, compile, and platform yamls - full_combined = cy.combined_compile_existcheck(combined, yml, platform, target) + # If force-checkout defined: re-combine model, compile, and platform yamls + if force_checkout: + comb = cy.init_compile_yaml(yml,platform,target) + full_combined = cy.get_combined_compileyaml(comb) + else: + ## If combined yaml exists, note message of its existence + ## If combined yaml does not exist, combine model, compile, and platform yamls + combined = Path(f"combined-{name}.yaml") + full_combined = cy.combined_compile_existcheck(combined,yml,platform,target) ## Get the variables in the model yaml freVars = varsfre.frevars(full_combined) From bca152fc243f60536d40af2d0b1f725a7625bf21 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 21 Jan 2025 15:19:52 -0500 Subject: [PATCH 28/39] #221 Fix variables --- fre/make/create_docker_script.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/fre/make/create_docker_script.py b/fre/make/create_docker_script.py index f312655c..bb36f919 100644 --- a/fre/make/create_docker_script.py +++ b/fre/make/create_docker_script.py @@ -9,7 +9,7 @@ from .gfdlfremake import varsfre, targetfre, yamlfre, buildDocker import fre.yamltools.combine_yamls as cy -def dockerfile_write_steps(yaml_obj,img,run_env,target,td,cr,cb,cd): +def dockerfile_write_steps(yaml_obj,img,run_env,target,mkTemplate,td,cr,cb,cd): """ Go through steps to write the Dockerfile """ @@ -17,7 +17,8 @@ def dockerfile_write_steps(yaml_obj,img,run_env,target,td,cr,cb,cd): exp = yaml_obj["experiment"], libs = yaml_obj["container_addlibs"], RUNenv = run_env, - target = target) + target = target, + mkTemplate = mkTemplate) dockerBuild.writeDockerfileCheckout("checkout.sh", td+"/checkout.sh") dockerBuild.writeDockerfileMakefile(td+"/Makefile", td+"/linkline.sh") @@ -81,10 +82,10 @@ def dockerfile_create(yamlfile,platform,target,execute,force_dockerfile): curr_dir = os.getcwd() if not os.path.exists(f"{curr_dir}/Dockerfile"): dockerfile_write_steps(yaml_obj = fremakeYaml, - #makefile_obj = freMakefile, img = image, run_env = platform["RUNenv"], target = targetObject, + mkTemplate = platform["mkTemplate"], td = tmp_dir, cr = platform["containerRun"], cb = platform["containerBuild"], @@ -98,13 +99,13 @@ def dockerfile_create(yamlfile,platform,target,execute,force_dockerfile): # Create the checkout script print("Re-creating Dockerfile...") dockerfile_write_steps(yaml_obj = fremakeYaml, -# makefile_obj = freMakefile, img = image, - run_env = RUNenv, + run_env = platform["RUNenv"], target = targetObject, + mkTemplate = platform["mkTemplate"], td = tmp_dir, - cr = containerRun, - cb = containerBuild, + cr = platform["containerRun"], + cb = platform["containerBuild"], cd = curr_dir) else: print(f"Dockerfile PREVIOUSLY created here: {curr_dir}/Dockerfile") From 5b2d0374faeb94b719b451f7f10d0112890d1d45 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 21 Jan 2025 15:20:11 -0500 Subject: [PATCH 29/39] #221 Fix print statement --- fre/make/create_makefile_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/make/create_makefile_script.py b/fre/make/create_makefile_script.py index e1f80b8f..79564250 100644 --- a/fre/make/create_makefile_script.py +++ b/fre/make/create_makefile_script.py @@ -77,7 +77,7 @@ def makefile_create(yamlfile, platform, target): for c in fremakeYaml['src']: freMakefile.addComponent(c['component'], c['requires'], c['makeOverrides']) freMakefile.writeMakefile() - print("\nMakefile created here: ./" + tmpDir + "/Makefile" + "\n") + print("\nMakefile created here: " + tmpDir + "/Makefile" + "\n") if __name__ == "__main__": makefile_create() From 621ba950de9804a989244e994ddd064a11f4ae8a Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 21 Jan 2025 15:25:20 -0500 Subject: [PATCH 30/39] #221 Add force options --- fre/make/tests/test_create_compile.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/fre/make/tests/test_create_compile.py b/fre/make/tests/test_create_compile.py index fae1781d..d2eb85f9 100644 --- a/fre/make/tests/test_create_compile.py +++ b/fre/make/tests/test_create_compile.py @@ -68,7 +68,8 @@ def test_compile_creation(): jobs = 4, parallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile=False) # Check for creation of compile script assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists() @@ -92,7 +93,8 @@ def test_compile_executable_failure(): jobs = 4, parallel = 1, execute = True, - verbose = False) + verbose = False, + force_compile=False) # Check for creation of compile script, FMS directory, # log.compile file, the executable @@ -119,7 +121,8 @@ def test_bad_platform(): jobs = 4, parallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile=False) def test_bad_platform_compilelog(): """ @@ -139,7 +142,8 @@ def test_bad_platform_compilelog(): jobs = 4, parallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile=False) except: assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{BAD_PLATFORM}-{TARGET}/exec/log.compile") @@ -161,7 +165,8 @@ def test_bad_target(): jobs = 4, parallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile=False) def test_bad_target_compilelog(): """ @@ -181,7 +186,8 @@ def test_bad_target_compilelog(): jobs = 4, parallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile=False) except: assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{BAD_PLATFORM}-{TARGET}/exec/log.compile") @@ -201,7 +207,8 @@ def test_multi_target(): jobs = 4, parallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile=False) assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{PLATFORM[0]}-{MULTI_TARGET[0]}/exec/compile.sh").exists() assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{PLATFORM[0]}-{MULTI_TARGET[1]}/exec/compile.sh").exists() From 923d63c2631e4683465d1814c17a1752f9e05e60 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 21 Jan 2025 15:41:37 -0500 Subject: [PATCH 31/39] #221 Remove path check - this shouldn't exist anyway in this failure --- fre/make/tests/test_create_compile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fre/make/tests/test_create_compile.py b/fre/make/tests/test_create_compile.py index d2eb85f9..b983ca07 100644 --- a/fre/make/tests/test_create_compile.py +++ b/fre/make/tests/test_create_compile.py @@ -99,7 +99,6 @@ def test_compile_executable_failure(): # Check for creation of compile script, FMS directory, # log.compile file, the executable assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists() - assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/FMS").is_dir() assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/log.compile").exists() assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/null_model_full.x").exists() == False From a0d0bc40b55c7b414d022505dacd6622414e7d28 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 22 Jan 2025 13:47:02 -0500 Subject: [PATCH 32/39] #221 Add tests for `force-compile` option --- fre/make/tests/test_create_compile.py | 35 ++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/fre/make/tests/test_create_compile.py b/fre/make/tests/test_create_compile.py index b983ca07..e963b642 100644 --- a/fre/make/tests/test_create_compile.py +++ b/fre/make/tests/test_create_compile.py @@ -73,7 +73,36 @@ def test_compile_creation(): # Check for creation of compile script assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists() -def test_compile_executable_failure(): +def test_compile_creation_force_compile(capfd): + """ + Check for the re-creation of the compile script + if --force-compile is defined + """ + # Set environment variable for use in ci.gnu platform + os.environ["TEST_BUILD_DIR"] = OUT + + plat = PLATFORM[0] + targ = TARGET[0] + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" + + # Create the compile script + create_compile_script.compile_create(yamlfile = yamlfile_path, + platform = PLATFORM, + target = TARGET, + jobs = 4, + parallel = 1, + execute = False, + verbose = False, + force_compile=True) + + #Capture output + out,err=capfd.readouterr() + if "Re-creating the compile script" in out: + assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists() + else: + assert False + +def test_compile_executablefail_compilelog(): """ Check for the failure in execution of the compile script. Fails because it would need the makefile and checked out @@ -96,11 +125,11 @@ def test_compile_executable_failure(): verbose = False, force_compile=False) - # Check for creation of compile script, FMS directory, + # Check for creation of compile script, # log.compile file, the executable assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists() - assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/log.compile").exists() assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/null_model_full.x").exists() == False + assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/log.compile").exists() @pytest.mark.xfail(raises=ValueError) def test_bad_platform(): From 6411f2b29c1bc93b1e0fbf90cce9ce348dc35303 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 22 Jan 2025 14:18:00 -0500 Subject: [PATCH 33/39] #221 Use `run` function directly - instead of passing fremake object to run function, just pass compile script path to run - this allows an already created compile script to also run in parallel (if not being created or if no `force-compile` is used) --- fre/make/create_compile_script.py | 16 ++++--- fre/make/gfdlfremake/buildBaremetal.py | 62 ++++++++++---------------- fre/make/run_fremake_script.py | 23 ++++------ 3 files changed, 43 insertions(+), 58 deletions(-) diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index b8bdfffb..f68fe431 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -1,5 +1,5 @@ ''' -TODO: make docstring +Create the compile script to compile model ''' import os import sys @@ -28,8 +28,10 @@ def compile_script_write_steps(yaml_obj,mkTemplate,src_dir,bld_dir,target,module fremakeBuild.writeBuildComponents(c) fremakeBuild.writeScript() - print(" Compile script created in " + bld_dir + "/compile.sh" + "\n") - return fremakeBuild + print(" Compile script created here: " + bld_dir + "/compile.sh" + "\n") + compile_script = f"{bld_dir}/compile.sh" + + return compile_script def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_compile): """ @@ -100,6 +102,7 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_ modules = platform["modules"], modulesInit = platform["modulesInit"], jobs = jobs) + # Append the compile script created fremakeBuildList.append(fremakeBuild) else: if force_compile: @@ -115,14 +118,17 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_ modules = platform["modules"], modulesInit = platform["modulesInit"], jobs = jobs) + # Append the returned compile script created fremakeBuildList.append(fremakeBuild) else: - print("Compile script PREVIOUSLY created here: " + bld_dir + "/compile.sh" + "\n") + print("Compile script PREVIOUSLY created here: " + bld_dir + "/compile.sh" + "\n") + # Append the compile script + fremakeBuildList.append(f"{bld_dir}/compile.sh") if execute: if baremetalRun: pool = Pool(processes=nparallel) # Create a multiprocessing Pool - pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) # process data_inputs iterable with pool + pool.map(buildBaremetal.run,fremakeBuildList) # process data_inputs iterable with pool else: return diff --git a/fre/make/gfdlfremake/buildBaremetal.py b/fre/make/gfdlfremake/buildBaremetal.py index 3159baf6..c3a3458a 100644 --- a/fre/make/gfdlfremake/buildBaremetal.py +++ b/fre/make/gfdlfremake/buildBaremetal.py @@ -1,20 +1,33 @@ -''' - \date 2023 - \author Tom Robinson - \email thomas.robinson@noaa.gov - \description -''' - import subprocess import os -def fremake_parallel(fremakeBuildList): +def run(cs): """ - Brief: Called for parallel execution purposes. Runs the builds. + Brief: Run the build script Param: - - fremakeBuildList : fremakeBuild object list passes by pool.map + - cs : The created compile script """ - fremakeBuildList.run() + #command = [bld_dir, "compile.sh"] + command = cs + bld_dir = os.path.dirname(command) + + # Run compile script + p1 = subprocess.Popen(command, stdout=subprocess.PIPE,stderr=subprocess.STDOUT) + + # Direct output to log file as well + p2 = subprocess.Popen(["tee",bld_dir+"/log.compile"], stdin=p1.stdout) + + # Allow process1 to receive SIGPIPE is process2 exits + p1.stdout.close() + p2.communicate() + + # wait for process1 to finish before checking return code + p1.wait() + if p1.returncode != 0: + print(f"\nThere was an error running {bld_dir}/compile.sh") + print(f"Check the log file: {bld_dir}/log.compile") + else: + print(f"\nSuccessful run of {bld_dir}/compile.sh") class buildBaremetal(): """ @@ -125,30 +138,3 @@ def writeScript(self): # Make compile script executable os.chmod(self.bld+"/compile.sh", 0o744) - -## TODO run as a batch job on the login cluster - def run(self): - """ - Brief: Run the build script - Param: - - self : The dockerfile object - """ - command = [self.bld+"/compile.sh"] - - # Run compile script - p1 = subprocess.Popen(command, stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - - # Direct output to log file as well - p2 = subprocess.Popen(["tee",self.bld+"/log.compile"], stdin=p1.stdout) - - # Allow process1 to receive SIGPIPE is process2 exits - p1.stdout.close() - p2.communicate() - - # wait for process1 to finish before checking return code - p1.wait() - if p1.returncode != 0: - print(f"\nThere was an error running {self.bld}/compile.sh") - print(f"Check the log file: {self.bld}/log.compile") - else: - print(f"\nSuccessful run of {self.bld}/compile.sh") diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index 8a2bee66..51b28d21 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -58,7 +58,8 @@ def compile_script_write_steps(yaml_obj,mkTemplate,src_dir,bld_dir,target,module fremakeBuild.writeScript() print(" Compile script created here: " + bld_dir + "/compile.sh" + "\n") - return fremakeBuild + compile_script = f"{bld_dir}/compile.sh" + return compile_script def dockerfile_write_steps(yaml_obj,makefile_obj,img,run_env,target,mkTemplate,td,cr,cb,cd): """ @@ -228,9 +229,6 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec modulesInit = platform["modulesInit"], jobs = jobs) fremakeBuildList.append(fremakeBuild) - ## Run the build if --execute option given, otherwise print out compile script path - if execute: - fremakeBuild.run() else: if force_compile or force_checkout: # Remove compile script @@ -246,13 +244,9 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec modulesInit = platform["modulesInit"], jobs = jobs) fremakeBuildList.append(fremakeBuild) - if execute: - fremakeBuild.run() else: print("Compile script PREVIOUSLY created here: " + bld_dir + "/compile.sh" + "\n") - if execute: - subprocess.run(args=[bld_dir+"/compile.sh"], check=True) - ##TO-DO: log file here + fremakeBuildList.append(f"{bld_dir}/compile.sh") else: ###################### container stuff below ####################################### ## Run the checkout script @@ -340,12 +334,11 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec #buildDockerfile(fremakeYaml, image) if baremetalRun: - if __name__ == '__main__': - if execute: - # Create a multiprocessing Pool - pool = Pool(processes=nparallel) - # process data_inputs iterable with pool - pool.map(buildBaremetal.fremake_parallel, fremakeBuildList) + if execute: + # Create a multiprocessing Pool + pool = Pool(processes=nparallel) + # process data_inputs iterable with pool + pool.map(buildBaremetal.run, fremakeBuildList) if __name__ == "__main__": fremake_run() From 1e840ccb115b42f402bb080ae4907c136ee39338 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 28 Jan 2025 15:42:44 -0500 Subject: [PATCH 34/39] #221 Add force options for tests --- fre/make/tests/test_run_fremake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/make/tests/test_run_fremake.py b/fre/make/tests/test_run_fremake.py index 80bf6329..9ecf1749 100644 --- a/fre/make/tests/test_run_fremake.py +++ b/fre/make/tests/test_run_fremake.py @@ -150,7 +150,7 @@ def test_run_fremake_run_script_creation_container(): # tests container 2 stage build script/makefile/dockerfile creation def test_run_fremake_2stage_container(): '''run run-fremake with options for containerized build''' - run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLAT2, TARGET, False, 1, True, False, VERBOSE) + run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLAT2, TARGET, False, 1, True, False, VERBOSE, False, False, False) def test_run_fremake_2stage_build_script_creation_container(): ''' checks container build script creation from previous test ''' From 821a70c5fb39a39951a5c13c150cad54ca3d2cb8 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 28 Jan 2025 15:54:33 -0500 Subject: [PATCH 35/39] #221 Add comma --- fre/make/create_docker_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/make/create_docker_script.py b/fre/make/create_docker_script.py index 876df5c4..e997ec97 100644 --- a/fre/make/create_docker_script.py +++ b/fre/make/create_docker_script.py @@ -18,7 +18,7 @@ def dockerfile_write_steps(yaml_obj,img,run_env,target,mkTemplate,s2i,td,cr,cb,c libs = yaml_obj["container_addlibs"], RUNenv = run_env, target = target, - mkTemplate = mkTemplate + mkTemplate = mkTemplate, stage2base = s2i) dockerBuild.writeDockerfileCheckout("checkout.sh", td+"/checkout.sh") From bb3c966b3af9686a458cc006133eae028ea15fa4 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 28 Jan 2025 17:26:20 -0500 Subject: [PATCH 36/39] #221 Add print statement --- fre/make/run_fremake_script.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index 1d4bc878..c364b092 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -81,6 +81,7 @@ def dockerfile_write_steps(yaml_obj,makefile_obj,img,run_env,target,mkTemplate,s dockerBuild.writeDockerfileMkmf(c) dockerBuild.writeRunscript(run_env,cr,td+"/execrunscript.sh") + print(f" Runscript created here: {td}/execrunscript.sh") print(f" Dockerfile created here: {cd}") # Create build script for container @@ -173,14 +174,6 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec freCheckout.run() else: print("\nCheckout script PREVIOUSLY created and run here: "+ src_dir + "/checkout.sh") -# try: -# subprocess.run(args=[src_dir+"/checkout.sh"], check=True) -# except: -# print("\nThere was an error with the checkout script "+src_dir+"/checkout.sh.", -# "\nTry:\n", -# " - removing test folder: " + modelRoot + "\n", -# " - add --force-checkout option\n") -# raise fremakeBuildList = [] ## Loop through platforms and targets @@ -251,7 +244,6 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec else: ###################### container stuff below ####################################### ## Run the checkout script - # image="hpc-me-intel:2021.1.1" image=modelYaml.platforms.getContainerImage(platformName) stage2image = modelYaml.platforms.getContainer2base(platformName) src_dir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/src" @@ -329,7 +321,6 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec print(f"Dockerfile PREVIOUSLY created here: {curr_dir}/Dockerfile") print(f"Container build script created here: {curr_dir}/createContainer.sh\n") - # Execute if flag is given if execute: subprocess.run(args=[dockerBuild.userScriptPath], check=True) From 987f60c3b927a963c8a1e7d7e4a45d7141b4ca1e Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 28 Jan 2025 17:26:43 -0500 Subject: [PATCH 37/39] #221 Remove Dockerfile and buildScript before running `run-fremake` again --- fre/make/tests/test_run_fremake.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fre/make/tests/test_run_fremake.py b/fre/make/tests/test_run_fremake.py index 9ecf1749..3d8a7486 100644 --- a/fre/make/tests/test_run_fremake.py +++ b/fre/make/tests/test_run_fremake.py @@ -102,7 +102,7 @@ def test_run_fremake_serial_force_compile(capfd): if "Re-creating the compile script" in out: assert Path(f"{SERIAL_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/compile.sh").exists() else: - assert False + assert False # same tests with multijob compile and non-parallel-checkout options enabled def test_run_fremake_multijob(): @@ -150,6 +150,12 @@ def test_run_fremake_run_script_creation_container(): # tests container 2 stage build script/makefile/dockerfile creation def test_run_fremake_2stage_container(): '''run run-fremake with options for containerized build''' + # Without force-dockerfile or force-checkout option, clean files first + # or else it'll read that they exist already and not make the execrunscript.sh + if Path("Dockerfile").exists() or Path("createContainer.sh").exists(): + os.remove(Path("Dockerfile")) + os.remove(Path("createContainer.sh")) + run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLAT2, TARGET, False, 1, True, False, VERBOSE, False, False, False) def test_run_fremake_2stage_build_script_creation_container(): From 6e0ead35268f56616bae0f0741bf96b1da2aa869 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 31 Jan 2025 14:38:20 -0500 Subject: [PATCH 38/39] #221 Fix variable referenced --- fre/make/run_fremake_script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index 02b8efee..437821c8 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -21,7 +21,7 @@ def baremetal_checkout_write_steps(model_yaml,src_dir,jobs,pc): """ fre_checkout = checkout.checkout("checkout.sh",src_dir) fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc) - fre_checkout.finish(modelYaml.compile.getCompileYaml(),pc) + fre_checkout.finish(model_yaml.compile.getCompileYaml(),pc) # Make checkout script executable os.chmod(src_dir+"/checkout.sh", 0o744) @@ -35,7 +35,7 @@ def container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc): """ fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir) fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc) - fre_checkout.finish(modelYaml.compile.getCompileYaml(),pc) + fre_checkout.finish(model_yaml.compile.getCompileYaml(),pc) print(" Checkout script created here: " + tmp_dir + "/checkout.sh") return fre_checkout From baa1baad0573ecdbb919260988ba2933c79f3942 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 31 Jan 2025 16:32:40 -0500 Subject: [PATCH 39/39] #221 Add `force-dockerfile` argument --- fre/make/tests/test_create_dockerfile.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fre/make/tests/test_create_dockerfile.py b/fre/make/tests/test_create_dockerfile.py index 5d036e6b..c55d9636 100644 --- a/fre/make/tests/test_create_dockerfile.py +++ b/fre/make/tests/test_create_dockerfile.py @@ -42,30 +42,30 @@ def test_platformyaml_exists(): @pytest.mark.xfail() def test_bad_platform_option(): ''' test -fremake with a invalid platform option''' - create_docker_script.dockerfile_create(YAMLPATH, BADOPT, TARGET, False) + create_docker_script.dockerfile_create(YAMLPATH, BADOPT, TARGET, False, False) @pytest.mark.xfail() def test_bad_target_option(): ''' test create-dockerfile with a invalid target option''' - create_docker_script.dockerfile_create(YAMLPATH, PLATFORM, BADOPT, False) + create_docker_script.dockerfile_create(YAMLPATH, PLATFORM, BADOPT, False, False) @pytest.mark.xfail() def test_bad_yamlpath_option(): - ''' test create-dockerfile with a invalid target option''' - create_docker_script.dockerfile_create(BADOPT[0], PLATFORM, TARGET, False) + ''' test create-dockerfile with a invalid yaml option''' + create_docker_script.dockerfile_create(BADOPT[0], PLATFORM, TARGET, False, False) def test_no_op_platform(): '''test create-dockerfile will do nothing if non-container platform is given''' if Path(os.getcwd()+"/tmp").exists(): rmtree(os.getcwd()+"/tmp") # clear out any past runs - create_docker_script.dockerfile_create(YAMLPATH, ["ci.gnu"], TARGET, False) + create_docker_script.dockerfile_create(YAMLPATH, ["ci.gnu"], TARGET, False, False) assert not Path(os.getcwd()+"/tmp").exists() # tests container build script/makefile/dockerfile creation def test_create_dockerfile(): '''run create-dockerfile with options for containerized build''' - create_docker_script.dockerfile_create(YAMLPATH, PLATFORM, TARGET, False) + create_docker_script.dockerfile_create(YAMLPATH, PLATFORM, TARGET, False, False) def test_container_dir_creation(): '''check directories are created'''