221.Add force checkout and compile functionality #236
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,21 +1,49 @@ | ||
| ''' | ||
| Checks out source code | ||
| ''' | ||
|
|
||
| import os | ||
| import subprocess | ||
| import logging | ||
| import sys | ||
| import shutil | ||
| from pathlib import Path | ||
|
|
||
| import fre.yamltools.combine_yamls as cy | ||
| from .gfdlfremake import varsfre, yamlfre, checkout, targetfre | ||
|
|
||
| def baremetal_checkout_write_steps(model_yaml,src_dir,jobs,pc): | ||
| """ | ||
| Go through steps to write the checkout script for bare-metal build | ||
| """ | ||
| fre_checkout = checkout.checkout("checkout.sh",src_dir) | ||
| fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc) | ||
| fre_checkout.finish(model_yaml.compile.getCompileYaml(),pc) | ||
|
|
||
| # Make checkout script executable | ||
| os.chmod(src_dir+"/checkout.sh", 0o744) | ||
| print(" Checkout script created in "+ src_dir + "/checkout.sh \n") | ||
|
|
||
| return fre_checkout | ||
|
|
||
| def container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc): | ||
| """ | ||
| Go through steps to write the checkout script for container | ||
| """ | ||
| fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir) | ||
| fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc) | ||
| fre_checkout.finish(model_yaml.compile.getCompileYaml(),pc) | ||
| print(" Checkout script created at " + tmp_dir + "/checkout.sh" + "\n") | ||
|
|
||
| return fre_checkout | ||
|
|
||
| def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose,force_checkout): | ||
| """ | ||
| Call gfdlfremake/checkout.py to create the checkout script | ||
| """ | ||
| # Define variables | ||
| yml = yamlfile | ||
| name = yamlfile.split(".")[0] | ||
| jobs = str(jobs) | ||
| pcheck = no_parallel_checkout | ||
|
|
||
|
|
@@ -37,10 +65,16 @@ def checkout_create(yamlfile, platform, target, no_parallel_checkout, jobs, exec | |
| plist = platform | ||
| tlist = target | ||
|
|
||
| # If force-checkout defined: re-combine model, compile, and platform yamls | ||
| if force_checkout: | ||
| print("Re-combine yaml files") | ||
| comb = cy.init_compile_yaml(yml,platform,target) | ||
| full_combined = cy.get_combined_compileyaml(comb) | ||
| else: | ||
| ## If combined yaml exists, note message of its existence | ||
| ## If combined yaml does not exist, combine model, compile, and platform yamls | ||
| combined = Path(f"combined-{name}.yaml") | ||
| full_combined = cy.combined_compile_existcheck(combined,yml,platform,target) | ||
|
There was a problem hiding this comment. I think if the combined yaml exists, it should combined them again and compare them and output a warning if they are not the same and suggest the For example, if you change something in your yaml and you forget to delete the combined yaml, fre-canopy is going to use the combined-yaml that already exist and won't do what you expect (I have done this sooooo many times :/) This is what bronx outpust: There was a problem hiding this comment. I agree with this 200%. This is making me wonder though, why not go one step further and just not even write the combined yaml to a file? I will admit, I never really understood why we combine the yamls and write it out in the first place, so maybe there's a better reason then I'm seeing. But if the current fremake yamls (platform/compile/etc) are what we are going to be modifying and using in place of xmls, the combined yamls don't really need to exist as files, they can stay in memory. Otherwise, they just kinda get in the way of whatever yaml files the user specifies. There was a problem hiding this comment. I agree with both of you, this is great input! @rem1776 - At first, I was trying to load the yamls and just have/use the dictionary it's parsed as, and then update/add to dictionaries in order to dump a combined yaml file with all the info needed. BUT, the main issue I was seeing here was from the reusable variables. The yaml won't even load (it will give "undefined alias" issue) if there is a key that uses a reusable variable that isn't defined in I think this is related to your confusion on why we even create these combined yamls. If there's a better way, I'm definitely up for it though. |
||
|
|
||
| ## Get the variables in the model yaml | ||
| fre_vars = varsfre.frevars(full_combined) | ||
|
|
@@ -65,29 +99,35 @@ def checkout_create(yamlfile, platform, target, no_parallel_checkout, jobs, exec | |
|
|
||
| platform = model_yaml.platforms.getPlatformFromName(platform_name) | ||
|
|
||
| # Create the source directory for the platform | ||
| if not platform["container"]: | ||
| src_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/src" | ||
| # if the source directory does not exist, it is created | ||
| if not os.path.exists(src_dir): | ||
| os.system("mkdir -p " + src_dir) | ||
| # if the checkout script does not exist, it is created | ||
| if not os.path.exists(src_dir+"/checkout.sh"): | ||
| print("\nCreating checkout script...") | ||
| fre_checkout = baremetal_checkout_write_steps(model_yaml,src_dir,jobs,pc) | ||
|
|
||
| # Run the checkout script | ||
| if execute: | ||
| fre_checkout.run() | ||
| else: | ||
| return | ||
| else: | ||
| if force_checkout: | ||
| # Remove previous checkout | ||
| print("\nRemoving previously checkout script and checked out source code") | ||
|
There was a problem hiding this comment. I think rather than removing it you should back up the src and exec directory. This what bronx ouputs |
||
| shutil.rmtree(src_dir) | ||
|
|
||
| # Create checkout script | ||
| print("Re-creating the checkout script...") | ||
| fre_checkout = baremetal_checkout_write_steps(model_yaml,src_dir,jobs,pc) | ||
| else: | ||
| print("\nCheckout script PREVIOUSLY created in "+ src_dir + "/checkout.sh \n") | ||
|
|
||
| if execute: | ||
| try: | ||
| subprocess.run(args=[src_dir+"/checkout.sh"], check=True) | ||
| except: | ||
|
|
@@ -101,10 +141,21 @@ def checkout_create(yamlfile, platform, target, no_parallel_checkout, jobs, exec | |
| src_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/src" | ||
| bld_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/exec" | ||
| tmp_dir = "tmp/"+platform_name | ||
| if not os.path.exists(tmp_dir+"/checkout.sh"): | ||
| # Create the checkout script | ||
| print("Creating checkout script...") | ||
| container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc) | ||
| else: | ||
| if force_checkout: | ||
| # Remove the checkout script | ||
| print("\nRemoving previously made checkout script") | ||
| os.remove(tmp_dir+"/checkout.sh") | ||
|
|
||
| # Create the checkout script | ||
| print("Re-creating the checkout script...") | ||
| container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc) | ||
| else: | ||
| print("\nCheckout script PREVIOUSLY created in "+ tmp_dir + "/checkout.sh" + "\n") | ||
|
|
||
| if __name__ == "__main__": | ||
| checkout_create() | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
just curious, is this returned for a reason?