diff --git a/views_pipeline_core/templates/model/template_config_deployment.py b/views_pipeline_core/templates/model/template_config_deployment.py index de49e44..60b312a 100644 --- a/views_pipeline_core/templates/model/template_config_deployment.py +++ b/views_pipeline_core/templates/model/template_config_deployment.py @@ -3,12 +3,10 @@ from pathlib import Path import logging -logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) - def generate( - script_dir: Path, + script_path: Path, deployment_type: str = "shadow", additional_settings: Dict[str, any] = None, ) -> bool: @@ -16,7 +14,7 @@ def generate( Generates a script that defines the `get_deployment_config` function for configuring the deployment status and settings. Parameters: - script_dir (Path): The directory where the generated deployment configuration script will be saved. + script_path (Path): The path where the generated deployment configuration script will be saved. This should be a valid writable path. deployment_type (str, optional): The type of deployment. Must be one of "shadow", "deployed", "baseline", or "deprecated". @@ -72,4 +70,4 @@ def get_deployment_config(): deployment_config = {deployment_config} return deployment_config """ - return save_script(script_dir, code) + return save_script(script_path, code) diff --git a/views_pipeline_core/templates/model/template_config_hyperparameters.py b/views_pipeline_core/templates/model/template_config_hyperparameters.py index 358c56c..025ccda 100644 --- a/views_pipeline_core/templates/model/template_config_hyperparameters.py +++ b/views_pipeline_core/templates/model/template_config_hyperparameters.py @@ -1,15 +1,17 @@ from views_pipeline_core.templates.utils import save_script from pathlib import Path +import logging +logger = logging.getLogger(__name__) -def generate(script_dir: Path) -> bool: +def generate(script_path: Path) -> bool: """ Generates a script that defines a function for obtaining hyperparameter configurations necessary for model training. Parameters: - script_dir (Path): - The directory where the generated deployment configuration script will be saved. + script_path (Path): + The path where the generated deployment configuration script will be saved. This should be a valid writable path. Returns: @@ -32,4 +34,4 @@ def get_hp_config(): }} return hyperparameters """ - return save_script(script_dir, code) + return save_script(script_path, code) diff --git a/views_pipeline_core/templates/model/template_config_meta.py b/views_pipeline_core/templates/model/template_config_meta.py index 489f2f4..9bbf18c 100644 --- a/views_pipeline_core/templates/model/template_config_meta.py +++ b/views_pipeline_core/templates/model/template_config_meta.py @@ -1,14 +1,15 @@ from views_pipeline_core.templates.utils import save_script from pathlib import Path +import logging +logger = logging.getLogger(__name__) - -def generate(script_dir: Path, model_name: str, model_algorithm: str) -> bool: +def generate(script_path: Path, model_name: str, model_algorithm: str) -> bool: """ Generates a script that defines the `get_meta_config` function for model metadata. Parameters: - script_dir (Path): - The directory where the generated deployment configuration script will be saved. + script_path (Path): + The path where the generated deployment configuration script will be saved. This should be a valid writable path. model_name (str): @@ -41,4 +42,4 @@ def generate(script_dir: Path, model_name: str, model_algorithm: str) -> bool: }} return meta_config """ - return save_script(script_dir, code) + return save_script(script_path, code) diff --git a/views_pipeline_core/templates/model/template_config_queryset.py b/views_pipeline_core/templates/model/template_config_queryset.py index 9f88871..90f4f5e 100644 --- a/views_pipeline_core/templates/model/template_config_queryset.py +++ b/views_pipeline_core/templates/model/template_config_queryset.py @@ -1,14 +1,15 @@ from views_pipeline_core.templates.utils import save_script from pathlib import Path +import logging +logger = logging.getLogger(__name__) - -def generate(script_dir: Path, model_name: str) -> bool: +def generate(script_path: Path, model_name: str) -> bool: """ Generates a Python script that defines the `generate` function for configuring input data for model training using the viewser library. Args: - script_dir (Path): The directory where the generated deployment configuration script will be saved. + script_path (Path): The path where the generated deployment configuration script will be saved. This should be a valid writable path. model_name (str): The name of the model for which the input data configuration is being generated. This name is used to identify the relevant data in the viewser queryset. @@ -58,4 +59,4 @@ def generate(): return queryset_base """ - return save_script(script_dir, code) + return save_script(script_path, code) diff --git a/views_pipeline_core/templates/model/template_config_sweep.py b/views_pipeline_core/templates/model/template_config_sweep.py index 260ee1c..f1dd458 100644 --- a/views_pipeline_core/templates/model/template_config_sweep.py +++ b/views_pipeline_core/templates/model/template_config_sweep.py @@ -1,14 +1,15 @@ from views_pipeline_core.templates.utils import save_script from pathlib import Path +import logging +logger = logging.getLogger(__name__) - -def generate(script_dir: Path, model_name: str, model_algorithm: str) -> bool: +def generate(script_path: Path, model_name: str, model_algorithm: str) -> bool: """ Generates a script that defines the `get_sweep_config` function for hyperparameter sweeps. Parameters: - script_dir (Path): - The directory where the generated deployment configuration script will be saved. + script_path (Path): + The path where the generated deployment configuration script will be saved. This should be a valid writable path. model_name (str): @@ -52,4 +53,4 @@ def get_sweep_config(): return sweep_config """ - return save_script(script_dir, code) + return save_script(script_path, code) diff --git a/views_pipeline_core/templates/model/template_main.py b/views_pipeline_core/templates/model/template_main.py index c85ea60..255539b 100644 --- a/views_pipeline_core/templates/model/template_main.py +++ b/views_pipeline_core/templates/model/template_main.py @@ -1,8 +1,9 @@ from views_pipeline_core.templates.utils import save_script from pathlib import Path +import logging +logger = logging.getLogger(__name__) - -def generate(script_dir: Path) -> bool: +def generate(script_path: Path) -> bool: """ Generates a Python script that sets up and executes model runs with Weights & Biases (WandB) integration. @@ -11,8 +12,8 @@ def generate(script_dir: Path) -> bool: generated script includes command-line argument parsing, validation, and runtime logging. Parameters: - script_dir (Path): - The directory where the generated Python script will be saved. This should be a valid writable + script_path (Path): + The path where the generated Python script will be saved. This should be a valid writable path that exists within the project structure. Returns: @@ -62,4 +63,4 @@ def generate(script_dir: Path) -> bool: else: # YourModelManager(model_path=model_path).execute_single_run(args) ''' - return save_script(script_dir, code) \ No newline at end of file + return save_script(script_path, code) \ No newline at end of file diff --git a/views_pipeline_core/templates/model/template_run_sh.py b/views_pipeline_core/templates/model/template_run_sh.py new file mode 100644 index 0000000..a00003a --- /dev/null +++ b/views_pipeline_core/templates/model/template_run_sh.py @@ -0,0 +1,87 @@ +from pathlib import Path + +def generate(script_path: Path, package_name: str) -> bool: + """ + Generates a shell script to set up the environment and run a Python script. + + This function creates a shell script that: + - Checks if the operating system is macOS (Darwin) and updates the user's `.zshrc` file with necessary environment variables for `libomp`. + - Determines the script's directory and the project's root directory. + - Sets up the path for the Conda environment specific to the project. + - Activates the Conda environment if it exists, or creates a new one if it doesn't. + - Installs any missing or outdated Python packages listed in `requirements.txt`. + - Runs the main Python script with any provided command-line arguments. + + Parameters: + script_path (Path): + The path where the generated shell script will be saved. This should be a valid writable path. + package_name (str): + The name of the package to be used in the script. + + Returns: + bool: + True if the script was successfully written to the specified directory, False otherwise. + + The generated shell script includes the following steps: + - Checks if the operating system is macOS and updates the `.zshrc` file with necessary environment variables for `libomp`. + - Determines the script's directory and the project's root directory. + - Sets up the path for the Conda environment specific to the project. + - Activates the Conda environment if it exists, or creates a new one if it doesn't. + - Installs any missing or outdated Python packages listed in `requirements.txt`. + - Runs the main Python script with any provided command-line arguments. + + Note: + - Ensure that the `requirements.txt` file is present in the same directory as the generated shell script. + - The generated shell script is designed to be executed in a zsh shell. + """ + code = f"""#!/bin/zsh + +if [[ "$OSTYPE" == "darwin"* ]]; then + if ! grep -q 'export LDFLAGS="-L/opt/homebrew/opt/libomp/lib"' ~/.zshrc; then + echo 'export LDFLAGS="-L/opt/homebrew/opt/libomp/lib"' >> ~/.zshrc + fi + if ! grep -q 'export CPPFLAGS="-I/opt/homebrew/opt/libomp/include"' ~/.zshrc; then + echo 'export CPPFLAGS="-I/opt/homebrew/opt/libomp/include"' >> ~/.zshrc + fi + if ! grep -q 'export DYLD_LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$DYLD_LIBRARY_PATH"' ~/.zshrc; then + echo 'export DYLD_LIBRARY_PATH="/opt/homebrew/opt/libomp/lib:$DYLD_LIBRARY_PATH"' >> ~/.zshrc + fi + source ~/.zshrc +fi + +script_path=$(dirname "$(realpath $0)") +project_path="$( cd "$script_path/../../" >/dev/null 2>&1 && pwd )" +env_path="$project_path/envs/{package_name}" + +eval "$(conda shell.bash hook)" + +if [ -d "$env_path" ]; then + echo "Conda environment already exists at $env_path. Checking dependencies..." + conda activate "$env_path" + echo "$env_path is activated" + + missing_packages=$(pip install --dry-run -r $script_path/requirements.txt 2>&1 | grep -v "Requirement already satisfied" | wc -l) + if [ "$missing_packages" -gt 0 ]; then + echo "Installing missing or outdated packages..." + pip install -r $script_path/requirements.txt + else + echo "All packages are up-to-date." + fi +else + echo "Creating new Conda environment at $env_path..." + conda create --prefix "$env_path" python=3.11 -y + conda activate "$env_path" + pip install -r $script_path/requirements.txt +fi + +echo "Running $script_path/main.py " +python $script_path/main.py "$@" +""" + + try: + with open(script_path, "w", encoding="utf-8") as script_file: + script_file.write(code) + return True + except IOError as e: + print(f"An error occurred while writing to {script_path}: {e}") + return False \ No newline at end of file