Rename and change pyrapids to spark_rapids_tools (#570)

Signed-off-by: Ahmed Hussein (amahussein) <[email protected]>

Fixes #567

user_tools/pyproject.toml

@@ -45,7 +45,7 @@ dynamic=["entry-points", "version"]
 
 [project.scripts]
 spark_rapids_user_tools = "spark_rapids_pytools.wrapper:main"
-ascli = "pyrapids.cmdli.pyrapids_cli:main"
+ascli = "spark_rapids_tools.cmdli.tools_cli:main"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

user_tools/src/spark_rapids_pytools/cloud_api/databricks_aws.py

@@ -18,7 +18,7 @@
 from dataclasses import dataclass, field
 from typing import Any, List
 
-from pyrapids import CspEnv
+from spark_rapids_tools import CspEnv
 from spark_rapids_pytools.cloud_api.databricks_aws_job import DBAWSLocalRapidsJob
 from spark_rapids_pytools.cloud_api.emr import EMRNode, EMRPlatform
 from spark_rapids_pytools.cloud_api.s3storage import S3StorageDriver

user_tools/src/spark_rapids_pytools/cloud_api/databricks_azure.py

@@ -20,7 +20,7 @@
 from dataclasses import dataclass, field
 from typing import Any, List
 
-from pyrapids import CspEnv
+from spark_rapids_tools import CspEnv
 from spark_rapids_pytools.cloud_api.azurestorage import AzureStorageDriver
 from spark_rapids_pytools.cloud_api.databricks_azure_job import DBAzureLocalRapidsJob
 from spark_rapids_pytools.cloud_api.sp_types import CMDDriverBase, ClusterBase, ClusterNode, \

user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py

@@ -19,7 +19,7 @@
 from dataclasses import dataclass, field
 from typing import Any, List
 
-from pyrapids import CspEnv
+from spark_rapids_tools import CspEnv
 from spark_rapids_pytools.cloud_api.dataproc_job import DataprocLocalRapidsJob
 from spark_rapids_pytools.cloud_api.gstorage import GStorageDriver
 from spark_rapids_pytools.cloud_api.sp_types import PlatformBase, CMDDriverBase, \

user_tools/src/spark_rapids_pytools/cloud_api/emr.py

@@ -19,7 +19,7 @@
 from dataclasses import field, dataclass
 from typing import Any, List
 
-from pyrapids import CspEnv
+from spark_rapids_tools import CspEnv
 from spark_rapids_pytools.cloud_api.emr_job import EmrLocalRapidsJob
 from spark_rapids_pytools.cloud_api.s3storage import S3StorageDriver
 from spark_rapids_pytools.cloud_api.sp_types import PlatformBase, ClusterBase, CMDDriverBase, \

user_tools/src/spark_rapids_pytools/cloud_api/onprem.py

@@ -18,7 +18,7 @@
 from dataclasses import dataclass
 from typing import Any, List
 
-from pyrapids import CspEnv
+from spark_rapids_tools import CspEnv
 from spark_rapids_pytools.rapids.rapids_job import RapidsLocalJob
 from spark_rapids_pytools.cloud_api.sp_types import PlatformBase, ClusterBase, ClusterNode, \
     CMDDriverBase, ClusterGetAccessor, GpuDevice, \

user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py

@@ -22,7 +22,7 @@
 from logging import Logger
 from typing import Type, Any, List, Callable
 
-from pyrapids import EnumeratedType, CspEnv
+from spark_rapids_tools import EnumeratedType, CspEnv
 from spark_rapids_pytools.common.prop_manager import AbstractPropertiesContainer, JSONPropertiesContainer, \
     get_elem_non_safe
 from spark_rapids_pytools.common.sys_storage import StorageDriver, FSUtil

user_tools/src/spark_rapids_pytools/common/prop_manager.py

@@ -22,7 +22,7 @@
 
 import yaml
 
-from pyrapids import get_elem_from_dict, get_elem_non_safe
+from spark_rapids_tools import get_elem_from_dict, get_elem_non_safe
 
 
 def convert_dict_to_camel_case(dic: dict):

user_tools/src/spark_rapids_pytools/pricing/emr_pricing.py

@@ -16,7 +16,7 @@
 
 from dataclasses import dataclass, field
 
-from pyrapids import get_elem_from_dict, get_elem_non_safe
+from spark_rapids_tools import get_elem_from_dict, get_elem_non_safe
 from spark_rapids_pytools.common.prop_manager import JSONPropertiesContainer
 from spark_rapids_pytools.common.sys_storage import FSUtil
 from spark_rapids_pytools.pricing.price_provider import PriceProvider

user_tools/src/spark_rapids_pytools/rapids/qualification.py

@@ -22,7 +22,7 @@
 import pandas as pd
 from tabulate import tabulate
 
-from pyrapids.enums import QualFilterApp, QualGpuClusterReshapeType
+from spark_rapids_tools.enums import QualFilterApp, QualGpuClusterReshapeType
 from spark_rapids_pytools.cloud_api.sp_types import ClusterReshape, NodeHWInfo
 from spark_rapids_pytools.common.sys_storage import FSUtil
 from spark_rapids_pytools.common.utilities import Utils, TemplateGenerator

user_tools/src/spark_rapids_pytools/rapids/rapids_tool.py

@@ -26,7 +26,7 @@
 from logging import Logger
 from typing import Any, Callable, Dict, List
 
-from pyrapids import CspEnv
+from spark_rapids_tools import CspEnv
 from spark_rapids_pytools.cloud_api.sp_types import get_platform, \
     ClusterBase, DeployMode, NodeHWInfo
 from spark_rapids_pytools.common.prop_manager import YAMLPropertiesContainer

user_tools/src/spark_rapids_pytools/rapids/tool_ctxt.py

@@ -21,7 +21,7 @@
 from logging import Logger
 from typing import Type, Any, ClassVar, List
 
-from pyrapids import CspEnv
+from spark_rapids_tools import CspEnv
 from spark_rapids_pytools.cloud_api.sp_types import PlatformBase
 from spark_rapids_pytools.common.prop_manager import YAMLPropertiesContainer
 from spark_rapids_pytools.common.sys_storage import FSUtil

user_tools/src/spark_rapids_pytools/resources/dev/prepackage_mgr.py

@@ -25,7 +25,7 @@
 
 import fire
 
-from pyrapids import CspEnv
+from spark_rapids_tools import CspEnv
 from spark_rapids_pytools.common.prop_manager import JSONPropertiesContainer
 from spark_rapids_pytools.common.sys_storage import FSUtil
 from spark_rapids_pytools.common.utilities import Utils

user_tools/src/spark_rapids_pytools/wrappers/databricks_aws_wrapper.py

@@ -14,7 +14,7 @@
 
 
 """Wrapper class to run tools associated with RAPIDS Accelerator for Apache Spark plugin on DATABRICKS_AWS."""
-from pyrapids import CspEnv
+from spark_rapids_tools import CspEnv
 from spark_rapids_pytools.cloud_api.sp_types import DeployMode
 from spark_rapids_pytools.common.utilities import ToolLogging
 from spark_rapids_pytools.rapids.profiling import ProfilingAsLocal

user_tools/src/spark_rapids_pytools/wrappers/databricks_azure_wrapper.py

@@ -14,7 +14,7 @@
 
 
 """Wrapper class to run tools associated with RAPIDS Accelerator for Apache Spark plugin on DATABRICKS_AZURE."""
-from pyrapids import CspEnv
+from spark_rapids_tools import CspEnv
 from spark_rapids_pytools.cloud_api.sp_types import DeployMode
 from spark_rapids_pytools.common.utilities import ToolLogging
 from spark_rapids_pytools.rapids.profiling import ProfilingAsLocal

user_tools/src/spark_rapids_pytools/wrappers/dataproc_wrapper.py

@@ -14,7 +14,7 @@
 
 """Wrapper class to run tools associated with RAPIDS Accelerator for Apache Spark plugin on Dataproc."""
 
-from pyrapids import CspEnv
+from spark_rapids_tools import CspEnv
 from spark_rapids_pytools.cloud_api.sp_types import DeployMode
 from spark_rapids_pytools.common.utilities import ToolLogging
 from spark_rapids_pytools.rapids.bootstrap import Bootstrap

user_tools/src/spark_rapids_pytools/wrappers/emr_wrapper.py

@@ -14,7 +14,7 @@
 
 
 """Wrapper class to run tools associated with RAPIDS Accelerator for Apache Spark plugin on AWS-EMR."""
-from pyrapids import CspEnv
+from spark_rapids_tools import CspEnv
 from spark_rapids_pytools.cloud_api.sp_types import DeployMode
 from spark_rapids_pytools.common.utilities import ToolLogging
 from spark_rapids_pytools.rapids.bootstrap import Bootstrap

user_tools/src/spark_rapids_pytools/wrappers/onprem_wrapper.py

@@ -14,7 +14,7 @@
 
 
 """Wrapper class to run tools associated with RAPIDS Accelerator for Apache Spark plugin on On-Prem cluster."""
-from pyrapids import CspEnv
+from spark_rapids_tools import CspEnv
 from spark_rapids_pytools.cloud_api.sp_types import DeployMode
 from spark_rapids_pytools.common.utilities import ToolLogging
 from spark_rapids_pytools.rapids.profiling import ProfilingAsLocal

user_tools/src/pyrapids/cloud/databricks/dbcluster.py → user_tools/src/spark_rapids_tools/cloud/databricks/dbcluster.py

@@ -18,8 +18,8 @@
 
 from typing import ClassVar, Type, Optional
 
-from pyrapids.cloud.cluster import register_client_cluster, register_cluster_prop_mgr, ClusterPropMgr, ClientCluster
-from pyrapids.utils.propmanager import PropValidatorSchema
+from spark_rapids_tools.cloud.cluster import register_client_cluster, register_cluster_prop_mgr, ClusterPropMgr, ClientCluster
+from spark_rapids_tools.utils.propmanager import PropValidatorSchema
 
 
 class DBAwsClusterSchema(PropValidatorSchema):

user_tools/src/pyrapids/cloud/dataproc/dataproccluster.py → user_tools/src/spark_rapids_tools/cloud/dataproc/dataproccluster.py

@@ -18,8 +18,8 @@
 
 from typing import ClassVar, Type
 
-from pyrapids.cloud.cluster import ClientCluster, register_client_cluster, ClusterPropMgr, register_cluster_prop_mgr
-from pyrapids.utils.propmanager import PropValidatorSchemaCamel, PropValidatorSchema
+from spark_rapids_tools.cloud.cluster import ClientCluster, register_client_cluster, ClusterPropMgr, register_cluster_prop_mgr
+from spark_rapids_tools.utils.propmanager import PropValidatorSchemaCamel, PropValidatorSchema
 
 
 class DataprocClusterSchema(PropValidatorSchemaCamel):

user_tools/src/pyrapids/cloud/emr/emrcluster.py → user_tools/src/spark_rapids_tools/cloud/emr/emrcluster.py

@@ -18,8 +18,8 @@
 
 from typing import ClassVar, Type
 
-from pyrapids.cloud.cluster import register_cluster_prop_mgr, register_client_cluster, ClusterPropMgr, ClientCluster
-from pyrapids.utils.propmanager import PropValidatorSchemaUpper, PropValidatorSchema
+from spark_rapids_tools.cloud.cluster import register_cluster_prop_mgr, register_client_cluster, ClusterPropMgr, ClientCluster
+from spark_rapids_tools.utils.propmanager import PropValidatorSchemaUpper, PropValidatorSchema
 
 
 class EmrClusterSchema(PropValidatorSchemaUpper):

user_tools/src/pyrapids/cloud/onprem/onpremcluster.py → user_tools/src/spark_rapids_tools/cloud/onprem/onpremcluster.py

@@ -19,9 +19,9 @@
 from typing import ClassVar, Type
 from typing_extensions import TypedDict
 from pydantic import ConfigDict
-from pyrapids.cloud.cluster import ClientCluster, ClusterPropMgr, register_cluster_prop_mgr, register_client_cluster
-from pyrapids.utils.propmanager import PropValidatorSchema
-from pyrapids.utils.util import to_camel_case
+from spark_rapids_tools.cloud.cluster import ClientCluster, ClusterPropMgr, register_cluster_prop_mgr, register_client_cluster
+from spark_rapids_tools.utils.propmanager import PropValidatorSchema
+from spark_rapids_tools.utils.util import to_camel_case
 
 
 class OnPremDriverConfigSchema(TypedDict):

user_tools/src/pyrapids/cmdli/__init__.py → user_tools/src/spark_rapids_tools/cmdli/__init__.py

@@ -14,8 +14,8 @@
 
 """init file of the user CLI used to run the tools"""
 
-from .pyrapids_cli import PyRapids
+from .tools_cli import ToolsCLI
 
 __all__ = [
-    'PyRapids'
+    'ToolsCLI'
 ]

user_tools/src/pyrapids/cmdli/argprocessor.py → user_tools/src/spark_rapids_tools/cmdli/argprocessor.py

@@ -25,9 +25,9 @@
 from pydantic.dataclasses import dataclass
 from pydantic_core import PydanticCustomError
 
-from pyrapids.cloud import ClientCluster
-from pyrapids.exceptions import IllegalArgumentError
-from pyrapids.utils import AbstractPropContainer, is_http_file
+from spark_rapids_tools.cloud import ClientCluster
+from spark_rapids_tools.exceptions import IllegalArgumentError
+from spark_rapids_tools.utils import AbstractPropContainer, is_http_file
 from spark_rapids_pytools.cloud_api.sp_types import DeployMode
 from spark_rapids_pytools.common.utilities import ToolLogging
 from spark_rapids_pytools.rapids.qualification import QualGpuClusterReshapeType
@@ -105,7 +105,7 @@ class AbsToolUserArgModel:
         'meta': {},
         'toolArgs': {}
     })
-    logger: ClassVar[Logger] = ToolLogging.get_and_setup_logger('pyrapids.argparser')
+    logger: ClassVar[Logger] = ToolLogging.get_and_setup_logger('spark_rapids_tools.argparser')
     tool_name: ClassVar[str] = None
 
     @classmethod

user_tools/src/pyrapids/cmdli/pyrapids_cli.py → user_tools/src/spark_rapids_tools/cmdli/tools_cli.py

@@ -17,15 +17,15 @@
 
 import fire
 
-from pyrapids.enums import QualGpuClusterReshapeType
-from pyrapids.utils.util import gen_app_banner
+from spark_rapids_tools.enums import QualGpuClusterReshapeType
+from spark_rapids_tools.utils.util import gen_app_banner
 from spark_rapids_pytools.rapids.bootstrap import Bootstrap
 from spark_rapids_pytools.rapids.profiling import ProfilingAsLocal
 from spark_rapids_pytools.rapids.qualification import QualificationAsLocal
 from .argprocessor import AbsToolUserArgModel
 
 
-class PyRapids(object):  # pylint: disable=too-few-public-methods
+class ToolsCLI(object):  # pylint: disable=too-few-public-methods
     """CLI that provides a runtime environment that simplifies running cost and performance analysis
     using the RAPIDS Accelerator for Apache Spark.
 
@@ -167,7 +167,7 @@ def main():
     # Make Python Fire not use a pager when it prints a help text
     fire.core.Display = lambda lines, out: out.write('\n'.join(lines) + '\n')
     print(gen_app_banner())
-    fire.Fire(PyRapids())
+    fire.Fire(ToolsCLI())
 
 
 if __name__ == '__main__':

user_tools/src/pyrapids/storagelib/csppath.py → user_tools/src/spark_rapids_tools/storagelib/csppath.py

@@ -190,18 +190,18 @@ class CspPath(metaclass=CspPathMeta):
     Create a new path subclass from a gcs URI:
 
     >>> gs_path = CspPath('gs://bucket-name/folder_00/subfolder_01')
-    <pyrapids.storagelib.gcs.gcpath.GcsPath object at ...>
+    <spark_rapids_tools.storagelib.gcs.gcpath.GcsPath object at ...>
 
     or from S3 URI:
 
     >>> s3_path = CspPath('s3://bucket-name/folder_00/subfolder_01')
-    <pyrapids.storagelib.s3.s3path.S3Path object at ...>
+    <spark_rapids_tools.storagelib.s3.s3path.S3Path object at ...>
 
     or from local file URI:
 
     >>> local_path1, local_path2 = (CspPath('~/my_folder'), CspPath('file:///my_folder'))
-    <pyrapids.storagelib.local.localpath.LocalPath object at ...,
-      pyrapids.storagelib.local.localpath.LocalPath object at ...>
+    <spark_rapids_tools.storagelib.local.localpath.LocalPath object at ...,
+      spark_rapids_tools.storagelib.local.localpath.LocalPath object at ...>
 
     Print the data from the file with `open_input_file()`:
 

user_tools/src/pyrapids/storagelib/s3/s3fs.py → user_tools/src/spark_rapids_tools/storagelib/s3/s3fs.py

@@ -15,7 +15,7 @@
 """Wrapper for the S3 File system"""
 
 
-from pyrapids.storagelib.cspfs import register_fs_class, CspFs
+from spark_rapids_tools.storagelib.cspfs import register_fs_class, CspFs
 
 
 @register_fs_class('s3', 'S3FileSystem')

user_tools/src/pyrapids/tools/autotuner.py → user_tools/src/spark_rapids_tools/tools/autotuner.py

@@ -16,7 +16,7 @@
 
 from typing import Optional, ClassVar, Type
 
-from pyrapids.utils.propmanager import PropValidatorSchemaCamel, PropValidatorSchema, AbstractPropContainer
+from spark_rapids_tools.utils.propmanager import PropValidatorSchemaCamel, PropValidatorSchema, AbstractPropContainer
 
 
 class AutoTunerInputSchema(PropValidatorSchemaCamel):

user_tools/src/pyrapids/utils/propmanager.py → user_tools/src/spark_rapids_tools/utils/propmanager.py

@@ -23,9 +23,9 @@
 import yaml
 from pydantic import BaseModel, ConfigDict, model_validator, ValidationError
 
-from pyrapids.exceptions import JsonLoadException, YamlLoadException, InvalidPropertiesSchema
-from pyrapids.storagelib.csppath import CspPath, CspPathT
-from pyrapids.utils.util import to_camel_case, to_camel_capital_case, get_elem_from_dict, get_elem_non_safe
+from spark_rapids_tools.exceptions import JsonLoadException, YamlLoadException, InvalidPropertiesSchema
+from spark_rapids_tools.storagelib.csppath import CspPath, CspPathT
+from spark_rapids_tools.utils.util import to_camel_case, to_camel_capital_case, get_elem_from_dict, get_elem_non_safe
 
 
 def load_json(file_path: Union[str, CspPathT]) -> Any:

user_tools/src/pyrapids/utils/util.py → user_tools/src/spark_rapids_tools/utils/util.py

@@ -26,7 +26,7 @@
 from pydantic import ValidationError, AnyHttpUrl, TypeAdapter
 
 import spark_rapids_pytools
-from pyrapids.exceptions import CspPathAttributeError
+from spark_rapids_tools.exceptions import CspPathAttributeError
 
 
 def get_elem_from_dict(data, keys):
@@ -87,8 +87,8 @@ def to_snake_case(word: str) -> str:
 
 
 def dump_tool_usage(tool_name: Optional[str], raise_sys_exit: Optional[bool] = True):
-    imported_module = __import__('pyrapids.cmdli', globals(), locals(), ['PyRapids'])
-    wrapper_clzz = getattr(imported_module, 'PyRapids')
+    imported_module = __import__('spark_rapids_tools.cmdli', globals(), locals(), ['ToolsCLI'])
+    wrapper_clzz = getattr(imported_module, 'ToolsCLI')
     help_name = 'ascli'
     usage_cmd = f'{tool_name} --help'
     try:

user_tools/tests/pyrapids_unit/conftest.py → user_tools/tests/spark_rapids_tools_ut/conftest.py

@@ -25,7 +25,7 @@ def get_test_resources_path():
         import importlib_resources
     else:
         import importlib.resources as importlib_resources
-    pkg = importlib_resources.files('tests.pyrapids_unit')
+    pkg = importlib_resources.files('tests.spark_rapids_tools_ut')
     return pkg / 'resources'
 
 
@@ -47,7 +47,7 @@ def gen_cpu_cluster_props():
 all_csps = csps + ['onprem']
 
 
-class PyrapidsUnitTest:   # pylint: disable=too-few-public-methods
+class SparkRapidsToolsUT:   # pylint: disable=too-few-public-methods
 
     @pytest.fixture(autouse=True)
     def get_ut_data_dir(self):

user_tools/tests/pyrapids_unit/test_cluster.py → user_tools/tests/spark_rapids_tools_ut/test_cluster.py

@@ -16,13 +16,13 @@
 
 import pytest  # pylint: disable=import-error
 
-from pyrapids import CspPath
-from pyrapids.cloud import ClientCluster
-from pyrapids.exceptions import InvalidPropertiesSchema
-from .conftest import PyrapidsUnitTest, all_cpu_cluster_props
+from spark_rapids_tools import CspPath
+from spark_rapids_tools.cloud import ClientCluster
+from spark_rapids_tools.exceptions import InvalidPropertiesSchema
+from .conftest import SparkRapidsToolsUT, all_cpu_cluster_props
 
 
-class TestClusterCSP(PyrapidsUnitTest):  # pylint: disable=too-few-public-methods
+class TestClusterCSP(SparkRapidsToolsUT):  # pylint: disable=too-few-public-methods
     """
     Class testing identifying the cluster type by comparing the properties to
     the defined Schema