Skip to content

Commit

Permalink
Rename and change pyrapids to spark_rapids_tools
Browse files Browse the repository at this point in the history
Signed-off-by: Ahmed Hussein (amahussein) <[email protected]>

Fixes #567
  • Loading branch information
amahussein committed Sep 13, 2023
1 parent 4a50bf1 commit 02ef108
Show file tree
Hide file tree
Showing 67 changed files with 61 additions and 61 deletions.
2 changes: 1 addition & 1 deletion user_tools/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ dynamic=["entry-points", "version"]

[project.scripts]
spark_rapids_user_tools = "spark_rapids_pytools.wrapper:main"
ascli = "pyrapids.cmdli.pyrapids_cli:main"
ascli = "spark_rapids_tools.cmdli.tools_cli:main"

[tool.setuptools]
package-dir = {"" = "src"}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from dataclasses import dataclass, field
from typing import Any, List

from pyrapids import CspEnv
from spark_rapids_tools import CspEnv
from spark_rapids_pytools.cloud_api.databricks_aws_job import DBAWSLocalRapidsJob
from spark_rapids_pytools.cloud_api.emr import EMRNode, EMRPlatform
from spark_rapids_pytools.cloud_api.s3storage import S3StorageDriver
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from dataclasses import dataclass, field
from typing import Any, List

from pyrapids import CspEnv
from spark_rapids_tools import CspEnv
from spark_rapids_pytools.cloud_api.azurestorage import AzureStorageDriver
from spark_rapids_pytools.cloud_api.databricks_azure_job import DBAzureLocalRapidsJob
from spark_rapids_pytools.cloud_api.sp_types import CMDDriverBase, ClusterBase, ClusterNode, \
Expand Down
2 changes: 1 addition & 1 deletion user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from dataclasses import dataclass, field
from typing import Any, List

from pyrapids import CspEnv
from spark_rapids_tools import CspEnv
from spark_rapids_pytools.cloud_api.dataproc_job import DataprocLocalRapidsJob
from spark_rapids_pytools.cloud_api.gstorage import GStorageDriver
from spark_rapids_pytools.cloud_api.sp_types import PlatformBase, CMDDriverBase, \
Expand Down
2 changes: 1 addition & 1 deletion user_tools/src/spark_rapids_pytools/cloud_api/emr.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from dataclasses import field, dataclass
from typing import Any, List

from pyrapids import CspEnv
from spark_rapids_tools import CspEnv
from spark_rapids_pytools.cloud_api.emr_job import EmrLocalRapidsJob
from spark_rapids_pytools.cloud_api.s3storage import S3StorageDriver
from spark_rapids_pytools.cloud_api.sp_types import PlatformBase, ClusterBase, CMDDriverBase, \
Expand Down
2 changes: 1 addition & 1 deletion user_tools/src/spark_rapids_pytools/cloud_api/onprem.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from dataclasses import dataclass
from typing import Any, List

from pyrapids import CspEnv
from spark_rapids_tools import CspEnv
from spark_rapids_pytools.rapids.rapids_job import RapidsLocalJob
from spark_rapids_pytools.cloud_api.sp_types import PlatformBase, ClusterBase, ClusterNode, \
CMDDriverBase, ClusterGetAccessor, GpuDevice, \
Expand Down
2 changes: 1 addition & 1 deletion user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from logging import Logger
from typing import Type, Any, List, Callable

from pyrapids import EnumeratedType, CspEnv
from spark_rapids_tools import EnumeratedType, CspEnv
from spark_rapids_pytools.common.prop_manager import AbstractPropertiesContainer, JSONPropertiesContainer, \
get_elem_non_safe
from spark_rapids_pytools.common.sys_storage import StorageDriver, FSUtil
Expand Down
2 changes: 1 addition & 1 deletion user_tools/src/spark_rapids_pytools/common/prop_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

import yaml

from pyrapids import get_elem_from_dict, get_elem_non_safe
from spark_rapids_tools import get_elem_from_dict, get_elem_non_safe


def convert_dict_to_camel_case(dic: dict):
Expand Down
2 changes: 1 addition & 1 deletion user_tools/src/spark_rapids_pytools/pricing/emr_pricing.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

from dataclasses import dataclass, field

from pyrapids import get_elem_from_dict, get_elem_non_safe
from spark_rapids_tools import get_elem_from_dict, get_elem_non_safe
from spark_rapids_pytools.common.prop_manager import JSONPropertiesContainer
from spark_rapids_pytools.common.sys_storage import FSUtil
from spark_rapids_pytools.pricing.price_provider import PriceProvider
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import pandas as pd
from tabulate import tabulate

from pyrapids.enums import QualFilterApp, QualGpuClusterReshapeType
from spark_rapids_tools.enums import QualFilterApp, QualGpuClusterReshapeType
from spark_rapids_pytools.cloud_api.sp_types import ClusterReshape, NodeHWInfo
from spark_rapids_pytools.common.sys_storage import FSUtil
from spark_rapids_pytools.common.utilities import Utils, TemplateGenerator
Expand Down
2 changes: 1 addition & 1 deletion user_tools/src/spark_rapids_pytools/rapids/rapids_tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from logging import Logger
from typing import Any, Callable, Dict, List

from pyrapids import CspEnv
from spark_rapids_tools import CspEnv
from spark_rapids_pytools.cloud_api.sp_types import get_platform, \
ClusterBase, DeployMode, NodeHWInfo
from spark_rapids_pytools.common.prop_manager import YAMLPropertiesContainer
Expand Down
2 changes: 1 addition & 1 deletion user_tools/src/spark_rapids_pytools/rapids/tool_ctxt.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from logging import Logger
from typing import Type, Any, ClassVar, List

from pyrapids import CspEnv
from spark_rapids_tools import CspEnv
from spark_rapids_pytools.cloud_api.sp_types import PlatformBase
from spark_rapids_pytools.common.prop_manager import YAMLPropertiesContainer
from spark_rapids_pytools.common.sys_storage import FSUtil
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@

import fire

from pyrapids import CspEnv
from spark_rapids_tools import CspEnv
from spark_rapids_pytools.common.prop_manager import JSONPropertiesContainer
from spark_rapids_pytools.common.sys_storage import FSUtil
from spark_rapids_pytools.common.utilities import Utils
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@


"""Wrapper class to run tools associated with RAPIDS Accelerator for Apache Spark plugin on DATABRICKS_AWS."""
from pyrapids import CspEnv
from spark_rapids_tools import CspEnv
from spark_rapids_pytools.cloud_api.sp_types import DeployMode
from spark_rapids_pytools.common.utilities import ToolLogging
from spark_rapids_pytools.rapids.profiling import ProfilingAsLocal
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@


"""Wrapper class to run tools associated with RAPIDS Accelerator for Apache Spark plugin on DATABRICKS_AZURE."""
from pyrapids import CspEnv
from spark_rapids_tools import CspEnv
from spark_rapids_pytools.cloud_api.sp_types import DeployMode
from spark_rapids_pytools.common.utilities import ToolLogging
from spark_rapids_pytools.rapids.profiling import ProfilingAsLocal
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

"""Wrapper class to run tools associated with RAPIDS Accelerator for Apache Spark plugin on Dataproc."""

from pyrapids import CspEnv
from spark_rapids_tools import CspEnv
from spark_rapids_pytools.cloud_api.sp_types import DeployMode
from spark_rapids_pytools.common.utilities import ToolLogging
from spark_rapids_pytools.rapids.bootstrap import Bootstrap
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@


"""Wrapper class to run tools associated with RAPIDS Accelerator for Apache Spark plugin on AWS-EMR."""
from pyrapids import CspEnv
from spark_rapids_tools import CspEnv
from spark_rapids_pytools.cloud_api.sp_types import DeployMode
from spark_rapids_pytools.common.utilities import ToolLogging
from spark_rapids_pytools.rapids.bootstrap import Bootstrap
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@


"""Wrapper class to run tools associated with RAPIDS Accelerator for Apache Spark plugin on On-Prem cluster."""
from pyrapids import CspEnv
from spark_rapids_tools import CspEnv
from spark_rapids_pytools.cloud_api.sp_types import DeployMode
from spark_rapids_pytools.common.utilities import ToolLogging
from spark_rapids_pytools.rapids.profiling import ProfilingAsLocal
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@

from typing import ClassVar, Type, Optional

from pyrapids.cloud.cluster import register_client_cluster, register_cluster_prop_mgr, ClusterPropMgr, ClientCluster
from pyrapids.utils.propmanager import PropValidatorSchema
from spark_rapids_tools.cloud.cluster import register_client_cluster, register_cluster_prop_mgr, ClusterPropMgr, ClientCluster
from spark_rapids_tools.utils.propmanager import PropValidatorSchema


class DBAwsClusterSchema(PropValidatorSchema):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@

from typing import ClassVar, Type

from pyrapids.cloud.cluster import ClientCluster, register_client_cluster, ClusterPropMgr, register_cluster_prop_mgr
from pyrapids.utils.propmanager import PropValidatorSchemaCamel, PropValidatorSchema
from spark_rapids_tools.cloud.cluster import ClientCluster, register_client_cluster, ClusterPropMgr, register_cluster_prop_mgr
from spark_rapids_tools.utils.propmanager import PropValidatorSchemaCamel, PropValidatorSchema


class DataprocClusterSchema(PropValidatorSchemaCamel):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@

from typing import ClassVar, Type

from pyrapids.cloud.cluster import register_cluster_prop_mgr, register_client_cluster, ClusterPropMgr, ClientCluster
from pyrapids.utils.propmanager import PropValidatorSchemaUpper, PropValidatorSchema
from spark_rapids_tools.cloud.cluster import register_cluster_prop_mgr, register_client_cluster, ClusterPropMgr, ClientCluster
from spark_rapids_tools.utils.propmanager import PropValidatorSchemaUpper, PropValidatorSchema


class EmrClusterSchema(PropValidatorSchemaUpper):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@
from typing import ClassVar, Type
from typing_extensions import TypedDict
from pydantic import ConfigDict
from pyrapids.cloud.cluster import ClientCluster, ClusterPropMgr, register_cluster_prop_mgr, register_client_cluster
from pyrapids.utils.propmanager import PropValidatorSchema
from pyrapids.utils.util import to_camel_case
from spark_rapids_tools.cloud.cluster import ClientCluster, ClusterPropMgr, register_cluster_prop_mgr, register_client_cluster
from spark_rapids_tools.utils.propmanager import PropValidatorSchema
from spark_rapids_tools.utils.util import to_camel_case


class OnPremDriverConfigSchema(TypedDict):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@

"""init file of the user CLI used to run the tools"""

from .pyrapids_cli import PyRapids
from .tools_cli import ToolsCLI

__all__ = [
'PyRapids'
'ToolsCLI'
]
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@
from pydantic.dataclasses import dataclass
from pydantic_core import PydanticCustomError

from pyrapids.cloud import ClientCluster
from pyrapids.exceptions import IllegalArgumentError
from pyrapids.utils import AbstractPropContainer, is_http_file
from spark_rapids_tools.cloud import ClientCluster
from spark_rapids_tools.exceptions import IllegalArgumentError
from spark_rapids_tools.utils import AbstractPropContainer, is_http_file
from spark_rapids_pytools.cloud_api.sp_types import DeployMode
from spark_rapids_pytools.common.utilities import ToolLogging
from spark_rapids_pytools.rapids.qualification import QualGpuClusterReshapeType
Expand Down Expand Up @@ -105,7 +105,7 @@ class AbsToolUserArgModel:
'meta': {},
'toolArgs': {}
})
logger: ClassVar[Logger] = ToolLogging.get_and_setup_logger('pyrapids.argparser')
logger: ClassVar[Logger] = ToolLogging.get_and_setup_logger('spark_rapids_tools.argparser')
tool_name: ClassVar[str] = None

@classmethod
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,15 @@

import fire

from pyrapids.enums import QualGpuClusterReshapeType
from pyrapids.utils.util import gen_app_banner
from spark_rapids_tools.enums import QualGpuClusterReshapeType
from spark_rapids_tools.utils.util import gen_app_banner
from spark_rapids_pytools.rapids.bootstrap import Bootstrap
from spark_rapids_pytools.rapids.profiling import ProfilingAsLocal
from spark_rapids_pytools.rapids.qualification import QualificationAsLocal
from .argprocessor import AbsToolUserArgModel


class PyRapids(object): # pylint: disable=too-few-public-methods
class ToolsCLI(object): # pylint: disable=too-few-public-methods
"""CLI that provides a runtime environment that simplifies running cost and performance analysis
using the RAPIDS Accelerator for Apache Spark.
Expand Down Expand Up @@ -167,7 +167,7 @@ def main():
# Make Python Fire not use a pager when it prints a help text
fire.core.Display = lambda lines, out: out.write('\n'.join(lines) + '\n')
print(gen_app_banner())
fire.Fire(PyRapids())
fire.Fire(ToolsCLI())


if __name__ == '__main__':
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -190,18 +190,18 @@ class CspPath(metaclass=CspPathMeta):
Create a new path subclass from a gcs URI:
>>> gs_path = CspPath('gs://bucket-name/folder_00/subfolder_01')
<pyrapids.storagelib.gcs.gcpath.GcsPath object at ...>
<spark_rapids_tools.storagelib.gcs.gcpath.GcsPath object at ...>
or from S3 URI:
>>> s3_path = CspPath('s3://bucket-name/folder_00/subfolder_01')
<pyrapids.storagelib.s3.s3path.S3Path object at ...>
<spark_rapids_tools.storagelib.s3.s3path.S3Path object at ...>
or from local file URI:
>>> local_path1, local_path2 = (CspPath('~/my_folder'), CspPath('file:///my_folder'))
<pyrapids.storagelib.local.localpath.LocalPath object at ...,
pyrapids.storagelib.local.localpath.LocalPath object at ...>
<spark_rapids_tools.storagelib.local.localpath.LocalPath object at ...,
spark_rapids_tools.storagelib.local.localpath.LocalPath object at ...>
Print the data from the file with `open_input_file()`:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"""Wrapper for the S3 File system"""


from pyrapids.storagelib.cspfs import register_fs_class, CspFs
from spark_rapids_tools.storagelib.cspfs import register_fs_class, CspFs


@register_fs_class('s3', 'S3FileSystem')
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

from typing import Optional, ClassVar, Type

from pyrapids.utils.propmanager import PropValidatorSchemaCamel, PropValidatorSchema, AbstractPropContainer
from spark_rapids_tools.utils.propmanager import PropValidatorSchemaCamel, PropValidatorSchema, AbstractPropContainer


class AutoTunerInputSchema(PropValidatorSchemaCamel):
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@
import yaml
from pydantic import BaseModel, ConfigDict, model_validator, ValidationError

from pyrapids.exceptions import JsonLoadException, YamlLoadException, InvalidPropertiesSchema
from pyrapids.storagelib.csppath import CspPath, CspPathT
from pyrapids.utils.util import to_camel_case, to_camel_capital_case, get_elem_from_dict, get_elem_non_safe
from spark_rapids_tools.exceptions import JsonLoadException, YamlLoadException, InvalidPropertiesSchema
from spark_rapids_tools.storagelib.csppath import CspPath, CspPathT
from spark_rapids_tools.utils.util import to_camel_case, to_camel_capital_case, get_elem_from_dict, get_elem_non_safe


def load_json(file_path: Union[str, CspPathT]) -> Any:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from pydantic import ValidationError, AnyHttpUrl, TypeAdapter

import spark_rapids_pytools
from pyrapids.exceptions import CspPathAttributeError
from spark_rapids_tools.exceptions import CspPathAttributeError


def get_elem_from_dict(data, keys):
Expand Down Expand Up @@ -87,8 +87,8 @@ def to_snake_case(word: str) -> str:


def dump_tool_usage(tool_name: Optional[str], raise_sys_exit: Optional[bool] = True):
imported_module = __import__('pyrapids.cmdli', globals(), locals(), ['PyRapids'])
wrapper_clzz = getattr(imported_module, 'PyRapids')
imported_module = __import__('spark_rapids_tools.cmdli', globals(), locals(), ['ToolsCLI'])
wrapper_clzz = getattr(imported_module, 'ToolsCLI')
help_name = 'ascli'
usage_cmd = f'{tool_name} --help'
try:
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def get_test_resources_path():
import importlib_resources
else:
import importlib.resources as importlib_resources
pkg = importlib_resources.files('tests.pyrapids_unit')
pkg = importlib_resources.files('tests.spark_rapids_tools_ut')
return pkg / 'resources'


Expand All @@ -47,7 +47,7 @@ def gen_cpu_cluster_props():
all_csps = csps + ['onprem']


class PyrapidsUnitTest: # pylint: disable=too-few-public-methods
class SparkRapidsToolsUT: # pylint: disable=too-few-public-methods

@pytest.fixture(autouse=True)
def get_ut_data_dir(self):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@

import pytest # pylint: disable=import-error

from pyrapids import CspPath
from pyrapids.cloud import ClientCluster
from pyrapids.exceptions import InvalidPropertiesSchema
from .conftest import PyrapidsUnitTest, all_cpu_cluster_props
from spark_rapids_tools import CspPath
from spark_rapids_tools.cloud import ClientCluster
from spark_rapids_tools.exceptions import InvalidPropertiesSchema
from .conftest import SparkRapidsToolsUT, all_cpu_cluster_props


class TestClusterCSP(PyrapidsUnitTest): # pylint: disable=too-few-public-methods
class TestClusterCSP(SparkRapidsToolsUT): # pylint: disable=too-few-public-methods
"""
Class testing identifying the cluster type by comparing the properties to
the defined Schema
Expand Down
Loading

0 comments on commit 02ef108

Please sign in to comment.