From bec93f066e1e24320d96ca12d5e8a1867ccdae86 Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Mon, 12 Aug 2024 13:33:25 -0700 Subject: [PATCH] Remove unused argument `--target_platform` (#1279) Signed-off-by: Partho Sarthi --- .../cloud_api/sp_types.py | 6 ----- .../rapids/qualification.py | 2 -- .../spark_rapids_tools/cmdli/argprocessor.py | 22 +------------------ .../src/spark_rapids_tools/cmdli/tools_cli.py | 10 +-------- 4 files changed, 2 insertions(+), 38 deletions(-) diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py b/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py index c3a3fd4d1..d61e12393 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py @@ -87,12 +87,6 @@ class ClusterState(EnumeratedType): UNKNOWN = 'unknown' -class TargetPlatform(EnumeratedType): - """Determine CostSavings for target platform based on OnPrem cluster configuration""" - DATAPROC = 'dataproc' - NONE = None - - class SparkNodeType(EnumeratedType): """ Node type from Spark perspective. We either have a master node or a worker node. diff --git a/user_tools/src/spark_rapids_pytools/rapids/qualification.py b/user_tools/src/spark_rapids_pytools/rapids/qualification.py index b1e3c1dbe..d2ae44174 100644 --- a/user_tools/src/spark_rapids_pytools/rapids/qualification.py +++ b/user_tools/src/spark_rapids_pytools/rapids/qualification.py @@ -275,8 +275,6 @@ def _process_custom_args(self) -> None: cuda_arg = self.wrapper_options.get('cuda') if cuda_arg is not None: cuda = cuda_arg - target_platform = self.wrapper_options.get('targetPlatform') - self.ctxt.set_ctxt('targetPlatform', target_platform) self.ctxt.set_ctxt('gpuPerMachine', gpu_per_machine) self.ctxt.set_ctxt('gpuDevice', gpu_device) self.ctxt.set_ctxt('cuda', cuda) diff --git a/user_tools/src/spark_rapids_tools/cmdli/argprocessor.py b/user_tools/src/spark_rapids_tools/cmdli/argprocessor.py index 66f6f9e4b..7d53bfd0e 100644 --- a/user_tools/src/spark_rapids_tools/cmdli/argprocessor.py +++ b/user_tools/src/spark_rapids_tools/cmdli/argprocessor.py @@ -446,7 +446,6 @@ class QualifyUserArgModel(ToolUserArgModel): Represents the arguments collected by the user to run the qualification tool. This is used as doing preliminary validation against some of the common pattern """ - target_platform: Optional[CspEnv] = None filter_apps: Optional[QualFilterApp] = None gpu_cluster_recommendation: Optional[QualGpuClusterReshapeType] = None estimation_model_args: Optional[Dict] = dataclasses.field(default_factory=dict) @@ -454,7 +453,6 @@ class QualifyUserArgModel(ToolUserArgModel): def init_tool_args(self) -> None: self.p_args['toolArgs']['platform'] = self.platform self.p_args['toolArgs']['savingsCalculations'] = False - self.p_args['toolArgs']['targetPlatform'] = self.target_platform # check the filter_apps argument if self.filter_apps is None: self.p_args['toolArgs']['filterApps'] = QualFilterApp.get_default() @@ -487,22 +485,6 @@ def build_tools_args(self) -> dict: # At this point, if the platform is still none, then we can set it to the default value # which is the onPrem platform. runtime_platform = self.get_or_set_platform() - # check the targetPlatform argument - if self.p_args['toolArgs']['targetPlatform']: - equivalent_pricing_list = runtime_platform.get_equivalent_pricing_platform() - if not equivalent_pricing_list: - # no target_platform for that runtime environment - self.logger.info( - 'Argument target_platform does not support the current cluster [%s]', runtime_platform) - self.p_args['toolArgs']['targetPlatform'] = None - else: - if not self.p_args['toolArgs']['targetPlatform'] in equivalent_pricing_list: - target_platform = self.p_args['toolArgs']['targetPlatform'] - raise PydanticCustomError( - 'invalid_argument', - f'The platform [{target_platform}] is currently ' - f'not supported to calculate savings from [{runtime_platform}] cluster\n Error:') - # process JVM arguments self.process_jvm_args() @@ -512,9 +494,7 @@ def build_tools_args(self) -> dict: 'outputFolder': self.output_folder, 'platformOpts': { 'credentialFile': None, - 'deployMode': DeployMode.LOCAL, - # used to be sent to the scala core java cmd - 'targetPlatform': self.p_args['toolArgs']['targetPlatform'] + 'deployMode': DeployMode.LOCAL }, 'migrationClustersProps': { 'cpuCluster': self.cluster, diff --git a/user_tools/src/spark_rapids_tools/cmdli/tools_cli.py b/user_tools/src/spark_rapids_tools/cmdli/tools_cli.py index 6a4ecab25..5780ef54d 100644 --- a/user_tools/src/spark_rapids_tools/cmdli/tools_cli.py +++ b/user_tools/src/spark_rapids_tools/cmdli/tools_cli.py @@ -39,7 +39,6 @@ def qualification(self, eventlogs: str = None, cluster: str = None, platform: str = None, - target_platform: str = None, output_folder: str = None, filter_apps: str = None, custom_model_file: str = None, @@ -66,12 +65,6 @@ def qualification(self, Name or ID (for databricks platforms) of cluster or path to cluster-properties. :param platform: defines one of the following "onprem", "emr", "dataproc", "dataproc-gke", "databricks-aws", and "databricks-azure". - :param target_platform: Cost savings and speedup recommendation for comparable cluster in - target_platform based on on-premises cluster configuration. - - Currently only `dataproc` is supported for target_platform. - If not provided, the final report will be limited to GPU speedups only without - cost-savings. :param output_folder: path to store the output :param tools_jar: Path to a bundled jar including Rapids tool. The path is a local filesystem, or remote cloud storage url. If missing, the wrapper downloads the latest rapids-4-spark-tools_*.jar @@ -97,7 +90,7 @@ def qualification(self, https://docs.nvidia.com/spark-rapids/user-guide/latest/qualification/jar-usage.html#running-the-qualification-tool-standalone-on-spark-event-logs """ platform = Utils.get_value_or_pop(platform, rapids_options, 'p') - target_platform = Utils.get_value_or_pop(target_platform, rapids_options, 't') + tools_jar = Utils.get_value_or_pop(tools_jar, rapids_options, 't') output_folder = Utils.get_value_or_pop(output_folder, rapids_options, 'o') filter_apps = Utils.get_value_or_pop(filter_apps, rapids_options, 'f') verbose = Utils.get_value_or_pop(verbose, rapids_options, 'v', False) @@ -117,7 +110,6 @@ def qualification(self, eventlogs=eventlogs, cluster=cluster, platform=platform, - target_platform=target_platform, output_folder=output_folder, tools_jar=tools_jar, jvm_heap_size=jvm_heap_size,