[GSProcessing CI Update] Add black check to GSProcessing Lint Check

.github/workflow_scripts/gsprocessing_lint.sh

@@ -4,6 +4,7 @@ cd ../../
 set -ex
 
 pip install pylint==2.17.5
-
 pylint --rcfile=./tests/lint/pylintrc ./graphstorm-processing/graphstorm_processing/
 
+pip install black==24.2.0
+black --check ./graphstorm-processing/

graphstorm-processing/graphstorm_processing/config/categorical_configs.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 from typing import Mapping
 from .feature_config_base import FeatureConfig
 

graphstorm-processing/graphstorm_processing/config/config_conversion/__init__.py

@@ -1,4 +1,5 @@
 """
 This module contains the classes to convert different specifications to GSProcessing
 """
+
 from .gconstruct_converter import GConstructConfigConverter

graphstorm-processing/graphstorm_processing/config/config_conversion/converter_base.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 import abc
 from abc import abstractmethod
 from typing import Any

graphstorm-processing/graphstorm_processing/config/config_conversion/meta_configuration.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 from dataclasses import dataclass
 from typing import Optional
 

graphstorm-processing/graphstorm_processing/config/feature_config_base.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 import abc
 from typing import Any, Mapping, Sequence
 

graphstorm-processing/graphstorm_processing/config/label_config_base.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 import abc
 from typing import Any, Dict, Optional
 

graphstorm-processing/graphstorm_processing/config/numerical_configs.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 from typing import Mapping
 import numbers
 

graphstorm-processing/graphstorm_processing/data_transformations/dist_label_loader.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 from dataclasses import dataclass
 from typing import Dict, List
 

graphstorm-processing/graphstorm_processing/data_transformations/dist_transformations/base_dist_transformation.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 from abc import ABC, abstractmethod
 from typing import Sequence
 

graphstorm-processing/graphstorm_processing/data_transformations/dist_transformations/dist_bucket_numerical_transformation.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 from typing import List
 
 from pyspark.sql import DataFrame

graphstorm-processing/graphstorm_processing/data_transformations/dist_transformations/dist_category_transformation.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 from typing import Dict, List, Optional, Sequence
 
 import numpy as np

graphstorm-processing/graphstorm_processing/data_transformations/dist_transformations/dist_label_transformation.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 import json
 from typing import Dict, Sequence
 

graphstorm-processing/graphstorm_processing/data_transformations/dist_transformations/dist_noop_transformation.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 from typing import List, Optional
 
 from pyspark.sql import DataFrame

graphstorm-processing/graphstorm_processing/data_transformations/dist_transformations/dist_numerical_transformation.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 import logging
 from typing import Optional, Sequence
 import uuid

graphstorm-processing/graphstorm_processing/data_transformations/s3_utils.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 import logging
 from typing import List, Optional
 

graphstorm-processing/graphstorm_processing/data_transformations/spark_utils.py

@@ -8,6 +8,7 @@
 that only exists for the Python 3.9/Spark 3.2 container
 https://github.com/aws/sagemaker-spark-container/blob/4ef476fd535040f245def3d38c59fe43062e88a9/src/smspark/bootstrapper.py#L375
 """
+
 import logging
 import uuid
 from typing import Tuple, Sequence

graphstorm-processing/graphstorm_processing/distributed_executor.py

@@ -45,6 +45,7 @@
     When set to true (default), we add reverse edges for each edge type.
 
 """
+
 import dataclasses
 import argparse
 import json

graphstorm-processing/graphstorm_processing/graph_loaders/heterogeneous_graphloader.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 from .loader_base import GraphLoader
 
 

graphstorm-processing/graphstorm_processing/graph_loaders/loader_base.py

@@ -15,6 +15,7 @@
 
 Base class for graph data processing.
 """
+
 from abc import ABC
 from typing import Dict, List
 import os

graphstorm-processing/graphstorm_processing/graph_loaders/row_count_utils.py

@@ -15,6 +15,7 @@
 
 This module is used to determine row counts for Parquet files.
 """
+
 import logging
 import os
 from typing import Dict, List, Sequence

graphstorm-processing/graphstorm_processing/repartition_files.py

@@ -24,6 +24,7 @@
 the same edge/node type have the same number of rows per corresponding part-file.
 The output is written to storage and a new metadata JSON file is generated.
 """
+
 import argparse
 import json
 import logging
@@ -931,10 +932,10 @@ def main():
                         "Repartitioning structure files for reverse edge type '%s'",
                         reverse_edge_type_name,
                     )
-                    edge_structure_meta[
-                        reverse_edge_type_name
-                    ] = repartitioner.repartition_parquet_files(
-                        edge_structure_meta[reverse_edge_type_name], most_frequent_counts
+                    edge_structure_meta[reverse_edge_type_name] = (
+                        repartitioner.repartition_parquet_files(
+                            edge_structure_meta[reverse_edge_type_name], most_frequent_counts
+                        )
                     )
 
             # Repartition edge feature files if the row counts don't match the most frequent

graphstorm-processing/pyproject.toml

@@ -36,7 +36,7 @@ coverage = ">=7.0.0"
 sphinx = ">=6.0.0"
 mypy = ">=1.0.0"
 types-psutil = "^5.9.5.15"
-black = "~23.7.0"
+black = "~24.2.0"
 pre-commit = "^3.3.3"
 types-mock = "^5.1.0.1"
 pylint = "~2.17.5"

graphstorm-processing/scripts/convert_gconstruct_config.py

@@ -20,6 +20,7 @@
 Entry point for graph spec conversion. Allows us to convert a graph
 data specification from GConstruct to the format used by GSProcessing.
 """
+
 import argparse
 import json
 

graphstorm-processing/scripts/run_distributed_processing.py

@@ -65,6 +65,7 @@
     --sm-estimator-parameters \"volume_size=100 subnets=['subnet-123','subnet-345']
     security_group_ids=['sg-1234','sg-3456']\"
 """
+
 import argparse
 import logging
 from pathlib import Path

graphstorm-processing/scripts/run_repartitioning.py

@@ -57,6 +57,7 @@
     security_group_ids=['sg-1234','sg-3456']\"``
 
 """
+
 import argparse
 import logging
 from time import strftime, gmtime

graphstorm-processing/scripts/script_utils.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 import argparse
 from ast import literal_eval
 from typing import Any, Dict, Sequence, Optional

graphstorm-processing/tests/conftest.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 import os
 import sys
 import logging

graphstorm-processing/tests/test_dist_bucket_transformation.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 from pyspark.sql import DataFrame, SparkSession
 import numpy as np
 from numpy.testing import assert_array_equal

graphstorm-processing/tests/test_dist_category_transformation.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 from typing import Tuple, Iterator
 import os
 import pytest

graphstorm-processing/tests/test_dist_label_loader.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 from typing import List
 
 import numpy as np

graphstorm-processing/tests/test_dist_noop_transformation.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 from numpy.testing import assert_array_equal
 from pyspark.sql import functions as F, DataFrame, SparkSession
 

graphstorm-processing/tests/test_dist_numerical_transformation.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 import os
 
 import pytest

graphstorm-processing/tests/test_repartition_files.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 import json
 from pathlib import Path
 import os