From 8874ffc78e660064a14858b26112657dd23ba17f Mon Sep 17 00:00:00 2001 From: JalenCato Date: Thu, 21 Sep 2023 23:16:05 +0000 Subject: [PATCH 1/4] test --- .github/workflow_scripts/gsprocessing_pytest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflow_scripts/gsprocessing_pytest.sh b/.github/workflow_scripts/gsprocessing_pytest.sh index 820f1a2c14..9cbe8875c4 100644 --- a/.github/workflow_scripts/gsprocessing_pytest.sh +++ b/.github/workflow_scripts/gsprocessing_pytest.sh @@ -1,4 +1,5 @@ cd /usr/lib/spark/graphstorm/graphstorm-processing/ pip install . +echo "hello" python3 -m pytest . From cdce76aab1637903042d5225e4359ccd24e028e1 Mon Sep 17 00:00:00 2001 From: JalenCato Date: Thu, 21 Sep 2023 23:18:42 +0000 Subject: [PATCH 2/4] test --- graphstorm-processing/graphstorm_processing/constants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/graphstorm-processing/graphstorm_processing/constants.py b/graphstorm-processing/graphstorm_processing/constants.py index 510d8a06bc..a77191f98b 100644 --- a/graphstorm-processing/graphstorm_processing/constants.py +++ b/graphstorm-processing/graphstorm_processing/constants.py @@ -22,7 +22,8 @@ ################ Multi-numerical Limits ##################### MAX_COLUMNS_TO_IMPUTE = 50 -SUPPORTED_FILE_TYPES = ["csv", "parquet"] +SUPPORTED_FILE_TYPES = ["csv", "parquet", "test1"] + ################### Label Properties ######################## MIN_VALUE = "MIN_VALUE" From d4bc9bae1618a715d29ecdfe350fa3ad0884d6fb Mon Sep 17 00:00:00 2001 From: JalenCato Date: Thu, 21 Sep 2023 23:24:20 +0000 Subject: [PATCH 3/4] add script --- .../graphstorm_processing/data_transformations/s3_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/graphstorm-processing/graphstorm_processing/data_transformations/s3_utils.py b/graphstorm-processing/graphstorm_processing/data_transformations/s3_utils.py index 82826a7ddf..7ada6966c1 100644 --- a/graphstorm-processing/graphstorm_processing/data_transformations/s3_utils.py +++ b/graphstorm-processing/graphstorm_processing/data_transformations/s3_utils.py @@ -72,6 +72,7 @@ def get_bucket_region(bucket: str, s3_boto_client: boto3.client = None) -> str: def s3_path_remove_trailing(s3_path: str) -> str: """ Removes trailing slash from S3 path. + Test """ if s3_path.endswith("/"): return s3_path[:-1] From 941fc05f697d02fa817c8bd920733ff740cb0fe0 Mon Sep 17 00:00:00 2001 From: JalenCato Date: Thu, 21 Sep 2023 23:28:20 +0000 Subject: [PATCH 4/4] allow slash in the part --- .github/workflow_scripts/gsprocessing_pytest.sh | 1 - .github/workflows/continuous-integration.yml | 4 ++-- .github/workflows/gsprocessing-workflow.yml | 4 ++-- graphstorm-processing/graphstorm_processing/constants.py | 3 +-- .../graphstorm_processing/data_transformations/s3_utils.py | 1 - 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflow_scripts/gsprocessing_pytest.sh b/.github/workflow_scripts/gsprocessing_pytest.sh index 9cbe8875c4..820f1a2c14 100644 --- a/.github/workflow_scripts/gsprocessing_pytest.sh +++ b/.github/workflow_scripts/gsprocessing_pytest.sh @@ -1,5 +1,4 @@ cd /usr/lib/spark/graphstorm/graphstorm-processing/ pip install . -echo "hello" python3 -m pytest . diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 8aa5d49651..98b916e22d 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -10,14 +10,14 @@ on: paths-ignore: - '.github/workflow_scripts/gsprocessing_pytest.sh' - '.github/workflow_scripts/gsprocessing_lint.sh' - - 'graphstorm-processing/*' + - 'graphstorm-processing/**' - '.github/workflows/gsprocessing-workflow.yml' pull_request_target: types: [ labeled, opened, reopened, synchronize, ready_for_review ] paths-ignore: - '.github/workflow_scripts/gsprocessing_pytest.sh' - '.github/workflow_scripts/gsprocessing_lint.sh' - - 'graphstorm-processing/*' + - 'graphstorm-processing/**' - '.github/workflows/gsprocessing-workflow.yml' permissions: diff --git a/.github/workflows/gsprocessing-workflow.yml b/.github/workflows/gsprocessing-workflow.yml index 4fc17f708b..b19a3d0855 100644 --- a/.github/workflows/gsprocessing-workflow.yml +++ b/.github/workflows/gsprocessing-workflow.yml @@ -10,14 +10,14 @@ on: paths: - '.github/workflow_scripts/gsprocessing_pytest.sh' - '.github/workflow_scripts/gsprocessing_lint.sh' - - 'graphstorm-processing/*' + - 'graphstorm-processing/**' - '.github/workflows/gsprocessing-workflow.yml' pull_request_target: types: [ labeled, opened, reopened, synchronize, ready_for_review ] paths: - '.github/workflow_scripts/gsprocessing_pytest.sh' - '.github/workflow_scripts/gsprocessing_lint.sh' - - 'graphstorm-processing/*' + - 'graphstorm-processing/**' - '.github/workflows/gsprocessing-workflow.yml' permissions: diff --git a/graphstorm-processing/graphstorm_processing/constants.py b/graphstorm-processing/graphstorm_processing/constants.py index a77191f98b..510d8a06bc 100644 --- a/graphstorm-processing/graphstorm_processing/constants.py +++ b/graphstorm-processing/graphstorm_processing/constants.py @@ -22,8 +22,7 @@ ################ Multi-numerical Limits ##################### MAX_COLUMNS_TO_IMPUTE = 50 -SUPPORTED_FILE_TYPES = ["csv", "parquet", "test1"] - +SUPPORTED_FILE_TYPES = ["csv", "parquet"] ################### Label Properties ######################## MIN_VALUE = "MIN_VALUE" diff --git a/graphstorm-processing/graphstorm_processing/data_transformations/s3_utils.py b/graphstorm-processing/graphstorm_processing/data_transformations/s3_utils.py index 7ada6966c1..82826a7ddf 100644 --- a/graphstorm-processing/graphstorm_processing/data_transformations/s3_utils.py +++ b/graphstorm-processing/graphstorm_processing/data_transformations/s3_utils.py @@ -72,7 +72,6 @@ def get_bucket_region(bucket: str, s3_boto_client: boto3.client = None) -> str: def s3_path_remove_trailing(s3_path: str) -> str: """ Removes trailing slash from S3 path. - Test """ if s3_path.endswith("/"): return s3_path[:-1]