From 179e24794d2bc420452b51f4fa225f7a19b2f7ee Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 19 Apr 2024 11:09:15 -0400 Subject: [PATCH 01/20] Add verify-alpha-spec hook --- pyproject.toml | 3 + src/rapids_pre_commit_hooks/alpha_spec.py | 197 ++++++++++++ .../test_alpha_spec.py | 294 ++++++++++++++++++ 3 files changed, 494 insertions(+) create mode 100644 src/rapids_pre_commit_hooks/alpha_spec.py create mode 100644 test/rapids_pre_commit_hooks/test_alpha_spec.py diff --git a/pyproject.toml b/pyproject.toml index b7ec7fc..1eb9d05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,8 +32,10 @@ classifiers = [ ] requires-python = ">=3.9" dependencies = [ + "PyYAML", "bashlex", "gitpython", + "packaging", "rich", ] @@ -46,6 +48,7 @@ test = [ [project.scripts] verify-conda-yes = "rapids_pre_commit_hooks.shell.verify_conda_yes:main" verify-copyright = "rapids_pre_commit_hooks.copyright:main" +verify-alpha-spec = "rapids_pre_commit_hooks.alpha_spec:main" [tool.setuptools] packages = { "find" = { where = ["src"] } } diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py new file mode 100644 index 0000000..75a1ac8 --- /dev/null +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -0,0 +1,197 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import reduce + +import yaml +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet + +from .lint import LintMain + +RAPIDS_VERSIONED_PACKAGES = { + "rmm", + "pylibcugraphops", + "pylibcugraph", + "nx-cugraph", + "dask-cudf", + "cuspatial", + "cuproj", + "cuml", + "cugraph", + "cudf", + "ptxcompiler", + "cubinlinker", + "cugraph-dgl", + "cugraph-pyg", + "cugraph-equivariant", + "raft-dask", + "pylibwholegraph", + "pylibraft", + "cuxfilter", + "cucim", + "ucx-py", + "ucxx", + "pynvjitlink", + "distributed-ucxx", +} + +ALPHA_SPECIFIER = ">=0.0.0a0" + + +def check_package_spec(linter, args, node): + if node.tag == "tag:yaml.org,2002:str": + req = Requirement(node.value) + if req.name in RAPIDS_VERSIONED_PACKAGES: + has_alpha_spec = any( + filter(lambda s: str(s) == ALPHA_SPECIFIER, req.specifier) + ) + if args.mode == "development" and not has_alpha_spec: + req.specifier &= ALPHA_SPECIFIER + linter.add_warning( + (node.start_mark.index, node.end_mark.index), + f"add alpha spec for RAPIDS package {req.name}", + ).add_replacement( + (node.start_mark.index, node.end_mark.index), str(req) + ) + elif args.mode == "release" and has_alpha_spec: + req.specifier = reduce( + lambda ss, s: ss & str(s), + filter(lambda s: str(s) != ALPHA_SPECIFIER, req.specifier), + SpecifierSet(), + ) + linter.add_warning( + (node.start_mark.index, node.end_mark.index), + f"remove alpha spec for RAPIDS package {req.name}", + ).add_replacement( + (node.start_mark.index, node.end_mark.index), str(req) + ) + + +def check_packages(linter, args, node): + if node.tag == "tag:yaml.org,2002:seq": + for package_spec in node.value: + check_package_spec(linter, args, package_spec) + + +def check_common(linter, args, node): + if node.tag == "tag:yaml.org,2002:seq": + for dependency_set in node.value: + if dependency_set.tag == "tag:yaml.org,2002:map": + for dependency_set_key, dependency_set_value in dependency_set.value: + if ( + dependency_set_key.tag == "tag:yaml.org,2002:str" + and dependency_set_key.value == "output_types" + ): + output_types = dependency_set_value + elif ( + dependency_set_key.tag == "tag:yaml.org,2002:str" + and dependency_set_key.value == "packages" + ): + packages = dependency_set_value + if output_types.tag == "tag:yaml.org,2002:seq" and any( + t.tag == "tag:yaml.org,2002:str" and t.value == "pyproject" + for t in output_types.value + ): + check_packages(linter, args, packages) + elif ( + output_types.tag == "tag:yaml.org,2002:str" + and output_types.value == "pyproject" + ): + check_packages(linter, args, packages) + + +def check_matrices(linter, args, node): + if node.tag == "tag:yaml.org,2002:seq": + for item in node.value: + if item.tag == "tag:yaml.org,2002:map": + for matrix_key, matrix_value in item.value: + if ( + matrix_key.tag == "tag:yaml.org,2002:str" + and matrix_key.value == "packages" + ): + check_packages(linter, args, matrix_value) + + +def check_specific(linter, args, node): + if node.tag == "tag:yaml.org,2002:seq": + for matrix_matcher in node.value: + if matrix_matcher.tag == "tag:yaml.org,2002:map": + for matrix_matcher_key, matrix_matcher_value in matrix_matcher.value: + if ( + matrix_matcher_key.tag == "tag:yaml.org,2002:str" + and matrix_matcher_key.value == "output_types" + ): + output_types = matrix_matcher_value + elif ( + matrix_matcher_key.tag == "tag:yaml.org,2002:str" + and matrix_matcher_key.value == "matrices" + ): + matrices = matrix_matcher_value + if output_types.tag == "tag:yaml.org,2002:seq" and any( + t.tag == "tag:yaml.org,2002:str" and t.value == "pyproject" + for t in output_types.value + ): + check_matrices(linter, args, matrices) + elif ( + output_types.tag == "tag:yaml.org,2002:str" + and output_types.value == "pyproject" + ): + check_matrices(linter, args, matrices) + + +def check_dependencies(linter, args, node): + if node.tag == "tag:yaml.org,2002:map": + for _, dependencies_value in node.value: + if dependencies_value.tag == "tag:yaml.org,2002:map": + for dependency_key, dependency_value in dependencies_value.value: + if dependency_key.tag == "tag:yaml.org,2002:str": + if dependency_key.value == "common": + check_common(linter, args, dependency_value) + elif dependency_key.value == "specific": + check_specific(linter, args, dependency_value) + + +def check_root(linter, args, node): + if node.tag == "tag:yaml.org,2002:map": + for root_key, root_value in node.value: + if ( + root_key.tag == "tag:yaml.org,2002:str" + and root_key.value == "dependencies" + ): + check_dependencies(linter, args, root_value) + + +def check_alpha_spec(linter, args): + check_root(linter, args, yaml.compose(linter.content)) + + +def main(): + m = LintMain() + m.argparser.description = ( + "Verify that RAPIDS packages in dependencies.yaml do (or do not) have " + "the alpha spec." + ) + m.argparser.add_argument( + "--mode", + help="mode to use (development has alpha spec, release does not)", + choices=["development", "release"], + default="development", + ) + with m.execute() as ctx: + ctx.add_check(check_alpha_spec) + + +if __name__ == "__main__": + main() diff --git a/test/rapids_pre_commit_hooks/test_alpha_spec.py b/test/rapids_pre_commit_hooks/test_alpha_spec.py new file mode 100644 index 0000000..98073dc --- /dev/null +++ b/test/rapids_pre_commit_hooks/test_alpha_spec.py @@ -0,0 +1,294 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from itertools import chain +from textwrap import dedent +from unittest.mock import Mock, call, patch + +import pytest +import yaml + +from rapids_pre_commit_hooks import alpha_spec, lint + + +@pytest.mark.parametrize( + ["package", "content", "mode", "replacement"], + [ + *chain( + *( + [ + (p, p, "development", f"{p}>=0.0.0a0"), + (p, p, "release", None), + (p, f"{p}>=0.0.0a0", "development", None), + (p, f"{p}>=0.0.0a0", "release", p), + ] + for p in alpha_spec.RAPIDS_VERSIONED_PACKAGES + ) + ), + ("cuml", "cuml>=24.04,<=24.06", "development", "cuml<=24.06,>=0.0.0a0,>=24.04"), + ("cuml", "cuml>=24.04,<=24.06,>=0.0.0a0", "release", "cuml<=24.06,>=24.04"), + ("packaging", "packaging", "development", None), + ], +) +def test_check_package_spec(package, content, mode, replacement): + args = Mock(mode=mode) + linter = lint.Linter("dependencies.yaml", content) + composed = yaml.compose(content) + alpha_spec.check_package_spec(linter, args, composed) + if replacement is None: + assert linter.warnings == [] + else: + expected_linter = lint.Linter("dependencies.yaml", content) + expected_linter.add_warning( + (composed.start_mark.index, composed.end_mark.index), + f"{'add' if mode == 'development' else 'remove'} " + f"alpha spec for RAPIDS package {package}", + ).add_replacement((0, len(content)), replacement) + assert linter.warnings == expected_linter.warnings + + +@pytest.mark.parametrize( + ["content", "indices"], + [ + ( + dedent( + """\ + - package_a + - package_b + """ + ), + [0, 1], + ), + ( + "null", + [], + ), + ], +) +def test_check_packages(content, indices): + with patch( + "rapids_pre_commit_hooks.alpha_spec.check_package_spec", Mock() + ) as mock_check_package_spec: + args = Mock() + linter = lint.Linter("dependencies.yaml", content) + composed = yaml.compose(content) + alpha_spec.check_packages(linter, args, composed) + assert mock_check_package_spec.mock_calls == [ + call(linter, args, composed.value[i]) for i in indices + ] + + +@pytest.mark.parametrize( + ["content", "indices"], + [ + ( + dedent( + """\ + - output_types: [pyproject, conda] + packages: + - package_a + - output_types: [conda] + packages: + - package_b + - packages: + - package_c + output_types: pyproject + """ + ), + [(0, 1), (2, 0)], + ), + ], +) +def test_check_common(content, indices): + with patch( + "rapids_pre_commit_hooks.alpha_spec.check_packages", Mock() + ) as mock_check_packages: + args = Mock() + linter = lint.Linter("dependencies.yaml", content) + composed = yaml.compose(content) + alpha_spec.check_common(linter, args, composed) + assert mock_check_packages.mock_calls == [ + call(linter, args, composed.value[i].value[j][1]) for i, j in indices + ] + + +@pytest.mark.parametrize( + ["content", "indices"], + [ + ( + dedent( + """\ + - matrix: + arch: x86_64 + packages: + - package_a + - packages: + - package_b + matrix: + """ + ), + [(0, 1), (1, 0)], + ), + ], +) +def test_check_matrices(content, indices): + with patch( + "rapids_pre_commit_hooks.alpha_spec.check_packages", Mock() + ) as mock_check_packages: + args = Mock() + linter = lint.Linter("dependencies.yaml", content) + composed = yaml.compose(content) + alpha_spec.check_matrices(linter, args, composed) + assert mock_check_packages.mock_calls == [ + call(linter, args, composed.value[i].value[j][1]) for i, j in indices + ] + + +@pytest.mark.parametrize( + ["content", "indices"], + [ + ( + dedent( + """\ + - output_types: [pyproject, conda] + matrices: + - matrix: + arch: x86_64 + packages: + - package_a + - output_types: [conda] + matrices: + - matrix: + arch: x86_64 + packages: + - package_b + - matrices: + - matrix: + arch: x86_64 + packages: + - package_c + output_types: pyproject + """ + ), + [(0, 1), (2, 0)], + ), + ], +) +def test_check_specific(content, indices): + with patch( + "rapids_pre_commit_hooks.alpha_spec.check_matrices", Mock() + ) as mock_check_matrices: + args = Mock() + linter = lint.Linter("dependencies.yaml", content) + composed = yaml.compose(content) + alpha_spec.check_specific(linter, args, composed) + assert mock_check_matrices.mock_calls == [ + call(linter, args, composed.value[i].value[j][1]) for i, j in indices + ] + + +@pytest.mark.parametrize( + ["content", "common_indices", "specific_indices"], + [ + ( + dedent( + """\ + set_a: + common: + - output_types: [pyproject] + packages: + - package_a + specific: + - output_types: [pyproject] + matrices: + - matrix: + arch: x86_64 + packages: + - package_b + set_b: + specific: + - output_types: [pyproject] + matrices: + - matrix: + arch: x86_64 + packages: + - package_c + common: + - output_types: [pyproject] + packages: + - package_d + """ + ), + [(0, 0), (1, 1)], + [(0, 1), (1, 0)], + ), + ], +) +def test_check_dependencies(content, common_indices, specific_indices): + with patch( + "rapids_pre_commit_hooks.alpha_spec.check_common", Mock() + ) as mock_check_common, patch( + "rapids_pre_commit_hooks.alpha_spec.check_specific", Mock() + ) as mock_check_specific: + args = Mock() + linter = lint.Linter("dependencies.yaml", content) + composed = yaml.compose(content) + alpha_spec.check_dependencies(linter, args, composed) + assert mock_check_common.mock_calls == [ + call(linter, args, composed.value[i][1].value[j][1]) for i, j in common_indices + ] + assert mock_check_specific.mock_calls == [ + call(linter, args, composed.value[i][1].value[j][1]) + for i, j in specific_indices + ] + + +@pytest.mark.parametrize( + ["content", "indices"], + [ + ( + dedent( + """\ + files: {} + channels: [] + dependencies: {} + """ + ), + [2], + ), + ], +) +def test_check_root(content, indices): + with patch( + "rapids_pre_commit_hooks.alpha_spec.check_dependencies", Mock() + ) as mock_check_dependencies: + args = Mock() + linter = lint.Linter("dependencies.yaml", content) + composed = yaml.compose(content) + alpha_spec.check_root(linter, args, composed) + assert mock_check_dependencies.mock_calls == [ + call(linter, args, composed.value[i][1]) for i in indices + ] + + +def test_check_alpha_spec(): + CONTENT = "dependencies: []" + with patch( + "rapids_pre_commit_hooks.alpha_spec.check_root", Mock() + ) as mock_check_root, patch("yaml.compose", Mock()) as mock_yaml_compose: + args = Mock() + linter = lint.Linter("dependencies.yaml", CONTENT) + alpha_spec.check_alpha_spec(linter, args) + mock_yaml_compose.assert_called_once_with(CONTENT) + mock_check_root.assert_called_once_with(linter, args, mock_yaml_compose()) From f106215b9fd5d3f46013f64d89b14a4aad4faea4 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 19 Apr 2024 11:13:47 -0400 Subject: [PATCH 02/20] Add to .pre-commit-hooks.yaml --- .pre-commit-hooks.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 566452c..d63ae4c 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -32,3 +32,10 @@ pyproject[.]toml$| setup[.]cfg$ args: [--fix] +- id: verify-alpha-spec + name: verify-alpha-spec + description: make sure RAPIDS package have correct alpha spec + entry: verify-alpha-spec + language: python + files: dependencies[.]yaml$ + args: [--fix] From b63ef5d9deefb18dc29e19dae0365576d2175b41 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 19 Apr 2024 12:13:40 -0400 Subject: [PATCH 03/20] Add alpha spec to requirements.txt too --- src/rapids_pre_commit_hooks/alpha_spec.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py index 75a1ac8..0a30533 100644 --- a/src/rapids_pre_commit_hooks/alpha_spec.py +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -49,6 +49,11 @@ ALPHA_SPECIFIER = ">=0.0.0a0" +ALPHA_SPEC_OUTPUT_TYPES = { + "pyproject", + "requirements", +} + def check_package_spec(linter, args, node): if node.tag == "tag:yaml.org,2002:str": @@ -101,13 +106,14 @@ def check_common(linter, args, node): ): packages = dependency_set_value if output_types.tag == "tag:yaml.org,2002:seq" and any( - t.tag == "tag:yaml.org,2002:str" and t.value == "pyproject" + t.tag == "tag:yaml.org,2002:str" + and t.value in ALPHA_SPEC_OUTPUT_TYPES for t in output_types.value ): check_packages(linter, args, packages) elif ( output_types.tag == "tag:yaml.org,2002:str" - and output_types.value == "pyproject" + and output_types.value in ALPHA_SPEC_OUTPUT_TYPES ): check_packages(linter, args, packages) @@ -140,13 +146,14 @@ def check_specific(linter, args, node): ): matrices = matrix_matcher_value if output_types.tag == "tag:yaml.org,2002:seq" and any( - t.tag == "tag:yaml.org,2002:str" and t.value == "pyproject" + t.tag == "tag:yaml.org,2002:str" + and t.value in ALPHA_SPEC_OUTPUT_TYPES for t in output_types.value ): check_matrices(linter, args, matrices) elif ( output_types.tag == "tag:yaml.org,2002:str" - and output_types.value == "pyproject" + and output_types.value in ALPHA_SPEC_OUTPUT_TYPES ): check_matrices(linter, args, matrices) From 7a29eea3f9a468ecf16502301934927d2325853d Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 26 Apr 2024 15:12:25 -0400 Subject: [PATCH 04/20] Fix formatting --- .../test_alpha_spec.py | 126 +++++++++--------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/test/rapids_pre_commit_hooks/test_alpha_spec.py b/test/rapids_pre_commit_hooks/test_alpha_spec.py index 98073dc..6dacb9e 100644 --- a/test/rapids_pre_commit_hooks/test_alpha_spec.py +++ b/test/rapids_pre_commit_hooks/test_alpha_spec.py @@ -64,9 +64,9 @@ def test_check_package_spec(package, content, mode, replacement): ( dedent( """\ - - package_a - - package_b - """ + - package_a + - package_b + """ ), [0, 1], ), @@ -95,16 +95,16 @@ def test_check_packages(content, indices): ( dedent( """\ - - output_types: [pyproject, conda] - packages: - - package_a - - output_types: [conda] - packages: - - package_b - - packages: - - package_c - output_types: pyproject - """ + - output_types: [pyproject, conda] + packages: + - package_a + - output_types: [conda] + packages: + - package_b + - packages: + - package_c + output_types: pyproject + """ ), [(0, 1), (2, 0)], ), @@ -129,14 +129,14 @@ def test_check_common(content, indices): ( dedent( """\ - - matrix: - arch: x86_64 - packages: - - package_a - - packages: - - package_b - matrix: - """ + - matrix: + arch: x86_64 + packages: + - package_a + - packages: + - package_b + matrix: + """ ), [(0, 1), (1, 0)], ), @@ -161,25 +161,25 @@ def test_check_matrices(content, indices): ( dedent( """\ - - output_types: [pyproject, conda] - matrices: - - matrix: - arch: x86_64 - packages: - - package_a - - output_types: [conda] - matrices: - - matrix: - arch: x86_64 - packages: - - package_b - - matrices: - - matrix: - arch: x86_64 - packages: - - package_c - output_types: pyproject - """ + - output_types: [pyproject, conda] + matrices: + - matrix: + arch: x86_64 + packages: + - package_a + - output_types: [conda] + matrices: + - matrix: + arch: x86_64 + packages: + - package_b + - matrices: + - matrix: + arch: x86_64 + packages: + - package_c + output_types: pyproject + """ ), [(0, 1), (2, 0)], ), @@ -204,31 +204,31 @@ def test_check_specific(content, indices): ( dedent( """\ - set_a: - common: - - output_types: [pyproject] - packages: - - package_a - specific: - - output_types: [pyproject] - matrices: - - matrix: - arch: x86_64 + set_a: + common: + - output_types: [pyproject] packages: - - package_b - set_b: - specific: - - output_types: [pyproject] - matrices: - - matrix: - arch: x86_64 + - package_a + specific: + - output_types: [pyproject] + matrices: + - matrix: + arch: x86_64 + packages: + - package_b + set_b: + specific: + - output_types: [pyproject] + matrices: + - matrix: + arch: x86_64 + packages: + - package_c + common: + - output_types: [pyproject] packages: - - package_c - common: - - output_types: [pyproject] - packages: - - package_d - """ + - package_d + """ ), [(0, 0), (1, 1)], [(0, 1), (1, 0)], From b5ca3cc8437e190b714486652f614d9e6baa7535 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 17 May 2024 17:42:05 -0400 Subject: [PATCH 05/20] Add alpha spec regardless of output type --- src/rapids_pre_commit_hooks/alpha_spec.py | 34 ++----------------- .../test_alpha_spec.py | 4 +-- 2 files changed, 4 insertions(+), 34 deletions(-) diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py index 0a30533..58f1cf7 100644 --- a/src/rapids_pre_commit_hooks/alpha_spec.py +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -96,26 +96,11 @@ def check_common(linter, args, node): if dependency_set.tag == "tag:yaml.org,2002:map": for dependency_set_key, dependency_set_value in dependency_set.value: if ( - dependency_set_key.tag == "tag:yaml.org,2002:str" - and dependency_set_key.value == "output_types" - ): - output_types = dependency_set_value - elif ( dependency_set_key.tag == "tag:yaml.org,2002:str" and dependency_set_key.value == "packages" ): packages = dependency_set_value - if output_types.tag == "tag:yaml.org,2002:seq" and any( - t.tag == "tag:yaml.org,2002:str" - and t.value in ALPHA_SPEC_OUTPUT_TYPES - for t in output_types.value - ): - check_packages(linter, args, packages) - elif ( - output_types.tag == "tag:yaml.org,2002:str" - and output_types.value in ALPHA_SPEC_OUTPUT_TYPES - ): - check_packages(linter, args, packages) + check_packages(linter, args, packages) def check_matrices(linter, args, node): @@ -136,26 +121,11 @@ def check_specific(linter, args, node): if matrix_matcher.tag == "tag:yaml.org,2002:map": for matrix_matcher_key, matrix_matcher_value in matrix_matcher.value: if ( - matrix_matcher_key.tag == "tag:yaml.org,2002:str" - and matrix_matcher_key.value == "output_types" - ): - output_types = matrix_matcher_value - elif ( matrix_matcher_key.tag == "tag:yaml.org,2002:str" and matrix_matcher_key.value == "matrices" ): matrices = matrix_matcher_value - if output_types.tag == "tag:yaml.org,2002:seq" and any( - t.tag == "tag:yaml.org,2002:str" - and t.value in ALPHA_SPEC_OUTPUT_TYPES - for t in output_types.value - ): - check_matrices(linter, args, matrices) - elif ( - output_types.tag == "tag:yaml.org,2002:str" - and output_types.value in ALPHA_SPEC_OUTPUT_TYPES - ): - check_matrices(linter, args, matrices) + check_matrices(linter, args, matrices) def check_dependencies(linter, args, node): diff --git a/test/rapids_pre_commit_hooks/test_alpha_spec.py b/test/rapids_pre_commit_hooks/test_alpha_spec.py index 6dacb9e..322be56 100644 --- a/test/rapids_pre_commit_hooks/test_alpha_spec.py +++ b/test/rapids_pre_commit_hooks/test_alpha_spec.py @@ -106,7 +106,7 @@ def test_check_packages(content, indices): output_types: pyproject """ ), - [(0, 1), (2, 0)], + [(0, 1), (1, 1), (2, 0)], ), ], ) @@ -181,7 +181,7 @@ def test_check_matrices(content, indices): output_types: pyproject """ ), - [(0, 1), (2, 0)], + [(0, 1), (1, 1), (2, 0)], ), ], ) From adc54a7fe8c3b1142531f89834334efb8e0d6d2d Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 17 May 2024 18:48:30 -0400 Subject: [PATCH 06/20] Check -cu* suffixed packages --- src/rapids_pre_commit_hooks/alpha_spec.py | 33 ++++++++++++++++++- .../test_alpha_spec.py | 21 ++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py index 58f1cf7..d7f5b90 100644 --- a/src/rapids_pre_commit_hooks/alpha_spec.py +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -47,6 +47,29 @@ "distributed-ucxx", } +RAPIDS_CUDA_VERSIONED_PACKAGES = { + "rmm", + "pylibcugraphops", + "pylibcugraph", + "nx-cugraph", + "dask-cudf", + "cuspatial", + "cuproj", + "cuml", + "cugraph", + "cudf", + "ptxcompiler", + "cubinlinker", + "cugraph-dgl", + "cugraph-pyg", + "cugraph-equivariant", + "raft-dask", + "pylibwholegraph", + "pylibraft", + "cuxfilter", + "cucim", +} + ALPHA_SPECIFIER = ">=0.0.0a0" ALPHA_SPEC_OUTPUT_TYPES = { @@ -55,10 +78,18 @@ } +def is_rapids_cuda_versioned_package(name): + return any( + name.startswith(f"{package}-cu") for package in RAPIDS_CUDA_VERSIONED_PACKAGES + ) + + def check_package_spec(linter, args, node): if node.tag == "tag:yaml.org,2002:str": req = Requirement(node.value) - if req.name in RAPIDS_VERSIONED_PACKAGES: + if req.name in RAPIDS_VERSIONED_PACKAGES or is_rapids_cuda_versioned_package( + req.name + ): has_alpha_spec = any( filter(lambda s: str(s) == ALPHA_SPECIFIER, req.specifier) ) diff --git a/test/rapids_pre_commit_hooks/test_alpha_spec.py b/test/rapids_pre_commit_hooks/test_alpha_spec.py index 322be56..687bee5 100644 --- a/test/rapids_pre_commit_hooks/test_alpha_spec.py +++ b/test/rapids_pre_commit_hooks/test_alpha_spec.py @@ -36,6 +36,27 @@ for p in alpha_spec.RAPIDS_VERSIONED_PACKAGES ) ), + *chain( + *( + [ + (f"{p}-cu12", f"{p}-cu12", "development", f"{p}-cu12>=0.0.0a0"), + (f"{p}-cu12", f"{p}-cu12", "release", None), + (f"{p}-cu12", f"{p}-cu12>=0.0.0a0", "development", None), + (f"{p}-cu12", f"{p}-cu12>=0.0.0a0", "release", f"{p}-cu12"), + ] + for p in alpha_spec.RAPIDS_CUDA_VERSIONED_PACKAGES + ) + ), + *chain( + *( + [ + (f"{p}-cu12", f"{p}-cu12", "development", None), + (f"{p}-cu12", f"{p}-cu12>=0.0.0a0", "release", None), + ] + for p in alpha_spec.RAPIDS_VERSIONED_PACKAGES + - alpha_spec.RAPIDS_CUDA_VERSIONED_PACKAGES + ) + ), ("cuml", "cuml>=24.04,<=24.06", "development", "cuml<=24.06,>=0.0.0a0,>=24.04"), ("cuml", "cuml>=24.04,<=24.06,>=0.0.0a0", "release", "cuml<=24.06,>=24.04"), ("packaging", "packaging", "development", None), From 20add368e69627a23ea904cec6b97ab5de507b1b Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 17 May 2024 19:20:57 -0400 Subject: [PATCH 07/20] Add test for reference --- test/rapids_pre_commit_hooks/test_alpha_spec.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/rapids_pre_commit_hooks/test_alpha_spec.py b/test/rapids_pre_commit_hooks/test_alpha_spec.py index 687bee5..db707da 100644 --- a/test/rapids_pre_commit_hooks/test_alpha_spec.py +++ b/test/rapids_pre_commit_hooks/test_alpha_spec.py @@ -59,6 +59,12 @@ ), ("cuml", "cuml>=24.04,<=24.06", "development", "cuml<=24.06,>=0.0.0a0,>=24.04"), ("cuml", "cuml>=24.04,<=24.06,>=0.0.0a0", "release", "cuml<=24.06,>=24.04"), + ( + "cuml", + "&cuml cuml>=24.04,<=24.06,>=0.0.0a0", + "release", + "cuml<=24.06,>=24.04", + ), ("packaging", "packaging", "development", None), ], ) @@ -86,7 +92,7 @@ def test_check_package_spec(package, content, mode, replacement): dedent( """\ - package_a - - package_b + - &package_b package_b """ ), [0, 1], From a69e3a48b35b17b746850c835b07f90591f2397c Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 21 May 2024 11:41:41 -0400 Subject: [PATCH 08/20] Refactoring --- src/rapids_pre_commit_hooks/alpha_spec.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py index d7f5b90..d5dafa2 100644 --- a/src/rapids_pre_commit_hooks/alpha_spec.py +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -130,8 +130,7 @@ def check_common(linter, args, node): dependency_set_key.tag == "tag:yaml.org,2002:str" and dependency_set_key.value == "packages" ): - packages = dependency_set_value - check_packages(linter, args, packages) + check_packages(linter, args, dependency_set_value) def check_matrices(linter, args, node): @@ -155,8 +154,7 @@ def check_specific(linter, args, node): matrix_matcher_key.tag == "tag:yaml.org,2002:str" and matrix_matcher_key.value == "matrices" ): - matrices = matrix_matcher_value - check_matrices(linter, args, matrices) + check_matrices(linter, args, matrix_matcher_value) def check_dependencies(linter, args, node): From cf1b6edd80ce407840ac05b4ffc2963e04c6c023 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 21 May 2024 12:41:17 -0400 Subject: [PATCH 09/20] Simplify tag checking --- src/rapids_pre_commit_hooks/alpha_spec.py | 39 ++++++++++++----------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py index d5dafa2..de126c1 100644 --- a/src/rapids_pre_commit_hooks/alpha_spec.py +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -78,6 +78,10 @@ } +def node_has_type(node, tag_type): + return node.tag == f"tag:yaml.org,2002:{tag_type}" + + def is_rapids_cuda_versioned_package(name): return any( name.startswith(f"{package}-cu") for package in RAPIDS_CUDA_VERSIONED_PACKAGES @@ -85,7 +89,7 @@ def is_rapids_cuda_versioned_package(name): def check_package_spec(linter, args, node): - if node.tag == "tag:yaml.org,2002:str": + if node_has_type(node, "str"): req = Requirement(node.value) if req.name in RAPIDS_VERSIONED_PACKAGES or is_rapids_cuda_versioned_package( req.name @@ -116,53 +120,53 @@ def check_package_spec(linter, args, node): def check_packages(linter, args, node): - if node.tag == "tag:yaml.org,2002:seq": + if node_has_type(node, "seq"): for package_spec in node.value: check_package_spec(linter, args, package_spec) def check_common(linter, args, node): - if node.tag == "tag:yaml.org,2002:seq": + if node_has_type(node, "seq"): for dependency_set in node.value: - if dependency_set.tag == "tag:yaml.org,2002:map": + if node_has_type(dependency_set, "map"): for dependency_set_key, dependency_set_value in dependency_set.value: if ( - dependency_set_key.tag == "tag:yaml.org,2002:str" + node_has_type(dependency_set_key, "str") and dependency_set_key.value == "packages" ): check_packages(linter, args, dependency_set_value) def check_matrices(linter, args, node): - if node.tag == "tag:yaml.org,2002:seq": + if node_has_type(node, "seq"): for item in node.value: - if item.tag == "tag:yaml.org,2002:map": + if node_has_type(item, "map"): for matrix_key, matrix_value in item.value: if ( - matrix_key.tag == "tag:yaml.org,2002:str" + node_has_type(matrix_key, "str") and matrix_key.value == "packages" ): check_packages(linter, args, matrix_value) def check_specific(linter, args, node): - if node.tag == "tag:yaml.org,2002:seq": + if node_has_type(node, "seq"): for matrix_matcher in node.value: - if matrix_matcher.tag == "tag:yaml.org,2002:map": + if node_has_type(matrix_matcher, "map"): for matrix_matcher_key, matrix_matcher_value in matrix_matcher.value: if ( - matrix_matcher_key.tag == "tag:yaml.org,2002:str" + node_has_type(matrix_matcher_key, "str") and matrix_matcher_key.value == "matrices" ): check_matrices(linter, args, matrix_matcher_value) def check_dependencies(linter, args, node): - if node.tag == "tag:yaml.org,2002:map": + if node_has_type(node, "map"): for _, dependencies_value in node.value: - if dependencies_value.tag == "tag:yaml.org,2002:map": + if node_has_type(dependencies_value, "map"): for dependency_key, dependency_value in dependencies_value.value: - if dependency_key.tag == "tag:yaml.org,2002:str": + if node_has_type(dependency_key, "str"): if dependency_key.value == "common": check_common(linter, args, dependency_value) elif dependency_key.value == "specific": @@ -170,12 +174,9 @@ def check_dependencies(linter, args, node): def check_root(linter, args, node): - if node.tag == "tag:yaml.org,2002:map": + if node_has_type(node, "map"): for root_key, root_value in node.value: - if ( - root_key.tag == "tag:yaml.org,2002:str" - and root_key.value == "dependencies" - ): + if node_has_type(root_key, "str") and root_key.value == "dependencies": check_dependencies(linter, args, root_value) From 5c0c9cc38c6c1e7bd87c10f11b0a94c3a57ff534 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 21 May 2024 12:48:51 -0400 Subject: [PATCH 10/20] Review feedback --- .pre-commit-hooks.yaml | 2 +- src/rapids_pre_commit_hooks/alpha_spec.py | 14 ++++++-------- test/rapids_pre_commit_hooks/test_alpha_spec.py | 8 ++++---- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 33ab4e9..7b33a35 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -43,7 +43,7 @@ args: [--fix] - id: verify-alpha-spec name: verify-alpha-spec - description: make sure RAPIDS package have correct alpha spec + description: make sure RAPIDS packages have correct alpha spec entry: verify-alpha-spec language: python files: dependencies[.]yaml$ diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py index de126c1..68ec043 100644 --- a/src/rapids_pre_commit_hooks/alpha_spec.py +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -20,7 +20,7 @@ from .lint import LintMain -RAPIDS_VERSIONED_PACKAGES = { +RAPIDS_ALPHA_SPEC_PACKAGES = { "rmm", "pylibcugraphops", "pylibcugraph", @@ -47,7 +47,7 @@ "distributed-ucxx", } -RAPIDS_CUDA_VERSIONED_PACKAGES = { +RAPIDS_CUDA_SUFFIXED_PACKAGES = { "rmm", "pylibcugraphops", "pylibcugraph", @@ -82,21 +82,19 @@ def node_has_type(node, tag_type): return node.tag == f"tag:yaml.org,2002:{tag_type}" -def is_rapids_cuda_versioned_package(name): +def is_rapids_cuda_suffixed_package(name): return any( - name.startswith(f"{package}-cu") for package in RAPIDS_CUDA_VERSIONED_PACKAGES + name.startswith(f"{package}-cu") for package in RAPIDS_CUDA_SUFFIXED_PACKAGES ) def check_package_spec(linter, args, node): if node_has_type(node, "str"): req = Requirement(node.value) - if req.name in RAPIDS_VERSIONED_PACKAGES or is_rapids_cuda_versioned_package( + if req.name in RAPIDS_ALPHA_SPEC_PACKAGES or is_rapids_cuda_suffixed_package( req.name ): - has_alpha_spec = any( - filter(lambda s: str(s) == ALPHA_SPECIFIER, req.specifier) - ) + has_alpha_spec = any(str(s) == ALPHA_SPECIFIER for s in req.specifier) if args.mode == "development" and not has_alpha_spec: req.specifier &= ALPHA_SPECIFIER linter.add_warning( diff --git a/test/rapids_pre_commit_hooks/test_alpha_spec.py b/test/rapids_pre_commit_hooks/test_alpha_spec.py index db707da..b39434d 100644 --- a/test/rapids_pre_commit_hooks/test_alpha_spec.py +++ b/test/rapids_pre_commit_hooks/test_alpha_spec.py @@ -33,7 +33,7 @@ (p, f"{p}>=0.0.0a0", "development", None), (p, f"{p}>=0.0.0a0", "release", p), ] - for p in alpha_spec.RAPIDS_VERSIONED_PACKAGES + for p in alpha_spec.RAPIDS_ALPHA_SPEC_PACKAGES ) ), *chain( @@ -44,7 +44,7 @@ (f"{p}-cu12", f"{p}-cu12>=0.0.0a0", "development", None), (f"{p}-cu12", f"{p}-cu12>=0.0.0a0", "release", f"{p}-cu12"), ] - for p in alpha_spec.RAPIDS_CUDA_VERSIONED_PACKAGES + for p in alpha_spec.RAPIDS_CUDA_SUFFIXED_PACKAGES ) ), *chain( @@ -53,8 +53,8 @@ (f"{p}-cu12", f"{p}-cu12", "development", None), (f"{p}-cu12", f"{p}-cu12>=0.0.0a0", "release", None), ] - for p in alpha_spec.RAPIDS_VERSIONED_PACKAGES - - alpha_spec.RAPIDS_CUDA_VERSIONED_PACKAGES + for p in alpha_spec.RAPIDS_ALPHA_SPEC_PACKAGES + - alpha_spec.RAPIDS_CUDA_SUFFIXED_PACKAGES ) ), ("cuml", "cuml>=24.04,<=24.06", "development", "cuml<=24.06,>=0.0.0a0,>=24.04"), From ae9b62a0f3a9e7a4a56375db9aa51fee1d1ed1d8 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 21 May 2024 13:01:00 -0400 Subject: [PATCH 11/20] All packages have CUDA suffix --- src/rapids_pre_commit_hooks/alpha_spec.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py index 68ec043..70a97f0 100644 --- a/src/rapids_pre_commit_hooks/alpha_spec.py +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -68,6 +68,10 @@ "pylibraft", "cuxfilter", "cucim", + "ucx-py", + "ucxx", + "pynvjitlink", + "distributed-ucxx", } ALPHA_SPECIFIER = ">=0.0.0a0" From 1e1e2df189ba057726304048586fa5afd5d95dc9 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 21 May 2024 14:19:01 -0400 Subject: [PATCH 12/20] Deduplicate list of packages --- src/rapids_pre_commit_hooks/alpha_spec.py | 27 +---------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py index 70a97f0..07659c8 100644 --- a/src/rapids_pre_commit_hooks/alpha_spec.py +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -47,32 +47,7 @@ "distributed-ucxx", } -RAPIDS_CUDA_SUFFIXED_PACKAGES = { - "rmm", - "pylibcugraphops", - "pylibcugraph", - "nx-cugraph", - "dask-cudf", - "cuspatial", - "cuproj", - "cuml", - "cugraph", - "cudf", - "ptxcompiler", - "cubinlinker", - "cugraph-dgl", - "cugraph-pyg", - "cugraph-equivariant", - "raft-dask", - "pylibwholegraph", - "pylibraft", - "cuxfilter", - "cucim", - "ucx-py", - "ucxx", - "pynvjitlink", - "distributed-ucxx", -} +RAPIDS_CUDA_SUFFIXED_PACKAGES = set(RAPIDS_ALPHA_SPEC_PACKAGES) ALPHA_SPECIFIER = ">=0.0.0a0" From f7e047e14ce1c90329b3535818a9c38ab34f3ef5 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 24 May 2024 11:41:05 -0400 Subject: [PATCH 13/20] Add more alpha spec packages --- src/rapids_pre_commit_hooks/alpha_spec.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py index 07659c8..298f076 100644 --- a/src/rapids_pre_commit_hooks/alpha_spec.py +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -45,6 +45,8 @@ "ucxx", "pynvjitlink", "distributed-ucxx", + "librmm", + "libucx", } RAPIDS_CUDA_SUFFIXED_PACKAGES = set(RAPIDS_ALPHA_SPEC_PACKAGES) From 8b7e9e1ed0b5170f809ebf1de5fc8d5aab7fac3c Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 28 May 2024 13:53:05 -0400 Subject: [PATCH 14/20] Alphabetize package list --- src/rapids_pre_commit_hooks/alpha_spec.py | 38 +++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py index 298f076..5b8ca84 100644 --- a/src/rapids_pre_commit_hooks/alpha_spec.py +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -21,32 +21,32 @@ from .lint import LintMain RAPIDS_ALPHA_SPEC_PACKAGES = { - "rmm", - "pylibcugraphops", - "pylibcugraph", - "nx-cugraph", - "dask-cudf", - "cuspatial", - "cuproj", - "cuml", - "cugraph", - "cudf", - "ptxcompiler", "cubinlinker", + "cucim", + "cudf", + "cugraph", "cugraph-dgl", - "cugraph-pyg", "cugraph-equivariant", - "raft-dask", - "pylibwholegraph", - "pylibraft", + "cugraph-pyg", + "cuml", + "cuproj", + "cuspatial", "cuxfilter", - "cucim", - "ucx-py", - "ucxx", - "pynvjitlink", + "dask-cudf", "distributed-ucxx", "librmm", "libucx", + "nx-cugraph", + "ptxcompiler", + "pylibcugraph", + "pylibcugraphops", + "pylibraft", + "pylibwholegraph", + "pynvjitlink", + "raft-dask", + "rmm", + "ucx-py", + "ucxx", } RAPIDS_CUDA_SUFFIXED_PACKAGES = set(RAPIDS_ALPHA_SPEC_PACKAGES) From 53f6eeca925472f08c7a59265d5c6bdae79ebc93 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 28 May 2024 13:54:39 -0400 Subject: [PATCH 15/20] Change a few tests to CUDA 11 --- test/rapids_pre_commit_hooks/test_alpha_spec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/rapids_pre_commit_hooks/test_alpha_spec.py b/test/rapids_pre_commit_hooks/test_alpha_spec.py index b39434d..63da475 100644 --- a/test/rapids_pre_commit_hooks/test_alpha_spec.py +++ b/test/rapids_pre_commit_hooks/test_alpha_spec.py @@ -40,9 +40,9 @@ *( [ (f"{p}-cu12", f"{p}-cu12", "development", f"{p}-cu12>=0.0.0a0"), - (f"{p}-cu12", f"{p}-cu12", "release", None), + (f"{p}-cu11", f"{p}-cu11", "release", None), (f"{p}-cu12", f"{p}-cu12>=0.0.0a0", "development", None), - (f"{p}-cu12", f"{p}-cu12>=0.0.0a0", "release", f"{p}-cu12"), + (f"{p}-cu11", f"{p}-cu11>=0.0.0a0", "release", f"{p}-cu11"), ] for p in alpha_spec.RAPIDS_CUDA_SUFFIXED_PACKAGES ) From dab4d781941e1e4275ac431850a529aa1c53fa0c Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 28 May 2024 14:04:02 -0400 Subject: [PATCH 16/20] Alphabetize entry points --- .pre-commit-hooks.yaml | 14 +++++++------- pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 7b33a35..daa5d93 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -11,6 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +- id: verify-alpha-spec + name: verify-alpha-spec + description: make sure RAPIDS packages have correct alpha spec + entry: verify-alpha-spec + language: python + files: dependencies[.]yaml$ + args: [--fix] - id: verify-conda-yes name: pass -y/--yes to conda description: make sure that all calls to conda pass -y/--yes @@ -41,10 +48,3 @@ (?x) pyproject[.]toml$ args: [--fix] -- id: verify-alpha-spec - name: verify-alpha-spec - description: make sure RAPIDS packages have correct alpha spec - entry: verify-alpha-spec - language: python - files: dependencies[.]yaml$ - args: [--fix] diff --git a/pyproject.toml b/pyproject.toml index 7f6fc91..8ee457e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,10 +47,10 @@ test = [ ] [project.scripts] +verify-alpha-spec = "rapids_pre_commit_hooks.alpha_spec:main" verify-conda-yes = "rapids_pre_commit_hooks.shell.verify_conda_yes:main" verify-copyright = "rapids_pre_commit_hooks.copyright:main" verify-pyproject-license = "rapids_pre_commit_hooks.pyproject_license:main" -verify-alpha-spec = "rapids_pre_commit_hooks.alpha_spec:main" [tool.setuptools] packages = { "find" = { where = ["src"] } } From 36f2881a4a7b343883aa267fb0cfbdba5e7f3197 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 28 May 2024 14:16:45 -0400 Subject: [PATCH 17/20] Use regex to search for CUDA suffix --- src/rapids_pre_commit_hooks/alpha_spec.py | 6 +++- .../test_alpha_spec.py | 30 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py index 5b8ca84..aed845b 100644 --- a/src/rapids_pre_commit_hooks/alpha_spec.py +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re from functools import reduce import yaml @@ -58,6 +59,8 @@ "requirements", } +CUDA_SUFFIX_REGEX = re.compile(r"^(?P.*)-cu[0-9]{2}$") + def node_has_type(node, tag_type): return node.tag == f"tag:yaml.org,2002:{tag_type}" @@ -65,7 +68,8 @@ def node_has_type(node, tag_type): def is_rapids_cuda_suffixed_package(name): return any( - name.startswith(f"{package}-cu") for package in RAPIDS_CUDA_SUFFIXED_PACKAGES + (match := CUDA_SUFFIX_REGEX.search(name)) and match.group("package") == package + for package in RAPIDS_CUDA_SUFFIXED_PACKAGES ) diff --git a/test/rapids_pre_commit_hooks/test_alpha_spec.py b/test/rapids_pre_commit_hooks/test_alpha_spec.py index 63da475..d84b678 100644 --- a/test/rapids_pre_commit_hooks/test_alpha_spec.py +++ b/test/rapids_pre_commit_hooks/test_alpha_spec.py @@ -22,6 +22,36 @@ from rapids_pre_commit_hooks import alpha_spec, lint +@pytest.mark.parametrize( + ["name", "is_suffixed"], + [ + *chain( + *( + [ + (f"{p}-cu11", True), + (f"{p}-cu12", True), + (f"{p}-cuda", False), + ] + for p in alpha_spec.RAPIDS_CUDA_SUFFIXED_PACKAGES + ) + ), + *chain( + *( + [ + (f"{p}-cu11", False), + (f"{p}-cu12", False), + (f"{p}-cuda", False), + ] + for p in alpha_spec.RAPIDS_ALPHA_SPEC_PACKAGES + - alpha_spec.RAPIDS_CUDA_SUFFIXED_PACKAGES + ) + ), + ], +) +def test_is_rapids_cuda_suffixed_package(name, is_suffixed): + assert alpha_spec.is_rapids_cuda_suffixed_package(name) == is_suffixed + + @pytest.mark.parametrize( ["package", "content", "mode", "replacement"], [ From bf5f737c0cf916373f03eba5d7044ba46be44a2e Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 28 May 2024 14:18:18 -0400 Subject: [PATCH 18/20] s/<=/=24.04,<=24.06", "development", "cuml<=24.06,>=0.0.0a0,>=24.04"), - ("cuml", "cuml>=24.04,<=24.06,>=0.0.0a0", "release", "cuml<=24.06,>=24.04"), + ("cuml", "cuml>=24.04,<24.06", "development", "cuml<24.06,>=0.0.0a0,>=24.04"), + ("cuml", "cuml>=24.04,<24.06,>=0.0.0a0", "release", "cuml<24.06,>=24.04"), ( "cuml", - "&cuml cuml>=24.04,<=24.06,>=0.0.0a0", + "&cuml cuml>=24.04,<24.06,>=0.0.0a0", "release", - "cuml<=24.06,>=24.04", + "cuml<24.06,>=24.04", ), ("packaging", "packaging", "development", None), ], From 6841fcfbbb409a1ac3a7e3b8cf0de594dfdaf720 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 28 May 2024 15:15:17 -0400 Subject: [PATCH 19/20] Pretty-sort version specifier --- src/rapids_pre_commit_hooks/alpha_spec.py | 48 +++++++++++++++---- .../test_alpha_spec.py | 6 +-- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py index aed845b..95515d2 100644 --- a/src/rapids_pre_commit_hooks/alpha_spec.py +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -13,11 +13,10 @@ # limitations under the License. import re -from functools import reduce +from functools import total_ordering import yaml from packaging.requirements import Requirement -from packaging.specifiers import SpecifierSet from .lint import LintMain @@ -74,6 +73,29 @@ def is_rapids_cuda_suffixed_package(name): def check_package_spec(linter, args, node): + @total_ordering + class SpecPriority: + def __init__(self, spec): + self.spec = spec + + def __eq__(self, other): + return self.spec == other.spec + + def __lt__(self, other): + if self.spec == other.spec: + return False + if self.spec == ALPHA_SPECIFIER: + return False + if other.spec == ALPHA_SPECIFIER: + return True + return self.sort_str() < other.sort_str() + + def sort_str(self): + return "".join(c for c in self.spec if c not in "<>=") + + def create_specifier_string(specifiers): + return ",".join(sorted(specifiers, key=SpecPriority)) + if node_has_type(node, "str"): req = Requirement(node.value) if req.name in RAPIDS_ALPHA_SPEC_PACKAGES or is_rapids_cuda_suffixed_package( @@ -81,24 +103,30 @@ def check_package_spec(linter, args, node): ): has_alpha_spec = any(str(s) == ALPHA_SPECIFIER for s in req.specifier) if args.mode == "development" and not has_alpha_spec: - req.specifier &= ALPHA_SPECIFIER linter.add_warning( (node.start_mark.index, node.end_mark.index), f"add alpha spec for RAPIDS package {req.name}", ).add_replacement( - (node.start_mark.index, node.end_mark.index), str(req) + (node.start_mark.index, node.end_mark.index), + str( + req.name + + create_specifier_string( + {str(s) for s in req.specifier} | {ALPHA_SPECIFIER} + ) + ), ) elif args.mode == "release" and has_alpha_spec: - req.specifier = reduce( - lambda ss, s: ss & str(s), - filter(lambda s: str(s) != ALPHA_SPECIFIER, req.specifier), - SpecifierSet(), - ) linter.add_warning( (node.start_mark.index, node.end_mark.index), f"remove alpha spec for RAPIDS package {req.name}", ).add_replacement( - (node.start_mark.index, node.end_mark.index), str(req) + (node.start_mark.index, node.end_mark.index), + str( + req.name + + create_specifier_string( + {str(s) for s in req.specifier} - {ALPHA_SPECIFIER} + ) + ), ) diff --git a/test/rapids_pre_commit_hooks/test_alpha_spec.py b/test/rapids_pre_commit_hooks/test_alpha_spec.py index f75a547..9661cf5 100644 --- a/test/rapids_pre_commit_hooks/test_alpha_spec.py +++ b/test/rapids_pre_commit_hooks/test_alpha_spec.py @@ -87,13 +87,13 @@ def test_is_rapids_cuda_suffixed_package(name, is_suffixed): - alpha_spec.RAPIDS_CUDA_SUFFIXED_PACKAGES ) ), - ("cuml", "cuml>=24.04,<24.06", "development", "cuml<24.06,>=0.0.0a0,>=24.04"), - ("cuml", "cuml>=24.04,<24.06,>=0.0.0a0", "release", "cuml<24.06,>=24.04"), + ("cuml", "cuml>=24.04,<24.06", "development", "cuml>=24.04,<24.06,>=0.0.0a0"), + ("cuml", "cuml>=24.04,<24.06,>=0.0.0a0", "release", "cuml>=24.04,<24.06"), ( "cuml", "&cuml cuml>=24.04,<24.06,>=0.0.0a0", "release", - "cuml<24.06,>=24.04", + "cuml>=24.04,<24.06", ), ("packaging", "packaging", "development", None), ], From f118ae8dbc6de8c2e7707a0a48d1e97fcea62c11 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 28 May 2024 16:19:44 -0400 Subject: [PATCH 20/20] dask-cuda has alpha spec but no CUDA suffix --- src/rapids_pre_commit_hooks/alpha_spec.py | 9 ++++++++- test/rapids_pre_commit_hooks/test_alpha_spec.py | 6 ++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py index 95515d2..aa74591 100644 --- a/src/rapids_pre_commit_hooks/alpha_spec.py +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -32,6 +32,7 @@ "cuproj", "cuspatial", "cuxfilter", + "dask-cuda", "dask-cudf", "distributed-ucxx", "librmm", @@ -49,7 +50,13 @@ "ucxx", } -RAPIDS_CUDA_SUFFIXED_PACKAGES = set(RAPIDS_ALPHA_SPEC_PACKAGES) +RAPIDS_NON_CUDA_SUFFIXED_PACKAGES = { + "dask-cuda", +} + +RAPIDS_CUDA_SUFFIXED_PACKAGES = ( + RAPIDS_ALPHA_SPEC_PACKAGES - RAPIDS_NON_CUDA_SUFFIXED_PACKAGES +) ALPHA_SPECIFIER = ">=0.0.0a0" diff --git a/test/rapids_pre_commit_hooks/test_alpha_spec.py b/test/rapids_pre_commit_hooks/test_alpha_spec.py index 9661cf5..425c2b4 100644 --- a/test/rapids_pre_commit_hooks/test_alpha_spec.py +++ b/test/rapids_pre_commit_hooks/test_alpha_spec.py @@ -42,8 +42,7 @@ (f"{p}-cu12", False), (f"{p}-cuda", False), ] - for p in alpha_spec.RAPIDS_ALPHA_SPEC_PACKAGES - - alpha_spec.RAPIDS_CUDA_SUFFIXED_PACKAGES + for p in alpha_spec.RAPIDS_NON_CUDA_SUFFIXED_PACKAGES ) ), ], @@ -83,8 +82,7 @@ def test_is_rapids_cuda_suffixed_package(name, is_suffixed): (f"{p}-cu12", f"{p}-cu12", "development", None), (f"{p}-cu12", f"{p}-cu12>=0.0.0a0", "release", None), ] - for p in alpha_spec.RAPIDS_ALPHA_SPEC_PACKAGES - - alpha_spec.RAPIDS_CUDA_SUFFIXED_PACKAGES + for p in alpha_spec.RAPIDS_NON_CUDA_SUFFIXED_PACKAGES ) ), ("cuml", "cuml>=24.04,<24.06", "development", "cuml>=24.04,<24.06,>=0.0.0a0"),