From e2b4764606da9156dedc13e4110de76acafee953 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Wed, 1 Nov 2023 12:26:34 -0700 Subject: [PATCH 01/24] fix large pandas dataframe --- .../cugraph_pyg/data/cugraph_store.py | 87 +++++++++++++++---- 1 file changed, 71 insertions(+), 16 deletions(-) diff --git a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py index fd2172e6ade..b71ea2f9b41 100644 --- a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py @@ -455,22 +455,64 @@ def __construct_graph( ] ) - df = pandas.DataFrame( - { - "src": pandas.Series(na_dst) - if order == "CSC" - else pandas.Series(na_src), - "dst": pandas.Series(na_src) - if order == "CSC" - else pandas.Series(na_dst), - "etp": pandas.Series(na_etp), - } - ) - vertex_dtype = df.src.dtype + vertex_dtype = na_src.dtype if multi_gpu: - nworkers = len(distributed.get_client().scheduler_info()["workers"]) - df = dd.from_pandas(df, npartitions=nworkers if len(df) > 32 else 1) + client = distributed.get_client() + nworkers = len(client.scheduler_info()["workers"]) + npartitions = nworkers * 4 + + na_src = np.array_split(na_src, npartitions) + na_dst = np.array_split(na_dst, npartitions) + na_etp = np.array_split(na_etp.astype('int64'), npartitions) + + scna_src = client.scatter(na_src) + shapes = [n.shape for n in na_src] + del na_src + + scna_dst = client.scatter(na_dst) + del na_dst + + scna_etp = client.scatter(na_etp) + del na_etp + + import dask.array as dar + src_dar = dar.concatenate( + [ + dar.from_delayed(s, shape=shapes[i], dtype=vertex_dtype) + for i, s in enumerate(scna_src) + ] + ) + del scna_src + + dst_dar = dar.concatenate( + [ + dar.from_delayed(s, shape=shapes[i], dtype=vertex_dtype) + for i, s in enumerate(scna_dst) + ] + ) + del scna_dst + + etp_dar = dar.concatenate( + [ + dar.from_delayed(s, shape=shapes[i], dtype='int32') + for i, s in enumerate(scna_etp) + ] + ) + del scna_etp + + df = dd.from_dask_array( + dar.stack( + [dst_dar, src_dar, etp_dar] if order == 'CSC' else [src_dar, dst_dar, etp_dar], + axis=1 + ), + columns=['src','dst','etp'] + ) + del src_dar + del dst_dar + del etp_dar + + df.etp=df.etp.astype('int32') # Ensure the dataframe is constructed on each partition # instead of adding additional synchronization head from potential @@ -491,9 +533,21 @@ def get_empty_df(): if len(f) > 0 else get_empty_df(), meta=get_empty_df(), - ).reset_index(drop=True) + ).reset_index(drop=True) # should be ok for dask else: - df = cudf.from_pandas(df).reset_index(drop=True) + df = pandas.DataFrame( + { + "src": pandas.Series(na_dst) + if order == "CSC" + else pandas.Series(na_src), + "dst": pandas.Series(na_src) + if order == "CSC" + else pandas.Series(na_dst), + "etp": pandas.Series(na_etp), + } + ) + df = cudf.from_pandas(df) + df.reset_index(drop=True, inplace=True) graph = cugraph.MultiGraph(directed=True) if multi_gpu: @@ -512,6 +566,7 @@ def get_empty_df(): edge_type="etp", ) + del df return graph @property From f9305e5a4e94e99494fbb11de08501d6fd49fdd6 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 6 Nov 2023 09:22:34 -0800 Subject: [PATCH 02/24] c --- .../cugraph_pyg/data/cugraph_store.py | 79 ++++++++----------- 1 file changed, 33 insertions(+), 46 deletions(-) diff --git a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py index b71ea2f9b41..616bf7755df 100644 --- a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py @@ -27,11 +27,12 @@ import cugraph import warnings -from cugraph.utilities.utils import import_optional, MissingModule +import dask.array as dar +import dask.dataframe as dd +import dask.distributed as distributed +import dask_cudf -dd = import_optional("dask.dataframe") -distributed = import_optional("dask.distributed") -dask_cudf = import_optional("dask_cudf") +from cugraph.utilities.utils import import_optional, MissingModule torch = import_optional("torch") torch_geometric = import_optional("torch_geometric") @@ -358,6 +359,22 @@ def __infer_offsets( } ) + def __dask_array_from_numpy(self, array: np.ndarray, client: distributed.client.Client, npartitions: int): + split_array = np.array_split(array, npartitions) + shapes = [n.shape for n in split_array] + scattered_array = client.scatter(split_array) + del split_array + + dask_array = dar.concatenate( + [ + dar.from_delayed(s, shape=shapes[i], dtype=array.dtype) + for i, s in enumerate(scattered_array) + ] + ) + del scattered_array + + return dask_array + def __construct_graph( self, edge_info: Dict[Tuple[str, str, str], List[TensorType]], @@ -462,57 +479,27 @@ def __construct_graph( nworkers = len(client.scheduler_info()["workers"]) npartitions = nworkers * 4 - na_src = np.array_split(na_src, npartitions) - na_dst = np.array_split(na_dst, npartitions) - na_etp = np.array_split(na_etp.astype('int64'), npartitions) - - scna_src = client.scatter(na_src) - shapes = [n.shape for n in na_src] + src_dar = self.__dask_array_from_numpy(na_src, client, npartitions) del na_src - scna_dst = client.scatter(na_dst) + dst_dar = self.__dask_array_from_numpy(na_dst, client, npartitions) del na_dst - - scna_etp = client.scatter(na_etp) - del na_etp - import dask.array as dar - src_dar = dar.concatenate( - [ - dar.from_delayed(s, shape=shapes[i], dtype=vertex_dtype) - for i, s in enumerate(scna_src) - ] - ) - del scna_src + etp_dar = self.__dask_array_from_numpy(na_etp, client, npartitions) + del na_etp - dst_dar = dar.concatenate( - [ - dar.from_delayed(s, shape=shapes[i], dtype=vertex_dtype) - for i, s in enumerate(scna_dst) - ] - ) - del scna_dst + df = dd.from_dask_array(etp_dar, columns=['etp']) + df['src'] = dst_dar if order == "CSC" else src_dar + df['dst'] = src_dar if order == "CSC" else dst_dar - etp_dar = dar.concatenate( - [ - dar.from_delayed(s, shape=shapes[i], dtype='int32') - for i, s in enumerate(scna_etp) - ] - ) - del scna_etp - - df = dd.from_dask_array( - dar.stack( - [dst_dar, src_dar, etp_dar] if order == 'CSC' else [src_dar, dst_dar, etp_dar], - axis=1 - ), - columns=['src','dst','etp'] - ) del src_dar del dst_dar del etp_dar - df.etp=df.etp.astype('int32') + if df.etp.dtype != 'int32': + raise ValueError("Edge type must be int32!") + + print(df) # Ensure the dataframe is constructed on each partition # instead of adding additional synchronization head from potential @@ -520,9 +507,9 @@ def __construct_graph( def get_empty_df(): return cudf.DataFrame( { + "etp": cudf.Series([], dtype="int32"), "src": cudf.Series([], dtype=vertex_dtype), "dst": cudf.Series([], dtype=vertex_dtype), - "etp": cudf.Series([], dtype="int32"), } ) From f1ef95f520c5afe3c04c52d5d50c5e5d34621607 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 6 Nov 2023 09:25:46 -0800 Subject: [PATCH 03/24] fix loader bugs --- .../cugraph_pyg/loader/cugraph_node_loader.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py index 8552e7412e0..fb9f3f9c70a 100644 --- a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py +++ b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py @@ -159,7 +159,10 @@ def __init__( if batch_size is None or batch_size < 1: raise ValueError("Batch size must be >= 1") - self.__directory = tempfile.TemporaryDirectory(dir=directory) + self.__directory = ( + tempfile.TemporaryDirectory() if directory is None + else directory + ) if isinstance(num_neighbors, dict): raise ValueError("num_neighbors dict is currently unsupported!") @@ -175,7 +178,7 @@ def __init__( bulk_sampler = BulkSampler( batch_size, - self.__directory.name, + self.__directory if isinstance(self.__directory, str) else self.__directory.name, self.__graph_store._subgraph(edge_types), fanout_vals=num_neighbors, with_replacement=replace, @@ -219,7 +222,11 @@ def __init__( ) bulk_sampler.flush() - self.__input_files = iter(os.listdir(self.__directory.name)) + self.__input_files = iter(os.listdir( + self.__directory + if isinstance(self.__directory, str) + else self.__directory.name + )) def __next__(self): from time import perf_counter @@ -437,11 +444,10 @@ def __next__(self): # Account for CSR format in cuGraph vs. CSC format in PyG if self.__coo and self.__graph_store.order == "CSC": - for node_type in out.edge_index_dict: - out[node_type].edge_index[0], out[node_type].edge_index[1] = ( - out[node_type].edge_index[1], - out[node_type].edge_index[0], - ) + for edge_type in out.edge_index_dict: + src = out[edge_type].edge_index[0] + dst = out[edge_type].edge_index[1] + out[edge_type].edge_index = torch.stack([dst,src]) out.set_value_dict("num_sampled_nodes", sampler_output.num_sampled_nodes) out.set_value_dict("num_sampled_edges", sampler_output.num_sampled_edges) From d5da756c8ac8bbbb26df137bcb2eae755a457031 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 6 Nov 2023 10:41:46 -0800 Subject: [PATCH 04/24] test updates --- .../cugraph_pyg/loader/cugraph_node_loader.py | 22 ++++++---- .../cugraph_pyg/tests/test_cugraph_loader.py | 41 +++++++++++++++++-- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py index fb9f3f9c70a..242a8b081c0 100644 --- a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py +++ b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py @@ -15,6 +15,7 @@ import os import re +import warnings import cupy import cudf @@ -167,14 +168,21 @@ def __init__( if isinstance(num_neighbors, dict): raise ValueError("num_neighbors dict is currently unsupported!") - renumber = ( - True - if ( - (len(self.__graph_store.node_types) == 1) - and (len(self.__graph_store.edge_types) == 1) + if 'renumber' in kwargs: + warnings.warn( + "Setting renumbering manually could result in invalid output," + " please ensure you intended to do this." + ) + renumber=kwargs.pop('renumber') + else: + renumber = ( + True + if ( + (len(self.__graph_store.node_types) == 1) + and (len(self.__graph_store.edge_types) == 1) + ) + else False ) - else False - ) bulk_sampler = BulkSampler( batch_size, diff --git a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py index 853836dc2a6..28a90c137fe 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py @@ -18,6 +18,7 @@ import cudf import cupy +import numpy as np from cugraph_pyg.loader import CuGraphNeighborLoader from cugraph_pyg.loader import BulkSampleLoader @@ -27,6 +28,8 @@ from cugraph.gnn import FeatureStore from cugraph.utilities.utils import import_optional, MissingModule +from typing import Dict, Tuple + torch = import_optional("torch") torch_geometric = import_optional("torch_geometric") trim_to_layer = import_optional("torch_geometric.utils.trim_to_layer") @@ -40,7 +43,7 @@ @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") -def test_cugraph_loader_basic(karate_gnn): +def test_cugraph_loader_basic(karate_gnn: Tuple[FeatureStore, Dict[str, np.ndarray], Dict[str, int]]): F, G, N = karate_gnn cugraph_store = CuGraphStore(F, G, N, order="CSR") loader = CuGraphNeighborLoader( @@ -66,7 +69,7 @@ def test_cugraph_loader_basic(karate_gnn): @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") -def test_cugraph_loader_hetero(karate_gnn): +def test_cugraph_loader_hetero(karate_gnn: Tuple[FeatureStore, Dict[str, np.ndarray], Dict[str, int]]): F, G, N = karate_gnn cugraph_store = CuGraphStore(F, G, N, order="CSR") loader = CuGraphNeighborLoader( @@ -342,7 +345,7 @@ def test_cugraph_loader_e2e_coo(): @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") @pytest.mark.skipif(not HAS_TORCH_SPARSE, reason="torch-sparse not available") @pytest.mark.parametrize("framework", ["pyg", "cugraph-ops"]) -def test_cugraph_loader_e2e_csc(framework): +def test_cugraph_loader_e2e_csc(framework: str): m = [2, 9, 99, 82, 9, 3, 18, 1, 12] x = torch.randint(3000, (256, 256)).to(torch.float32) F = FeatureStore() @@ -442,3 +445,35 @@ def test_cugraph_loader_e2e_csc(framework): x = x.narrow(dim=0, start=0, length=s - num_sampled_nodes[1]) assert list(x.shape) == [1, 1] + + +@pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") +@pytest.mark.parametrize("directory", ["local", "temp"]) +def test_load_directory(karate_gnn: Tuple[FeatureStore, Dict[str, np.ndarray], Dict[str, int]], directory: str): + if directory == "local": + local_dir = tempfile.TemporaryDirectory(dir=".") + + cugraph_store = CuGraphStore(*karate_gnn) + cugraph_loader = CuGraphNeighborLoader( + (cugraph_store, cugraph_store), + torch.arange(8, dtype=torch.int64), + 2, + num_neighbors=[8, 4, 2], + random_state=62, + replace=False, + directory=None if directory == "temp" else local_dir.name, + batches_per_partition=1 + ) + + it = iter(cugraph_loader) + next_batch = next(it) + assert next_batch is not None + + if directory == "local": + assert len(os.listdir(local_dir.name)) == 4 + + count = 1 + while next(it, None) is not None: + count += 1 + + assert count == 4 From 77c4525888ae6f20c7cc39d3f9201b3409aa1102 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 6 Nov 2023 10:48:39 -0800 Subject: [PATCH 05/24] style --- .../cugraph_pyg/loader/cugraph_node_loader.py | 25 +++++++++++-------- .../cugraph_pyg/tests/test_cugraph_loader.py | 17 +++++++++---- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py index 242a8b081c0..689ae0f69da 100644 --- a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py +++ b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py @@ -161,19 +161,18 @@ def __init__( raise ValueError("Batch size must be >= 1") self.__directory = ( - tempfile.TemporaryDirectory() if directory is None - else directory + tempfile.TemporaryDirectory() if directory is None else directory ) if isinstance(num_neighbors, dict): raise ValueError("num_neighbors dict is currently unsupported!") - if 'renumber' in kwargs: + if "renumber" in kwargs: warnings.warn( "Setting renumbering manually could result in invalid output," " please ensure you intended to do this." ) - renumber=kwargs.pop('renumber') + renumber = kwargs.pop("renumber") else: renumber = ( True @@ -186,7 +185,9 @@ def __init__( bulk_sampler = BulkSampler( batch_size, - self.__directory if isinstance(self.__directory, str) else self.__directory.name, + self.__directory + if isinstance(self.__directory, str) + else self.__directory.name, self.__graph_store._subgraph(edge_types), fanout_vals=num_neighbors, with_replacement=replace, @@ -230,11 +231,13 @@ def __init__( ) bulk_sampler.flush() - self.__input_files = iter(os.listdir( - self.__directory - if isinstance(self.__directory, str) - else self.__directory.name - )) + self.__input_files = iter( + os.listdir( + self.__directory + if isinstance(self.__directory, str) + else self.__directory.name + ) + ) def __next__(self): from time import perf_counter @@ -455,7 +458,7 @@ def __next__(self): for edge_type in out.edge_index_dict: src = out[edge_type].edge_index[0] dst = out[edge_type].edge_index[1] - out[edge_type].edge_index = torch.stack([dst,src]) + out[edge_type].edge_index = torch.stack([dst, src]) out.set_value_dict("num_sampled_nodes", sampler_output.num_sampled_nodes) out.set_value_dict("num_sampled_edges", sampler_output.num_sampled_edges) diff --git a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py index 28a90c137fe..a0af3ed975e 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py @@ -43,7 +43,9 @@ @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") -def test_cugraph_loader_basic(karate_gnn: Tuple[FeatureStore, Dict[str, np.ndarray], Dict[str, int]]): +def test_cugraph_loader_basic( + karate_gnn: Tuple[FeatureStore, Dict[str, np.ndarray], Dict[str, int]] +): F, G, N = karate_gnn cugraph_store = CuGraphStore(F, G, N, order="CSR") loader = CuGraphNeighborLoader( @@ -69,7 +71,9 @@ def test_cugraph_loader_basic(karate_gnn: Tuple[FeatureStore, Dict[str, np.ndarr @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") -def test_cugraph_loader_hetero(karate_gnn: Tuple[FeatureStore, Dict[str, np.ndarray], Dict[str, int]]): +def test_cugraph_loader_hetero( + karate_gnn: Tuple[FeatureStore, Dict[str, np.ndarray], Dict[str, int]] +): F, G, N = karate_gnn cugraph_store = CuGraphStore(F, G, N, order="CSR") loader = CuGraphNeighborLoader( @@ -449,7 +453,10 @@ def test_cugraph_loader_e2e_csc(framework: str): @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") @pytest.mark.parametrize("directory", ["local", "temp"]) -def test_load_directory(karate_gnn: Tuple[FeatureStore, Dict[str, np.ndarray], Dict[str, int]], directory: str): +def test_load_directory( + karate_gnn: Tuple[FeatureStore, Dict[str, np.ndarray], Dict[str, int]], + directory: str, +): if directory == "local": local_dir = tempfile.TemporaryDirectory(dir=".") @@ -462,7 +469,7 @@ def test_load_directory(karate_gnn: Tuple[FeatureStore, Dict[str, np.ndarray], D random_state=62, replace=False, directory=None if directory == "temp" else local_dir.name, - batches_per_partition=1 + batches_per_partition=1, ) it = iter(cugraph_loader) @@ -475,5 +482,5 @@ def test_load_directory(karate_gnn: Tuple[FeatureStore, Dict[str, np.ndarray], D count = 1 while next(it, None) is not None: count += 1 - + assert count == 4 From 1a6124f63e9448f85e91a33bc1873602b475bf71 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Tue, 7 Nov 2023 08:55:50 -0800 Subject: [PATCH 06/24] update dependencies.yaml --- dependencies.yaml | 4 ++-- python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index b127d9bd29e..cdce32c179c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -503,9 +503,9 @@ dependencies: - output_types: [conda] packages: - cugraph==23.12.* - - pytorch==2.0 + - pytorch==2.1 - pytorch-cuda==11.8 - - pyg=2.3.1=*torch_2.0.0*cu118* + - pyg=2.4.0=*torch_2.1.0*cu118* depends_on_rmm: common: diff --git a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml index f98eab430ba..4252e0db3d2 100644 --- a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml +++ b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml @@ -13,13 +13,13 @@ dependencies: - cugraph==23.12.* - pandas - pre-commit -- pyg=2.3.1=*torch_2.0.0*cu118* +- pyg=2.4.0=*torch_2.1.0*cu118* - pylibcugraphops==23.12.* - pytest - pytest-benchmark - pytest-cov - pytest-xdist - pytorch-cuda==11.8 -- pytorch==2.0 +- pytorch==2.1 - scipy name: cugraph_pyg_dev_cuda-118 From cf93084bdda5d846d0f66baf1064c0572d26d35d Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Tue, 7 Nov 2023 09:14:49 -0800 Subject: [PATCH 07/24] style --- .../cugraph_pyg/data/cugraph_store.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py index cd11adbc199..f68ef207171 100644 --- a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py @@ -365,7 +365,9 @@ def __infer_offsets( } ) - def __dask_array_from_numpy(self, array: np.ndarray, client: distributed.client.Client, npartitions: int): + def __dask_array_from_numpy( + self, array: np.ndarray, client: distributed.client.Client, npartitions: int + ): split_array = np.array_split(array, npartitions) shapes = [n.shape for n in split_array] scattered_array = client.scatter(split_array) @@ -494,17 +496,17 @@ def __construct_graph( etp_dar = self.__dask_array_from_numpy(na_etp, client, npartitions) del na_etp - df = dd.from_dask_array(etp_dar, columns=['etp']) - df['src'] = dst_dar if order == "CSC" else src_dar - df['dst'] = src_dar if order == "CSC" else dst_dar + df = dd.from_dask_array(etp_dar, columns=["etp"]) + df["src"] = dst_dar if order == "CSC" else src_dar + df["dst"] = src_dar if order == "CSC" else dst_dar del src_dar del dst_dar del etp_dar - if df.etp.dtype != 'int32': + if df.etp.dtype != "int32": raise ValueError("Edge type must be int32!") - + print(df) # Ensure the dataframe is constructed on each partition @@ -526,7 +528,9 @@ def get_empty_df(): if len(f) > 0 else get_empty_df(), meta=get_empty_df(), - ).reset_index(drop=True) # should be ok for dask + ).reset_index( + drop=True + ) # should be ok for dask else: df = pandas.DataFrame( { From f6199319f5789ecc872ee5401bde3d2e93330bb3 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Tue, 7 Nov 2023 11:07:32 -0800 Subject: [PATCH 08/24] update test --- .../cugraph_pyg/data/cugraph_store.py | 2 -- .../tests/mg/test_mg_cugraph_loader.py | 1 - .../tests/mg/test_mg_cugraph_store.py | 28 ++++++++++++++++++- .../cugraph_pyg/tests/test_cugraph_store.py | 2 +- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py index 8590e0052a5..20f21218b49 100644 --- a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py @@ -507,8 +507,6 @@ def __construct_graph( if df.etp.dtype != "int32": raise ValueError("Edge type must be int32!") - print(df) - # Ensure the dataframe is constructed on each partition # instead of adding additional synchronization head from potential # host to device copies. diff --git a/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_loader.py b/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_loader.py index 55aebf305da..f5035a38621 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_loader.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_loader.py @@ -15,7 +15,6 @@ from cugraph_pyg.loader import CuGraphNeighborLoader from cugraph_pyg.data import CuGraphStore - from cugraph.utilities.utils import import_optional, MissingModule torch = import_optional("torch") diff --git a/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py index ed7f70034e2..392b6708abc 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py @@ -120,7 +120,7 @@ def test_get_edge_index(graph, edge_index_type, dask_client): G[et][0] = dask_cudf.from_cudf(cudf.Series(G[et][0]), npartitions=1) G[et][1] = dask_cudf.from_cudf(cudf.Series(G[et][1]), npartitions=1) - cugraph_store = CuGraphStore(F, G, N, multi_gpu=True) + cugraph_store = CuGraphStore(F, G, N, multi_gpu=True, order="CSC") for pyg_can_edge_type in G: src, dst = cugraph_store.get_edge_index( @@ -386,3 +386,29 @@ def test_mg_frame_handle(graph, dask_client): F, G, N = graph cugraph_store = CuGraphStore(F, G, N, multi_gpu=True) assert isinstance(cugraph_store._EXPERIMENTAL__CuGraphStore__graph._plc_graph, dict) + + +@pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") +def test_cugraph_loader_large_index(dask_client): + large_index = ( + np.random.randint(0, 1_000_000, (100_000_000,)), + np.random.randint(0, 1_000_000, (100_000_000,)), + ) + + large_features = np.random.randint(0, 50, (1_000_000,)) + F = cugraph.gnn.FeatureStore(backend="torch") + F.add_data(large_features, "N", "f") + + store = CuGraphStore( + F, + {("N", "e", "N"): large_index}, + {"N": 1_000_000}, + multi_gpu=True, + ) + + graph = store._subgraph() + assert isinstance(graph, cugraph.Graph) + + el = graph.view_edge_list().compute() + assert (el["src"].values_host - large_index[0]).sum() == 0 + assert (el["dst"].values_host - large_index[1]).sum() == 0 diff --git a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py index da3043760d4..b39ebad8254 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py @@ -113,7 +113,7 @@ def test_get_edge_index(graph, edge_index_type): G[et][0] = cudf.Series(G[et][0]) G[et][1] = cudf.Series(G[et][1]) - cugraph_store = CuGraphStore(F, G, N) + cugraph_store = CuGraphStore(F, G, N, order="CSC") for pyg_can_edge_type in G: src, dst = cugraph_store.get_edge_index( From 0f4693cd793131ea101930b5ec2063815e191828 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Wed, 8 Nov 2023 10:25:34 -0800 Subject: [PATCH 09/24] attempt to fix package resolution issue --- ci/test_python.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ci/test_python.sh b/ci/test_python.sh index 1690ce2f15b..79b5079b5ae 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -200,15 +200,21 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then # Install pytorch rapids-mamba-retry install \ --force-reinstall \ - --channel pyg \ --channel pytorch \ --channel nvidia \ - 'pyg=2.3' \ - 'pytorch=2.0.0' \ + --channel conda-forge \ + 'pytorch=2.1.0' \ 'pytorch-cuda=11.8' # Install pyg dependencies (which requires pip) - pip install pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv -f https://data.pyg.org/whl/torch-2.0.0+cu118.html + pip install \ + torch_geometric==2.4.0 + pyg_lib \ + torch_scatter \ + torch_sparse \ + torch_cluster \ + torch_spline_conv \ + -f https://data.pyg.org/whl/torch-2.1.0+cu118.html rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ From 9713ceed0fc02fccd7749315f71468d147d35e7f Mon Sep 17 00:00:00 2001 From: Naim Date: Thu, 9 Nov 2023 18:16:06 +0100 Subject: [PATCH 10/24] Add ucx to cugraph recipe --- conda/recipes/cugraph/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/recipes/cugraph/meta.yaml b/conda/recipes/cugraph/meta.yaml index 65403bc8d73..7cd0708fffe 100644 --- a/conda/recipes/cugraph/meta.yaml +++ b/conda/recipes/cugraph/meta.yaml @@ -88,6 +88,7 @@ requirements: - requests - ucx-proc=*=gpu - ucx-py {{ ucx_py_version }} + - ucx >=1.14.0 tests: requirements: From bc14fd79d24f665b989411941f242e3bd1bd6749 Mon Sep 17 00:00:00 2001 From: Naim Date: Thu, 9 Nov 2023 18:22:06 +0100 Subject: [PATCH 11/24] Add ucx to libcugraph recipe --- conda/recipes/cugraph/meta.yaml | 1 - conda/recipes/libcugraph/meta.yaml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/cugraph/meta.yaml b/conda/recipes/cugraph/meta.yaml index 7cd0708fffe..65403bc8d73 100644 --- a/conda/recipes/cugraph/meta.yaml +++ b/conda/recipes/cugraph/meta.yaml @@ -88,7 +88,6 @@ requirements: - requests - ucx-proc=*=gpu - ucx-py {{ ucx_py_version }} - - ucx >=1.14.0 tests: requirements: diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index 66f72e6b6b5..5171725f896 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -77,6 +77,7 @@ requirements: - librmm ={{ minor_version }} - nccl {{ nccl_version }} - ucx-proc=*=gpu + - ucx >=1.14.0 outputs: - name: libcugraph From 2064c37462bf4ff4f67cfd49e7c8740a4e03ba3c Mon Sep 17 00:00:00 2001 From: Naim <110031745+naimnv@users.noreply.github.com> Date: Thu, 9 Nov 2023 20:41:48 +0100 Subject: [PATCH 12/24] Update conda/recipes/libcugraph/meta.yaml Co-authored-by: Ray Douglass <3107146+raydouglass@users.noreply.github.com> --- conda/recipes/libcugraph/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index 5171725f896..ecfabcef390 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -77,7 +77,7 @@ requirements: - librmm ={{ minor_version }} - nccl {{ nccl_version }} - ucx-proc=*=gpu - - ucx >=1.14.0 + - ucx >=1.14.1,<1.16.0 outputs: - name: libcugraph From 11ff5bf5a08bc1a7e1b01312745ae043a4ba3abf Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 10 Nov 2023 14:58:29 -0800 Subject: [PATCH 13/24] fix typing --- python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py | 12 +++++++++--- .../cugraph_pyg/tests/test_cugraph_loader.py | 12 +++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py index d1b24543956..edeeface4c4 100644 --- a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py @@ -210,7 +210,10 @@ class EXPERIMENTAL__CuGraphStore: def __init__( self, F: cugraph.gnn.FeatureStore, - G: Union[Dict[str, Tuple[TensorType]], Dict[str, int]], + G: Union[ + Dict[Tuple[str, str, str], Tuple[TensorType]], + Dict[Tuple[str, str, str], int], + ], num_nodes_dict: Dict[str, int], *, multi_gpu: bool = False, @@ -744,7 +747,7 @@ def _subgraph(self, edge_types: List[tuple] = None) -> cugraph.MultiGraph: def _get_vertex_groups_from_sample( self, nodes_of_interest: TensorType, is_sorted: bool = False - ) -> dict: + ) -> Dict[str, torch.Tensor]: """ Given a tensor of nodes of interest, this method a single dictionary, noi_index. @@ -808,7 +811,10 @@ def _get_sample_from_vertex_groups( def _get_renumbered_edge_groups_from_sample( self, sampling_results: cudf.DataFrame, noi_index: dict - ) -> Tuple[dict, dict]: + ) -> Tuple[ + Dict[Tuple[str, str, str], torch.Tensor], + Tuple[Dict[Tuple[str, str, str], torch.Tensor]], + ]: """ Given a cudf (NOT dask_cudf) DataFrame of sampling results and a dictionary of non-renumbered vertex ids grouped by vertex type, this method diff --git a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py index a0af3ed975e..27b73bf7d35 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_loader.py @@ -44,7 +44,9 @@ @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") def test_cugraph_loader_basic( - karate_gnn: Tuple[FeatureStore, Dict[str, np.ndarray], Dict[str, int]] + karate_gnn: Tuple[ + FeatureStore, Dict[Tuple[str, str, str], np.ndarray], Dict[str, int] + ] ): F, G, N = karate_gnn cugraph_store = CuGraphStore(F, G, N, order="CSR") @@ -72,7 +74,9 @@ def test_cugraph_loader_basic( @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") def test_cugraph_loader_hetero( - karate_gnn: Tuple[FeatureStore, Dict[str, np.ndarray], Dict[str, int]] + karate_gnn: Tuple[ + FeatureStore, Dict[Tuple[str, str, str], np.ndarray], Dict[str, int] + ] ): F, G, N = karate_gnn cugraph_store = CuGraphStore(F, G, N, order="CSR") @@ -454,7 +458,9 @@ def test_cugraph_loader_e2e_csc(framework: str): @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") @pytest.mark.parametrize("directory", ["local", "temp"]) def test_load_directory( - karate_gnn: Tuple[FeatureStore, Dict[str, np.ndarray], Dict[str, int]], + karate_gnn: Tuple[ + FeatureStore, Dict[Tuple[str, str, str], np.ndarray], Dict[str, int] + ], directory: str, ): if directory == "local": From 7b9396c24a028f55cdf1a8ac43df9996609f44bf Mon Sep 17 00:00:00 2001 From: Alex Barghi <105237337+alexbarghi-nv@users.noreply.github.com> Date: Fri, 10 Nov 2023 18:01:39 -0500 Subject: [PATCH 14/24] replace manual exchange with flip Co-authored-by: Tingyu Wang --- python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py index 689ae0f69da..b9792d12591 100644 --- a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py +++ b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py @@ -456,9 +456,7 @@ def __next__(self): # Account for CSR format in cuGraph vs. CSC format in PyG if self.__coo and self.__graph_store.order == "CSC": for edge_type in out.edge_index_dict: - src = out[edge_type].edge_index[0] - dst = out[edge_type].edge_index[1] - out[edge_type].edge_index = torch.stack([dst, src]) + out[edge_type].edge_index = out[edge_type].edge_index.flip(dims=[0]) out.set_value_dict("num_sampled_nodes", sampler_output.num_sampled_nodes) out.set_value_dict("num_sampled_edges", sampler_output.num_sampled_edges) From d1b5f223a80a98740794d41dfcb5a81a1388fae3 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Sat, 11 Nov 2023 15:01:29 -0800 Subject: [PATCH 15/24] various changes to get ci working --- ci/test_python.sh | 29 +++++++++++++++-------------- conda/recipes/cugraph-pyg/meta.yaml | 2 +- dependencies.yaml | 4 ++-- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/ci/test_python.sh b/ci/test_python.sh index 79b5079b5ae..29412120310 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -197,33 +197,34 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then conda activate test_cugraph_pyg set -u - # Install pytorch + rapids-mamba-retry install \ + --channel "${CPP_CHANNEL}" \ + --channel "${PYTHON_CHANNEL}" \ + libcugraph \ + pylibcugraph \ + pylibcugraphops \ + cugraph \ + cugraph-pyg + + # Install pytorch, pyg rapids-mamba-retry install \ --force-reinstall \ + --channel pyg \ --channel pytorch \ --channel nvidia \ --channel conda-forge \ - 'pytorch=2.1.0' \ - 'pytorch-cuda=11.8' + "pytorch>=2.0,<2.1" \ + "pytorch-cuda=11.8" \ + "pyg>=2.3.1" # Install pyg dependencies (which requires pip) pip install \ - torch_geometric==2.4.0 pyg_lib \ torch_scatter \ torch_sparse \ torch_cluster \ torch_spline_conv \ - -f https://data.pyg.org/whl/torch-2.1.0+cu118.html - - rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - libcugraph \ - pylibcugraph \ - pylibcugraphops \ - cugraph \ - cugraph-pyg + -f https://data.pyg.org/whl/torch-2.0.0+cu118.html rapids-print-env diff --git a/conda/recipes/cugraph-pyg/meta.yaml b/conda/recipes/cugraph-pyg/meta.yaml index 2714dcfa55a..89006c3bb55 100644 --- a/conda/recipes/cugraph-pyg/meta.yaml +++ b/conda/recipes/cugraph-pyg/meta.yaml @@ -34,7 +34,7 @@ requirements: - cupy >=12.0.0 - cugraph ={{ version }} - pylibcugraphops ={{ version }} - - pyg >=2.3,<2.4 + - pyg >=2.3,<2.5 tests: imports: diff --git a/dependencies.yaml b/dependencies.yaml index cdce32c179c..b4c42416a25 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -503,9 +503,9 @@ dependencies: - output_types: [conda] packages: - cugraph==23.12.* - - pytorch==2.1 + - pytorch>=2.0 - pytorch-cuda==11.8 - - pyg=2.4.0=*torch_2.1.0*cu118* + - pyg>=2.4.0 depends_on_rmm: common: From e941e10620a0d7fe1954f6cb74c3331bb969ed1d Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Sat, 11 Nov 2023 15:01:59 -0800 Subject: [PATCH 16/24] generator --- python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml index 4252e0db3d2..71d1c7e389c 100644 --- a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml +++ b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml @@ -13,13 +13,13 @@ dependencies: - cugraph==23.12.* - pandas - pre-commit -- pyg=2.4.0=*torch_2.1.0*cu118* +- pyg>=2.4.0 - pylibcugraphops==23.12.* - pytest - pytest-benchmark - pytest-cov - pytest-xdist - pytorch-cuda==11.8 -- pytorch==2.1 +- pytorch>=2.0 - scipy name: cugraph_pyg_dev_cuda-118 From 3f7ec7f59c449936db46e3e8321a4cb881a52784 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Sun, 12 Nov 2023 09:18:53 -0800 Subject: [PATCH 17/24] fix pylibcugraphops version, simplify test_python.sh --- bleh.txt | 167 ++++++++++++++++++++++++++++ ci/test_python.sh | 18 +-- conda/recipes/cugraph-pyg/meta.yaml | 2 +- env.yaml | 29 +++++ 4 files changed, 202 insertions(+), 14 deletions(-) create mode 100644 bleh.txt create mode 100644 env.yaml diff --git a/bleh.txt b/bleh.txt new file mode 100644 index 00000000000..3cb971f3038 --- /dev/null +++ b/bleh.txt @@ -0,0 +1,167 @@ +pytorch/linux-64 Using cache +pytorch/noarch Using cache +nvidia/linux-64 Using cache +nvidia/noarch Using cache +conda-forge/linux-64 Using cache +conda-forge/noarch Using cache +Transaction + + Prefix: /opt/conda/envs/test_cugraph_pyg + + Updating specs: + + - cugraph-pyg + - pytorch[version='>=2.0,<2.1'] + - pytorch-cuda=11.8 + - ca-certificates + - openssl + + + Package Version Build Channel Size +────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + Install: +────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + + + aiohttp 3.8.6 py310h2372a71_1 conda-forge/linux-64 Cached + + aiosignal 1.3.1 pyhd8ed1ab_0 conda-forge/noarch Cached + + async-timeout 4.0.3 pyhd8ed1ab_0 conda-forge/noarch Cached + + attrs 23.1.0 pyh71513ae_1 conda-forge/noarch Cached + + blas 2.119 mkl conda-forge/linux-64 Cached + + blas-devel 3.9.0 19_linux64_mkl conda-forge/linux-64 Cached + + bokeh 3.3.1 pyhd8ed1ab_0 conda-forge/noarch Cached + + brotli-python 1.1.0 py310hc6cd4ac_1 conda-forge/linux-64 Cached + + certifi 2023.7.22 pyhd8ed1ab_0 conda-forge/noarch Cached + + charset-normalizer 3.3.2 pyhd8ed1ab_0 conda-forge/noarch Cached + + click 8.1.7 unix_pyh707e725_0 conda-forge/noarch Cached + + cloudpickle 3.0.0 pyhd8ed1ab_0 conda-forge/noarch Cached + + contourpy 1.2.0 py310hd41b1e2_0 conda-forge/linux-64 Cached + + cuda-cudart 11.8.89 0 nvidia/linux-64 202kB + + cuda-cupti 11.8.87 0 nvidia/linux-64 27MB + + cuda-libraries 11.8.0 0 nvidia/linux-64 2kB + + cuda-nvrtc 11.8.89 0 nvidia/linux-64 20MB + + cuda-nvtx 11.8.86 0 nvidia/linux-64 58kB + + cuda-runtime 11.8.0 0 nvidia/linux-64 1kB + + cugraph 23.12.00a106 cuda11_py310_231111_ge941e106_106 /tmp/python_channel/linux-64 Cached + + cugraph-pyg 23.12.00a106 py310_231111_ge941e106_106 /tmp/python_channel/linux-64 95kB + + cytoolz 0.12.2 py310h2372a71_1 conda-forge/linux-64 Cached + + dask 2023.11.0 pyhd8ed1ab_0 conda-forge/noarch 7kB + + dask-core 2023.11.1a231110 py_gb2f11d026_0 dask/label/dev/noarch 933kB + + dask-cuda 23.12.00a26 py310_231109_ge5b240c_26 rapidsai-nightly/linux-64 Cached + + dask-cudf 23.12.00a686 cuda11_py310_231110_g87d2a36f04_686 rapidsai-nightly/linux-64 Cached + + distributed 2023.11.0 pyhd8ed1ab_0 conda-forge/noarch 792kB + + filelock 3.13.1 pyhd8ed1ab_0 conda-forge/noarch Cached + + freetype 2.12.1 h267a509_2 conda-forge/linux-64 Cached + + frozenlist 1.4.0 py310h2372a71_1 conda-forge/linux-64 Cached + + gmp 6.3.0 h59595ed_0 conda-forge/linux-64 563kB + + gmpy2 2.1.2 py310h3ec546c_1 conda-forge/linux-64 220kB + + idna 3.4 pyhd8ed1ab_0 conda-forge/noarch Cached + + importlib-metadata 6.8.0 pyha770c72_0 conda-forge/noarch Cached + + importlib_metadata 6.8.0 hd8ed1ab_0 conda-forge/noarch Cached + + jinja2 3.1.2 pyhd8ed1ab_1 conda-forge/noarch Cached + + lcms2 2.15 hb7c19ff_3 conda-forge/linux-64 Cached + + lerc 4.0.0 h27087fc_0 conda-forge/linux-64 Cached + + libcufft 10.9.0.58 0 nvidia/linux-64 150MB + + libcugraph 23.12.00a106 cuda11_231111_ge941e106_106 /tmp/cpp_channel/linux-64 Cached + + libcugraphops 23.12.00a14 cuda11_231110_g207773d3_14 rapidsai-nightly/linux-64 Cached + + libdeflate 1.19 hd590300_0 conda-forge/linux-64 Cached + + libhwloc 2.9.3 default_h554bfaf_1009 conda-forge/linux-64 Cached + + libjpeg-turbo 3.0.0 hd590300_1 conda-forge/linux-64 Cached + + liblapacke 3.9.0 19_linux64_mkl conda-forge/linux-64 Cached + + libnpp 11.8.0.86 0 nvidia/linux-64 155MB + + libnvjpeg 11.9.0.86 0 nvidia/linux-64 3MB + + libpng 1.6.39 h753d276_0 conda-forge/linux-64 Cached + + libtiff 4.6.0 ha9c0a0a_2 conda-forge/linux-64 Cached + + libwebp-base 1.3.2 hd590300_0 conda-forge/linux-64 Cached + + libxcb 1.15 h0b41bf4_0 conda-forge/linux-64 Cached + + llvm-openmp 17.0.4 h4dfa4b3_0 conda-forge/linux-64 Cached + + locket 1.0.0 pyhd8ed1ab_0 conda-forge/noarch Cached + + lz4 4.3.2 py310h350c4a5_1 conda-forge/linux-64 Cached + + markupsafe 2.1.3 py310h2372a71_1 conda-forge/linux-64 Cached + + mkl 2023.2.0 h84fe81f_50496 conda-forge/linux-64 Cached + + mkl-devel 2023.2.0 ha770c72_50496 conda-forge/linux-64 Cached + + mkl-include 2023.2.0 h84fe81f_50496 conda-forge/linux-64 Cached + + mpc 1.3.1 hfe3b2da_0 conda-forge/linux-64 116kB + + mpfr 4.2.1 h9458935_0 conda-forge/linux-64 642kB + + mpmath 1.3.0 pyhd8ed1ab_0 conda-forge/noarch 438kB + + msgpack-python 1.0.6 py310hd41b1e2_0 conda-forge/linux-64 Cached + + multidict 6.0.4 py310h2372a71_1 conda-forge/linux-64 Cached + + openjpeg 2.5.0 h488ebb8_3 conda-forge/linux-64 Cached + + partd 1.4.1 pyhd8ed1ab_0 conda-forge/noarch Cached + + pillow 10.1.0 py310h01dd4db_0 conda-forge/linux-64 Cached + + psutil 5.9.5 py310h2372a71_1 conda-forge/linux-64 Cached + + pthread-stubs 0.4 h36c2ea0_1001 conda-forge/linux-64 Cached + + pyarrow-hotfix 0.4 pyhd8ed1ab_0 conda-forge/noarch 13kB + + pyg 2.4.0 py310_torch_2.0.0_cu118 pyg/linux-64 1MB + + pylibcugraph 23.12.00a106 cuda11_py310_231111_ge941e106_106 /tmp/python_channel/linux-64 Cached + + pylibcugraphops 23.12.00a14 cuda11_py310_231110_g207773d3_14 rapidsai-nightly/linux-64 Cached + + pylibraft 23.12.00a85 cuda11_py310_231108_g42512f73_85 rapidsai-nightly/linux-64 Cached + + pynvml 11.4.1 pyhd8ed1ab_0 conda-forge/noarch Cached + + pyparsing 3.1.1 pyhd8ed1ab_0 conda-forge/noarch 90kB + + pysocks 1.7.1 pyha2e5f31_6 conda-forge/noarch Cached + + pytorch 2.0.1 py3.10_cuda11.8_cudnn8.7.0_0 pytorch/linux-64 2GB + + pytorch-cuda 11.8 h7e8668a_5 pytorch/linux-64 4kB + + pytorch-mutex 1.0 cuda pytorch/noarch 3kB + + pyyaml 6.0.1 py310h2372a71_1 conda-forge/linux-64 Cached + + raft-dask 23.12.00a85 cuda11_py310_231108_g42512f73_85 rapidsai-nightly/linux-64 Cached + + requests 2.31.0 pyhd8ed1ab_0 conda-forge/noarch Cached + + sortedcontainers 2.4.0 pyhd8ed1ab_0 conda-forge/noarch Cached + + sympy 1.12 pypyh9d50eac_103 conda-forge/noarch 4MB + + tbb 2021.10.0 h00ab1b0_2 conda-forge/linux-64 Cached + + tblib 2.0.0 pyhd8ed1ab_0 conda-forge/noarch Cached + + toolz 0.12.0 pyhd8ed1ab_0 conda-forge/noarch Cached + + torchtriton 2.0.0 py310 pytorch/linux-64 66MB + + tornado 6.3.3 py310h2372a71_1 conda-forge/linux-64 Cached + + tqdm 4.66.1 pyhd8ed1ab_0 conda-forge/noarch 89kB + + typing-extensions 4.8.0 hd8ed1ab_0 conda-forge/noarch Cached + + ucx-proc 1.0.0 gpu rapidsai-nightly/linux-64 6kB + + ucx-py 0.35.00a13 py310_231109_g78c734c_13 rapidsai-nightly/linux-64 Cached + + urllib3 2.0.7 pyhd8ed1ab_0 conda-forge/noarch Cached + + xorg-libxau 1.0.11 hd590300_0 conda-forge/linux-64 Cached + + xorg-libxdmcp 1.1.3 h7f98852_0 conda-forge/linux-64 Cached + + xyzservices 2023.10.1 pyhd8ed1ab_0 conda-forge/noarch Cached + + yaml 0.2.5 h7f98852_2 conda-forge/linux-64 Cached + + yarl 1.9.2 py310h2372a71_1 conda-forge/linux-64 Cached + + zict 3.0.0 pyhd8ed1ab_0 conda-forge/noarch Cached + + zipp 3.17.0 pyhd8ed1ab_0 conda-forge/noarch Cached + + Change: +────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + + - _openmp_mutex 4.5 2_gnu conda-forge + + _openmp_mutex 4.5 2_kmp_llvm conda-forge/linux-64 Cached + - libblas 3.9.0 19_linux64_openblas conda-forge + + libblas 3.9.0 19_linux64_mkl conda-forge/linux-64 Cached + - libcblas 3.9.0 19_linux64_openblas conda-forge + + libcblas 3.9.0 19_linux64_mkl conda-forge/linux-64 Cached + - liblapack 3.9.0 19_linux64_openblas conda-forge + + liblapack 3.9.0 19_linux64_mkl conda-forge/linux-64 Cached + + Downgrade: +────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + + - libraft 23.12.00a91 cuda11_231110_g047bfb2a_91 rapidsai-nightly + + libraft 23.12.00a85 cuda11_231108_g42512f73_85 rapidsai-nightly/linux-64 639MB + - libraft-headers 23.12.00a91 cuda11_231110_g047bfb2a_91 rapidsai-nightly + + libraft-headers 23.12.00a85 cuda11_231108_g42512f73_85 rapidsai-nightly/linux-64 Cached + - libraft-headers-only 23.12.00a91 cuda11_231110_g047bfb2a_91 rapidsai-nightly + + libraft-headers-only 23.12.00a85 cuda11_231108_g42512f73_85 rapidsai-nightly/linux-64 Cached + + Summary: + + Install: 101 packages + Change: 4 packages + Downgrade: 3 packages + + Total download: 3GB + +────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + + + +Looking for: ['cugraph-pyg', "pytorch[version='>=2.0,<2.1']", 'pytorch-cuda=11.8'] + + +Pinned packages: + - python 3.10.* + + diff --git a/ci/test_python.sh b/ci/test_python.sh index 29412120310..273d3c93482 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -197,25 +197,17 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then conda activate test_cugraph_pyg set -u + # Will automatically install built dependencies of cuGraph-PyG rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - libcugraph \ - pylibcugraph \ - pylibcugraphops \ - cugraph \ - cugraph-pyg - - # Install pytorch, pyg - rapids-mamba-retry install \ - --force-reinstall \ - --channel pyg \ --channel pytorch \ --channel nvidia \ - --channel conda-forge \ + --channel pyg \ + --channel rapidsai-nightly \ + "cugraph-pyg" \ "pytorch>=2.0,<2.1" \ - "pytorch-cuda=11.8" \ - "pyg>=2.3.1" + "pytorch-cuda=11.8" # Install pyg dependencies (which requires pip) pip install \ diff --git a/conda/recipes/cugraph-pyg/meta.yaml b/conda/recipes/cugraph-pyg/meta.yaml index 89006c3bb55..2d03b2edae9 100644 --- a/conda/recipes/cugraph-pyg/meta.yaml +++ b/conda/recipes/cugraph-pyg/meta.yaml @@ -33,7 +33,7 @@ requirements: - pytorch >=2.0 - cupy >=12.0.0 - cugraph ={{ version }} - - pylibcugraphops ={{ version }} + - pylibcugraphops ={{ minor_version }} - pyg >=2.3,<2.5 tests: diff --git a/env.yaml b/env.yaml new file mode 100644 index 00000000000..836c545211d --- /dev/null +++ b/env.yaml @@ -0,0 +1,29 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- dask/label/dev +- pytorch +- pyg +- dglteam/label/cu118 +- conda-forge +- nvidia +dependencies: +- cuda-version=11.8 +- cudatoolkit +- cudf==23.12.* +- networkx>=2.5.1 +- numpy>=1.21 +- pandas +- pylibwholegraph==23.12.* +- pytest +- pytest-benchmark +- pytest-cov +- pytest-xdist +- python-louvain +- python=3.10 +- scikit-learn>=0.23.1 +- scipy +name: test_python_cuda-118_arch-x86_64_py-310 + From 46c446068bcecce0ed45164e77c9a77a94e0c4fb Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Sun, 12 Nov 2023 09:19:21 -0800 Subject: [PATCH 18/24] remove unwanted files --- bleh.txt | 167 ------------------------------------------------------- env.yaml | 29 ---------- 2 files changed, 196 deletions(-) delete mode 100644 bleh.txt delete mode 100644 env.yaml diff --git a/bleh.txt b/bleh.txt deleted file mode 100644 index 3cb971f3038..00000000000 --- a/bleh.txt +++ /dev/null @@ -1,167 +0,0 @@ -pytorch/linux-64 Using cache -pytorch/noarch Using cache -nvidia/linux-64 Using cache -nvidia/noarch Using cache -conda-forge/linux-64 Using cache -conda-forge/noarch Using cache -Transaction - - Prefix: /opt/conda/envs/test_cugraph_pyg - - Updating specs: - - - cugraph-pyg - - pytorch[version='>=2.0,<2.1'] - - pytorch-cuda=11.8 - - ca-certificates - - openssl - - - Package Version Build Channel Size -────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── - Install: -────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── - - + aiohttp 3.8.6 py310h2372a71_1 conda-forge/linux-64 Cached - + aiosignal 1.3.1 pyhd8ed1ab_0 conda-forge/noarch Cached - + async-timeout 4.0.3 pyhd8ed1ab_0 conda-forge/noarch Cached - + attrs 23.1.0 pyh71513ae_1 conda-forge/noarch Cached - + blas 2.119 mkl conda-forge/linux-64 Cached - + blas-devel 3.9.0 19_linux64_mkl conda-forge/linux-64 Cached - + bokeh 3.3.1 pyhd8ed1ab_0 conda-forge/noarch Cached - + brotli-python 1.1.0 py310hc6cd4ac_1 conda-forge/linux-64 Cached - + certifi 2023.7.22 pyhd8ed1ab_0 conda-forge/noarch Cached - + charset-normalizer 3.3.2 pyhd8ed1ab_0 conda-forge/noarch Cached - + click 8.1.7 unix_pyh707e725_0 conda-forge/noarch Cached - + cloudpickle 3.0.0 pyhd8ed1ab_0 conda-forge/noarch Cached - + contourpy 1.2.0 py310hd41b1e2_0 conda-forge/linux-64 Cached - + cuda-cudart 11.8.89 0 nvidia/linux-64 202kB - + cuda-cupti 11.8.87 0 nvidia/linux-64 27MB - + cuda-libraries 11.8.0 0 nvidia/linux-64 2kB - + cuda-nvrtc 11.8.89 0 nvidia/linux-64 20MB - + cuda-nvtx 11.8.86 0 nvidia/linux-64 58kB - + cuda-runtime 11.8.0 0 nvidia/linux-64 1kB - + cugraph 23.12.00a106 cuda11_py310_231111_ge941e106_106 /tmp/python_channel/linux-64 Cached - + cugraph-pyg 23.12.00a106 py310_231111_ge941e106_106 /tmp/python_channel/linux-64 95kB - + cytoolz 0.12.2 py310h2372a71_1 conda-forge/linux-64 Cached - + dask 2023.11.0 pyhd8ed1ab_0 conda-forge/noarch 7kB - + dask-core 2023.11.1a231110 py_gb2f11d026_0 dask/label/dev/noarch 933kB - + dask-cuda 23.12.00a26 py310_231109_ge5b240c_26 rapidsai-nightly/linux-64 Cached - + dask-cudf 23.12.00a686 cuda11_py310_231110_g87d2a36f04_686 rapidsai-nightly/linux-64 Cached - + distributed 2023.11.0 pyhd8ed1ab_0 conda-forge/noarch 792kB - + filelock 3.13.1 pyhd8ed1ab_0 conda-forge/noarch Cached - + freetype 2.12.1 h267a509_2 conda-forge/linux-64 Cached - + frozenlist 1.4.0 py310h2372a71_1 conda-forge/linux-64 Cached - + gmp 6.3.0 h59595ed_0 conda-forge/linux-64 563kB - + gmpy2 2.1.2 py310h3ec546c_1 conda-forge/linux-64 220kB - + idna 3.4 pyhd8ed1ab_0 conda-forge/noarch Cached - + importlib-metadata 6.8.0 pyha770c72_0 conda-forge/noarch Cached - + importlib_metadata 6.8.0 hd8ed1ab_0 conda-forge/noarch Cached - + jinja2 3.1.2 pyhd8ed1ab_1 conda-forge/noarch Cached - + lcms2 2.15 hb7c19ff_3 conda-forge/linux-64 Cached - + lerc 4.0.0 h27087fc_0 conda-forge/linux-64 Cached - + libcufft 10.9.0.58 0 nvidia/linux-64 150MB - + libcugraph 23.12.00a106 cuda11_231111_ge941e106_106 /tmp/cpp_channel/linux-64 Cached - + libcugraphops 23.12.00a14 cuda11_231110_g207773d3_14 rapidsai-nightly/linux-64 Cached - + libdeflate 1.19 hd590300_0 conda-forge/linux-64 Cached - + libhwloc 2.9.3 default_h554bfaf_1009 conda-forge/linux-64 Cached - + libjpeg-turbo 3.0.0 hd590300_1 conda-forge/linux-64 Cached - + liblapacke 3.9.0 19_linux64_mkl conda-forge/linux-64 Cached - + libnpp 11.8.0.86 0 nvidia/linux-64 155MB - + libnvjpeg 11.9.0.86 0 nvidia/linux-64 3MB - + libpng 1.6.39 h753d276_0 conda-forge/linux-64 Cached - + libtiff 4.6.0 ha9c0a0a_2 conda-forge/linux-64 Cached - + libwebp-base 1.3.2 hd590300_0 conda-forge/linux-64 Cached - + libxcb 1.15 h0b41bf4_0 conda-forge/linux-64 Cached - + llvm-openmp 17.0.4 h4dfa4b3_0 conda-forge/linux-64 Cached - + locket 1.0.0 pyhd8ed1ab_0 conda-forge/noarch Cached - + lz4 4.3.2 py310h350c4a5_1 conda-forge/linux-64 Cached - + markupsafe 2.1.3 py310h2372a71_1 conda-forge/linux-64 Cached - + mkl 2023.2.0 h84fe81f_50496 conda-forge/linux-64 Cached - + mkl-devel 2023.2.0 ha770c72_50496 conda-forge/linux-64 Cached - + mkl-include 2023.2.0 h84fe81f_50496 conda-forge/linux-64 Cached - + mpc 1.3.1 hfe3b2da_0 conda-forge/linux-64 116kB - + mpfr 4.2.1 h9458935_0 conda-forge/linux-64 642kB - + mpmath 1.3.0 pyhd8ed1ab_0 conda-forge/noarch 438kB - + msgpack-python 1.0.6 py310hd41b1e2_0 conda-forge/linux-64 Cached - + multidict 6.0.4 py310h2372a71_1 conda-forge/linux-64 Cached - + openjpeg 2.5.0 h488ebb8_3 conda-forge/linux-64 Cached - + partd 1.4.1 pyhd8ed1ab_0 conda-forge/noarch Cached - + pillow 10.1.0 py310h01dd4db_0 conda-forge/linux-64 Cached - + psutil 5.9.5 py310h2372a71_1 conda-forge/linux-64 Cached - + pthread-stubs 0.4 h36c2ea0_1001 conda-forge/linux-64 Cached - + pyarrow-hotfix 0.4 pyhd8ed1ab_0 conda-forge/noarch 13kB - + pyg 2.4.0 py310_torch_2.0.0_cu118 pyg/linux-64 1MB - + pylibcugraph 23.12.00a106 cuda11_py310_231111_ge941e106_106 /tmp/python_channel/linux-64 Cached - + pylibcugraphops 23.12.00a14 cuda11_py310_231110_g207773d3_14 rapidsai-nightly/linux-64 Cached - + pylibraft 23.12.00a85 cuda11_py310_231108_g42512f73_85 rapidsai-nightly/linux-64 Cached - + pynvml 11.4.1 pyhd8ed1ab_0 conda-forge/noarch Cached - + pyparsing 3.1.1 pyhd8ed1ab_0 conda-forge/noarch 90kB - + pysocks 1.7.1 pyha2e5f31_6 conda-forge/noarch Cached - + pytorch 2.0.1 py3.10_cuda11.8_cudnn8.7.0_0 pytorch/linux-64 2GB - + pytorch-cuda 11.8 h7e8668a_5 pytorch/linux-64 4kB - + pytorch-mutex 1.0 cuda pytorch/noarch 3kB - + pyyaml 6.0.1 py310h2372a71_1 conda-forge/linux-64 Cached - + raft-dask 23.12.00a85 cuda11_py310_231108_g42512f73_85 rapidsai-nightly/linux-64 Cached - + requests 2.31.0 pyhd8ed1ab_0 conda-forge/noarch Cached - + sortedcontainers 2.4.0 pyhd8ed1ab_0 conda-forge/noarch Cached - + sympy 1.12 pypyh9d50eac_103 conda-forge/noarch 4MB - + tbb 2021.10.0 h00ab1b0_2 conda-forge/linux-64 Cached - + tblib 2.0.0 pyhd8ed1ab_0 conda-forge/noarch Cached - + toolz 0.12.0 pyhd8ed1ab_0 conda-forge/noarch Cached - + torchtriton 2.0.0 py310 pytorch/linux-64 66MB - + tornado 6.3.3 py310h2372a71_1 conda-forge/linux-64 Cached - + tqdm 4.66.1 pyhd8ed1ab_0 conda-forge/noarch 89kB - + typing-extensions 4.8.0 hd8ed1ab_0 conda-forge/noarch Cached - + ucx-proc 1.0.0 gpu rapidsai-nightly/linux-64 6kB - + ucx-py 0.35.00a13 py310_231109_g78c734c_13 rapidsai-nightly/linux-64 Cached - + urllib3 2.0.7 pyhd8ed1ab_0 conda-forge/noarch Cached - + xorg-libxau 1.0.11 hd590300_0 conda-forge/linux-64 Cached - + xorg-libxdmcp 1.1.3 h7f98852_0 conda-forge/linux-64 Cached - + xyzservices 2023.10.1 pyhd8ed1ab_0 conda-forge/noarch Cached - + yaml 0.2.5 h7f98852_2 conda-forge/linux-64 Cached - + yarl 1.9.2 py310h2372a71_1 conda-forge/linux-64 Cached - + zict 3.0.0 pyhd8ed1ab_0 conda-forge/noarch Cached - + zipp 3.17.0 pyhd8ed1ab_0 conda-forge/noarch Cached - - Change: -────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── - - - _openmp_mutex 4.5 2_gnu conda-forge - + _openmp_mutex 4.5 2_kmp_llvm conda-forge/linux-64 Cached - - libblas 3.9.0 19_linux64_openblas conda-forge - + libblas 3.9.0 19_linux64_mkl conda-forge/linux-64 Cached - - libcblas 3.9.0 19_linux64_openblas conda-forge - + libcblas 3.9.0 19_linux64_mkl conda-forge/linux-64 Cached - - liblapack 3.9.0 19_linux64_openblas conda-forge - + liblapack 3.9.0 19_linux64_mkl conda-forge/linux-64 Cached - - Downgrade: -────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── - - - libraft 23.12.00a91 cuda11_231110_g047bfb2a_91 rapidsai-nightly - + libraft 23.12.00a85 cuda11_231108_g42512f73_85 rapidsai-nightly/linux-64 639MB - - libraft-headers 23.12.00a91 cuda11_231110_g047bfb2a_91 rapidsai-nightly - + libraft-headers 23.12.00a85 cuda11_231108_g42512f73_85 rapidsai-nightly/linux-64 Cached - - libraft-headers-only 23.12.00a91 cuda11_231110_g047bfb2a_91 rapidsai-nightly - + libraft-headers-only 23.12.00a85 cuda11_231108_g42512f73_85 rapidsai-nightly/linux-64 Cached - - Summary: - - Install: 101 packages - Change: 4 packages - Downgrade: 3 packages - - Total download: 3GB - -────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── - - - -Looking for: ['cugraph-pyg', "pytorch[version='>=2.0,<2.1']", 'pytorch-cuda=11.8'] - - -Pinned packages: - - python 3.10.* - - diff --git a/env.yaml b/env.yaml deleted file mode 100644 index 836c545211d..00000000000 --- a/env.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# This file is generated by `rapids-dependency-file-generator`. -# To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. -channels: -- rapidsai -- rapidsai-nightly -- dask/label/dev -- pytorch -- pyg -- dglteam/label/cu118 -- conda-forge -- nvidia -dependencies: -- cuda-version=11.8 -- cudatoolkit -- cudf==23.12.* -- networkx>=2.5.1 -- numpy>=1.21 -- pandas -- pylibwholegraph==23.12.* -- pytest -- pytest-benchmark -- pytest-cov -- pytest-xdist -- python-louvain -- python=3.10 -- scikit-learn>=0.23.1 -- scipy -name: test_python_cuda-118_arch-x86_64_py-310 - From d229de43615f91f3d41b5fdb7ec536352e9c851b Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Sun, 12 Nov 2023 09:49:55 -0800 Subject: [PATCH 19/24] remove strict csr/csr check --- python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py index b9792d12591..ad8d22e255e 100644 --- a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py +++ b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py @@ -441,9 +441,6 @@ def __next__(self): sampler_output.edge, ) else: - if self.__graph_store.order == "CSR": - raise ValueError("CSR format incompatible with CSC output") - out = filter_cugraph_store_csc( self.__feature_store, self.__graph_store, From 55516bd17aae65672b98c5bf3814534ecf94962c Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Sun, 12 Nov 2023 09:53:53 -0800 Subject: [PATCH 20/24] remove strict format check --- python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py index b9792d12591..ad8d22e255e 100644 --- a/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py +++ b/python/cugraph-pyg/cugraph_pyg/loader/cugraph_node_loader.py @@ -441,9 +441,6 @@ def __next__(self): sampler_output.edge, ) else: - if self.__graph_store.order == "CSR": - raise ValueError("CSR format incompatible with CSC output") - out = filter_cugraph_store_csc( self.__feature_store, self.__graph_store, From d1334ac4d08682c3ce363e661d4b5a27f6beb3a2 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 13 Nov 2023 18:33:00 -0800 Subject: [PATCH 21/24] test correction --- .../cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py | 2 +- python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py index ed7f70034e2..b139df44636 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py @@ -120,7 +120,7 @@ def test_get_edge_index(graph, edge_index_type, dask_client): G[et][0] = dask_cudf.from_cudf(cudf.Series(G[et][0]), npartitions=1) G[et][1] = dask_cudf.from_cudf(cudf.Series(G[et][1]), npartitions=1) - cugraph_store = CuGraphStore(F, G, N, multi_gpu=True) + cugraph_store = CuGraphStore(F, G, N, order='CSC', multi_gpu=True) for pyg_can_edge_type in G: src, dst = cugraph_store.get_edge_index( diff --git a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py index da3043760d4..982140984c2 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py @@ -113,7 +113,7 @@ def test_get_edge_index(graph, edge_index_type): G[et][0] = cudf.Series(G[et][0]) G[et][1] = cudf.Series(G[et][1]) - cugraph_store = CuGraphStore(F, G, N) + cugraph_store = CuGraphStore(F, G, N, order='CSC') for pyg_can_edge_type in G: src, dst = cugraph_store.get_edge_index( From 2706d87e0e0ee2b55f5798336598a939e1c18b3f Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 13 Nov 2023 18:34:28 -0800 Subject: [PATCH 22/24] style --- .../cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py | 2 +- python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py index b139df44636..13c9c90c7c2 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/mg/test_mg_cugraph_store.py @@ -120,7 +120,7 @@ def test_get_edge_index(graph, edge_index_type, dask_client): G[et][0] = dask_cudf.from_cudf(cudf.Series(G[et][0]), npartitions=1) G[et][1] = dask_cudf.from_cudf(cudf.Series(G[et][1]), npartitions=1) - cugraph_store = CuGraphStore(F, G, N, order='CSC', multi_gpu=True) + cugraph_store = CuGraphStore(F, G, N, order="CSC", multi_gpu=True) for pyg_can_edge_type in G: src, dst = cugraph_store.get_edge_index( diff --git a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py index 982140984c2..b39ebad8254 100644 --- a/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/tests/test_cugraph_store.py @@ -113,7 +113,7 @@ def test_get_edge_index(graph, edge_index_type): G[et][0] = cudf.Series(G[et][0]) G[et][1] = cudf.Series(G[et][1]) - cugraph_store = CuGraphStore(F, G, N, order='CSC') + cugraph_store = CuGraphStore(F, G, N, order="CSC") for pyg_can_edge_type in G: src, dst = cugraph_store.get_edge_index( From b9a9af38011195d411c614cbdee7cbeee4ebbb6a Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Thu, 16 Nov 2023 08:47:43 -0800 Subject: [PATCH 23/24] revert meta changes --- conda/recipes/libcugraph/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index ecfabcef390..66f72e6b6b5 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -77,7 +77,6 @@ requirements: - librmm ={{ minor_version }} - nccl {{ nccl_version }} - ucx-proc=*=gpu - - ucx >=1.14.1,<1.16.0 outputs: - name: libcugraph From adc94c50535f60e5b1e1b19650aff455004c0973 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 17 Nov 2023 11:18:39 -0800 Subject: [PATCH 24/24] remove unncessary argument from function --- python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py index b208e080de6..14dc5d84f90 100644 --- a/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py +++ b/python/cugraph-pyg/cugraph_pyg/data/cugraph_store.py @@ -368,9 +368,7 @@ def __infer_offsets( } ) - def __dask_array_from_numpy( - self, array: np.ndarray, client: distributed.client.Client, npartitions: int - ): + def __dask_array_from_numpy(self, array: np.ndarray, npartitions: int): return dar.from_array( array, meta=np.array([], dtype=array.dtype), @@ -481,13 +479,13 @@ def __construct_graph( nworkers = len(client.scheduler_info()["workers"]) npartitions = nworkers * 4 - src_dar = self.__dask_array_from_numpy(na_src, client, npartitions) + src_dar = self.__dask_array_from_numpy(na_src, npartitions) del na_src - dst_dar = self.__dask_array_from_numpy(na_dst, client, npartitions) + dst_dar = self.__dask_array_from_numpy(na_dst, npartitions) del na_dst - etp_dar = self.__dask_array_from_numpy(na_etp, client, npartitions) + etp_dar = self.__dask_array_from_numpy(na_etp, npartitions) del na_etp df = dd.from_dask_array(etp_dar, columns=["etp"])