From c43dc1b5b0cb4d0bf6166822d579c490b76015c1 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 31 May 2022 17:07:11 +0200 Subject: [PATCH 01/50] new setup and dependency configuration --- requirements/common.txt | 21 +++++++++++++++++++++ requirements/dev.txt | 4 ++++ requirements/prod.txt | 4 ++++ setup.py | 14 ++++++++++++-- 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 requirements/common.txt create mode 100644 requirements/dev.txt create mode 100644 requirements/prod.txt diff --git a/requirements/common.txt b/requirements/common.txt new file mode 100644 index 000000000..66f4444c0 --- /dev/null +++ b/requirements/common.txt @@ -0,0 +1,21 @@ +# Contains requirements common to all environments +cx_Freeze==6.10 +Cython==0.29.30 +dill==0.3.4 +dl==0.1.0 +docutils==0.17.1 +matplotlib==3.5.0 +numpy>=1.21.2 +ordereddict==1.1 +pandas>=1.1.0 +py2exe==0.11.1.0 +pytest==6.2.5 +pytorch_lightning==1.5.9 +scikit_learn==1.1.1 +scipy==1.7.1 +SQLAlchemy==1.4.27 +timer==0.2.2 +torch==1.9.1 +torch_geometric==2.0.1 +torch_scatter==2.0.9 +tqdm==4.62.3 \ No newline at end of file diff --git a/requirements/dev.txt b/requirements/dev.txt new file mode 100644 index 000000000..b4f0f18ae --- /dev/null +++ b/requirements/dev.txt @@ -0,0 +1,4 @@ +# Specifies only development-specific requirements +# This includes tasks such as code development or training. +# But imports the common ones too +-r common.txt \ No newline at end of file diff --git a/requirements/prod.txt b/requirements/prod.txt new file mode 100644 index 000000000..5f3a7a688 --- /dev/null +++ b/requirements/prod.txt @@ -0,0 +1,4 @@ +# Specifies only production-specific requirements +# This includes tasks such as running inference. +# But imports the common ones too +-r common.txt \ No newline at end of file diff --git a/setup.py b/setup.py index a1b7ca182..b842a8d08 100644 --- a/setup.py +++ b/setup.py @@ -1,21 +1,28 @@ +try: + from setuptools import setup, find_packages +except ImportError: + from distutils.core import setup + import subprocess import sys -from setuptools import setup, find_packages import versioneer - # Utility method(s) def install(package): subprocess.check_call( [sys.executable, "-m", "pip", "install", "--user", package] ) +with open('requirements.txt') as dependencies_file: + dependencies = dependencies_file.read() # Requirements definitions SETUP_REQUIRES = [ "setuptools == 59.5.0", ] +INSTALL_REQUIRES = [dependencies] + INSTALL_REQUIRES = [ "sqlalchemy", "pandas>=1.1.0", @@ -46,6 +53,9 @@ def install(package): "sphinx_rtd_theme", "versioneer", ], + "build": [ + "pytest", + ] } # Ensure pytorch is already installed (see e.g. From 2abfff2936bd7777dcbf03879364f051451e144c Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 31 May 2022 17:17:46 +0200 Subject: [PATCH 02/50] added missing requirement --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..27d2a4cb3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# install packages in the category from setup.py +-r requirements/prod.txt +-e requirements/dev.txt \ No newline at end of file From d0fe4bc011c793037a37c14359d3748929cfee99 Mon Sep 17 00:00:00 2001 From: Morten Date: Thu, 2 Jun 2022 08:37:22 +0200 Subject: [PATCH 03/50] Tried different approaches --- require.txt | 21 ++++++++++ requirements.txt | 25 +++++++++++- requirements/common.txt | 8 ++-- requirements/dev.txt | 2 +- requirements_standalone.txt | 26 ++++++++++++ setup.py | 79 ++++++++++++++++++++++++++++++++----- 6 files changed, 145 insertions(+), 16 deletions(-) create mode 100644 require.txt create mode 100644 requirements_standalone.txt diff --git a/require.txt b/require.txt new file mode 100644 index 000000000..baeea905c --- /dev/null +++ b/require.txt @@ -0,0 +1,21 @@ +cx_Freeze~=6.10 +Cython~=0.29.30 +dill~=0.3.4 +dl~=0.1.0 +docutils~=0.17.1 +matplotlib~=3.5.0 +ordereddict~=1.1 +pandas~=1.1.0 +py2exe~=0.11.1.0 +pytest~=6.2.5 +scikit_learn~=1.1.1 +scipy~=1.7.1 +SQLAlchemy~=1.4.27 +timer~=0.2.2 +tqdm~=4.62.3 +numpy~=1.21.2 + +torch==1.10.1+cu113 +pytorch_lightning~=1.5.9 +torch_scatter~=2.0.9 +torch_geometric~=2.0.1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 27d2a4cb3..a7945ab1f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,24 @@ # install packages in the category from setup.py --r requirements/prod.txt --e requirements/dev.txt \ No newline at end of file +#-r requirements/prod.txt +#-e requirements/dev.txt +# Contains requirements common to all environments +cx_Freeze~=6.10 +Cython~=0.29.30 +dill~=0.3.4 +dl~=0.1.0 +docutils~=0.17.1 +matplotlib~=3.5.0 +ordereddict~=1.1 +pandas~=1.1.0 +py2exe~=0.11.1.0 +pytest~=6.2.5 +scikit_learn~=1.1.1 +scipy~=1.7.1 +SQLAlchemy~=1.4.27 +timer~=0.2.2 +tqdm~=4.62.3 +numpy~=1.21.2 +https://download.pytorch.org/whl/torch_stable.html#egg=torch==1.10.1+cu113 +pytorch_lightning~=1.5.9 +https://data.pyg.org/whl/torch-1.10.0+cu113.html#egg=torch_scatter~=2.0.9 +torch_geometric~=2.0.1 \ No newline at end of file diff --git a/requirements/common.txt b/requirements/common.txt index 66f4444c0..d041e0231 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -10,12 +10,14 @@ ordereddict==1.1 pandas>=1.1.0 py2exe==0.11.1.0 pytest==6.2.5 -pytorch_lightning==1.5.9 scikit_learn==1.1.1 scipy==1.7.1 SQLAlchemy==1.4.27 timer==0.2.2 torch==1.9.1 +tqdm==4.62.3 +--find-links https://download.pytorch.org/whl/torch_stable.html +torch==1.10.1+cu111 +pytorch_lightning==1.5.9 torch_geometric==2.0.1 -torch_scatter==2.0.9 -tqdm==4.62.3 \ No newline at end of file +torch_scatter==2.0.9 \ No newline at end of file diff --git a/requirements/dev.txt b/requirements/dev.txt index b4f0f18ae..05b6aa4c1 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,4 +1,4 @@ # Specifies only development-specific requirements # This includes tasks such as code development or training. # But imports the common ones too --r common.txt \ No newline at end of file +-r common.txt diff --git a/requirements_standalone.txt b/requirements_standalone.txt new file mode 100644 index 000000000..532409226 --- /dev/null +++ b/requirements_standalone.txt @@ -0,0 +1,26 @@ +# install packages in the category from setup.py +#-r requirements/prod.txt +#-e requirements/dev.txt +# Contains requirements common to all environments +cx_Freeze~=6.10 +Cython~=0.29.30 +dill~=0.3.4 +dl~=0.1.0 +docutils~=0.17.1 +matplotlib~=3.5.0 +ordereddict~=1.1 +pandas~=1.1.0 +py2exe~=0.11.1.0 +pytest~=6.2.5 +scikit_learn~=1.1.1 +scipy~=1.7.1 +SQLAlchemy~=1.4.27 +timer~=0.2.2 +tqdm~=4.62.3 +numpy~=1.21.2 +--find-links https://download.pytorch.org/whl/torch_stable.html +torch==1.10.1+cu113 +pytorch_lightning~=1.5.9 +--find-links https://data.pyg.org/whl/torch-1.10.0+cu113.html +torch_scatter~=2.0.9 +torch_geometric~=2.0.1 \ No newline at end of file diff --git a/setup.py b/setup.py index b842a8d08..74432fff8 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,11 @@ +# inspiration: https://github.com/PyTorchLightning/pytorch-lightning try: from setuptools import setup, find_packages except ImportError: from distutils.core import setup import subprocess +import platform import sys import versioneer @@ -13,16 +15,39 @@ def install(package): [sys.executable, "-m", "pip", "install", "--user", package] ) +# from https://github.com/PyTorchLightning/pytorch-lightning +def _load_requirements(path_dir: str , file_name: str = 'requirements.txt', comment_char: str = '#') -> List[str]: + """Load requirements from a file + >>> _load_requirements(PROJECT_ROOT) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + ['numpy...', 'torch...', ...] + """ + with open(os.path.join(path_dir, file_name), 'r') as file: + lines = [ln.strip() for ln in file.readlines()] + reqs = [] + for ln in lines: + # filer all comments + if comment_char in ln: + ln = ln[:ln.index(comment_char)].strip() + # skip directly installed dependencies + if ln.startswith('http'): + continue + if ln: # if requirement is not empty + reqs.append(ln) + return reqs + +#def install_requirement(requirement): +# subprocess.check_call( +# [sys.executable, "-m", "pip", "install", "--user", "-r", requirement] +# ) + with open('requirements.txt') as dependencies_file: - dependencies = dependencies_file.read() + dependencies = [x.rstrip("\n") for x in dependencies_file.readlines()] # Requirements definitions SETUP_REQUIRES = [ "setuptools == 59.5.0", ] -INSTALL_REQUIRES = [dependencies] - INSTALL_REQUIRES = [ "sqlalchemy", "pandas>=1.1.0", @@ -40,6 +65,10 @@ def install(package): "matplotlib", ] +with open('require.txt') as dependencies_file: + depend = [x.rstrip("\n") for x in dependencies_file.readlines()] +INSTALL_REQUIRES = [depend] + EXTRAS_REQUIRE = { "develop": [ "black", @@ -58,12 +87,38 @@ def install(package): ] } +# https://pypi.org/classifiers/ +CLASSIFIER = { + "Development Status :: 3 - Alpha", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Environment :: CPU", + "Environment :: GPU", +} # Ensure pytorch is already installed (see e.g. # https://github.com/pyg-team/pytorch_geometric/issues/861#issuecomment-566424944) -try: - import torch # pyright: reportMissingImports=false -except ImportError: - install("torch==1.9.0") +#try: +# import torch # pyright: reportMissingImports=false +#except ImportError: +# install("torch==1.10.1") + +# Installs individual packages, does not '--find_links' into account +#for req in dependencies: +# if not req.startswith("#"): +# try: +# import req +# except ImportError: +# install(str(req)) + +# directly install requirements.txt file +#try: +# # try to import the needed packages +# for req in dependencies: +# if not req.startswith("#"): +# import req +# if any package is missing install the requirements.txt file +#except ImportError: +# install_requirement("requirements.txt") setup( name="graphnet", @@ -79,9 +134,13 @@ def install(package): packages=find_packages(where="src"), package_dir={"": "src"}, setup_requires=SETUP_REQUIRES, - install_requires=INSTALL_REQUIRES, + #install_requires=INSTALL_REQUIRES, # requires a list without --find_links + install_requires=_load_requirements(PATH_ROOT), extras_require=EXTRAS_REQUIRE, + #classifier=CLASSIFIER, dependency_links=[ - "https://data.pyg.org/whl/torch-1.9.0+cpu.html", - ], + "https://download.pytorch.org/whl/torch_stable.html#egg=torch==1.10.1+cu113" + "https://data.pyg.org/whl/torch-1.10.0+cu113.html#egg=torch_scatter~=2.0.9", + ], # maybe needs an additonal flag 'pip install -e .[develop] --allow-all-external' ) + From ab1fd7b1bb3bc2f9e5fcd54f38ad4b97b4aa5897 Mon Sep 17 00:00:00 2001 From: Morten Date: Thu, 2 Jun 2022 10:00:57 +0200 Subject: [PATCH 04/50] added missing package --- .gitignore | 3 ++- examples/train_model.py | 12 ++++++------ requirements_standalone.txt | 4 +++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 0a034f79b..d9a9ba735 100644 --- a/.gitignore +++ b/.gitignore @@ -138,4 +138,5 @@ lightning_logs/ wandb/ **.png **.pth -**.ipynb \ No newline at end of file +**.ipynb +datasets/dev_lvl7_robustness_muon_neutrino_0000.db diff --git a/examples/train_model.py b/examples/train_model.py index 916ec3e7a..f11c2a890 100644 --- a/examples/train_model.py +++ b/examples/train_model.py @@ -33,7 +33,7 @@ project="example-script", entity="graphnet-team", save_dir="./wandb/", - log_model=True, + log_model=False, ) @@ -47,12 +47,12 @@ def main(): config = { "db": "/groups/icecube/asogaard/data/sqlite/dev_lvl7_robustness_muon_neutrino_0000/data/dev_lvl7_robustness_muon_neutrino_0000.db", "pulsemap": "SRTTWOfflinePulsesDC", - "batch_size": 512, - "num_workers": 10, - "gpus": [1], + "batch_size": 256, + "num_workers": 2, + "gpus": [], "target": "energy", - "n_epochs": 5, - "patience": 5, + "n_epochs": 1, + "patience": 1, } archive = "/groups/icecube/asogaard/gnn/results/" run_name = "dynedge_{}_example".format(config["target"]) diff --git a/requirements_standalone.txt b/requirements_standalone.txt index 532409226..18c35fc3d 100644 --- a/requirements_standalone.txt +++ b/requirements_standalone.txt @@ -23,4 +23,6 @@ torch==1.10.1+cu113 pytorch_lightning~=1.5.9 --find-links https://data.pyg.org/whl/torch-1.10.0+cu113.html torch_scatter~=2.0.9 -torch_geometric~=2.0.1 \ No newline at end of file +torch_geometric~=2.0.1 +torch-spline-conv +torch-cluster \ No newline at end of file From e64df54763f79d0ba253f741ae48ba163b37c522 Mon Sep 17 00:00:00 2001 From: Morten Date: Thu, 2 Jun 2022 12:38:30 +0200 Subject: [PATCH 05/50] working requirements and setup file --- .gitignore | 2 ++ examples/train_model.py | 4 ++-- requirements_standalone.txt | 10 ++++---- setup.py | 46 ++++++++++++++++++------------------- 4 files changed, 33 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index d9a9ba735..24b9e5103 100644 --- a/.gitignore +++ b/.gitignore @@ -140,3 +140,5 @@ wandb/ **.pth **.ipynb datasets/dev_lvl7_robustness_muon_neutrino_0000.db +examples/train_model.py +examples/train_model.py diff --git a/examples/train_model.py b/examples/train_model.py index f11c2a890..f46e7b85d 100644 --- a/examples/train_model.py +++ b/examples/train_model.py @@ -45,7 +45,7 @@ def main(): # Configuration config = { - "db": "/groups/icecube/asogaard/data/sqlite/dev_lvl7_robustness_muon_neutrino_0000/data/dev_lvl7_robustness_muon_neutrino_0000.db", + "db": "\graphnet\datasets\dev_lvl7_robustness_muon_neutrino_0000.db", "pulsemap": "SRTTWOfflinePulsesDC", "batch_size": 256, "num_workers": 2, @@ -54,7 +54,7 @@ def main(): "n_epochs": 1, "patience": 1, } - archive = "/groups/icecube/asogaard/gnn/results/" + archive = "\Graphnet\graphnet_user" run_name = "dynedge_{}_example".format(config["target"]) # Log configuration to W&B diff --git a/requirements_standalone.txt b/requirements_standalone.txt index 18c35fc3d..dd607f119 100644 --- a/requirements_standalone.txt +++ b/requirements_standalone.txt @@ -18,11 +18,13 @@ SQLAlchemy~=1.4.27 timer~=0.2.2 tqdm~=4.62.3 numpy~=1.21.2 +wandb --find-links https://download.pytorch.org/whl/torch_stable.html torch==1.10.1+cu113 pytorch_lightning~=1.5.9 --find-links https://data.pyg.org/whl/torch-1.10.0+cu113.html -torch_scatter~=2.0.9 -torch_geometric~=2.0.1 -torch-spline-conv -torch-cluster \ No newline at end of file +torch-cluster==1.5.9 +torch-sparse==0.6.12 +torch-spline-conv==1.2.1 +torch_scatter==2.0.9 +torch_geometric==2.0.1 \ No newline at end of file diff --git a/setup.py b/setup.py index 74432fff8..3a9cd2af5 100644 --- a/setup.py +++ b/setup.py @@ -16,24 +16,24 @@ def install(package): ) # from https://github.com/PyTorchLightning/pytorch-lightning -def _load_requirements(path_dir: str , file_name: str = 'requirements.txt', comment_char: str = '#') -> List[str]: - """Load requirements from a file - >>> _load_requirements(PROJECT_ROOT) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE - ['numpy...', 'torch...', ...] - """ - with open(os.path.join(path_dir, file_name), 'r') as file: - lines = [ln.strip() for ln in file.readlines()] - reqs = [] - for ln in lines: - # filer all comments - if comment_char in ln: - ln = ln[:ln.index(comment_char)].strip() - # skip directly installed dependencies - if ln.startswith('http'): - continue - if ln: # if requirement is not empty - reqs.append(ln) - return reqs +#def _load_requirements(path_dir: str , file_name: str = 'requirements.txt', comment_char: str = '#') -> List[str]: +# """Load requirements from a file +# >>> _load_requirements(PROJECT_ROOT) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE +# ['numpy...', 'torch...', ...] +# """ +# with open(os.path.join(path_dir, file_name), 'r') as file: +# lines = [ln.strip() for ln in file.readlines()] +# reqs = [] +# for ln in lines: +# # filer all comments +# if comment_char in ln: +# ln = ln[:ln.index(comment_char)].strip() +# # skip directly installed dependencies +# if ln.startswith('http'): +# continue +# if ln: # if requirement is not empty +# reqs.append(ln) +# return reqs #def install_requirement(requirement): # subprocess.check_call( @@ -65,9 +65,9 @@ def _load_requirements(path_dir: str , file_name: str = 'requirements.txt', comm "matplotlib", ] -with open('require.txt') as dependencies_file: - depend = [x.rstrip("\n") for x in dependencies_file.readlines()] -INSTALL_REQUIRES = [depend] +#with open('require.txt') as dependencies_file: +# depend = [x.rstrip("\n") for x in dependencies_file.readlines()] +#INSTALL_REQUIRES = [depend] EXTRAS_REQUIRE = { "develop": [ @@ -134,8 +134,8 @@ def _load_requirements(path_dir: str , file_name: str = 'requirements.txt', comm packages=find_packages(where="src"), package_dir={"": "src"}, setup_requires=SETUP_REQUIRES, - #install_requires=INSTALL_REQUIRES, # requires a list without --find_links - install_requires=_load_requirements(PATH_ROOT), + install_requires=INSTALL_REQUIRES, # requires a list without --find_links + #install_requires=_load_requirements(PATH_ROOT), extras_require=EXTRAS_REQUIRE, #classifier=CLASSIFIER, dependency_links=[ From eaa7ee20f274f9f374e0574c71ad98a07d1ce1b3 Mon Sep 17 00:00:00 2001 From: Morten Date: Thu, 2 Jun 2022 14:21:45 +0200 Subject: [PATCH 06/50] cleaned code --- README.md | 2 +- require.txt | 21 ------- requirements.txt | 44 +++++++-------- requirements/common.txt | 23 -------- requirements/dev.txt | 4 -- requirements/prod.txt | 4 -- requirements_standalone.txt | 30 ---------- setup.py | 107 +++++++----------------------------- 8 files changed, 41 insertions(+), 194 deletions(-) delete mode 100644 require.txt delete mode 100644 requirements/common.txt delete mode 100644 requirements/dev.txt delete mode 100644 requirements/prod.txt delete mode 100644 requirements_standalone.txt diff --git a/README.md b/README.md index 9a8289ebd..82e0b2967 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ $ git clone git@github.com:/graphnet.git $ cd graphnet $ conda env create -f envs/gnn_py38.yml $ conda activate gnn_py38 -(gnn_py38) $ pip install -e .[develop] +(gnn_py38) $ pip install -r requirements.txt -e .[develop] ``` This should allow you to e.g. run the scripts in [examples/](./examples/) out of the box. diff --git a/require.txt b/require.txt deleted file mode 100644 index baeea905c..000000000 --- a/require.txt +++ /dev/null @@ -1,21 +0,0 @@ -cx_Freeze~=6.10 -Cython~=0.29.30 -dill~=0.3.4 -dl~=0.1.0 -docutils~=0.17.1 -matplotlib~=3.5.0 -ordereddict~=1.1 -pandas~=1.1.0 -py2exe~=0.11.1.0 -pytest~=6.2.5 -scikit_learn~=1.1.1 -scipy~=1.7.1 -SQLAlchemy~=1.4.27 -timer~=0.2.2 -tqdm~=4.62.3 -numpy~=1.21.2 - -torch==1.10.1+cu113 -pytorch_lightning~=1.5.9 -torch_scatter~=2.0.9 -torch_geometric~=2.0.1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a7945ab1f..6fe75c6ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,24 +1,22 @@ -# install packages in the category from setup.py -#-r requirements/prod.txt -#-e requirements/dev.txt -# Contains requirements common to all environments -cx_Freeze~=6.10 -Cython~=0.29.30 -dill~=0.3.4 -dl~=0.1.0 -docutils~=0.17.1 -matplotlib~=3.5.0 -ordereddict~=1.1 -pandas~=1.1.0 -py2exe~=0.11.1.0 -pytest~=6.2.5 +# Contains packages recommended for functional performance +setuptools>=62.3 +docutils>=0.18 +SQLAlchemy>=1.4.37 +pandas>=1.4 +numpy>=1.22 +timer>=0.2 +tqdm>=4.64 +dill>=0.3 +wandb>=0.12 +matplotlib>=3.5 scikit_learn~=1.1.1 -scipy~=1.7.1 -SQLAlchemy~=1.4.27 -timer~=0.2.2 -tqdm~=4.62.3 -numpy~=1.21.2 -https://download.pytorch.org/whl/torch_stable.html#egg=torch==1.10.1+cu113 -pytorch_lightning~=1.5.9 -https://data.pyg.org/whl/torch-1.10.0+cu113.html#egg=torch_scatter~=2.0.9 -torch_geometric~=2.0.1 \ No newline at end of file +scipy>=1.8 +--find-links https://download.pytorch.org/whl/torch_stable.html +torch==1.11+cu115 +--find-links https://data.pyg.org/whl/torch-1.10.0+cu113.html +torch-cluster==1.6.0 +torch_scatter==2.0.9 +torch-sparse==0.6.13 +torch-spline-conv==1.2.1 +torch_geometric==2.0.4 +pytorch_lightning~=1.6.0 diff --git a/requirements/common.txt b/requirements/common.txt deleted file mode 100644 index d041e0231..000000000 --- a/requirements/common.txt +++ /dev/null @@ -1,23 +0,0 @@ -# Contains requirements common to all environments -cx_Freeze==6.10 -Cython==0.29.30 -dill==0.3.4 -dl==0.1.0 -docutils==0.17.1 -matplotlib==3.5.0 -numpy>=1.21.2 -ordereddict==1.1 -pandas>=1.1.0 -py2exe==0.11.1.0 -pytest==6.2.5 -scikit_learn==1.1.1 -scipy==1.7.1 -SQLAlchemy==1.4.27 -timer==0.2.2 -torch==1.9.1 -tqdm==4.62.3 ---find-links https://download.pytorch.org/whl/torch_stable.html -torch==1.10.1+cu111 -pytorch_lightning==1.5.9 -torch_geometric==2.0.1 -torch_scatter==2.0.9 \ No newline at end of file diff --git a/requirements/dev.txt b/requirements/dev.txt deleted file mode 100644 index 05b6aa4c1..000000000 --- a/requirements/dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Specifies only development-specific requirements -# This includes tasks such as code development or training. -# But imports the common ones too --r common.txt diff --git a/requirements/prod.txt b/requirements/prod.txt deleted file mode 100644 index 5f3a7a688..000000000 --- a/requirements/prod.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Specifies only production-specific requirements -# This includes tasks such as running inference. -# But imports the common ones too --r common.txt \ No newline at end of file diff --git a/requirements_standalone.txt b/requirements_standalone.txt deleted file mode 100644 index dd607f119..000000000 --- a/requirements_standalone.txt +++ /dev/null @@ -1,30 +0,0 @@ -# install packages in the category from setup.py -#-r requirements/prod.txt -#-e requirements/dev.txt -# Contains requirements common to all environments -cx_Freeze~=6.10 -Cython~=0.29.30 -dill~=0.3.4 -dl~=0.1.0 -docutils~=0.17.1 -matplotlib~=3.5.0 -ordereddict~=1.1 -pandas~=1.1.0 -py2exe~=0.11.1.0 -pytest~=6.2.5 -scikit_learn~=1.1.1 -scipy~=1.7.1 -SQLAlchemy~=1.4.27 -timer~=0.2.2 -tqdm~=4.62.3 -numpy~=1.21.2 -wandb ---find-links https://download.pytorch.org/whl/torch_stable.html -torch==1.10.1+cu113 -pytorch_lightning~=1.5.9 ---find-links https://data.pyg.org/whl/torch-1.10.0+cu113.html -torch-cluster==1.5.9 -torch-sparse==0.6.12 -torch-spline-conv==1.2.1 -torch_scatter==2.0.9 -torch_geometric==2.0.1 \ No newline at end of file diff --git a/setup.py b/setup.py index 3a9cd2af5..d343efe3b 100644 --- a/setup.py +++ b/setup.py @@ -1,74 +1,37 @@ -# inspiration: https://github.com/PyTorchLightning/pytorch-lightning try: from setuptools import setup, find_packages except ImportError: from distutils.core import setup import subprocess -import platform import sys import versioneer -# Utility method(s) -def install(package): - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "--user", package] - ) - -# from https://github.com/PyTorchLightning/pytorch-lightning -#def _load_requirements(path_dir: str , file_name: str = 'requirements.txt', comment_char: str = '#') -> List[str]: -# """Load requirements from a file -# >>> _load_requirements(PROJECT_ROOT) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE -# ['numpy...', 'torch...', ...] -# """ -# with open(os.path.join(path_dir, file_name), 'r') as file: -# lines = [ln.strip() for ln in file.readlines()] -# reqs = [] -# for ln in lines: -# # filer all comments -# if comment_char in ln: -# ln = ln[:ln.index(comment_char)].strip() -# # skip directly installed dependencies -# if ln.startswith('http'): -# continue -# if ln: # if requirement is not empty -# reqs.append(ln) -# return reqs - -#def install_requirement(requirement): -# subprocess.check_call( -# [sys.executable, "-m", "pip", "install", "--user", "-r", requirement] -# ) - -with open('requirements.txt') as dependencies_file: - dependencies = [x.rstrip("\n") for x in dependencies_file.readlines()] - # Requirements definitions SETUP_REQUIRES = [ - "setuptools == 59.5.0", + "setuptools==62.3", ] INSTALL_REQUIRES = [ - "sqlalchemy", - "pandas>=1.1.0", - "numpy", - "timer", - "tqdm", - "torch-cluster==1.5.9", + "sqlalchemy>=1.4.37", + "pandas>=1.4", + "numpy>=1.22", + "timer>=0.2", + "tqdm>=4.64", + "dill>=0.3", + "wandb>=0.12", + "matplotlib>=3.5", + "scikit_learn~=1.1.1", + "scipy>=1.8", + "torch~=1.11", + "torch-cluster==1.6.0", "torch-scatter==2.0.9", - "torch-sparse==0.6.12", + "torch-sparse==0.6.13", "torch-spline-conv==1.2.1", - "torch-geometric==2.0.1", - "pytorch-lightning==1.5.6", - "dill", - "wandb", - "matplotlib", + "torch-geometric==2.0.4", + "pytorch-lightning>=1.6.0", ] -#with open('require.txt') as dependencies_file: -# depend = [x.rstrip("\n") for x in dependencies_file.readlines()] -#INSTALL_REQUIRES = [depend] - EXTRAS_REQUIRE = { "develop": [ "black", @@ -81,44 +44,18 @@ def install(package): "sphinx", "sphinx_rtd_theme", "versioneer", - ], - "build": [ - "pytest", ] } # https://pypi.org/classifiers/ CLASSIFIER = { "Development Status :: 3 - Alpha", + "Programming Language :: Python :: 3.7" "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Environment :: CPU", "Environment :: GPU", } -# Ensure pytorch is already installed (see e.g. -# https://github.com/pyg-team/pytorch_geometric/issues/861#issuecomment-566424944) -#try: -# import torch # pyright: reportMissingImports=false -#except ImportError: -# install("torch==1.10.1") - -# Installs individual packages, does not '--find_links' into account -#for req in dependencies: -# if not req.startswith("#"): -# try: -# import req -# except ImportError: -# install(str(req)) - -# directly install requirements.txt file -#try: -# # try to import the needed packages -# for req in dependencies: -# if not req.startswith("#"): -# import req -# if any package is missing install the requirements.txt file -#except ImportError: -# install_requirement("requirements.txt") setup( name="graphnet", @@ -134,13 +71,7 @@ def install(package): packages=find_packages(where="src"), package_dir={"": "src"}, setup_requires=SETUP_REQUIRES, - install_requires=INSTALL_REQUIRES, # requires a list without --find_links - #install_requires=_load_requirements(PATH_ROOT), + install_requires=INSTALL_REQUIRES, extras_require=EXTRAS_REQUIRE, - #classifier=CLASSIFIER, - dependency_links=[ - "https://download.pytorch.org/whl/torch_stable.html#egg=torch==1.10.1+cu113" - "https://data.pyg.org/whl/torch-1.10.0+cu113.html#egg=torch_scatter~=2.0.9", - ], # maybe needs an additonal flag 'pip install -e .[develop] --allow-all-external' + classifier=CLASSIFIER, ) - From 97e095c3972c67929df252003d0e43ceda26f7b7 Mon Sep 17 00:00:00 2001 From: Morten Date: Thu, 2 Jun 2022 14:38:05 +0200 Subject: [PATCH 07/50] bugfix link in requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6fe75c6ff..7339ab2ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ scikit_learn~=1.1.1 scipy>=1.8 --find-links https://download.pytorch.org/whl/torch_stable.html torch==1.11+cu115 ---find-links https://data.pyg.org/whl/torch-1.10.0+cu113.html +--find-links https://data.pyg.org/whl/torch-1.11.0+cu115.html torch-cluster==1.6.0 torch_scatter==2.0.9 torch-sparse==0.6.13 From c54fc64931a6db35e8a65a888c067abad3fb64cc Mon Sep 17 00:00:00 2001 From: Morten Date: Wed, 15 Jun 2022 10:25:13 +0200 Subject: [PATCH 08/50] version conflict and readme modifications --- .github/actions/install-cpu/action.yml | 10 +++++----- README.md | 4 ++-- requirements.txt | 4 ++-- setup.py | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/actions/install-cpu/action.yml b/.github/actions/install-cpu/action.yml index e295ec948..faf6f6d84 100644 --- a/.github/actions/install-cpu/action.yml +++ b/.github/actions/install-cpu/action.yml @@ -5,11 +5,11 @@ inputs: torch-verison: description: 'Version of pytorch to install' required: true - default: '1.9.0' + default: '1.11.0' pyg-verison: description: 'Version of pytorch-geometric to install' required: true - default: '2.0.1' + default: '2.0.4' editable: description: "Whether to install graphnet as editable" required: true @@ -25,11 +25,11 @@ runs: - name: "Install dependencies" run: | pip install --upgrade pip - pip install wheel setuptools==59.5.0 + pip install wheel setuptools>=62.3.0 pip install torch==${{ inputs.torch-verison }}+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install torch-cluster==1.5.9 -f https://data.pyg.org/whl/torch-${{ inputs.torch-verison }}+cpu.html + pip install torch-cluster==1.6.0 -f https://data.pyg.org/whl/torch-${{ inputs.torch-verison }}+cpu.html pip install torch-scatter==2.0.9 -f https://data.pyg.org/whl/torch-${{ inputs.torch-verison }}+cpu.html - pip install torch-sparse==0.6.12 -f https://data.pyg.org/whl/torch-${{ inputs.torch-verison }}+cpu.html + pip install torch-sparse==0.6.13 -f https://data.pyg.org/whl/torch-${{ inputs.torch-verison }}+cpu.html pip install torch-spline-conv==1.2.1 -f https://data.pyg.org/whl/torch-${{ inputs.torch-verison }}+cpu.html pip install torch-geometric==${{ inputs.pyg-verison }} shell: bash diff --git a/README.md b/README.md index 82e0b2967..12e26c18a 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,11 @@ ## :gear: Install -We recommend installing `graphnet` in a separate environment, e.g. using Anaconda (see details on installation [here](https://www.anaconda.com/products/individual)). The fastest way to get up and running is to install the package in the provided conda environment: +We recommend installing `graphnet` in a separate environment, e.g. using Anaconda (see details on installation [here](https://www.anaconda.com/products/individual)). The fastest way to get up and running is to install the package in the provided conda environment, specifying the python version {>3.8} and ensuring that the gcc compilers are up to date: ```bash $ git clone git@github.com:/graphnet.git $ cd graphnet -$ conda env create -f envs/gnn_py38.yml +$ conda create --name gnn_py38 python=3.xx gcc_linux-64 gxx_linux-64 libgcc -y $ conda activate gnn_py38 (gnn_py38) $ pip install -r requirements.txt -e .[develop] ``` diff --git a/requirements.txt b/requirements.txt index 7339ab2ba..d882e3baa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Contains packages recommended for functional performance -setuptools>=62.3 +setuptools>=62.3.0 docutils>=0.18 SQLAlchemy>=1.4.37 pandas>=1.4 @@ -19,4 +19,4 @@ torch_scatter==2.0.9 torch-sparse==0.6.13 torch-spline-conv==1.2.1 torch_geometric==2.0.4 -pytorch_lightning~=1.6.0 +pytorch_lightning~=1.6.0 \ No newline at end of file diff --git a/setup.py b/setup.py index d343efe3b..e180c725a 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ # Requirements definitions SETUP_REQUIRES = [ - "setuptools==62.3", + "setuptools>=62.3.0", ] INSTALL_REQUIRES = [ @@ -50,9 +50,9 @@ # https://pypi.org/classifiers/ CLASSIFIER = { "Development Status :: 3 - Alpha", - "Programming Language :: Python :: 3.7" "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Environment :: CPU", "Environment :: GPU", } From 4633ac537135ae98cb6f5e95cb6195d1f04c7338 Mon Sep 17 00:00:00 2001 From: Morten Date: Wed, 15 Jun 2022 10:45:26 +0200 Subject: [PATCH 09/50] reverted import statements to original --- setup.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/setup.py b/setup.py index e180c725a..7bcaa9276 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,6 @@ -try: - from setuptools import setup, find_packages -except ImportError: - from distutils.core import setup - import subprocess import sys +from setuptools import setup, find_packages import versioneer # Requirements definitions From 90929aed0730acaf3810baa71ad836b829ed0311 Mon Sep 17 00:00:00 2001 From: Morten Date: Wed, 15 Jun 2022 11:10:31 +0200 Subject: [PATCH 10/50] IceTray install description --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index 12e26c18a..f84c24f33 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,30 @@ This should allow you to e.g. run the scripts in [examples/](./examples/) out of You can also install the package in a python virtual environment, or in your system python, but then you will have to contend with C++ compiler versions; the non-standard interplay between [pytorch](https://pytorch.org/) and [pytorch-geometric](https://pytorch-geometric.readthedocs.io/en/latest/) (see e.g. [here](https://github.com/pyg-team/pytorch_geometric/issues/861#issuecomment-566424944)), which `graphnet` uses internally; etc. +## :gear: Database creation and Installation on IceTray + +Within the IceTray enviroment, the Graphnet module is used to create SQL databases from I3 files, using the provided 'convert_i3_to_sqlite.py' script. The recommended set up is as follows: + +```bash +$ cd graphnet +$ nano cvmfs.sh +``` + +Within the 'cvmfs.sh' copy the following and press 'ctrl+X' +```bash +eval `/cvmfs/icecube.opensciencegrid.org/py3-v4.1.0/setup.sh` +/cvmfs/icecube.opensciencegrid.org/py3-v4.1.0/RHEL_7_x86_64/metaprojects/combo/stable/env-shell.sh +``` + +Next, launch it and install the Graphnet module at a user level. +```bash +$ source cvmfs.sh +$ conda create --name gnn_icetray +$ conda activate gnn_icetray +$ pip install --user -r requirements.txt -e .[develop] +``` + +This should allow you to run 'convert_i3_to_sqlite.py' in [examples/](./examples/) with your preferred I3 files. ## :handshake: Contributing From 1b13bed559eb971187002045ae09b5ccba46b744 Mon Sep 17 00:00:00 2001 From: Morten Date: Wed, 15 Jun 2022 11:17:31 +0200 Subject: [PATCH 11/50] removed python 3.6 --- .github/workflows/build-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-matrix.yml b/.github/workflows/build-matrix.yml index 04a41f6d4..1645ad7aa 100644 --- a/.github/workflows/build-matrix.yml +++ b/.github/workflows/build-matrix.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From 15171627de27666cfd6e03f67264a0ad5b4f7fa6 Mon Sep 17 00:00:00 2001 From: Morten Date: Wed, 15 Jun 2022 12:09:27 +0200 Subject: [PATCH 12/50] version testing --- .github/workflows/build-matrix.yml | 2 +- setup.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-matrix.yml b/.github/workflows/build-matrix.yml index 1645ad7aa..86fb35030 100644 --- a/.github/workflows/build-matrix.yml +++ b/.github/workflows/build-matrix.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/setup.py b/setup.py index 5ca694449..4f5c43719 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,8 @@ # https://pypi.org/classifiers/ CLASSIFIER = { "Development Status :: 3 - Alpha", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", From aca3c347b69c115651c61702c0e4a10abe3b79cd Mon Sep 17 00:00:00 2001 From: Morten Date: Wed, 15 Jun 2022 12:12:10 +0200 Subject: [PATCH 13/50] corrected gitignore based on comment --- .gitignore | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index bd12ed635..4210443c1 100644 --- a/.gitignore +++ b/.gitignore @@ -141,6 +141,4 @@ test_data/ **.png **.pth **.ipynb -datasets/dev_lvl7_robustness_muon_neutrino_0000.db -examples/train_model.py -examples/train_model.py +**.db \ No newline at end of file From 70d168d3bf87e5ddf0b50f8dbd7eba663173d7e0 Mon Sep 17 00:00:00 2001 From: Morten Date: Wed, 15 Jun 2022 12:13:48 +0200 Subject: [PATCH 14/50] removed 3.10 as it registered as 3.1 --- .github/workflows/build-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-matrix.yml b/.github/workflows/build-matrix.yml index 86fb35030..04a41f6d4 100644 --- a/.github/workflows/build-matrix.yml +++ b/.github/workflows/build-matrix.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, 3.10] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From 8b5e3f2608958d8e4b346b78551bc7cb2a9cb888 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 11:32:17 +0200 Subject: [PATCH 15/50] new install action reflecting requirement file --- .github/actions/install-cpu/action.yml | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/.github/actions/install-cpu/action.yml b/.github/actions/install-cpu/action.yml index faf6f6d84..dc0cb2980 100644 --- a/.github/actions/install-cpu/action.yml +++ b/.github/actions/install-cpu/action.yml @@ -1,15 +1,7 @@ name: 'Install graphnet' -description: 'Installs graphnet package, with pytorch and pytorch-geometric for CPU' +description: 'Installs graphnet package' inputs: - torch-verison: - description: 'Version of pytorch to install' - required: true - default: '1.11.0' - pyg-verison: - description: 'Version of pytorch-geometric to install' - required: true - default: '2.0.4' editable: description: "Whether to install graphnet as editable" required: true @@ -22,17 +14,6 @@ inputs: runs: using: "composite" steps: - - name: "Install dependencies" - run: | - pip install --upgrade pip - pip install wheel setuptools>=62.3.0 - pip install torch==${{ inputs.torch-verison }}+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install torch-cluster==1.6.0 -f https://data.pyg.org/whl/torch-${{ inputs.torch-verison }}+cpu.html - pip install torch-scatter==2.0.9 -f https://data.pyg.org/whl/torch-${{ inputs.torch-verison }}+cpu.html - pip install torch-sparse==0.6.13 -f https://data.pyg.org/whl/torch-${{ inputs.torch-verison }}+cpu.html - pip install torch-spline-conv==1.2.1 -f https://data.pyg.org/whl/torch-${{ inputs.torch-verison }}+cpu.html - pip install torch-geometric==${{ inputs.pyg-verison }} - shell: bash - name: Infer installation flags run: | PIP_FLAGS=`[[ ${{ inputs.editable }} =~ (T|t)rue ]] && printf "%s\n" "-e" || echo " "` @@ -40,5 +21,5 @@ runs: shell: bash - name: Install package run: | - pip install ${{ env.PIP_FLAGS }} .${{ inputs.extras }} + pip install ${{ env.PIP_FLAGS }} -r requirements.txt .${{ inputs.extras }} shell: bash From 1b4cedc9b2b122bc2d01e28565c6089f949cb296 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 11:37:47 +0200 Subject: [PATCH 16/50] modifications to actions --- .github/actions/install-cpu/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/install-cpu/action.yml b/.github/actions/install-cpu/action.yml index dc0cb2980..5c460d32b 100644 --- a/.github/actions/install-cpu/action.yml +++ b/.github/actions/install-cpu/action.yml @@ -19,6 +19,10 @@ runs: PIP_FLAGS=`[[ ${{ inputs.editable }} =~ (T|t)rue ]] && printf "%s\n" "-e" || echo " "` echo "PIP_FLAGS=${PIP_FLAGS}" >> $GITHUB_ENV shell: bash + - name: Install dependencies + run: | + pip install --upgrade pip + pip install wheel setuptools>=62.3.0 - name: Install package run: | pip install ${{ env.PIP_FLAGS }} -r requirements.txt .${{ inputs.extras }} From d11b9e3f99996612eb2692a2c720e1bdec2e6dc1 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 11:39:00 +0200 Subject: [PATCH 17/50] fixed bug --- .github/actions/install-cpu/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/install-cpu/action.yml b/.github/actions/install-cpu/action.yml index 5c460d32b..fbfb84757 100644 --- a/.github/actions/install-cpu/action.yml +++ b/.github/actions/install-cpu/action.yml @@ -23,6 +23,7 @@ runs: run: | pip install --upgrade pip pip install wheel setuptools>=62.3.0 + shell: bash - name: Install package run: | pip install ${{ env.PIP_FLAGS }} -r requirements.txt .${{ inputs.extras }} From ff663b97902f26d2ab247f0eb525355bbf510438 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 11:41:00 +0200 Subject: [PATCH 18/50] fixed naming bug --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4f5c43719..eba6a0c67 100644 --- a/setup.py +++ b/setup.py @@ -71,5 +71,5 @@ setup_requires=SETUP_REQUIRES, install_requires=INSTALL_REQUIRES, extras_require=EXTRAS_REQUIRE, - classifier=CLASSIFIER, + classifiers=CLASSIFIER, ) From 25aed5ed8b3430ce94374180d8660a5bc8ad8151 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 11:42:46 +0200 Subject: [PATCH 19/50] fixed set error to list --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index eba6a0c67..0e5ead21e 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ } # https://pypi.org/classifiers/ -CLASSIFIER = { +CLASSIFIERS = [ "Development Status :: 3 - Alpha", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", @@ -53,7 +53,7 @@ "Programming Language :: Python :: 3.10", "Environment :: CPU", "Environment :: GPU", -} +] setup( name="graphnet", @@ -71,5 +71,5 @@ setup_requires=SETUP_REQUIRES, install_requires=INSTALL_REQUIRES, extras_require=EXTRAS_REQUIRE, - classifiers=CLASSIFIER, + classifiers=CLASSIFIERS, ) From cf9ac1a279d01a9a08828e879c74b83c0b5ebee4 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 11:53:20 +0200 Subject: [PATCH 20/50] changed setuptools version --- .github/actions/install-cpu/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-cpu/action.yml b/.github/actions/install-cpu/action.yml index fbfb84757..c0abf6906 100644 --- a/.github/actions/install-cpu/action.yml +++ b/.github/actions/install-cpu/action.yml @@ -22,7 +22,7 @@ runs: - name: Install dependencies run: | pip install --upgrade pip - pip install wheel setuptools>=62.3.0 + pip install wheel setuptools==59.5.0 shell: bash - name: Install package run: | From 1ac2611d1fa59e9a6e3b53fe938724bce8c2bae3 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 11:55:36 +0200 Subject: [PATCH 21/50] changed pip version --- .github/actions/install-cpu/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-cpu/action.yml b/.github/actions/install-cpu/action.yml index c0abf6906..86b792731 100644 --- a/.github/actions/install-cpu/action.yml +++ b/.github/actions/install-cpu/action.yml @@ -21,7 +21,7 @@ runs: shell: bash - name: Install dependencies run: | - pip install --upgrade pip + pip install --upgrade pip==22.0.4 pip install wheel setuptools==59.5.0 shell: bash - name: Install package From 49fea9707a924cb3e4053f18bc8809fc8b381553 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 11:57:20 +0200 Subject: [PATCH 22/50] changed pip version specifier to include python 3.6 --- .github/actions/install-cpu/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-cpu/action.yml b/.github/actions/install-cpu/action.yml index 86b792731..0c03e051d 100644 --- a/.github/actions/install-cpu/action.yml +++ b/.github/actions/install-cpu/action.yml @@ -21,7 +21,7 @@ runs: shell: bash - name: Install dependencies run: | - pip install --upgrade pip==22.0.4 + pip install --upgrade pip<=22.0.4 pip install wheel setuptools==59.5.0 shell: bash - name: Install package From dd1c2dea72cc00dbac9a65457da9900f1111753e Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 12:00:42 +0200 Subject: [PATCH 23/50] multiple pip version bounds --- .github/actions/install-cpu/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-cpu/action.yml b/.github/actions/install-cpu/action.yml index 0c03e051d..e9afa4a50 100644 --- a/.github/actions/install-cpu/action.yml +++ b/.github/actions/install-cpu/action.yml @@ -21,7 +21,7 @@ runs: shell: bash - name: Install dependencies run: | - pip install --upgrade pip<=22.0.4 + pip install --upgrade 'pip>=20,<22.1' pip install wheel setuptools==59.5.0 shell: bash - name: Install package From 91b2b2cbc240772f07e4dcbd2feb42ac4dbe12c5 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 12:02:38 +0200 Subject: [PATCH 24/50] setuptools version lowered --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0e5ead21e..0df720a1c 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ # Requirements definitions SETUP_REQUIRES = [ - "setuptools>=62.3.0", + "setuptools>=59.5.0", ] INSTALL_REQUIRES = [ From b4d5db0776af7fb2b5973d40edcac4a42b5024c4 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 12:08:24 +0200 Subject: [PATCH 25/50] fixed versions --- .github/workflows/build-matrix.yml | 2 +- setup.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-matrix.yml b/.github/workflows/build-matrix.yml index 04a41f6d4..7e5896049 100644 --- a/.github/workflows/build-matrix.yml +++ b/.github/workflows/build-matrix.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.7, 3.8, 3.9, '3.10'] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/setup.py b/setup.py index 0df720a1c..1d0eddfb5 100644 --- a/setup.py +++ b/setup.py @@ -9,8 +9,8 @@ ] INSTALL_REQUIRES = [ - "sqlalchemy>=1.4.37", - "pandas>=1.4", + "sqlalchemy>=1.4", + "pandas>=1.3", "numpy>=1.22", "timer>=0.2", "tqdm>=4.64", @@ -19,13 +19,13 @@ "matplotlib>=3.5", "scikit_learn~=1.1.1", "scipy>=1.8", - "torch~=1.11", + "torch==1.11", "torch-cluster==1.6.0", "torch-scatter==2.0.9", "torch-sparse==0.6.13", "torch-spline-conv==1.2.1", "torch-geometric==2.0.4", - "pytorch-lightning>=1.6.0", + "pytorch-lightning>=1.6", ] EXTRAS_REQUIRE = { From 62bd93ce3f027ef8e2de52d8120221d00bb87e88 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 12:15:10 +0200 Subject: [PATCH 26/50] fixed requirements --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index d882e3baa..5eb6c037c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ # Contains packages recommended for functional performance -setuptools>=62.3.0 +setuptools>=55.9 docutils>=0.18 -SQLAlchemy>=1.4.37 -pandas>=1.4 +SQLAlchemy>=1.4 +pandas>=1.35 numpy>=1.22 timer>=0.2 tqdm>=4.64 From 645a3b8c141d9d091124ed6e306be4bc1d63bf7d Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 12:16:44 +0200 Subject: [PATCH 27/50] fixed bug --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5eb6c037c..1279d5be8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ setuptools>=55.9 docutils>=0.18 SQLAlchemy>=1.4 -pandas>=1.35 +pandas>=1.3 numpy>=1.22 timer>=0.2 tqdm>=4.64 From 062691df51280134e831da1c8cec5b2fc84f9724 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 12:18:11 +0200 Subject: [PATCH 28/50] fixed numpy --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1279d5be8..c79b81ffe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ setuptools>=55.9 docutils>=0.18 SQLAlchemy>=1.4 pandas>=1.3 -numpy>=1.22 +numpy>=1.21 timer>=0.2 tqdm>=4.64 dill>=0.3 From 8ae749c61f25536bff2d9d1104582479e427c066 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 12:18:29 +0200 Subject: [PATCH 29/50] fixed numpy version v2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1d0eddfb5..473e47c97 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ INSTALL_REQUIRES = [ "sqlalchemy>=1.4", "pandas>=1.3", - "numpy>=1.22", + "numpy>=1.21", "timer>=0.2", "tqdm>=4.64", "dill>=0.3", From 06afbc61ed8dd592ae0ea8d1f91618ded51429c3 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 12:22:13 +0200 Subject: [PATCH 30/50] fixed scikit version --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index c79b81ffe..f8e1f652f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ tqdm>=4.64 dill>=0.3 wandb>=0.12 matplotlib>=3.5 -scikit_learn~=1.1.1 +scikit_learn>=1.0 scipy>=1.8 --find-links https://download.pytorch.org/whl/torch_stable.html torch==1.11+cu115 diff --git a/setup.py b/setup.py index 473e47c97..de5e5d394 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ "dill>=0.3", "wandb>=0.12", "matplotlib>=3.5", - "scikit_learn~=1.1.1", + "scikit_learn>=1.0", "scipy>=1.8", "torch==1.11", "torch-cluster==1.6.0", From df98c31cae3be4d5cdf3da823cb4b2dde4fffd28 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 12:24:02 +0200 Subject: [PATCH 31/50] fixed scipy version --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f8e1f652f..8faa0fbf9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ dill>=0.3 wandb>=0.12 matplotlib>=3.5 scikit_learn>=1.0 -scipy>=1.8 +scipy>=1.7 --find-links https://download.pytorch.org/whl/torch_stable.html torch==1.11+cu115 --find-links https://data.pyg.org/whl/torch-1.11.0+cu115.html diff --git a/setup.py b/setup.py index de5e5d394..0cdde45ce 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ "wandb>=0.12", "matplotlib>=3.5", "scikit_learn>=1.0", - "scipy>=1.8", + "scipy>=1.7", "torch==1.11", "torch-cluster==1.6.0", "torch-scatter==2.0.9", From 2d559a65608af48bfdf74f6be0400f0ffce7d134 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 12:36:17 +0200 Subject: [PATCH 32/50] divided cpu and gpu requirements --- .github/actions/install-cpu/action.yml | 2 +- requirements_cpu.txt | 8 ++++++++ requirements.txt => requirements_gpu.txt | 15 +-------------- 3 files changed, 10 insertions(+), 15 deletions(-) create mode 100644 requirements_cpu.txt rename requirements.txt => requirements_gpu.txt (58%) diff --git a/.github/actions/install-cpu/action.yml b/.github/actions/install-cpu/action.yml index e9afa4a50..f00b70d56 100644 --- a/.github/actions/install-cpu/action.yml +++ b/.github/actions/install-cpu/action.yml @@ -26,5 +26,5 @@ runs: shell: bash - name: Install package run: | - pip install ${{ env.PIP_FLAGS }} -r requirements.txt .${{ inputs.extras }} + pip install ${{ env.PIP_FLAGS }} -r requirements_cpu.txt .${{ inputs.extras }} shell: bash diff --git a/requirements_cpu.txt b/requirements_cpu.txt new file mode 100644 index 000000000..0c469e494 --- /dev/null +++ b/requirements_cpu.txt @@ -0,0 +1,8 @@ +--find-links https://download.pytorch.org/whl/torch_stable.html +torch==1.11+cpu +--find-links https://data.pyg.org/whl/torch-1.11.0+cpu.html +torch-cluster==1.6.0 +torch_scatter==2.0.9 +torch-sparse==0.6.13 +torch-spline-conv==1.2.1 +torch_geometric==2.0.4 \ No newline at end of file diff --git a/requirements.txt b/requirements_gpu.txt similarity index 58% rename from requirements.txt rename to requirements_gpu.txt index 8faa0fbf9..23988eca4 100644 --- a/requirements.txt +++ b/requirements_gpu.txt @@ -1,16 +1,4 @@ # Contains packages recommended for functional performance -setuptools>=55.9 -docutils>=0.18 -SQLAlchemy>=1.4 -pandas>=1.3 -numpy>=1.21 -timer>=0.2 -tqdm>=4.64 -dill>=0.3 -wandb>=0.12 -matplotlib>=3.5 -scikit_learn>=1.0 -scipy>=1.7 --find-links https://download.pytorch.org/whl/torch_stable.html torch==1.11+cu115 --find-links https://data.pyg.org/whl/torch-1.11.0+cu115.html @@ -18,5 +6,4 @@ torch-cluster==1.6.0 torch_scatter==2.0.9 torch-sparse==0.6.13 torch-spline-conv==1.2.1 -torch_geometric==2.0.4 -pytorch_lightning~=1.6.0 \ No newline at end of file +torch_geometric==2.0.4 \ No newline at end of file From fac43d58a2a25ed2bb2bd6c5f254ad2cb01ead53 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 13:11:20 +0200 Subject: [PATCH 33/50] changed setup structure and readme --- .github/actions/install-cpu/action.yml | 4 +-- README.md | 20 +++++++------- envs/gnn_py38.yml | 10 ------- .../torch_cpu.txt | 0 .../torch_gpu.txt | 0 setup.py | 27 +++++++++++++------ 6 files changed, 30 insertions(+), 31 deletions(-) delete mode 100644 envs/gnn_py38.yml rename requirements_cpu.txt => requirements/torch_cpu.txt (100%) rename requirements_gpu.txt => requirements/torch_gpu.txt (100%) diff --git a/.github/actions/install-cpu/action.yml b/.github/actions/install-cpu/action.yml index f00b70d56..4e3da9ee4 100644 --- a/.github/actions/install-cpu/action.yml +++ b/.github/actions/install-cpu/action.yml @@ -9,7 +9,7 @@ inputs: extras: description: "Extras flags for pip installation of graphnet" required: true - default: "[develop]" + default: "[develop,torch]" runs: using: "composite" @@ -26,5 +26,5 @@ runs: shell: bash - name: Install package run: | - pip install ${{ env.PIP_FLAGS }} -r requirements_cpu.txt .${{ inputs.extras }} + pip install ${{ env.PIP_FLAGS }} -r requirements/torch_cpu.txt .${{ inputs.extras }} shell: bash diff --git a/README.md b/README.md index f84c24f33..2386ef944 100644 --- a/README.md +++ b/README.md @@ -9,24 +9,22 @@ [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) - +![Supported python versions](https://img.shields.io/badge/python-3.7%20%7C%203.8%20%7C%203.9%20%7C%203.10-blue) ## :gear: Install -We recommend installing `graphnet` in a separate environment, e.g. using Anaconda (see details on installation [here](https://www.anaconda.com/products/individual)). The fastest way to get up and running is to install the package in the provided conda environment, specifying the python version {>3.8} and ensuring that the gcc compilers are up to date: +We recommend installing `graphnet` in a separate environment, e.g. using Anaconda (see details on installation [here](https://www.anaconda.com/products/individual)) or python virtual environment. The fastest way to get up and running is to install the package as shown below, specifying the python version and ensuring that the gcc compilers are up to date: ```bash $ git clone git@github.com:/graphnet.git $ cd graphnet -$ conda create --name gnn_py38 python=3.xx gcc_linux-64 gxx_linux-64 libgcc -y -$ conda activate gnn_py38 -(gnn_py38) $ pip install -r requirements.txt -e .[develop] +$ conda create --name graphnet python=3.8 gcc_linux-64 gxx_linux-64 libgcc -y +$ conda activate graphnet +(graphnet) $ pip install -r requirements/torch_[gpu/cpu].txt -e .[develop,torch] ``` This should allow you to e.g. run the scripts in [examples/](./examples/) out of the box. -You can also install the package in a python virtual environment, or in your system python, but then you will have to contend with C++ compiler versions; the non-standard interplay between [pytorch](https://pytorch.org/) and [pytorch-geometric](https://pytorch-geometric.readthedocs.io/en/latest/) (see e.g. [here](https://github.com/pyg-team/pytorch_geometric/issues/861#issuecomment-566424944)), which `graphnet` uses internally; etc. - -## :gear: Database creation and Installation on IceTray +### :gear: Database creation and Installation on IceTray Within the IceTray enviroment, the Graphnet module is used to create SQL databases from I3 files, using the provided 'convert_i3_to_sqlite.py' script. The recommended set up is as follows: @@ -44,9 +42,9 @@ eval `/cvmfs/icecube.opensciencegrid.org/py3-v4.1.0/setup.sh` Next, launch it and install the Graphnet module at a user level. ```bash $ source cvmfs.sh -$ conda create --name gnn_icetray -$ conda activate gnn_icetray -$ pip install --user -r requirements.txt -e .[develop] +$ conda create --name graphnet_icetray +$ conda activate graphnet_icetray +$ pip install --user -e .[develop] ``` This should allow you to run 'convert_i3_to_sqlite.py' in [examples/](./examples/) with your preferred I3 files. diff --git a/envs/gnn_py38.yml b/envs/gnn_py38.yml deleted file mode 100644 index 754a9a9b7..000000000 --- a/envs/gnn_py38.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: gnn_py38 -channels: - - nvidia - - conda-forge -dependencies: - - python=3.8 - - cudatoolkit=11.3 - - pip - - pip: - - setuptools==59.5.0 diff --git a/requirements_cpu.txt b/requirements/torch_cpu.txt similarity index 100% rename from requirements_cpu.txt rename to requirements/torch_cpu.txt diff --git a/requirements_gpu.txt b/requirements/torch_gpu.txt similarity index 100% rename from requirements_gpu.txt rename to requirements/torch_gpu.txt diff --git a/setup.py b/setup.py index 0cdde45ce..f4edc9962 100644 --- a/setup.py +++ b/setup.py @@ -19,13 +19,6 @@ "matplotlib>=3.5", "scikit_learn>=1.0", "scipy>=1.7", - "torch==1.11", - "torch-cluster==1.6.0", - "torch-scatter==2.0.9", - "torch-sparse==0.6.13", - "torch-spline-conv==1.2.1", - "torch-geometric==2.0.4", - "pytorch-lightning>=1.6", ] EXTRAS_REQUIRE = { @@ -40,19 +33,37 @@ "sphinx", "sphinx_rtd_theme", "versioneer", + ], + "torch": [ + "torch==1.11", + "torch-cluster==1.6.0", + "torch-scatter==2.0.9", + "torch-sparse==0.6.13", + "torch-spline-conv==1.2.1", + "torch-geometric==2.0.4", + "pytorch-lightning>=1.6", ] } # https://pypi.org/classifiers/ CLASSIFIERS = [ "Development Status :: 3 - Alpha", - "Programming Language :: Python :: 3.6", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Environment :: CPU", "Environment :: GPU", + "License :: OSI Approved :: BSD License", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", ] setup( From e1ae275a01ec4f0682bbd6ca09ee1de215f2cc82 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 13:14:12 +0200 Subject: [PATCH 34/50] black --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f4edc9962..4f4dcfc2a 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ "torch-spline-conv==1.2.1", "torch-geometric==2.0.4", "pytorch-lightning>=1.6", - ] + ], } # https://pypi.org/classifiers/ From dab2160a13d3729c443c932d6a6c2972f516a951 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 14:26:40 +0200 Subject: [PATCH 35/50] removed 'wheel setuptools==59.5.0' --- .github/actions/install-cpu/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/install-cpu/action.yml b/.github/actions/install-cpu/action.yml index 4e3da9ee4..d10838c70 100644 --- a/.github/actions/install-cpu/action.yml +++ b/.github/actions/install-cpu/action.yml @@ -22,7 +22,6 @@ runs: - name: Install dependencies run: | pip install --upgrade 'pip>=20,<22.1' - pip install wheel setuptools==59.5.0 shell: bash - name: Install package run: | From fd0b9120c2fff7772bd56de6fde1aea5c5a37309 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 14:28:25 +0200 Subject: [PATCH 36/50] reverted prior change --- .github/actions/install-cpu/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/install-cpu/action.yml b/.github/actions/install-cpu/action.yml index d10838c70..4e3da9ee4 100644 --- a/.github/actions/install-cpu/action.yml +++ b/.github/actions/install-cpu/action.yml @@ -22,6 +22,7 @@ runs: - name: Install dependencies run: | pip install --upgrade 'pip>=20,<22.1' + pip install wheel setuptools==59.5.0 shell: bash - name: Install package run: | From 34ce1ea66eca21e5a9353ebe39f82b2d594c08ee Mon Sep 17 00:00:00 2001 From: Morten Holm <33733987+MortenHolmRep@users.noreply.github.com> Date: Tue, 21 Jun 2022 14:29:36 +0200 Subject: [PATCH 37/50] Update setup.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit changed License Co-authored-by: Andreas Søgaard --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4f4dcfc2a..8c51f15be 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ "Programming Language :: Python :: 3.10", "Environment :: CPU", "Environment :: GPU", - "License :: OSI Approved :: BSD License", + "License :: OSI Approved :: Apache Software License", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Mathematics", "Topic :: Scientific/Engineering :: Artificial Intelligence", From 63ff64902a3d873e29677c7cb1fa9d6090b51c4b Mon Sep 17 00:00:00 2001 From: Morten Holm <33733987+MortenHolmRep@users.noreply.github.com> Date: Tue, 21 Jun 2022 14:29:53 +0200 Subject: [PATCH 38/50] Update setup.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removed intended audicent label for education Co-authored-by: Andreas Søgaard --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 8c51f15be..c4c9d3cb5 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,6 @@ CLASSIFIERS = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", - "Intended Audience :: Education", "Intended Audience :: Science/Research", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", From f972002c7e3225cfb92b3b829e2d38227b93a89a Mon Sep 17 00:00:00 2001 From: Morten Holm <33733987+MortenHolmRep@users.noreply.github.com> Date: Tue, 21 Jun 2022 14:30:17 +0200 Subject: [PATCH 39/50] Update setup.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added Physics as a label Co-authored-by: Andreas Søgaard --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index c4c9d3cb5..fea1c3b82 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,7 @@ "Environment :: GPU", "License :: OSI Approved :: Apache Software License", "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Physics", "Topic :: Scientific/Engineering :: Mathematics", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Software Development", From 74d5d46bdc0a4d913119b88f03d1a6c894a56ef7 Mon Sep 17 00:00:00 2001 From: Morten Holm <33733987+MortenHolmRep@users.noreply.github.com> Date: Tue, 21 Jun 2022 14:31:03 +0200 Subject: [PATCH 40/50] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit modified readme Co-authored-by: Andreas Søgaard --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2386ef944..30e72d46c 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,11 @@ ## :gear: Install -We recommend installing `graphnet` in a separate environment, e.g. using Anaconda (see details on installation [here](https://www.anaconda.com/products/individual)) or python virtual environment. The fastest way to get up and running is to install the package as shown below, specifying the python version and ensuring that the gcc compilers are up to date: +We recommend installing `graphnet` in a separate environment, e.g. using Anaconda (see details on installation [here](https://www.anaconda.com/products/individual)) or python virtual environment. This requires specifying a supported python version (see above) and ensuring that the C++ compilers (gcc) are up to date. + +### Installing stand-alone + +If you don't need to interface with [IceTray](https://github.com/icecube/icetray/) (e.g., for reading data from I3 files or running inference on these), the following commands should provide a fast way to get up and running on most UNIX systems: ```bash $ git clone git@github.com:/graphnet.git $ cd graphnet From d2ab050fc51a642d9bbb9b3e792890e1614c11c1 Mon Sep 17 00:00:00 2001 From: Morten Holm <33733987+MortenHolmRep@users.noreply.github.com> Date: Tue, 21 Jun 2022 14:31:51 +0200 Subject: [PATCH 41/50] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added bash syntax in readme Co-authored-by: Andreas Søgaard --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 30e72d46c..939fbf19e 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,8 @@ $ nano cvmfs.sh Within the 'cvmfs.sh' copy the following and press 'ctrl+X' ```bash -eval `/cvmfs/icecube.opensciencegrid.org/py3-v4.1.0/setup.sh` -/cvmfs/icecube.opensciencegrid.org/py3-v4.1.0/RHEL_7_x86_64/metaprojects/combo/stable/env-shell.sh +$ eval `/cvmfs/icecube.opensciencegrid.org/py3-v4.1.0/setup.sh` +$ /cvmfs/icecube.opensciencegrid.org/py3-v4.1.0/RHEL_7_x86_64/metaprojects/combo/stable/env-shell.sh ``` Next, launch it and install the Graphnet module at a user level. From 8ea900fed2f2ca2874fbe0f12a2e83db9ee7a32d Mon Sep 17 00:00:00 2001 From: Morten Holm <33733987+MortenHolmRep@users.noreply.github.com> Date: Tue, 21 Jun 2022 14:32:28 +0200 Subject: [PATCH 42/50] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit modified icetray specific readme of database creation segment Co-authored-by: Andreas Søgaard --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 939fbf19e..b5b77eaae 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Within the 'cvmfs.sh' copy the following and press 'ctrl+X' $ eval `/cvmfs/icecube.opensciencegrid.org/py3-v4.1.0/setup.sh` $ /cvmfs/icecube.opensciencegrid.org/py3-v4.1.0/RHEL_7_x86_64/metaprojects/combo/stable/env-shell.sh ``` +Optionally, you can alias these commands or save them as a bash script for convenience, as you will have to run these commands every time you want to use IceTray (with `graphnet`) in a clean shell. Next, launch it and install the Graphnet module at a user level. ```bash From fb37b38b307c80323969d39347dff2fa30b19ac1 Mon Sep 17 00:00:00 2001 From: Morten Holm <33733987+MortenHolmRep@users.noreply.github.com> Date: Tue, 21 Jun 2022 14:32:52 +0200 Subject: [PATCH 43/50] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removed unnecessary instruction in icetray segment Co-authored-by: Andreas Søgaard --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index b5b77eaae..12c5c173d 100644 --- a/README.md +++ b/README.md @@ -32,12 +32,6 @@ This should allow you to e.g. run the scripts in [examples/](./examples/) out of Within the IceTray enviroment, the Graphnet module is used to create SQL databases from I3 files, using the provided 'convert_i3_to_sqlite.py' script. The recommended set up is as follows: -```bash -$ cd graphnet -$ nano cvmfs.sh -``` - -Within the 'cvmfs.sh' copy the following and press 'ctrl+X' ```bash $ eval `/cvmfs/icecube.opensciencegrid.org/py3-v4.1.0/setup.sh` $ /cvmfs/icecube.opensciencegrid.org/py3-v4.1.0/RHEL_7_x86_64/metaprojects/combo/stable/env-shell.sh From 5ed386a167cfafcc6e0a60f18901a59250c3dbdd Mon Sep 17 00:00:00 2001 From: Morten Holm <33733987+MortenHolmRep@users.noreply.github.com> Date: Tue, 21 Jun 2022 14:33:35 +0200 Subject: [PATCH 44/50] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit improved description in icetray segment Co-authored-by: Andreas Søgaard --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 12c5c173d..0ce81ff0c 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ $ /cvmfs/icecube.opensciencegrid.org/py3-v4.1.0/RHEL_7_x86_64/metaprojects/combo ``` Optionally, you can alias these commands or save them as a bash script for convenience, as you will have to run these commands every time you want to use IceTray (with `graphnet`) in a clean shell. -Next, launch it and install the Graphnet module at a user level. +With the IceTray environment active, you can now install `graphnet` at a user level. In the example below, we are installing a light-weight version of `graphnet` without the `torch` extras, i.e., without the machine learning packages (pytorch and pytorch-geometric). This is useful when you just want to convert data from I3 files to, e.g., SQLite, and won't be running inference on I3 files later on. In this case, you don't need to specify a requirements file, compared to the example below. ```bash $ source cvmfs.sh $ conda create --name graphnet_icetray From dd6badb9eeada52d7bc30f135ab26fae54e52ad8 Mon Sep 17 00:00:00 2001 From: Morten Holm <33733987+MortenHolmRep@users.noreply.github.com> Date: Tue, 21 Jun 2022 14:34:23 +0200 Subject: [PATCH 45/50] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit better pointer for script Co-authored-by: Andreas Søgaard --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0ce81ff0c..aea87b5e1 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ $ conda activate graphnet_icetray $ pip install --user -e .[develop] ``` -This should allow you to run 'convert_i3_to_sqlite.py' in [examples/](./examples/) with your preferred I3 files. +This should allow you to run the [examples/convert_i3_to_sqlite.py](examples/convert_i3_to_sqlite.py) script with your preferred I3 files. ## :handshake: Contributing From 613d49e476ff6a7162d8f0c5247001242cf2b6b9 Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 14:37:37 +0200 Subject: [PATCH 46/50] reverted train_model script to original --- examples/train_model.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/train_model.py b/examples/train_model.py index af2c4e118..28491a102 100644 --- a/examples/train_model.py +++ b/examples/train_model.py @@ -36,7 +36,7 @@ project="example-script", entity="graphnet-team", save_dir="./wandb/", - log_model=False, + log_model=True, ) @@ -48,16 +48,16 @@ def main(): # Configuration config = { - "db": "\graphnet\datasets\dev_lvl7_robustness_muon_neutrino_0000.db", + "db": "/groups/icecube/asogaard/data/sqlite/dev_lvl7_robustness_muon_neutrino_0000/data/dev_lvl7_robustness_muon_neutrino_0000.db", "pulsemap": "SRTTWOfflinePulsesDC", - "batch_size": 256, - "num_workers": 2, - "gpus": [], + "batch_size": 512, + "num_workers":10, + "gpus": [1], "target": "energy", - "n_epochs": 1, - "patience": 1, + "n_epochs": 5, + "patience": 5, } - archive = "\Graphnet\graphnet_user" + archive = "/groups/icecube/asogaard/gnn/results/" run_name = "dynedge_{}_example".format(config["target"]) # Log configuration to W&B From eabb8f43cf7911102acac1d9d5dfa9c9a76458be Mon Sep 17 00:00:00 2001 From: Morten Holm Date: Tue, 21 Jun 2022 14:40:18 +0200 Subject: [PATCH 47/50] black --- examples/train_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/train_model.py b/examples/train_model.py index 28491a102..adba222d9 100644 --- a/examples/train_model.py +++ b/examples/train_model.py @@ -51,7 +51,7 @@ def main(): "db": "/groups/icecube/asogaard/data/sqlite/dev_lvl7_robustness_muon_neutrino_0000/data/dev_lvl7_robustness_muon_neutrino_0000.db", "pulsemap": "SRTTWOfflinePulsesDC", "batch_size": 512, - "num_workers":10, + "num_workers": 10, "gpus": [1], "target": "energy", "n_epochs": 5, From 130bd8c8dffa5457e410d26a0fce0b19d60a56c9 Mon Sep 17 00:00:00 2001 From: Morten Holm <33733987+MortenHolmRep@users.noreply.github.com> Date: Tue, 21 Jun 2022 15:32:01 +0200 Subject: [PATCH 48/50] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andreas Søgaard --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aea87b5e1..a16e660f2 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ $ conda activate graphnet This should allow you to e.g. run the scripts in [examples/](./examples/) out of the box. -### :gear: Database creation and Installation on IceTray +### Installing with IceTray Within the IceTray enviroment, the Graphnet module is used to create SQL databases from I3 files, using the provided 'convert_i3_to_sqlite.py' script. The recommended set up is as follows: From 47dc59e9e61ee8c320f3604bac5fd3bc0012462c Mon Sep 17 00:00:00 2001 From: Morten Holm <33733987+MortenHolmRep@users.noreply.github.com> Date: Tue, 21 Jun 2022 15:32:35 +0200 Subject: [PATCH 49/50] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andreas Søgaard --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a16e660f2..1c44a33f9 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ $ conda activate graphnet (graphnet) $ pip install -r requirements/torch_[gpu/cpu].txt -e .[develop,torch] ``` -This should allow you to e.g. run the scripts in [examples/](./examples/) out of the box. +This should allow you to e.g. run the scripts in [examples/](./examples/) out of the box. Here, we have installed recent C++ compilers using conda (`gcc_linux-64 gxx_linux-64 libgcc`), but if your system already have recent versions (`$gcc --version` should be > 5, at least) you should be able to omit these from the setup. ### Installing with IceTray From db8b454cf15dc8bb5452ab0f5cfd3dc6425bfba3 Mon Sep 17 00:00:00 2001 From: Morten Holm <33733987+MortenHolmRep@users.noreply.github.com> Date: Tue, 21 Jun 2022 15:33:00 +0200 Subject: [PATCH 50/50] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andreas Søgaard --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1c44a33f9..020b98037 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,9 @@ This should allow you to e.g. run the scripts in [examples/](./examples/) out of ### Installing with IceTray -Within the IceTray enviroment, the Graphnet module is used to create SQL databases from I3 files, using the provided 'convert_i3_to_sqlite.py' script. The recommended set up is as follows: +In some instances, you might want `graphnet` to be able to interface with IceTray, e.g., when converting I3 files to an intermediate file format for training GNN models(e.g., SQLite or parquet), as shown in the [examples/convert_i3_to_sqlite.py](examples/convert_i3_to_sqlite.py) script, or when running GNN inference as part of an IceTray chain. In these cases, you need to install `graphnet` in a python runtime that has IceTray installed. + +To achieve this, we recommend running the following commands in a clean bash shell: ```bash $ eval `/cvmfs/icecube.opensciencegrid.org/py3-v4.1.0/setup.sh`