diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86629b0d..f98b771a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,28 +1,9 @@ -name: Test - -on: [push,pull_request,workflow_call] +on: + push: + branches: + - main + pull_request: jobs: - qa: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: pre-commit/action@v3.0.1 - test: - needs: qa - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - python-version: [ "3.10" ] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: pip install -r dev-requirements.txt - - name: Run tests - run: pytest \ No newline at end of file + uses: ./.github/workflows/ci_template.yml \ No newline at end of file diff --git a/.github/workflows/ci_template.yml b/.github/workflows/ci_template.yml new file mode 100644 index 00000000..839c0f0d --- /dev/null +++ b/.github/workflows/ci_template.yml @@ -0,0 +1,38 @@ +name: Test + +on: + workflow_call: + inputs: + os: + description: 'Operating system' + default: '["ubuntu-latest", "windows-latest"]' + type: string + python-version: + description: 'Python version' + default: '["3.10"]' + type: string + +jobs: + qa: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pre-commit/action@v3.0.1 + + test: + needs: qa + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ${{fromJson(inputs.os)}} + python-version: ${{fromJson(inputs.python-version)}} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: pip install .[dev] + - name: Run tests + run: pytest \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 228edfb1..ec52321b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,8 +1,73 @@ on: [release] +permissions: + contents: write + attestations: write + id-token: write + jobs: test: - uses: ./.github/workflows/ci.yml + uses: ./.github/workflows/ci_template.yml + with: + os: '["ubuntu-latest", "windows-latest", "macos-latest"]' + python-version: '["3.10", "3.11", "3.12"]' + + build-wheel: + needs: test + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: hynek/build-and-inspect-python-package@v2 + with: + attest-build-provenance-github: true + + publish-TestPyPI: + needs: build-wheel + name: Publish SWMManywhere to TestPyPI + runs-on: ubuntu-latest + + steps: + - name: Download packages built by build-and-inspect-python-package + uses: actions/download-artifact@v4 + with: + name: Packages + path: dist + + - name: Generate artifact attestation for sdist and wheel + uses: actions/attest-build-provenance@v1 + with: + subject-path: dist + + - name: Publish package distributions to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + skip-existing: true + + publish-PyPI: + needs: publish-TestPyPI + name: Publish SWMManywhere to PyPI + runs-on: ubuntu-latest + + steps: + - name: Download packages built by build-and-inspect-python-package + uses: actions/download-artifact@v4 + with: + name: Packages + path: dist + + - name: Generate artifact attestation for sdist and wheel + uses: actions/attest-build-provenance@v1 + with: + subject-path: dist + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + publish-docs: needs: publish-PyPI @@ -16,29 +81,7 @@ jobs: python-version: "3.11" - name: Install dependencies - run: pip install -r doc-requirements.txt + run: pip install .[doc] - name: Deploy Docs run: mkdocs gh-deploy --force - - # publish: - # runs-on: ubuntu-latest - # needs: test - # # The following steps to build a Docker image and publish to the GitHub container registry on release. Alternatively, can replace with other publishing steps (ie. publishing to PyPI, deploying documentation etc.) - # steps: - # - name: Login to GitHub Container Registry - # uses: docker/login-action@v3 - # with: - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} - # - name: Get image metadata - # id: meta - # uses: docker/metadata-action@v5 - # with: - # images: ghcr.io/${{ github.repository }} - # - name: Build and push Docker image - # uses: docker/build-push-action@v5 - # with: - # push: true - # tags: ${{ steps.meta.outputs.tags }} diff --git a/.gitignore b/.gitignore index b17a5bb4..f9cabf09 100644 --- a/.gitignore +++ b/.gitignore @@ -132,4 +132,8 @@ dmypy.json cache/ # Documentation generated models -swmmanywhere_models/ \ No newline at end of file +swmmanywhere_models/ +whiteboxtools_binaries.zip + +# Hatch +_version.py \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a9e24ae0..7ca9211b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,4 +42,4 @@ repos: - id: refurb name: Modernizing Python codebases using Refurb additional_dependencies: - - numpy \ No newline at end of file + - numpy diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index c09b373c..00000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,326 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --extra=dev --output-file=dev-requirements.txt pyproject.toml -# -aenum==3.1.11 - # via - # pyswmm - # swmm-toolkit -affine==2.4.0 - # via - # pyflwdir - # rasterio -annotated-types==0.7.0 - # via pydantic -attrs==23.2.0 - # via - # cads-api-client - # fiona - # jsonschema - # pytest-mypy - # rasterio - # referencing -build==1.2.1 - # via pip-tools -cads-api-client==1.2.0 - # via cdsapi -cdsapi==0.7.0 - # via swmmanywhere (pyproject.toml) -certifi==2024.7.4 - # via - # fiona - # netcdf4 - # pyogrio - # pyproj - # rasterio - # requests -cfgv==3.4.0 - # via pre-commit -cftime==1.6.4 - # via netcdf4 -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # click-plugins - # cligj - # fiona - # pip-tools - # planetary-computer - # rasterio -click-plugins==1.1.1 - # via - # fiona - # rasterio -cligj==0.7.2 - # via - # fiona - # rasterio -colorama==0.4.6 - # via - # build - # click - # loguru - # pytest - # tqdm -coverage[toml]==7.5.4 - # via pytest-cov -cramjam==2.8.3 - # via fastparquet -cytoolz==0.12.3 - # via swmmanywhere (pyproject.toml) -distlib==0.3.8 - # via virtualenv -fastparquet==2024.5.0 - # via swmmanywhere (pyproject.toml) -filelock==3.15.4 - # via - # pytest-mypy - # virtualenv -fiona==1.9.6 - # via swmmanywhere (pyproject.toml) -fsspec==2024.6.1 - # via fastparquet -geographiclib==2.0 - # via geopy -geopandas==1.0.1 - # via - # osmnx - # swmmanywhere (pyproject.toml) -geopy==2.4.1 - # via swmmanywhere (pyproject.toml) -gitdb==4.0.11 - # via gitpython -gitpython==3.1.43 - # via swmmanywhere (pyproject.toml) -identify==2.6.0 - # via pre-commit -idna==3.8 - # via requests -iniconfig==2.0.0 - # via pytest -joblib==1.4.2 - # via swmmanywhere (pyproject.toml) -jsonschema==4.23.0 - # via - # pystac - # swmmanywhere (pyproject.toml) -jsonschema-specifications==2023.12.1 - # via jsonschema -julian==0.14 - # via pyswmm -llvmlite==0.43.0 - # via numba -loguru==0.7.2 - # via swmmanywhere (pyproject.toml) -multiurl==0.3.1 - # via cads-api-client -mypy==1.10.1 - # via - # pytest-mypy - # swmmanywhere (pyproject.toml) -mypy-extensions==1.0.0 - # via mypy -netcdf4==1.7.1.post1 - # via swmmanywhere (pyproject.toml) -netcomp @ git+https://github.com/barneydobson/NetComp.git - # via swmmanywhere (pyproject.toml) -networkx==3.3 - # via - # netcomp - # osmnx - # swmmanywhere (pyproject.toml) -nodeenv==1.9.1 - # via pre-commit -numba==0.60.0 - # via pyflwdir -numpy==1.26.4 - # via - # cftime - # fastparquet - # geopandas - # netcdf4 - # netcomp - # numba - # osmnx - # pandas - # pyarrow - # pyflwdir - # pyogrio - # rasterio - # rioxarray - # scipy - # shapely - # snuggs - # swmmanywhere (pyproject.toml) - # xarray -osmnx==1.9.3 - # via swmmanywhere (pyproject.toml) -packaging==24.1 - # via - # build - # fastparquet - # geopandas - # planetary-computer - # pyogrio - # pyswmm - # pytest - # rioxarray - # xarray -pandas==2.2.2 - # via - # fastparquet - # geopandas - # osmnx - # swmmanywhere (pyproject.toml) - # xarray -pip-tools==7.4.1 - # via swmmanywhere (pyproject.toml) -planetary-computer==1.0.0 - # via swmmanywhere (pyproject.toml) -platformdirs==4.2.2 - # via virtualenv -pluggy==1.5.0 - # via pytest -pre-commit==3.7.1 - # via swmmanywhere (pyproject.toml) -pyarrow==16.1.0 - # via swmmanywhere (pyproject.toml) -pydantic==2.8.2 - # via - # planetary-computer - # swmmanywhere (pyproject.toml) -pydantic-core==2.20.1 - # via pydantic -pyflwdir==0.5.8 - # via swmmanywhere (pyproject.toml) -pyogrio==0.9.0 - # via geopandas -pyparsing==3.1.2 - # via snuggs -pyproj==3.6.1 - # via - # geopandas - # rioxarray -pyproject-hooks==1.1.0 - # via - # build - # pip-tools -pystac[validation]==1.10.1 - # via - # planetary-computer - # pystac-client -pystac-client==0.8.3 - # via - # planetary-computer - # swmmanywhere (pyproject.toml) -pyswmm==2.0.1 - # via swmmanywhere (pyproject.toml) -pytest==8.2.2 - # via - # pytest-cov - # pytest-mock - # pytest-mypy - # swmmanywhere (pyproject.toml) -pytest-cov==5.0.0 - # via swmmanywhere (pyproject.toml) -pytest-mock==3.14.0 - # via swmmanywhere (pyproject.toml) -pytest-mypy==0.10.3 - # via swmmanywhere (pyproject.toml) -python-dateutil==2.9.0.post0 - # via - # multiurl - # pandas - # pystac - # pystac-client -python-dotenv==1.0.1 - # via planetary-computer -pytz==2024.1 - # via - # multiurl - # pandas - # planetary-computer -pywbt==0.1.1 - # via swmmanywhere (pyproject.toml) -pyyaml==6.0.1 - # via - # pre-commit - # swmmanywhere (pyproject.toml) -rasterio==1.3.10 - # via - # rioxarray - # swmmanywhere (pyproject.toml) -referencing==0.35.1 - # via - # jsonschema - # jsonschema-specifications -requests==2.32.3 - # via - # cads-api-client - # cdsapi - # multiurl - # osmnx - # planetary-computer - # pystac-client -rioxarray==0.17.0 - # via swmmanywhere (pyproject.toml) -rpds-py==0.19.1 - # via - # jsonschema - # referencing -ruff==0.5.5 - # via swmmanywhere (pyproject.toml) -scipy==1.14.0 - # via - # netcomp - # pyflwdir - # swmmanywhere (pyproject.toml) -shapely==2.0.5 - # via - # geopandas - # osmnx - # swmmanywhere (pyproject.toml) -six==1.16.0 - # via - # fiona - # python-dateutil -smmap==5.0.1 - # via gitdb -snuggs==1.4.7 - # via rasterio -swmm-toolkit==0.15.5 - # via pyswmm -toolz==0.12.1 - # via cytoolz -tqdm==4.66.4 - # via - # cdsapi - # multiurl - # swmmanywhere (pyproject.toml) -typing-extensions==4.12.2 - # via - # cads-api-client - # mypy - # pydantic - # pydantic-core -tzdata==2024.1 - # via pandas -urllib3==2.2.2 - # via requests -virtualenv==20.26.3 - # via pre-commit -wheel==0.43.0 - # via pip-tools -win32-setctime==1.1.0 - # via loguru -xarray==2024.6.0 - # via - # rioxarray - # swmmanywhere (pyproject.toml) - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/doc-requirements.txt b/doc-requirements.txt deleted file mode 100644 index d4cc7566..00000000 --- a/doc-requirements.txt +++ /dev/null @@ -1,484 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --extra=doc --output-file=doc-requirements.txt pyproject.toml -# -aenum==3.1.11 - # via - # pyswmm - # swmm-toolkit -affine==2.4.0 - # via - # pyflwdir - # rasterio -annotated-types==0.7.0 - # via pydantic -asttokens==2.4.1 - # via stack-data -attrs==23.2.0 - # via - # cads-api-client - # fiona - # jsonschema - # rasterio - # referencing -babel==2.15.0 - # via mkdocs-material -beautifulsoup4==4.12.3 - # via nbconvert -bleach==6.1.0 - # via nbconvert -bracex==2.4 - # via wcmatch -cads-api-client==1.2.0 - # via cdsapi -cdsapi==0.7.0 - # via swmmanywhere (pyproject.toml) -certifi==2024.7.4 - # via - # fiona - # netcdf4 - # pyogrio - # pyproj - # rasterio - # requests -cftime==1.6.4 - # via netcdf4 -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # click-plugins - # cligj - # fiona - # mkdocs - # mkdocstrings - # planetary-computer - # rasterio -click-plugins==1.1.1 - # via - # fiona - # rasterio -cligj==0.7.2 - # via - # fiona - # rasterio -colorama==0.4.6 - # via - # click - # griffe - # ipython - # loguru - # mkdocs - # mkdocs-material - # tqdm -comm==0.2.2 - # via ipykernel -cramjam==2.8.3 - # via fastparquet -cytoolz==0.12.3 - # via swmmanywhere (pyproject.toml) -debugpy==1.8.5 - # via ipykernel -decorator==5.1.1 - # via ipython -defusedxml==0.7.1 - # via nbconvert -executing==2.0.1 - # via stack-data -fastjsonschema==2.20.0 - # via nbformat -fastparquet==2024.5.0 - # via swmmanywhere (pyproject.toml) -fiona==1.9.6 - # via swmmanywhere (pyproject.toml) -fsspec==2024.6.1 - # via fastparquet -geographiclib==2.0 - # via geopy -geopandas==1.0.1 - # via - # osmnx - # swmmanywhere (pyproject.toml) -geopy==2.4.1 - # via swmmanywhere (pyproject.toml) -ghp-import==2.1.0 - # via mkdocs -gitdb==4.0.11 - # via gitpython -gitpython==3.1.43 - # via swmmanywhere (pyproject.toml) -griffe==0.47.0 - # via mkdocstrings-python -idna==3.8 - # via requests -ipykernel==6.29.5 - # via mkdocs-jupyter -ipython==8.26.0 - # via ipykernel -jedi==0.19.1 - # via ipython -jinja2==3.1.4 - # via - # mkdocs - # mkdocs-material - # mkdocstrings - # nbconvert -joblib==1.4.2 - # via swmmanywhere (pyproject.toml) -jsonschema==4.23.0 - # via - # nbformat - # pystac - # swmmanywhere (pyproject.toml) -jsonschema-specifications==2023.12.1 - # via jsonschema -julian==0.14 - # via pyswmm -jupyter-client==8.6.2 - # via - # ipykernel - # nbclient -jupyter-core==5.7.2 - # via - # ipykernel - # jupyter-client - # nbclient - # nbconvert - # nbformat -jupyterlab-pygments==0.3.0 - # via nbconvert -jupytext==1.16.3 - # via mkdocs-jupyter -llvmlite==0.43.0 - # via numba -loguru==0.7.2 - # via swmmanywhere (pyproject.toml) -markdown==3.6 - # via - # mkdocs - # mkdocs-autorefs - # mkdocs-material - # mkdocstrings - # pymdown-extensions -markdown-it-py==3.0.0 - # via - # jupytext - # mdit-py-plugins -markupsafe==2.1.5 - # via - # jinja2 - # mkdocs - # mkdocs-autorefs - # mkdocstrings - # nbconvert -matplotlib-inline==0.1.7 - # via - # ipykernel - # ipython -mdit-py-plugins==0.4.1 - # via jupytext -mdurl==0.1.2 - # via markdown-it-py -mergedeep==1.3.4 - # via - # mkdocs - # mkdocs-get-deps -mistune==3.0.2 - # via nbconvert -mkdocs==1.6.0 - # via - # mkdocs-autorefs - # mkdocs-coverage - # mkdocs-include-markdown-plugin - # mkdocs-jupyter - # mkdocs-material - # mkdocstrings - # swmmanywhere (pyproject.toml) -mkdocs-autorefs==1.0.1 - # via mkdocstrings -mkdocs-coverage==1.1.0 - # via swmmanywhere (pyproject.toml) -mkdocs-get-deps==0.2.0 - # via mkdocs -mkdocs-include-markdown-plugin==6.2.1 - # via swmmanywhere (pyproject.toml) -mkdocs-jupyter==0.24.8 - # via swmmanywhere (pyproject.toml) -mkdocs-material==9.5.28 - # via - # mkdocs-jupyter - # swmmanywhere (pyproject.toml) -mkdocs-material-extensions==1.3.1 - # via - # mkdocs-material - # swmmanywhere (pyproject.toml) -mkdocstrings[python]==0.25.2 - # via - # mkdocstrings-python - # swmmanywhere (pyproject.toml) -mkdocstrings-python==1.10.5 - # via mkdocstrings -multiurl==0.3.1 - # via cads-api-client -nbclient==0.10.0 - # via nbconvert -nbconvert==7.16.4 - # via mkdocs-jupyter -nbformat==5.10.4 - # via - # jupytext - # nbclient - # nbconvert -nest-asyncio==1.6.0 - # via ipykernel -netcdf4==1.7.1.post1 - # via swmmanywhere (pyproject.toml) -netcomp @ git+https://github.com/barneydobson/NetComp.git - # via swmmanywhere (pyproject.toml) -networkx==3.3 - # via - # netcomp - # osmnx - # swmmanywhere (pyproject.toml) -numba==0.60.0 - # via pyflwdir -numpy==1.26.4 - # via - # cftime - # fastparquet - # geopandas - # netcdf4 - # netcomp - # numba - # osmnx - # pandas - # pyarrow - # pyflwdir - # pyogrio - # rasterio - # rioxarray - # scipy - # shapely - # snuggs - # swmmanywhere (pyproject.toml) - # xarray -osmnx==1.9.3 - # via swmmanywhere (pyproject.toml) -packaging==24.1 - # via - # fastparquet - # geopandas - # ipykernel - # jupytext - # mkdocs - # nbconvert - # planetary-computer - # pyogrio - # pyswmm - # rioxarray - # xarray -paginate==0.5.7 - # via mkdocs-material -pandas==2.2.2 - # via - # fastparquet - # geopandas - # osmnx - # swmmanywhere (pyproject.toml) - # xarray -pandocfilters==1.5.1 - # via nbconvert -parso==0.8.4 - # via jedi -pathspec==0.12.1 - # via mkdocs -planetary-computer==1.0.0 - # via swmmanywhere (pyproject.toml) -platformdirs==4.2.2 - # via - # jupyter-core - # mkdocs-get-deps - # mkdocstrings -prompt-toolkit==3.0.47 - # via ipython -psutil==6.0.0 - # via ipykernel -pure-eval==0.2.2 - # via stack-data -pyarrow==16.1.0 - # via swmmanywhere (pyproject.toml) -pydantic==2.8.2 - # via - # planetary-computer - # swmmanywhere (pyproject.toml) -pydantic-core==2.20.1 - # via pydantic -pyflwdir==0.5.8 - # via swmmanywhere (pyproject.toml) -pygments==2.18.0 - # via - # ipython - # mkdocs-jupyter - # mkdocs-material - # nbconvert -pymdown-extensions==10.8.1 - # via - # mkdocs-material - # mkdocstrings -pyogrio==0.9.0 - # via geopandas -pyparsing==3.1.2 - # via snuggs -pyproj==3.6.1 - # via - # geopandas - # rioxarray -pystac[validation]==1.10.1 - # via - # planetary-computer - # pystac-client -pystac-client==0.8.3 - # via - # planetary-computer - # swmmanywhere (pyproject.toml) -pyswmm==2.0.1 - # via swmmanywhere (pyproject.toml) -python-dateutil==2.9.0.post0 - # via - # ghp-import - # jupyter-client - # multiurl - # pandas - # pystac - # pystac-client -python-dotenv==1.0.1 - # via planetary-computer -pytz==2024.1 - # via - # multiurl - # pandas - # planetary-computer -pywbt==0.1.1 - # via swmmanywhere (pyproject.toml) -pywin32==306 - # via jupyter-core -pyyaml==6.0.1 - # via - # jupytext - # mkdocs - # mkdocs-get-deps - # pymdown-extensions - # pyyaml-env-tag - # swmmanywhere (pyproject.toml) -pyyaml-env-tag==0.1 - # via mkdocs -pyzmq==26.0.3 - # via - # ipykernel - # jupyter-client -rasterio==1.3.10 - # via - # rioxarray - # swmmanywhere (pyproject.toml) -referencing==0.35.1 - # via - # jsonschema - # jsonschema-specifications -regex==2024.5.15 - # via mkdocs-material -requests==2.32.3 - # via - # cads-api-client - # cdsapi - # mkdocs-material - # multiurl - # osmnx - # planetary-computer - # pystac-client -rioxarray==0.17.0 - # via swmmanywhere (pyproject.toml) -rpds-py==0.19.1 - # via - # jsonschema - # referencing -scipy==1.14.0 - # via - # netcomp - # pyflwdir - # swmmanywhere (pyproject.toml) -shapely==2.0.5 - # via - # geopandas - # osmnx - # swmmanywhere (pyproject.toml) -six==1.16.0 - # via - # asttokens - # bleach - # fiona - # python-dateutil -smmap==5.0.1 - # via gitdb -snuggs==1.4.7 - # via rasterio -soupsieve==2.5 - # via beautifulsoup4 -stack-data==0.6.3 - # via ipython -swmm-toolkit==0.15.5 - # via pyswmm -tinycss2==1.3.0 - # via nbconvert -toolz==0.12.1 - # via cytoolz -tornado==6.4.1 - # via - # ipykernel - # jupyter-client -tqdm==4.66.4 - # via - # cdsapi - # multiurl - # swmmanywhere (pyproject.toml) -traitlets==5.14.3 - # via - # comm - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbclient - # nbconvert - # nbformat -typing-extensions==4.12.2 - # via - # cads-api-client - # ipython - # pydantic - # pydantic-core -tzdata==2024.1 - # via pandas -urllib3==2.2.2 - # via requests -watchdog==4.0.1 - # via mkdocs -wcmatch==8.5.2 - # via mkdocs-include-markdown-plugin -wcwidth==0.2.13 - # via prompt-toolkit -webencodings==0.5.1 - # via - # bleach - # tinycss2 -win32-setctime==1.1.0 - # via loguru -xarray==2024.6.0 - # via - # rioxarray - # swmmanywhere (pyproject.toml) - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/docs/graphfcns_guide.md b/docs/graphfcns_guide.md index 61f214cb..1570344b 100644 --- a/docs/graphfcns_guide.md +++ b/docs/graphfcns_guide.md @@ -9,11 +9,11 @@ an updated graph. ## Using graph functions -Let's look at a [graph function](reference-graph-utilities.md#swmmanywhere.graph_utilities.to_undirected) +Let's look at a [graph function](reference-graph-utilities.md#swmmanywhere.graphfcns.network_cleaning_graphfcns.to_undirected) that is simply a wrapper for [`networkx.to_undirected`](https://networkx.org/documentation/stable/reference/classes/generated/networkx.DiGraph.to_undirected.html): -:::swmmanywhere.graph_utilities.to_undirected +:::swmmanywhere.graphfcns.network_cleaning_graphfcns.to_undirected handler: python options: members: no @@ -64,9 +64,9 @@ view the [`parameters`](reference-parameters.md) and [`FilePaths`](reference-filepaths.md) APIs. We can see an example of using a parameter category with this -[graph function](reference-graph-utilities.md#swmmanywhere.graph_utilities.remove_non_pipe_allowable_links): +[graph function](reference-graph-utilities.md#swmmanywhere.graphfcns.network_cleaning_graphfcns.remove_non_pipe_allowable_links): -:::swmmanywhere.graph_utilities.remove_non_pipe_allowable_links +:::swmmanywhere.graphfcns.network_cleaning_graphfcns.remove_non_pipe_allowable_links handler: python options: members: no @@ -138,9 +138,9 @@ are not provided, `iterate_graphfcns` uses the default values for all Furthermore, this `graphfcn_list` also provides opportunities for validation. For example, see the -[following graph function](reference-graph-utilities.md#swmmanywhere.graph_utilities.set_surface_slope): +[following graph function](reference-graph-utilities.md#swmmanywhere.graphfcns.topology_graphfcns.set_surface_slope): -:::swmmanywhere.graph_utilities.set_surface_slope +:::swmmanywhere.graphfcns.topology_graphfcns.set_surface_slope handler: python options: members: no @@ -168,7 +168,7 @@ can be used to specify what, if any, parameters are added to the graph by the graph function. Let us inspect the `set_elevation` graph function: -:::swmmanywhere.graph_utilities.set_elevation +:::swmmanywhere.graphfcns.topology_graphfcns.set_elevation handler: python options: members: no diff --git a/docs/metrics_guide.md b/docs/metrics_guide.md index 7245a365..0c9c2305 100644 --- a/docs/metrics_guide.md +++ b/docs/metrics_guide.md @@ -39,15 +39,15 @@ registered metrics to be called from one place. ``` py >>> from swmmanywhere.metric_utilities import metrics >>> print(metrics.keys()) -dict_keys(['outlet_nse_flow', 'outlet_kge_flow', 'outlet_relerror_flow', -'outlet_relerror_length', 'outlet_relerror_npipes', 'outlet_relerror_nmanholes', -'outlet_relerror_diameter', 'outlet_nse_flooding', 'outlet_kge_flooding', -'outlet_relerror_flooding', 'grid_nse_flooding', 'grid_kge_flooding', +dict_keys(['outfall_nse_flow', 'outfall_kge_flow', 'outfall_relerror_flow', +'outfall_relerror_length', 'outfall_relerror_npipes', 'outfall_relerror_nmanholes', +'outfall_relerror_diameter', 'outfall_nse_flooding', 'outfall_kge_flooding', +'outfall_relerror_flooding', 'grid_nse_flooding', 'grid_kge_flooding', 'grid_relerror_flooding', 'subcatchment_nse_flooding', 'subcatchment_kge_flooding', 'subcatchment_relerror_flooding', 'nc_deltacon0', 'nc_laplacian_dist', 'nc_laplacian_norm_dist', 'nc_adjacency_dist', 'nc_vertex_edge_distance', 'nc_resistance_distance', 'bias_flood_depth', -'kstest_edge_betweenness', 'kstest_betweenness', 'outlet_kstest_diameters']) +'kstest_edge_betweenness', 'kstest_betweenness', 'outfall_kstest_diameters']) ``` We will later demonstrate how to [add a new metric](#add-a-new-metric) to the @@ -66,9 +66,9 @@ any `metric` has access to a range of arguments for calculation: - the [`MetricEvaluation`](reference-parameters.md#swmmanywhere.parameters.MetricEvaluation) parameters category. -For example, see the [following metric](reference-metric-utilities.md#swmmanywhere.metric_utilities.outlet_kstest_diameters) +For example, see the [following metric](reference-metric-utilities.md#swmmanywhere.metric_utilities.outfall_kstest_diameters) -:::swmmanywhere.metric_utilities.outlet_kstest_diameters +:::swmmanywhere.metric_utilities.outfall_kstest_diameters handler: python options: members: no @@ -184,7 +184,7 @@ us create a metric with `metric_factory`: ``` py >>> from swmmanywhere.metric_utilities import metric_factory ->>> metric_factory('outlet_nse_flow') +>>> metric_factory('outfall_nse_flow') .new_metric at 0x000001EECEA7C220> ``` @@ -243,7 +243,7 @@ in `metrics` we can also register that. ... metrics ... ) >>> import numpy as np ->>> metric_factory('outlet_rmse_flow') # Try creating the metric +>>> metric_factory('outfall_rmse_flow') # Try creating the metric Traceback (most recent call last): ... KeyError: 'rmse' @@ -253,9 +253,9 @@ KeyError: 'rmse' >>> print(coef_registry.keys()) dict_keys(['relerror', 'nse', 'kge', 'rmse']) ->>> metrics.register(metric_factory('outlet_rmse_flow')) # Create and register new metric +>>> metrics.register(metric_factory('outfall_rmse_flow')) # Create and register new metric .new_metric at 0x00000227D219E020> ->>> 'outlet_rmse_flow' in metrics # Check that the metric is available for use +>>> 'outfall_rmse_flow' in metrics # Check that the metric is available for use True ``` @@ -267,7 +267,7 @@ As with coefficients, these are stored in a registry. ``` py >>> from swmmanywhere.metric_utilities import scale_registry >>> print(scale_registry.keys()) -dict_keys(['subcatchment', 'grid', 'outlet']) +dict_keys(['subcatchment', 'grid', 'outfall']) ``` For example, `subcatchment` aligns `real` and `synthesised` subcatchments @@ -305,7 +305,7 @@ is a description of the designed UDM. ``` py >>> from swmmanywhere.metric_utilities import metric_factory ->>> metric_factory('outlet_nse_npipes') +>>> metric_factory('outfall_nse_npipes') Traceback (most recent call last): ... , in restriction_on_metric raise ValueError(f"Variable {variable} only valid with relerror metric") diff --git a/mkdocs.yml b/mkdocs.yml index e4926910..c561e5fd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -23,6 +23,7 @@ plugins: - include-markdown repo_url: https://github.com/ImperialCollegeLondon/SWMManywhere +site_url: https://imperialcollegelondon.github.io/SWMManywhere/ markdown_extensions: - footnotes diff --git a/netcomp/LICENSE.txt b/netcomp/LICENSE.txt new file mode 100644 index 00000000..8aa26455 --- /dev/null +++ b/netcomp/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) [year] [fullname] + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/netcomp/README.md b/netcomp/README.md new file mode 100644 index 00000000..e4e18526 --- /dev/null +++ b/netcomp/README.md @@ -0,0 +1,5 @@ +# NetComp + +See [original repo](https://github.com/barneydobson/NetComp) for more details on NetComp. + +NetComp is distributed under a MIT license diff --git a/netcomp/__init__.py b/netcomp/__init__.py new file mode 100644 index 00000000..700fa492 --- /dev/null +++ b/netcomp/__init__.py @@ -0,0 +1,25 @@ +"""NetComp v0.2.2 +============== + + NetComp is a Python package for comparing networks using pairwise distances, + and for performing anomaly detection on a time series of networks. It is + built on top of the NetworkX package. For details on usage and installation, + see README.md. + +""" + +from __future__ import annotations + +__author__ = "Peter Wills " +__version__ = "0.2.2" +__license__ = "MIT" + +import sys + +if sys.version_info[0] < 3: + m = "Python 3.x required (%d.%d detected)." + raise ImportError(m % sys.version_info[:2]) +del sys + +from .linalg import * # noqa +from .distance import * # noqa diff --git a/netcomp/distance/__init__.py b/netcomp/distance/__init__.py new file mode 100644 index 00000000..1ae92672 --- /dev/null +++ b/netcomp/distance/__init__.py @@ -0,0 +1,10 @@ +"""********* +Distances +********* + +Calculation of distances between graphs. +""" +from __future__ import annotations + +from .exact import * +from .features import * diff --git a/netcomp/distance/exact.py b/netcomp/distance/exact.py new file mode 100644 index 00000000..6ca7f969 --- /dev/null +++ b/netcomp/distance/exact.py @@ -0,0 +1,329 @@ +"""*************** +Exact Distances +*************** + +Calculation of exact distances between graphs. Generally slow (quadratic in +graph size). +""" + +from __future__ import annotations + +import networkx as nx +import numpy as np +from numpy import linalg as la + +from ..exception import InputError +from ..linalg import ( # type: ignore + fast_bp, + normalized_laplacian_eig, + renormalized_res_mat, + resistance_matrix, +) +from ..linalg.eigenstuff import _eigs, _flat +from ..linalg.matrices import _pad, laplacian_matrix +from .features import aggregate_features, get_features + +###################### +## Helper Functions ## +###################### + + +def _canberra_dist(v1, v2): + """The canberra distance between two vectors. We need to carefully handle + the case in which both v1 and v2 are zero in a certain dimension. + """ + eps = 10 ** (-15) + v1, v2 = [_flat(v) for v in [v1, v2]] + d_can = 0 + for u, w in zip(v1, v2): + if np.abs(u) < eps and np.abs(w) < eps: + d_update = 1 + else: + d_update = np.abs(u - w) / (np.abs(u) + np.abs(w)) + d_can += d_update + return d_can + + +############################# +## Distance Between Graphs ## +############################# + + +def edit_distance(A1, A2): + """The edit distance between graphs, defined as the number of changes one + needs to make to put the edge lists in correspondence. + + Parameters + ---------- + A1, A2 : NumPy matrices + Adjacency matrices of graphs to be compared + + Returns: + ------- + dist : float + The edit distance between the two graphs + """ + dist = np.abs((A1 - A2)).sum() / 2 + return dist + + +def vertex_edge_overlap(A1, A2): + """Vertex-edge overlap. Basically a souped-up edit distance, but in similarity + form. The VEO similarity is defined as + + VEO(G1,G2) = (|V1&V2| + |E1&E2|) / (|V1|+|V2|+|E1|+|E2|) + + where |S| is the size of a set S and U&T is the union of U and T. + + Parameters + ---------- + A1, A2 : NumPy matrices + Adjacency matrices of graphs to be compared + + Returns: + ------- + sim : float + The similarity between the two graphs + + + References: + ---------- + + """ + try: + [G1, G2] = [nx.from_scipy_sparse_array(A) for A in [A1, A2]] + except AttributeError: + [G1, G2] = [nx.from_numpy_array(A) for A in [A1, A2]] + V1, V2 = [set(G.nodes()) for G in [G1, G2]] + E1, E2 = [set(G.edges()) for G in [G1, G2]] + V_overlap = len(V1 | V2) # set union + E_overlap = len(E1 | E2) + sim = (V_overlap + E_overlap) / (len(V1) + len(V2) + len(E1) + len(E2)) + return sim + + +def vertex_edge_distance(A1, A2): + """Vertex-edge overlap transformed into a distance via + + D = (1-VEO)/VEO + + which is the inversion of the common distance-to-similarity function + + sim = 1/(1+D). + + Parameters + ---------- + A1, A2 : NumPy matrices + Adjacency matrices of graphs to be compared + + Returns: + ------- + dist : float + The distance between the two graphs + """ + sim = vertex_edge_overlap(A1, A2) + dist = (1 - sim) / sim + return dist + + +def lambda_dist(A1, A2, k=None, p=2, kind="laplacian"): + """The lambda distance between graphs, which is defined as + + d(G1,G2) = norm(L_1 - L_2) + + where L_1 is a vector of the top k eigenvalues of the appropriate matrix + associated with G1, and L2 is defined similarly. + + Parameters + ---------- + A1, A2 : NumPy matrices + Adjacency matrices of graphs to be compared + + k : Integer + The number of eigenvalues to be compared + + p : non-zero Float + The p-norm is used to compare the resulting vector of eigenvalues. + + kind : String , in {'laplacian','laplacian_norm','adjacency'} + The matrix for which eigenvalues will be calculated. + + Returns: + ------- + dist : float + The distance between the two graphs + + Notes: + ----- + The norm can be any p-norm; by default we use p=2. If p<0 is used, the + result is not a mathematical norm, but may still be interesting and/or + useful. + + If k is provided, then we use the k SMALLEST eigenvalues for the Laplacian + distances, and we use the k LARGEST eigenvalues for the adjacency + distance. This is because the corresponding order flips, as L = D-A. + + References: + ---------- + + See Also: + -------- + netcomp.linalg._eigs + normalized_laplacian_eigs + + """ + # ensure valid k + n1, n2 = [A.shape[0] for A in [A1, A2]] + N = min(n1, n2) # minimum size between the two graphs + if k is None or k > N: + k = N + if kind == "laplacian": + # form matrices + L1, L2 = [laplacian_matrix(A) for A in [A1, A2]] + # get eigenvalues, ignore eigenvectors + evals1, evals2 = [_eigs(L)[0] for L in [L1, L2]] + elif kind == "laplacian_norm": + # use our function to graph evals of normalized laplacian + evals1, evals2 = [normalized_laplacian_eig(A)[0] for A in [A1, A2]] + elif kind == "adjacency": + evals1, evals2 = [_eigs(A)[0] for A in [A1, A2]] + # reverse, so that we are sorted from large to small, since we care + # about the k LARGEST eigenvalues for the adjacency distance + evals1, evals2 = [evals[::-1] for evals in [evals1, evals2]] + else: + raise InputError( + "Invalid type, choose from 'laplacian', " + "'laplacian_norm', and 'adjacency'." + ) + dist = la.norm(evals1[:k] - evals2[:k], ord=p) + return dist + + +def netsimile(A1, A2): + """NetSimile distance between two graphs. + + Parameters + ---------- + A1, A2 : SciPy sparse array + Adjacency matrices of the graphs in question. + + Returns: + ------- + d_can : Float + The distance between the two graphs. + + Notes: + ----- + NetSimile works on graphs without node correspondence. Graphs to not need to + be the same size. + + See Also: + -------- + + References: + ---------- + """ + feat_A1, feat_A2 = [get_features(A) for A in [A1, A2]] + agg_A1, agg_A2 = [aggregate_features(feat) for feat in [feat_A1, feat_A2]] + # calculate Canberra distance between two aggregate vectors + d_can = _canberra_dist(agg_A1, agg_A2) + return d_can + + +def resistance_distance( + A1, A2, p=2, renormalized=False, attributed=False, check_connected=True, beta=1 +): + """Compare two graphs using resistance distance (possibly renormalized). + + Parameters + ---------- + A1, A2 : NumPy Matrices + Adjacency matrices of graphs to be compared. + + p : float + Function returns the p-norm of the flattened matrices. + + renormalized : Boolean, optional (default = False) + If true, then renormalized resistance distance is computed. + + attributed : Boolean, optional (default=False) + If true, then the resistance distance PER NODE is returned. + + check_connected : Boolean, optional (default=True) + If false, then no check on connectivity is performed. See Notes of + resistance_matrix for more information. + + beta : float, optional (default=1) + A parameter used in the calculation of the renormalized resistance + matrix. If using regular resistance, this is irrelevant. + + Returns: + ------- + dist : float of numpy array + The RR distance between the two graphs. If attributed is True, then + vector distance per node is returned. + + Notes: + ----- + The distance is calculated by assuming the nodes are in correspondence, and + any nodes not present are treated as isolated by renormalized resistance. + + References: + ---------- + + See Also: + -------- + resistance_matrix + """ + # Calculate resistance matrices and compare + if renormalized: + # pad smaller adj. mat. so they're the same size + n1, n2 = [A.shape[0] for A in [A1, A2]] + N = max(n1, n2) + A1, A2 = [_pad(A, N) for A in [A1, A2]] + R1, R2 = [renormalized_res_mat(A, beta=beta) for A in [A1, A2]] + else: + R1, R2 = [ + resistance_matrix(A, check_connected=check_connected) for A in [A1, A2] + ] + try: + distance_vector = np.sum((R1 - R2) ** p, axis=1) + except ValueError: + raise InputError( + "Input matrices are different sizes. Please use " + "renormalized resistance distance." + ) + if attributed: + return distance_vector ** (1 / p) + + return np.sum(distance_vector) ** (1 / p) + + +def deltacon0(A1, A2, eps=None): + """DeltaCon0 distance between two graphs. The distance is the Frobenius norm + of the element-wise square root of the fast belief propagation matrix. + + Parameters + ---------- + A1, A2 : NumPy Matrices + Adjacency matrices of graphs to be compared. + + Returns: + ------- + dist : float + DeltaCon0 distance between graphs. + + References: + ---------- + + See Also: + -------- + fast_bp + """ + # pad smaller adj. mat. so they're the same size + n1, n2 = [A.shape[0] for A in [A1, A2]] + N = max(n1, n2) + A1, A2 = [_pad(A, N) for A in [A1, A2]] + S1, S2 = [fast_bp(A, eps=eps) for A in [A1, A2]] + dist = np.abs(np.sqrt(S1) - np.sqrt(S2)).sum() + return dist diff --git a/netcomp/distance/features.py b/netcomp/distance/features.py new file mode 100644 index 00000000..57ba3d9c --- /dev/null +++ b/netcomp/distance/features.py @@ -0,0 +1,141 @@ +"""******** +Features +******** + +Calculation of features for NetSimile algorithm. +""" +from __future__ import annotations + +import networkx as nx +import numpy as np +from scipy import stats + +from ..linalg.matrices import _eps # type: ignore + + +def get_features(A): + """Feature grabber for NetSimile algorithm. Features used are + + 1. Degree of node + 2. Clustering coefficient of node + 3. Average degree of node's neighbors + 4. Average clustering coefficient of node's neighbors + 5. Number of edges in node's egonet + 6. Number of neighbors of node's egonet + 7. Number of outgoing edges from node's egonet + + Parameters + --------- + A : NumPy matrix + Adjacency matrix of graph in question. Preferably a SciPy sparse matrix + for large graphs. + + Returns: + ------- + feature_mat : NumPy array + An n by 7 array of features, where n = A.shape[0] + + References: + ----- + [Berlingerio 2012] + + """ + try: + G = nx.from_scipy_sparse_array(A) + except AttributeError: + G = nx.from_numpy_array(A) + n = len(G) + # degrees, array so we can slice nice + d_vec = np.array(list(dict(G.degree()).values())) + # list of clustering coefficient + clust_vec = np.array(list(nx.clustering(G).values())) + neighbors = [G.neighbors(i) for i in range(n)] + # average degree of neighbors (0 if node is isolated) + neighbors = [list(G.neighbors(i)) for i in range(n)] + # average degree of neighbors (0 if node is isolated) + neighbor_deg = [ + d_vec[neighbors[i]].sum() / d_vec[i] if d_vec[i] > _eps else 0 for i in range(n) + ] + # avg. clustering coefficient of neighbors (0 if node is isolated) + neighbor_clust = [ + clust_vec[neighbors[i]].sum() / d_vec[i] if d_vec[i] > _eps else 0 + for i in range(n) + ] + egonets = [nx.ego_graph(G, i) for i in range(n)] + # number of edges in egonet + ego_size = [G.number_of_edges() for G in egonets] + # number of neighbors of egonet + ego_neighbors = [ + len( + set.union(*[set(neighbors[j]) for j in egonets[i].nodes()]) + - set(egonets[i].nodes()) + ) + for i in range(n) + ] + # number of edges outgoing from egonet + outgoing_edges = [ + len( + [ + edge + for edge in G.edges(egonets[i].nodes()) + if edge[1] not in egonets[i].nodes() + ] + ) + for i in range(n) + ] + # use mat.T so that each node is a row (standard format) + feature_mat = np.array( + [ + d_vec, + clust_vec, + neighbor_deg, + neighbor_clust, + ego_size, + ego_neighbors, + outgoing_edges, + ] + ).T + return feature_mat + + +def aggregate_features(feature_mat, row_var=False, as_matrix=False): + """Returns column-wise descriptive statistics of a feature matrix. + + Parameters + ---------- + feature_mat : NumPy array + Matrix on which statistics are to be calculated. Assumed to be formatted + so each row is an observation (a node, in the case of NetSimile). + + row_var : Boolean, optional (default=False) + If true, then each variable has it's own row, and statistics are + computed along rows rather than columns. + + as_matrix : Boolean, optional (default=False) + If true, then description is returned as matrix. Otherwise, it is + flattened into a vector. + + Returns: + ------- + description : NumPy array + Descriptive statistics of feature_mat + + Notes: + ----- + + References: + ---------- + """ + axis = int(row_var) # 0 if column-oriented, 1 if not + description = np.array( + [ + feature_mat.mean(axis=axis), + np.median(feature_mat, axis=axis), + np.std(feature_mat, axis=axis), + stats.skew(feature_mat, axis=axis), + stats.kurtosis(feature_mat, axis=axis), + ] + ) + if not as_matrix: + description = description.flatten() + return description diff --git a/netcomp/exception.py b/netcomp/exception.py new file mode 100644 index 00000000..53dc70ed --- /dev/null +++ b/netcomp/exception.py @@ -0,0 +1,19 @@ +"""********** +Exceptions +********** + +Custom exceptions for NetComp. +""" +from __future__ import annotations + + +class UndefinedException(Exception): + """Raised when matrix to be returned is undefined""" + + +class InputError(Exception): + """Raised when input to algorithm is invalid""" + + +class KathleenError(Exception): + """Raised when food is gross or it is too cold""" diff --git a/netcomp/linalg/__init__.py b/netcomp/linalg/__init__.py new file mode 100644 index 00000000..8cfe75da --- /dev/null +++ b/netcomp/linalg/__init__.py @@ -0,0 +1,14 @@ +"""************** +Linear Algebra +************** + +Linear algebraic functions, calculations of important matrices. +""" +from __future__ import annotations + +from .eigenstuff import * +from .fast_bp import * +from .matrices import * + +# import helper functions for use in other places +from .resistance import * diff --git a/netcomp/linalg/eigenstuff.py b/netcomp/linalg/eigenstuff.py new file mode 100644 index 00000000..c6ddf57b --- /dev/null +++ b/netcomp/linalg/eigenstuff.py @@ -0,0 +1,133 @@ +"""********** +Eigenstuff +********** + +Functions for calculating eigenstuff of graphs. +""" + +from __future__ import annotations + +from contextlib import suppress + +import numpy as np +from numpy import linalg as la +from scipy import sparse as sps +from scipy.sparse import issparse +from scipy.sparse import linalg as spla + +from .matrices import _eps, _flat + +###################### +## Helper Functions ## +###################### + + +def _eigs(M, which="SR", k=None): + """Helper function for getting eigenstuff. + + Parameters + ---------- + M : matrix, numpy or scipy sparse + The matrix for which we hope to get eigenstuff. + which : string in {'SR','LR'} + If 'SR', get eigenvalues with smallest real part. If 'LR', get largest. + k : int + Number of eigenvalues to return + + Returns: + ------- + evals, evecs : numpy arrays + Eigenvalues and eigenvectors of matrix M, sorted in ascending or + descending order, depending on 'which'. + + See Also: + -------- + numpy.linalg.eig + scipy.sparse.eigs + """ + n, _ = M.shape + if k is None: + k = n + if which not in ["LR", "SR"]: + raise ValueError("which must be either 'LR' or 'SR'.") + M = M.astype(float) + if issparse(M) and k < n - 1: + evals, evecs = spla.eigs(M, k=k, which=which) + else: + with suppress(Exception): + M = M.todense() + + evals, evecs = la.eig(M) + # sort dem eigenvalues + inds = np.argsort(evals) + if which == "LR": + inds = inds[::-1] + else: + pass + inds = inds[:k] + evals = evals[inds] + evecs = np.matrix(evecs[:, inds]) + return np.real(evals), np.real(evecs) + + +##################### +## Get Eigenstuff ## +##################### + + +def normalized_laplacian_eig(A, k=None): + """Return the eigenstuff of the normalized Laplacian matrix of graph + associated with adjacency matrix A. + + Calculates via eigenvalues if + + K = D^(-1/2) A D^(-1/2) + + where `A` is the adjacency matrix and `D` is the diagonal matrix of + node degrees. Since L = I - K, the eigenvalues and vectors of L can + be easily recovered. + + Parameters + ---------- + A : NumPy matrix + Adjacency matrix of a graph + + k : int, 0 < k < A.shape[0]-1 + The number of eigenvalues to grab. + + Returns: + ------- + lap_evals : NumPy array + Eigenvalues of L + + evecs : NumPy matrix + Columns are the eigenvectors of L + + Notes: + ----- + This way of calculating the eigenvalues of the normalized graph laplacian is + more numerically stable than simply forming the matrix L = I - K and doing + numpy.linalg.eig on the result. This is because the eigenvalues of L are + close to zero, whereas the eigenvalues of K are close to 1. + + References: + ---------- + + See Also: + -------- + nx.laplacian_matrix + nx.normalized_laplacian_matrix + """ + n, m = A.shape + ## + ## TODO: implement checks on the adjacency matrix + ## + degs = _flat(A.sum(axis=1)) + # the below will break if + inv_root_degs = [d ** (-1 / 2) if d > _eps else 0 for d in degs] + inv_rootD = sps.spdiags(inv_root_degs, [0], n, n, format="csr") + # build normalized diffusion matrix + K = inv_rootD * A * inv_rootD + evals, evecs = _eigs(K, k=k, which="LR") + lap_evals = 1 - evals + return np.real(lap_evals), np.real(evecs) diff --git a/netcomp/linalg/fast_bp.py b/netcomp/linalg/fast_bp.py new file mode 100644 index 00000000..b1d1e8b2 --- /dev/null +++ b/netcomp/linalg/fast_bp.py @@ -0,0 +1,57 @@ +"""*********************** +Fast Belief Propagation +*********************** + +The fast approximation of the Belief propagation matrix. +""" + +from __future__ import annotations + +import numpy as np +from numpy import linalg as la +from scipy import sparse as sps + + +def fast_bp(A, eps=None): + """Return the fast belief propagation matrix of graph associated with A. + + Parameters + ---------- + A : NumPy matrix or Scipy sparse matrix + Adjacency matrix of a graph. If sparse, can be any format; CSC or CSR + recommended. + + eps : float, optional (default=None) + Small parameter used in calculation of matrix. If not provided, it is + set to 1/(1+d_max) where d_max is the maximum degree. + + Returns: + ------- + S : NumPy matrix or Scipy sparse matrix + The fast belief propagation matrix. If input is sparse, will be returned + as (sparse) CSC matrix. + + Notes: + ----- + + References: + ---------- + + """ + n, m = A.shape + ## + ## TODO: implement checks on the adjacency matrix + ## + degs = np.array(A.sum(axis=1)).flatten() + if eps is None: + eps = 1 / (1 + max(degs)) + identity = sps.identity(n) + D = sps.dia_matrix((degs, [0]), shape=(n, n)) + # form inverse of S and invert (slow!) + Sinv = identity + eps**2 * D - eps * A + try: + S = la.inv(Sinv) + except Exception: + Sinv = sps.csc_matrix(Sinv) + S = sps.linalg.inv(Sinv) + return S diff --git a/netcomp/linalg/matrices.py b/netcomp/linalg/matrices.py new file mode 100644 index 00000000..e5a6207d --- /dev/null +++ b/netcomp/linalg/matrices.py @@ -0,0 +1,94 @@ +"""******** +Matrices +******** + +Matrices associated with graphs. Also contains linear algebraic helper functions. +""" +from __future__ import annotations + +import numpy as np +from scipy import sparse as sps +from scipy.sparse import issparse + +_eps = 10 ** (-10) # a small parameter + +###################### +## Helper Functions ## +###################### + + +def _flat(D): + """Flatten column or row matrices, as well as arrays.""" + if issparse(D): + raise ValueError("Cannot flatten sparse matrix.") + d_flat = np.array(D).flatten() + return d_flat + + +def _pad(A, N): + """Pad A so A.shape is (N,N)""" + n, _ = A.shape + if n >= N: + return A + else: + if issparse(A): + # thrown if we try to np.concatenate sparse matrices + side = sps.csr_matrix((n, N - n)) + bottom = sps.csr_matrix((N - n, N)) + A_pad = sps.hstack([A, side]) + A_pad = sps.vstack([A_pad, bottom]) + else: + side = np.zeros((n, N - n)) + bottom = np.zeros((N - n, N)) + A_pad = np.concatenate([A, side], axis=1) + A_pad = np.concatenate([A_pad, bottom]) + return A_pad + + +######################## +## Matrices of Graphs ## +######################## + + +def degree_matrix(A): + """Diagonal degree matrix of graph with adjacency matrix A + + Parameters + ---------- + A : matrix + Adjacency matrix + + Returns: + ------- + D : SciPy sparse matrix + Diagonal matrix of degrees. + """ + n, m = A.shape + degs = _flat(A.sum(axis=1)) + D = sps.spdiags(degs, [0], n, n, format="csr") + return D + + +def laplacian_matrix(A, normalized=False): + """Diagonal degree matrix of graph with adjacency matrix A + + Parameters + ---------- + A : matrix + Adjacency matrix + normalized : Bool, optional (default=False) + If true, then normalized laplacian is returned. + + Returns: + ------- + L : SciPy sparse matrix + Combinatorial laplacian matrix. + """ + n, m = A.shape + D = degree_matrix(A) + L = D - A + if normalized: + degs = _flat(A.sum(axis=1)) + rootD = sps.spdiags(np.power(degs, -1 / 2), [0], n, n, format="csr") + L = rootD * L * rootD + return L diff --git a/netcomp/linalg/resistance.py b/netcomp/linalg/resistance.py new file mode 100644 index 00000000..e24c2265 --- /dev/null +++ b/netcomp/linalg/resistance.py @@ -0,0 +1,260 @@ +"""********** +Resistance +********** + +Resistance matrix. Renormalized version, as well as conductance and commute matrices. +""" +from __future__ import annotations + +from contextlib import suppress + +import networkx as nx +import numpy as np +from numpy import linalg as la +from scipy import linalg as spla +from scipy.sparse import issparse + +from ..exception import UndefinedException +from .matrices import laplacian_matrix + + +def resistance_matrix(A, check_connected=True): + """Return the resistance matrix of G. + + Parameters + ---------- + A : NumPy matrix or SciPy sparse matrix + Adjacency matrix of a graph. + + check_connected : Boolean, optional (default=True) + If false, then the resistance matrix will be computed even for + disconnected matrices. See Notes. + + Returns: + ------- + R : NumPy matrix + Matrix of pairwise resistances between nodes. + + Notes: + ----- + Uses formula for resistance matrix R in terms of Moore-Penrose of + pseudoinverse (non-normalized) graph Laplacian. See e.g. Theorem 2.1 in [1]. + + This formula can be computed even for disconnected graphs, although the + interpretation in this case is unclear. Thus, the usage of + check_connected=False is recommended only to reduce computation time in a + scenario in which the user is confident the graph in question is, in fact, + connected. + + Since we do not expect the pseudoinverse of the laplacian to be sparse, we + convert L to dense form before running np.linalg.pinv(). The returned + resistance matrix is dense. + + See Also: + -------- + nx.laplacian_matrix + + References: + ---------- + .. [1] W. Ellens, et al. (2011) + Effective graph resistance. + Linear Algebra and its Applications, 435 (2011) + + """ + n, m = A.shape + # check if graph is connected + if check_connected: + if issparse(A): + G = nx.from_scipy_sparse_array(A) + else: + G = nx.from_numpy_array(A) + if not nx.is_connected(G): + raise UndefinedException( + "Graph is not connected. " "Resistance matrix is undefined." + ) + L = laplacian_matrix(A) + with suppress(Exception): + L = L.todense() + + M = la.pinv(L) + # calculate R in terms of M + d = np.reshape(np.diag(M), (n, 1)) + ones = np.ones((n, 1)) + R = np.dot(d, ones.T) + np.dot(ones, d.T) - M - M.T + return R + + +def commute_matrix(A): + """Return the commute matrix of the graph associated with adj. matrix A. + + Parameters + ---------- + A : NumPy matrix or SciPy sparse matrix + Adjacency matrix of a graph. + + Returns: + ------- + C : NumPy matrix + Matrix of pairwise resistances between nodes. + + Notes: + ----- + Uses formula for commute time matrix in terms of resistance matrix, + + C = R*2*|E| + + where |E| is the number of edges in G. See e.g. Theorem 2.8 in [1]. + + See Also: + -------- + laplacian_matrix + resistance_matrix + + References: + ---------- + .. [1] W. Ellens, et al. (2011) + Effective graph resistance. + Linear Algebra and its Applications, 435 (2011) + + """ + R = resistance_matrix(A) + E = A.sum() / 2 # number of edges in graph + C = 2 * E * R + return C + + +def renormalized_res_mat(A, beta=1): + """Return the renormalized resistance matrix of graph associated with A. + + To renormalize a resistance R, we apply the function + + R' = R / (R + beta) + + In this way, the renormalized resistance of nodes in disconnected components + is 1. The parameter beta determines the penalty for disconnection. If we set + beta to be approximately the maximum resistance found in the network, then + the penalty for disconnection is at least 1/2. + + Parameters + ---------- + A : NumPy matrix or SciPy sparse matrix + Adjacency matrix of a graph. + + nodelist : list, optional + The rows and columns are ordered according to the nodes in nodelist. If + nodelist is None, then the ordering is produced by G.nodes(). + + weight : string or None, optional (default='weight') + The edge data key used to compute each value in the matrix. + If None, then each edge has weight 1. + + beta : float, optional + Scaling parameter in renormalization. Must be greater than or equal to + 1. Determines how heavily disconnection is penalized. + + Returns: + ------- + R : NumPy array + Matrix of pairwise renormalized resistances between nodes. + + Notes: + ----- + This function converts to a NetworkX graph, as it uses the algorithms + therein for identifying connected components. + + See Also: + -------- + resistance_matrix + + """ + if issparse(A): + G = nx.from_scipy_sparse_array(A) + else: + G = nx.from_numpy_array(A) + + if isinstance(G, nx.Graph): + cc = nx.connected_components + else: + cc = nx.weakly_connected_components + + n = len(G) + subgraphR = [] + for c in cc(G): + subgraph = G.subgraph(c).copy() + a_sub = nx.adjacency_matrix(subgraph) + r_sub = resistance_matrix(a_sub) + subgraphR.append(r_sub) + R = spla.block_diag(*subgraphR) + # now, resort R so that it matches the original node list + component_order = [] + for component in cc(G): + component_order += list(component) + component_order = list(np.argsort(component_order)) + R = R[component_order, :] + R = R[:, component_order] + renorm = np.vectorize(lambda r: r / (r + beta)) + R = renorm(R) + # set resistance for different components to 1 + R[R == 0] = 1 + R = R - np.eye(n) # don't want diagonal to be 1 + return R + + +def conductance_matrix(A): + """Return the conductance matrix of G. + + The conductance matrix of G is the element-wise inverse of the resistance + matrix. The diagonal is set to 0, although it is formally infinite. Nodes in + disconnected components have 0 conductance. + + Parameters + ---------- + G : graph + A NetworkX graph + + nodelist : list, optional + The rows and columns are ordered according to the nodes in nodelist. If + nodelist is None, then the ordering is produced by G.nodes(). + + weight : string or None, optional (default='weight') + The edge data key used to compute each value in the matrix. + If None, then each edge has weight 1. + + Returns: + ------- + C : NumPy array + Matrix of pairwise conductances between nodes. + + + See Also: + -------- + resistance_matrix + renormalized_res_mat + + """ + if issparse(A): + G = nx.from_scipy_sparse_array(A) + else: + G = nx.from_numpy_array(A) + if isinstance(G, nx.Graph): + cc = nx.connected_components + else: + cc = nx.weakly_connected_components + subgraphC = [] + for c in cc(G): + subgraph = G.subgraph(c).copy() + a_sub = nx.adjacency_matrix(subgraph) + r_sub = resistance_matrix(a_sub) + m = len(subgraph) + # add one to diagonal, invert, remove one from diagonal: + c_sub = 1 / (r_sub + np.eye(m)) - np.eye(m) + subgraphC.append(c_sub) + C = spla.block_diag(*subgraphC) + # resort C so that it matches the original node list + component_order = [] + for component in cc(G): + component_order += list(component) + component_order = list(np.argsort(component_order)) + C = C[component_order, :] + C = C[:, component_order] + return C diff --git a/pyproject.toml b/pyproject.toml index 98a9d170..58115f86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,43 +1,46 @@ [build-system] -build-backend = "setuptools.build_meta" +build-backend = "hatchling.build" requires = [ - "setuptools", - "setuptools-scm", + "hatch-vcs", + "hatchling", ] -[tool.setuptools.packages.find] -exclude = ["htmlcov"] # Exclude the coverage report file from setuptools package finder - [project] name = "swmmanywhere" -version = "0.0.1" authors = [ { name = "Barnaby Dobson", email = "b.dobson@imperial.ac.uk" }, { name = "Imperial College London RSE Team", email = "ict-rse-team@imperial.ac.uk" } ] requires-python = ">=3.10" classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: GIS", + "Typing :: Typed", +] +dynamic = [ + "version", ] dependencies = [ "cdsapi", "cytoolz", - "fastparquet", - "fiona", - "geopandas", + "geopandas>=1", "geopy", - "GitPython", "joblib", "jsonschema", "loguru", "netcdf4", - "netcomp@ git+https://github.com/barneydobson/NetComp.git", "networkx>=3", - "numpy", - "osmnx", + "numpy>=2", + "osmnx>=1.9.3", "pandas", "planetary_computer", "pyarrow", @@ -45,7 +48,7 @@ dependencies = [ "pyflwdir", "pystac_client", "pyswmm", - "pywbt", + "pywbt>=0.2.2", "PyYAML", "rasterio", "rioxarray", @@ -74,29 +77,28 @@ doc = [ "mkdocs-material-extensions", "mkdocstrings[python]", ] +[project.urls] +Documentation = "https://imperialcollegelondon.github.io/SWMManywhere/" +Issues = "https://github.com/ImperialCollegeLondon/SWMManywhere/issues" +Source = "https://github.com/ImperialCollegeLondon/SWMManywhere" -[tool.mypy] -disallow_any_explicit = false -disallow_any_generics = false -warn_unreachable = true -warn_unused_ignores = false -disallow_untyped_defs = false -exclude = [".venv/"] +[tool.hatch.build.targets.sdist] +only-include = ["swmmanywhere", "netcomp"] -[[tool.mypy.overrides]] -module = "tests.*" -disallow_untyped_defs = false +[tool.hatch.version] +source = "vcs" -[tool.pytest.ini_options] -addopts = "-v -p no:warnings --cov=swmmanywhere --cov-report=html --doctest-modules --ignore=swmmanywhere/logging.py" +[tool.hatch.build.hooks.vcs] +version-file = "_version.py" -[tool.ruff] +[tool.ruff.lint] select = ["D", "E", "F", "I"] # pydocstyle, pycodestyle, Pyflakes, isort -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "tests/*" = ["D100", "D104"] +"netcomp/*" = ["D", "F"] # Ignore all checks for netcomp -[tool.ruff.pydocstyle] +[tool.ruff.lint.pydocstyle] convention = "google" [tool.ruff.lint.isort] @@ -106,6 +108,24 @@ required-imports = ["from __future__ import annotations"] skip = "swmmanywhere/defs/iso_converter.yml,*.inp" ignore-words-list = "gage,gages" +[tool.pytest.ini_options] +addopts = "-v --import-mode=importlib --cov=swmmanywhere --cov-report=html --doctest-modules --ignore=swmmanywhere/logging.py" +markers = [ + "downloads: mark a test as requiring downloads", +] + +[tool.mypy] +disallow_any_explicit = false +disallow_any_generics = false +warn_unreachable = true +warn_unused_ignores = false +disallow_untyped_defs = false +exclude = [".venv/"] + +[[tool.mypy.overrides]] +module = "tests.*" +disallow_untyped_defs = false + [tool.refurb] ignore = [ 184, # Because some frankly bizarre suggestions diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 5f23ec4f..00000000 --- a/requirements.txt +++ /dev/null @@ -1,262 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile -# -aenum==3.1.11 - # via - # pyswmm - # swmm-toolkit -affine==2.4.0 - # via - # pyflwdir - # rasterio -annotated-types==0.7.0 - # via pydantic -attrs==23.2.0 - # via - # cads-api-client - # fiona - # jsonschema - # rasterio - # referencing -cads-api-client==1.2.0 - # via cdsapi -cdsapi==0.7.0 - # via swmmanywhere (pyproject.toml) -certifi==2024.7.4 - # via - # fiona - # netcdf4 - # pyogrio - # pyproj - # rasterio - # requests -cftime==1.6.4 - # via netcdf4 -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # click-plugins - # cligj - # fiona - # planetary-computer - # rasterio -click-plugins==1.1.1 - # via - # fiona - # rasterio -cligj==0.7.2 - # via - # fiona - # rasterio -colorama==0.4.6 - # via - # click - # loguru - # tqdm -cramjam==2.8.3 - # via fastparquet -cytoolz==0.12.3 - # via swmmanywhere (pyproject.toml) -fastparquet==2024.5.0 - # via swmmanywhere (pyproject.toml) -fiona==1.9.6 - # via swmmanywhere (pyproject.toml) -fsspec==2024.6.1 - # via fastparquet -geographiclib==2.0 - # via geopy -geopandas==1.0.1 - # via - # osmnx - # swmmanywhere (pyproject.toml) -geopy==2.4.1 - # via swmmanywhere (pyproject.toml) -gitdb==4.0.11 - # via gitpython -gitpython==3.1.43 - # via swmmanywhere (pyproject.toml) -idna==3.8 - # via requests -joblib==1.4.2 - # via swmmanywhere (pyproject.toml) -jsonschema==4.23.0 - # via - # pystac - # swmmanywhere (pyproject.toml) -jsonschema-specifications==2023.12.1 - # via jsonschema -julian==0.14 - # via pyswmm -llvmlite==0.43.0 - # via numba -loguru==0.7.2 - # via swmmanywhere (pyproject.toml) -multiurl==0.3.1 - # via cads-api-client -netcdf4==1.7.1.post1 - # via swmmanywhere (pyproject.toml) -netcomp @ git+https://github.com/barneydobson/NetComp.git - # via swmmanywhere (pyproject.toml) -networkx==3.3 - # via - # netcomp - # osmnx - # swmmanywhere (pyproject.toml) -numba==0.60.0 - # via pyflwdir -numpy==1.26.4 - # via - # cftime - # fastparquet - # geopandas - # netcdf4 - # netcomp - # numba - # osmnx - # pandas - # pyarrow - # pyflwdir - # pyogrio - # rasterio - # rioxarray - # scipy - # shapely - # snuggs - # swmmanywhere (pyproject.toml) - # xarray -osmnx==1.9.3 - # via swmmanywhere (pyproject.toml) -packaging==24.1 - # via - # fastparquet - # geopandas - # planetary-computer - # pyogrio - # pyswmm - # rioxarray - # xarray -pandas==2.2.2 - # via - # fastparquet - # geopandas - # osmnx - # swmmanywhere (pyproject.toml) - # xarray -planetary-computer==1.0.0 - # via swmmanywhere (pyproject.toml) -pyarrow==16.1.0 - # via swmmanywhere (pyproject.toml) -pydantic==2.8.2 - # via - # planetary-computer - # swmmanywhere (pyproject.toml) -pydantic-core==2.20.1 - # via pydantic -pyflwdir==0.5.8 - # via swmmanywhere (pyproject.toml) -pyogrio==0.9.0 - # via geopandas -pyparsing==3.1.2 - # via snuggs -pyproj==3.6.1 - # via - # geopandas - # rioxarray -pystac[validation]==1.10.1 - # via - # planetary-computer - # pystac-client -pystac-client==0.8.3 - # via - # planetary-computer - # swmmanywhere (pyproject.toml) -pyswmm==2.0.1 - # via swmmanywhere (pyproject.toml) -python-dateutil==2.9.0.post0 - # via - # multiurl - # pandas - # pystac - # pystac-client -python-dotenv==1.0.1 - # via planetary-computer -pytz==2024.1 - # via - # multiurl - # pandas - # planetary-computer -pywbt==0.1.1 - # via swmmanywhere (pyproject.toml) -pyyaml==6.0.1 - # via swmmanywhere (pyproject.toml) -rasterio==1.3.10 - # via - # rioxarray - # swmmanywhere (pyproject.toml) -referencing==0.35.1 - # via - # jsonschema - # jsonschema-specifications -requests==2.32.3 - # via - # cads-api-client - # cdsapi - # multiurl - # osmnx - # planetary-computer - # pystac-client -rioxarray==0.17.0 - # via swmmanywhere (pyproject.toml) -rpds-py==0.19.1 - # via - # jsonschema - # referencing -scipy==1.14.0 - # via - # netcomp - # pyflwdir - # swmmanywhere (pyproject.toml) -shapely==2.0.5 - # via - # geopandas - # osmnx - # swmmanywhere (pyproject.toml) -six==1.16.0 - # via - # fiona - # python-dateutil -smmap==5.0.1 - # via gitdb -snuggs==1.4.7 - # via rasterio -swmm-toolkit==0.15.5 - # via pyswmm -toolz==0.12.1 - # via cytoolz -tqdm==4.66.4 - # via - # cdsapi - # multiurl - # swmmanywhere (pyproject.toml) -typing-extensions==4.12.2 - # via - # cads-api-client - # pydantic - # pydantic-core -tzdata==2024.1 - # via pandas -urllib3==2.2.2 - # via requests -win32-setctime==1.1.0 - # via loguru -xarray==2024.6.0 - # via - # rioxarray - # swmmanywhere (pyproject.toml) - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/swmmanywhere/__init__.py b/swmmanywhere/__init__.py index 6da981e7..65cd1222 100644 --- a/swmmanywhere/__init__.py +++ b/swmmanywhere/__init__.py @@ -1,7 +1,12 @@ """The main module for MyProject.""" from __future__ import annotations -__version__ = "0.1.0" +from importlib.metadata import PackageNotFoundError, version + +try: + __version__ = version("swmmanywhere") +except PackageNotFoundError: + __version__ = "999" # Importing module to register the graphfcns and made them available from . import graphfcns # noqa: F401 diff --git a/swmmanywhere/geospatial_utilities.py b/swmmanywhere/geospatial_utilities.py index c990d970..c5c83ecd 100644 --- a/swmmanywhere/geospatial_utilities.py +++ b/swmmanywhere/geospatial_utilities.py @@ -642,10 +642,12 @@ def flwdir_whitebox(fid: Path) -> np.array: "D8Pointer": ["-i=dem_corr.tif", "-o=fdir.tif"], } whitebox_tools( + temp_path, wbt_args, - work_dir=temp_path, + save_dir=temp_path, verbose=verbose(), wbt_root=temp_path / "WBT", + zip_path=fid.parent / "whiteboxtools_binaries.zip", max_procs=1, ) diff --git a/swmmanywhere/graphfcns/topology_graphfcns.py b/swmmanywhere/graphfcns/topology_graphfcns.py index 292dbdfc..b3ab4c3d 100644 --- a/swmmanywhere/graphfcns/topology_graphfcns.py +++ b/swmmanywhere/graphfcns/topology_graphfcns.py @@ -209,7 +209,7 @@ def __call__( G (nx.Graph): A graph """ # Calculate bounds to normalise between - bounds: Dict[Any, List[float]] = defaultdict(lambda: [np.Inf, -np.Inf]) + bounds: Dict[Any, List[float]] = defaultdict(lambda: [np.inf, -np.inf]) for w in topology_derivation.weights: bounds[w][0] = min(nx.get_edge_attributes(G, w).values()) # lower bound diff --git a/swmmanywhere/metric_utilities.py b/swmmanywhere/metric_utilities.py index 0e6d57ae..b469d625 100644 --- a/swmmanywhere/metric_utilities.py +++ b/swmmanywhere/metric_utilities.py @@ -12,13 +12,13 @@ import cytoolz.curried as tlz import geopandas as gpd import joblib -import netcomp import networkx as nx import numpy as np import pandas as pd import shapely from scipy import stats +import netcomp from swmmanywhere.logging import logger from swmmanywhere.parameters import MetricEvaluation diff --git a/swmmanywhere/post_processing.py b/swmmanywhere/post_processing.py index 8c95271b..2c78e1fb 100644 --- a/swmmanywhere/post_processing.py +++ b/swmmanywhere/post_processing.py @@ -433,7 +433,8 @@ def _fill_backslash_columns( # Extract SWMM order and default values columns = conversion_dict[key]["iwcolumns"] - shp = shp.fillna(0) + numeric_cols = shp.select_dtypes(include=[np.number]).columns + shp[numeric_cols] = shp[numeric_cols].fillna(0) # Find columns with a default specified cols_default = [c[1:] for c in columns if c.startswith("/")] diff --git a/tests/test_swmmanywhere.py b/tests/test_swmmanywhere.py index 4886beef..afa91d2a 100644 --- a/tests/test_swmmanywhere.py +++ b/tests/test_swmmanywhere.py @@ -9,16 +9,11 @@ import pytest import yaml -from swmmanywhere import __version__, swmmanywhere +from swmmanywhere import swmmanywhere from swmmanywhere.graph_utilities import graphfcns from swmmanywhere.metric_utilities import metrics -def test_version(): - """Check that the version is acceptable.""" - assert __version__ == "0.1.0" - - def test_run(): """Test the run function.""" demo_dir = Path(__file__).parent.parent / "swmmanywhere" / "defs"