diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 594ba8c3c4..dc12ab2ade 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -33,5 +33,5 @@ ENV PYTHONDONTWRITEBYTECODE="1" ENV SCCACHE_REGION="us-east-2" ENV SCCACHE_BUCKET="rapids-sccache-devs" -ENV VAULT_HOST="https://vault.ops.k8s.rapids.ai" +ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs" ENV HISTFILE="/home/coder/.cache/._bash_history" diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json index a2b12764c5..008bf8730a 100644 --- a/.devcontainer/cuda11.8-conda/devcontainer.json +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.10-cpp-cuda11.8-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.12-cpp-cuda11.8-mambaforge-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda11.8-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda11.8-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index f319536b18..75aed80f9f 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -5,24 +5,24 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.10-cpp-cuda11.8-ucx1.17.0-openmpi-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.12-cpp-cuda11.8-ucx1.17.0-openmpi-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda11.8-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda11.8-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/cuda:24.10": { + "ghcr.io/rapidsai/devcontainers/features/cuda:24.12": { "version": "11.8", "installcuBLAS": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true }, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/ucx", diff --git a/.devcontainer/cuda12.5-conda/devcontainer.json b/.devcontainer/cuda12.5-conda/devcontainer.json index adc79408a3..240ba02131 100644 --- a/.devcontainer/cuda12.5-conda/devcontainer.json +++ b/.devcontainer/cuda12.5-conda/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.10-cpp-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.12-cpp-mambaforge-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.5-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda12.5-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda12.5-pip/devcontainer.json b/.devcontainer/cuda12.5-pip/devcontainer.json index 26b9a56e48..c23c79017a 100644 --- a/.devcontainer/cuda12.5-pip/devcontainer.json +++ b/.devcontainer/cuda12.5-pip/devcontainer.json @@ -5,24 +5,24 @@ "args": { "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.10-cpp-cuda12.5-ucx1.17.0-openmpi-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.12-cpp-cuda12.5-ucx1.17.0-openmpi-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.5-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.12-cuda12.5-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/cuda:24.10": { + "ghcr.io/rapidsai/devcontainers/features/cuda:24.12": { "version": "12.5", "installcuBLAS": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true }, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/ucx", diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2b0ae5099c..945589dc12 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,18 +46,17 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - skip_upload_pkgs: libraft-template docs-build: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +68,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibraft: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -79,7 +78,7 @@ jobs: wheel-publish-pylibraft: needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -88,7 +87,7 @@ jobs: package-name: pylibraft wheel-build-raft-dask: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -98,7 +97,7 @@ jobs: wheel-publish-raft-dask: needs: wheel-build-raft-dask secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 381ca6b378..9c22edf74c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,6 +12,7 @@ concurrency: jobs: pr-builder: needs: + - changed-files - checks - conda-cpp-build - conda-cpp-tests @@ -25,49 +26,77 @@ jobs: - wheel-tests-raft-dask - devcontainer secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 + if: always() + with: + needs: ${{ toJSON(needs) }} + changed-files: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.12 + with: + files_yaml: | + test_cpp: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!python/**' + - '!thirdparty/LICENSES/**' + test_python: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!thirdparty/LICENSES/**' checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 with: build_type: pull-request node_type: cpu16 conda-cpp-tests: - needs: conda-cpp-build + needs: [conda-cpp-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp with: build_type: pull-request conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 with: build_type: pull-request enable_check_symbols: true - symbol_exclusions: raft_cutlass conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 with: build_type: pull-request conda-python-tests: - needs: conda-python-build + needs: [conda-python-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -77,38 +106,40 @@ jobs: wheel-build-pylibraft: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_pylibraft.sh wheel-tests-pylibraft: - needs: wheel-build-pylibraft + needs: [wheel-build-pylibraft, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request script: ci/test_wheel_pylibraft.sh wheel-build-raft-dask: needs: wheel-tests-pylibraft secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: "ci/build_wheel_raft_dask.sh" wheel-tests-raft-dask: - needs: wheel-build-raft-dask + needs: [wheel-build-raft-dask, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request script: ci/test_wheel_raft_dask.sh devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 with: arch: '["amd64"]' cuda: '["12.5"]' build_command: | sccache -z; - build-all -DBUILD_PRIMS_BENCH=ON -DBUILD_ANN_BENCH=ON --verbose; + build-all -DBUILD_PRIMS_BENCH=ON --verbose; sccache -s; diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ad0456d526..92020f6a76 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,17 +16,16 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} enable_check_symbols: true - symbol_exclusions: raft_cutlass conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +33,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -42,7 +41,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibraft: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -51,7 +50,7 @@ jobs: script: ci/test_wheel_pylibraft.sh wheel-tests-raft-dask: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/.gitignore b/.gitignore index 11b7bc3eba..3d6c84a83f 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,6 @@ log dask-worker-space/ *.egg-info/ *.bin -bench/ann/data temporary_*.json ## scikit-build diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 458d8b1b51..e3b3c8c440 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,7 +62,7 @@ repos: entry: ./cpp/scripts/run-cmake-format.sh cmake-format language: python types: [cmake] - exclude: .*/thirdparty/.*|.*FindAVX.cmake.* + exclude: .*/thirdparty/.* # Note that pre-commit autoupdate does not update the versions # of dependencies, so we'll have to update this manually. additional_dependencies: @@ -93,7 +93,10 @@ repos: - id: codespell additional_dependencies: [tomli] args: ["--toml", "pyproject.toml"] - exclude: (?x)^(^CHANGELOG.md$) + exclude: | + (?x) + ^CHANGELOG[.]md$| + ^cpp/cmake/patches/cutlass/build-export[.]patch$ - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: @@ -114,10 +117,9 @@ repos: cpp/include/raft/neighbors/detail/faiss_select/| cpp/include/raft/thirdparty/| docs/source/sphinxext/github_link[.]py| - cpp/cmake/modules/FindAVX[.]cmake - id: verify-alpha-spec - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.13.11 + rev: v1.16.0 hooks: - id: rapids-dependency-file-generator args: ["--clean"] diff --git a/CHANGELOG.md b/CHANGELOG.md index 9caa5ef571..1d7c641b21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,42 @@ +# raft 24.12.00 (11 Dec 2024) + +## 🚨 Breaking Changes + +- Do not initialize the pinned mdarray at construction time ([#2478](https://github.com/rapidsai/raft/pull/2478)) [@achirkin](https://github.com/achirkin) + +## 🐛 Bug Fixes + +- Skip gtests for new lanczos solver when CUDA version is 11.4 or below. ([#2520](https://github.com/rapidsai/raft/pull/2520)) [@cjnolet](https://github.com/cjnolet) +- Switch `assert` to `static_assert` ([#2510](https://github.com/rapidsai/raft/pull/2510)) [@divyegala](https://github.com/divyegala) +- Revert use of new Lanczos solver in spectral clustering ([#2507](https://github.com/rapidsai/raft/pull/2507)) [@lowener](https://github.com/lowener) +- Put a ceiling on cuda-python ([#2486](https://github.com/rapidsai/raft/pull/2486)) [@bdice](https://github.com/bdice) +- Don't presume pointers location infers usability. ([#2480](https://github.com/rapidsai/raft/pull/2480)) [@robertmaynard](https://github.com/robertmaynard) +- Use Python for sccache hit rate computation. ([#2474](https://github.com/rapidsai/raft/pull/2474)) [@bdice](https://github.com/bdice) +- Allow compilation with CUDA 12.6.1 ([#2469](https://github.com/rapidsai/raft/pull/2469)) [@robertmaynard](https://github.com/robertmaynard) + +## 🚀 New Features + +- [FEA] Lanczos solver v2 ([#2481](https://github.com/rapidsai/raft/pull/2481)) [@lowener](https://github.com/lowener) + +## 🛠️ Improvements + +- Skip gtests for Rmat Lanczos tests with cuda <= 11.4 ([#2525](https://github.com/rapidsai/raft/pull/2525)) [@benfred](https://github.com/benfred) +- Upgrade to latest cutlass version ([#2503](https://github.com/rapidsai/raft/pull/2503)) [@vyasr](https://github.com/vyasr) +- Removing some left over places where implicit instantiations were being ignored in headers ([#2501](https://github.com/rapidsai/raft/pull/2501)) [@cjnolet](https://github.com/cjnolet) +- Remove leftover template project code. ([#2500](https://github.com/rapidsai/raft/pull/2500)) [@bdice](https://github.com/bdice) +- 2412 remove libraft vss instantiations ([#2498](https://github.com/rapidsai/raft/pull/2498)) [@cjnolet](https://github.com/cjnolet) +- Remove raft-ann-bench ([#2497](https://github.com/rapidsai/raft/pull/2497)) [@cjnolet](https://github.com/cjnolet) +- Pin FAISS Version for raft-ann-bench ([#2496](https://github.com/rapidsai/raft/pull/2496)) [@tarang-jain](https://github.com/tarang-jain) +- enforce wheel size limits and README formatting in CI, put a ceiling on Cython dependency ([#2490](https://github.com/rapidsai/raft/pull/2490)) [@jameslamb](https://github.com/jameslamb) +- Do not initialize the pinned mdarray at construction time ([#2478](https://github.com/rapidsai/raft/pull/2478)) [@achirkin](https://github.com/achirkin) +- Use environment variables in cache hit rate computation. ([#2475](https://github.com/rapidsai/raft/pull/2475)) [@bdice](https://github.com/bdice) +- devcontainer: replace `VAULT_HOST` with `AWS_ROLE_ARN` ([#2472](https://github.com/rapidsai/raft/pull/2472)) [@jjacobelli](https://github.com/jjacobelli) +- print sccache stats in builds ([#2470](https://github.com/rapidsai/raft/pull/2470)) [@jameslamb](https://github.com/jameslamb) +- make package installations in CI stricter ([#2467](https://github.com/rapidsai/raft/pull/2467)) [@jameslamb](https://github.com/jameslamb) +- Prune workflows based on changed files ([#2466](https://github.com/rapidsai/raft/pull/2466)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA) +- Merge branch-24.10 into branch-24.12 ([#2461](https://github.com/rapidsai/raft/pull/2461)) [@jameslamb](https://github.com/jameslamb) +- Update all rmm imports to use pylibrmm/librmm ([#2451](https://github.com/rapidsai/raft/pull/2451)) [@Matt711](https://github.com/Matt711) + # raft 24.10.00 (9 Oct 2024) ## 🚨 Breaking Changes diff --git a/README.md b/README.md index af2219fdd1..898c5c22c3 100755 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ #
 RAFT: Reusable Accelerated Functions and Tools for Vector Search and More
> [!IMPORTANT] -> The vector search and clustering algorithms in RAFT are being migrated to a new library dedicated to vector search called [cuVS](https://github.com/rapidsai/cuvs). We will continue to support the vector search algorithms in RAFT during this move, but will no longer update them after the RAPIDS 24.06 (June) release. We plan to complete the migration by RAPIDS 24.10 (October) release and will be removing them altogether in the 24.12 (December) release. +> The vector search and clustering algorithms in RAFT have been formally migrated to a new library dedicated to vector search called [cuVS](https://github.com/rapidsai/cuvs). The headers for the vector search and clustering algorithms in RAFT will remain for a bried period, but will no longer be tested, benchmarked, included in the pre-compiled libraft binary, or otherwise updated after the 24.12 (December 2024) release. We will be removing these headers altogether in a future release. It is strongly suggested to use cuVS for these routines, which include any headers in the `distance`, `neighbors`, `cluster` and `spatial` directories, and use the RAFT versions at your own risk. ![RAFT tech stack](img/raft-tech-stack-vss.png) @@ -27,7 +27,6 @@ - [RAFT Reference Documentation](https://docs.rapids.ai/api/raft/stable/): API Documentation. - [RAFT Getting Started](./docs/source/quick_start.md): Getting started with RAFT. - [Build and Install RAFT](./docs/source/build.md): Instructions for installing and building RAFT. -- [Example Notebooks](./notebooks): Example jupyter notebooks - [RAPIDS Community](https://rapids.ai/community.html): Get help, contribute, and collaborate. - [GitHub repository](https://github.com/rapidsai/raft): Download the RAFT source code. - [Issue tracker](https://github.com/rapidsai/raft/issues): Report issues or request features. @@ -120,13 +119,13 @@ auto metric = raft::distance::DistanceType::L2SqrtExpanded; raft::distance::pairwise_distance(handle, input.view(), input.view(), output.view(), metric); ``` -It's also possible to create `raft::device_mdspan` views to invoke the same API with raw pointers and shape information: +It's also possible to create `raft::device_mdspan` views to invoke the same API with raw pointers and shape information. Take this example from the [NVIDIA cuVS](https://github.com/rapidsai/cuvs) library: ```c++ #include #include #include -#include +#include raft::device_resources handle; @@ -147,8 +146,8 @@ auto output_view = raft::make_device_matrix_view(output, n_samples, n_samples); raft::random::make_blobs(handle, input_view, labels_view); -auto metric = raft::distance::DistanceType::L2SqrtExpanded; -raft::distance::pairwise_distance(handle, input_view, input_view, output_view, metric); +auto metric = cuvs::distance::DistanceType::L2SqrtExpanded; +cuvs::distance::pairwise_distance(handle, input_view, input_view, output_view, metric); ``` @@ -156,12 +155,12 @@ raft::distance::pairwise_distance(handle, input_view, input_view, output_view, m The `pylibraft` package contains a Python API for RAFT algorithms and primitives. `pylibraft` integrates nicely into other libraries by being very lightweight with minimal dependencies and accepting any object that supports the `__cuda_array_interface__`, such as [CuPy's ndarray](https://docs.cupy.dev/en/stable/user_guide/interoperability.html#rmm). The number of RAFT algorithms exposed in this package is continuing to grow from release to release. -The example below demonstrates computing the pairwise Euclidean distances between CuPy arrays. Note that CuPy is not a required dependency for `pylibraft`. +The example below demonstrates computing the pairwise Euclidean distances between CuPy arrays using the [NVIDIA cuVS](https://github.com/rapidsai/cuvs) library. Note that CuPy is not a required dependency for `pylibraft`. ```python import cupy as cp -from pylibraft.distance import pairwise_distance +from cuvs.distance import pairwise_distance n_samples = 5000 n_features = 50 @@ -208,7 +207,7 @@ pylibraft.config.set_output_as(lambda device_ndarray: return device_ndarray.copy ```python import cupy as cp -from pylibraft.distance import pairwise_distance +from cuvs.distance import pairwise_distance n_samples = 5000 n_features = 50 @@ -223,18 +222,15 @@ pairwise_distance(in1, in2, out=output, metric="euclidean") ## Installing -RAFT's C++ and Python libraries can both be installed through Conda and the Python libraries through Pip. +RAFT's C++ and Python libraries can both be installed through Conda and the Python libraries through Pip. ### Installing C++ and Python through Conda The easiest way to install RAFT is through conda and several packages are provided. - `libraft-headers` C++ headers -- `libraft` (optional) C++ shared library containing pre-compiled template instantiations and runtime API. - `pylibraft` (optional) Python library - `raft-dask` (optional) Python library for deployment of multi-node multi-GPU algorithms that use the RAFT `raft::comms` abstraction layer in Dask clusters. -- `raft-ann-bench` (optional) Benchmarking tool for easily producing benchmarks that compare RAFT's vector search algorithms against other state-of-the-art implementations. -- `raft-ann-bench-cpu` (optional) Reproducible benchmarking tool similar to above, but doesn't require CUDA to be installed on the machine. Can be used to test in environments with competitive CPUs. Use the following command, depending on your CUDA version, to install all of the RAFT packages with conda (replace `rapidsai` with `rapidsai-nightly` to install more up-to-date but less stable nightly packages). `mamba` is preferred over the `conda` command. ```bash @@ -255,8 +251,6 @@ You can also install the conda packages individually using the `mamba` command a mamba install -c rapidsai -c conda-forge -c nvidia libraft libraft-headers cuda-version=12.5 ``` -If installing the C++ APIs please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-24.10/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. - ### Installing Python through Pip `pylibraft` and `raft-dask` both have experimental packages that can be [installed through pip](https://rapids.ai/pip.html#install): @@ -265,12 +259,10 @@ pip install pylibraft-cu11 --extra-index-url=https://pypi.nvidia.com pip install raft-dask-cu11 --extra-index-url=https://pypi.nvidia.com ``` -These packages statically build RAFT's pre-compiled instantiations and so the C++ headers and pre-compiled shared library won't be readily available to use in your code. +These packages statically build RAFT's pre-compiled instantiations and so the C++ headers won't be readily available to use in your code. The [build instructions](https://docs.rapids.ai/api/raft/nightly/build/) contain more details on building RAFT from source and including it in downstream projects. You can also find a more comprehensive version of the above CPM code snippet the [Building RAFT C++ and Python from source](https://docs.rapids.ai/api/raft/nightly/build/#building-c-and-python-from-source) section of the build instructions. -You can find an example [RAFT project template](cpp/template/README.md) in the `cpp/template` directory, which demonstrates how to build a new application with RAFT or incorporate RAFT into an existing CMake project. - ## Contributing @@ -284,7 +276,7 @@ When citing RAFT generally, please consider referencing this Github project. title={Rapidsai/raft: RAFT contains fundamental widely-used algorithms and primitives for data science, Graph and machine learning.}, url={https://github.com/rapidsai/raft}, journal={GitHub}, - publisher={Nvidia RAPIDS}, + publisher={NVIDIA RAPIDS}, author={Rapidsai}, year={2022} } diff --git a/VERSION b/VERSION index 7c7ba04436..af28c42b52 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.10.00 +24.12.00 diff --git a/build.sh b/build.sh index a77dd188f4..a95cb8ee23 100755 --- a/build.sh +++ b/build.sh @@ -18,8 +18,8 @@ ARGS=$* # scripts, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libraft pylibraft raft-dask docs tests template bench-prims bench-ann clean --uninstall -v -g -n --compile-lib --compile-static-lib --allgpuarch --no-nvtx --cpu-only --show_depr_warn --incl-cache-stats --time -h" -HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool=] [--limit-tests=] [--limit-bench-prims=] [--limit-bench-ann=] [--build-metrics=] +VALIDARGS="clean libraft pylibraft raft-dask docs tests bench-prims clean --uninstall -v -g -n --compile-lib --compile-static-lib --allgpuarch --no-nvtx --show_depr_warn --incl-cache-stats --time -h" +HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool=] [--limit-tests=] [--limit-bench-prims=] [--build-metrics=] where is: clean - remove all existing build artifacts and configuration (start over) libraft - build the raft C++ code only. Also builds the C-wrapper library @@ -29,8 +29,6 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool= is: -v - verbose build mode @@ -39,10 +37,8 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool= VERSION +sccache --zero-stats + # TODO: Remove `--no-test` flags once importing on a CPU # node works correctly rapids-conda-retry mambabuild \ @@ -29,28 +31,15 @@ rapids-conda-retry mambabuild \ --channel "${CPP_CHANNEL}" \ conda/recipes/pylibraft -rapids-conda-retry mambabuild \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/raft-dask +sccache --show-adv-stats +sccache --zero-stats -# Build ann-bench for each cuda and python version rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/raft-ann-bench + conda/recipes/raft-dask -# Build ann-bench-cpu only in CUDA 11 jobs since it only depends on python -# version -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then - rapids-conda-retry mambabuild \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/raft-ann-bench-cpu -fi +sccache --show-adv-stats rapids-upload-conda-to-s3 python diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index e7ae52f33a..326ee9a4c7 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -16,7 +16,7 @@ source rapids-date-string RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -rapids-generate-version > VERSION +rapids-generate-version > ./VERSION cd "${package_dir}" @@ -39,10 +39,20 @@ case "${RAPIDS_CUDA_VERSION}" in ;; esac -# Hardcode the output dir -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +sccache --zero-stats + +rapids-logger "Building '${package_name}' wheel" + +python -m pip wheel \ + -w dist \ + -v \ + --no-deps \ + --disable-pip-version-check \ + . + +sccache --show-adv-stats mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* -RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist +RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh index ce9f0ed172..dacaa1190e 100755 --- a/ci/build_wheel_pylibraft.sh +++ b/ci/build_wheel_pylibraft.sh @@ -3,6 +3,8 @@ set -euo pipefail +package_dir="python/pylibraft" + case "${RAPIDS_CUDA_VERSION}" in 12.*) EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" @@ -15,4 +17,5 @@ esac # Set up skbuild options. Enable sccache in skbuild config options export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF${EXTRA_CMAKE_ARGS}" -ci/build_wheel.sh pylibraft python/pylibraft +ci/build_wheel.sh pylibraft ${package_dir} +ci/validate_wheel.sh ${package_dir} final_dist diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh index feba2d7a5b..e4f3f0a833 100755 --- a/ci/build_wheel_raft_dask.sh +++ b/ci/build_wheel_raft_dask.sh @@ -3,7 +3,10 @@ set -euo pipefail +package_dir="python/raft-dask" + # Set up skbuild options. Enable sccache in skbuild config options export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" -ci/build_wheel.sh raft-dask python/raft-dask +ci/build_wheel.sh raft-dask ${package_dir} +ci/validate_wheel.sh ${package_dir} final_dist diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 032b88b4aa..a70fed9ec8 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -35,7 +35,6 @@ function sed_runner() { sed -i.bak ''"$1"'' $2 && rm -f ${2}.bak } -sed_runner "s/set(RAPIDS_VERSION .*)/set(RAPIDS_VERSION \"${NEXT_SHORT_TAG}\")/g" cpp/template/cmake/thirdparty/fetch_rapids.cmake sed_runner 's/'"find_and_configure_ucxx(VERSION .*"'/'"find_and_configure_ucxx(VERSION ${NEXT_UCXX_SHORT_TAG_PEP440}"'/g' python/raft-dask/cmake/thirdparty/get_ucxx.cmake sed_runner 's/'"branch-.*"'/'"branch-${NEXT_UCXX_SHORT_TAG_PEP440}"'/g' python/raft-dask/cmake/thirdparty/get_ucxx.cmake @@ -81,8 +80,6 @@ sed_runner "/^set(RAFT_VERSION/ s|\".*\"|\"${NEXT_SHORT_TAG}\"|g" docs/source/bu sed_runner "s|branch-[0-9][0-9].[0-9][0-9]|branch-${NEXT_SHORT_TAG}|g" docs/source/build.md sed_runner "/rapidsai\/raft/ s|branch-[0-9][0-9].[0-9][0-9]|branch-${NEXT_SHORT_TAG}|g" docs/source/developer_guide.md -sed_runner "s|:[0-9][0-9].[0-9][0-9]|:${NEXT_SHORT_TAG}|g" docs/source/raft_ann_benchmarks.md - sed_runner "s|branch-[0-9][0-9].[0-9][0-9]|branch-${NEXT_SHORT_TAG}|g" README.md # .devcontainer files diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 05323e4f5d..9d0edc6b21 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -8,6 +8,8 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh +RAPIDS_VERSION="$(rapids-version)" + rapids-logger "Generate C++ testing dependencies" rapids-dependency-file-generator \ --output conda \ @@ -29,7 +31,9 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ - libraft-headers libraft libraft-tests + "libraft-headers=${RAPIDS_VERSION}" \ + "libraft=${RAPIDS_VERSION}" \ + "libraft-tests=${RAPIDS_VERSION}" rapids-logger "Check GPU usage" nvidia-smi diff --git a/ci/test_python.sh b/ci/test_python.sh index 01e5ac9456..af93d2e04b 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -8,6 +8,8 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh +RAPIDS_VERSION="$(rapids-version)" + rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ @@ -34,7 +36,10 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - libraft libraft-headers pylibraft raft-dask + "libraft=${RAPIDS_VERSION}" \ + "libraft-headers=${RAPIDS_VERSION}" \ + "pylibraft=${RAPIDS_VERSION}" \ + "raft-dask=${RAPIDS_VERSION}" rapids-logger "Check GPU usage" nvidia-smi diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index 9b1187592d..a778a3ec51 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -9,10 +9,11 @@ RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels # Download the pylibraft built in the previous step RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibraft-dep -python -m pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install -v "$(echo ./dist/raft_dask_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" +python -m pip install -v \ + ./local-pylibraft-dep/pylibraft*.whl \ + "$(echo ./dist/raft_dask_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" test_dir="python/raft-dask/raft_dask/test" diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh new file mode 100755 index 0000000000..5910a5c59f --- /dev/null +++ b/ci/validate_wheel.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_dir=$1 +wheel_dir_relative_path=$2 + +cd "${package_dir}" + +rapids-logger "validate packages with 'pydistcheck'" + +pydistcheck \ + --inspect \ + "$(echo ${wheel_dir_relative_path}/*.whl)" + +rapids-logger "validate packages with 'twine'" + +twine check \ + --strict \ + "$(echo ${wheel_dir_relative_path}/*.whl)" diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 0b84772fad..6098cd12bf 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -14,14 +14,14 @@ dependencies: - cmake>=3.26.4,!=3.30.0 - cuda-nvtx=11.8 - cuda-profiler-api=11.8.86 -- cuda-python>=11.7.1,<12.0a0 +- cuda-python>=11.7.1,<12.0a0,<=11.8.3 - cuda-version=11.8 - cudatoolkit - cupy>=12.0.0 - cxx-compiler -- cython>=3.0.0 -- dask-cuda==24.10.*,>=0.0.0a0 -- distributed-ucxx==0.40.*,>=0.0.0a0 +- cython>=3.0.0,<3.1.0a0 +- dask-cuda==24.12.*,>=0.0.0a0 +- distributed-ucxx==0.41.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-aarch64=11.* - graphviz @@ -35,7 +35,7 @@ dependencies: - libcusolver=11.4.1.48 - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 -- libucxx==0.40.*,>=0.0.0a0 +- libucxx==0.41.*,>=0.0.0a0 - nccl>=2.19 - ninja - numba>=0.57 @@ -44,18 +44,18 @@ dependencies: - nvcc_linux-aarch64=11.8 - pre-commit - pydata-sphinx-theme -- pylibraft==24.10.*,>=0.0.0a0 +- pylibraft==24.12.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.10.*,>=0.0.0a0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - recommonmark -- rmm==24.10.*,>=0.0.0a0 +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-aarch64==2.17 -- ucx-py==0.40.*,>=0.0.0a0 +- ucx-py==0.41.*,>=0.0.0a0 name: all_cuda-118_arch-aarch64 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index d1c01f1b16..0fe8fbab39 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -14,14 +14,14 @@ dependencies: - cmake>=3.26.4,!=3.30.0 - cuda-nvtx=11.8 - cuda-profiler-api=11.8.86 -- cuda-python>=11.7.1,<12.0a0 +- cuda-python>=11.7.1,<12.0a0,<=11.8.3 - cuda-version=11.8 - cudatoolkit - cupy>=12.0.0 - cxx-compiler -- cython>=3.0.0 -- dask-cuda==24.10.*,>=0.0.0a0 -- distributed-ucxx==0.40.*,>=0.0.0a0 +- cython>=3.0.0,<3.1.0a0 +- dask-cuda==24.12.*,>=0.0.0a0 +- distributed-ucxx==0.41.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-64=11.* - graphviz @@ -35,7 +35,7 @@ dependencies: - libcusolver=11.4.1.48 - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 -- libucxx==0.40.*,>=0.0.0a0 +- libucxx==0.41.*,>=0.0.0a0 - nccl>=2.19 - ninja - numba>=0.57 @@ -44,18 +44,18 @@ dependencies: - nvcc_linux-64=11.8 - pre-commit - pydata-sphinx-theme -- pylibraft==24.10.*,>=0.0.0a0 +- pylibraft==24.12.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.10.*,>=0.0.0a0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - recommonmark -- rmm==24.10.*,>=0.0.0a0 +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-64==2.17 -- ucx-py==0.40.*,>=0.0.0a0 +- ucx-py==0.41.*,>=0.0.0a0 name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index 4c506f5297..dfb9ac0b97 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -16,13 +16,13 @@ dependencies: - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api -- cuda-python>=12.0,<13.0a0 +- cuda-python>=12.0,<13.0a0,<=12.6.0 - cuda-version=12.5 - cupy>=12.0.0 - cxx-compiler -- cython>=3.0.0 -- dask-cuda==24.10.*,>=0.0.0a0 -- distributed-ucxx==0.40.*,>=0.0.0a0 +- cython>=3.0.0,<3.1.0a0 +- dask-cuda==24.12.*,>=0.0.0a0 +- distributed-ucxx==0.41.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-aarch64=11.* - graphviz @@ -32,7 +32,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libucxx==0.40.*,>=0.0.0a0 +- libucxx==0.41.*,>=0.0.0a0 - nccl>=2.19 - ninja - numba>=0.57 @@ -40,18 +40,18 @@ dependencies: - numpydoc - pre-commit - pydata-sphinx-theme -- pylibraft==24.10.*,>=0.0.0a0 +- pylibraft==24.12.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.10.*,>=0.0.0a0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - recommonmark -- rmm==24.10.*,>=0.0.0a0 +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-aarch64==2.17 -- ucx-py==0.40.*,>=0.0.0a0 +- ucx-py==0.41.*,>=0.0.0a0 name: all_cuda-125_arch-aarch64 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index a123950e3a..bf6f5d6462 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -16,13 +16,13 @@ dependencies: - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api -- cuda-python>=12.0,<13.0a0 +- cuda-python>=12.0,<13.0a0,<=12.6.0 - cuda-version=12.5 - cupy>=12.0.0 - cxx-compiler -- cython>=3.0.0 -- dask-cuda==24.10.*,>=0.0.0a0 -- distributed-ucxx==0.40.*,>=0.0.0a0 +- cython>=3.0.0,<3.1.0a0 +- dask-cuda==24.12.*,>=0.0.0a0 +- distributed-ucxx==0.41.*,>=0.0.0a0 - doxygen>=1.8.20 - gcc_linux-64=11.* - graphviz @@ -32,7 +32,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libucxx==0.40.*,>=0.0.0a0 +- libucxx==0.41.*,>=0.0.0a0 - nccl>=2.19 - ninja - numba>=0.57 @@ -40,18 +40,18 @@ dependencies: - numpydoc - pre-commit - pydata-sphinx-theme -- pylibraft==24.10.*,>=0.0.0a0 +- pylibraft==24.12.*,>=0.0.0a0 - pytest-cov - pytest==7.* - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.10.*,>=0.0.0a0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - recommonmark -- rmm==24.10.*,>=0.0.0a0 +- rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scikit-learn - scipy - sphinx-copybutton - sphinx-markdown-tables - sysroot_linux-64==2.17 -- ucx-py==0.40.*,>=0.0.0a0 +- ucx-py==0.41.*,>=0.0.0a0 name: all_cuda-125_arch-x86_64 diff --git a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml deleted file mode 100644 index 864eb2130b..0000000000 --- a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# This file is generated by `rapids-dependency-file-generator`. -# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -channels: -- rapidsai -- rapidsai-nightly -- dask/label/dev -- conda-forge -- nvidia -dependencies: -- benchmark>=1.8.2 -- c-compiler -- clang-tools=16.0.6 -- clang==16.0.6 -- cmake>=3.26.4,!=3.30.0 -- cuda-nvtx=11.8 -- cuda-profiler-api=11.8.86 -- cuda-version=11.8 -- cudatoolkit -- cxx-compiler -- cython>=3.0.0 -- gcc_linux-aarch64=11.* -- glog>=0.6.0 -- h5py>=3.8.0 -- hnswlib=0.7.0 -- libcublas-dev=11.11.3.6 -- libcublas=11.11.3.6 -- libcurand-dev=10.3.0.86 -- libcurand=10.3.0.86 -- libcusolver-dev=11.4.1.48 -- libcusolver=11.4.1.48 -- libcusparse-dev=11.7.5.86 -- libcusparse=11.7.5.86 -- libucxx==0.40.*,>=0.0.0a0 -- matplotlib -- nccl>=2.19 -- ninja -- nlohmann_json>=3.11.2 -- nvcc_linux-aarch64=11.8 -- openblas -- pandas -- pyyaml -- rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.10.*,>=0.0.0a0 -- scikit-build-core>=0.10.0 -- sysroot_linux-aarch64==2.17 -name: bench_ann_cuda-118_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml deleted file mode 100644 index 5da6eaf17e..0000000000 --- a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# This file is generated by `rapids-dependency-file-generator`. -# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -channels: -- rapidsai -- rapidsai-nightly -- dask/label/dev -- conda-forge -- nvidia -dependencies: -- benchmark>=1.8.2 -- c-compiler -- clang-tools=16.0.6 -- clang==16.0.6 -- cmake>=3.26.4,!=3.30.0 -- cuda-nvtx=11.8 -- cuda-profiler-api=11.8.86 -- cuda-version=11.8 -- cudatoolkit -- cxx-compiler -- cython>=3.0.0 -- gcc_linux-64=11.* -- glog>=0.6.0 -- h5py>=3.8.0 -- hnswlib=0.7.0 -- libcublas-dev=11.11.3.6 -- libcublas=11.11.3.6 -- libcurand-dev=10.3.0.86 -- libcurand=10.3.0.86 -- libcusolver-dev=11.4.1.48 -- libcusolver=11.4.1.48 -- libcusparse-dev=11.7.5.86 -- libcusparse=11.7.5.86 -- libucxx==0.40.*,>=0.0.0a0 -- matplotlib -- nccl>=2.19 -- ninja -- nlohmann_json>=3.11.2 -- nvcc_linux-64=11.8 -- openblas -- pandas -- pyyaml -- rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.10.*,>=0.0.0a0 -- scikit-build-core>=0.10.0 -- sysroot_linux-64==2.17 -name: bench_ann_cuda-118_arch-x86_64 diff --git a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml deleted file mode 100644 index 65de97c170..0000000000 --- a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# This file is generated by `rapids-dependency-file-generator`. -# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -channels: -- rapidsai -- rapidsai-nightly -- dask/label/dev -- conda-forge -- nvidia -dependencies: -- benchmark>=1.8.2 -- c-compiler -- clang-tools=16.0.6 -- clang==16.0.6 -- cmake>=3.26.4,!=3.30.0 -- cuda-cudart-dev -- cuda-nvcc -- cuda-nvtx-dev -- cuda-profiler-api -- cuda-version=12.0 -- cxx-compiler -- cython>=3.0.0 -- gcc_linux-aarch64=11.* -- glog>=0.6.0 -- h5py>=3.8.0 -- hnswlib=0.7.0 -- libcublas-dev -- libcurand-dev -- libcusolver-dev -- libcusparse-dev -- libucxx==0.40.*,>=0.0.0a0 -- matplotlib -- nccl>=2.19 -- ninja -- nlohmann_json>=3.11.2 -- openblas -- pandas -- pyyaml -- rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.10.*,>=0.0.0a0 -- scikit-build-core>=0.10.0 -- sysroot_linux-aarch64==2.17 -name: bench_ann_cuda-120_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml deleted file mode 100644 index 7e1adbc483..0000000000 --- a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# This file is generated by `rapids-dependency-file-generator`. -# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -channels: -- rapidsai -- rapidsai-nightly -- dask/label/dev -- conda-forge -- nvidia -dependencies: -- benchmark>=1.8.2 -- c-compiler -- clang-tools=16.0.6 -- clang==16.0.6 -- cmake>=3.26.4,!=3.30.0 -- cuda-cudart-dev -- cuda-nvcc -- cuda-nvtx-dev -- cuda-profiler-api -- cuda-version=12.0 -- cxx-compiler -- cython>=3.0.0 -- gcc_linux-64=11.* -- glog>=0.6.0 -- h5py>=3.8.0 -- hnswlib=0.7.0 -- libcublas-dev -- libcurand-dev -- libcusolver-dev -- libcusparse-dev -- libucxx==0.40.*,>=0.0.0a0 -- matplotlib -- nccl>=2.19 -- ninja -- nlohmann_json>=3.11.2 -- openblas -- pandas -- pyyaml -- rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rmm==24.10.*,>=0.0.0a0 -- scikit-build-core>=0.10.0 -- sysroot_linux-64==2.17 -name: bench_ann_cuda-120_arch-x86_64 diff --git a/conda/recipes/libraft/build_libraft_template.sh b/conda/recipes/libraft/build_libraft_template.sh deleted file mode 100644 index 86c0fa11b6..0000000000 --- a/conda/recipes/libraft/build_libraft_template.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. - -# Just building template so we verify it uses libraft.so and fail if it doesn't build -./build.sh template --no-nvtx diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index bc0ff1fae7..4857f12cd1 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -19,21 +19,6 @@ c_stdlib_version: cmake_version: - ">=3.26.4,!=3.30.0" -nccl_version: - - ">=2.19" - -glog_version: - - ">=0.6.0" - -faiss_version: - - ">=1.7.1" - -h5py_version: - - ">=3.8.0" - -nlohmann_json_version: - - ">=3.11.2" - # The CTK libraries below are missing from the conda-forge::cudatoolkit package # for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages # and the "*_run_*" version specifiers correspond to `11.x` packages. diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index a075308500..503c4cb6fb 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -322,57 +322,3 @@ outputs: home: https://rapids.ai/ license: Apache-2.0 summary: libraft tests - - name: libraft-template - version: {{ version }} - script: build_libraft_template.sh - build: - script_env: *script_env - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports_from: - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} - {% else %} - - {{ compiler('cuda') }} - - cuda-cudart-dev - - libcublas-dev - {% endif %} - requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - cmake {{ cmake_version }} - - ninja - - {{ stdlib("c") }} - host: - - {{ pin_subpackage('libraft', exact=True) }} - - {{ pin_subpackage('libraft-headers', exact=True) }} - - cuda-version ={{ cuda_version }} - {% if cuda_major == "11" %} - - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - libcublas {{ cuda11_libcublas_host_version }} - - libcublas-dev {{ cuda11_libcublas_host_version }} - {% else %} - - cuda-cudart-dev - - cuda-profiler-api - - libcublas-dev - {% endif %} - run: - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {% if cuda_major == "11" %} - - cudatoolkit - {% else %} - - cuda-cudart - - libcublas - {% endif %} - - {{ pin_subpackage('libraft', exact=True) }} - about: - home: https://rapids.ai/ - license: Apache-2.0 - summary: libraft template diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 9d91af712e..01a9d61f0f 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -26,6 +26,7 @@ build: - {{ compiler('cuda') }} - cuda-cudart-dev {% endif %} + - cuda-python requirements: build: @@ -42,14 +43,14 @@ requirements: - {{ stdlib("c") }} host: {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0a0 + - cuda-python >=11.7.1,<12.0a0,<=11.8.3 - cudatoolkit {% else %} - - cuda-python >=12.0,<13.0a0 + - cuda-python >=12.0,<13.0a0,<=12.6.0 - cuda-cudart-dev {% endif %} - cuda-version ={{ cuda_version }} - - cython >=3.0.0 + - cython >=3.0.0,<3.1.0a0 - libraft {{ version }} - libraft-headers {{ version }} - python x.x @@ -60,8 +61,10 @@ requirements: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit + - cuda-python >=11.7.1,<12.0a0,<=11.8.3 {% else %} - cuda-cudart + - cuda-python >=12.0,<13.0a0,<=12.6.0 {% endif %} - libraft {{ version }} - libraft-headers {{ version }} diff --git a/conda/recipes/raft-ann-bench-cpu/build.sh b/conda/recipes/raft-ann-bench-cpu/build.sh deleted file mode 100644 index 4462d5124b..0000000000 --- a/conda/recipes/raft-ann-bench-cpu/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2023, NVIDIA CORPORATION. - -./build.sh bench-ann --cpu-only --no-nvtx --build-metrics=bench_ann_cpu --incl-cache-stats -cmake --install cpp/build --component ann_bench diff --git a/conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml b/conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml deleted file mode 100644 index ed6f708e14..0000000000 --- a/conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml +++ /dev/null @@ -1,29 +0,0 @@ -c_compiler_version: - - 11 - -cxx_compiler_version: - - 11 - -c_stdlib: - - sysroot - -c_stdlib_version: - - "2.17" - -cmake_version: - - ">=3.26.4,!=3.30.0" - -glog_version: - - ">=0.6.0" - -h5py_version: - - ">=3.8.0" - -nlohmann_json_version: - - ">=3.11.2" - -spdlog_version: - - ">=1.14.1,<1.15" - -fmt_version: - - ">=11.0.2,<12" diff --git a/conda/recipes/raft-ann-bench-cpu/meta.yaml b/conda/recipes/raft-ann-bench-cpu/meta.yaml deleted file mode 100644 index 94f7102726..0000000000 --- a/conda/recipes/raft-ann-bench-cpu/meta.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. - -# Usage: -# conda build . -c conda-forge -c nvidia -c rapidsai -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ['CONDA_PY'] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: raft-ann-bench-cpu - version: {{ version }} - script: build.sh - -source: - path: ../../.. - -build: - script_env: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_GENERATOR - - PARALLEL_LEVEL - - RAPIDS_ARTIFACTS_DIR - - SCCACHE_BUCKET - - SCCACHE_IDLE_TIMEOUT - - SCCACHE_REGION - - SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64] - - SCCACHE_S3_USE_SSL - number: {{ GIT_DESCRIBE_NUMBER }} - string: py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - -requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - cmake {{ cmake_version }} - - ninja - - {{ stdlib("c") }} - - host: - - glog {{ glog_version }} - - matplotlib - - nlohmann_json {{ nlohmann_json_version }} - - spdlog {{ spdlog_version }} - - fmt {{ fmt_version }} - - python - - pyyaml - - pandas - - rapids-build-backend>=0.3.0,<0.4.0.dev0 - - run: - - glog {{ glog_version }} - - h5py {{ h5py_version }} - - matplotlib - - python - - pyyaml - - pandas - - benchmark -about: - home: https://rapids.ai/ - license: Apache-2.0 - summary: RAFT ANN CPU benchmarks diff --git a/conda/recipes/raft-ann-bench/build.sh b/conda/recipes/raft-ann-bench/build.sh deleted file mode 100644 index 00078792a1..0000000000 --- a/conda/recipes/raft-ann-bench/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2023, NVIDIA CORPORATION. - -./build.sh bench-ann --allgpuarch --no-nvtx --build-metrics=bench_ann --incl-cache-stats -cmake --install cpp/build --component ann_bench diff --git a/conda/recipes/raft-ann-bench/conda_build_config.yaml b/conda/recipes/raft-ann-bench/conda_build_config.yaml deleted file mode 100644 index 47bd730daf..0000000000 --- a/conda/recipes/raft-ann-bench/conda_build_config.yaml +++ /dev/null @@ -1,70 +0,0 @@ -c_compiler_version: - - 11 - -cxx_compiler_version: - - 11 - -cuda_compiler: - - cuda-nvcc - -cuda11_compiler: - - nvcc - -c_stdlib: - - sysroot - -c_stdlib_version: - - "2.17" - -cmake_version: - - ">=3.26.4,!=3.30.0" - -nccl_version: - - ">=2.19" - -glog_version: - - ">=0.6.0" - -h5py_version: - - ">=3.8.0" - -nlohmann_json_version: - - ">=3.11.2" - -# The CTK libraries below are missing from the conda-forge::cudatoolkit package -# for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages -# and the "*_run_*" version specifiers correspond to `11.x` packages. - -cuda11_libcublas_host_version: - - "=11.11.3.6" - -cuda11_libcublas_run_version: - - ">=11.5.2.43,<12.0.0" - -cuda11_libcurand_host_version: - - "=10.3.0.86" - -cuda11_libcurand_run_version: - - ">=10.2.5.43,<10.3.1" - -cuda11_libcusolver_host_version: - - "=11.4.1.48" - -cuda11_libcusolver_run_version: - - ">=11.2.0.43,<11.4.2" - -cuda11_libcusparse_host_version: - - "=11.7.5.86" - -cuda11_libcusparse_run_version: - - ">=11.6.0.43,<12.0.0" - -# `cuda-profiler-api` only has `11.8.0` and `12.0.0` packages for all -# architectures. The "*_host_*" version specifiers correspond to `11.8` packages and the -# "*_run_*" version specifiers correspond to `11.x` packages. - -cuda11_cuda_profiler_api_host_version: - - "=11.8.86" - -cuda11_cuda_profiler_api_run_version: - - ">=11.4.240,<12" diff --git a/conda/recipes/raft-ann-bench/meta.yaml b/conda/recipes/raft-ann-bench/meta.yaml deleted file mode 100644 index d6aeb5f860..0000000000 --- a/conda/recipes/raft-ann-bench/meta.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. - -# Usage: -# conda build . -c conda-forge -c nvidia -c rapidsai -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ['CONDA_PY'] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: raft-ann-bench - version: {{ version }} - script: build.sh - -source: - path: ../../.. - -build: - script_env: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_GENERATOR - - PARALLEL_LEVEL - - RAPIDS_ARTIFACTS_DIR - - SCCACHE_BUCKET - - SCCACHE_IDLE_TIMEOUT - - SCCACHE_REGION - - SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64] - - SCCACHE_S3_USE_SSL - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports_from: - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} - {% else %} - - {{ compiler('cuda') }} - - cuda-cudart-dev - - libcublas-dev - {% endif %} - -requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - cmake {{ cmake_version }} - - ninja - - {{ stdlib("c") }} - - host: - - python - - libraft {{ version }} - - cuda-version ={{ cuda_version }} - {% if cuda_major == "11" %} - - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - libcublas {{ cuda11_libcublas_host_version }} - - libcublas-dev {{ cuda11_libcublas_host_version }} - {% else %} - - cuda-cudart-dev - - cuda-profiler-api - - libcublas-dev - {% endif %} - - glog {{ glog_version }} - - nlohmann_json {{ nlohmann_json_version }} - - h5py {{ h5py_version }} - - benchmark - - matplotlib - - python - - pandas - - pyyaml - # rmm is needed to determine if package is gpu-enabled - - rmm ={{ minor_version }} - - rapids-build-backend>=0.3.0,<0.4.0.dev0 - - run: - - python - - libraft {{ version }} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {% if cuda_major == "11" %} - - cudatoolkit - {% else %} - - cuda-cudart - - libcublas - {% endif %} - - glog {{ glog_version }} - - h5py {{ h5py_version }} - - benchmark - - glog {{ glog_version }} - - matplotlib - - python - - pandas - - pyyaml - # rmm is needed to determine if package is gpu-enabled - - rmm ={{ minor_version }} -about: - home: https://rapids.ai/ - license: Apache-2.0 - summary: RAFT ANN GPU and CPU benchmarks diff --git a/conda/recipes/raft-dask/conda_build_config.yaml b/conda/recipes/raft-dask/conda_build_config.yaml index 65c589fc0c..d7d2f68b42 100644 --- a/conda/recipes/raft-dask/conda_build_config.yaml +++ b/conda/recipes/raft-dask/conda_build_config.yaml @@ -17,10 +17,10 @@ c_stdlib_version: - "2.17" ucx_py_version: - - "0.40.*" + - "0.41.*" ucxx_version: - - "0.40.*" + - "0.41.*" cmake_version: - ">=3.26.4,!=3.30.0" diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index bc13d352b7..02a8957b06 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -26,6 +26,7 @@ build: - {{ compiler('cuda') }} - cuda-cudart-dev {% endif %} + - cuda-python requirements: build: @@ -42,14 +43,14 @@ requirements: - {{ stdlib("c") }} host: {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0a0 + - cuda-python >=11.7.1,<12.0a0,<=11.8.3 - cudatoolkit {% else %} - - cuda-python >=12.0,<13.0a0 + - cuda-python >=12.0,<13.0a0,<=12.6.0 - cuda-cudart-dev {% endif %} - cuda-version ={{ cuda_version }} - - cython >=3.0.0 + - cython >=3.0.0,<3.1.0a0 - nccl {{ nccl_version }} - pylibraft {{ version }} - python x.x @@ -61,8 +62,10 @@ requirements: run: {% if cuda_major == "11" %} - cudatoolkit + - cuda-python >=11.7.1,<12.0a0,<=11.8.3 {% else %} - cuda-cudart + - cuda-python >=12.0,<13.0a0,<=12.6.0 {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - dask-cuda ={{ minor_version }} diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d7eeb60b27..780f6f8581 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -17,17 +17,13 @@ include(rapids-cpm) include(rapids-export) include(rapids-find) -option(BUILD_CPU_ONLY "Build CPU only components. Applies to RAFT ANN benchmarks currently" OFF) - # workaround for rapids_cuda_init_architectures not working for arch detection with # enable_language(CUDA) set(lang_list "CXX") -if(NOT BUILD_CPU_ONLY) - include(rapids-cuda) - rapids_cuda_init_architectures(RAFT) - list(APPEND lang_list "CUDA") -endif() +include(rapids-cuda) +rapids_cuda_init_architectures(RAFT) +list(APPEND lang_list "CUDA") project( RAFT @@ -53,8 +49,6 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) option(BUILD_SHARED_LIBS "Build raft shared libraries" ON) option(BUILD_TESTS "Build raft unit-tests" ON) option(BUILD_PRIMS_BENCH "Build raft C++ benchmark tests" OFF) -option(BUILD_ANN_BENCH "Build raft ann benchmarks" OFF) -option(BUILD_CAGRA_HNSWLIB "Build CAGRA+hnswlib interface" ON) option(CUDA_ENABLE_KERNELINFO "Enable kernel resource usage info" OFF) option(CUDA_ENABLE_LINEINFO "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF @@ -68,23 +62,13 @@ option(DISABLE_OPENMP "Disable OpenMP" OFF) option(RAFT_NVTX "Enable nvtx markers" OFF) set(RAFT_COMPILE_LIBRARY_DEFAULT OFF) -if((BUILD_TESTS - OR BUILD_PRIMS_BENCH - OR BUILD_ANN_BENCH - ) - AND NOT BUILD_CPU_ONLY -) +if(BUILD_TESTS OR BUILD_PRIMS_BENCH) set(RAFT_COMPILE_LIBRARY_DEFAULT ON) endif() option(RAFT_COMPILE_LIBRARY "Enable building raft shared library instantiations" ${RAFT_COMPILE_LIBRARY_DEFAULT} ) -if(BUILD_CPU_ONLY) - set(BUILD_SHARED_LIBS OFF) - set(BUILD_TESTS OFF) -endif() - # Needed because GoogleBenchmark changes the state of FindThreads.cmake, causing subsequent runs to # have different values for the `Threads::Threads` target. Setting this flag ensures # `Threads::Threads` is the same value across all builds so that cache hits occur @@ -97,20 +81,14 @@ include(CMakeDependentOption) message(VERBOSE "RAFT: Building optional components: ${raft_FIND_COMPONENTS}") message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Building raft C++ benchmarks: ${BUILD_PRIMS_BENCH}") -message(VERBOSE "RAFT: Building ANN benchmarks: ${BUILD_ANN_BENCH}") -message(VERBOSE "RAFT: Build CPU only components: ${BUILD_CPU_ONLY}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") message(VERBOSE "RAFT: Disable depreaction warnings " ${DISABLE_DEPRECATION_WARNINGS}) message(VERBOSE "RAFT: Disable OpenMP: ${DISABLE_OPENMP}") message(VERBOSE "RAFT: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") message(VERBOSE "RAFT: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") message(VERBOSE "RAFT: Enable nvtx markers: ${RAFT_NVTX}") -message(VERBOSE - "RAFT: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}" -) -message(VERBOSE - "RAFT: Statically link the CUDA math libraries: ${CUDA_STATIC_MATH_LIBRARIES}" -) +message(VERBOSE "RAFT: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") +message(VERBOSE "RAFT: Statically link the CUDA math libraries: ${CUDA_STATIC_MATH_LIBRARIES}") # Set RMM logging level set(RMM_LOGGING_LEVEL @@ -143,21 +121,17 @@ if(CUDA_STATIC_MATH_LIBRARIES) set(_ctk_static_suffix "_static") endif() -if(NOT BUILD_CPU_ONLY) - # CUDA runtime - rapids_cuda_init_runtime(USE_STATIC ${CUDA_STATIC_RUNTIME}) - # * find CUDAToolkit package - # * determine GPU architectures - # * enable the CMake CUDA language - # * set other CUDA compilation flags - rapids_find_package( - CUDAToolkit REQUIRED - BUILD_EXPORT_SET raft-exports - INSTALL_EXPORT_SET raft-exports - ) -else() - add_compile_definitions(BUILD_CPU_ONLY) -endif() +# CUDA runtime +rapids_cuda_init_runtime(USE_STATIC ${CUDA_STATIC_RUNTIME}) +# * find CUDAToolkit package +# * determine GPU architectures +# * enable the CMake CUDA language +# * set other CUDA compilation flags +rapids_find_package( + CUDAToolkit REQUIRED + BUILD_EXPORT_SET raft-exports + INSTALL_EXPORT_SET raft-exports +) if(NOT DISABLE_OPENMP) rapids_find_package( @@ -178,30 +152,24 @@ include(cmake/modules/ConfigureCUDA.cmake) # add third party dependencies using CPM rapids_cpm_init() -if(NOT BUILD_CPU_ONLY) - # CCCL before rmm/cuco so we get the right version of CCCL - include(cmake/thirdparty/get_cccl.cmake) - include(cmake/thirdparty/get_rmm.cmake) - include(cmake/thirdparty/get_cutlass.cmake) +# CCCL before rmm/cuco so we get the right version of CCCL +include(cmake/thirdparty/get_cccl.cmake) +include(cmake/thirdparty/get_rmm.cmake) +include(cmake/thirdparty/get_cutlass.cmake) - include(${rapids-cmake-dir}/cpm/cuco.cmake) - rapids_cpm_cuco(BUILD_EXPORT_SET raft-exports INSTALL_EXPORT_SET raft-exports) -endif() +include(${rapids-cmake-dir}/cpm/cuco.cmake) +rapids_cpm_cuco(BUILD_EXPORT_SET raft-exports INSTALL_EXPORT_SET raft-exports) if(BUILD_TESTS) include(${rapids-cmake-dir}/cpm/gtest.cmake) rapids_cpm_gtest(BUILD_STATIC) endif() -if(BUILD_PRIMS_BENCH OR BUILD_ANN_BENCH) +if(BUILD_PRIMS_BENCH) include(${rapids-cmake-dir}/cpm/gbench.cmake) rapids_cpm_gbench(BUILD_STATIC) endif() -if(BUILD_CAGRA_HNSWLIB) - include(cmake/thirdparty/get_hnswlib.cmake) -endif() - # ################################################################################################## # * raft --------------------------------------------------------------------- add_library(raft INTERFACE) @@ -210,14 +178,9 @@ add_library(raft::raft ALIAS raft) target_include_directories( raft INTERFACE "$" "$" ) -if(BUILD_CAGRA_HNSWLIB) - target_link_libraries(raft INTERFACE hnswlib::hnswlib) -endif() -if(NOT BUILD_CPU_ONLY) - # Keep RAFT as lightweight as possible. Only CUDA libs and rmm should be used in global target. - target_link_libraries(raft INTERFACE rmm::rmm cuco::cuco nvidia::cutlass::cutlass CCCL::CCCL) -endif() +# Keep RAFT as lightweight as possible. Only CUDA libs and rmm should be used in global target. +target_link_libraries(raft INTERFACE rmm::rmm cuco::cuco nvidia::cutlass::cutlass CCCL::CCCL) target_compile_features(raft INTERFACE cxx_std_17 $) target_compile_options( @@ -300,277 +263,15 @@ if(RAFT_COMPILE_LIBRARY) add_library( raft_objs OBJECT src/core/logger.cpp - src/distance/detail/pairwise_matrix/dispatch_canberra_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_canberra_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_correlation_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_correlation_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_cosine_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_cosine_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_dice_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_dice_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_kl_divergence_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_kl_divergence_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_l1_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_l1_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_l2_expanded_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_l2_expanded_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_l_inf_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_l_inf_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_float_float_float_int.cu - src/distance/detail/pairwise_matrix/dispatch_rbf.cu - src/distance/detail/pairwise_matrix/dispatch_russel_rao_double_double_double_int.cu - src/distance/detail/pairwise_matrix/dispatch_russel_rao_float_float_float_int.cu - src/distance/distance.cu - src/distance/fused_l2_nn.cu - src/distance/fused_distance_nn.cu src/linalg/detail/coalesced_reduction.cu - src/matrix/detail/select_k_double_int64_t.cu - src/matrix/detail/select_k_double_uint32_t.cu - src/matrix/detail/select_k_float_int64_t.cu - src/matrix/detail/select_k_float_uint32_t.cu - src/matrix/detail/select_k_float_int32.cu - src/matrix/detail/select_k_half_int64_t.cu - src/matrix/detail/select_k_half_uint32_t.cu - src/neighbors/ball_cover.cu - src/neighbors/brute_force_fused_l2_knn_float_int64_t.cu - src/neighbors/brute_force_knn_int64_t_float_int64_t.cu - src/neighbors/brute_force_knn_int64_t_float_uint32_t.cu - src/neighbors/brute_force_knn_int_float_int.cu - src/neighbors/brute_force_knn_uint32_t_float_uint32_t.cu - src/neighbors/brute_force_knn_index_float.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_single_cta_float_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_single_cta_float_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_single_cta_float_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_single_cta_float_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_single_cta_half_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_single_cta_half_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_single_cta_half_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_single_cta_half_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim128_t8.cu - src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim256_t16.cu - src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim512_t32.cu - src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim1024_t32.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu - src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu - src/neighbors/detail/ivf_flat_interleaved_scan_float_float_int64_t.cu - src/neighbors/detail/ivf_flat_interleaved_scan_half_half_int64_t.cu - src/neighbors/detail/ivf_flat_interleaved_scan_int8_t_int32_t_int64_t.cu - src/neighbors/detail/ivf_flat_interleaved_scan_uint8_t_uint32_t_int64_t.cu - src/neighbors/detail/ivf_flat_search.cu - src/neighbors/detail/ivf_pq_compute_similarity_float_float.cu - src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false.cu - src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true.cu - src/neighbors/detail/ivf_pq_compute_similarity_float_half.cu - src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false.cu - src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true.cu - src/neighbors/detail/ivf_pq_compute_similarity_half_half.cu - src/neighbors/detail/refine_host_float_float.cpp - src/neighbors/detail/refine_host_half_float.cpp - src/neighbors/detail/refine_host_int8_t_float.cpp - src/neighbors/detail/refine_host_uint8_t_float.cpp - src/neighbors/ivf_flat_build_float_int64_t.cu - src/neighbors/ivf_flat_build_int8_t_int64_t.cu - src/neighbors/ivf_flat_build_uint8_t_int64_t.cu - src/neighbors/ivf_flat_extend_float_int64_t.cu - src/neighbors/ivf_flat_extend_int8_t_int64_t.cu - src/neighbors/ivf_flat_extend_uint8_t_int64_t.cu - src/neighbors/ivf_flat_search_float_int64_t.cu - src/neighbors/ivf_flat_search_int8_t_int64_t.cu - src/neighbors/ivf_flat_search_uint8_t_int64_t.cu - src/neighbors/ivfpq_build_float_int64_t.cu - src/neighbors/ivfpq_build_half_int64_t.cu - src/neighbors/ivfpq_build_int8_t_int64_t.cu - src/neighbors/ivfpq_build_uint8_t_int64_t.cu - src/neighbors/ivfpq_extend_float_int64_t.cu - src/neighbors/ivfpq_extend_half_int64_t.cu - src/neighbors/ivfpq_extend_int8_t_int64_t.cu - src/neighbors/ivfpq_extend_uint8_t_int64_t.cu - src/neighbors/ivfpq_search_float_int64_t.cu - src/neighbors/ivfpq_search_half_int64_t.cu - src/neighbors/ivfpq_search_int8_t_int64_t.cu - src/neighbors/ivfpq_search_uint8_t_int64_t.cu - src/neighbors/refine_float_float.cu - src/neighbors/refine_half_float.cu - src/neighbors/refine_int8_t_float.cu - src/neighbors/refine_uint8_t_float.cu - src/raft_runtime/cluster/cluster_cost.cuh - src/raft_runtime/cluster/cluster_cost_double.cu - src/raft_runtime/cluster/cluster_cost_float.cu - src/raft_runtime/cluster/kmeans_fit_double.cu - src/raft_runtime/cluster/kmeans_fit_float.cu - src/raft_runtime/cluster/kmeans_init_plus_plus_double.cu - src/raft_runtime/cluster/kmeans_init_plus_plus_float.cu - src/raft_runtime/cluster/update_centroids.cuh - src/raft_runtime/cluster/update_centroids_double.cu - src/raft_runtime/cluster/update_centroids_float.cu - src/raft_runtime/distance/fused_distance_min_arg.cu - src/raft_runtime/distance/fused_l2_min_arg.cu - src/raft_runtime/distance/pairwise_distance.cu - src/raft_runtime/matrix/select_k_float_int64_t.cu - src/raft_runtime/neighbors/brute_force_knn_int64_t_float.cu - src/raft_runtime/neighbors/cagra_build.cu - src/raft_runtime/neighbors/cagra_search.cu - src/raft_runtime/neighbors/cagra_serialize.cu - src/raft_runtime/neighbors/eps_neighborhood.cu - $<$:src/raft_runtime/neighbors/hnsw.cpp> - src/raft_runtime/neighbors/ivf_flat_build.cu - src/raft_runtime/neighbors/ivf_flat_search.cu - src/raft_runtime/neighbors/ivf_flat_serialize.cu - src/raft_runtime/neighbors/ivfpq_build.cu - src/raft_runtime/neighbors/ivfpq_deserialize.cu - src/raft_runtime/neighbors/ivfpq_search_float_int64_t.cu - src/raft_runtime/neighbors/ivfpq_search_int8_t_int64_t.cu - src/raft_runtime/neighbors/ivfpq_search_uint8_t_int64_t.cu - src/raft_runtime/neighbors/ivfpq_serialize.cu - src/raft_runtime/neighbors/refine_d_int64_t_float.cu - src/raft_runtime/neighbors/refine_d_int64_t_int8_t.cu - src/raft_runtime/neighbors/refine_d_int64_t_uint8_t.cu - src/raft_runtime/neighbors/refine_h_int64_t_float.cu - src/raft_runtime/neighbors/refine_h_int64_t_int8_t.cu - src/raft_runtime/neighbors/refine_h_int64_t_uint8_t.cu src/raft_runtime/random/rmat_rectangular_generator_int64_double.cu src/raft_runtime/random/rmat_rectangular_generator_int64_float.cu src/raft_runtime/random/rmat_rectangular_generator_int_double.cu src/raft_runtime/random/rmat_rectangular_generator_int_float.cu - src/spatial/knn/detail/ball_cover/registers_eps_pass_euclidean.cu - src/spatial/knn/detail/ball_cover/registers_pass_one_2d_dist.cu - src/spatial/knn/detail/ball_cover/registers_pass_one_2d_euclidean.cu - src/spatial/knn/detail/ball_cover/registers_pass_one_2d_haversine.cu - src/spatial/knn/detail/ball_cover/registers_pass_one_3d_dist.cu - src/spatial/knn/detail/ball_cover/registers_pass_one_3d_euclidean.cu - src/spatial/knn/detail/ball_cover/registers_pass_one_3d_haversine.cu - src/spatial/knn/detail/ball_cover/registers_pass_two_2d_dist.cu - src/spatial/knn/detail/ball_cover/registers_pass_two_2d_euclidean.cu - src/spatial/knn/detail/ball_cover/registers_pass_two_2d_haversine.cu - src/spatial/knn/detail/ball_cover/registers_pass_two_3d_dist.cu - src/spatial/knn/detail/ball_cover/registers_pass_two_3d_euclidean.cu - src/spatial/knn/detail/ball_cover/registers_pass_two_3d_haversine.cu - src/spatial/knn/detail/fused_l2_knn_int32_t_float.cu - src/spatial/knn/detail/fused_l2_knn_int64_t_float.cu - src/spatial/knn/detail/fused_l2_knn_uint32_t_float.cu + src/raft_runtime/solver/lanczos_solver_int64_double.cu + src/raft_runtime/solver/lanczos_solver_int64_float.cu + src/raft_runtime/solver/lanczos_solver_int_double.cu + src/raft_runtime/solver/lanczos_solver_int_float.cu ) set_target_properties( raft_objs @@ -847,10 +548,3 @@ endif() if(BUILD_PRIMS_BENCH) add_subdirectory(bench/prims/) endif() - -# ################################################################################################## -# * build ann benchmark executable ----------------------------------------------- - -if(BUILD_ANN_BENCH) - add_subdirectory(bench/ann/) -endif() diff --git a/cpp/bench/ann/CMakeLists.txt b/cpp/bench/ann/CMakeLists.txt deleted file mode 100644 index 35df378438..0000000000 --- a/cpp/bench/ann/CMakeLists.txt +++ /dev/null @@ -1,349 +0,0 @@ -# ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -list(APPEND CMAKE_MODULE_PATH "${RAFT_SOURCE_DIR}") - -# ################################################################################################## -# * benchmark options ------------------------------------------------------------------------------ - -option(RAFT_ANN_BENCH_USE_FAISS_GPU_FLAT "Include faiss' brute-force knn algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_FAISS_GPU_IVF_FLAT "Include faiss' ivf flat algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_FAISS_GPU_IVF_PQ "Include faiss' ivf pq algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_FAISS_CPU_FLAT "Include faiss' cpu brute-force algorithm in benchmark" ON) - -option(RAFT_ANN_BENCH_USE_FAISS_CPU_IVF_FLAT "Include faiss' cpu ivf flat algorithm in benchmark" - ON -) -option(RAFT_ANN_BENCH_USE_FAISS_CPU_IVF_PQ "Include faiss' cpu ivf pq algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT "Include raft's ivf flat algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ "Include raft's ivf pq algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_RAFT_CAGRA "Include raft's CAGRA in benchmark" ON) -option(RAFT_ANN_BENCH_USE_RAFT_BRUTE_FORCE "Include raft's brute force knn in benchmark" ON) -option(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB "Include raft's CAGRA in benchmark" ON) -option(RAFT_ANN_BENCH_USE_HNSWLIB "Include hnsw algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_GGNN "Include ggnn algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_SINGLE_EXE - "Make a single executable with benchmark as shared library modules" OFF -) - -# ################################################################################################## -# * Process options ---------------------------------------------------------- - -find_package(Threads REQUIRED) - -set(RAFT_ANN_BENCH_USE_FAISS ON) -set(RAFT_FAISS_ENABLE_GPU ON) -set(RAFT_USE_FAISS_STATIC ON) - -if(BUILD_CPU_ONLY) - - # Include necessary logging dependencies - include(cmake/thirdparty/get_fmt) - include(cmake/thirdparty/get_spdlog) - set(RAFT_FAISS_ENABLE_GPU OFF) - set(RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT OFF) - set(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ OFF) - set(RAFT_ANN_BENCH_USE_RAFT_CAGRA OFF) - set(RAFT_ANN_BENCH_USE_RAFT_BRUTE_FORCE OFF) - set(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB OFF) - set(RAFT_ANN_BENCH_USE_GGNN OFF) -endif() - -set(RAFT_ANN_BENCH_USE_RAFT OFF) -if(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ - OR RAFT_ANN_BENCH_USE_RAFT_BRUTE_FORCE - OR RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT - OR RAFT_ANN_BENCH_USE_RAFT_CAGRA - OR RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB -) - set(RAFT_ANN_BENCH_USE_RAFT ON) -endif() - -# ################################################################################################## -# * Fetch requirements ------------------------------------------------------------- - -if(RAFT_ANN_BENCH_USE_HNSWLIB OR RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB) - include(cmake/thirdparty/get_hnswlib) -endif() - -include(cmake/thirdparty/get_nlohmann_json) - -if(RAFT_ANN_BENCH_USE_GGNN) - include(cmake/thirdparty/get_ggnn) -endif() - -if(RAFT_ANN_BENCH_USE_FAISS) - include(cmake/thirdparty/get_faiss) -endif() - -# ################################################################################################## -# * Enable NVTX if available - -# Note: ANN_BENCH wrappers have extra NVTX code not related to raft::nvtx.They track gbench -# benchmark cases and iterations. This is to make limited NVTX available to all algos, not just -# raft. -if(TARGET CUDA::nvtx3) - set(_CMAKE_REQUIRED_INCLUDES_ORIG ${CMAKE_REQUIRED_INCLUDES}) - get_target_property(CMAKE_REQUIRED_INCLUDES CUDA::nvtx3 INTERFACE_INCLUDE_DIRECTORIES) - unset(NVTX3_HEADERS_FOUND CACHE) - # Check the headers explicitly to make sure the cpu-only build succeeds - CHECK_INCLUDE_FILE_CXX(nvtx3/nvToolsExt.h NVTX3_HEADERS_FOUND) - set(CMAKE_REQUIRED_INCLUDES ${_CMAKE_REQUIRED_INCLUDES_ORIG}) -endif() - -# ################################################################################################## -# * Configure tests function------------------------------------------------------------- - -function(ConfigureAnnBench) - - set(oneValueArgs NAME) - set(multiValueArgs PATH LINKS CXXFLAGS) - - if(NOT BUILD_CPU_ONLY) - set(GPU_BUILD ON) - endif() - - cmake_parse_arguments( - ConfigureAnnBench "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} - ) - - set(BENCH_NAME ${ConfigureAnnBench_NAME}_ANN_BENCH) - - if(RAFT_ANN_BENCH_SINGLE_EXE) - add_library(${BENCH_NAME} SHARED ${ConfigureAnnBench_PATH}) - string(TOLOWER ${BENCH_NAME} BENCH_LIB_NAME) - set_target_properties(${BENCH_NAME} PROPERTIES OUTPUT_NAME ${BENCH_LIB_NAME}) - add_dependencies(${BENCH_NAME} ANN_BENCH) - else() - add_executable(${BENCH_NAME} ${ConfigureAnnBench_PATH}) - target_compile_definitions( - ${BENCH_NAME} PRIVATE ANN_BENCH_BUILD_MAIN - $<$:ANN_BENCH_NVTX3_HEADERS_FOUND> - ) - target_link_libraries( - ${BENCH_NAME} PRIVATE benchmark::benchmark $<$:CUDA::nvtx3> - ) - endif() - - target_link_libraries( - ${BENCH_NAME} - PRIVATE raft::raft - nlohmann_json::nlohmann_json - ${ConfigureAnnBench_LINKS} - Threads::Threads - $<$:${RAFT_CTK_MATH_DEPENDENCIES}> - $ - $ - $<$:fmt::fmt-header-only> - $<$:spdlog::spdlog_header_only> - ) - - set_target_properties( - ${BENCH_NAME} - PROPERTIES # set target compile options - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON - BUILD_RPATH "\$ORIGIN" - INSTALL_RPATH "\$ORIGIN" - ) - - set(${ConfigureAnnBench_CXXFLAGS} ${RAFT_CXX_FLAGS} ${ConfigureAnnBench_CXXFLAGS}) - - target_compile_options( - ${BENCH_NAME} PRIVATE "$<$:${ConfigureAnnBench_CXXFLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) - - if(RAFT_ANN_BENCH_USE_${ConfigureAnnBench_NAME}) - target_compile_definitions( - ${BENCH_NAME} - PUBLIC - RAFT_ANN_BENCH_USE_${ConfigureAnnBench_NAME}=RAFT_ANN_BENCH_USE_${ConfigureAnnBench_NAME} - ) - endif() - - target_include_directories( - ${BENCH_NAME} - PUBLIC "$" - PRIVATE ${ConfigureAnnBench_INCLUDES} - ) - - install( - TARGETS ${BENCH_NAME} - COMPONENT ann_bench - DESTINATION bin/ann - ) -endfunction() - -# ################################################################################################## -# * Configure tests------------------------------------------------------------- - -if(RAFT_ANN_BENCH_USE_HNSWLIB) - ConfigureAnnBench( - NAME HNSWLIB PATH src/hnswlib/hnswlib_benchmark.cpp LINKS hnswlib::hnswlib - ) - -endif() - -if(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ) - ConfigureAnnBench( - NAME - RAFT_IVF_PQ - PATH - src/raft/raft_benchmark.cu - src/raft/raft_ivf_pq.cu - LINKS - raft::compiled - ) -endif() - -if(RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT) - ConfigureAnnBench( - NAME - RAFT_IVF_FLAT - PATH - src/raft/raft_benchmark.cu - src/raft/raft_ivf_flat.cu - LINKS - raft::compiled - ) -endif() - -if(RAFT_ANN_BENCH_USE_RAFT_BRUTE_FORCE) - ConfigureAnnBench( - NAME RAFT_BRUTE_FORCE PATH src/raft/raft_benchmark.cu LINKS raft::compiled - ) -endif() - -if(RAFT_ANN_BENCH_USE_RAFT_CAGRA) - ConfigureAnnBench( - NAME - RAFT_CAGRA - PATH - src/raft/raft_benchmark.cu - src/raft/raft_cagra_float.cu - src/raft/raft_cagra_half.cu - src/raft/raft_cagra_int8_t.cu - src/raft/raft_cagra_uint8_t.cu - LINKS - raft::compiled - ) -endif() - -if(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB) - ConfigureAnnBench( - NAME RAFT_CAGRA_HNSWLIB PATH src/raft/raft_cagra_hnswlib.cu LINKS raft::compiled - hnswlib::hnswlib - ) -endif() - -message("RAFT_FAISS_TARGETS: ${RAFT_FAISS_TARGETS}") -message("CUDAToolkit_LIBRARY_DIR: ${CUDAToolkit_LIBRARY_DIR}") -if(RAFT_ANN_BENCH_USE_FAISS_CPU_FLAT) - ConfigureAnnBench( - NAME FAISS_CPU_FLAT PATH src/faiss/faiss_cpu_benchmark.cpp LINKS - ${RAFT_FAISS_TARGETS} - ) -endif() - -if(RAFT_ANN_BENCH_USE_FAISS_CPU_IVF_FLAT) - ConfigureAnnBench( - NAME FAISS_CPU_IVF_FLAT PATH src/faiss/faiss_cpu_benchmark.cpp LINKS - ${RAFT_FAISS_TARGETS} - ) -endif() - -if(RAFT_ANN_BENCH_USE_FAISS_CPU_IVF_PQ) - ConfigureAnnBench( - NAME FAISS_CPU_IVF_PQ PATH src/faiss/faiss_cpu_benchmark.cpp LINKS - ${RAFT_FAISS_TARGETS} - ) -endif() - -if(RAFT_ANN_BENCH_USE_FAISS_GPU_IVF_FLAT AND RAFT_FAISS_ENABLE_GPU) - ConfigureAnnBench( - NAME FAISS_GPU_IVF_FLAT PATH src/faiss/faiss_gpu_benchmark.cu LINKS - ${RAFT_FAISS_TARGETS} - ) -endif() - -if(RAFT_ANN_BENCH_USE_FAISS_GPU_IVF_PQ AND RAFT_FAISS_ENABLE_GPU) - ConfigureAnnBench( - NAME FAISS_GPU_IVF_PQ PATH src/faiss/faiss_gpu_benchmark.cu LINKS - ${RAFT_FAISS_TARGETS} - ) -endif() - -if(RAFT_ANN_BENCH_USE_FAISS_GPU_FLAT AND RAFT_FAISS_ENABLE_GPU) - ConfigureAnnBench( - NAME FAISS_GPU_FLAT PATH src/faiss/faiss_gpu_benchmark.cu LINKS ${RAFT_FAISS_TARGETS} - ) -endif() - -if(RAFT_ANN_BENCH_USE_GGNN) - include(cmake/thirdparty/get_glog) - ConfigureAnnBench(NAME GGNN PATH src/ggnn/ggnn_benchmark.cu LINKS glog::glog ggnn::ggnn) -endif() - -# ################################################################################################## -# * Dynamically-loading ANN_BENCH executable ------------------------------------------------------- -if(RAFT_ANN_BENCH_SINGLE_EXE) - add_executable(ANN_BENCH src/common/benchmark.cpp) - - target_include_directories(ANN_BENCH PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) - - target_link_libraries( - ANN_BENCH - PRIVATE raft::raft - nlohmann_json::nlohmann_json - benchmark::benchmark - dl - -static-libgcc - fmt::fmt-header-only - spdlog::spdlog_header_only - -static-libstdc++ - $<$:CUDA::nvtx3> - ) - set_target_properties( - ANN_BENCH - PROPERTIES # set target compile options - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON - BUILD_RPATH "\$ORIGIN" - INSTALL_RPATH "\$ORIGIN" - ) - target_compile_definitions( - ANN_BENCH - PRIVATE - $<$:ANN_BENCH_LINK_CUDART="libcudart.so.${CUDAToolkit_VERSION_MAJOR}.${CUDAToolkit_VERSION_MINOR}.${CUDAToolkit_VERSION_PATCH}"> - $<$:ANN_BENCH_NVTX3_HEADERS_FOUND> - ) - - target_link_options(ANN_BENCH PRIVATE -export-dynamic) - - install( - TARGETS ANN_BENCH - COMPONENT ann_bench - DESTINATION bin/ann - EXCLUDE_FROM_ALL - ) -endif() diff --git a/cpp/bench/ann/README.md b/cpp/bench/ann/README.md deleted file mode 100644 index 1a8af2e448..0000000000 --- a/cpp/bench/ann/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# RAFT CUDA ANN Benchmarks - -Please see the [ANN Benchmarks](https://docs.rapids.ai/api/raft/stable/cuda_ann_benchmarks.html) section of the RAFT documentation for instructions on building and using the ANN benchmarks. \ No newline at end of file diff --git a/cpp/bench/ann/src/common/ann_types.hpp b/cpp/bench/ann/src/common/ann_types.hpp deleted file mode 100644 index b010063dee..0000000000 --- a/cpp/bench/ann/src/common/ann_types.hpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "cuda_stub.hpp" // cudaStream_t - -#include -#include -#include -#include - -namespace raft::bench::ann { - -enum Objective { - THROUGHPUT, // See how many vectors we can push through - LATENCY // See how fast we can push a vector through -}; - -enum class MemoryType { - Host, - HostMmap, - Device, -}; - -enum class Metric { - kInnerProduct, - kEuclidean, -}; - -inline auto parse_metric(const std::string& metric_str) -> Metric -{ - if (metric_str == "inner_product") { - return raft::bench::ann::Metric::kInnerProduct; - } else if (metric_str == "euclidean") { - return raft::bench::ann::Metric::kEuclidean; - } else { - throw std::runtime_error("invalid metric: '" + metric_str + "'"); - } -} - -inline auto parse_memory_type(const std::string& memory_type) -> MemoryType -{ - if (memory_type == "host") { - return MemoryType::Host; - } else if (memory_type == "mmap") { - return MemoryType::HostMmap; - } else if (memory_type == "device") { - return MemoryType::Device; - } else { - throw std::runtime_error("invalid memory type: '" + memory_type + "'"); - } -} - -struct AlgoProperty { - MemoryType dataset_memory_type; - // neighbors/distances should have same memory type as queries - MemoryType query_memory_type; -}; - -class AnnBase { - public: - using index_type = size_t; - - inline AnnBase(Metric metric, int dim) : metric_(metric), dim_(dim) {} - virtual ~AnnBase() noexcept = default; - - protected: - Metric metric_; - int dim_; -}; - -/** - * The GPU-based algorithms, which do not perform CPU synchronization at the end of their build or - * search methods, must implement this interface. - * - * The `cuda_timer` / `cuda_lap` from `util.hpp` uses this stream to record GPU times with events - * and, if necessary, also synchronize (via events) between iterations. - * - * If the algo does not implement this interface, GPU timings are disabled. - */ -class AnnGPU { - public: - /** - * Return the main cuda stream for this algorithm. - * If any work is done in multiple streams, they should synchornize with the main stream at the - * end. - */ - [[nodiscard]] virtual auto get_sync_stream() const noexcept -> cudaStream_t = 0; - /** - * By default a GPU algorithm uses a fixed stream to order GPU operations. - * However, an algorithm may need to synchronize with the host at the end of its execution. - * In that case, also synchronizing with a benchmark event would put it at disadvantage. - * - * We can disable event sync by passing `false` here - * - ONLY IF THE ALGORITHM HAS PRODUCED ITS OUTPUT BY THE TIME IT SYNCHRONIZES WITH CPU. - */ - [[nodiscard]] virtual auto uses_stream() const noexcept -> bool { return true; } - virtual ~AnnGPU() noexcept = default; -}; - -template -class ANN : public AnnBase { - public: - struct AnnSearchParam { - Objective metric_objective = Objective::LATENCY; - virtual ~AnnSearchParam() = default; - [[nodiscard]] virtual auto needs_dataset() const -> bool { return false; }; - }; - - inline ANN(Metric metric, int dim) : AnnBase(metric, dim) {} - virtual ~ANN() noexcept override = default; - - virtual void build(const T* dataset, size_t nrow) = 0; - - virtual void set_search_param(const AnnSearchParam& param) = 0; - // TODO: this assumes that an algorithm can always return k results. - // This is not always possible. - virtual void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const = 0; - - virtual void save(const std::string& file) const = 0; - virtual void load(const std::string& file) = 0; - - virtual AlgoProperty get_preference() const = 0; - - // Some algorithms don't save the building dataset in their indices. - // So they should be given the access to that dataset during searching. - // The advantage of this way is that index has smaller size - // and many indices can share one dataset. - // - // SearchParam::needs_dataset() of such algorithm should be true, - // and set_search_dataset() should save the passed-in pointer somewhere. - // The client code should call set_search_dataset() before searching, - // and should not release dataset before searching is finished. - virtual void set_search_dataset(const T* /*dataset*/, size_t /*nrow*/){}; - - /** - * Make a shallow copy of the ANN wrapper that shares the resources and ensures thread-safe access - * to them. */ - virtual auto copy() -> std::unique_ptr> = 0; -}; - -} // namespace raft::bench::ann - -#define REGISTER_ALGO_INSTANCE(DataT) \ - template auto raft::bench::ann::create_algo( \ - const std::string&, const std::string&, int, const nlohmann::json&, const std::vector&) \ - ->std::unique_ptr>; \ - template auto raft::bench::ann::create_search_param(const std::string&, \ - const nlohmann::json&) \ - ->std::unique_ptr::AnnSearchParam>; diff --git a/cpp/bench/ann/src/common/benchmark.cpp b/cpp/bench/ann/src/common/benchmark.cpp deleted file mode 100644 index 5510abf42f..0000000000 --- a/cpp/bench/ann/src/common/benchmark.cpp +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// clang-format off -#include "cuda_stub.hpp" // must go first -// clang-format on - -#include "ann_types.hpp" - -#include -#define JSON_DIAGNOSTICS 1 -#include - -#include -#include -#include - -namespace raft::bench::ann { - -struct lib_handle { - void* handle{nullptr}; - explicit lib_handle(const std::string& name) - { - handle = dlopen(name.c_str(), RTLD_LAZY | RTLD_LOCAL); - if (handle == nullptr) { - auto error_msg = "Failed to load " + name; - auto err = dlerror(); - if (err != nullptr && err[0] != '\0') { error_msg += ": " + std::string(err); } - throw std::runtime_error(error_msg); - } - } - ~lib_handle() noexcept - { - if (handle != nullptr) { dlclose(handle); } - } -}; - -auto load_lib(const std::string& algo) -> void* -{ - static std::unordered_map libs{}; - auto found = libs.find(algo); - - if (found != libs.end()) { return found->second.handle; } - auto lib_name = "lib" + algo + "_ann_bench.so"; - return libs.emplace(algo, lib_name).first->second.handle; -} - -auto get_fun_name(void* addr) -> std::string -{ - Dl_info dl_info; - if (dladdr(addr, &dl_info) != 0) { - if (dl_info.dli_sname != nullptr && dl_info.dli_sname[0] != '\0') { - return std::string{dl_info.dli_sname}; - } - } - throw std::logic_error("Failed to find out name of the looked up function"); -} - -template -auto create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -> std::unique_ptr> -{ - static auto fname = get_fun_name(reinterpret_cast(&create_algo)); - auto handle = load_lib(algo); - auto fun_addr = dlsym(handle, fname.c_str()); - if (fun_addr == nullptr) { - throw std::runtime_error("Couldn't load the create_algo function (" + algo + ")"); - } - auto fun = reinterpret_cast)>(fun_addr); - return fun(algo, distance, dim, conf, dev_list); -} - -template -std::unique_ptr::AnnSearchParam> create_search_param( - const std::string& algo, const nlohmann::json& conf) -{ - static auto fname = get_fun_name(reinterpret_cast(&create_search_param)); - auto handle = load_lib(algo); - auto fun_addr = dlsym(handle, fname.c_str()); - if (fun_addr == nullptr) { - throw std::runtime_error("Couldn't load the create_search_param function (" + algo + ")"); - } - auto fun = reinterpret_cast)>(fun_addr); - return fun(algo, conf); -} - -}; // namespace raft::bench::ann - -REGISTER_ALGO_INSTANCE(float); -REGISTER_ALGO_INSTANCE(std::int8_t); -REGISTER_ALGO_INSTANCE(std::uint8_t); - -#include "benchmark.hpp" - -int main(int argc, char** argv) { return raft::bench::ann::run_main(argc, argv); } diff --git a/cpp/bench/ann/src/common/benchmark.hpp b/cpp/bench/ann/src/common/benchmark.hpp deleted file mode 100644 index 185d54a0a3..0000000000 --- a/cpp/bench/ann/src/common/benchmark.hpp +++ /dev/null @@ -1,736 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "ann_types.hpp" -#include "conf.hpp" -#include "dataset.hpp" -#include "util.hpp" - -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -static inline std::unique_ptr current_algo{nullptr}; -static inline std::unique_ptr current_algo_props{nullptr}; - -using kv_series = std::vector>>; - -inline auto apply_overrides(const std::vector& configs, - const kv_series& overrides, - std::size_t override_idx = 0) -> std::vector -{ - std::vector results{}; - if (override_idx >= overrides.size()) { - auto n = configs.size(); - for (size_t i = 0; i < n; i++) { - auto c = configs[i]; - c["override_suffix"] = n > 1 ? "/" + std::to_string(i) : ""; - results.push_back(c); - } - return results; - } - auto rec_configs = apply_overrides(configs, overrides, override_idx + 1); - auto [key, vals] = overrides[override_idx]; - auto n = vals.size(); - for (size_t i = 0; i < n; i++) { - const auto& val = vals[i]; - for (auto rc : rec_configs) { - if (n > 1) { - rc["override_suffix"] = - static_cast(rc["override_suffix"]) + "/" + std::to_string(i); - } - rc[key] = val; - results.push_back(rc); - } - } - return results; -} - -inline auto apply_overrides(const nlohmann::json& config, - const kv_series& overrides, - std::size_t override_idx = 0) -{ - return apply_overrides(std::vector{config}, overrides, 0); -} - -inline void dump_parameters(::benchmark::State& state, nlohmann::json params) -{ - std::string label = ""; - bool label_empty = true; - for (auto& [key, val] : params.items()) { - if (val.is_number()) { - state.counters.insert({{key, val}}); - } else if (val.is_boolean()) { - state.counters.insert({{key, val ? 1.0 : 0.0}}); - } else { - auto kv = key + "=" + val.dump(); - if (label_empty) { - label = kv; - } else { - label += "#" + kv; - } - label_empty = false; - } - } - if (!label_empty) { state.SetLabel(label); } -} - -inline auto parse_algo_property(AlgoProperty prop, const nlohmann::json& conf) -> AlgoProperty -{ - if (conf.contains("dataset_memory_type")) { - prop.dataset_memory_type = parse_memory_type(conf.at("dataset_memory_type")); - } - if (conf.contains("query_memory_type")) { - prop.query_memory_type = parse_memory_type(conf.at("query_memory_type")); - } - return prop; -}; - -template -void bench_build(::benchmark::State& state, - std::shared_ptr> dataset, - Configuration::Index index, - bool force_overwrite) -{ - // NB: these two thread-local vars can be used within algo wrappers - raft::bench::ann::benchmark_thread_id = state.thread_index(); - raft::bench::ann::benchmark_n_threads = state.threads(); - dump_parameters(state, index.build_param); - if (file_exists(index.file)) { - if (force_overwrite) { - log_info("Overwriting file: %s", index.file.c_str()); - } else { - return state.SkipWithMessage( - "Index file already exists (use --force to overwrite the index)."); - } - } - - std::unique_ptr> algo; - try { - algo = ann::create_algo( - index.algo, dataset->distance(), dataset->dim(), index.build_param, index.dev_list); - } catch (const std::exception& e) { - return state.SkipWithError("Failed to create an algo: " + std::string(e.what())); - } - - const auto algo_property = parse_algo_property(algo->get_preference(), index.build_param); - - const T* base_set = dataset->base_set(algo_property.dataset_memory_type); - std::size_t index_size = dataset->base_set_size(); - - cuda_timer gpu_timer{algo}; - { - nvtx_case nvtx{state.name()}; - for (auto _ : state) { - [[maybe_unused]] auto ntx_lap = nvtx.lap(); - [[maybe_unused]] auto gpu_lap = gpu_timer.lap(); - try { - algo->build(base_set, index_size); - } catch (const std::exception& e) { - state.SkipWithError(std::string(e.what())); - } - } - } - if (gpu_timer.active()) { - state.counters.insert({"GPU", {gpu_timer.total_time(), benchmark::Counter::kAvgIterations}}); - } - state.counters.insert({{"index_size", index_size}}); - - if (state.skipped()) { return; } - make_sure_parent_dir_exists(index.file); - algo->save(index.file); -} - -template -void bench_search(::benchmark::State& state, - Configuration::Index index, - std::size_t search_param_ix, - std::shared_ptr> dataset, - Objective metric_objective) -{ - // NB: these two thread-local vars can be used within algo wrappers - raft::bench::ann::benchmark_thread_id = state.thread_index(); - raft::bench::ann::benchmark_n_threads = state.threads(); - std::size_t queries_processed = 0; - - const auto& sp_json = index.search_params[search_param_ix]; - - if (state.thread_index() == 0) { dump_parameters(state, sp_json); } - - // NB: `k` and `n_queries` are guaranteed to be populated in conf.cpp - const std::uint32_t k = sp_json["k"]; - // Amount of data processes in one go - const std::size_t n_queries = sp_json["n_queries"]; - // Round down the query data to a multiple of the batch size to loop over full batches of data - const std::size_t query_set_size = (dataset->query_set_size() / n_queries) * n_queries; - - if (dataset->query_set_size() < n_queries) { - std::stringstream msg; - msg << "Not enough queries in benchmark set. Expected " << n_queries << ", actual " - << dataset->query_set_size(); - state.SkipWithError(msg.str()); - return; - } - - // Each thread start from a different offset, so that the queries that they process do not - // overlap. - std::ptrdiff_t batch_offset = (state.thread_index() * n_queries) % query_set_size; - std::ptrdiff_t queries_stride = state.threads() * n_queries; - // Output is saved into a contiguous buffer (separate buffers for each thread). - std::ptrdiff_t out_offset = 0; - - const T* query_set = nullptr; - - if (!file_exists(index.file)) { - state.SkipWithError("Index file is missing. Run the benchmark in the build mode first."); - return; - } - - /** - * Make sure the first thread loads the algo and dataset - */ - progress_barrier load_barrier{}; - if (load_barrier.arrive(1) == 0) { - // algo is static to cache it between close search runs to save time on index loading - static std::string index_file = ""; - if (index.file != index_file) { - current_algo.reset(); - index_file = index.file; - } - - std::unique_ptr::AnnSearchParam> search_param; - ANN* algo; - try { - if (!current_algo || (algo = dynamic_cast*>(current_algo.get())) == nullptr) { - auto ualgo = ann::create_algo( - index.algo, dataset->distance(), dataset->dim(), index.build_param, index.dev_list); - algo = ualgo.get(); - algo->load(index_file); - current_algo = std::move(ualgo); - } - search_param = ann::create_search_param(index.algo, sp_json); - search_param->metric_objective = metric_objective; - } catch (const std::exception& e) { - state.SkipWithError("Failed to create an algo: " + std::string(e.what())); - return; - } - - current_algo_props = std::make_unique( - std::move(parse_algo_property(algo->get_preference(), sp_json))); - - if (search_param->needs_dataset()) { - try { - algo->set_search_dataset(dataset->base_set(current_algo_props->dataset_memory_type), - dataset->base_set_size()); - } catch (const std::exception& ex) { - state.SkipWithError("The algorithm '" + index.name + - "' requires the base set, but it's not available. " + - "Exception: " + std::string(ex.what())); - return; - } - } - try { - algo->set_search_param(*search_param); - } catch (const std::exception& ex) { - state.SkipWithError("An error occurred setting search parameters: " + std::string(ex.what())); - return; - } - - query_set = dataset->query_set(current_algo_props->query_memory_type); - load_barrier.arrive(state.threads()); - } else { - // All other threads will wait for the first thread to initialize the algo. - load_barrier.wait(state.threads() * 2); - // gbench ensures that all threads are synchronized at the start of the benchmark loop. - // We are accessing shared variables (like current_algo, current_algo_probs) before the - // benchmark loop, therefore the synchronization here is necessary. - } - query_set = dataset->query_set(current_algo_props->query_memory_type); - - /** - * Each thread will manage its own outputs - */ - using index_type = AnnBase::index_type; - constexpr size_t kAlignResultBuf = 64; - size_t result_elem_count = k * query_set_size; - result_elem_count = - ((result_elem_count + kAlignResultBuf - 1) / kAlignResultBuf) * kAlignResultBuf; - auto& result_buf = - get_result_buffer_from_global_pool(result_elem_count * (sizeof(float) + sizeof(index_type))); - auto* neighbors_ptr = - reinterpret_cast(result_buf.data(current_algo_props->query_memory_type)); - auto* distances_ptr = reinterpret_cast(neighbors_ptr + result_elem_count); - - { - nvtx_case nvtx{state.name()}; - - std::unique_ptr> algo{nullptr}; - try { - dynamic_cast*>(current_algo.get())->copy().swap(algo); - } catch (const std::exception& e) { - state.SkipWithError("Algo::copy: " + std::string(e.what())); - return; - } - // Initialize with algo, so that the timer.lap() object can sync with algo::get_sync_stream() - cuda_timer gpu_timer{algo}; - auto start = std::chrono::high_resolution_clock::now(); - for (auto _ : state) { - [[maybe_unused]] auto ntx_lap = nvtx.lap(); - [[maybe_unused]] auto gpu_lap = gpu_timer.lap(); - try { - algo->search(query_set + batch_offset * dataset->dim(), - n_queries, - k, - neighbors_ptr + out_offset * k, - distances_ptr + out_offset * k); - } catch (const std::exception& e) { - state.SkipWithError("Benchmark loop: " + std::string(e.what())); - break; - } - - // advance to the next batch - batch_offset = (batch_offset + queries_stride) % query_set_size; - out_offset = (out_offset + n_queries) % query_set_size; - - queries_processed += n_queries; - } - auto end = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast>(end - start).count(); - if (state.thread_index() == 0) { state.counters.insert({{"end_to_end", duration}}); } - state.counters.insert({"Latency", {duration, benchmark::Counter::kAvgIterations}}); - - if (gpu_timer.active()) { - state.counters.insert({"GPU", {gpu_timer.total_time(), benchmark::Counter::kAvgIterations}}); - } - } - - state.SetItemsProcessed(queries_processed); - - // This will be the total number of queries across all threads - state.counters.insert({{"total_queries", queries_processed}}); - - if (state.skipped()) { return; } - - // Each thread calculates recall on their partition of queries. - // evaluate recall - if (dataset->max_k() >= k) { - const std::int32_t* gt = dataset->gt_set(); - const std::uint32_t max_k = dataset->max_k(); - result_buf.transfer_data(MemoryType::Host, current_algo_props->query_memory_type); - auto* neighbors_host = reinterpret_cast(result_buf.data(MemoryType::Host)); - std::size_t rows = std::min(queries_processed, query_set_size); - std::size_t match_count = 0; - std::size_t total_count = rows * static_cast(k); - - // We go through the groundtruth with same stride as the benchmark loop. - size_t out_offset = 0; - size_t batch_offset = (state.thread_index() * n_queries) % query_set_size; - while (out_offset < rows) { - for (std::size_t i = 0; i < n_queries; i++) { - size_t i_orig_idx = batch_offset + i; - size_t i_out_idx = out_offset + i; - if (i_out_idx < rows) { - for (std::uint32_t j = 0; j < k; j++) { - auto act_idx = std::int32_t(neighbors_host[i_out_idx * k + j]); - for (std::uint32_t l = 0; l < k; l++) { - auto exp_idx = gt[i_orig_idx * max_k + l]; - if (act_idx == exp_idx) { - match_count++; - break; - } - } - } - } - } - out_offset += n_queries; - batch_offset = (batch_offset + queries_stride) % query_set_size; - } - double actual_recall = static_cast(match_count) / static_cast(total_count); - state.counters.insert({"Recall", {actual_recall, benchmark::Counter::kAvgThreads}}); - } -} - -inline void printf_usage() -{ - ::benchmark::PrintDefaultHelp(); - fprintf(stdout, - " [--build|--search] \n" - " [--force]\n" - " [--data_prefix=]\n" - " [--index_prefix=]\n" - " [--override_kv=]\n" - " [--mode=\n" - " [--threads=min[:max]]\n" - " .json\n" - "\n" - "Note the non-standard benchmark parameters:\n" - " --build: build mode, will build index\n" - " --search: search mode, will search using the built index\n" - " one and only one of --build and --search should be specified\n" - " --force: force overwriting existing index files\n" - " --data_prefix=:" - " prepend to dataset file paths specified in the .json (default = " - "'data/').\n" - " --index_prefix=:" - " prepend to index file paths specified in the .json (default = " - "'index/').\n" - " --override_kv=:" - " override a build/search key one or more times multiplying the number of configurations;" - " you can use this parameter multiple times to get the Cartesian product of benchmark" - " configs.\n" - " --mode=" - " run the benchmarks in latency (accumulate times spent in each batch) or " - " throughput (pipeline batches and measure end-to-end) mode\n" - " --threads=min[:max] specify the number threads to use for throughput benchmark." - " Power of 2 values between 'min' and 'max' will be used. If only 'min' is specified," - " then a single test is run with 'min' threads. By default min=1, max=.\n"); -} - -template -void register_build(std::shared_ptr> dataset, - std::vector indices, - bool force_overwrite) -{ - for (auto index : indices) { - auto suf = static_cast(index.build_param["override_suffix"]); - auto file_suf = suf; - index.build_param.erase("override_suffix"); - std::replace(file_suf.begin(), file_suf.end(), '/', '-'); - index.file += file_suf; - auto* b = ::benchmark::RegisterBenchmark( - index.name + suf, bench_build, dataset, index, force_overwrite); - b->Unit(benchmark::kSecond); - b->MeasureProcessCPUTime(); - b->UseRealTime(); - } -} - -template -void register_search(std::shared_ptr> dataset, - std::vector indices, - Objective metric_objective, - const std::vector& threads) -{ - for (auto index : indices) { - for (std::size_t i = 0; i < index.search_params.size(); i++) { - auto suf = static_cast(index.search_params[i]["override_suffix"]); - index.search_params[i].erase("override_suffix"); - - auto* b = ::benchmark::RegisterBenchmark( - index.name + suf, bench_search, index, i, dataset, metric_objective) - ->Unit(benchmark::kMillisecond) - /** - * The following are important for getting accuracy QPS measurements on both CPU - * and GPU These make sure that - * - `end_to_end` ~ (`Time` * `Iterations`) - * - `items_per_second` ~ (`total_queries` / `end_to_end`) - * - Throughput = `items_per_second` - */ - ->MeasureProcessCPUTime() - ->UseRealTime(); - if (metric_objective == Objective::THROUGHPUT) { - if (index.algo.find("faiss_gpu") != std::string::npos) { - log_warn( - "FAISS GPU does not work in throughput mode because the underlying " - "StandardGpuResources object is not thread-safe. This will cause unexpected results"); - } - b->ThreadRange(threads[0], threads[1]); - } - } - } -} - -template -void dispatch_benchmark(const Configuration& conf, - bool force_overwrite, - bool build_mode, - bool search_mode, - std::string data_prefix, - std::string index_prefix, - kv_series override_kv, - Objective metric_objective, - const std::vector& threads) -{ - if (cudart.found()) { - for (auto [key, value] : cuda_info()) { - ::benchmark::AddCustomContext(key, value); - } - } - const auto dataset_conf = conf.get_dataset_conf(); - auto base_file = combine_path(data_prefix, dataset_conf.base_file); - auto query_file = combine_path(data_prefix, dataset_conf.query_file); - auto gt_file = dataset_conf.groundtruth_neighbors_file; - if (gt_file.has_value()) { gt_file.emplace(combine_path(data_prefix, gt_file.value())); } - auto dataset = std::make_shared>(dataset_conf.name, - base_file, - dataset_conf.subset_first_row, - dataset_conf.subset_size, - query_file, - dataset_conf.distance, - gt_file); - ::benchmark::AddCustomContext("dataset", dataset_conf.name); - ::benchmark::AddCustomContext("distance", dataset_conf.distance); - std::vector indices = conf.get_indices(); - if (build_mode) { - if (file_exists(base_file)) { - log_info("Using the dataset file '%s'", base_file.c_str()); - ::benchmark::AddCustomContext("n_records", std::to_string(dataset->base_set_size())); - ::benchmark::AddCustomContext("dim", std::to_string(dataset->dim())); - } else { - log_warn("Dataset file '%s' does not exist; benchmarking index building is impossible.", - base_file.c_str()); - } - std::vector more_indices{}; - for (auto& index : indices) { - for (auto param : apply_overrides(index.build_param, override_kv)) { - auto modified_index = index; - modified_index.build_param = param; - modified_index.file = combine_path(index_prefix, modified_index.file); - more_indices.push_back(modified_index); - } - } - register_build(dataset, more_indices, force_overwrite); - } else if (search_mode) { - if (file_exists(query_file)) { - log_info("Using the query file '%s'", query_file.c_str()); - ::benchmark::AddCustomContext("max_n_queries", std::to_string(dataset->query_set_size())); - ::benchmark::AddCustomContext("dim", std::to_string(dataset->dim())); - if (gt_file.has_value()) { - if (file_exists(*gt_file)) { - log_info("Using the ground truth file '%s'", gt_file->c_str()); - ::benchmark::AddCustomContext("max_k", std::to_string(dataset->max_k())); - } else { - log_warn("Ground truth file '%s' does not exist; the recall won't be reported.", - gt_file->c_str()); - } - } else { - log_warn( - "Ground truth file is not provided; the recall won't be reported. NB: use " - "the 'groundtruth_neighbors_file' alongside the 'query_file' key to specify the " - "path to " - "the ground truth in your conf.json."); - } - } else { - log_warn("Query file '%s' does not exist; benchmarking search is impossible.", - query_file.c_str()); - } - for (auto& index : indices) { - index.search_params = apply_overrides(index.search_params, override_kv); - index.file = combine_path(index_prefix, index.file); - } - register_search(dataset, indices, metric_objective, threads); - } -} - -inline auto parse_bool_flag(const char* arg, const char* pat, bool& result) -> bool -{ - if (strcmp(arg, pat) == 0) { - result = true; - return true; - } - return false; -} - -inline auto parse_string_flag(const char* arg, const char* pat, std::string& result) -> bool -{ - auto n = strlen(pat); - if (strncmp(pat, arg, strlen(pat)) == 0) { - result = arg + n + 1; - return true; - } - return false; -} - -inline auto run_main(int argc, char** argv) -> int -{ - bool force_overwrite = false; - bool build_mode = false; - bool search_mode = false; - std::string data_prefix = "data"; - std::string index_prefix = "index"; - std::string new_override_kv = ""; - std::string mode = "latency"; - std::string threads_arg_txt = ""; - std::vector threads = {1, -1}; // min_thread, max_thread - std::string log_level_str = ""; - int raft_log_level = raft::logger::get(RAFT_NAME).get_level(); - kv_series override_kv{}; - - char arg0_default[] = "benchmark"; // NOLINT - char* args_default = arg0_default; - if (!argv) { - argc = 1; - argv = &args_default; - } - if (argc == 1) { - printf_usage(); - return -1; - } - - char* conf_path = argv[--argc]; - std::ifstream conf_stream(conf_path); - - for (int i = 1; i < argc; i++) { - if (parse_bool_flag(argv[i], "--force", force_overwrite) || - parse_bool_flag(argv[i], "--build", build_mode) || - parse_bool_flag(argv[i], "--search", search_mode) || - parse_string_flag(argv[i], "--data_prefix", data_prefix) || - parse_string_flag(argv[i], "--index_prefix", index_prefix) || - parse_string_flag(argv[i], "--mode", mode) || - parse_string_flag(argv[i], "--override_kv", new_override_kv) || - parse_string_flag(argv[i], "--threads", threads_arg_txt) || - parse_string_flag(argv[i], "--raft_log_level", log_level_str)) { - if (!log_level_str.empty()) { - raft_log_level = std::stoi(log_level_str); - log_level_str = ""; - } - if (!threads_arg_txt.empty()) { - auto threads_arg = split(threads_arg_txt, ':'); - threads[0] = std::stoi(threads_arg[0]); - if (threads_arg.size() > 1) { - threads[1] = std::stoi(threads_arg[1]); - } else { - threads[1] = threads[0]; - } - threads_arg_txt = ""; - } - if (!new_override_kv.empty()) { - auto kvv = split(new_override_kv, ':'); - auto key = kvv[0]; - std::vector vals{}; - for (std::size_t j = 1; j < kvv.size(); j++) { - vals.push_back(nlohmann::json::parse(kvv[j])); - } - override_kv.emplace_back(key, vals); - new_override_kv = ""; - } - for (int j = i; j < argc - 1; j++) { - argv[j] = argv[j + 1]; - } - argc--; - i--; - } - } - - raft::logger::get(RAFT_NAME).set_level(raft_log_level); - - Objective metric_objective = Objective::LATENCY; - if (mode == "throughput") { metric_objective = Objective::THROUGHPUT; } - - int max_threads = - (metric_objective == Objective::THROUGHPUT) ? std::thread::hardware_concurrency() : 1; - if (threads[1] == -1) threads[1] = max_threads; - - if (metric_objective == Objective::LATENCY) { - if (threads[0] != 1 || threads[1] != 1) { - log_warn("Latency mode enabled. Overriding threads arg, running with single thread."); - threads = {1, 1}; - } - } - - if (build_mode == search_mode) { - log_error("One and only one of --build and --search should be specified"); - printf_usage(); - return -1; - } - - if (!conf_stream) { - log_error("Can't open configuration file: %s", conf_path); - return -1; - } - - if (cudart.needed() && !cudart.found()) { - log_warn("cudart library is not found, GPU-based indices won't work."); - } - - Configuration conf(conf_stream); - std::string dtype = conf.get_dataset_conf().dtype; - - if (dtype == "float") { - dispatch_benchmark(conf, - force_overwrite, - build_mode, - search_mode, - data_prefix, - index_prefix, - override_kv, - metric_objective, - threads); - } else if (dtype == "half") { - dispatch_benchmark(conf, - force_overwrite, - build_mode, - search_mode, - data_prefix, - index_prefix, - override_kv, - metric_objective, - threads); - } else if (dtype == "uint8") { - dispatch_benchmark(conf, - force_overwrite, - build_mode, - search_mode, - data_prefix, - index_prefix, - override_kv, - metric_objective, - threads); - } else if (dtype == "int8") { - dispatch_benchmark(conf, - force_overwrite, - build_mode, - search_mode, - data_prefix, - index_prefix, - override_kv, - metric_objective, - threads); - } else { - log_error("datatype '%s' is not supported", dtype.c_str()); - return -1; - } - - ::benchmark::Initialize(&argc, argv, printf_usage); - if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return -1; - ::benchmark::RunSpecifiedBenchmarks(); - ::benchmark::Shutdown(); - // Release a possibly cached ANN object, so that it cannot be alive longer than the handle - // to a shared library it depends on (dynamic benchmark executable). - current_algo.reset(); - current_algo_props.reset(); - reset_global_device_resources(); - return 0; -} -}; // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/common/conf.hpp b/cpp/bench/ann/src/common/conf.hpp deleted file mode 100644 index 92ba86c6cf..0000000000 --- a/cpp/bench/ann/src/common/conf.hpp +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "util.hpp" - -#include -#include -#include -#include -#include - -#define JSON_DIAGNOSTICS 1 -#include - -namespace raft::bench::ann { - -class Configuration { - public: - struct Index { - std::string name; - std::string algo; - nlohmann::json build_param; - std::string file; - std::vector dev_list; - - int batch_size; - int k; - std::vector search_params; - }; - - struct DatasetConf { - std::string name; - std::string base_file; - // use only a subset of base_file, - // the range of rows is [subset_first_row, subset_first_row + subset_size) - // however, subset_size = 0 means using all rows after subset_first_row - // that is, the subset is [subset_first_row, #rows in base_file) - size_t subset_first_row{0}; - size_t subset_size{0}; - std::string query_file; - std::string distance; - std::optional groundtruth_neighbors_file{std::nullopt}; - - // data type of input dataset, possible values ["float", "int8", "uint8"] - std::string dtype; - }; - - explicit inline Configuration(std::istream& conf_stream) - { - // to enable comments in json - auto conf = nlohmann::json::parse(conf_stream, nullptr, true, true); - - parse_dataset_(conf.at("dataset")); - parse_index_(conf.at("index"), conf.at("search_basic_param")); - } - - [[nodiscard]] inline auto get_dataset_conf() const -> DatasetConf { return dataset_conf_; } - [[nodiscard]] inline auto get_indices() const -> std::vector { return indices_; }; - - private: - inline void parse_dataset_(const nlohmann::json& conf) - { - dataset_conf_.name = conf.at("name"); - dataset_conf_.base_file = conf.at("base_file"); - dataset_conf_.query_file = conf.at("query_file"); - dataset_conf_.distance = conf.at("distance"); - - if (conf.contains("groundtruth_neighbors_file")) { - dataset_conf_.groundtruth_neighbors_file = conf.at("groundtruth_neighbors_file"); - } - if (conf.contains("subset_first_row")) { - dataset_conf_.subset_first_row = conf.at("subset_first_row"); - } - if (conf.contains("subset_size")) { dataset_conf_.subset_size = conf.at("subset_size"); } - - if (conf.contains("dtype")) { - dataset_conf_.dtype = conf.at("dtype"); - } else { - auto filename = dataset_conf_.base_file; - if (filename.size() > 6 && filename.compare(filename.size() - 6, 6, "f16bin") == 0) { - dataset_conf_.dtype = "half"; - } else if (filename.size() > 9 && - filename.compare(filename.size() - 9, 9, "fp16.fbin") == 0) { - dataset_conf_.dtype = "half"; - } else if (filename.size() > 4 && filename.compare(filename.size() - 4, 4, "fbin") == 0) { - dataset_conf_.dtype = "float"; - } else if (filename.size() > 5 && filename.compare(filename.size() - 5, 5, "u8bin") == 0) { - dataset_conf_.dtype = "uint8"; - } else if (filename.size() > 5 && filename.compare(filename.size() - 5, 5, "i8bin") == 0) { - dataset_conf_.dtype = "int8"; - } else { - log_error("Could not determine data type of the dataset %s", filename.c_str()); - } - } - } - inline void parse_index_(const nlohmann::json& index_conf, - const nlohmann::json& search_basic_conf) - { - const int batch_size = search_basic_conf.at("batch_size"); - const int k = search_basic_conf.at("k"); - - for (const auto& conf : index_conf) { - Index index; - index.name = conf.at("name"); - index.algo = conf.at("algo"); - index.build_param = conf.at("build_param"); - index.file = conf.at("file"); - index.batch_size = batch_size; - index.k = k; - - if (conf.contains("multigpu")) { - for (auto it : conf.at("multigpu")) { - index.dev_list.push_back(it); - } - if (index.dev_list.empty()) { throw std::runtime_error("dev_list shouln't be empty!"); } - index.dev_list.shrink_to_fit(); - index.build_param["multigpu"] = conf["multigpu"]; - } - - for (auto param : conf.at("search_params")) { - /* ### Special parameters for backward compatibility ### - - - Local values of `k` and `n_queries` take priority. - - The legacy "batch_size" renamed to `n_queries`. - - Basic search params are used otherwise. - */ - if (!param.contains("k")) { param["k"] = k; } - if (!param.contains("n_queries")) { - if (param.contains("batch_size")) { - param["n_queries"] = param["batch_size"]; - param.erase("batch_size"); - } else { - param["n_queries"] = batch_size; - } - } - index.search_params.push_back(param); - } - - indices_.push_back(index); - } - } - - DatasetConf dataset_conf_; - std::vector indices_; -}; - -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/common/cuda_huge_page_resource.hpp b/cpp/bench/ann/src/common/cuda_huge_page_resource.hpp deleted file mode 100644 index 27be26dfe9..0000000000 --- a/cpp/bench/ann/src/common/cuda_huge_page_resource.hpp +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include - -#include -#include - -#include - -#include -#include - -namespace raft::mr { -/** - * @brief `device_memory_resource` derived class that uses mmap to allocate memory. - * This class enables memory allocation using huge pages. - * It is assumed that the allocated memory is directly accessible on device. This currently only - * works on GH systems. - * - * TODO(tfeher): consider improving or removing this helper once we made progress with - * https://github.com/rapidsai/raft/issues/1819 - */ -class cuda_huge_page_resource final : public rmm::mr::device_memory_resource { - public: - cuda_huge_page_resource() = default; - ~cuda_huge_page_resource() override = default; - cuda_huge_page_resource(cuda_huge_page_resource const&) = default; - cuda_huge_page_resource(cuda_huge_page_resource&&) = default; - cuda_huge_page_resource& operator=(cuda_huge_page_resource const&) = default; - cuda_huge_page_resource& operator=(cuda_huge_page_resource&&) = default; - - private: - /** - * @brief Allocates memory of size at least `bytes` using cudaMalloc. - * - * The returned pointer has at least 256B alignment. - * - * @note Stream argument is ignored - * - * @throws `rmm::bad_alloc` if the requested allocation could not be fulfilled - * - * @param bytes The size, in bytes, of the allocation - * @return void* Pointer to the newly allocated memory - */ - void* do_allocate(std::size_t bytes, rmm::cuda_stream_view) override - { - void* _addr{nullptr}; - _addr = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (_addr == MAP_FAILED) { RAFT_FAIL("huge_page_resource::MAP FAILED"); } - if (madvise(_addr, bytes, MADV_HUGEPAGE) == -1) { - munmap(_addr, bytes); - RAFT_FAIL("huge_page_resource::madvise MADV_HUGEPAGE"); - } - memset(_addr, 0, bytes); - return _addr; - } - - /** - * @brief Deallocate memory pointed to by \p p. - * - * @note Stream argument is ignored. - * - * @throws Nothing. - * - * @param p Pointer to be deallocated - */ - void do_deallocate(void* ptr, std::size_t size, rmm::cuda_stream_view) override - { - if (munmap(ptr, size) == -1) { RAFT_FAIL("huge_page_resource::munmap"); } - } - - /** - * @brief Compare this resource to another. - * - * Two cuda_huge_page_resources always compare equal, because they can each - * deallocate memory allocated by the other. - * - * @throws Nothing. - * - * @param other The other resource to compare to - * @return true If the two resources are equivalent - * @return false If the two resources are not equal - */ - [[nodiscard]] bool do_is_equal(device_memory_resource const& other) const noexcept override - { - return dynamic_cast(&other) != nullptr; - } -}; -} // namespace raft::mr diff --git a/cpp/bench/ann/src/common/cuda_pinned_resource.hpp b/cpp/bench/ann/src/common/cuda_pinned_resource.hpp deleted file mode 100644 index 3256fc293c..0000000000 --- a/cpp/bench/ann/src/common/cuda_pinned_resource.hpp +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include -#include - -#include - -namespace raft::mr { -/** - * @brief `device_memory_resource` derived class that uses cudaMallocHost/Free for - * allocation/deallocation. - * - * This is almost the same as rmm::mr::host::pinned_memory_resource, but it has - * device_memory_resource as base class. Pinned memory can be accessed from device, - * and using this allocator we can create device_mdarray backed by pinned allocator. - * - * TODO(tfeher): it would be preferred to just rely on the existing allocator from rmm - * (pinned_memory_resource), but that is incompatible with the container_policy class - * for device matrix, because the latter expects a device_memory_resource. We shall - * revise this once we progress with Issue https://github.com/rapidsai/raft/issues/1819 - */ -class cuda_pinned_resource final : public rmm::mr::device_memory_resource { - public: - cuda_pinned_resource() = default; - ~cuda_pinned_resource() override = default; - cuda_pinned_resource(cuda_pinned_resource const&) = default; - cuda_pinned_resource(cuda_pinned_resource&&) = default; - cuda_pinned_resource& operator=(cuda_pinned_resource const&) = default; - cuda_pinned_resource& operator=(cuda_pinned_resource&&) = default; - - private: - /** - * @brief Allocates memory of size at least `bytes` using cudaMalloc. - * - * The returned pointer has at least 256B alignment. - * - * @note Stream argument is ignored - * - * @throws `rmm::bad_alloc` if the requested allocation could not be fulfilled - * - * @param bytes The size, in bytes, of the allocation - * @return void* Pointer to the newly allocated memory - */ - void* do_allocate(std::size_t bytes, rmm::cuda_stream_view) override - { - void* ptr{nullptr}; - RMM_CUDA_TRY_ALLOC(cudaMallocHost(&ptr, bytes)); - return ptr; - } - - /** - * @brief Deallocate memory pointed to by \p p. - * - * @note Stream argument is ignored. - * - * @throws Nothing. - * - * @param p Pointer to be deallocated - */ - void do_deallocate(void* ptr, std::size_t, rmm::cuda_stream_view) override - { - RMM_ASSERT_CUDA_SUCCESS(cudaFreeHost(ptr)); - } - - /** - * @brief Compare this resource to another. - * - * Two cuda_pinned_resources always compare equal, because they can each - * deallocate memory allocated by the other. - * - * @throws Nothing. - * - * @param other The other resource to compare to - * @return true If the two resources are equivalent - * @return false If the two resources are not equal - */ - [[nodiscard]] bool do_is_equal(device_memory_resource const& other) const noexcept override - { - return dynamic_cast(&other) != nullptr; - } -}; -} // namespace raft::mr diff --git a/cpp/bench/ann/src/common/cuda_stub.hpp b/cpp/bench/ann/src/common/cuda_stub.hpp deleted file mode 100644 index 5ed138a86d..0000000000 --- a/cpp/bench/ann/src/common/cuda_stub.hpp +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* -The content of this header is governed by two preprocessor definitions: - - - BUILD_CPU_ONLY - whether none of the CUDA functions are used. - - ANN_BENCH_LINK_CUDART - dynamically link against this string if defined. - -___________________________________________________________________________________ -|BUILD_CPU_ONLY | ANN_BENCH_LINK_CUDART | cudart | cuda_runtime_api.h | -| | | found | needed | included | -|---------------|-----------------------|-----------|---------|--------------------| -| ON | | false | false | NO | -| ON | "cudart.so.xx.xx" | false | false | NO | -| OFF | | true | true | YES | -| OFF | "cudart.so.xx.xx" | | true | YES | ------------------------------------------------------------------------------------- -*/ - -#pragma once - -#ifndef BUILD_CPU_ONLY -#include -#include -#ifdef ANN_BENCH_LINK_CUDART -#include - -#include -#endif -#else -#include - -typedef void* cudaStream_t; -typedef void* cudaEvent_t; -typedef uint16_t half; -#endif - -namespace raft::bench::ann { - -struct cuda_lib_handle { - void* handle{nullptr}; - explicit cuda_lib_handle() - { -#ifdef ANN_BENCH_LINK_CUDART - constexpr int kFlags = RTLD_NOW | RTLD_GLOBAL | RTLD_DEEPBIND | RTLD_NODELETE; - // The full name of the linked cudart library 'cudart.so.MAJOR.MINOR.PATCH' - char libname[] = ANN_BENCH_LINK_CUDART; // NOLINT - handle = dlopen(ANN_BENCH_LINK_CUDART, kFlags); - if (handle != nullptr) { return; } - // try strip the PATCH - auto p = strrchr(libname, '.'); - p[0] = 0; - handle = dlopen(libname, kFlags); - if (handle != nullptr) { return; } - // try set the MINOR version to 0 - p = strrchr(libname, '.'); - p[1] = '0'; - p[2] = 0; - handle = dlopen(libname, kFlags); - if (handle != nullptr) { return; } - // try strip the MINOR - p[0] = 0; - handle = dlopen(libname, kFlags); - if (handle != nullptr) { return; } - // try strip the MAJOR - p = strrchr(libname, '.'); - p[0] = 0; - handle = dlopen(libname, kFlags); -#endif - } - ~cuda_lib_handle() noexcept - { -#ifdef ANN_BENCH_LINK_CUDART - if (handle != nullptr) { dlclose(handle); } -#endif - } - - template - auto sym(const char* name) -> Symbol - { -#ifdef ANN_BENCH_LINK_CUDART - return reinterpret_cast(dlsym(handle, name)); -#else - return nullptr; -#endif - } - - /** Whether this is NOT a cpu-only package. */ - [[nodiscard]] constexpr inline auto needed() const -> bool - { -#if defined(BUILD_CPU_ONLY) - return false; -#else - return true; -#endif - } - - /** CUDA found, either at compile time or at runtime. */ - [[nodiscard]] inline auto found() const -> bool - { -#if defined(BUILD_CPU_ONLY) - return false; -#elif defined(ANN_BENCH_LINK_CUDART) - return handle != nullptr; -#else - return true; -#endif - } -}; - -static inline cuda_lib_handle cudart{}; - -#ifdef ANN_BENCH_LINK_CUDART -namespace stub { - -[[gnu::weak, gnu::noinline]] cudaError_t cudaMemcpy(void* dst, - const void* src, - size_t count, - enum cudaMemcpyKind kind) -{ - return cudaSuccess; -} - -[[gnu::weak, gnu::noinline]] cudaError_t cudaMalloc(void** ptr, size_t size) -{ - *ptr = nullptr; - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaMemset(void* devPtr, int value, size_t count) -{ - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaFree(void* devPtr) { return cudaSuccess; } -[[gnu::weak, gnu::noinline]] cudaError_t cudaStreamCreate(cudaStream_t* pStream) -{ - *pStream = 0; - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaStreamCreateWithFlags(cudaStream_t* pStream, - unsigned int flags) -{ - *pStream = 0; - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaStreamDestroy(cudaStream_t pStream) -{ - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaDeviceSynchronize() { return cudaSuccess; } - -[[gnu::weak, gnu::noinline]] cudaError_t cudaStreamSynchronize(cudaStream_t pStream) -{ - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaEventCreate(cudaEvent_t* event) -{ - *event = 0; - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaEventRecord(cudaEvent_t event, cudaStream_t stream) -{ - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaEventSynchronize(cudaEvent_t event) -{ - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaEventElapsedTime(float* ms, - cudaEvent_t start, - cudaEvent_t end) -{ - *ms = 0; - return cudaSuccess; -} -[[gnu::weak, gnu::noinline]] cudaError_t cudaEventDestroy(cudaEvent_t event) { return cudaSuccess; } -[[gnu::weak, gnu::noinline]] cudaError_t cudaGetDevice(int* device) -{ - *device = 0; - return cudaSuccess; -}; -[[gnu::weak, gnu::noinline]] cudaError_t cudaDriverGetVersion(int* driver) -{ - *driver = 0; - return cudaSuccess; -}; -[[gnu::weak, gnu::noinline]] cudaError_t cudaRuntimeGetVersion(int* runtime) -{ - *runtime = 0; - return cudaSuccess; -}; -[[gnu::weak, gnu::noinline]] cudaError_t cudaGetDeviceProperties(struct cudaDeviceProp* prop, - int device) -{ - *prop = cudaDeviceProp{}; - return cudaSuccess; -} - -} // namespace stub - -#define RAFT_DECLARE_CUDART(fun) \ - static inline decltype(&stub::fun) fun = \ - cudart.found() ? cudart.sym(#fun) : &stub::fun - -RAFT_DECLARE_CUDART(cudaMemcpy); -RAFT_DECLARE_CUDART(cudaMalloc); -RAFT_DECLARE_CUDART(cudaMemset); -RAFT_DECLARE_CUDART(cudaFree); -RAFT_DECLARE_CUDART(cudaStreamCreate); -RAFT_DECLARE_CUDART(cudaStreamCreateWithFlags); -RAFT_DECLARE_CUDART(cudaStreamDestroy); -RAFT_DECLARE_CUDART(cudaDeviceSynchronize); -RAFT_DECLARE_CUDART(cudaStreamSynchronize); -RAFT_DECLARE_CUDART(cudaEventCreate); -RAFT_DECLARE_CUDART(cudaEventRecord); -RAFT_DECLARE_CUDART(cudaEventSynchronize); -RAFT_DECLARE_CUDART(cudaEventElapsedTime); -RAFT_DECLARE_CUDART(cudaEventDestroy); -RAFT_DECLARE_CUDART(cudaGetDevice); -RAFT_DECLARE_CUDART(cudaDriverGetVersion); -RAFT_DECLARE_CUDART(cudaRuntimeGetVersion); -RAFT_DECLARE_CUDART(cudaGetDeviceProperties); - -#undef RAFT_DECLARE_CUDART -#endif - -}; // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/common/dataset.hpp b/cpp/bench/ann/src/common/dataset.hpp deleted file mode 100644 index 8fcff77d3c..0000000000 --- a/cpp/bench/ann/src/common/dataset.hpp +++ /dev/null @@ -1,495 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "util.hpp" - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -// http://big-ann-benchmarks.com/index.html: -// binary format that starts with 8 bytes of data consisting of num_points(uint32_t) -// num_dimensions(uint32) followed by num_pts x num_dimensions x sizeof(type) bytes of -// data stored one vector after another. -// Data files will have suffixes .fbin, .u8bin, and .i8bin to represent float32, uint8 -// and int8 type data. -// As extensions for this benchmark, half and int data files will have suffixes .f16bin -// and .ibin, respectively. -template -class BinFile { - public: - BinFile(const std::string& file, - const std::string& mode, - uint32_t subset_first_row = 0, - uint32_t subset_size = 0); - ~BinFile() - { - if (mapped_ptr_ != nullptr) { unmap(); } - if (fp_ != nullptr) { fclose(fp_); } - } - BinFile(const BinFile&) = delete; - BinFile& operator=(const BinFile&) = delete; - - void get_shape(size_t* nrows, int* ndims) const - { - assert(read_mode_); - if (!fp_) { open_file_(); } - *nrows = nrows_; - *ndims = ndims_; - } - - void read(T* data) const - { - assert(read_mode_); - if (!fp_) { open_file_(); } - size_t total = static_cast(nrows_) * ndims_; - if (fread(data, sizeof(T), total, fp_) != total) { - throw std::runtime_error("fread() BinFile " + file_ + " failed"); - } - } - - void write(const T* data, uint32_t nrows, uint32_t ndims) - { - assert(!read_mode_); - if (!fp_) { open_file_(); } - if (fwrite(&nrows, sizeof(uint32_t), 1, fp_) != 1) { - throw std::runtime_error("fwrite() BinFile " + file_ + " failed"); - } - if (fwrite(&ndims, sizeof(uint32_t), 1, fp_) != 1) { - throw std::runtime_error("fwrite() BinFile " + file_ + " failed"); - } - - size_t total = static_cast(nrows) * ndims; - if (fwrite(data, sizeof(T), total, fp_) != total) { - throw std::runtime_error("fwrite() BinFile " + file_ + " failed"); - } - } - - T* map() const - { - assert(read_mode_); - if (!fp_) { open_file_(); } - int fid = fileno(fp_); - mapped_ptr_ = mmap(nullptr, file_size_, PROT_READ, MAP_PRIVATE, fid, 0); - if (mapped_ptr_ == MAP_FAILED) { - mapped_ptr_ = nullptr; - throw std::runtime_error("mmap error: Value of errno " + std::to_string(errno) + ", " + - std::string(strerror(errno))); - } - return reinterpret_cast(reinterpret_cast(mapped_ptr_) + 2 * sizeof(uint32_t) + - subset_first_row_ * ndims_ * sizeof(T)); - } - - void unmap() const - { - if (munmap(mapped_ptr_, file_size_) == -1) { - throw std::runtime_error("munmap error: " + std::string(strerror(errno))); - } - } - - private: - void check_suffix_(); - void open_file_() const; - - std::string file_; - bool read_mode_; - uint32_t subset_first_row_; - uint32_t subset_size_; - - mutable FILE* fp_{nullptr}; - mutable uint32_t nrows_; - mutable uint32_t ndims_; - mutable size_t file_size_; - mutable void* mapped_ptr_{nullptr}; -}; - -template -BinFile::BinFile(const std::string& file, - const std::string& mode, - uint32_t subset_first_row, - uint32_t subset_size) - : file_(file), - read_mode_(mode == "r"), - subset_first_row_(subset_first_row), - subset_size_(subset_size), - fp_(nullptr) -{ - check_suffix_(); - - if (!read_mode_) { - if (mode == "w") { - if (subset_first_row != 0) { - throw std::runtime_error("subset_first_row should be zero for write mode"); - } - if (subset_size != 0) { - throw std::runtime_error("subset_size should be zero for write mode"); - } - } else { - throw std::runtime_error("BinFile's mode must be either 'r' or 'w': " + file_); - } - } -} - -template -void BinFile::open_file_() const -{ - fp_ = fopen(file_.c_str(), read_mode_ ? "r" : "w"); - if (!fp_) { throw std::runtime_error("open BinFile failed: " + file_); } - - if (read_mode_) { - struct stat statbuf; - if (stat(file_.c_str(), &statbuf) != 0) { throw std::runtime_error("stat() failed: " + file_); } - file_size_ = statbuf.st_size; - - uint32_t header[2]; - if (fread(header, sizeof(uint32_t), 2, fp_) != 2) { - throw std::runtime_error("read header of BinFile failed: " + file_); - } - nrows_ = header[0]; - ndims_ = header[1]; - - size_t expected_file_size = - 2 * sizeof(uint32_t) + static_cast(nrows_) * ndims_ * sizeof(T); - if (file_size_ != expected_file_size) { - throw std::runtime_error("expected file size of " + file_ + " is " + - std::to_string(expected_file_size) + ", however, actual size is " + - std::to_string(file_size_)); - } - - if (subset_first_row_ >= nrows_) { - throw std::runtime_error(file_ + ": subset_first_row (" + std::to_string(subset_first_row_) + - ") >= nrows (" + std::to_string(nrows_) + ")"); - } - if (subset_first_row_ + subset_size_ > nrows_) { - throw std::runtime_error(file_ + ": subset_first_row (" + std::to_string(subset_first_row_) + - ") + subset_size (" + std::to_string(subset_size_) + ") > nrows (" + - std::to_string(nrows_) + ")"); - } - - if (subset_first_row_) { - static_assert(sizeof(long) == 8, "fseek() don't support 64-bit offset"); - if (fseek(fp_, sizeof(T) * subset_first_row_ * ndims_, SEEK_CUR) == -1) { - throw std::runtime_error(file_ + ": fseek failed"); - } - nrows_ -= subset_first_row_; - } - if (subset_size_) { nrows_ = subset_size_; } - } -} - -template -void BinFile::check_suffix_() -{ - auto pos = file_.rfind('.'); - if (pos == std::string::npos) { - throw std::runtime_error("name of BinFile doesn't have a suffix: " + file_); - } - std::string suffix = file_.substr(pos + 1); - - if constexpr (std::is_same_v) { - if (suffix != "fbin") { - throw std::runtime_error("BinFile should has .fbin suffix: " + file_); - } - } else if constexpr (std::is_same_v) { - if (suffix != "f16bin" && suffix != "fbin") { - throw std::runtime_error("BinFile should has .f16bin suffix: " + file_); - } - } else if constexpr (std::is_same_v) { - if (suffix != "ibin") { - throw std::runtime_error("BinFile should has .ibin suffix: " + file_); - } - } else if constexpr (std::is_same_v) { - if (suffix != "u8bin") { - throw std::runtime_error("BinFile should has .u8bin suffix: " + file_); - } - } else if constexpr (std::is_same_v) { - if (suffix != "i8bin") { - throw std::runtime_error("BinFile should has .i8bin suffix: " + file_); - } - } else { - throw std::runtime_error( - "T of BinFile should be one of float, half, int, uint8_t, or int8_t"); - } -} - -template -class Dataset { - public: - Dataset(const std::string& name) : name_(name) {} - Dataset(const std::string& name, const std::string& distance) : name_(name), distance_(distance) - { - } - Dataset(const Dataset&) = delete; - Dataset& operator=(const Dataset&) = delete; - virtual ~Dataset(); - - std::string name() const { return name_; } - std::string distance() const { return distance_; } - virtual int dim() const = 0; - virtual uint32_t max_k() const = 0; - virtual size_t base_set_size() const = 0; - virtual size_t query_set_size() const = 0; - - // load data lazily, so don't pay the overhead of reading unneeded set - // e.g. don't load base set when searching - const T* base_set() const - { - if (!base_set_) { load_base_set_(); } - return base_set_; - } - - const T* query_set() const - { - if (!query_set_) { load_query_set_(); } - return query_set_; - } - - const int32_t* gt_set() const - { - if (!gt_set_) { load_gt_set_(); } - return gt_set_; - } - - const T* base_set_on_gpu() const; - const T* query_set_on_gpu() const; - const T* mapped_base_set() const; - - auto query_set(MemoryType memory_type) const -> const T* - { - switch (memory_type) { - case MemoryType::Device: return query_set_on_gpu(); - default: return query_set(); - } - } - - auto base_set(MemoryType memory_type) const -> const T* - { - switch (memory_type) { - case MemoryType::Device: return base_set_on_gpu(); - case MemoryType::Host: return base_set(); - case MemoryType::HostMmap: return mapped_base_set(); - default: return nullptr; - } - } - - protected: - virtual void load_base_set_() const = 0; - virtual void load_gt_set_() const = 0; - virtual void load_query_set_() const = 0; - virtual void map_base_set_() const = 0; - - std::string name_; - std::string distance_; - - mutable T* base_set_ = nullptr; - mutable T* query_set_ = nullptr; - mutable T* d_base_set_ = nullptr; - mutable T* d_query_set_ = nullptr; - mutable T* mapped_base_set_ = nullptr; - mutable int32_t* gt_set_ = nullptr; -}; - -template -Dataset::~Dataset() -{ - delete[] base_set_; - delete[] query_set_; - delete[] gt_set_; -#ifndef BUILD_CPU_ONLY - if (d_base_set_) { cudaFree(d_base_set_); } - if (d_query_set_) { cudaFree(d_query_set_); } -#endif -} - -template -const T* Dataset::base_set_on_gpu() const -{ -#ifndef BUILD_CPU_ONLY - if (!d_base_set_) { - base_set(); - cudaMalloc((void**)&d_base_set_, base_set_size() * dim() * sizeof(T)); - cudaMemcpy(d_base_set_, base_set_, base_set_size() * dim() * sizeof(T), cudaMemcpyHostToDevice); - } -#endif - return d_base_set_; -} - -template -const T* Dataset::query_set_on_gpu() const -{ -#ifndef BUILD_CPU_ONLY - if (!d_query_set_) { - query_set(); - cudaMalloc((void**)&d_query_set_, query_set_size() * dim() * sizeof(T)); - cudaMemcpy( - d_query_set_, query_set_, query_set_size() * dim() * sizeof(T), cudaMemcpyHostToDevice); - } -#endif - return d_query_set_; -} - -template -const T* Dataset::mapped_base_set() const -{ - if (!mapped_base_set_) { map_base_set_(); } - return mapped_base_set_; -} - -template -class BinDataset : public Dataset { - public: - BinDataset(const std::string& name, - const std::string& base_file, - size_t subset_first_row, - size_t subset_size, - const std::string& query_file, - const std::string& distance, - const std::optional& groundtruth_neighbors_file); - - int dim() const override; - uint32_t max_k() const override; - size_t base_set_size() const override; - size_t query_set_size() const override; - - private: - void load_base_set_() const override; - void load_query_set_() const override; - void load_gt_set_() const override; - void map_base_set_() const override; - - mutable int dim_ = 0; - mutable uint32_t max_k_ = 0; - mutable size_t base_set_size_ = 0; - mutable size_t query_set_size_ = 0; - - BinFile base_file_; - BinFile query_file_; - std::optional> gt_file_{std::nullopt}; -}; - -template -BinDataset::BinDataset(const std::string& name, - const std::string& base_file, - size_t subset_first_row, - size_t subset_size, - const std::string& query_file, - const std::string& distance, - const std::optional& groundtruth_neighbors_file) - : Dataset(name, distance), - base_file_(base_file, "r", subset_first_row, subset_size), - query_file_(query_file, "r") -{ - if (groundtruth_neighbors_file.has_value()) { - gt_file_.emplace(groundtruth_neighbors_file.value(), "r"); - } -} - -template -int BinDataset::dim() const -{ - if (dim_ > 0) { return dim_; } - if (base_set_size() > 0) { return dim_; } - if (query_set_size() > 0) { return dim_; } - return dim_; -} - -template -uint32_t BinDataset::max_k() const -{ - if (!this->gt_set_) { load_gt_set_(); } - return max_k_; -} - -template -size_t BinDataset::query_set_size() const -{ - if (query_set_size_ > 0) { return query_set_size_; } - int dim; - query_file_.get_shape(&query_set_size_, &dim); - if (query_set_size_ == 0) { throw std::runtime_error("Zero query set size"); } - if (dim == 0) { throw std::runtime_error("Zero query set dim"); } - if (dim_ == 0) { - dim_ = dim; - } else if (dim_ != dim) { - throw std::runtime_error("base set dim (" + std::to_string(dim_) + ") != query set dim (" + - std::to_string(dim)); - } - return query_set_size_; -} - -template -size_t BinDataset::base_set_size() const -{ - if (base_set_size_ > 0) { return base_set_size_; } - int dim; - base_file_.get_shape(&base_set_size_, &dim); - if (base_set_size_ == 0) { throw std::runtime_error("Zero base set size"); } - if (dim == 0) { throw std::runtime_error("Zero base set dim"); } - if (dim_ == 0) { - dim_ = dim; - } else if (dim_ != dim) { - throw std::runtime_error("base set dim (" + std::to_string(dim) + ") != query set dim (" + - std::to_string(dim_)); - } - return base_set_size_; -} - -template -void BinDataset::load_base_set_() const -{ - this->base_set_ = new T[base_set_size() * dim()]; - base_file_.read(this->base_set_); -} - -template -void BinDataset::load_query_set_() const -{ - this->query_set_ = new T[query_set_size() * dim()]; - query_file_.read(this->query_set_); -} - -template -void BinDataset::load_gt_set_() const -{ - if (gt_file_.has_value()) { - size_t queries; - int k; - gt_file_->get_shape(&queries, &k); - this->gt_set_ = new std::int32_t[queries * k]; - gt_file_->read(this->gt_set_); - max_k_ = k; - } -} - -template -void BinDataset::map_base_set_() const -{ - this->mapped_base_set_ = base_file_.map(); -} - -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/common/thread_pool.hpp b/cpp/bench/ann/src/common/thread_pool.hpp deleted file mode 100644 index 4a5684ecb3..0000000000 --- a/cpp/bench/ann/src/common/thread_pool.hpp +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include - -class FixedThreadPool { - public: - FixedThreadPool(int num_threads) - { - if (num_threads < 1) { - throw std::runtime_error("num_threads must >= 1"); - } else if (num_threads == 1) { - return; - } - - tasks_ = new Task_[num_threads]; - - threads_.reserve(num_threads); - for (int i = 0; i < num_threads; ++i) { - threads_.emplace_back([&, i] { - auto& task = tasks_[i]; - while (true) { - std::unique_lock lock(task.mtx); - task.cv.wait(lock, - [&] { return task.has_task || finished_.load(std::memory_order_relaxed); }); - if (finished_.load(std::memory_order_relaxed)) { break; } - - task.task(); - task.has_task = false; - } - }); - } - } - - ~FixedThreadPool() - { - if (threads_.empty()) { return; } - - finished_.store(true, std::memory_order_relaxed); - for (unsigned i = 0; i < threads_.size(); ++i) { - auto& task = tasks_[i]; - std::lock_guard(task.mtx); - - task.cv.notify_one(); - threads_[i].join(); - } - - delete[] tasks_; - } - - template - void submit(Func f, IdxT len) - { - // Run functions in main thread if thread pool has no threads - if (threads_.empty()) { - for (IdxT i = 0; i < len; ++i) { - f(i); - } - return; - } - - const int num_threads = threads_.size(); - // one extra part for competition among threads - const IdxT items_per_thread = len / (num_threads + 1); - std::atomic cnt(items_per_thread * num_threads); - - // Wrap function - auto wrapped_f = [&](IdxT start, IdxT end) { - for (IdxT i = start; i < end; ++i) { - f(i); - } - - while (true) { - IdxT i = cnt.fetch_add(1, std::memory_order_relaxed); - if (i >= len) { break; } - f(i); - } - }; - - std::vector> futures; - futures.reserve(num_threads); - for (int i = 0; i < num_threads; ++i) { - IdxT start = i * items_per_thread; - auto& task = tasks_[i]; - { - std::lock_guard lock(task.mtx); - (void)lock; // stop nvcc warning - task.task = std::packaged_task([=] { wrapped_f(start, start + items_per_thread); }); - futures.push_back(task.task.get_future()); - task.has_task = true; - } - task.cv.notify_one(); - } - - for (auto& fut : futures) { - fut.wait(); - } - return; - } - - private: - struct alignas(64) Task_ { - std::mutex mtx; - std::condition_variable cv; - bool has_task = false; - std::packaged_task task; - }; - - Task_* tasks_; - std::vector threads_; - std::atomic finished_{false}; -}; diff --git a/cpp/bench/ann/src/common/util.hpp b/cpp/bench/ann/src/common/util.hpp deleted file mode 100644 index 96185c79eb..0000000000 --- a/cpp/bench/ann/src/common/util.hpp +++ /dev/null @@ -1,557 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "ann_types.hpp" -#include "cuda_stub.hpp" // cuda-related utils - -#ifdef ANN_BENCH_NVTX3_HEADERS_FOUND -#include -#endif - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -/** - * Current thread id as given by the benchmark State. - * It's populated on every call of a benchmark case. - * It's relevant in the 'throughput' mode of the search benchmarks, - * where some algorithms might want to coordinate allocation of the resources. - */ -inline thread_local int benchmark_thread_id = 0; -/** - * Total concurrent thread count as given by the benchmark State. - * It's populated on every call of a benchmark case. - * It's relevant in the 'throughput' mode of the search benchmarks, - * where some algorithms might want to coordinate allocation of the resources. - */ -inline thread_local int benchmark_n_threads = 1; - -struct cuda_timer { - private: - std::optional stream_; - cudaEvent_t start_{nullptr}; - cudaEvent_t stop_{nullptr}; - double total_time_{0}; - - template - static inline auto extract_stream(AnnT* algo) -> std::optional - { - auto gpu_ann = dynamic_cast(algo); - if (gpu_ann != nullptr && gpu_ann->uses_stream()) { - return std::make_optional(gpu_ann->get_sync_stream()); - } - return std::nullopt; - } - - public: - struct cuda_lap { - private: - cudaStream_t stream_; - cudaEvent_t start_; - cudaEvent_t stop_; - double& total_time_; - - public: - cuda_lap(cudaStream_t stream, cudaEvent_t start, cudaEvent_t stop, double& total_time) - : start_(start), stop_(stop), stream_(stream), total_time_(total_time) - { -#ifndef BUILD_CPU_ONLY - cudaEventRecord(start_, stream_); -#endif - } - cuda_lap() = delete; - - ~cuda_lap() noexcept - { -#ifndef BUILD_CPU_ONLY - cudaEventRecord(stop_, stream_); - cudaEventSynchronize(stop_); - float milliseconds = 0.0f; - cudaEventElapsedTime(&milliseconds, start_, stop_); - total_time_ += milliseconds / 1000.0; -#endif - } - }; - - explicit cuda_timer(std::optional stream) : stream_{stream} - { -#ifndef BUILD_CPU_ONLY - if (stream_.has_value()) { - cudaEventCreate(&stop_); - cudaEventCreate(&start_); - } -#endif - } - - template - explicit cuda_timer(const std::unique_ptr& algo) : cuda_timer{extract_stream(algo.get())} - { - } - - ~cuda_timer() noexcept - { -#ifndef BUILD_CPU_ONLY - if (stream_.has_value()) { - cudaStreamSynchronize(stream_.value()); - cudaEventDestroy(start_); - cudaEventDestroy(stop_); - } -#endif - } - - cuda_timer() = delete; - cuda_timer(cuda_timer const&) = delete; - cuda_timer(cuda_timer&&) = delete; - auto operator=(cuda_timer const&) -> cuda_timer& = delete; - auto operator=(cuda_timer&&) -> cuda_timer& = delete; - - [[nodiscard]] auto stream() const -> std::optional { return stream_; } - - [[nodiscard]] auto active() const -> bool { return stream_.has_value(); } - - [[nodiscard]] auto total_time() const -> double { return total_time_; } - - [[nodiscard]] auto lap(bool enabled = true) -> std::optional - { - return enabled && stream_.has_value() - ? std::make_optional(stream_.value(), start_, stop_, total_time_) - : std::nullopt; - } -}; - -#ifndef BUILD_CPU_ONLY -// ATM, rmm::stream does not support passing in flags; hence this helper type. -struct non_blocking_stream { - non_blocking_stream() { cudaStreamCreateWithFlags(&stream_, cudaStreamNonBlocking); } - ~non_blocking_stream() noexcept - { - if (stream_ != nullptr) { cudaStreamDestroy(stream_); } - } - non_blocking_stream(non_blocking_stream const&) = delete; - non_blocking_stream(non_blocking_stream&& other) noexcept { std::swap(stream_, other.stream_); } - auto operator=(non_blocking_stream const&) -> non_blocking_stream& = delete; - auto operator=(non_blocking_stream&&) -> non_blocking_stream& = delete; - [[nodiscard]] auto view() const noexcept -> cudaStream_t { return stream_; } - - private: - cudaStream_t stream_{nullptr}; -}; - -namespace detail { -inline std::vector global_stream_pool(0); -inline std::mutex gsp_mutex; -} // namespace detail -#endif - -/** - * Get a stream associated with the current benchmark thread. - * - * Note, the streams are reused between the benchmark cases. - * This makes it easier to profile and analyse multiple benchmark cases in one timeline using tools - * like nsys. - */ -inline auto get_stream_from_global_pool() -> cudaStream_t -{ -#ifndef BUILD_CPU_ONLY - std::lock_guard guard(detail::gsp_mutex); - if (int(detail::global_stream_pool.size()) < benchmark_n_threads) { - detail::global_stream_pool.resize(benchmark_n_threads); - } - return detail::global_stream_pool[benchmark_thread_id].view(); -#else - return nullptr; -#endif -} - -struct result_buffer { - explicit result_buffer(size_t size, cudaStream_t stream) : size_{size}, stream_{stream} - { - if (size_ == 0) { return; } - data_host_ = malloc(size_); -#ifndef BUILD_CPU_ONLY - cudaMallocAsync(&data_device_, size_, stream_); - cudaStreamSynchronize(stream_); -#endif - } - result_buffer() = delete; - result_buffer(result_buffer&&) = delete; - result_buffer& operator=(result_buffer&&) = delete; - result_buffer(const result_buffer&) = delete; - result_buffer& operator=(const result_buffer&) = delete; - ~result_buffer() noexcept - { - if (size_ == 0) { return; } -#ifndef BUILD_CPU_ONLY - cudaFreeAsync(data_device_, stream_); - cudaStreamSynchronize(stream_); -#endif - free(data_host_); - } - - [[nodiscard]] auto size() const noexcept { return size_; } - [[nodiscard]] auto data(ann::MemoryType loc) const noexcept - { - switch (loc) { - case MemoryType::Device: return data_device_; - default: return data_host_; - } - } - - void transfer_data(ann::MemoryType dst, ann::MemoryType src) - { - auto dst_ptr = data(dst); - auto src_ptr = data(src); - if (dst_ptr == src_ptr) { return; } -#ifndef BUILD_CPU_ONLY - cudaMemcpyAsync(dst_ptr, src_ptr, size_, cudaMemcpyDefault, stream_); - cudaStreamSynchronize(stream_); -#endif - } - - private: - size_t size_{0}; - cudaStream_t stream_ = nullptr; - void* data_host_ = nullptr; - void* data_device_ = nullptr; -}; - -namespace detail { -inline std::vector> global_result_buffer_pool(0); -inline std::mutex grp_mutex; -} // namespace detail - -/** - * Get a result buffer associated with the current benchmark thread. - * - * Note, the allocations are reused between the benchmark cases. - * This reduces the setup overhead and number of times the context is being blocked - * (this is relevant if there is a persistent kernel running across multiples benchmark cases). - */ -inline auto get_result_buffer_from_global_pool(size_t size) -> result_buffer& -{ - auto stream = get_stream_from_global_pool(); - auto& rb = [stream, size]() -> result_buffer& { - std::lock_guard guard(detail::grp_mutex); - if (static_cast(detail::global_result_buffer_pool.size()) < benchmark_n_threads) { - detail::global_result_buffer_pool.resize(benchmark_n_threads); - } - auto& rb = detail::global_result_buffer_pool[benchmark_thread_id]; - if (!rb || rb->size() < size) { rb = std::make_unique(size, stream); } - return *rb; - }(); - - memset(rb.data(MemoryType::Host), 0, size); -#ifndef BUILD_CPU_ONLY - cudaMemsetAsync(rb.data(MemoryType::Device), 0, size, stream); - cudaStreamSynchronize(stream); -#endif - return rb; -} - -/** - * Delete all streams and memory allocations in the global pool. - * It's called at the end of the `main` function - before global/static variables and cuda context - * is destroyed - to make sure they are destroyed gracefully and correctly seen by analysis tools - * such as nsys. - */ -inline void reset_global_device_resources() -{ -#ifndef BUILD_CPU_ONLY - std::lock_guard guard(detail::gsp_mutex); - detail::global_result_buffer_pool.resize(0); - detail::global_stream_pool.resize(0); -#endif -} - -inline auto cuda_info() -{ - std::vector> props; -#ifndef BUILD_CPU_ONLY - int dev, driver = 0, runtime = 0; - cudaDriverGetVersion(&driver); - cudaRuntimeGetVersion(&runtime); - - cudaDeviceProp device_prop; - cudaGetDevice(&dev); - cudaGetDeviceProperties(&device_prop, dev); - props.emplace_back("gpu_name", std::string(device_prop.name)); - props.emplace_back("gpu_sm_count", std::to_string(device_prop.multiProcessorCount)); - props.emplace_back("gpu_sm_freq", std::to_string(device_prop.clockRate * 1e3)); - props.emplace_back("gpu_mem_freq", std::to_string(device_prop.memoryClockRate * 1e3)); - props.emplace_back("gpu_mem_bus_width", std::to_string(device_prop.memoryBusWidth)); - props.emplace_back("gpu_mem_global_size", std::to_string(device_prop.totalGlobalMem)); - props.emplace_back("gpu_mem_shared_size", std::to_string(device_prop.sharedMemPerMultiprocessor)); - props.emplace_back("gpu_driver_version", - std::to_string(driver / 1000) + "." + std::to_string((driver % 100) / 10)); - props.emplace_back("gpu_runtime_version", - std::to_string(runtime / 1000) + "." + std::to_string((runtime % 100) / 10)); -#endif - return props; -} - -struct nvtx_case { -#ifdef ANN_BENCH_NVTX3_HEADERS_FOUND - private: - std::string case_name_; - std::array iter_name_{0}; - nvtxDomainHandle_t domain_; - int64_t iteration_ = 0; - nvtxEventAttributes_t case_attrib_{0}; - nvtxEventAttributes_t iter_attrib_{0}; -#endif - - public: - struct nvtx_lap { -#ifdef ANN_BENCH_NVTX3_HEADERS_FOUND - private: - nvtxDomainHandle_t domain_; - - public: - nvtx_lap(nvtxDomainHandle_t domain, nvtxEventAttributes_t* attr) : domain_(domain) - { - nvtxDomainRangePushEx(domain_, attr); - } - nvtx_lap() = delete; - ~nvtx_lap() noexcept { nvtxDomainRangePop(domain_); } -#endif - }; - -#ifdef ANN_BENCH_NVTX3_HEADERS_FOUND - explicit nvtx_case(std::string case_name) - : case_name_(std::move(case_name)), domain_(nvtxDomainCreateA("ANN benchmark")) - { - case_attrib_.version = NVTX_VERSION; - iter_attrib_.version = NVTX_VERSION; - case_attrib_.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; - iter_attrib_.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; - case_attrib_.colorType = NVTX_COLOR_ARGB; - iter_attrib_.colorType = NVTX_COLOR_ARGB; - case_attrib_.messageType = NVTX_MESSAGE_TYPE_ASCII; - iter_attrib_.messageType = NVTX_MESSAGE_TYPE_ASCII; - case_attrib_.message.ascii = case_name_.c_str(); - auto c = std::hash{}(case_name_); - case_attrib_.color = c | 0xA0A0A0; - nvtxDomainRangePushEx(domain_, &case_attrib_); - } - - ~nvtx_case() - { - nvtxDomainRangePop(domain_); - nvtxDomainDestroy(domain_); - } -#else - explicit nvtx_case(std::string) {} -#endif - - [[nodiscard]] auto lap() -> nvtx_case::nvtx_lap - { -#ifdef ANN_BENCH_NVTX3_HEADERS_FOUND - auto i = iteration_++; - uint32_t c = (i % 5); - uint32_t r = 150 + c * 20; - uint32_t g = 200 + c * 10; - uint32_t b = 220 + c * 5; - std::snprintf(iter_name_.data(), iter_name_.size(), "Lap %zd", i); - iter_attrib_.message.ascii = iter_name_.data(); - iter_attrib_.color = (r << 16) + (g << 8) + b; - return nvtx_lap{domain_, &iter_attrib_}; -#else - return nvtx_lap{}; -#endif - } -}; - -/** - * A progress tracker that allows syncing threads multiple times and resets the global - * progress once the threads are done. - */ -struct progress_barrier { - progress_barrier() = default; - ~progress_barrier() noexcept - { - { - // Lock makes sure the notified threads see the updates to `done_`. - std::unique_lock lk(mutex_); - done_.store(true, std::memory_order_relaxed); - cv_.notify_all(); - } - // This is the only place where the order of the updates to thread_progress_ and done_ is - // important. They are not guarded by the mutex, and `done_` must not be reset to `true` by - // other threads after the `total_progress_` is zero. - // Hence the default memory order (std::memory_order_seq_cst). - auto rem = total_progress_.fetch_sub(thread_progress_); - if (rem == thread_progress_) { - // the last thread to exit clears the progress state. - done_.store(false); - } - } - - /** - * Advance the progress counter by `n` and return the previous `progress` value. - * - * This can be used to track which thread arrives on the call site first. - * - * @return the previous progress counter value (before incrementing it by `n`). - */ - auto arrive(int n) - { - thread_progress_ += n; - // Lock makes sure the notified threads see the updates to `total_progress_`. - std::unique_lock lk(mutex_); - auto prev = total_progress_.fetch_add(n, std::memory_order_relaxed); - cv_.notify_all(); - return prev; - } - - /** - * Wait till the progress counter reaches `n` or finishes abnormally. - * - * @return the latest observed value of the progress counter. - */ - auto wait(int limit) - { - int cur = total_progress_.load(std::memory_order_relaxed); - if (cur >= limit) { return cur; } - auto done = done_.load(std::memory_order_relaxed); - if (done) { return cur; } - std::unique_lock lk(mutex_); - while (cur < limit && !done) { - using namespace std::chrono_literals; - cv_.wait_for(lk, 10ms); - cur = total_progress_.load(std::memory_order_relaxed); - done = done_.load(std::memory_order_relaxed); - } - return cur; - } - - private: - static inline std::atomic total_progress_; - static inline std::atomic done_; - static inline std::mutex mutex_; - static inline std::condition_variable cv_; - int thread_progress_{0}; -}; - -inline std::vector split(const std::string& s, char delimiter) -{ - std::vector tokens; - std::string token; - std::istringstream iss(s); - while (getline(iss, token, delimiter)) { - if (!token.empty()) { tokens.push_back(token); } - } - return tokens; -} - -inline bool file_exists(const std::string& filename) -{ - struct stat statbuf; - if (stat(filename.c_str(), &statbuf) != 0) { return false; } - return S_ISREG(statbuf.st_mode); -} - -inline bool dir_exists(const std::string& dir) -{ - struct stat statbuf; - if (stat(dir.c_str(), &statbuf) != 0) { return false; } - return S_ISDIR(statbuf.st_mode); -} - -inline bool create_dir(const std::string& dir) -{ - const auto path = split(dir, '/'); - - std::string cwd; - if (!dir.empty() && dir[0] == '/') { cwd += '/'; } - - for (const auto& p : path) { - cwd += p + "/"; - if (!dir_exists(cwd)) { - int ret = mkdir(cwd.c_str(), S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); - if (ret != 0) { return false; } - } - } - return true; -} - -inline void make_sure_parent_dir_exists(const std::string& file_path) -{ - const auto pos = file_path.rfind('/'); - if (pos != std::string::npos) { - auto dir = file_path.substr(0, pos); - if (!dir_exists(dir)) { create_dir(dir); } - } -} - -inline auto combine_path(const std::string& dir, const std::string& path) -{ - std::filesystem::path p_dir(dir); - std::filesystem::path p_suf(path); - return (p_dir / p_suf).string(); -} - -template -void log_(const char* level, const Ts&... vs) -{ - char buf[20]; - std::time_t now = std::time(nullptr); - std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", std::localtime(&now)); - printf("%s [%s] ", buf, level); - if constexpr (sizeof...(Ts) == 1) { - printf("%s", vs...); - } else { - printf(vs...); - } - printf("\n"); - fflush(stdout); -} - -template -void log_info(Ts&&... vs) -{ - log_("info", std::forward(vs)...); -} - -template -void log_warn(Ts&&... vs) -{ - log_("warn", std::forward(vs)...); -} - -template -void log_error(Ts&&... vs) -{ - log_("error", std::forward(vs)...); -} - -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp b/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp deleted file mode 100644 index 234b33d80a..0000000000 --- a/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../common/ann_types.hpp" -#include "faiss_cpu_wrapper.h" - -#define JSON_DIAGNOSTICS 1 -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -template -void parse_base_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissCpu::BuildParam& param) -{ - param.nlist = conf.at("nlist"); - if (conf.contains("ratio")) { param.ratio = conf.at("ratio"); } -} - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissCpuIVFFlat::BuildParam& param) -{ - parse_base_build_param(conf, param); -} - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissCpuIVFPQ::BuildParam& param) -{ - parse_base_build_param(conf, param); - param.M = conf.at("M"); - if (conf.contains("use_precomputed_table")) { - param.use_precomputed_table = conf.at("use_precomputed_table"); - } else { - param.use_precomputed_table = false; - } - if (conf.contains("bitsPerCode")) { - param.bitsPerCode = conf.at("bitsPerCode"); - } else { - param.bitsPerCode = 8; - } -} - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissCpuIVFSQ::BuildParam& param) -{ - parse_base_build_param(conf, param); - param.quantizer_type = conf.at("quantizer_type"); -} - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissCpu::SearchParam& param) -{ - param.nprobe = conf.at("nprobe"); - if (conf.contains("refine_ratio")) { param.refine_ratio = conf.at("refine_ratio"); } - if (conf.contains("numThreads")) { param.num_threads = conf.at("numThreads"); } -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - return std::make_unique>(metric, dim, param); -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - - (void)dev_list; - return std::make_unique>(metric, dim, param); -} - -template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - // stop compiler warning; not all algorithms support multi-GPU so it may not be used - (void)dev_list; - - std::unique_ptr> ann; - - if constexpr (std::is_same_v) { - raft::bench::ann::Metric metric = parse_metric(distance); - if (algo == "faiss_cpu_ivf_flat") { - ann = make_algo(metric, dim, conf, dev_list); - } else if (algo == "faiss_cpu_ivf_pq") { - ann = make_algo(metric, dim, conf); - } else if (algo == "faiss_cpu_ivf_sq") { - ann = make_algo(metric, dim, conf); - } else if (algo == "faiss_cpu_flat") { - ann = std::make_unique>(metric, dim); - } - } - - if constexpr (std::is_same_v) {} - - if (!ann) { throw std::runtime_error("invalid algo: '" + algo + "'"); } - - return ann; -} - -template -std::unique_ptr::AnnSearchParam> create_search_param( - const std::string& algo, const nlohmann::json& conf) -{ - if (algo == "faiss_cpu_ivf_flat" || algo == "faiss_cpu_ivf_pq" || algo == "faiss_cpu_ivf_sq") { - auto param = std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } else if (algo == "faiss_cpu_flat") { - auto param = std::make_unique::SearchParam>(); - return param; - } - // else - throw std::runtime_error("invalid algo: '" + algo + "'"); -} - -} // namespace raft::bench::ann - -REGISTER_ALGO_INSTANCE(float); -REGISTER_ALGO_INSTANCE(std::int8_t); -REGISTER_ALGO_INSTANCE(std::uint8_t); - -#ifdef ANN_BENCH_BUILD_MAIN -#include "../common/benchmark.hpp" -int main(int argc, char** argv) { return raft::bench::ann::run_main(argc, argv); } -#endif diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h b/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h deleted file mode 100644 index c7ce4595b5..0000000000 --- a/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h +++ /dev/null @@ -1,326 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../common/ann_types.hpp" -#include "../common/thread_pool.hpp" - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace { - -faiss::MetricType parse_metric_type(raft::bench::ann::Metric metric) -{ - if (metric == raft::bench::ann::Metric::kInnerProduct) { - return faiss::METRIC_INNER_PRODUCT; - } else if (metric == raft::bench::ann::Metric::kEuclidean) { - return faiss::METRIC_L2; - } else { - throw std::runtime_error("faiss supports only metric type of inner product and L2"); - } -} -} // namespace - -namespace raft::bench::ann { - -template -class FaissCpu : public ANN { - public: - using typename ANN::AnnSearchParam; - struct SearchParam : public AnnSearchParam { - int nprobe; - float refine_ratio = 1.0; - int num_threads = omp_get_num_procs(); - }; - - struct BuildParam { - int nlist = 1; - int ratio = 2; - }; - - FaissCpu(Metric metric, int dim, const BuildParam& param) - : ANN(metric, dim), - metric_type_(parse_metric_type(metric)), - nlist_{param.nlist}, - training_sample_fraction_{1.0 / double(param.ratio)} - { - static_assert(std::is_same_v, "faiss support only float type"); - } - - void build(const T* dataset, size_t nrow) final; - - void set_search_param(const AnnSearchParam& param) override; - - void init_quantizer(int dim) - { - if (this->metric_type_ == faiss::MetricType::METRIC_L2) { - this->quantizer_ = std::make_shared(dim); - } else if (this->metric_type_ == faiss::MetricType::METRIC_INNER_PRODUCT) { - this->quantizer_ = std::make_shared(dim); - } - } - - // TODO: if the number of results is less than k, the remaining elements of 'neighbors' - // will be filled with (size_t)-1 - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const final; - - AlgoProperty get_preference() const override - { - AlgoProperty property; - // to enable building big dataset which is larger than memory - property.dataset_memory_type = MemoryType::Host; - property.query_memory_type = MemoryType::Host; - return property; - } - - protected: - template - void save_(const std::string& file) const; - - template - void load_(const std::string& file); - - std::shared_ptr index_; - std::shared_ptr quantizer_; - std::shared_ptr index_refine_; - faiss::MetricType metric_type_; - int nlist_; - double training_sample_fraction_; - - int num_threads_; - std::shared_ptr thread_pool_; -}; - -template -void FaissCpu::build(const T* dataset, size_t nrow) -{ - auto index_ivf = dynamic_cast(index_.get()); - if (index_ivf != nullptr) { - // set the min/max training size for clustering to use the whole provided training set. - double trainset_size = training_sample_fraction_ * static_cast(nrow); - double points_per_centroid = trainset_size / static_cast(nlist_); - int max_ppc = std::ceil(points_per_centroid); - int min_ppc = std::floor(points_per_centroid); - if (min_ppc < index_ivf->cp.min_points_per_centroid) { - RAFT_LOG_WARN( - "The suggested training set size %zu (data size %zu, training sample ratio %f) yields %d " - "points per cluster (n_lists = %d). This is smaller than the FAISS default " - "min_points_per_centroid = %d.", - static_cast(trainset_size), - nrow, - training_sample_fraction_, - min_ppc, - nlist_, - index_ivf->cp.min_points_per_centroid); - } - index_ivf->cp.max_points_per_centroid = max_ppc; - index_ivf->cp.min_points_per_centroid = min_ppc; - } - index_->train(nrow, dataset); // faiss::IndexFlat::train() will do nothing - assert(index_->is_trained); - index_->add(nrow, dataset); - index_refine_ = std::make_shared(this->index_.get(), dataset); -} - -template -void FaissCpu::set_search_param(const AnnSearchParam& param) -{ - auto search_param = dynamic_cast(param); - int nprobe = search_param.nprobe; - assert(nprobe <= nlist_); - dynamic_cast(index_.get())->nprobe = nprobe; - - if (search_param.refine_ratio > 1.0) { - this->index_refine_.get()->k_factor = search_param.refine_ratio; - } - - if (!thread_pool_ || num_threads_ != search_param.num_threads) { - num_threads_ = search_param.num_threads; - thread_pool_ = std::make_shared(num_threads_); - } -} - -template -void FaissCpu::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - static_assert(sizeof(size_t) == sizeof(faiss::idx_t), - "sizes of size_t and faiss::idx_t are different"); - - thread_pool_->submit( - [&](int i) { - // Use thread pool for batch size = 1. FAISS multi-threads internally for batch size > 1. - index_->search(batch_size, queries, k, distances, reinterpret_cast(neighbors)); - }, - 1); -} - -template -template -void FaissCpu::save_(const std::string& file) const -{ - faiss::write_index(index_.get(), file.c_str()); -} - -template -template -void FaissCpu::load_(const std::string& file) -{ - index_ = std::shared_ptr(dynamic_cast(faiss::read_index(file.c_str()))); -} - -template -class FaissCpuIVFFlat : public FaissCpu { - public: - using typename FaissCpu::BuildParam; - - FaissCpuIVFFlat(Metric metric, int dim, const BuildParam& param) : FaissCpu(metric, dim, param) - { - this->init_quantizer(dim); - this->index_ = std::make_shared( - this->quantizer_.get(), dim, param.nlist, this->metric_type_); - } - - void save(const std::string& file) const override - { - this->template save_(file); - } - void load(const std::string& file) override { this->template load_(file); } - - std::unique_ptr> copy() - { - return std::make_unique>(*this); // use copy constructor - } -}; - -template -class FaissCpuIVFPQ : public FaissCpu { - public: - struct BuildParam : public FaissCpu::BuildParam { - int M; - int bitsPerCode; - bool use_precomputed_table; - }; - - FaissCpuIVFPQ(Metric metric, int dim, const BuildParam& param) : FaissCpu(metric, dim, param) - { - this->init_quantizer(dim); - this->index_ = std::make_shared( - this->quantizer_.get(), dim, param.nlist, param.M, param.bitsPerCode, this->metric_type_); - } - - void save(const std::string& file) const override - { - this->template save_(file); - } - void load(const std::string& file) override { this->template load_(file); } - - std::unique_ptr> copy() - { - return std::make_unique>(*this); // use copy constructor - } -}; - -// TODO: Enable this in cmake -// ref: https://github.com/rapidsai/raft/issues/1876 -template -class FaissCpuIVFSQ : public FaissCpu { - public: - struct BuildParam : public FaissCpu::BuildParam { - std::string quantizer_type; - }; - - FaissCpuIVFSQ(Metric metric, int dim, const BuildParam& param) : FaissCpu(metric, dim, param) - { - faiss::ScalarQuantizer::QuantizerType qtype; - if (param.quantizer_type == "fp16") { - qtype = faiss::ScalarQuantizer::QT_fp16; - } else if (param.quantizer_type == "int8") { - qtype = faiss::ScalarQuantizer::QT_8bit; - } else { - throw std::runtime_error("FaissCpuIVFSQ supports only fp16 and int8 but got " + - param.quantizer_type); - } - - this->init_quantizer(dim); - this->index_ = std::make_shared( - this->quantizer_.get(), dim, param.nlist, qtype, this->metric_type_, true); - } - - void save(const std::string& file) const override - { - this->template save_(file); - } - void load(const std::string& file) override - { - this->template load_(file); - } - - std::unique_ptr> copy() - { - return std::make_unique>(*this); // use copy constructor - } -}; - -template -class FaissCpuFlat : public FaissCpu { - public: - FaissCpuFlat(Metric metric, int dim) - : FaissCpu(metric, dim, typename FaissCpu::BuildParam{}) - { - this->index_ = std::make_shared(dim, this->metric_type_); - } - - // class FaissCpu is more like a IVF class, so need special treating here - void set_search_param(const typename ANN::AnnSearchParam& param) override - { - auto search_param = dynamic_cast::SearchParam&>(param); - if (!this->thread_pool_ || this->num_threads_ != search_param.num_threads) { - this->num_threads_ = search_param.num_threads; - this->thread_pool_ = std::make_shared(this->num_threads_); - } - }; - - void save(const std::string& file) const override - { - this->template save_(file); - } - void load(const std::string& file) override { this->template load_(file); } - - std::unique_ptr> copy() - { - return std::make_unique>(*this); // use copy constructor - } -}; - -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu b/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu deleted file mode 100644 index b47c497e3d..0000000000 --- a/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../common/ann_types.hpp" - -#undef WARP_SIZE -#include "faiss_gpu_wrapper.h" - -#define JSON_DIAGNOSTICS 1 -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -template -void parse_base_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissGpu::BuildParam& param) -{ - param.nlist = conf.at("nlist"); - if (conf.contains("ratio")) { param.ratio = conf.at("ratio"); } -} - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissGpuIVFFlat::BuildParam& param) -{ - parse_base_build_param(conf, param); - if (conf.contains("use_raft")) { - param.use_raft = conf.at("use_raft"); - } else { - param.use_raft = false; - } -} - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissGpuIVFPQ::BuildParam& param) -{ - parse_base_build_param(conf, param); - param.M = conf.at("M"); - if (conf.contains("usePrecomputed")) { - param.usePrecomputed = conf.at("usePrecomputed"); - } else { - param.usePrecomputed = false; - } - if (conf.contains("useFloat16")) { - param.useFloat16 = conf.at("useFloat16"); - } else { - param.useFloat16 = false; - } - if (conf.contains("use_raft")) { - param.use_raft = conf.at("use_raft"); - } else { - param.use_raft = false; - } - if (conf.contains("bitsPerCode")) { - param.bitsPerCode = conf.at("bitsPerCode"); - } else { - param.bitsPerCode = 8; - } -} - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissGpuIVFSQ::BuildParam& param) -{ - parse_base_build_param(conf, param); - param.quantizer_type = conf.at("quantizer_type"); -} - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissGpu::SearchParam& param) -{ - param.nprobe = conf.at("nprobe"); - if (conf.contains("refine_ratio")) { param.refine_ratio = conf.at("refine_ratio"); } -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - return std::make_unique>(metric, dim, param); -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - - (void)dev_list; - return std::make_unique>(metric, dim, param); -} - -template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - // stop compiler warning; not all algorithms support multi-GPU so it may not be used - (void)dev_list; - - std::unique_ptr> ann; - - if constexpr (std::is_same_v) { - raft::bench::ann::Metric metric = parse_metric(distance); - if (algo == "faiss_gpu_ivf_flat") { - ann = make_algo(metric, dim, conf, dev_list); - } else if (algo == "faiss_gpu_ivf_pq") { - ann = make_algo(metric, dim, conf); - } else if (algo == "faiss_gpu_ivf_sq") { - ann = make_algo(metric, dim, conf); - } else if (algo == "faiss_gpu_flat") { - ann = std::make_unique>(metric, dim); - } - } - - if constexpr (std::is_same_v) {} - - if (!ann) { throw std::runtime_error("invalid algo: '" + algo + "'"); } - - return ann; -} - -template -std::unique_ptr::AnnSearchParam> create_search_param( - const std::string& algo, const nlohmann::json& conf) -{ - if (algo == "faiss_gpu_ivf_flat" || algo == "faiss_gpu_ivf_pq" || algo == "faiss_gpu_ivf_sq") { - auto param = std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } else if (algo == "faiss_gpu_flat") { - auto param = std::make_unique::SearchParam>(); - return param; - } - // else - throw std::runtime_error("invalid algo: '" + algo + "'"); -} - -} // namespace raft::bench::ann - -REGISTER_ALGO_INSTANCE(float); -REGISTER_ALGO_INSTANCE(std::int8_t); -REGISTER_ALGO_INSTANCE(std::uint8_t); - -#ifdef ANN_BENCH_BUILD_MAIN -#include "../common/benchmark.hpp" -int main(int argc, char** argv) -{ - rmm::mr::cuda_memory_resource cuda_mr; - // Construct a resource that uses a coalescing best-fit pool allocator - // and is initially sized to half of free device memory. - rmm::mr::pool_memory_resource pool_mr{ - &cuda_mr, rmm::percent_of_free_device_memory(50)}; - // Updates the current device resource pointer to `pool_mr` - auto old_mr = rmm::mr::set_current_device_resource(&pool_mr); - auto ret = raft::bench::ann::run_main(argc, argv); - // Restores the current device resource pointer to its previous value - rmm::mr::set_current_device_resource(old_mr); - return ret; -} -#endif diff --git a/cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h b/cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h deleted file mode 100644 index 6955201c5d..0000000000 --- a/cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h +++ /dev/null @@ -1,515 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef FAISS_WRAPPER_H_ -#define FAISS_WRAPPER_H_ - -#include "../common/ann_types.hpp" -#include "../raft/raft_ann_bench_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace { - -faiss::MetricType parse_metric_faiss(raft::bench::ann::Metric metric) -{ - if (metric == raft::bench::ann::Metric::kInnerProduct) { - return faiss::METRIC_INNER_PRODUCT; - } else if (metric == raft::bench::ann::Metric::kEuclidean) { - return faiss::METRIC_L2; - } else { - throw std::runtime_error("faiss supports only metric type of inner product and L2"); - } -} - -// note BLAS library can still use multi-threading, and -// setting environment variable like OPENBLAS_NUM_THREADS can control it -class OmpSingleThreadScope { - public: - OmpSingleThreadScope() - { - max_threads_ = omp_get_max_threads(); - omp_set_num_threads(1); - } - ~OmpSingleThreadScope() - { - // the best we can do - omp_set_num_threads(max_threads_); - } - - private: - int max_threads_; -}; - -} // namespace - -namespace raft::bench::ann { - -template -class FaissGpu : public ANN, public AnnGPU { - public: - using typename ANN::AnnSearchParam; - struct SearchParam : public AnnSearchParam { - int nprobe; - float refine_ratio = 1.0; - auto needs_dataset() const -> bool override { return refine_ratio > 1.0f; } - }; - - struct BuildParam { - int nlist = 1; - int ratio = 2; - }; - - FaissGpu(Metric metric, int dim, const BuildParam& param) - : ANN(metric, dim), - gpu_resource_{std::make_shared()}, - metric_type_(parse_metric_faiss(metric)), - nlist_{param.nlist}, - training_sample_fraction_{1.0 / double(param.ratio)} - { - static_assert(std::is_same_v, "faiss support only float type"); - RAFT_CUDA_TRY(cudaGetDevice(&device_)); - } - - void build(const T* dataset, size_t nrow) final; - - virtual void set_search_param(const FaissGpu::AnnSearchParam& param) {} - - void set_search_dataset(const T* dataset, size_t nrow) override { dataset_ = dataset; } - - // TODO: if the number of results is less than k, the remaining elements of 'neighbors' - // will be filled with (size_t)-1 - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const final; - - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override - { - return gpu_resource_->getDefaultStream(device_); - } - - AlgoProperty get_preference() const override - { - AlgoProperty property; - // to enable building big dataset which is larger than GPU memory - property.dataset_memory_type = MemoryType::Host; - property.query_memory_type = MemoryType::Device; - return property; - } - - protected: - template - void save_(const std::string& file) const; - - template - void load_(const std::string& file); - - /** [NOTE Multithreading] - * - * `gpu_resource_` is a shared resource: - * 1. It uses a shared_ptr under the hood, so the copies of it refer to the same - * resource implementation instance - * 2. GpuIndex is probably keeping a reference to it, as it's passed to the constructor - * - * To avoid copying the index (database) in each thread, we make both the index and - * the gpu_resource shared. - * This means faiss GPU streams are possibly shared among the CPU threads; - * the throughput search mode may be inaccurate. - * - * WARNING: we haven't investigated whether faiss::gpu::GpuIndex or - * faiss::gpu::StandardGpuResources are thread-safe. - * - */ - mutable std::shared_ptr gpu_resource_; - std::shared_ptr index_; - std::shared_ptr index_refine_{nullptr}; - faiss::MetricType metric_type_; - int nlist_; - int device_; - double training_sample_fraction_; - std::shared_ptr search_params_; - std::shared_ptr refine_search_params_{nullptr}; - const T* dataset_; - float refine_ratio_ = 1.0; - Objective metric_objective_; -}; - -template -void FaissGpu::build(const T* dataset, size_t nrow) -{ - OmpSingleThreadScope omp_single_thread; - auto index_ivf = dynamic_cast(index_.get()); - if (index_ivf != nullptr) { - // set the min/max training size for clustering to use the whole provided training set. - double trainset_size = training_sample_fraction_ * static_cast(nrow); - double points_per_centroid = trainset_size / static_cast(nlist_); - int max_ppc = std::ceil(points_per_centroid); - int min_ppc = std::floor(points_per_centroid); - if (min_ppc < index_ivf->cp.min_points_per_centroid) { - RAFT_LOG_WARN( - "The suggested training set size %zu (data size %zu, training sample ratio %f) yields %d " - "points per cluster (n_lists = %d). This is smaller than the FAISS default " - "min_points_per_centroid = %d.", - static_cast(trainset_size), - nrow, - training_sample_fraction_, - min_ppc, - nlist_, - index_ivf->cp.min_points_per_centroid); - } - index_ivf->cp.max_points_per_centroid = max_ppc; - index_ivf->cp.min_points_per_centroid = min_ppc; - } - index_->train(nrow, dataset); // faiss::gpu::GpuIndexFlat::train() will do nothing - assert(index_->is_trained); - index_->add(nrow, dataset); -} - -template -void FaissGpu::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - ASSERT(Objective::LATENCY, "l2Knn: rowMajorIndex and rowMajorQuery should have same layout"); - using IdxT = faiss::idx_t; - static_assert(sizeof(size_t) == sizeof(faiss::idx_t), - "sizes of size_t and faiss::idx_t are different"); - - if (refine_ratio_ > 1.0) { - if (raft::get_device_for_address(queries) >= 0) { - uint32_t k0 = static_cast(refine_ratio_ * k); - auto distances_tmp = raft::make_device_matrix( - gpu_resource_->getRaftHandle(device_), batch_size, k0); - auto candidates = - raft::make_device_matrix(gpu_resource_->getRaftHandle(device_), batch_size, k0); - index_->search(batch_size, - queries, - k0, - distances_tmp.data_handle(), - candidates.data_handle(), - this->search_params_.get()); - - auto queries_host = raft::make_host_matrix(batch_size, index_->d); - auto candidates_host = raft::make_host_matrix(batch_size, k0); - auto neighbors_host = raft::make_host_matrix(batch_size, k); - auto distances_host = raft::make_host_matrix(batch_size, k); - auto dataset_v = raft::make_host_matrix_view( - this->dataset_, index_->ntotal, index_->d); - - raft::device_resources handle_ = gpu_resource_->getRaftHandle(device_); - - raft::copy(queries_host.data_handle(), queries, queries_host.size(), handle_.get_stream()); - raft::copy(candidates_host.data_handle(), - candidates.data_handle(), - candidates_host.size(), - handle_.get_stream()); - - // wait for the queries to copy to host in 'stream` - handle_.sync_stream(); - - raft::runtime::neighbors::refine(handle_, - dataset_v, - queries_host.view(), - candidates_host.view(), - neighbors_host.view(), - distances_host.view(), - parse_metric_type(this->metric_)); - - raft::copy(neighbors, - (size_t*)neighbors_host.data_handle(), - neighbors_host.size(), - handle_.get_stream()); - raft::copy( - distances, distances_host.data_handle(), distances_host.size(), handle_.get_stream()); - } else { - index_refine_->search(batch_size, - queries, - k, - distances, - reinterpret_cast(neighbors), - this->refine_search_params_.get()); - } - } else { - index_->search(batch_size, - queries, - k, - distances, - reinterpret_cast(neighbors), - this->search_params_.get()); - } -} - -template -template -void FaissGpu::save_(const std::string& file) const -{ - OmpSingleThreadScope omp_single_thread; - - auto cpu_index = std::make_unique(); - dynamic_cast(index_.get())->copyTo(cpu_index.get()); - faiss::write_index(cpu_index.get(), file.c_str()); -} - -template -template -void FaissGpu::load_(const std::string& file) -{ - OmpSingleThreadScope omp_single_thread; - - std::unique_ptr cpu_index(dynamic_cast(faiss::read_index(file.c_str()))); - assert(cpu_index); - - try { - dynamic_cast(index_.get())->copyFrom(cpu_index.get()); - - } catch (const std::exception& e) { - std::cout << "Error loading index file: " << std::string(e.what()) << std::endl; - } -} - -template -class FaissGpuIVFFlat : public FaissGpu { - public: - struct BuildParam : public FaissGpu::BuildParam { - bool use_raft; - }; - - FaissGpuIVFFlat(Metric metric, int dim, const BuildParam& param) : FaissGpu(metric, dim, param) - { - faiss::gpu::GpuIndexIVFFlatConfig config; - config.device = this->device_; - config.use_raft = param.use_raft; - this->index_ = std::make_shared( - this->gpu_resource_.get(), dim, param.nlist, this->metric_type_, config); - } - - void set_search_param(const typename FaissGpu::AnnSearchParam& param) override - { - auto search_param = dynamic_cast::SearchParam&>(param); - int nprobe = search_param.nprobe; - assert(nprobe <= nlist_); - - faiss::IVFSearchParameters faiss_search_params; - faiss_search_params.nprobe = nprobe; - this->search_params_ = std::make_shared(faiss_search_params); - this->refine_ratio_ = search_param.refine_ratio; - } - - void save(const std::string& file) const override - { - this->template save_(file); - } - void load(const std::string& file) override - { - this->template load_(file); - } - std::unique_ptr> copy() override { return std::make_unique>(*this); }; -}; - -template -class FaissGpuIVFPQ : public FaissGpu { - public: - struct BuildParam : public FaissGpu::BuildParam { - int M; - bool useFloat16; - bool usePrecomputed; - bool use_raft; - int bitsPerCode; - }; - - FaissGpuIVFPQ(Metric metric, int dim, const BuildParam& param) : FaissGpu(metric, dim, param) - { - faiss::gpu::GpuIndexIVFPQConfig config; - config.useFloat16LookupTables = param.useFloat16; - config.usePrecomputedTables = param.usePrecomputed; - config.use_raft = param.use_raft; - config.interleavedLayout = param.use_raft; - config.device = this->device_; - - this->index_ = std::make_shared(this->gpu_resource_.get(), - dim, - param.nlist, - param.M, - param.bitsPerCode, - this->metric_type_, - config); - } - - void set_search_param(const typename FaissGpu::AnnSearchParam& param) override - { - auto search_param = dynamic_cast::SearchParam&>(param); - int nprobe = search_param.nprobe; - assert(nprobe <= nlist_); - this->refine_ratio_ = search_param.refine_ratio; - faiss::IVFPQSearchParameters faiss_search_params; - faiss_search_params.nprobe = nprobe; - - this->search_params_ = std::make_shared(faiss_search_params); - - if (search_param.refine_ratio > 1.0) { - this->index_refine_ = - std::make_shared(this->index_.get(), this->dataset_); - this->index_refine_.get()->k_factor = search_param.refine_ratio; - faiss::IndexRefineSearchParameters faiss_refine_search_params; - faiss_refine_search_params.k_factor = this->index_refine_.get()->k_factor; - faiss_refine_search_params.base_index_params = this->search_params_.get(); - this->refine_search_params_ = - std::make_unique(faiss_refine_search_params); - } - } - - void save(const std::string& file) const override - { - this->template save_(file); - } - void load(const std::string& file) override - { - this->template load_(file); - } - std::unique_ptr> copy() override { return std::make_unique>(*this); }; -}; - -// TODO: Enable this in cmake -// ref: https://github.com/rapidsai/raft/issues/1876 -template -class FaissGpuIVFSQ : public FaissGpu { - public: - struct BuildParam : public FaissGpu::BuildParam { - std::string quantizer_type; - }; - - FaissGpuIVFSQ(Metric metric, int dim, const BuildParam& param) : FaissGpu(metric, dim, param) - { - faiss::ScalarQuantizer::QuantizerType qtype; - if (param.quantizer_type == "fp16") { - qtype = faiss::ScalarQuantizer::QT_fp16; - } else if (param.quantizer_type == "int8") { - qtype = faiss::ScalarQuantizer::QT_8bit; - } else { - throw std::runtime_error("FaissGpuIVFSQ supports only fp16 and int8 but got " + - param.quantizer_type); - } - - faiss::gpu::GpuIndexIVFScalarQuantizerConfig config; - config.device = this->device_; - this->index_ = std::make_shared( - this->gpu_resource_.get(), dim, param.nlist, qtype, this->metric_type_, true, config); - } - - void set_search_param(const typename FaissGpu::AnnSearchParam& param) override - { - auto search_param = dynamic_cast::SearchParam&>(param); - int nprobe = search_param.nprobe; - assert(nprobe <= nlist_); - - faiss::IVFSearchParameters faiss_search_params; - faiss_search_params.nprobe = nprobe; - - this->search_params_ = std::make_shared(faiss_search_params); - this->refine_ratio_ = search_param.refine_ratio; - if (search_param.refine_ratio > 1.0) { - this->index_refine_ = - std::make_shared(this->index_.get(), this->dataset_); - this->index_refine_.get()->k_factor = search_param.refine_ratio; - faiss::IndexRefineSearchParameters faiss_refine_search_params; - faiss_refine_search_params.k_factor = this->index_refine_.get()->k_factor; - faiss_refine_search_params.base_index_params = this->search_params_.get(); - this->refine_search_params_ = - std::make_unique(faiss_refine_search_params); - } - } - - void save(const std::string& file) const override - { - this->template save_( - file); - } - void load(const std::string& file) override - { - this->template load_( - file); - } - std::unique_ptr> copy() override { return std::make_unique>(*this); }; -}; - -template -class FaissGpuFlat : public FaissGpu { - public: - FaissGpuFlat(Metric metric, int dim) - : FaissGpu(metric, dim, typename FaissGpu::BuildParam{}) - { - faiss::gpu::GpuIndexFlatConfig config; - config.device = this->device_; - this->index_ = std::make_shared( - this->gpu_resource_.get(), dim, this->metric_type_, config); - } - void set_search_param(const typename FaissGpu::AnnSearchParam& param) override - { - auto search_param = dynamic_cast::SearchParam&>(param); - int nprobe = search_param.nprobe; - assert(nprobe <= nlist_); - - this->search_params_ = std::make_shared(); - } - - void save(const std::string& file) const override - { - this->template save_(file); - } - void load(const std::string& file) override - { - this->template load_(file); - } - std::unique_ptr> copy() override { return std::make_unique>(*this); }; -}; - -} // namespace raft::bench::ann - -#endif diff --git a/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu b/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu deleted file mode 100644 index 48d41388d4..0000000000 --- a/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../common/ann_types.hpp" -#include "ggnn_wrapper.cuh" - -#define JSON_DIAGNOSTICS 1 -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::Ggnn::BuildParam& param) -{ - param.k = conf.at("k"); - - if (conf.contains("k_build")) { param.k_build = conf.at("k_build"); } - if (conf.contains("segment_size")) { param.segment_size = conf.at("segment_size"); } - if (conf.contains("num_layers")) { param.num_layers = conf.at("num_layers"); } - if (conf.contains("tau")) { param.tau = conf.at("tau"); } - if (conf.contains("refine_iterations")) { - param.refine_iterations = conf.at("refine_iterations"); - } -} - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::Ggnn::SearchParam& param) -{ - param.tau = conf.at("tau"); - - if (conf.contains("block_dim")) { param.block_dim = conf.at("block_dim"); } - if (conf.contains("max_iterations")) { param.max_iterations = conf.at("max_iterations"); } - if (conf.contains("cache_size")) { param.cache_size = conf.at("cache_size"); } - if (conf.contains("sorted_size")) { param.sorted_size = conf.at("sorted_size"); } -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - return std::make_unique>(metric, dim, param); -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - - (void)dev_list; - return std::make_unique>(metric, dim, param); -} - -template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - // stop compiler warning; not all algorithms support multi-GPU so it may not be used - (void)dev_list; - - raft::bench::ann::Metric metric = parse_metric(distance); - std::unique_ptr> ann; - - if constexpr (std::is_same_v || std::is_same_v || - std::is_same_v) { - if (algo == "ggnn") { ann = make_algo(metric, dim, conf); } - } - if (!ann) { throw std::runtime_error("invalid algo: '" + algo + "'"); } - - return ann; -} - -template -std::unique_ptr::AnnSearchParam> create_search_param( - const std::string& algo, const nlohmann::json& conf) -{ - if constexpr (std::is_same_v || std::is_same_v || - std::is_same_v) { - if (algo == "ggnn") { - auto param = std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } - } - // else - throw std::runtime_error("invalid algo: '" + algo + "'"); -} - -} // namespace raft::bench::ann - -REGISTER_ALGO_INSTANCE(float); -REGISTER_ALGO_INSTANCE(std::int8_t); -REGISTER_ALGO_INSTANCE(std::uint8_t); - -#ifdef ANN_BENCH_BUILD_MAIN -#include "../common/benchmark.hpp" -int main(int argc, char** argv) { return raft::bench::ann::run_main(argc, argv); } -#endif diff --git a/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh b/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh deleted file mode 100644 index 59cf3df806..0000000000 --- a/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh +++ /dev/null @@ -1,322 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "../common/ann_types.hpp" -#include "../common/util.hpp" - -#include - -#include - -#include -#include - -namespace raft::bench::ann { - -template -class GgnnImpl; - -template -class Ggnn : public ANN, public AnnGPU { - public: - struct BuildParam { - int k_build{24}; // KBuild - int segment_size{32}; // S - int num_layers{4}; // L - float tau{0.5}; - int refine_iterations{2}; - int k; // GGNN requires to know k during building - }; - - using typename ANN::AnnSearchParam; - struct SearchParam : public AnnSearchParam { - float tau; - int block_dim{32}; - int max_iterations{400}; - int cache_size{512}; - int sorted_size{256}; - auto needs_dataset() const -> bool override { return true; } - }; - - Ggnn(Metric metric, int dim, const BuildParam& param); - - void build(const T* dataset, size_t nrow) override { impl_->build(dataset, nrow); } - - void set_search_param(const AnnSearchParam& param) override { impl_->set_search_param(param); } - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const override - { - impl_->search(queries, batch_size, k, neighbors, distances); - } - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override - { - return dynamic_cast(impl_.get())->get_sync_stream(); - } - - void save(const std::string& file) const override { impl_->save(file); } - void load(const std::string& file) override { impl_->load(file); } - std::unique_ptr> copy() override { return std::make_unique>(*this); }; - - AlgoProperty get_preference() const override { return impl_->get_preference(); } - - void set_search_dataset(const T* dataset, size_t nrow) override - { - impl_->set_search_dataset(dataset, nrow); - }; - - private: - std::shared_ptr> impl_; -}; - -template -Ggnn::Ggnn(Metric metric, int dim, const BuildParam& param) : ANN(metric, dim) -{ - // ggnn/src/sift1m.cu - if (metric == Metric::kEuclidean && dim == 128 && param.k_build == 24 && param.k == 10 && - param.segment_size == 32) { - impl_ = std::make_shared>(metric, dim, param); - } - // ggnn/src/deep1b_multi_gpu.cu, and adapt it deep1B - else if (metric == Metric::kEuclidean && dim == 96 && param.k_build == 24 && param.k == 10 && - param.segment_size == 32) { - impl_ = std::make_shared>(metric, dim, param); - } else if (metric == Metric::kInnerProduct && dim == 96 && param.k_build == 24 && param.k == 10 && - param.segment_size == 32) { - impl_ = std::make_shared>(metric, dim, param); - } else if (metric == Metric::kInnerProduct && dim == 96 && param.k_build == 96 && param.k == 10 && - param.segment_size == 64) { - impl_ = std::make_shared>(metric, dim, param); - } - // ggnn/src/glove200.cu, adapt it to glove100 - else if (metric == Metric::kInnerProduct && dim == 100 && param.k_build == 96 && param.k == 10 && - param.segment_size == 64) { - impl_ = std::make_shared>(metric, dim, param); - } else { - throw std::runtime_error( - "ggnn: not supported combination of metric, dim and build param; " - "see Ggnn's constructor in ggnn_wrapper.cuh for available combinations"); - } -} - -template -class GgnnImpl : public ANN, public AnnGPU { - public: - using typename ANN::AnnSearchParam; - - GgnnImpl(Metric metric, int dim, const typename Ggnn::BuildParam& param); - - void build(const T* dataset, size_t nrow) override; - - void set_search_param(const AnnSearchParam& param) override; - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const override; - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override { return stream_; } - - void save(const std::string& file) const override; - void load(const std::string& file) override; - std::unique_ptr> copy() override - { - auto r = std::make_unique>(*this); - // set the thread-local stream to the copied handle. - r->stream_ = raft::bench::ann::get_stream_from_global_pool(); - return r; - }; - - AlgoProperty get_preference() const override - { - AlgoProperty property; - property.dataset_memory_type = MemoryType::Device; - property.query_memory_type = MemoryType::Device; - return property; - } - - void set_search_dataset(const T* dataset, size_t nrow) override; - - private: - using ANN::metric_; - using ANN::dim_; - - using GGNNGPUInstance = GGNNGPUInstance; - std::shared_ptr ggnn_; - typename Ggnn::BuildParam build_param_; - typename Ggnn::SearchParam search_param_; - cudaStream_t stream_; - const T* base_dataset = nullptr; - size_t base_n_rows = 0; - std::optional graph_file = std::nullopt; - - void load_impl() - { - if (base_dataset == nullptr) { return; } - if (base_n_rows == 0) { return; } - int device; - RAFT_CUDA_TRY(cudaGetDevice(&device)); - ggnn_ = std::make_shared( - device, base_n_rows, build_param_.num_layers, true, build_param_.tau); - ggnn_->set_base_data(base_dataset); - ggnn_->set_stream(get_sync_stream()); - if (graph_file.has_value()) { - auto& ggnn_host = ggnn_->ggnn_cpu_buffers.at(0); - auto& ggnn_device = ggnn_->ggnn_shards.at(0); - ggnn_->set_stream(get_sync_stream()); - - ggnn_host.load(graph_file.value()); - ggnn_host.uploadAsync(ggnn_device); - RAFT_CUDA_TRY(cudaStreamSynchronize(ggnn_device.stream)); - } - } -}; - -template -GgnnImpl::GgnnImpl(Metric metric, - int dim, - const typename Ggnn::BuildParam& param) - : ANN(metric, dim), - build_param_(param), - stream_(raft::bench::ann::get_stream_from_global_pool()) -{ - if (metric_ == Metric::kInnerProduct) { - if (measure != Cosine) { throw std::runtime_error("mis-matched metric"); } - } else if (metric_ == Metric::kEuclidean) { - if (measure != Euclidean) { throw std::runtime_error("mis-matched metric"); } - } else { - throw std::runtime_error( - "ggnn supports only metric type of InnerProduct, Cosine and Euclidean"); - } - - if (dim != D) { throw std::runtime_error("mis-matched dim"); } -} - -template -void GgnnImpl::build(const T* dataset, size_t nrow) -{ - base_dataset = dataset; - base_n_rows = nrow; - graph_file = std::nullopt; - load_impl(); - ggnn_->build(0); - for (int i = 0; i < build_param_.refine_iterations; ++i) { - ggnn_->refine(); - } -} - -template -void GgnnImpl::set_search_dataset(const T* dataset, size_t nrow) -{ - if (base_dataset != dataset || base_n_rows != nrow) { - base_dataset = dataset; - base_n_rows = nrow; - load_impl(); - } -} - -template -void GgnnImpl::set_search_param(const AnnSearchParam& param) -{ - search_param_ = dynamic_cast::SearchParam&>(param); -} - -template -void GgnnImpl::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - static_assert(sizeof(size_t) == sizeof(int64_t), "sizes of size_t and GGNN's KeyT are different"); - if (k != KQuery) { - throw std::runtime_error( - "k = " + std::to_string(k) + - ", but this GGNN instance only supports k = " + std::to_string(KQuery)); - } - - ggnn_->set_stream(get_sync_stream()); - RAFT_CUDA_TRY(cudaMemcpyToSymbol(c_tau_query, &search_param_.tau, sizeof(float))); - - const int block_dim = search_param_.block_dim; - const int max_iterations = search_param_.max_iterations; - const int cache_size = search_param_.cache_size; - const int sorted_size = search_param_.sorted_size; - // default value - if (block_dim == 32 && max_iterations == 400 && cache_size == 512 && sorted_size == 256) { - ggnn_->template queryLayer<32, 400, 512, 256, false>( - queries, batch_size, reinterpret_cast(neighbors), distances); - } - // ggnn/src/sift1m.cu - else if (block_dim == 32 && max_iterations == 200 && cache_size == 256 && sorted_size == 64) { - ggnn_->template queryLayer<32, 200, 256, 64, false>( - queries, batch_size, reinterpret_cast(neighbors), distances); - } - // ggnn/src/sift1m.cu - else if (block_dim == 32 && max_iterations == 400 && cache_size == 448 && sorted_size == 64) { - ggnn_->template queryLayer<32, 400, 448, 64, false>( - queries, batch_size, reinterpret_cast(neighbors), distances); - } - // ggnn/src/glove200.cu - else if (block_dim == 128 && max_iterations == 2000 && cache_size == 2048 && sorted_size == 32) { - ggnn_->template queryLayer<128, 2000, 2048, 32, false>( - queries, batch_size, reinterpret_cast(neighbors), distances); - } - // for glove100 - else if (block_dim == 64 && max_iterations == 400 && cache_size == 512 && sorted_size == 32) { - ggnn_->template queryLayer<64, 400, 512, 32, false>( - queries, batch_size, reinterpret_cast(neighbors), distances); - } else if (block_dim == 128 && max_iterations == 2000 && cache_size == 1024 && - sorted_size == 32) { - ggnn_->template queryLayer<128, 2000, 1024, 32, false>( - queries, batch_size, reinterpret_cast(neighbors), distances); - } else { - throw std::runtime_error("ggnn: not supported search param"); - } -} - -template -void GgnnImpl::save(const std::string& file) const -{ - auto& ggnn_host = ggnn_->ggnn_cpu_buffers.at(0); - auto& ggnn_device = ggnn_->ggnn_shards.at(0); - ggnn_->set_stream(get_sync_stream()); - - ggnn_host.downloadAsync(ggnn_device); - RAFT_CUDA_TRY(cudaStreamSynchronize(ggnn_device.stream)); - ggnn_host.store(file); -} - -template -void GgnnImpl::load(const std::string& file) -{ - if (!graph_file.has_value() || graph_file.value() != file) { - graph_file = file; - load_impl(); - } -} - -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp b/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp deleted file mode 100644 index df82c68830..0000000000 --- a/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../common/ann_types.hpp" -#include "hnswlib_wrapper.h" - -#define JSON_DIAGNOSTICS 1 -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::HnswLib::BuildParam& param) -{ - param.ef_construction = conf.at("efConstruction"); - param.M = conf.at("M"); - if (conf.contains("numThreads")) { param.num_threads = conf.at("numThreads"); } -} - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::HnswLib::SearchParam& param) -{ - param.ef = conf.at("ef"); - if (conf.contains("numThreads")) { param.num_threads = conf.at("numThreads"); } -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - return std::make_unique>(metric, dim, param); -} - -template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - typename Algo::BuildParam param; - parse_build_param(conf, param); - - (void)dev_list; - return std::make_unique>(metric, dim, param); -} - -template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - // stop compiler warning; not all algorithms support multi-GPU so it may not be used - (void)dev_list; - - raft::bench::ann::Metric metric = parse_metric(distance); - std::unique_ptr> ann; - - if constexpr (std::is_same_v) { - if (algo == "hnswlib") { ann = make_algo(metric, dim, conf); } - } - - if constexpr (std::is_same_v) { - if (algo == "hnswlib") { ann = make_algo(metric, dim, conf); } - } - - if (!ann) { throw std::runtime_error("invalid algo: '" + algo + "'"); } - return ann; -} - -template -std::unique_ptr::AnnSearchParam> create_search_param( - const std::string& algo, const nlohmann::json& conf) -{ - if (algo == "hnswlib") { - auto param = std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } - // else - throw std::runtime_error("invalid algo: '" + algo + "'"); -} - -}; // namespace raft::bench::ann - -REGISTER_ALGO_INSTANCE(float); -REGISTER_ALGO_INSTANCE(std::int8_t); -REGISTER_ALGO_INSTANCE(std::uint8_t); - -#ifdef ANN_BENCH_BUILD_MAIN -#include "../common/benchmark.hpp" -int main(int argc, char** argv) { return raft::bench::ann::run_main(argc, argv); } -#endif diff --git a/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h b/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h deleted file mode 100644 index 5743632bf4..0000000000 --- a/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../common/ann_types.hpp" -#include "../common/thread_pool.hpp" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -template -struct hnsw_dist_t { - using type = void; -}; - -template <> -struct hnsw_dist_t { - using type = float; -}; - -template <> -struct hnsw_dist_t { - using type = int; -}; - -template <> -struct hnsw_dist_t { - using type = int; -}; - -template -class HnswLib : public ANN { - public: - // https://github.com/nmslib/hnswlib/blob/master/ALGO_PARAMS.md - struct BuildParam { - int M; - int ef_construction; - int num_threads = omp_get_num_procs(); - }; - - using typename ANN::AnnSearchParam; - struct SearchParam : public AnnSearchParam { - int ef; - int num_threads = 1; - }; - - HnswLib(Metric metric, int dim, const BuildParam& param); - - void build(const T* dataset, size_t nrow) override; - - void set_search_param(const AnnSearchParam& param) override; - void search(const T* query, - int batch_size, - int k, - AnnBase::index_type* indices, - float* distances) const override; - - void save(const std::string& path_to_index) const override; - void load(const std::string& path_to_index) override; - std::unique_ptr> copy() override { return std::make_unique>(*this); }; - - AlgoProperty get_preference() const override - { - AlgoProperty property; - property.dataset_memory_type = MemoryType::Host; - property.query_memory_type = MemoryType::Host; - return property; - } - - void set_base_layer_only() { appr_alg_->base_layer_only = true; } - - private: - void get_search_knn_results_(const T* query, - int k, - AnnBase::index_type* indices, - float* distances) const; - - std::shared_ptr::type>> appr_alg_; - std::shared_ptr::type>> space_; - - using ANN::metric_; - using ANN::dim_; - int ef_construction_; - int m_; - int num_threads_; - std::shared_ptr thread_pool_; - Objective metric_objective_; -}; - -template -HnswLib::HnswLib(Metric metric, int dim, const BuildParam& param) : ANN(metric, dim) -{ - assert(dim_ > 0); - static_assert(std::is_same_v || std::is_same_v); - if constexpr (std::is_same_v) { - if (metric_ != Metric::kEuclidean) { - throw std::runtime_error("hnswlib only supports Euclidean distance"); - } - } - - ef_construction_ = param.ef_construction; - m_ = param.M; - num_threads_ = param.num_threads; -} - -template -void HnswLib::build(const T* dataset, size_t nrow) -{ - if constexpr (std::is_same_v) { - if (metric_ == Metric::kInnerProduct) { - space_ = std::make_shared(dim_); - } else { - space_ = std::make_shared(dim_); - } - } else if constexpr (std::is_same_v) { - space_ = std::make_shared>(dim_); - } - - appr_alg_ = std::make_shared::type>>( - space_.get(), nrow, m_, ef_construction_); - - thread_pool_ = std::make_shared(num_threads_); - const size_t items_per_thread = nrow / (num_threads_ + 1); - - thread_pool_->submit( - [&](size_t i) { - if (i < items_per_thread && i % 10000 == 0) { - char buf[20]; - std::time_t now = std::time(nullptr); - std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", std::localtime(&now)); - printf("%s building %zu / %zu\n", buf, i, items_per_thread); - fflush(stdout); - } - - appr_alg_->addPoint(dataset + i * dim_, i); - }, - nrow); -} - -template -void HnswLib::set_search_param(const AnnSearchParam& param_) -{ - auto param = dynamic_cast(param_); - appr_alg_->ef_ = param.ef; - metric_objective_ = param.metric_objective; - num_threads_ = param.num_threads; - - // Create a pool if multiple query threads have been set and the pool hasn't been created already - bool create_pool = (metric_objective_ == Objective::LATENCY && num_threads_ > 1 && !thread_pool_); - if (create_pool) { thread_pool_ = std::make_shared(num_threads_); } -} - -template -void HnswLib::search( - const T* query, int batch_size, int k, AnnBase::index_type* indices, float* distances) const -{ - auto f = [&](int i) { - // hnsw can only handle a single vector at a time. - get_search_knn_results_(query + i * dim_, k, indices + i * k, distances + i * k); - }; - if (metric_objective_ == Objective::LATENCY && num_threads_ > 1) { - thread_pool_->submit(f, batch_size); - } else { - for (int i = 0; i < batch_size; i++) { - f(i); - } - } -} - -template -void HnswLib::save(const std::string& path_to_index) const -{ - appr_alg_->saveIndex(std::string(path_to_index)); -} - -template -void HnswLib::load(const std::string& path_to_index) -{ - if constexpr (std::is_same_v) { - if (metric_ == Metric::kInnerProduct) { - space_ = std::make_shared(dim_); - } else { - space_ = std::make_shared(dim_); - } - } else if constexpr (std::is_same_v) { - space_ = std::make_shared>(dim_); - } - - appr_alg_ = std::make_shared::type>>( - space_.get(), path_to_index); -} - -template -void HnswLib::get_search_knn_results_(const T* query, - int k, - AnnBase::index_type* indices, - float* distances) const -{ - auto result = appr_alg_->searchKnn(query, k); - assert(result.size() >= static_cast(k)); - - for (int i = k - 1; i >= 0; --i) { - indices[i] = result.top().second; - distances[i] = result.top().first; - result.pop(); - } -} - -}; // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_ann_bench_param_parser.h b/cpp/bench/ann/src/raft/raft_ann_bench_param_parser.h deleted file mode 100644 index 48bf1d70d8..0000000000 --- a/cpp/bench/ann/src/raft/raft_ann_bench_param_parser.h +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#define JSON_DIAGNOSTICS 1 -#include - -#undef WARP_SIZE -#ifdef RAFT_ANN_BENCH_USE_RAFT_BRUTE_FORCE -#include "raft_wrapper.h" -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT -#include "raft_ivf_flat_wrapper.h" -extern template class raft::bench::ann::RaftIvfFlatGpu; -extern template class raft::bench::ann::RaftIvfFlatGpu; -extern template class raft::bench::ann::RaftIvfFlatGpu; -#endif -#if defined(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ) || defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA) || \ - defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB) -#include "raft_ivf_pq_wrapper.h" -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_IVF_PQ -extern template class raft::bench::ann::RaftIvfPQ; -extern template class raft::bench::ann::RaftIvfPQ; -extern template class raft::bench::ann::RaftIvfPQ; -#endif -#if defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA) || defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB) -#include "raft_cagra_wrapper.h" -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_CAGRA -extern template class raft::bench::ann::RaftCagra; -extern template class raft::bench::ann::RaftCagra; -extern template class raft::bench::ann::RaftCagra; -extern template class raft::bench::ann::RaftCagra; -#endif - -#ifdef RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftIvfFlatGpu::BuildParam& param) -{ - param.n_lists = conf.at("nlist"); - if (conf.contains("niter")) { param.kmeans_n_iters = conf.at("niter"); } - if (conf.contains("ratio")) { param.kmeans_trainset_fraction = 1.0 / (double)conf.at("ratio"); } -} - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftIvfFlatGpu::SearchParam& param) -{ - param.ivf_flat_params.n_probes = conf.at("nprobe"); -} -#endif - -#if defined(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ) || defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA) || \ - defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB) -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftIvfPQ::BuildParam& param) -{ - if (conf.contains("nlist")) { param.n_lists = conf.at("nlist"); } - if (conf.contains("niter")) { param.kmeans_n_iters = conf.at("niter"); } - if (conf.contains("ratio")) { param.kmeans_trainset_fraction = 1.0 / (double)conf.at("ratio"); } - if (conf.contains("pq_bits")) { param.pq_bits = conf.at("pq_bits"); } - if (conf.contains("pq_dim")) { param.pq_dim = conf.at("pq_dim"); } - if (conf.contains("codebook_kind")) { - std::string kind = conf.at("codebook_kind"); - if (kind == "cluster") { - param.codebook_kind = raft::neighbors::ivf_pq::codebook_gen::PER_CLUSTER; - } else if (kind == "subspace") { - param.codebook_kind = raft::neighbors::ivf_pq::codebook_gen::PER_SUBSPACE; - } else { - throw std::runtime_error("codebook_kind: '" + kind + - "', should be either 'cluster' or 'subspace'"); - } - } -} - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftIvfPQ::SearchParam& param) -{ - if (conf.contains("nprobe")) { param.pq_param.n_probes = conf.at("nprobe"); } - if (conf.contains("internalDistanceDtype")) { - std::string type = conf.at("internalDistanceDtype"); - if (type == "float") { - param.pq_param.internal_distance_dtype = CUDA_R_32F; - } else if (type == "half") { - param.pq_param.internal_distance_dtype = CUDA_R_16F; - } else { - throw std::runtime_error("internalDistanceDtype: '" + type + - "', should be either 'float' or 'half'"); - } - } else { - // set half as default type - param.pq_param.internal_distance_dtype = CUDA_R_16F; - } - - if (conf.contains("smemLutDtype")) { - std::string type = conf.at("smemLutDtype"); - if (type == "float") { - param.pq_param.lut_dtype = CUDA_R_32F; - } else if (type == "half") { - param.pq_param.lut_dtype = CUDA_R_16F; - } else if (type == "fp8") { - param.pq_param.lut_dtype = CUDA_R_8U; - } else { - throw std::runtime_error("smemLutDtype: '" + type + - "', should be either 'float', 'half' or 'fp8'"); - } - } else { - // set half as default - param.pq_param.lut_dtype = CUDA_R_16F; - } - if (conf.contains("refine_ratio")) { - param.refine_ratio = conf.at("refine_ratio"); - if (param.refine_ratio < 1.0f) { throw std::runtime_error("refine_ratio should be >= 1.0"); } - } -} -#endif - -#if defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA) || defined(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB) -template -void parse_build_param(const nlohmann::json& conf, - raft::neighbors::experimental::nn_descent::index_params& param) -{ - if (conf.contains("graph_degree")) { param.graph_degree = conf.at("graph_degree"); } - if (conf.contains("intermediate_graph_degree")) { - param.intermediate_graph_degree = conf.at("intermediate_graph_degree"); - } - // we allow niter shorthand for max_iterations - if (conf.contains("niter")) { param.max_iterations = conf.at("niter"); } - if (conf.contains("max_iterations")) { param.max_iterations = conf.at("max_iterations"); } - if (conf.contains("termination_threshold")) { - param.termination_threshold = conf.at("termination_threshold"); - } -} - -inline void parse_build_param(const nlohmann::json& conf, raft::neighbors::vpq_params& param) -{ - if (conf.contains("pq_bits")) { param.pq_bits = conf.at("pq_bits"); } - if (conf.contains("pq_dim")) { param.pq_dim = conf.at("pq_dim"); } - if (conf.contains("vq_n_centers")) { param.vq_n_centers = conf.at("vq_n_centers"); } - if (conf.contains("kmeans_n_iters")) { param.kmeans_n_iters = conf.at("kmeans_n_iters"); } - if (conf.contains("vq_kmeans_trainset_fraction")) { - param.vq_kmeans_trainset_fraction = conf.at("vq_kmeans_trainset_fraction"); - } - if (conf.contains("pq_kmeans_trainset_fraction")) { - param.pq_kmeans_trainset_fraction = conf.at("pq_kmeans_trainset_fraction"); - } -} - -nlohmann::json collect_conf_with_prefix(const nlohmann::json& conf, - const std::string& prefix, - bool remove_prefix = true) -{ - nlohmann::json out; - for (auto& i : conf.items()) { - if (i.key().compare(0, prefix.size(), prefix) == 0) { - auto new_key = remove_prefix ? i.key().substr(prefix.size()) : i.key(); - out[new_key] = i.value(); - } - } - return out; -} - -template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftCagra::BuildParam& param) -{ - if (conf.contains("graph_degree")) { - param.cagra_params.graph_degree = conf.at("graph_degree"); - param.cagra_params.intermediate_graph_degree = param.cagra_params.graph_degree * 2; - } - if (conf.contains("intermediate_graph_degree")) { - param.cagra_params.intermediate_graph_degree = conf.at("intermediate_graph_degree"); - } - if (conf.contains("graph_build_algo")) { - if (conf.at("graph_build_algo") == "IVF_PQ") { - param.cagra_params.build_algo = raft::neighbors::cagra::graph_build_algo::IVF_PQ; - } else if (conf.at("graph_build_algo") == "NN_DESCENT") { - param.cagra_params.build_algo = raft::neighbors::cagra::graph_build_algo::NN_DESCENT; - } - } - nlohmann::json ivf_pq_build_conf = collect_conf_with_prefix(conf, "ivf_pq_build_"); - if (!ivf_pq_build_conf.empty()) { - raft::neighbors::ivf_pq::index_params bparam; - parse_build_param(ivf_pq_build_conf, bparam); - param.ivf_pq_build_params = bparam; - } - nlohmann::json ivf_pq_search_conf = collect_conf_with_prefix(conf, "ivf_pq_search_"); - if (!ivf_pq_search_conf.empty()) { - typename raft::bench::ann::RaftIvfPQ::SearchParam sparam; - parse_search_param(ivf_pq_search_conf, sparam); - param.ivf_pq_search_params = sparam.pq_param; - param.ivf_pq_refine_rate = sparam.refine_ratio; - } - nlohmann::json nn_descent_conf = collect_conf_with_prefix(conf, "nn_descent_"); - if (!nn_descent_conf.empty()) { - raft::neighbors::experimental::nn_descent::index_params nn_param; - nn_param.intermediate_graph_degree = 1.5 * param.cagra_params.intermediate_graph_degree; - parse_build_param(nn_descent_conf, nn_param); - if (nn_param.graph_degree != param.cagra_params.intermediate_graph_degree) { - nn_param.graph_degree = param.cagra_params.intermediate_graph_degree; - } - param.nn_descent_params = nn_param; - } - nlohmann::json comp_search_conf = collect_conf_with_prefix(conf, "compression_"); - if (!comp_search_conf.empty()) { - raft::neighbors::vpq_params vpq_pams; - parse_build_param(comp_search_conf, vpq_pams); - param.cagra_params.compression.emplace(vpq_pams); - } -} - -raft::bench::ann::AllocatorType parse_allocator(std::string mem_type) -{ - if (mem_type == "device") { - return raft::bench::ann::AllocatorType::Device; - } else if (mem_type == "host_pinned") { - return raft::bench::ann::AllocatorType::HostPinned; - } else if (mem_type == "host_huge_page") { - return raft::bench::ann::AllocatorType::HostHugePage; - } - THROW( - "Invalid value for memory type %s, must be one of [\"device\", \"host_pinned\", " - "\"host_huge_page\"", - mem_type.c_str()); -} - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftCagra::SearchParam& param) -{ - if (conf.contains("itopk")) { param.p.itopk_size = conf.at("itopk"); } - if (conf.contains("search_width")) { param.p.search_width = conf.at("search_width"); } - if (conf.contains("max_iterations")) { param.p.max_iterations = conf.at("max_iterations"); } - if (conf.contains("algo")) { - if (conf.at("algo") == "single_cta") { - param.p.algo = raft::neighbors::experimental::cagra::search_algo::SINGLE_CTA; - } else if (conf.at("algo") == "multi_cta") { - param.p.algo = raft::neighbors::experimental::cagra::search_algo::MULTI_CTA; - } else if (conf.at("algo") == "multi_kernel") { - param.p.algo = raft::neighbors::experimental::cagra::search_algo::MULTI_KERNEL; - } else if (conf.at("algo") == "auto") { - param.p.algo = raft::neighbors::experimental::cagra::search_algo::AUTO; - } else { - std::string tmp = conf.at("algo"); - THROW("Invalid value for algo: %s", tmp.c_str()); - } - } - if (conf.contains("graph_memory_type")) { - param.graph_mem = parse_allocator(conf.at("graph_memory_type")); - } - if (conf.contains("internal_dataset_memory_type")) { - param.dataset_mem = parse_allocator(conf.at("internal_dataset_memory_type")); - } - // Same ratio as in IVF-PQ - param.refine_ratio = conf.value("refine_ratio", 1.0f); -} -#endif diff --git a/cpp/bench/ann/src/raft/raft_ann_bench_utils.h b/cpp/bench/ann/src/raft/raft_ann_bench_utils.h deleted file mode 100644 index 9b086fdb23..0000000000 --- a/cpp/bench/ann/src/raft/raft_ann_bench_utils.h +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../common/util.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace raft::bench::ann { - -inline raft::distance::DistanceType parse_metric_type(raft::bench::ann::Metric metric) -{ - if (metric == raft::bench::ann::Metric::kInnerProduct) { - return raft::distance::DistanceType::InnerProduct; - } else if (metric == raft::bench::ann::Metric::kEuclidean) { - // Even for L2 expanded RAFT IVF Flat uses unexpanded formula - return raft::distance::DistanceType::L2Expanded; - } else { - throw std::runtime_error("raft supports only metric type of inner product and L2"); - } -} - -/** Report a more verbose error with a backtrace when OOM occurs on RMM side. */ -inline auto rmm_oom_callback(std::size_t bytes, void*) -> bool -{ - auto cuda_status = cudaGetLastError(); - size_t free = 0; - size_t total = 0; - RAFT_CUDA_TRY_NO_THROW(cudaMemGetInfo(&free, &total)); - RAFT_FAIL( - "Failed to allocate %zu bytes using RMM memory resource. " - "NB: latest cuda status = %s, free memory = %zu, total memory = %zu.", - bytes, - cudaGetErrorName(cuda_status), - free, - total); -} - -/** - * This container keeps the part of raft state that should be shared among multiple copies of raft - * handles (in different CPU threads). - * An example of this is an RMM memory resource: if we had an RMM memory pool per thread, we'd - * quickly run out of memory. - */ -class shared_raft_resources { - public: - using pool_mr_type = rmm::mr::pool_memory_resource; - using mr_type = rmm::mr::failure_callback_resource_adaptor; - using large_mr_type = rmm::mr::managed_memory_resource; - - shared_raft_resources() - try : orig_resource_{rmm::mr::get_current_device_resource()}, - pool_resource_(orig_resource_, 1024 * 1024 * 1024ull), - resource_(&pool_resource_, rmm_oom_callback, nullptr), large_mr_() { - rmm::mr::set_current_device_resource(&resource_); - } catch (const std::exception& e) { - auto cuda_status = cudaGetLastError(); - size_t free = 0; - size_t total = 0; - RAFT_CUDA_TRY_NO_THROW(cudaMemGetInfo(&free, &total)); - RAFT_FAIL( - "Failed to initialize shared raft resources (NB: latest cuda status = %s, free memory = %zu, " - "total memory = %zu): %s", - cudaGetErrorName(cuda_status), - free, - total, - e.what()); - } - - shared_raft_resources(shared_raft_resources&&) = delete; - shared_raft_resources& operator=(shared_raft_resources&&) = delete; - shared_raft_resources(const shared_raft_resources& res) = delete; - shared_raft_resources& operator=(const shared_raft_resources& other) = delete; - - ~shared_raft_resources() noexcept { rmm::mr::set_current_device_resource(orig_resource_); } - - auto get_large_memory_resource() noexcept - { - return static_cast(&large_mr_); - } - - private: - rmm::mr::device_memory_resource* orig_resource_; - pool_mr_type pool_resource_; - mr_type resource_; - large_mr_type large_mr_; -}; - -/** - * This struct is used by multiple raft benchmark wrappers. It serves as a thread-safe keeper of - * shared and private GPU resources (see below). - * - * - Accessing the same `configured_raft_resources` from concurrent threads is not safe. - * - Accessing the copies of `configured_raft_resources` from concurrent threads is safe. - * - There must be at most one "original" `configured_raft_resources` at any time, but as many - * copies of it as needed (modifies the program static state). - */ -class configured_raft_resources { - public: - /** - * This constructor has the shared state passed unmodified but creates the local state anew. - * It's used by the copy constructor. - */ - explicit configured_raft_resources(const std::shared_ptr& shared_res) - : shared_res_{shared_res}, - res_{std::make_unique( - rmm::cuda_stream_view(get_stream_from_global_pool()))} - { - // set the large workspace resource to the raft handle, but without the deleter - // (this resource is managed by the shared_res). - raft::resource::set_large_workspace_resource( - *res_, - std::shared_ptr(shared_res_->get_large_memory_resource(), - raft::void_op{})); - } - - /** Default constructor creates all resources anew. */ - configured_raft_resources() : configured_raft_resources{std::make_shared()} - { - } - - configured_raft_resources(configured_raft_resources&&); - configured_raft_resources& operator=(configured_raft_resources&&); - ~configured_raft_resources() = default; - configured_raft_resources(const configured_raft_resources& res) - : configured_raft_resources{res.shared_res_} - { - } - configured_raft_resources& operator=(const configured_raft_resources& other) - { - this->shared_res_ = other.shared_res_; - return *this; - } - - operator raft::resources&() noexcept { return *res_; } - operator const raft::resources&() const noexcept { return *res_; } - - /** Get the main stream */ - [[nodiscard]] auto get_sync_stream() const noexcept { return resource::get_cuda_stream(*res_); } - - private: - /** The resources shared among multiple raft handles / threads. */ - std::shared_ptr shared_res_; - /** - * Until we make the use of copies of raft::resources thread-safe, each benchmark wrapper must - * have its own copy of it. - */ - std::unique_ptr res_ = std::make_unique(); -}; - -inline configured_raft_resources::configured_raft_resources(configured_raft_resources&&) = default; -inline configured_raft_resources& configured_raft_resources::operator=( - configured_raft_resources&&) = default; - -/** A helper to refine the neighbors when the data is on device or on host. */ -template -void refine_helper(const raft::resources& res, - DatasetT dataset, - QueriesT queries, - CandidatesT candidates, - int k, - AnnBase::index_type* neighbors, - float* distances, - raft::distance::DistanceType metric) -{ - using data_type = typename DatasetT::value_type; - using index_type = AnnBase::index_type; - using extents_type = index_type; // device-side refine requires this - - static_assert(std::is_same_v); - static_assert(std::is_same_v); - static_assert(std::is_same_v); - - extents_type batch_size = queries.extent(0); - extents_type dim = queries.extent(1); - extents_type k0 = candidates.extent(1); - - if (raft::get_device_for_address(dataset.data_handle()) >= 0) { - auto dataset_device = raft::make_device_matrix_view( - dataset.data_handle(), dataset.extent(0), dataset.extent(1)); - auto queries_device = raft::make_device_matrix_view( - queries.data_handle(), batch_size, dim); - auto candidates_device = raft::make_device_matrix_view( - candidates.data_handle(), batch_size, k0); - auto neighbors_device = - raft::make_device_matrix_view(neighbors, batch_size, k); - auto distances_device = - raft::make_device_matrix_view(distances, batch_size, k); - - raft::neighbors::refine(res, - dataset_device, - queries_device, - candidates_device, - neighbors_device, - distances_device, - metric); - } else { - auto dataset_host = raft::make_host_matrix_view( - dataset.data_handle(), dataset.extent(0), dataset.extent(1)); - auto queries_host = raft::make_host_matrix(batch_size, dim); - auto candidates_host = raft::make_host_matrix(batch_size, k0); - auto neighbors_host = raft::make_host_matrix(batch_size, k); - auto distances_host = raft::make_host_matrix(batch_size, k); - - auto stream = resource::get_cuda_stream(res); - raft::copy(queries_host.data_handle(), queries.data_handle(), queries_host.size(), stream); - raft::copy( - candidates_host.data_handle(), candidates.data_handle(), candidates_host.size(), stream); - - raft::resource::sync_stream(res); // wait for the queries and candidates - raft::neighbors::refine(res, - dataset_host, - queries_host.view(), - candidates_host.view(), - neighbors_host.view(), - distances_host.view(), - metric); - - raft::copy(neighbors, neighbors_host.data_handle(), neighbors_host.size(), stream); - raft::copy(distances, distances_host.data_handle(), distances_host.size(), stream); - } -} - -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_benchmark.cu b/cpp/bench/ann/src/raft/raft_benchmark.cu deleted file mode 100644 index 8bb4d9423c..0000000000 --- a/cpp/bench/ann/src/raft/raft_benchmark.cu +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../common/ann_types.hpp" -#include "raft_ann_bench_param_parser.h" - -#include - -#include - -#define JSON_DIAGNOSTICS 1 -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - // stop compiler warning; not all algorithms support multi-GPU so it may not be used - (void)dev_list; - - [[maybe_unused]] raft::bench::ann::Metric metric = parse_metric(distance); - std::unique_ptr> ann; - - if constexpr (std::is_same_v) { -#ifdef RAFT_ANN_BENCH_USE_RAFT_BRUTE_FORCE - if (algo == "raft_brute_force") { - ann = std::make_unique>(metric, dim); - } -#endif - } - - if constexpr (std::is_same_v) {} - -#ifdef RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT - if constexpr (std::is_same_v || std::is_same_v || - std::is_same_v) { - if (algo == "raft_ivf_flat") { - typename raft::bench::ann::RaftIvfFlatGpu::BuildParam param; - parse_build_param(conf, param); - ann = std::make_unique>(metric, dim, param); - } - } -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_IVF_PQ - if (algo == "raft_ivf_pq") { - typename raft::bench::ann::RaftIvfPQ::BuildParam param; - parse_build_param(conf, param); - ann = std::make_unique>(metric, dim, param); - } -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_CAGRA - if (algo == "raft_cagra") { - typename raft::bench::ann::RaftCagra::BuildParam param; - parse_build_param(conf, param); - ann = std::make_unique>(metric, dim, param); - } -#endif - - if (!ann) { throw std::runtime_error("invalid algo: '" + algo + "'"); } - - return ann; -} - -template -std::unique_ptr::AnnSearchParam> create_search_param( - const std::string& algo, const nlohmann::json& conf) -{ -#ifdef RAFT_ANN_BENCH_USE_RAFT_BRUTE_FORCE - if (algo == "raft_brute_force") { - auto param = std::make_unique::AnnSearchParam>(); - return param; - } -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT - if constexpr (std::is_same_v || std::is_same_v || - std::is_same_v) { - if (algo == "raft_ivf_flat") { - auto param = - std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } - } -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_IVF_PQ - if (algo == "raft_ivf_pq") { - auto param = std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } -#endif -#ifdef RAFT_ANN_BENCH_USE_RAFT_CAGRA - if (algo == "raft_cagra") { - auto param = std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } -#endif - - // else - throw std::runtime_error("invalid algo: '" + algo + "'"); -} - -}; // namespace raft::bench::ann - -REGISTER_ALGO_INSTANCE(float); -REGISTER_ALGO_INSTANCE(half); -REGISTER_ALGO_INSTANCE(std::int8_t); -REGISTER_ALGO_INSTANCE(std::uint8_t); - -#ifdef ANN_BENCH_BUILD_MAIN -#include "../common/benchmark.hpp" -int main(int argc, char** argv) { return raft::bench::ann::run_main(argc, argv); } -#endif diff --git a/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu b/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu deleted file mode 100644 index d9ef1d74a3..0000000000 --- a/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../common/ann_types.hpp" -#include "raft_ann_bench_param_parser.h" -#include "raft_cagra_hnswlib_wrapper.h" - -#include -#include -#include - -#define JSON_DIAGNOSTICS 1 -#include - -namespace raft::bench::ann { - -template -void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftCagraHnswlib::SearchParam& param) -{ - param.ef = conf.at("ef"); - if (conf.contains("numThreads")) { param.num_threads = conf.at("numThreads"); } -} - -template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) -{ - // stop compiler warning; not all algorithms support multi-GPU so it may not be used - (void)dev_list; - - [[maybe_unused]] raft::bench::ann::Metric metric = parse_metric(distance); - std::unique_ptr> ann; - - if constexpr (std::is_same_v or std::is_same_v) { - if (algo == "raft_cagra_hnswlib") { - typename raft::bench::ann::RaftCagraHnswlib::BuildParam param; - parse_build_param(conf, param); - ann = std::make_unique>(metric, dim, param); - } - } - - if (!ann) { throw std::runtime_error("invalid algo: '" + algo + "'"); } - - return ann; -} - -template -std::unique_ptr::AnnSearchParam> create_search_param( - const std::string& algo, const nlohmann::json& conf) -{ - if (algo == "raft_cagra_hnswlib") { - auto param = - std::make_unique::SearchParam>(); - parse_search_param(conf, *param); - return param; - } - - throw std::runtime_error("invalid algo: '" + algo + "'"); -} - -} // namespace raft::bench::ann - -REGISTER_ALGO_INSTANCE(float); -REGISTER_ALGO_INSTANCE(std::int8_t); -REGISTER_ALGO_INSTANCE(std::uint8_t); - -#ifdef ANN_BENCH_BUILD_MAIN -#include "../common/benchmark.hpp" -int main(int argc, char** argv) -{ - rmm::mr::cuda_memory_resource cuda_mr; - // Construct a resource that uses a coalescing best-fit pool allocator - // and is initially sized to half of free device memory. - rmm::mr::pool_memory_resource pool_mr{ - &cuda_mr, rmm::percent_of_free_device_memory(50)}; - // Updates the current device resource pointer to `pool_mr` - auto old_mr = rmm::mr::set_current_device_resource(&pool_mr); - auto ret = raft::bench::ann::run_main(argc, argv); - // Restores the current device resource pointer to its previous value - rmm::mr::set_current_device_resource(old_mr); - return ret; -} -#endif diff --git a/cpp/bench/ann/src/raft/raft_cagra_hnswlib_wrapper.h b/cpp/bench/ann/src/raft/raft_cagra_hnswlib_wrapper.h deleted file mode 100644 index 1d2a1076ab..0000000000 --- a/cpp/bench/ann/src/raft/raft_cagra_hnswlib_wrapper.h +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../hnswlib/hnswlib_wrapper.h" -#include "raft_cagra_wrapper.h" - -#include - -namespace raft::bench::ann { - -template -class RaftCagraHnswlib : public ANN, public AnnGPU { - public: - using typename ANN::AnnSearchParam; - using BuildParam = typename RaftCagra::BuildParam; - using SearchParam = typename HnswLib::SearchParam; - - RaftCagraHnswlib(Metric metric, int dim, const BuildParam& param, int concurrent_searches = 1) - : ANN(metric, dim), - cagra_build_{metric, dim, param, concurrent_searches, true}, - // HnswLib param values don't matter since we don't build with HnswLib - hnswlib_search_{metric, dim, typename HnswLib::BuildParam{50, 100}} - { - } - - void build(const T* dataset, size_t nrow) final; - - void set_search_param(const AnnSearchParam& param) override; - - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const override; - - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override - { - return cagra_build_.get_sync_stream(); - } - - // to enable dataset access from GPU memory - AlgoProperty get_preference() const override - { - AlgoProperty property; - property.dataset_memory_type = MemoryType::HostMmap; - property.query_memory_type = MemoryType::Host; - return property; - } - - void save(const std::string& file) const override; - void load(const std::string&) override; - std::unique_ptr> copy() override - { - return std::make_unique>(*this); - } - - private: - RaftCagra cagra_build_; - HnswLib hnswlib_search_; -}; - -template -void RaftCagraHnswlib::build(const T* dataset, size_t nrow) -{ - cagra_build_.build(dataset, nrow); -} - -template -void RaftCagraHnswlib::set_search_param(const AnnSearchParam& param_) -{ - hnswlib_search_.set_search_param(param_); -} - -template -void RaftCagraHnswlib::save(const std::string& file) const -{ - cagra_build_.save_to_hnswlib(file); -} - -template -void RaftCagraHnswlib::load(const std::string& file) -{ - hnswlib_search_.load(file); - hnswlib_search_.set_base_layer_only(); -} - -template -void RaftCagraHnswlib::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - hnswlib_search_.search(queries, batch_size, k, neighbors, distances); -} - -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_cagra_wrapper.h b/cpp/bench/ann/src/raft/raft_cagra_wrapper.h deleted file mode 100644 index b03f875a8e..0000000000 --- a/cpp/bench/ann/src/raft/raft_cagra_wrapper.h +++ /dev/null @@ -1,339 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../common/ann_types.hpp" -#include "../common/cuda_huge_page_resource.hpp" -#include "../common/cuda_pinned_resource.hpp" -#include "raft_ann_bench_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -enum class AllocatorType { HostPinned, HostHugePage, Device }; -template -class RaftCagra : public ANN, public AnnGPU { - public: - using typename ANN::AnnSearchParam; - - struct SearchParam : public AnnSearchParam { - raft::neighbors::experimental::cagra::search_params p; - float refine_ratio; - AllocatorType graph_mem = AllocatorType::Device; - AllocatorType dataset_mem = AllocatorType::Device; - auto needs_dataset() const -> bool override { return true; } - }; - - struct BuildParam { - raft::neighbors::cagra::index_params cagra_params; - std::optional nn_descent_params = - std::nullopt; - std::optional ivf_pq_refine_rate = std::nullopt; - std::optional ivf_pq_build_params = std::nullopt; - std::optional ivf_pq_search_params = std::nullopt; - }; - - RaftCagra(Metric metric, - int dim, - const BuildParam& param, - int concurrent_searches = 1, - bool shall_include_dataset = false) - : ANN(metric, dim), - index_params_(param), - dimension_(dim), - need_dataset_update_(true), - shall_include_dataset_(shall_include_dataset), - dataset_(std::make_shared>( - std::move(make_device_matrix(handle_, 0, 0)))), - graph_(std::make_shared>( - std::move(make_device_matrix(handle_, 0, 0)))), - input_dataset_v_( - std::make_shared>(nullptr, 0, 0)), - graph_mem_(AllocatorType::Device), - dataset_mem_(AllocatorType::Device) - { - index_params_.cagra_params.metric = parse_metric_type(metric); - index_params_.ivf_pq_build_params->metric = parse_metric_type(metric); - } - - void build(const T* dataset, size_t nrow) final; - - void set_search_param(const AnnSearchParam& param) override; - - void set_search_dataset(const T* dataset, size_t nrow) override; - - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const override; - void search_base(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const; - - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override - { - return handle_.get_sync_stream(); - } - - // to enable dataset access from GPU memory - AlgoProperty get_preference() const override - { - AlgoProperty property; - property.dataset_memory_type = MemoryType::HostMmap; - property.query_memory_type = MemoryType::Device; - return property; - } - void save(const std::string& file) const override; - void load(const std::string&) override; - void save_to_hnswlib(const std::string& file) const; - std::unique_ptr> copy() override; - - private: - // handle_ must go first to make sure it dies last and all memory allocated in pool - configured_raft_resources handle_{}; - raft::mr::cuda_pinned_resource mr_pinned_; - raft::mr::cuda_huge_page_resource mr_huge_page_; - AllocatorType graph_mem_; - AllocatorType dataset_mem_; - float refine_ratio_; - BuildParam index_params_; - bool need_dataset_update_; - bool shall_include_dataset_; - raft::neighbors::cagra::search_params search_params_; - std::shared_ptr> index_; - int dimension_; - std::shared_ptr> graph_; - std::shared_ptr> dataset_; - std::shared_ptr> input_dataset_v_; - - inline rmm::device_async_resource_ref get_mr(AllocatorType mem_type) - { - switch (mem_type) { - case (AllocatorType::HostPinned): return &mr_pinned_; - case (AllocatorType::HostHugePage): return &mr_huge_page_; - default: return rmm::mr::get_current_device_resource(); - } - } -}; - -template -void RaftCagra::build(const T* dataset, size_t nrow) -{ - auto dataset_view = - raft::make_host_matrix_view(dataset, IdxT(nrow), dimension_); - - auto& params = index_params_.cagra_params; - - // Do include the compressed dataset for the CAGRA-Q - bool include_dataset = params.compression.has_value() || shall_include_dataset_; - - index_ = std::make_shared>( - std::move(raft::neighbors::cagra::detail::build(handle_, - params, - dataset_view, - index_params_.nn_descent_params, - index_params_.ivf_pq_refine_rate, - index_params_.ivf_pq_build_params, - index_params_.ivf_pq_search_params, - include_dataset))); -} - -inline std::string allocator_to_string(AllocatorType mem_type) -{ - if (mem_type == AllocatorType::Device) { - return "device"; - } else if (mem_type == AllocatorType::HostPinned) { - return "host_pinned"; - } else if (mem_type == AllocatorType::HostHugePage) { - return "host_huge_page"; - } - return ""; -} - -template -void RaftCagra::set_search_param(const AnnSearchParam& param) -{ - auto search_param = dynamic_cast(param); - search_params_ = search_param.p; - refine_ratio_ = search_param.refine_ratio; - if (search_param.graph_mem != graph_mem_) { - // Move graph to correct memory space - graph_mem_ = search_param.graph_mem; - RAFT_LOG_DEBUG("moving graph to new memory space: %s", allocator_to_string(graph_mem_).c_str()); - // We create a new graph and copy to it from existing graph - auto mr = get_mr(graph_mem_); - auto new_graph = make_device_mdarray( - handle_, mr, make_extents(index_->graph().extent(0), index_->graph_degree())); - - raft::copy(new_graph.data_handle(), - index_->graph().data_handle(), - index_->graph().size(), - resource::get_cuda_stream(handle_)); - - index_->update_graph(handle_, make_const_mdspan(new_graph.view())); - // update_graph() only stores a view in the index. We need to keep the graph object alive. - *graph_ = std::move(new_graph); - } - - if (search_param.dataset_mem != dataset_mem_ || need_dataset_update_) { - dataset_mem_ = search_param.dataset_mem; - - // First free up existing memory - *dataset_ = make_device_matrix(handle_, 0, 0); - index_->update_dataset(handle_, make_const_mdspan(dataset_->view())); - - // Allocate space using the correct memory resource. - RAFT_LOG_DEBUG("moving dataset to new memory space: %s", - allocator_to_string(dataset_mem_).c_str()); - - auto mr = get_mr(dataset_mem_); - raft::neighbors::cagra::detail::copy_with_padding(handle_, *dataset_, *input_dataset_v_, mr); - - auto dataset_view = raft::make_device_strided_matrix_view( - dataset_->data_handle(), dataset_->extent(0), this->dim_, dataset_->extent(1)); - index_->update_dataset(handle_, dataset_view); - - need_dataset_update_ = false; - } -} - -template -void RaftCagra::set_search_dataset(const T* dataset, size_t nrow) -{ - using ds_idx_type = decltype(index_->data().n_rows()); - bool is_vpq = - dynamic_cast*>(&index_->data()) || - dynamic_cast*>(&index_->data()); - // It can happen that we are re-using a previous algo object which already has - // the dataset set. Check if we need update. - if (static_cast(input_dataset_v_->extent(0)) != nrow || - input_dataset_v_->data_handle() != dataset) { - *input_dataset_v_ = make_device_matrix_view(dataset, nrow, this->dim_); - need_dataset_update_ = !is_vpq; // ignore update if this is a VPQ dataset. - } -} - -template -void RaftCagra::save(const std::string& file) const -{ - raft::neighbors::cagra::serialize(handle_, file, *index_); -} - -template -void RaftCagra::save_to_hnswlib(const std::string& file) const -{ - raft::neighbors::cagra::serialize_to_hnswlib(handle_, file, *index_); -} - -template -void RaftCagra::load(const std::string& file) -{ - index_ = std::make_shared>( - std::move(raft::neighbors::cagra::deserialize(handle_, file))); -} - -template -std::unique_ptr> RaftCagra::copy() -{ - return std::make_unique>(*this); // use copy constructor -} - -template -void RaftCagra::search_base( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - static_assert(std::is_integral_v); - static_assert(std::is_integral_v); - - IdxT* neighbors_IdxT; - std::optional> neighbors_storage{std::nullopt}; - if constexpr (sizeof(IdxT) == sizeof(AnnBase::index_type)) { - neighbors_IdxT = reinterpret_cast(neighbors); - } else { - neighbors_storage.emplace(batch_size * k, resource::get_cuda_stream(handle_)); - neighbors_IdxT = neighbors_storage->data(); - } - - auto queries_view = - raft::make_device_matrix_view(queries, batch_size, dimension_); - auto neighbors_view = raft::make_device_matrix_view(neighbors_IdxT, batch_size, k); - auto distances_view = raft::make_device_matrix_view(distances, batch_size, k); - - raft::neighbors::cagra::search( - handle_, search_params_, *index_, queries_view, neighbors_view, distances_view); - - if constexpr (sizeof(IdxT) != sizeof(AnnBase::index_type)) { - raft::linalg::unaryOp(neighbors, - neighbors_IdxT, - batch_size * k, - raft::cast_op(), - raft::resource::get_cuda_stream(handle_)); - } -} - -template -void RaftCagra::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - auto k0 = static_cast(refine_ratio_ * k); - const bool disable_refinement = k0 <= static_cast(k); - const raft::resources& res = handle_; - - if (disable_refinement) { - search_base(queries, batch_size, k, neighbors, distances); - } else { - auto queries_v = - raft::make_device_matrix_view(queries, batch_size, dimension_); - auto candidate_ixs = - raft::make_device_matrix(res, batch_size, k0); - auto candidate_dists = - raft::make_device_matrix(res, batch_size, k0); - search_base( - queries, batch_size, k0, candidate_ixs.data_handle(), candidate_dists.data_handle()); - refine_helper( - res, *input_dataset_v_, queries_v, candidate_ixs, k, neighbors, distances, index_->metric()); - } -} -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_ivf_flat.cu b/cpp/bench/ann/src/raft/raft_ivf_flat.cu deleted file mode 100644 index bcd23723a4..0000000000 --- a/cpp/bench/ann/src/raft/raft_ivf_flat.cu +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "raft_ivf_flat_wrapper.h" - -namespace raft::bench::ann { -template class RaftIvfFlatGpu; -template class RaftIvfFlatGpu; -template class RaftIvfFlatGpu; -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_ivf_flat_wrapper.h b/cpp/bench/ann/src/raft/raft_ivf_flat_wrapper.h deleted file mode 100644 index 83a3a63aba..0000000000 --- a/cpp/bench/ann/src/raft/raft_ivf_flat_wrapper.h +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../common/ann_types.hpp" -#include "raft_ann_bench_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace raft::bench::ann { - -template -class RaftIvfFlatGpu : public ANN, public AnnGPU { - public: - using typename ANN::AnnSearchParam; - - struct SearchParam : public AnnSearchParam { - raft::neighbors::ivf_flat::search_params ivf_flat_params; - }; - - using BuildParam = raft::neighbors::ivf_flat::index_params; - - RaftIvfFlatGpu(Metric metric, int dim, const BuildParam& param) - : ANN(metric, dim), index_params_(param), dimension_(dim) - { - index_params_.metric = parse_metric_type(metric); - index_params_.conservative_memory_allocation = true; - RAFT_CUDA_TRY(cudaGetDevice(&device_)); - } - - void build(const T* dataset, size_t nrow) final; - - void set_search_param(const AnnSearchParam& param) override; - - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const override; - - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override - { - return handle_.get_sync_stream(); - } - - // to enable dataset access from GPU memory - AlgoProperty get_preference() const override - { - AlgoProperty property; - property.dataset_memory_type = MemoryType::HostMmap; - property.query_memory_type = MemoryType::Device; - return property; - } - void save(const std::string& file) const override; - void load(const std::string&) override; - std::unique_ptr> copy() override; - - private: - // handle_ must go first to make sure it dies last and all memory allocated in pool - configured_raft_resources handle_{}; - BuildParam index_params_; - raft::neighbors::ivf_flat::search_params search_params_; - std::shared_ptr> index_; - int device_; - int dimension_; -}; - -template -void RaftIvfFlatGpu::build(const T* dataset, size_t nrow) -{ - index_ = std::make_shared>(std::move( - raft::neighbors::ivf_flat::build(handle_, index_params_, dataset, IdxT(nrow), dimension_))); -} - -template -void RaftIvfFlatGpu::set_search_param(const AnnSearchParam& param) -{ - auto search_param = dynamic_cast(param); - search_params_ = search_param.ivf_flat_params; - assert(search_params_.n_probes <= index_params_.n_lists); -} - -template -void RaftIvfFlatGpu::save(const std::string& file) const -{ - raft::neighbors::ivf_flat::serialize(handle_, file, *index_); - return; -} - -template -void RaftIvfFlatGpu::load(const std::string& file) -{ - index_ = std::make_shared>( - std::move(raft::neighbors::ivf_flat::deserialize(handle_, file))); - return; -} - -template -std::unique_ptr> RaftIvfFlatGpu::copy() -{ - return std::make_unique>(*this); // use copy constructor -} - -template -void RaftIvfFlatGpu::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - static_assert(std::is_integral_v); - static_assert(std::is_integral_v); - - IdxT* neighbors_IdxT; - std::optional> neighbors_storage{std::nullopt}; - if constexpr (sizeof(IdxT) == sizeof(AnnBase::index_type)) { - neighbors_IdxT = reinterpret_cast(neighbors); - } else { - neighbors_storage.emplace(batch_size * k, resource::get_cuda_stream(handle_)); - neighbors_IdxT = neighbors_storage->data(); - } - raft::neighbors::ivf_flat::search(handle_, - search_params_, - *index_, - queries, - batch_size, - k, - neighbors_IdxT, - distances, - resource::get_workspace_resource(handle_)); - if constexpr (sizeof(IdxT) != sizeof(AnnBase::index_type)) { - raft::linalg::unaryOp(neighbors, - neighbors_IdxT, - batch_size * k, - raft::cast_op(), - raft::resource::get_cuda_stream(handle_)); - } -} -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_ivf_pq.cu b/cpp/bench/ann/src/raft/raft_ivf_pq.cu deleted file mode 100644 index d4f68c1c7d..0000000000 --- a/cpp/bench/ann/src/raft/raft_ivf_pq.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "raft_ivf_pq_wrapper.h" - -namespace raft::bench::ann { -template class RaftIvfPQ; -template class RaftIvfPQ; -template class RaftIvfPQ; -template class RaftIvfPQ; -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_ivf_pq_wrapper.h b/cpp/bench/ann/src/raft/raft_ivf_pq_wrapper.h deleted file mode 100644 index 7201467969..0000000000 --- a/cpp/bench/ann/src/raft/raft_ivf_pq_wrapper.h +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../common/ann_types.hpp" -#include "raft_ann_bench_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace raft::bench::ann { - -template -class RaftIvfPQ : public ANN, public AnnGPU { - public: - using typename ANN::AnnSearchParam; - using ANN::dim_; - - struct SearchParam : public AnnSearchParam { - raft::neighbors::ivf_pq::search_params pq_param; - float refine_ratio = 1.0f; - auto needs_dataset() const -> bool override { return refine_ratio > 1.0f; } - }; - - using BuildParam = raft::neighbors::ivf_pq::index_params; - - RaftIvfPQ(Metric metric, int dim, const BuildParam& param) - : ANN(metric, dim), index_params_(param), dimension_(dim) - { - index_params_.metric = parse_metric_type(metric); - } - - void build(const T* dataset, size_t nrow) final; - - void set_search_param(const AnnSearchParam& param) override; - void set_search_dataset(const T* dataset, size_t nrow) override; - - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const override; - void search_base(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const; - - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override - { - return handle_.get_sync_stream(); - } - - // to enable dataset access from GPU memory - AlgoProperty get_preference() const override - { - AlgoProperty property; - property.dataset_memory_type = MemoryType::Host; - property.query_memory_type = MemoryType::Device; - return property; - } - void save(const std::string& file) const override; - void load(const std::string&) override; - std::unique_ptr> copy() override; - - private: - // handle_ must go first to make sure it dies last and all memory allocated in pool - configured_raft_resources handle_{}; - BuildParam index_params_; - raft::neighbors::ivf_pq::search_params search_params_; - std::shared_ptr> index_; - int dimension_; - float refine_ratio_ = 1.0; - raft::device_matrix_view dataset_; -}; - -template -void RaftIvfPQ::save(const std::string& file) const -{ - raft::neighbors::ivf_pq::serialize(handle_, file, *index_); -} - -template -void RaftIvfPQ::load(const std::string& file) -{ - index_ = std::make_shared>( - std::move(raft::neighbors::ivf_pq::deserialize(handle_, file))); -} - -template -void RaftIvfPQ::build(const T* dataset, size_t nrow) -{ - auto dataset_v = raft::make_device_matrix_view(dataset, IdxT(nrow), dim_); - std::make_shared>( - std::move(raft::neighbors::ivf_pq::build(handle_, index_params_, dataset_v))) - .swap(index_); -} - -template -std::unique_ptr> RaftIvfPQ::copy() -{ - return std::make_unique>(*this); // use copy constructor -} - -template -void RaftIvfPQ::set_search_param(const AnnSearchParam& param) -{ - auto search_param = dynamic_cast(param); - search_params_ = search_param.pq_param; - refine_ratio_ = search_param.refine_ratio; - assert(search_params_.n_probes <= index_params_.n_lists); -} - -template -void RaftIvfPQ::set_search_dataset(const T* dataset, size_t nrow) -{ - dataset_ = raft::make_device_matrix_view(dataset, nrow, index_->dim()); -} - -template -void RaftIvfPQ::search_base( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - static_assert(std::is_integral_v); - static_assert(std::is_integral_v); - - IdxT* neighbors_IdxT; - std::optional> neighbors_storage{std::nullopt}; - if constexpr (sizeof(IdxT) == sizeof(AnnBase::index_type)) { - neighbors_IdxT = reinterpret_cast(neighbors); - } else { - neighbors_storage.emplace(batch_size * k, resource::get_cuda_stream(handle_)); - neighbors_IdxT = neighbors_storage->data(); - } - - auto queries_view = - raft::make_device_matrix_view(queries, batch_size, dimension_); - auto neighbors_view = - raft::make_device_matrix_view(neighbors_IdxT, batch_size, k); - auto distances_view = raft::make_device_matrix_view(distances, batch_size, k); - - raft::neighbors::ivf_pq::search( - handle_, search_params_, *index_, queries_view, neighbors_view, distances_view); - - if constexpr (sizeof(IdxT) != sizeof(AnnBase::index_type)) { - raft::linalg::unaryOp(neighbors, - neighbors_IdxT, - batch_size * k, - raft::cast_op(), - raft::resource::get_cuda_stream(handle_)); - } -} - -template -void RaftIvfPQ::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - auto k0 = static_cast(refine_ratio_ * k); - const bool disable_refinement = k0 <= static_cast(k); - const raft::resources& res = handle_; - - if (disable_refinement) { - search_base(queries, batch_size, k, neighbors, distances); - } else { - auto queries_v = - raft::make_device_matrix_view(queries, batch_size, dimension_); - auto candidate_ixs = - raft::make_device_matrix(res, batch_size, k0); - auto candidate_dists = - raft::make_device_matrix(res, batch_size, k0); - search_base( - queries, batch_size, k0, candidate_ixs.data_handle(), candidate_dists.data_handle()); - refine_helper( - res, dataset_, queries_v, candidate_ixs, k, neighbors, distances, index_->metric()); - } -} -} // namespace raft::bench::ann diff --git a/cpp/bench/ann/src/raft/raft_wrapper.h b/cpp/bench/ann/src/raft/raft_wrapper.h deleted file mode 100644 index 2c996058b2..0000000000 --- a/cpp/bench/ann/src/raft/raft_wrapper.h +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../common/ann_types.hpp" -#include "raft_ann_bench_utils.h" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace raft_temp { - -inline raft::distance::DistanceType parse_metric_type(raft::bench::ann::Metric metric) -{ - switch (metric) { - case raft::bench::ann::Metric::kInnerProduct: return raft::distance::DistanceType::InnerProduct; - case raft::bench::ann::Metric::kEuclidean: return raft::distance::DistanceType::L2Expanded; - default: throw std::runtime_error("raft supports only metric type of inner product and L2"); - } -} -} // namespace raft_temp - -namespace raft::bench::ann { - -// brute force KNN - RAFT -template -class RaftGpu : public ANN, public AnnGPU { - public: - using typename ANN::AnnSearchParam; - - RaftGpu(Metric metric, int dim); - - void build(const T*, size_t) final; - - void set_search_param(const AnnSearchParam& param) override; - - void search(const T* queries, - int batch_size, - int k, - AnnBase::index_type* neighbors, - float* distances) const final; - - // to enable dataset access from GPU memory - AlgoProperty get_preference() const override - { - AlgoProperty property; - property.dataset_memory_type = MemoryType::Device; - property.query_memory_type = MemoryType::Device; - return property; - } - [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override - { - return handle_.get_sync_stream(); - } - void set_search_dataset(const T* dataset, size_t nrow) override; - void save(const std::string& file) const override; - void load(const std::string&) override; - std::unique_ptr> copy() override; - - protected: - // handle_ must go first to make sure it dies last and all memory allocated in pool - configured_raft_resources handle_{}; - std::shared_ptr> index_; - raft::distance::DistanceType metric_type_; - int device_; - const T* dataset_; - size_t nrow_; -}; - -template -RaftGpu::RaftGpu(Metric metric, int dim) - : ANN(metric, dim), metric_type_(raft_temp::parse_metric_type(metric)) -{ - static_assert(std::is_same_v || std::is_same_v, - "raft bfknn only supports float/double"); - RAFT_CUDA_TRY(cudaGetDevice(&device_)); -} - -template -void RaftGpu::build(const T* dataset, size_t nrow) -{ - auto dataset_view = raft::make_host_matrix_view(dataset, nrow, this->dim_); - index_ = std::make_shared>( - std::move(raft::neighbors::brute_force::build(handle_, dataset_view))); -} - -template -void RaftGpu::set_search_param(const AnnSearchParam&) -{ - // Nothing to set here as it is brute force implementation -} - -template -void RaftGpu::set_search_dataset(const T* dataset, size_t nrow) -{ - dataset_ = dataset; - nrow_ = nrow; -} - -template -void RaftGpu::save(const std::string& file) const -{ - raft::neighbors::brute_force::serialize(handle_, file, *index_); -} - -template -void RaftGpu::load(const std::string& file) -{ - index_ = std::make_shared>( - std::move(raft::neighbors::brute_force::deserialize(handle_, file))); -} - -template -void RaftGpu::search( - const T* queries, int batch_size, int k, AnnBase::index_type* neighbors, float* distances) const -{ - auto queries_view = - raft::make_device_matrix_view(queries, batch_size, this->dim_); - - auto neighbors_view = - raft::make_device_matrix_view(neighbors, batch_size, k); - auto distances_view = raft::make_device_matrix_view(distances, batch_size, k); - - raft::neighbors::brute_force::search( - handle_, *index_, queries_view, neighbors_view, distances_view); -} - -template -std::unique_ptr> RaftGpu::copy() -{ - return std::make_unique>(*this); // use copy constructor -} - -} // namespace raft::bench::ann diff --git a/cpp/bench/prims/CMakeLists.txt b/cpp/bench/prims/CMakeLists.txt index 52c63ad73b..cf03a36612 100644 --- a/cpp/bench/prims/CMakeLists.txt +++ b/cpp/bench/prims/CMakeLists.txt @@ -74,49 +74,9 @@ function(ConfigureBench) endfunction() if(BUILD_PRIMS_BENCH) - ConfigureBench( - NAME - CORE_BENCH - PATH - core/bitset.cu - core/copy.cu - main.cpp - ) + ConfigureBench(NAME CORE_BENCH PATH core/bitset.cu core/copy.cu main.cpp) - ConfigureBench( - NAME - UTIL_BENCH - PATH - util/popc.cu - main.cpp - ) - - ConfigureBench( - NAME CLUSTER_BENCH PATH cluster/kmeans_balanced.cu cluster/kmeans.cu - main.cpp OPTIONAL LIB EXPLICIT_INSTANTIATE_ONLY - ) - - ConfigureBench( - NAME TUNE_DISTANCE PATH distance/tune_pairwise/kernel.cu - distance/tune_pairwise/bench.cu main.cpp - ) - - ConfigureBench( - NAME - DISTANCE_BENCH - PATH - distance/distance_cosine.cu - distance/distance_exp_l2.cu - distance/distance_l1.cu - distance/distance_unexp_l2.cu - distance/fused_l2_nn.cu - distance/masked_nn.cu - distance/kernels.cu - main.cpp - OPTIONAL - LIB - EXPLICIT_INSTANTIATE_ONLY - ) + ConfigureBench(NAME UTIL_BENCH PATH util/popc.cu main.cpp) ConfigureBench( NAME @@ -137,54 +97,18 @@ if(BUILD_PRIMS_BENCH) ) ConfigureBench( - NAME MATRIX_BENCH PATH matrix/argmin.cu matrix/gather.cu - matrix/select_k.cu main.cpp OPTIONAL LIB EXPLICIT_INSTANTIATE_ONLY + NAME MATRIX_BENCH PATH matrix/argmin.cu matrix/gather.cu matrix/select_k.cu main.cpp OPTIONAL + LIB EXPLICIT_INSTANTIATE_ONLY ) ConfigureBench( - NAME RANDOM_BENCH PATH random/make_blobs.cu random/permute.cu - random/rng.cu random/subsample.cu main.cpp - ) - - ConfigureBench( - NAME - SPARSE_BENCH - PATH - sparse/bitmap_to_csr.cu - sparse/convert_csr.cu - sparse/select_k_csr.cu + NAME RANDOM_BENCH PATH random/make_blobs.cu random/permute.cu random/rng.cu random/subsample.cu main.cpp ) ConfigureBench( - NAME - NEIGHBORS_BENCH - PATH - neighbors/knn/brute_force_float_int64_t.cu - neighbors/knn/brute_force_float_uint32_t.cu - neighbors/knn/cagra_float_uint32_t.cu - neighbors/knn/ivf_flat_filter_float_int64_t.cu - neighbors/knn/ivf_flat_float_int64_t.cu - neighbors/knn/ivf_flat_int8_t_int64_t.cu - neighbors/knn/ivf_flat_uint8_t_int64_t.cu - neighbors/knn/ivf_pq_float_int64_t.cu - neighbors/knn/ivf_pq_filter_float_int64_t.cu - neighbors/knn/ivf_pq_int8_t_int64_t.cu - neighbors/knn/ivf_pq_uint8_t_int64_t.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_search_filtering_float_int64_t.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset64.cu - neighbors/refine_float_int64_t.cu - neighbors/refine_uint8_t_int64_t.cu + NAME SPARSE_BENCH PATH sparse/bitmap_to_csr.cu sparse/convert_csr.cu sparse/select_k_csr.cu main.cpp - OPTIONAL - LIB - EXPLICIT_INSTANTIATE_ONLY ) endif() diff --git a/cpp/bench/prims/cluster/kmeans.cu b/cpp/bench/prims/cluster/kmeans.cu deleted file mode 100644 index 6387211135..0000000000 --- a/cpp/bench/prims/cluster/kmeans.cu +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include - -namespace raft::bench::cluster { - -struct KMeansBenchParams { - DatasetParams data; - BlobsParams blobs; - raft::cluster::KMeansParams kmeans; -}; - -inline auto operator<<(std::ostream& os, const KMeansBenchParams& p) -> std::ostream& -{ - os << p.data.rows << "#" << p.data.cols << "#" << p.kmeans.n_clusters; - return os; -} - -template -struct KMeans : public BlobsFixture { - KMeans(const KMeansBenchParams& p) - : BlobsFixture(p.data, p.blobs), - params(p), - centroids(this->handle), - labels(this->handle) - { - } - - void run_benchmark(::benchmark::State& state) override - { - std::ostringstream label_stream; - label_stream << params; - state.SetLabel(label_stream.str()); - - raft::device_matrix_view X_view = this->X.view(); - std::optional> opt_weights_view = std::nullopt; - std::optional> centroids_view = - std::make_optional>(centroids.view()); - raft::device_vector_view labels_view = labels.view(); - raft::host_scalar_view inertia_view = raft::make_host_scalar_view(&inertia); - raft::host_scalar_view n_iter_view = raft::make_host_scalar_view(&n_iter); - - this->loop_on_state(state, [&]() { - raft::cluster::kmeans_fit_predict(this->handle, - params.kmeans, - X_view, - opt_weights_view, - centroids_view, - labels_view, - inertia_view, - n_iter_view); - }); - } - - void allocate_temp_buffers(const ::benchmark::State& state) override - { - centroids = - raft::make_device_matrix(this->handle, params.kmeans.n_clusters, params.data.cols); - labels = raft::make_device_vector(this->handle, params.data.rows); - } - - private: - KMeansBenchParams params; - raft::device_matrix centroids; - raft::device_vector labels; - T inertia; - IndexT n_iter; -}; // struct KMeans - -std::vector getKMeansInputs() -{ - std::vector out; - KMeansBenchParams p; - p.data.row_major = true; - p.blobs.cluster_std = 1.0; - p.blobs.shuffle = false; - p.blobs.center_box_min = -10.0; - p.blobs.center_box_max = 10.0; - p.blobs.seed = 12345ULL; - p.kmeans.init = raft::cluster::KMeansParams::KMeansPlusPlus; - p.kmeans.max_iter = 300; - p.kmeans.tol = 1e-4; - p.kmeans.verbosity = RAFT_LEVEL_INFO; - p.kmeans.metric = raft::distance::DistanceType::L2Expanded; - p.kmeans.inertia_check = true; - std::vector> row_cols_k = { - {1000000, 20, 1000}, - {3000000, 50, 20}, - {10000000, 50, 5}, - }; - for (auto& rck : row_cols_k) { - p.data.rows = std::get<0>(rck); - p.data.cols = std::get<1>(rck); - p.blobs.n_clusters = std::get<2>(rck); - p.kmeans.n_clusters = std::get<2>(rck); - out.push_back(p); - } - return out; -} - -// note(lsugy): commenting out int64_t because the templates are not compiled in the distance -// library, resulting in long compilation times. -RAFT_BENCH_REGISTER((KMeans), "", getKMeansInputs()); -RAFT_BENCH_REGISTER((KMeans), "", getKMeansInputs()); -// RAFT_BENCH_REGISTER((KMeans), "", getKMeansInputs()); -// RAFT_BENCH_REGISTER((KMeans), "", getKMeansInputs()); - -} // namespace raft::bench::cluster diff --git a/cpp/bench/prims/cluster/kmeans_balanced.cu b/cpp/bench/prims/cluster/kmeans_balanced.cu deleted file mode 100644 index dc05783989..0000000000 --- a/cpp/bench/prims/cluster/kmeans_balanced.cu +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include - -namespace raft::bench::cluster { - -struct KMeansBalancedBenchParams { - DatasetParams data; - uint32_t n_lists; - raft::cluster::kmeans_balanced_params kb_params; -}; - -template -struct KMeansBalanced : public fixture { - KMeansBalanced(const KMeansBalancedBenchParams& p) : params(p), X(handle), centroids(handle) {} - - void run_benchmark(::benchmark::State& state) override - { - this->loop_on_state(state, [this]() { - raft::device_matrix_view X_view = this->X.view(); - raft::device_matrix_view centroids_view = this->centroids.view(); - raft::cluster::kmeans_balanced::fit( - this->handle, this->params.kb_params, X_view, centroids_view); - }); - } - - void allocate_data(const ::benchmark::State& state) override - { - X = raft::make_device_matrix(handle, params.data.rows, params.data.cols); - - raft::random::RngState rng{1234}; - constexpr T kRangeMax = std::is_integral_v ? std::numeric_limits::max() : T(1); - constexpr T kRangeMin = std::is_integral_v ? std::numeric_limits::min() : T(-1); - if constexpr (std::is_integral_v) { - raft::random::uniformInt( - handle, rng, X.data_handle(), params.data.rows * params.data.cols, kRangeMin, kRangeMax); - } else { - raft::random::uniform( - handle, rng, X.data_handle(), params.data.rows * params.data.cols, kRangeMin, kRangeMax); - } - resource::sync_stream(handle, stream); - } - - void allocate_temp_buffers(const ::benchmark::State& state) override - { - centroids = - raft::make_device_matrix(this->handle, params.n_lists, params.data.cols); - } - - private: - KMeansBalancedBenchParams params; - raft::device_matrix X; - raft::device_matrix centroids; -}; // struct KMeansBalanced - -std::vector getKMeansBalancedInputs() -{ - std::vector out; - KMeansBalancedBenchParams p; - p.data.row_major = true; - p.kb_params.n_iters = 20; - p.kb_params.metric = raft::distance::DistanceType::L2Expanded; - std::vector> row_cols = { - {100000, 128}, {1000000, 128}, {10000000, 128}, - // The following dataset sizes are too large for most GPUs. - // {100000000, 128}, - }; - for (auto& rc : row_cols) { - p.data.rows = rc.first; - p.data.cols = rc.second; - for (auto n_lists : std::vector({1000, 10000, 100000})) { - p.n_lists = n_lists; - out.push_back(p); - } - } - return out; -} - -// Note: the datasets sizes are too large for 32-bit index types. -RAFT_BENCH_REGISTER((KMeansBalanced), "", getKMeansBalancedInputs()); - -} // namespace raft::bench::cluster diff --git a/cpp/bench/prims/distance/distance_common.cuh b/cpp/bench/prims/distance/distance_common.cuh deleted file mode 100644 index 8368062168..0000000000 --- a/cpp/bench/prims/distance/distance_common.cuh +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include - -#include - -namespace raft::bench::distance { - -struct distance_params { - int m, n, k; - bool isRowMajor; -}; // struct distance_params - -template -struct distance : public fixture { - distance(const distance_params& p) - : params(p), - x(p.m * p.k, stream), - y(p.n * p.k, stream), - out(p.m * p.n, stream), - workspace(0, stream) - { - RAFT_CUDA_TRY(cudaMemsetAsync(x.data(), 0, x.size() * sizeof(T), stream)); - RAFT_CUDA_TRY(cudaMemsetAsync(y.data(), 0, y.size() * sizeof(T), stream)); - RAFT_CUDA_TRY(cudaMemsetAsync(out.data(), 0, out.size() * sizeof(T), stream)); - worksize = raft::distance::getWorkspaceSize( - x.data(), y.data(), params.m, params.n, params.k); - workspace.resize(worksize, stream); - } - - void run_benchmark(::benchmark::State& state) override - { - loop_on_state(state, [this]() { - raft::distance::distance(handle, - x.data(), - y.data(), - out.data(), - params.m, - params.n, - params.k, - (void*)workspace.data(), - worksize, - params.isRowMajor); - }); - } - - private: - distance_params params; - rmm::device_uvector x, y, out; - rmm::device_uvector workspace; - size_t worksize; -}; // struct Distance - -const std::vector dist_input_vecs{ - {32, 16384, 16384, true}, {64, 16384, 16384, true}, {128, 16384, 16384, true}, - {256, 16384, 16384, true}, {512, 16384, 16384, true}, {1024, 16384, 16384, true}, - {16384, 32, 16384, true}, {16384, 64, 16384, true}, {16384, 128, 16384, true}, - {16384, 256, 16384, true}, {16384, 512, 16384, true}, {16384, 1024, 16384, true}, - {16384, 16384, 32, true}, {16384, 16384, 64, true}, {16384, 16384, 128, true}, - {16384, 16384, 256, true}, {16384, 16384, 512, true}, {16384, 16384, 1024, true}, - {16384, 16384, 16384, true}, {32, 16384, 16384, false}, {64, 16384, 16384, false}, - {128, 16384, 16384, false}, {256, 16384, 16384, false}, {512, 16384, 16384, false}, - {1024, 16384, 16384, false}, {16384, 32, 16384, false}, {16384, 64, 16384, false}, - {16384, 128, 16384, false}, {16384, 256, 16384, false}, {16384, 512, 16384, false}, - {16384, 1024, 16384, false}, {16384, 16384, 32, false}, {16384, 16384, 64, false}, - {16384, 16384, 128, false}, {16384, 16384, 256, false}, {16384, 16384, 512, false}, - {16384, 16384, 1024, false}, {16384, 16384, 16384, false} - -}; - -#define DIST_BENCH_REGISTER(Name, Metric) \ - using Name##F = distance; \ - RAFT_BENCH_REGISTER(Name##F, "", dist_input_vecs); \ - using Name##D = distance; \ - RAFT_BENCH_REGISTER(Name##D, "", dist_input_vecs); - -} // namespace raft::bench::distance diff --git a/cpp/bench/prims/distance/distance_cosine.cu b/cpp/bench/prims/distance/distance_cosine.cu deleted file mode 100644 index c8ac8067c8..0000000000 --- a/cpp/bench/prims/distance/distance_cosine.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "distance_common.cuh" - -namespace raft::bench::distance { - -DIST_BENCH_REGISTER(DistanceCosine, raft::distance::DistanceType::CosineExpanded); - -} // namespace raft::bench::distance diff --git a/cpp/bench/prims/distance/distance_exp_l2.cu b/cpp/bench/prims/distance/distance_exp_l2.cu deleted file mode 100644 index 52b7fff05c..0000000000 --- a/cpp/bench/prims/distance/distance_exp_l2.cu +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "distance_common.cuh" - -namespace raft::bench::distance { - -DIST_BENCH_REGISTER(DistanceL2Sq, raft::distance::DistanceType::L2Expanded); -DIST_BENCH_REGISTER(DistanceL2Sqrt, raft::distance::DistanceType::L2SqrtExpanded); - -} // namespace raft::bench::distance diff --git a/cpp/bench/prims/distance/distance_l1.cu b/cpp/bench/prims/distance/distance_l1.cu deleted file mode 100644 index e80db48ef0..0000000000 --- a/cpp/bench/prims/distance/distance_l1.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "distance_common.cuh" - -namespace raft::bench::distance { - -DIST_BENCH_REGISTER(DistanceL1, raft::distance::DistanceType::L1); - -} // namespace raft::bench::distance diff --git a/cpp/bench/prims/distance/distance_unexp_l2.cu b/cpp/bench/prims/distance/distance_unexp_l2.cu deleted file mode 100644 index 7ac1a8a4b5..0000000000 --- a/cpp/bench/prims/distance/distance_unexp_l2.cu +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "distance_common.cuh" - -namespace raft::bench::distance { - -DIST_BENCH_REGISTER(DistanceUnexpL2Sq, raft::distance::DistanceType::L2Unexpanded); -DIST_BENCH_REGISTER(DistanceUnexpL2Sqrt, raft::distance::DistanceType::L2SqrtUnexpanded); - -} // namespace raft::bench::distance diff --git a/cpp/bench/prims/distance/fused_l2_nn.cu b/cpp/bench/prims/distance/fused_l2_nn.cu deleted file mode 100644 index a263bef6ba..0000000000 --- a/cpp/bench/prims/distance/fused_l2_nn.cu +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include - -#include - -namespace raft::bench::distance { - -struct fusedl2nn_inputs { - int64_t m, n, k; -}; // struct fusedl2nn_inputs - -inline auto operator<<(std::ostream& os, const fusedl2nn_inputs& p) -> std::ostream& -{ - os << p.m << "#" << p.n << "#" << p.k; - return os; -} - -template -struct fusedl2nn : public fixture { - fusedl2nn(const fusedl2nn_inputs& p) - : params(p), - workspace(this->handle), - x(this->handle), - y(this->handle), - x_norm(this->handle), - y_norm(this->handle), - out(this->handle) - { - } - - void allocate_data(const ::benchmark::State& state) override - { - x = raft::make_device_matrix(handle, params.m, params.k); - y = raft::make_device_matrix(handle, params.n, params.k); - x_norm = raft::make_device_vector(handle, params.m); - y_norm = raft::make_device_vector(handle, params.n); - out = raft::make_device_vector(handle, params.m); - - raft::random::RngState rng{1234}; - raft::random::uniform( - handle, rng, x.data_handle(), params.m * params.k, (DataT)-1.0, (DataT)1.0); - raft::random::uniform( - handle, rng, y.data_handle(), params.n * params.k, (DataT)-1.0, (DataT)1.0); - - // Pre-compute norms - raft::linalg::rowNorm(x_norm.data_handle(), - x.data_handle(), - params.k, - params.m, - raft::linalg::L2Norm, - true, - stream); - raft::linalg::rowNorm(y_norm.data_handle(), - y.data_handle(), - params.k, - params.n, - raft::linalg::L2Norm, - true, - stream); - resource::sync_stream(handle, stream); - } - - void allocate_temp_buffers(const ::benchmark::State& state) override - { - workspace = raft::make_device_vector(handle, params.m * sizeof(IdxT)); - } - - void run_benchmark(::benchmark::State& state) override - { - std::ostringstream label_stream; - label_stream << params; - state.SetLabel(label_stream.str()); - - loop_on_state(state, [this]() { - raft::distance::fusedL2NNMinReduce(out.data_handle(), - x.data_handle(), - y.data_handle(), - x_norm.data_handle(), - y_norm.data_handle(), - static_cast(params.m), - static_cast(params.n), - static_cast(params.k), - (void*)workspace.data_handle(), - false, - true, - stream); - }); - - int64_t num_flops = 2 * params.m * params.n * params.k; - - int64_t read_elts = params.n * params.k + params.m * params.k; - int64_t write_elts = params.m; - - state.counters["FLOP/s"] = benchmark::Counter( - num_flops, benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1000); - - state.counters["BW Wr"] = benchmark::Counter(write_elts * sizeof(OutT), - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - state.counters["BW Rd"] = benchmark::Counter(read_elts * sizeof(DataT), - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - } - - private: - fusedl2nn_inputs params; - raft::device_matrix x, y; - raft::device_vector x_norm, y_norm; - raft::device_vector out; - raft::device_vector workspace; -}; // struct fusedl2nn - -template -std::vector getFusedL2NNInputs() -{ - std::vector inputs; - std::vector m_list = {100000, 1000000}; - if constexpr (sizeof(IdxT) == 8) { m_list.push_back(10000000); } - std::vector n_list = {100, 1000, 10000}; - std::vector k_list = {64, 128, 256}; - for (auto m : m_list) { - for (auto n : n_list) { - for (auto k : k_list) { - inputs.push_back({m, n, k}); - } - } - } - return inputs; -} - -#define FUSEDL2NN_BENCH(DataT, IdxT, OutT) \ - RAFT_BENCH_REGISTER((fusedl2nn), "", getFusedL2NNInputs()) - -FUSEDL2NN_BENCH(float, int, float); -FUSEDL2NN_BENCH(double, int, double); -FUSEDL2NN_BENCH(float, int, (raft::KeyValuePair)); -FUSEDL2NN_BENCH(double, int, (raft::KeyValuePair)); -FUSEDL2NN_BENCH(float, int64_t, float); -FUSEDL2NN_BENCH(double, int64_t, double); -FUSEDL2NN_BENCH(float, int64_t, (raft::KeyValuePair)); -FUSEDL2NN_BENCH(double, int64_t, (raft::KeyValuePair)); - -} // namespace raft::bench::distance diff --git a/cpp/bench/prims/distance/kernels.cu b/cpp/bench/prims/distance/kernels.cu deleted file mode 100644 index eb86330637..0000000000 --- a/cpp/bench/prims/distance/kernels.cu +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) 2019-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace raft::bench::distance::kernels { - -using namespace raft::distance::kernels; -struct GramTestParams { - int m; // m parameter of the GEMM - int k; // k parameter of the GEMM - int n; // n parameter of the GEMM - KernelParams kernel_params; - bool is_row_major; -}; // struct GramTestParams - -template -struct GramMatrix : public fixture { - GramMatrix(const GramTestParams& p) - : params(p), handle(stream), A(0, stream), B(0, stream), C(0, stream) - { - kernel = std::unique_ptr>( - KernelFactory::create(p.kernel_params, resource::get_cublas_handle(handle))); - - A.resize(params.m * params.k, stream); - B.resize(params.k * params.n, stream); - C.resize(params.m * params.n, stream); - raft::random::RngState rng(123456ULL); - raft::random::uniform(handle, rng, A.data(), params.m * params.k, T(-1.0), T(1.0)); - raft::random::uniform(handle, rng, B.data(), params.k * params.n, T(-1.0), T(1.0)); - } - - ~GramMatrix() - { - A.release(); - B.release(); - C.release(); - } - - void run_benchmark(::benchmark::State& state) override - { - if (!this->kernel) { state.SkipWithError("Kernel matrix is not initialized"); } - loop_on_state(state, [this]() { - (*this->kernel)(A.data(), - this->params.m, - this->params.k, - B.data(), - this->params.n, - C.data(), - this->params.is_row_major, - this->stream); - }); - } - - private: - const raft::device_resources handle; - std::unique_ptr> kernel; - GramTestParams params; - - rmm::device_uvector A; // input matrix A, size [m * k] - rmm::device_uvector B; // input matrix B, size [n * k] - rmm::device_uvector C; // output matrix C, size [m*n] -}; - -static std::vector getInputs() -{ - std::vector param_vec; - std::vector kernel_params{KernelParams{LINEAR, 3, 1, 0}, - KernelParams{POLYNOMIAL, 2, 1.3, 1}, - KernelParams{TANH, 2, 0.5, 2.4}, - KernelParams{RBF, 2, 0.5, 0}}; - struct TestSize { - int m; - int k; - int n; - }; - std::vector data_size{{4096, 10, 1024}, - {4096, 100, 1024}, - {4096, 1000, 1024}, - {4096, 10000, 1024}, - {100000, 10, 1024}, - {100000, 100, 1024}, - {100000, 1000, 1024}}; - - param_vec.reserve(kernel_params.size() * data_size.size()); - for (TestSize s : data_size) { - for (auto kernel : kernel_params) { - for (bool row_major : {false, true}) { - param_vec.push_back(GramTestParams{s.m, s.k, s.n, kernel, row_major}); - } - } - } - return param_vec; -} - -RAFT_BENCH_REGISTER(GramMatrix, "", getInputs()); -RAFT_BENCH_REGISTER(GramMatrix, "", getInputs()); - -} // namespace raft::bench::distance::kernels diff --git a/cpp/bench/prims/distance/masked_nn.cu b/cpp/bench/prims/distance/masked_nn.cu deleted file mode 100644 index 979d438b67..0000000000 --- a/cpp/bench/prims/distance/masked_nn.cu +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace raft::bench::distance::masked_nn { - -// Introduce various sparsity patterns -enum AdjacencyPattern { - checkerboard = 0, - checkerboard_4 = 1, - checkerboard_64 = 2, - all_true = 3, - all_false = 4 -}; - -struct Params { - int m, n, k, num_groups; - AdjacencyPattern pattern; -}; // struct Params - -RAFT_KERNEL init_adj(AdjacencyPattern pattern, - int n, - raft::device_matrix_view adj, - raft::device_vector_view group_idxs) -{ - int m = adj.extent(0); - int num_groups = adj.extent(1); - - for (int idx_m = blockIdx.y * blockDim.y + threadIdx.y; idx_m < m; - idx_m += blockDim.y * gridDim.y) { - for (int idx_g = blockIdx.x * blockDim.x + threadIdx.x; idx_g < num_groups; - idx_g += blockDim.x * gridDim.x) { - switch (pattern) { - case checkerboard: adj(idx_m, idx_g) = (idx_m + idx_g) % 2; break; - case checkerboard_4: adj(idx_m, idx_g) = (idx_m / 4 + idx_g) % 2; break; - case checkerboard_64: adj(idx_m, idx_g) = (idx_m / 64 + idx_g) % 2; break; - case all_true: adj(idx_m, idx_g) = true; break; - case all_false: adj(idx_m, idx_g) = false; break; - default: assert(false && "unknown pattern"); - } - } - } - // Each group is of size n / num_groups. - // - // - group_idxs[j] indicates the start of group j + 1 (i.e. is the inclusive - // scan of the group lengths) - // - // - The first group always starts at index zero, so we do not store it. - // - // - The group_idxs[num_groups - 1] should always equal n. - - if (blockIdx.y == 0 && threadIdx.y == 0) { - const int g_stride = blockDim.x * gridDim.x; - for (int idx_g = blockIdx.x * blockDim.x + threadIdx.x; idx_g < num_groups; idx_g += g_stride) { - group_idxs(idx_g) = (idx_g + 1) * (n / num_groups); - } - group_idxs(num_groups - 1) = n; - } -} - -template -struct masked_l2_nn : public fixture { - using DataT = T; - using IdxT = int; - using OutT = raft::KeyValuePair; - using RedOpT = raft::distance::MinAndDistanceReduceOp; - using PairRedOpT = raft::distance::KVPMinReduce; - using ParamT = raft::distance::masked_l2_nn_params; - - // Parameters - Params params; - // Data - raft::device_vector out; - raft::device_matrix x, y; - raft::device_vector xn, yn; - raft::device_matrix adj; - raft::device_vector group_idxs; - - masked_l2_nn(const Params& p) - : params(p), - out{raft::make_device_vector(handle, p.m)}, - x{raft::make_device_matrix(handle, p.m, p.k)}, - y{raft::make_device_matrix(handle, p.n, p.k)}, - xn{raft::make_device_vector(handle, p.m)}, - yn{raft::make_device_vector(handle, p.n)}, - adj{raft::make_device_matrix(handle, p.m, p.num_groups)}, - group_idxs{raft::make_device_vector(handle, p.num_groups)} - { - raft::random::RngState r(123456ULL); - - uniform(handle, r, x.data_handle(), p.m * p.k, T(-1.0), T(1.0)); - uniform(handle, r, y.data_handle(), p.n * p.k, T(-1.0), T(1.0)); - raft::linalg::rowNorm( - xn.data_handle(), x.data_handle(), p.k, p.m, raft::linalg::L2Norm, true, stream); - raft::linalg::rowNorm( - yn.data_handle(), y.data_handle(), p.k, p.n, raft::linalg::L2Norm, true, stream); - raft::distance::initialize, int>( - handle, out.data_handle(), p.m, std::numeric_limits::max(), RedOpT{}); - - dim3 block(32, 32); - dim3 grid(10, 10); - init_adj<<>>(p.pattern, p.n, adj.view(), group_idxs.view()); - RAFT_CUDA_TRY(cudaGetLastError()); - } - - void run_benchmark(::benchmark::State& state) override - { - bool init_out = true; - bool sqrt = false; - ParamT masked_l2_params{RedOpT{}, PairRedOpT{}, sqrt, init_out}; - - loop_on_state(state, [this, masked_l2_params]() { - // It is sufficient to only benchmark the L2-squared metric - raft::distance::masked_l2_nn(handle, - masked_l2_params, - x.view(), - y.view(), - xn.view(), - yn.view(), - adj.view(), - group_idxs.view(), - out.view()); - }); - - // Virtual flop count if no skipping had occurred. - size_t virtual_flops = size_t(2) * size_t(params.m) * size_t(params.n) * size_t(params.k); - - int64_t read_elts = params.n * params.k + params.m * params.k; - int64_t write_elts = params.m; - - // Virtual min flops is the number of flops that would have been executed if - // the algorithm had actually skipped each computation that it could have - // skipped. - size_t virtual_min_flops = 0; - switch (params.pattern) { - case checkerboard: - case checkerboard_4: - case checkerboard_64: virtual_min_flops = virtual_flops / 2; break; - case all_true: virtual_min_flops = virtual_flops; break; - case all_false: virtual_min_flops = 0; break; - default: assert(false && "unknown pattern"); - } - - // VFLOP/s is the "virtual" flop count that would have executed if there was - // no adjacency pattern. This is useful for comparing to fusedL2NN - state.counters["VFLOP/s"] = benchmark::Counter(virtual_flops, - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - // Virtual min flops is the number of flops that would have been executed if - // the algorithm had actually skipped each computation that it could have - // skipped. - state.counters["VminFLOP/s"] = benchmark::Counter(virtual_min_flops, - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - - state.counters["BW Wr"] = benchmark::Counter(write_elts * sizeof(OutT), - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - state.counters["BW Rd"] = benchmark::Counter(read_elts * sizeof(DataT), - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - - state.counters["m"] = benchmark::Counter(params.m); - state.counters["n"] = benchmark::Counter(params.n); - state.counters["k"] = benchmark::Counter(params.k); - state.counters["num_groups"] = benchmark::Counter(params.num_groups); - state.counters["group size"] = benchmark::Counter(params.n / params.num_groups); - state.counters["Pat"] = benchmark::Counter(static_cast(params.pattern)); - - state.counters["SM count"] = raft::getMultiProcessorCount(); - } -}; - -const std::vector masked_l2_nn_input_vecs = { - // Very fat matrices... - {32, 16384, 16384, 32, AdjacencyPattern::checkerboard}, - {64, 16384, 16384, 32, AdjacencyPattern::checkerboard}, - {128, 16384, 16384, 32, AdjacencyPattern::checkerboard}, - {256, 16384, 16384, 32, AdjacencyPattern::checkerboard}, - {512, 16384, 16384, 32, AdjacencyPattern::checkerboard}, - {1024, 16384, 16384, 32, AdjacencyPattern::checkerboard}, - {16384, 32, 16384, 32, AdjacencyPattern::checkerboard}, - {16384, 64, 16384, 32, AdjacencyPattern::checkerboard}, - {16384, 128, 16384, 32, AdjacencyPattern::checkerboard}, - {16384, 256, 16384, 32, AdjacencyPattern::checkerboard}, - {16384, 512, 16384, 32, AdjacencyPattern::checkerboard}, - {16384, 1024, 16384, 32, AdjacencyPattern::checkerboard}, - - // Representative matrices... - {16384, 16384, 32, 32, AdjacencyPattern::checkerboard}, - {16384, 16384, 64, 32, AdjacencyPattern::checkerboard}, - {16384, 16384, 128, 32, AdjacencyPattern::checkerboard}, - {16384, 16384, 256, 32, AdjacencyPattern::checkerboard}, - {16384, 16384, 512, 32, AdjacencyPattern::checkerboard}, - {16384, 16384, 1024, 32, AdjacencyPattern::checkerboard}, - {16384, 16384, 16384, 32, AdjacencyPattern::checkerboard}, - - {16384, 16384, 32, 32, AdjacencyPattern::checkerboard_4}, - {16384, 16384, 64, 32, AdjacencyPattern::checkerboard_4}, - {16384, 16384, 128, 32, AdjacencyPattern::checkerboard_4}, - {16384, 16384, 256, 32, AdjacencyPattern::checkerboard_4}, - {16384, 16384, 512, 32, AdjacencyPattern::checkerboard_4}, - {16384, 16384, 1024, 32, AdjacencyPattern::checkerboard_4}, - {16384, 16384, 16384, 32, AdjacencyPattern::checkerboard_4}, - - {16384, 16384, 32, 32, AdjacencyPattern::checkerboard_64}, - {16384, 16384, 64, 32, AdjacencyPattern::checkerboard_64}, - {16384, 16384, 128, 32, AdjacencyPattern::checkerboard_64}, - {16384, 16384, 256, 32, AdjacencyPattern::checkerboard_64}, - {16384, 16384, 512, 32, AdjacencyPattern::checkerboard_64}, - {16384, 16384, 1024, 32, AdjacencyPattern::checkerboard_64}, - {16384, 16384, 16384, 32, AdjacencyPattern::checkerboard_64}, - - {16384, 16384, 32, 32, AdjacencyPattern::all_true}, - {16384, 16384, 64, 32, AdjacencyPattern::all_true}, - {16384, 16384, 128, 32, AdjacencyPattern::all_true}, - {16384, 16384, 256, 32, AdjacencyPattern::all_true}, - {16384, 16384, 512, 32, AdjacencyPattern::all_true}, - {16384, 16384, 1024, 32, AdjacencyPattern::all_true}, - {16384, 16384, 16384, 32, AdjacencyPattern::all_true}, - - {16384, 16384, 32, 32, AdjacencyPattern::all_false}, - {16384, 16384, 64, 32, AdjacencyPattern::all_false}, - {16384, 16384, 128, 32, AdjacencyPattern::all_false}, - {16384, 16384, 256, 32, AdjacencyPattern::all_false}, - {16384, 16384, 512, 32, AdjacencyPattern::all_false}, - {16384, 16384, 1024, 32, AdjacencyPattern::all_false}, - {16384, 16384, 16384, 32, AdjacencyPattern::all_false}, -}; - -RAFT_BENCH_REGISTER(masked_l2_nn, "", masked_l2_nn_input_vecs); -// We don't benchmark double to keep compile times in check when not using the -// distance library. - -} // namespace raft::bench::distance::masked_nn diff --git a/cpp/bench/prims/distance/tune_pairwise/bench.cu b/cpp/bench/prims/distance/tune_pairwise/bench.cu deleted file mode 100644 index 81105cdefe..0000000000 --- a/cpp/bench/prims/distance/tune_pairwise/bench.cu +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Tuning benchmarks. -// -// Goals: -// -// 1. Fast compile times to maintain iteration speed. -// 2. Create benchmarks that can inform the design of the kernels. -// -// Non-goals: -// -// 1. Measure every distance operation. Instead measures just one distance -// operation at the same time. -// 2. Be useful for finding performance regressions. This is handled by the -// normal benchmarks. -// -// So far, both goals are partly achieved. -// -// RE (1), COMPILE TIMES: kernel.cu is fast to compile. This file is not. -// When the internals of a pairwise distance kernel is changed, this file is not -// recompiled. -// -// RE 2, benchmarks with intent: this file contains a benchmark to check the -// maximal throughput of a kernel. Measuring other things, like performance on -// skinny or wide matrices is not yet implemented. - -#include "kernel.cuh" // launch_kernel - -#include // RAFT_BENCH_REGISTER - -#include // pairwise_matrix_params - -#include // rmm::device_uvector - -#include // std::min -#include // std::vector - -namespace raft::bench::distance::tune { - -// Max throughput benchmark. -// -// Goal: Measure the maximum distances/sec that can be computed. -// -// To achieve this, we make sure that: -// -// - Input data size is a multiple of the block tile size. -// -// - Perfect distribution of work between SMs, i.e. the number of block tiles is -// a large multiple (num_waves) of the number of blocks (#SMs * occupancy). -// -// - Multiple iterations over Kblk are executed (num_k_iters). -struct throughput_param { - int num_waves; - int occupancy; - int num_k_iters; -}; - -const std::vector throughput_params{ - // 32 waves, requested occupancy of 4, and 32 k iterations typically achieves - // maximum throughput. No need to pick higher values. - {32, 4, 32}, -}; - -struct throughput_bench : public fixture { - const throughput_param p; - - throughput_bench(const throughput_param& p_) : p(p_) {} - - void run_benchmark(::benchmark::State& state) override - { - // Get block size: - int block_m, block_n, block_k; - get_block_size(block_m, block_n, block_k); - - // Determine number of blocks that will be launched. This informs the size - // of the inputs as well as the grid size. - const int num_sms = raft::getMultiProcessorCount(); - const int max_occupancy = get_max_occupancy(); - const int occupancy = std::min(p.occupancy, max_occupancy); - const int num_blocks = occupancy * num_sms; - dim3 grid(num_blocks); - - // Create input sizes that are a multiple of the block tile size. - size_t m = block_m; - size_t n = block_n * p.num_waves * num_blocks; - size_t k = block_k * p.num_k_iters; - - // DataT, OutT, IdxT, etc, are defined in tuned_kernel.cuh - rmm::device_uvector x_vec(m * k, stream); - rmm::device_uvector y_vec(n * k, stream); - rmm::device_uvector x_norm_vec(m, stream); - rmm::device_uvector y_norm_vec(n, stream); - rmm::device_uvector out_vec(m * n, stream); - - auto x = x_vec.data(); - auto y = y_vec.data(); - auto x_norm = x_norm_vec.data(); - auto y_norm = y_norm_vec.data(); - auto out = out_vec.data(); - FinOpT fin_op{}; - - // Create kernel parameter struct. Flip x and y if column major. - IdxT ldx = row_major ? k : m; - IdxT ldy = row_major ? k : n; - IdxT ld_out = row_major ? n : m; - - // Template parameters of pairwise_matrix_params are defined in kernel.cuh - pairwise_matrix_params kparams{ - IdxT(m), IdxT(n), IdxT(k), ldx, ldy, ld_out, x, y, x_norm, y_norm, out, fin_op, row_major}; - - // Run benchmark - loop_on_state(state, [&]() { launch_kernel(kparams, grid, stream); }); - - // Report metrics. We don't report flop/s because we do not know for each - // distance operation how many flops it costs. For L2_unexp and l1, we can - // double this number to get the flop/s. For l2 expanded, core_ops/s should - // equal flop/s (modulo the sqrt and subtracting from the norm). - size_t num_core_ops = m * n * k; - size_t read_elts = n * k + m * k; - size_t write_elts = m * n; - - state.counters["m"] = benchmark::Counter(m); - state.counters["n"] = benchmark::Counter(n); - state.counters["k"] = benchmark::Counter(k); - state.counters["occupancy"] = benchmark::Counter(occupancy); - state.counters["# waves"] = benchmark::Counter(p.num_waves); - state.counters["# k iters"] = benchmark::Counter(p.num_k_iters); - - state.counters["core_ops/s"] = benchmark::Counter(num_core_ops, - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - - state.counters["BW"] = benchmark::Counter(write_elts * sizeof(OutT) + read_elts * sizeof(DataT), - benchmark::Counter::kIsIterationInvariantRate, - benchmark::Counter::OneK::kIs1000); - } -}; - -RAFT_BENCH_REGISTER(throughput_bench, "", throughput_params); - -} // namespace raft::bench::distance::tune diff --git a/cpp/bench/prims/distance/tune_pairwise/kernel.cu b/cpp/bench/prims/distance/tune_pairwise/kernel.cu deleted file mode 100644 index 42173c51f5..0000000000 --- a/cpp/bench/prims/distance/tune_pairwise/kernel.cu +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "kernel.cuh" - -#include // pairwise_matrix_sm60_wrapper -#include // raft::linalg::Policy4x4 -#include // raft::util::arch::SM_compute_arch - -namespace raft::bench::distance::tune { - -// Distance op -using OpT = raft::distance::detail::ops::lp_unexp_distance_op; -constexpr float metric_arg = 2.0; -OpT distance_op{metric_arg}; - -// Kernel policy -constexpr int vec_len = 1; -using Policy = typename raft::linalg::Policy4x4::Policy; - -// Architecture -namespace arch = raft::util::arch; -constexpr auto sm_compat_range = arch::SM_range(arch::SM_min(), arch::SM_future()); - -void launch_kernel(pairwise_matrix_params params, dim3 grid, cudaStream_t stream) -{ - dim3 block(Policy::Nthreads); - int smem_size = OpT::shared_mem_size(); - - // Obtain function pointer to kernel - auto kernel = raft::distance::detail::pairwise_matrix_kernel; - - kernel<<>>(distance_op, params); - RAFT_CUDA_TRY(cudaGetLastError()); -} - -void get_block_size(int& m, int& n, int& k) -{ - m = Policy::Mblk; - n = Policy::Nblk; - k = Policy::Kblk; -} - -void* get_kernel_ptr() -{ - auto kernel = raft::distance::detail::pairwise_matrix_kernel; - return reinterpret_cast(kernel); -} - -int get_max_occupancy() -{ - void* kernel_ptr = get_kernel_ptr(); - int max_occupancy; - int smem_size = OpT::shared_mem_size(); - - RAFT_CUDA_TRY(cudaOccupancyMaxActiveBlocksPerMultiprocessor( - &max_occupancy, kernel_ptr, Policy::Nthreads, smem_size)); - - return max_occupancy; -} - -} // namespace raft::bench::distance::tune diff --git a/cpp/bench/prims/distance/tune_pairwise/kernel.cuh b/cpp/bench/prims/distance/tune_pairwise/kernel.cuh deleted file mode 100644 index 5da54a343c..0000000000 --- a/cpp/bench/prims/distance/tune_pairwise/kernel.cuh +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // lp_unexp_distance_op -#include // pairwise_matrix_params - -namespace raft::bench::distance::tune { - -// Launch one specific kernel with the following template parameters -constexpr bool row_major = true; -using DataT = float; -using AccT = float; -using OutT = DataT; -using IdxT = int; - -using FinOpT = raft::identity_op; - -using pairwise_matrix_params = - raft::distance::detail::pairwise_matrix_params; - -// Launches kernel -void launch_kernel(pairwise_matrix_params, dim3, cudaStream_t); - -// Describes the block size that is decided by the policy -void get_block_size(int& m, int& n, int& k); - -int get_max_occupancy(); - -} // namespace raft::bench::distance::tune diff --git a/cpp/bench/prims/neighbors/cagra_bench.cuh b/cpp/bench/prims/neighbors/cagra_bench.cuh deleted file mode 100644 index acbeba375a..0000000000 --- a/cpp/bench/prims/neighbors/cagra_bench.cuh +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include -#include -#include -#include - -#include - -#include - -namespace raft::bench::neighbors { - -struct params { - /** Size of the dataset. */ - size_t n_samples; - /** Number of dimensions in the dataset. */ - int n_dims; - /** The batch size -- number of KNN searches. */ - int n_queries; - /** Number of nearest neighbours to find for every probe. */ - int k; - /** kNN graph degree*/ - int degree; - int itopk_size; - int block_size; - int search_width; - int max_iterations; - /** Ratio of removed indices. */ - double removed_ratio; -}; - -template -struct CagraBench : public fixture { - explicit CagraBench(const params& ps) - : fixture(true), - params_(ps), - queries_(make_device_matrix(handle, ps.n_queries, ps.n_dims)), - dataset_(make_device_matrix(handle, ps.n_samples, ps.n_dims)), - knn_graph_(make_device_matrix(handle, ps.n_samples, ps.degree)), - removed_indices_bitset_(handle, ps.n_samples) - { - // Generate random dataset and queriees - raft::random::RngState state{42}; - constexpr T kRangeMax = std::is_integral_v ? std::numeric_limits::max() : T(1); - constexpr T kRangeMin = std::is_integral_v ? std::numeric_limits::min() : T(-1); - if constexpr (std::is_integral_v) { - raft::random::uniformInt( - handle, state, dataset_.data_handle(), dataset_.size(), kRangeMin, kRangeMax); - raft::random::uniformInt( - handle, state, queries_.data_handle(), queries_.size(), kRangeMin, kRangeMax); - } else { - raft::random::uniform( - handle, state, dataset_.data_handle(), dataset_.size(), kRangeMin, kRangeMax); - raft::random::uniform( - handle, state, queries_.data_handle(), queries_.size(), kRangeMin, kRangeMax); - } - - // Generate random knn graph - - raft::random::uniformInt( - handle, state, knn_graph_.data_handle(), knn_graph_.size(), 0, ps.n_samples - 1); - - auto metric = raft::distance::DistanceType::L2Expanded; - - auto removed_indices = - raft::make_device_vector(handle, ps.removed_ratio * ps.n_samples); - thrust::sequence( - resource::get_thrust_policy(handle), - thrust::device_pointer_cast(removed_indices.data_handle()), - thrust::device_pointer_cast(removed_indices.data_handle() + removed_indices.extent(0))); - removed_indices_bitset_.set(handle, removed_indices.view()); - index_.emplace(raft::neighbors::cagra::index( - handle, metric, make_const_mdspan(dataset_.view()), make_const_mdspan(knn_graph_.view()))); - } - - void run_benchmark(::benchmark::State& state) override - { - raft::neighbors::cagra::search_params search_params; - search_params.max_queries = 1024; - search_params.itopk_size = params_.itopk_size; - search_params.team_size = 0; - search_params.thread_block_size = params_.block_size; - search_params.search_width = params_.search_width; - - auto indices = make_device_matrix(handle, params_.n_queries, params_.k); - auto distances = make_device_matrix(handle, params_.n_queries, params_.k); - auto ind_v = make_device_matrix_view( - indices.data_handle(), params_.n_queries, params_.k); - auto dist_v = make_device_matrix_view( - distances.data_handle(), params_.n_queries, params_.k); - - auto queries_v = make_const_mdspan(queries_.view()); - if (params_.removed_ratio > 0) { - auto filter = raft::neighbors::filtering::bitset_filter(removed_indices_bitset_.view()); - loop_on_state(state, [&]() { - raft::neighbors::cagra::search_with_filtering( - this->handle, search_params, *this->index_, queries_v, ind_v, dist_v, filter); - }); - } else { - loop_on_state(state, [&]() { - raft::neighbors::cagra::search( - this->handle, search_params, *this->index_, queries_v, ind_v, dist_v); - }); - } - - double data_size = params_.n_samples * params_.n_dims * sizeof(T); - double graph_size = params_.n_samples * params_.degree * sizeof(IdxT); - - int iterations = params_.max_iterations; - if (iterations == 0) { - // see search_plan_impl::adjust_search_params() - double r = params_.itopk_size / static_cast(params_.search_width); - iterations = 1 + std::min(r * 1.1, r + 10); - } - state.counters["dataset (GiB)"] = data_size / (1 << 30); - state.counters["graph (GiB)"] = graph_size / (1 << 30); - state.counters["n_rows"] = params_.n_samples; - state.counters["n_cols"] = params_.n_dims; - state.counters["degree"] = params_.degree; - state.counters["n_queries"] = params_.n_queries; - state.counters["k"] = params_.k; - state.counters["itopk_size"] = params_.itopk_size; - state.counters["block_size"] = params_.block_size; - state.counters["search_width"] = params_.search_width; - state.counters["iterations"] = iterations; - state.counters["removed_ratio"] = params_.removed_ratio; - } - - private: - const params params_; - std::optional> index_; - raft::device_matrix queries_; - raft::device_matrix dataset_; - raft::device_matrix knn_graph_; - raft::core::bitset removed_indices_bitset_; -}; - -inline const std::vector generate_inputs() -{ - std::vector inputs = - raft::util::itertools::product({2000000ull}, // n_samples - {128, 256, 512, 1024}, // dataset dim - {1000}, // n_queries - {32}, // k - {64}, // knn graph degree - {64}, // itopk_size - {0}, // block_size - {1}, // search_width - {0}, // max_iterations - {0.0} // removed_ratio - ); - auto inputs2 = raft::util::itertools::product({2000000ull, 10000000ull}, // n_samples - {128}, // dataset dim - {1000}, // n_queries - {32}, // k - {64}, // knn graph degree - {64}, // itopk_size - {64, 128, 256, 512, 1024}, // block_size - {1}, // search_width - {0}, // max_iterations - {0.0} // removed_ratio - ); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - - inputs2 = raft::util::itertools::product( - {2000000ull, 10000000ull}, // n_samples - {128}, // dataset dim - {1, 10, 10000}, // n_queries - {255}, // k - {64}, // knn graph degree - {300}, // itopk_size - {256}, // block_size - {2}, // search_width - {0}, // max_iterations - {0.0, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64} // removed_ratio - ); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - return inputs; -} - -const std::vector kCagraInputs = generate_inputs(); - -#define CAGRA_REGISTER(ValT, IdxT, inputs) \ - namespace BENCHMARK_PRIVATE_NAME(knn) { \ - using AnnCagra = CagraBench; \ - RAFT_BENCH_REGISTER(AnnCagra, #ValT "/" #IdxT, inputs); \ - } - -} // namespace raft::bench::neighbors diff --git a/cpp/bench/prims/neighbors/knn.cuh b/cpp/bench/prims/neighbors/knn.cuh deleted file mode 100644 index 6499078623..0000000000 --- a/cpp/bench/prims/neighbors/knn.cuh +++ /dev/null @@ -1,516 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace raft::bench::spatial { - -struct params { - /** Size of the dataset. */ - size_t n_samples; - /** Number of dimensions in the dataset. */ - size_t n_dims; - /** The batch size -- number of KNN searches. */ - size_t n_queries; - /** Number of nearest neighbours to find for every probe. */ - size_t k; - /** Ratio of removed indices. */ - double removed_ratio; -}; - -inline auto operator<<(std::ostream& os, const params& p) -> std::ostream& -{ - os << p.n_samples << "#" << p.n_dims << "#" << p.n_queries << "#" << p.k << "#" - << p.removed_ratio; - return os; -} - -enum class TransferStrategy { NO_COPY, COPY_PLAIN, COPY_PINNED, MAP_PINNED, MANAGED }; // NOLINT -enum class Scope { BUILD, SEARCH, BUILD_SEARCH }; // NOLINT - -inline auto operator<<(std::ostream& os, const TransferStrategy& ts) -> std::ostream& -{ - switch (ts) { - case TransferStrategy::NO_COPY: os << "NO_COPY"; break; - case TransferStrategy::COPY_PLAIN: os << "COPY_PLAIN"; break; - case TransferStrategy::COPY_PINNED: os << "COPY_PINNED"; break; - case TransferStrategy::MAP_PINNED: os << "MAP_PINNED"; break; - case TransferStrategy::MANAGED: os << "MANAGED"; break; - default: os << "UNKNOWN"; - } - return os; -} - -inline auto operator<<(std::ostream& os, const Scope& s) -> std::ostream& -{ - switch (s) { - case Scope::BUILD: os << "BUILD"; break; - case Scope::SEARCH: os << "SEARCH"; break; - case Scope::BUILD_SEARCH: os << "BUILD_SEARCH"; break; - default: os << "UNKNOWN"; - } - return os; -} - -struct device_resource { - public: - explicit device_resource(bool managed) : managed_(managed) - { - if (managed_) { - res_ = new rmm::mr::managed_memory_resource(); - } else { - res_ = rmm::mr::get_current_device_resource(); - } - } - - ~device_resource() - { - if (managed_) { delete res_; } - } - - [[nodiscard]] auto get() const -> rmm::device_async_resource_ref { return res_; } - - private: - const bool managed_; - rmm::mr::device_memory_resource* res_; -}; - -template -struct host_uvector { - host_uvector(size_t n, bool pinned) : n_(n) - { - if (pinned) { - res_ = new rmm::mr::pinned_memory_resource(); - } else { - res_ = new rmm::mr::new_delete_resource(); - } - arr_ = static_cast(res_->allocate(n_ * sizeof(T))); - } - - ~host_uvector() noexcept - { - res_->deallocate(arr_, n_ * sizeof(T)); - delete res_; - } - - auto data() -> T* { return arr_; } - [[nodiscard]] auto size() const -> size_t { return n_; } - - private: - rmm::mr::host_memory_resource* res_; - size_t n_; - T* arr_; -}; - -template -struct ivf_flat_knn { - using dist_t = float; - - std::optional> index; - raft::neighbors::ivf_flat::index_params index_params; - raft::neighbors::ivf_flat::search_params search_params; - params ps; - - ivf_flat_knn(const raft::device_resources& handle, const params& ps, const ValT* data) : ps(ps) - { - index_params.n_lists = 4096; - index_params.metric = raft::distance::DistanceType::L2Expanded; - index.emplace(raft::neighbors::ivf_flat::build( - handle, index_params, data, IdxT(ps.n_samples), uint32_t(ps.n_dims))); - } - - void search(const raft::device_resources& handle, - const ValT* search_items, - dist_t* out_dists, - IdxT* out_idxs) - { - search_params.n_probes = 20; - raft::neighbors::ivf_flat::search(handle, - search_params, - *index, - search_items, - ps.n_queries, - ps.k, - out_idxs, - out_dists, - resource::get_workspace_resource(handle)); - } -}; - -template -struct ivf_pq_knn { - using dist_t = float; - - std::optional> index; - raft::neighbors::ivf_pq::index_params index_params; - raft::neighbors::ivf_pq::search_params search_params; - params ps; - - ivf_pq_knn(const raft::device_resources& handle, const params& ps, const ValT* data) : ps(ps) - { - index_params.n_lists = 4096; - index_params.metric = raft::distance::DistanceType::L2Expanded; - auto data_view = raft::make_device_matrix_view(data, ps.n_samples, ps.n_dims); - index.emplace(raft::neighbors::ivf_pq::build(handle, index_params, data_view)); - } - - void search(const raft::device_resources& handle, - const ValT* search_items, - dist_t* out_dists, - IdxT* out_idxs) - { - search_params.n_probes = 20; - auto queries_view = - raft::make_device_matrix_view(search_items, ps.n_queries, ps.n_dims); - auto idxs_view = raft::make_device_matrix_view(out_idxs, ps.n_queries, ps.k); - auto dists_view = - raft::make_device_matrix_view(out_dists, ps.n_queries, ps.k); - raft::neighbors::ivf_pq::search( - handle, search_params, *index, queries_view, idxs_view, dists_view); - } -}; - -template -struct brute_force_knn { - using dist_t = ValT; - - ValT* index; - params ps; - - brute_force_knn(const raft::device_resources& handle, const params& ps, const ValT* data) - : index(const_cast(data)), ps(ps) - { - } - - void search(const raft::device_resources& handle, - const ValT* search_items, - dist_t* out_dists, - IdxT* out_idxs) - { - std::vector input{index}; - std::vector sizes{ps.n_samples}; - raft::spatial::knn::brute_force_knn(handle, - input, - sizes, - ps.n_dims, - const_cast(search_items), - ps.n_queries, - out_idxs, - out_dists, - ps.k); - } -}; - -template -struct ivf_flat_filter_knn { - using dist_t = float; - - std::optional> index; - raft::neighbors::ivf_flat::index_params index_params; - raft::neighbors::ivf_flat::search_params search_params; - raft::core::bitset removed_indices_bitset_; - params ps; - - ivf_flat_filter_knn(const raft::device_resources& handle, const params& ps, const ValT* data) - : ps(ps), removed_indices_bitset_(handle, ps.n_samples) - { - index_params.n_lists = 4096; - index_params.metric = raft::distance::DistanceType::L2Expanded; - index.emplace(raft::neighbors::ivf_flat::build( - handle, index_params, data, IdxT(ps.n_samples), uint32_t(ps.n_dims))); - auto removed_indices = - raft::make_device_vector(handle, ps.removed_ratio * ps.n_samples); - thrust::sequence( - resource::get_thrust_policy(handle), - thrust::device_pointer_cast(removed_indices.data_handle()), - thrust::device_pointer_cast(removed_indices.data_handle() + removed_indices.extent(0))); - removed_indices_bitset_.set(handle, removed_indices.view()); - } - - void search(const raft::device_resources& handle, - const ValT* search_items, - dist_t* out_dists, - IdxT* out_idxs) - { - search_params.n_probes = 20; - auto queries_view = - raft::make_device_matrix_view(search_items, ps.n_queries, ps.n_dims); - auto neighbors_view = raft::make_device_matrix_view(out_idxs, ps.n_queries, ps.k); - auto distance_view = raft::make_device_matrix_view(out_dists, ps.n_queries, ps.k); - auto filter = raft::neighbors::filtering::bitset_filter(removed_indices_bitset_.view()); - - if (ps.removed_ratio > 0) { - raft::neighbors::ivf_flat::search_with_filtering( - handle, search_params, *index, queries_view, neighbors_view, distance_view, filter); - } else { - raft::neighbors::ivf_flat::search( - handle, search_params, *index, queries_view, neighbors_view, distance_view); - } - } -}; - -template -struct ivf_pq_filter_knn { - using dist_t = float; - - std::optional> index; - raft::neighbors::ivf_pq::index_params index_params; - raft::neighbors::ivf_pq::search_params search_params; - raft::core::bitset removed_indices_bitset_; - params ps; - - ivf_pq_filter_knn(const raft::device_resources& handle, const params& ps, const ValT* data) - : ps(ps), removed_indices_bitset_(handle, ps.n_samples) - { - index_params.n_lists = 4096; - index_params.metric = raft::distance::DistanceType::L2Expanded; - auto data_view = raft::make_device_matrix_view(data, ps.n_samples, ps.n_dims); - index.emplace(raft::neighbors::ivf_pq::build(handle, index_params, data_view)); - auto removed_indices = - raft::make_device_vector(handle, ps.removed_ratio * ps.n_samples); - thrust::sequence( - resource::get_thrust_policy(handle), - thrust::device_pointer_cast(removed_indices.data_handle()), - thrust::device_pointer_cast(removed_indices.data_handle() + removed_indices.extent(0))); - removed_indices_bitset_.set(handle, removed_indices.view()); - } - - void search(const raft::device_resources& handle, - const ValT* search_items, - dist_t* out_dists, - IdxT* out_idxs) - { - search_params.n_probes = 20; - auto queries_view = - raft::make_device_matrix_view(search_items, ps.n_queries, ps.n_dims); - auto neighbors_view = - raft::make_device_matrix_view(out_idxs, ps.n_queries, ps.k); - auto distance_view = - raft::make_device_matrix_view(out_dists, ps.n_queries, ps.k); - auto filter = raft::neighbors::filtering::bitset_filter(removed_indices_bitset_.view()); - - if (ps.removed_ratio > 0) { - raft::neighbors::ivf_pq::search_with_filtering( - handle, search_params, *index, queries_view, neighbors_view, distance_view, filter); - } else { - raft::neighbors::ivf_pq::search( - handle, search_params, *index, queries_view, neighbors_view, distance_view); - } - } -}; - -template -struct knn : public fixture { - explicit knn(const params& p, const TransferStrategy& strategy, const Scope& scope) - : fixture(true), - params_(p), - strategy_(strategy), - scope_(scope), - dev_mem_res_(strategy == TransferStrategy::MANAGED), - data_host_(0), - search_items_(p.n_queries * p.n_dims, stream), - out_dists_(p.n_queries * p.k, stream), - out_idxs_(p.n_queries * p.k, stream) - { - raft::random::RngState state{42}; - gen_data(state, search_items_, search_items_.size(), stream); - try { - size_t total_size = p.n_samples * p.n_dims; - data_host_.resize(total_size); - constexpr size_t kGenMinibatchSize = 1024 * 1024 * 1024; - rmm::device_uvector d(std::min(kGenMinibatchSize, total_size), stream); - for (size_t offset = 0; offset < total_size; offset += kGenMinibatchSize) { - size_t actual_size = std::min(total_size - offset, kGenMinibatchSize); - gen_data(state, d, actual_size, stream); - copy(data_host_.data() + offset, d.data(), actual_size, stream); - } - } catch (std::bad_alloc& e) { - data_does_not_fit_ = true; - } - } - - template - void gen_data(raft::random::RngState& state, // NOLINT - rmm::device_uvector& vec, - size_t n, - rmm::cuda_stream_view stream) - { - constexpr T kRangeMax = std::is_integral_v ? std::numeric_limits::max() : T(1); - constexpr T kRangeMin = std::is_integral_v ? std::numeric_limits::min() : T(-1); - if constexpr (std::is_integral_v) { - raft::random::uniformInt(handle, state, vec.data(), n, kRangeMin, kRangeMax); - } else { - raft::random::uniform(handle, state, vec.data(), n, kRangeMin, kRangeMax); - } - } - - void run_benchmark(::benchmark::State& state) override - { - if (data_does_not_fit_) { - state.SkipWithError("The data size is too big to fit into the host memory."); - } - if (scope_ == Scope::SEARCH && strategy_ != TransferStrategy::NO_COPY) { - state.SkipWithError( - "When benchmarking without index building (Scope::SEARCH), the data must be already on the " - "device (TransferStrategy::NO_COPY)"); - } - - try { - std::ostringstream label_stream; - label_stream << params_ << "#" << strategy_ << "#" << scope_; - state.SetLabel(label_stream.str()); - raft::device_resources handle(stream); - std::optional index; - - if (scope_ == Scope::SEARCH) { // also implies TransferStrategy::NO_COPY - rmm::device_uvector data(data_host_.size(), stream); - copy(data.data(), data_host_.data(), data_host_.size(), stream); - index.emplace(handle, params_, data.data()); - stream.synchronize(); - } - - // benchmark loop - for (auto _ : state) { - // managed or plain device memory initialized anew every time - rmm::device_uvector data(data_host_.size(), stream, dev_mem_res_.get()); - ValT* data_ptr = data.data(); - size_t allocation_size = data_host_.size() * sizeof(ValT); - - // Non-benchmarked part: using different methods to copy the data if necessary - switch (strategy_) { - case TransferStrategy::NO_COPY: // copy data to GPU before starting the timer. - copy(data_ptr, data_host_.data(), data_host_.size(), stream); - break; - case TransferStrategy::COPY_PINNED: - RAFT_CUDA_TRY( - cudaHostRegister(data_host_.data(), allocation_size, cudaHostRegisterDefault)); - break; - case TransferStrategy::MAP_PINNED: - RAFT_CUDA_TRY( - cudaHostRegister(data_host_.data(), allocation_size, cudaHostRegisterMapped)); - RAFT_CUDA_TRY(cudaHostGetDevicePointer(&data_ptr, data_host_.data(), 0)); - break; - case TransferStrategy::MANAGED: // sic! using std::memcpy rather than cuda copy - RAFT_CUDA_TRY(cudaMemAdvise(data_ptr, - allocation_size, - cudaMemAdviseSetPreferredLocation, - resource::get_device_id(handle))); - RAFT_CUDA_TRY(cudaMemAdvise(data_ptr, - allocation_size, - cudaMemAdviseSetAccessedBy, - resource::get_device_id(handle))); - RAFT_CUDA_TRY(cudaMemAdvise(data_ptr, - allocation_size, - cudaMemAdviseSetReadMostly, - resource::get_device_id(handle))); - std::memcpy(data_ptr, data_host_.data(), allocation_size); - break; - default: break; - } - - flush_L2_cache(); - { - // Timer synchronizes the stream, so all prior gpu work should be done before it sets off. - cuda_event_timer timer(state, stream); - switch (strategy_) { - case TransferStrategy::COPY_PLAIN: - case TransferStrategy::COPY_PINNED: - copy(data_ptr, data_host_.data(), data_host_.size(), stream); - default: break; - } - - if (scope_ != Scope::SEARCH) { index.emplace(handle, params_, data_ptr); } - if (scope_ != Scope::BUILD) { - index->search(handle, search_items_.data(), out_dists_.data(), out_idxs_.data()); - } - } - - if (scope_ != Scope::SEARCH) { index.reset(); } - - switch (strategy_) { - case TransferStrategy::COPY_PINNED: - case TransferStrategy::MAP_PINNED: - RAFT_CUDA_TRY(cudaHostUnregister(data_host_.data())); - break; - default: break; - } - } - } catch (raft::exception& e) { - state.SkipWithError(e.what()); - } catch (std::bad_alloc& e) { - state.SkipWithError(e.what()); - } - } - - private: - const params params_; - const TransferStrategy strategy_; - const Scope scope_; - device_resource dev_mem_res_; - bool data_does_not_fit_ = false; - - std::vector data_host_; - rmm::device_uvector search_items_; - rmm::device_uvector out_dists_; - rmm::device_uvector out_idxs_; -}; - -inline const std::vector kInputs{ - {2000000, 128, 1000, 32, 0}, {10000000, 128, 1000, 32, 0}, {10000, 8192, 1000, 32, 0}}; - -const std::vector kInputsFilter = - raft::util::itertools::product({size_t(10000000)}, // n_samples - {size_t(128)}, // n_dim - {size_t(1000)}, // n_queries - {size_t(255)}, // k - {0.0, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64} // removed_ratio - ); -inline const std::vector kAllStrategies{ - TransferStrategy::NO_COPY, TransferStrategy::MAP_PINNED, TransferStrategy::MANAGED}; -inline const std::vector kNoCopyOnly{TransferStrategy::NO_COPY}; - -inline const std::vector kScopeFull{Scope::BUILD_SEARCH}; -inline const std::vector kAllScopes{Scope::BUILD_SEARCH, Scope::SEARCH, Scope::BUILD}; - -#define KNN_REGISTER(ValT, IdxT, ImplT, inputs, strats, scope) \ - namespace BENCHMARK_PRIVATE_NAME(knn) { \ - using KNN = knn>; \ - RAFT_BENCH_REGISTER(KNN, #ValT "/" #IdxT "/" #ImplT, inputs, strats, scope); \ - } - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/brute_force_float_int64_t.cu b/cpp/bench/prims/neighbors/knn/brute_force_float_int64_t.cu deleted file mode 100644 index 7df0599670..0000000000 --- a/cpp/bench/prims/neighbors/knn/brute_force_float_int64_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(float, int64_t, brute_force_knn, kInputs, kAllStrategies, kScopeFull); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/brute_force_float_uint32_t.cu b/cpp/bench/prims/neighbors/knn/brute_force_float_uint32_t.cu deleted file mode 100644 index 9704d39e76..0000000000 --- a/cpp/bench/prims/neighbors/knn/brute_force_float_uint32_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(float, uint32_t, brute_force_knn, kInputs, kAllStrategies, kScopeFull); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/cagra_float_uint32_t.cu b/cpp/bench/prims/neighbors/knn/cagra_float_uint32_t.cu deleted file mode 100644 index 5d762f6e85..0000000000 --- a/cpp/bench/prims/neighbors/knn/cagra_float_uint32_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../cagra_bench.cuh" - -namespace raft::bench::neighbors { - -CAGRA_REGISTER(float, uint32_t, kCagraInputs); - -} // namespace raft::bench::neighbors diff --git a/cpp/bench/prims/neighbors/knn/ivf_flat_filter_float_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_flat_filter_float_int64_t.cu deleted file mode 100644 index bf5118ceae..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_flat_filter_float_int64_t.cu +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#undef RAFT_EXPLICIT_INSTANTIATE_ONLY // Enable instantiation of search with filter -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(float, int64_t, ivf_flat_filter_knn, kInputsFilter, kNoCopyOnly, kScopeFull); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/ivf_flat_float_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_flat_float_int64_t.cu deleted file mode 100644 index fbbb4f9acc..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_flat_float_int64_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(float, int64_t, ivf_flat_knn, kInputs, kNoCopyOnly, kAllScopes); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/ivf_flat_int8_t_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_flat_int8_t_int64_t.cu deleted file mode 100644 index 7067dbe1b6..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_flat_int8_t_int64_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(int8_t, int64_t, ivf_flat_knn, kInputs, kNoCopyOnly, kAllScopes); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/ivf_flat_uint8_t_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_flat_uint8_t_int64_t.cu deleted file mode 100644 index 91fada3c28..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_flat_uint8_t_int64_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(uint8_t, int64_t, ivf_flat_knn, kInputs, kNoCopyOnly, kAllScopes); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/ivf_pq_filter_float_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_pq_filter_float_int64_t.cu deleted file mode 100644 index 1840eca99d..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_pq_filter_float_int64_t.cu +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -#include -#include -namespace raft::bench::spatial { - -KNN_REGISTER(float, int64_t, ivf_pq_filter_knn, kInputsFilter, kNoCopyOnly, kScopeFull); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/ivf_pq_float_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_pq_float_int64_t.cu deleted file mode 100644 index 83c4973c3a..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_pq_float_int64_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(float, int64_t, ivf_pq_knn, kInputs, kNoCopyOnly, kAllScopes); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/ivf_pq_int8_t_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_pq_int8_t_int64_t.cu deleted file mode 100644 index 4ea281b11a..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_pq_int8_t_int64_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(int8_t, int64_t, ivf_pq_knn, kInputs, kNoCopyOnly, kAllScopes); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/knn/ivf_pq_uint8_t_int64_t.cu b/cpp/bench/prims/neighbors/knn/ivf_pq_uint8_t_int64_t.cu deleted file mode 100644 index 3313a49ba2..0000000000 --- a/cpp/bench/prims/neighbors/knn/ivf_pq_uint8_t_int64_t.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../knn.cuh" - -namespace raft::bench::spatial { - -KNN_REGISTER(uint8_t, int64_t, ivf_pq_knn, kInputs, kNoCopyOnly, kAllScopes); - -} // namespace raft::bench::spatial diff --git a/cpp/bench/prims/neighbors/refine.cuh b/cpp/bench/prims/neighbors/refine.cuh deleted file mode 100644 index 0360babd82..0000000000 --- a/cpp/bench/prims/neighbors/refine.cuh +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include - -#include -#include - -using namespace raft::neighbors; - -namespace raft::bench::neighbors { - -template -inline auto operator<<(std::ostream& os, const RefineInputs& p) -> std::ostream& -{ - os << p.n_rows << "#" << p.dim << "#" << p.n_queries << "#" << p.k0 << "#" << p.k << "#" - << (p.host_data ? "host" : "device"); - return os; -} - -template -class RefineAnn : public fixture { - public: - RefineAnn(RefineInputs p) : data(handle_, p) {} - - void run_benchmark(::benchmark::State& state) override - { - std::ostringstream label_stream; - label_stream << data.p; - state.SetLabel(label_stream.str()); - - auto old_mr = rmm::mr::get_current_device_resource(); - rmm::mr::pool_memory_resource pool_mr( - old_mr, rmm::percent_of_free_device_memory(50)); - rmm::mr::set_current_device_resource(&pool_mr); - - if (data.p.host_data) { - loop_on_state(state, [this]() { - raft::neighbors::refine(handle_, - data.dataset_host.view(), - data.queries_host.view(), - data.candidates_host.view(), - data.refined_indices_host.view(), - data.refined_distances_host.view(), - data.p.metric); - }); - } else { - loop_on_state(state, [&]() { - raft::neighbors::refine(handle_, - data.dataset.view(), - data.queries.view(), - data.candidates.view(), - data.refined_indices.view(), - data.refined_distances.view(), - data.p.metric); - }); - } - rmm::mr::set_current_device_resource(old_mr); - } - - private: - raft::device_resources handle_; - RefineHelper data; -}; - -template -std::vector> getInputs() -{ - std::vector> out; - raft::distance::DistanceType metric = raft::distance::DistanceType::L2Expanded; - for (bool host_data : {true, false}) { - for (T n_queries : {1000, 10000}) { - for (T dim : {128, 512}) { - out.push_back(RefineInputs{n_queries, 2000000, dim, 32, 128, metric, host_data}); - out.push_back(RefineInputs{n_queries, 2000000, dim, 10, 40, metric, host_data}); - } - } - } - return out; -} - -} // namespace raft::bench::neighbors diff --git a/cpp/bench/prims/neighbors/refine_float_int64_t.cu b/cpp/bench/prims/neighbors/refine_float_int64_t.cu deleted file mode 100644 index d69a157eca..0000000000 --- a/cpp/bench/prims/neighbors/refine_float_int64_t.cu +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "refine.cuh" - -#include - -using namespace raft::neighbors; - -namespace raft::bench::neighbors { -using refine_float_int64 = RefineAnn; -RAFT_BENCH_REGISTER(refine_float_int64, "", getInputs()); -} // namespace raft::bench::neighbors diff --git a/cpp/bench/prims/neighbors/refine_uint8_t_int64_t.cu b/cpp/bench/prims/neighbors/refine_uint8_t_int64_t.cu deleted file mode 100644 index 9da536b6c7..0000000000 --- a/cpp/bench/prims/neighbors/refine_uint8_t_int64_t.cu +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "refine.cuh" - -#include - -using namespace raft::neighbors; - -namespace raft::bench::neighbors { -using refine_uint8_int64 = RefineAnn; -RAFT_BENCH_REGISTER(refine_uint8_int64, "", getInputs()); -} // namespace raft::bench::neighbors diff --git a/cpp/cmake/config.json b/cpp/cmake/config.json index f7cc50e513..3c568d9766 100644 --- a/cpp/cmake/config.json +++ b/cpp/cmake/config.json @@ -9,7 +9,7 @@ "VERSION": "?", "GIT_SHALLOW": "?", "OPTIONS": "*", - "FIND_PACKAGE_ARGUMENTS": "*" + "FIND_PACKAGE_ARGUMENTS": "*" } }, "ConfigureTest": { diff --git a/cpp/cmake/modules/FindAVX.cmake b/cpp/cmake/modules/FindAVX.cmake deleted file mode 100644 index 7f3b2dfc76..0000000000 --- a/cpp/cmake/modules/FindAVX.cmake +++ /dev/null @@ -1,110 +0,0 @@ -# ============================================================================= -# Copyright (c) 2016- Facebook, Inc (Adam Paszke) -# Copyright (c) 2014- Facebook, Inc (Soumith Chintala) -# Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) -# Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) -# Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) -# Copyright (c) 2011-2013 NYU (Clement Farabet) -# Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) -# Copyright (c) 2006 Idiap Research Institute (Samy Bengio) -# Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) -# -# Note: This file was copied from PyTorch and modified for use in the RAFT library. -# Refer to thirdparty/LICENSES/LICENSE.pytorch for license and additional -# copyright information. -# ============================================================================= - -INCLUDE(CheckCXXSourceRuns) - -SET(AVX_CODE - " - #include - - int main() - { - __m256 a; - a = _mm256_set1_ps(0); - return 0; - } -" -) - -SET(AVX512_CODE - " - #include - - int main() - { - __m512i a = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0); - __m512i b = a; - __mmask64 equality_mask = _mm512_cmp_epi8_mask(a, b, _MM_CMPINT_EQ); - return 0; - } -" -) - -SET(AVX2_CODE - " - #include - - int main() - { - __m256i a = {0}; - a = _mm256_abs_epi16(a); - __m256i x; - _mm256_extract_epi64(x, 0); // we rely on this in our AVX2 code - return 0; - } -" -) - -MACRO(CHECK_SSE lang type flags) - SET(__FLAG_I 1) - SET(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS}) - FOREACH(__FLAG ${flags}) - IF(NOT ${lang}_${type}_FOUND) - SET(CMAKE_REQUIRED_FLAGS ${__FLAG}) - CHECK_CXX_SOURCE_RUNS("${${type}_CODE}" ${lang}_HAS_${type}_${__FLAG_I}) - IF(${lang}_HAS_${type}_${__FLAG_I}) - SET(${lang}_${type}_FOUND - TRUE - CACHE BOOL "${lang} ${type} support" - ) - SET(${lang}_${type}_FLAGS - "${__FLAG}" - CACHE STRING "${lang} ${type} flags" - ) - ENDIF() - MATH(EXPR __FLAG_I "${__FLAG_I}+1") - ENDIF() - ENDFOREACH() - SET(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE}) - - IF(NOT ${lang}_${type}_FOUND) - SET(${lang}_${type}_FOUND - FALSE - CACHE BOOL "${lang} ${type} support" - ) - SET(${lang}_${type}_FLAGS - "" - CACHE STRING "${lang} ${type} flags" - ) - ENDIF() - - MARK_AS_ADVANCED(${lang}_${type}_FOUND ${lang}_${type}_FLAGS) - -ENDMACRO() - -# CHECK_SSE(C "AVX" " ;-mavx;/arch:AVX") CHECK_SSE(C "AVX2" " ;-mavx2 -mfma;/arch:AVX2") CHECK_SSE(C -# "AVX512" " ;-mavx512f -mavx512dq -mavx512vl -mavx512bw -mfma;/arch:AVX512") -# -CHECK_SSE(CXX "AVX" " ;-mavx;/arch:AVX") -CHECK_SSE(CXX "AVX2" " ;-mavx2 -mfma;/arch:AVX2") -CHECK_SSE(CXX "AVX512" " ;-mavx512f -mavx512dq -mavx512vl -mavx512bw -mfma;/arch:AVX512") diff --git a/cpp/cmake/patches/cutlass/build-export.patch b/cpp/cmake/patches/cutlass/build-export.patch new file mode 100644 index 0000000000..a6423e9c08 --- /dev/null +++ b/cpp/cmake/patches/cutlass/build-export.patch @@ -0,0 +1,27 @@ +From e0a9597946257a01ae8444200f836ee51d5597ba Mon Sep 17 00:00:00 2001 +From: Kyle Edwards +Date: Wed, 20 Nov 2024 16:37:38 -0500 +Subject: [PATCH] Remove erroneous include directories + +These directories are left over from when CuTe was a separate +CMake project. Remove them. +--- + CMakeLists.txt | 2 -- + 1 file changed, 2 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 7419bdf5e..545384d82 100755 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -665,8 +665,6 @@ target_include_directories( + $ + $ + $ +- $ +- $ + ) + + # Mark CTK headers as system to supress warnings from them +-- +2.34.1 + diff --git a/cpp/cmake/patches/cutlass_override.json b/cpp/cmake/patches/cutlass_override.json new file mode 100644 index 0000000000..7bf818987f --- /dev/null +++ b/cpp/cmake/patches/cutlass_override.json @@ -0,0 +1,16 @@ +{ + "packages" : { + "cutlass" : { + "version": "3.5.1", + "git_url": "https://github.com/NVIDIA/cutlass.git", + "git_tag": "v${version}", + "patches" : [ + { + "file" : "${current_json_dir}/cutlass/build-export.patch", + "issue" : "Fix build directory export", + "fixed_in" : "" + } + ] + } + } +} diff --git a/cpp/cmake/patches/faiss_override.json b/cpp/cmake/patches/faiss_override.json deleted file mode 100644 index 19dad362b9..0000000000 --- a/cpp/cmake/patches/faiss_override.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "packages" : { - "faiss" : { - "version": "1.7.4", - "git_url": "https://github.com/facebookresearch/faiss.git", - "git_tag": "main" - } - } -} diff --git a/cpp/cmake/patches/ggnn.diff b/cpp/cmake/patches/ggnn.diff deleted file mode 100644 index fc45298803..0000000000 --- a/cpp/cmake/patches/ggnn.diff +++ /dev/null @@ -1,230 +0,0 @@ ---- a/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh -+++ b/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh -@@ -62,7 +62,7 @@ struct SimpleKNNSymCache { - const ValueT dist_half) - : dist_query(dist_query), dist_half(dist_half) {} - -- __device__ __forceinline__ DistQueryAndHalf() {} -+ DistQueryAndHalf() = default; - }; - - struct DistanceAndNorm { -@@ -98,8 +98,7 @@ struct SimpleKNNSymCache { - KeyT cache; - DistQueryAndHalf dist; - bool flag; -- -- __device__ __forceinline__ SyncTempStorage() {} -+ SyncTempStorage() = default; - }; - - public: -diff --git a/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh b/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh -index 8cbaf0d..6eb72ac 100644 ---- a/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh -+++ b/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh -@@ -41,7 +41,6 @@ limitations under the License. - #include "ggnn/sym/cuda_knn_sym_query_layer.cuh" - #include "ggnn/utils/cuda_knn_utils.cuh" - #include "ggnn/utils/cuda_knn_constants.cuh" --#include "ggnn/utils/cuda_knn_dataset.cuh" - - template - __global__ void divide(ValueT* res, ValueT* input, ValueT N) { -@@ -98,9 +97,7 @@ struct GGNNGPUInstance { - typedef GGNNGraphDevice GGNNGraphDevice; - typedef GGNNGraphHost GGNNGraphHost; - -- const Dataset* dataset; - GGNNGraphBuffer* ggnn_buffer {nullptr}; -- GGNNQuery ggnn_query; - - // Graph Shards resident on the GPU - std::vector ggnn_shards; -@@ -117,13 +114,12 @@ struct GGNNGPUInstance { - // number of shards that need to be processed by this instance - const int num_parts; - -- GGNNGPUInstance(const int gpu_id, const Dataset* dataset, -+ GGNNGPUInstance(const int gpu_id, - const int N_shard, const int L, - const bool enable_construction, const float tau_build, - const int num_parts=1, const int num_cpu_buffers=1) : - N_shard{N_shard}, L{L}, tau_build{tau_build}, -- dataset{dataset}, gpu_id{gpu_id}, -- ggnn_query{dataset->N_query, D, KQuery, num_parts}, -+ gpu_id{gpu_id}, - num_parts{num_parts} - { - CHECK_LE(L, MAX_LAYER); -@@ -135,7 +131,6 @@ struct GGNNGPUInstance { - CHECK_EQ(current_gpu_id, gpu_id) << "cudaSetDevice() needs to be called in advance!"; - } - -- ggnn_query.loadQueriesAsync(dataset->h_query, 0); - - computeGraphParameters(); - -@@ -186,7 +181,7 @@ struct GGNNGPUInstance { - } - - GGNNGPUInstance(const GGNNGPUInstance& other) -- : dataset{nullptr}, ggnn_query{0, D, KQuery}, -+ : - gpu_id{0}, N_shard{0}, num_parts{0} { - // this exists to allow using vector::emplace_back - // when it triggers a reallocation, this code will be called. -@@ -305,6 +300,7 @@ struct GGNNGPUInstance { - - // io - -+ /* - void waitForDiskIO(const int shard_id) { - auto& cpu_buffer = ggnn_cpu_buffers[shard_id%ggnn_cpu_buffers.size()]; - if (cpu_buffer.disk_io_thread.joinable()) -@@ -468,11 +464,12 @@ struct GGNNGPUInstance { - CHECK_CUDA(cudaDeviceSynchronize()); - CHECK_CUDA(cudaPeekAtLastError()); - } -+ */ - - // graph operations - - template -- void queryLayer(const int shard_id = 0) const { -+ void queryLayer(const BaseT* d_query, int batch_size, KeyT* d_query_result_ids, ValueT* d_query_result_dists, const int shard_id = 0) const { - CHECK_CUDA(cudaSetDevice(gpu_id)); - const auto& shard = ggnn_shards.at(shard_id%ggnn_shards.size()); - -@@ -482,21 +479,21 @@ struct GGNNGPUInstance { - - int* m_dist_statistics = nullptr; - if (DIST_STATS) -- cudaMallocManaged(&m_dist_statistics, dataset->N_query * sizeof(int)); -+ cudaMallocManaged(&m_dist_statistics, batch_size * sizeof(int)); - - QueryKernel query_kernel; - query_kernel.d_base = shard.d_base; -- query_kernel.d_query = ggnn_query.d_query; -+ query_kernel.d_query = d_query; - - query_kernel.d_graph = shard.d_graph; -- query_kernel.d_query_results = ggnn_query.d_query_result_ids; -- query_kernel.d_query_results_dists = ggnn_query.d_query_result_dists; -+ query_kernel.d_query_results = d_query_result_ids; -+ query_kernel.d_query_results_dists = d_query_result_dists; - - query_kernel.d_translation = shard.d_translation; - - query_kernel.d_nn1_stats = shard.d_nn1_stats; - -- query_kernel.N = dataset->N_query; -+ query_kernel.N = batch_size; - query_kernel.N_offset = 0; - - query_kernel.d_dist_stats = m_dist_statistics; -@@ -771,6 +768,16 @@ struct GGNNGPUInstance { - sym(layer, shard_id); - } - } -+ -+ void set_stream(cudaStream_t stream) { -+ assert(ggnn_shards.size() == 1); -+ ggnn_shards.at(0).stream = stream; -+ } -+ -+ void set_base_data(const BaseT* dataset) { -+ assert(ggnn_shards.size() == 1); -+ ggnn_shards.at(0).d_base = dataset; -+ } - }; - - #endif // INCLUDE_GGNN_CUDA_KNN_GGNN_GPU_INSTANCE_CUH_ -diff --git a/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh b/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh -index c94a8f1..781226d 100644 ---- a/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh -+++ b/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh -@@ -50,7 +50,7 @@ struct GGNNGraphDevice { - ValueT* d_nn1_stats; - - /// base data pointer for the shard. -- BaseT* d_base; -+ const BaseT* d_base; - - /// combined memory pool - char* d_memory; -@@ -69,7 +69,9 @@ struct GGNNGraphDevice { - const size_t selection_translation_size = align8(ST_all * sizeof(KeyT)); - const size_t nn1_stats_size = align8(2 * sizeof(ValueT)); - total_graph_size = graph_size + 2 * selection_translation_size + nn1_stats_size; -- base_size = align8(static_cast(N) * D * sizeof(BaseT)); -+ // base_size = align8(static_cast(N) * D * sizeof(BaseT)); -+ (void) N; -+ (void) D; - - const size_t total_size = base_size+total_graph_size; - -@@ -86,8 +88,7 @@ struct GGNNGraphDevice { - CHECK_CUDA(cudaMalloc(&d_memory, total_size)); - - size_t pos = 0; -- d_base = reinterpret_cast(d_memory+pos); -- pos += base_size; -+ d_base = nullptr; - d_graph = reinterpret_cast(d_memory+pos); - pos += graph_size; - d_translation = reinterpret_cast(d_memory+pos); -@@ -99,14 +100,14 @@ struct GGNNGraphDevice { - - CHECK_EQ(pos, total_size); - -- CHECK_CUDA(cudaStreamCreate(&stream)); -+ // CHECK_CUDA(cudaStreamCreate(&stream)); - - CHECK_CUDA(cudaPeekAtLastError()); - CHECK_CUDA(cudaDeviceSynchronize()); - CHECK_CUDA(cudaPeekAtLastError()); - } - -- GGNNGraphDevice(const GGNNGraphDevice& other) { -+ GGNNGraphDevice(const GGNNGraphDevice&) { - // this exists to allow using vector::emplace_back - // when it triggers a reallocation, this code will be called. - // always make sure that enough memory is reserved ahead of time. -@@ -116,7 +117,7 @@ struct GGNNGraphDevice { - ~GGNNGraphDevice() { - cudaFree(d_memory); - -- CHECK_CUDA(cudaStreamDestroy(stream)); -+ // CHECK_CUDA(cudaStreamDestroy(stream)); - } - }; - -diff --git a/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh b/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh -index 2055f9e..ef5843a 100644 ---- a/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh -+++ b/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh -@@ -92,7 +92,7 @@ struct GGNNGraphHost { - CHECK_CUDA(cudaPeekAtLastError()); - } - -- GGNNGraphHost(const GGNNGraphHost& other) { -+ GGNNGraphHost(const GGNNGraphHost&) { - // this exists to allow using vector::emplace_back - // when it triggers a reallocation, this code will be called. - // always make sure that enough memory is reserved ahead of time. -diff --git a/include/ggnn/select/cuda_knn_wrs_select_layer.cuh b/include/ggnn/select/cuda_knn_wrs_select_layer.cuh -index 49d76a1..eef69e6 100644 ---- a/include/ggnn/select/cuda_knn_wrs_select_layer.cuh -+++ b/include/ggnn/select/cuda_knn_wrs_select_layer.cuh -@@ -22,7 +22,6 @@ limitations under the License. - #include - #include - --#include - #include - - #include "ggnn/utils/cuda_knn_constants.cuh" --- -2.43.0 - diff --git a/cpp/cmake/patches/ggnn_override.json b/cpp/cmake/patches/ggnn_override.json deleted file mode 100644 index 768fae8b0c..0000000000 --- a/cpp/cmake/patches/ggnn_override.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "packages" : { - "ggnn" : { - "version": "0.5", - "git_url": "https://github.com/cgtuebingen/ggnn.git", - "git_tag": "release_${version}", - "patches" : [ - { - "file" : "${current_json_dir}/ggnn.diff", - "issue" : "Correct compilation issues", - "fixed_in" : "" - } - ] - } - } -} diff --git a/cpp/cmake/patches/hnswlib.diff b/cpp/cmake/patches/hnswlib.diff deleted file mode 100644 index e7f89a8cc9..0000000000 --- a/cpp/cmake/patches/hnswlib.diff +++ /dev/null @@ -1,188 +0,0 @@ ---- a/hnswlib/hnswalg.h -+++ b/hnswlib/hnswalg.h -@@ -3,6 +3,7 @@ - #include "visited_list_pool.h" - #include "hnswlib.h" - #include -+#include - #include - #include - #include -@@ -16,6 +17,8 @@ namespace hnswlib { - template - class HierarchicalNSW : public AlgorithmInterface { - public: -+ bool base_layer_only{false}; -+ int num_seeds=32; - static const tableint max_update_element_locks = 65536; - HierarchicalNSW(SpaceInterface *s) { - } -@@ -56,7 +59,7 @@ namespace hnswlib { - visited_list_pool_ = new VisitedListPool(1, max_elements); - - //initializations for special treatment of the first node -- enterpoint_node_ = -1; -+ enterpoint_node_ = std::numeric_limits::max(); - maxlevel_ = -1; - - linkLists_ = (char **) malloc(sizeof(void *) * max_elements_); -@@ -527,7 +530,7 @@ namespace hnswlib { - tableint *datal = (tableint *) (data + 1); - for (int i = 0; i < size; i++) { - tableint cand = datal[i]; -- if (cand < 0 || cand > max_elements_) -+ if (cand > max_elements_) - throw std::runtime_error("cand error"); - dist_t d = fstdistfunc_(query_data, getDataByInternalId(cand), dist_func_param_); - -@@ -1067,7 +1070,7 @@ namespace hnswlib { - tableint *datal = (tableint *) (data + 1); - for (int i = 0; i < size; i++) { - tableint cand = datal[i]; -- if (cand < 0 || cand > max_elements_) -+ if (cand > max_elements_) - throw std::runtime_error("cand error"); - dist_t d = fstdistfunc_(data_point, getDataByInternalId(cand), dist_func_param_); - if (d < curdist) { -@@ -1119,28 +1122,41 @@ namespace hnswlib { - tableint currObj = enterpoint_node_; - dist_t curdist = fstdistfunc_(query_data, getDataByInternalId(enterpoint_node_), dist_func_param_); - -- for (int level = maxlevel_; level > 0; level--) { -- bool changed = true; -- while (changed) { -- changed = false; -- unsigned int *data; -+ if (base_layer_only) { -+ // You can increase the number of seeds when testing large-scale dataset, num_seeds = 48 for 100M-scale -+ for (int i = 0; i < num_seeds; i++) { -+ tableint obj = i * (max_elements_ / num_seeds); -+ dist_t dist = fstdistfunc_(query_data, getDataByInternalId(obj), dist_func_param_); -+ if (dist < curdist) { -+ curdist = dist; -+ currObj = obj; -+ } -+ } -+ } -+ else{ -+ for (int level = maxlevel_; level > 0; level--) { -+ bool changed = true; -+ while (changed) { -+ changed = false; -+ unsigned int *data; - -- data = (unsigned int *) get_linklist(currObj, level); -- int size = getListCount(data); -- metric_hops++; -- metric_distance_computations+=size; -+ data = (unsigned int *) get_linklist(currObj, level); -+ int size = getListCount(data); -+ metric_hops++; -+ metric_distance_computations+=size; - -- tableint *datal = (tableint *) (data + 1); -- for (int i = 0; i < size; i++) { -- tableint cand = datal[i]; -- if (cand < 0 || cand > max_elements_) -- throw std::runtime_error("cand error"); -- dist_t d = fstdistfunc_(query_data, getDataByInternalId(cand), dist_func_param_); -+ tableint *datal = (tableint *) (data + 1); -+ for (int i = 0; i < size; i++) { -+ tableint cand = datal[i]; -+ if (cand > max_elements_) -+ throw std::runtime_error("cand error"); -+ dist_t d = fstdistfunc_(query_data, getDataByInternalId(cand), dist_func_param_); - -- if (d < curdist) { -- curdist = d; -- currObj = cand; -- changed = true; -+ if (d < curdist) { -+ curdist = d; -+ currObj = cand; -+ changed = true; -+ } - } - } - } -diff --git a/hnswlib/space_l2.h b/hnswlib/space_l2.h -index 4413537..c3240f3 100644 ---- a/hnswlib/space_l2.h -+++ b/hnswlib/space_l2.h -@@ -252,13 +252,14 @@ namespace hnswlib { - ~L2Space() {} - }; - -+ template - static int - L2SqrI4x(const void *__restrict pVect1, const void *__restrict pVect2, const void *__restrict qty_ptr) { - - size_t qty = *((size_t *) qty_ptr); - int res = 0; -- unsigned char *a = (unsigned char *) pVect1; -- unsigned char *b = (unsigned char *) pVect2; -+ T *a = (T *) pVect1; -+ T *b = (T *) pVect2; - - qty = qty >> 2; - for (size_t i = 0; i < qty; i++) { -@@ -279,11 +280,12 @@ namespace hnswlib { - return (res); - } - -+ template - static int L2SqrI(const void* __restrict pVect1, const void* __restrict pVect2, const void* __restrict qty_ptr) { - size_t qty = *((size_t*)qty_ptr); - int res = 0; -- unsigned char* a = (unsigned char*)pVect1; -- unsigned char* b = (unsigned char*)pVect2; -+ T* a = (T*)pVect1; -+ T* b = (T*)pVect2; - - for(size_t i = 0; i < qty; i++) - { -@@ -294,6 +296,7 @@ namespace hnswlib { - return (res); - } - -+ template - class L2SpaceI : public SpaceInterface { - - DISTFUNC fstdistfunc_; -@@ -302,10 +305,10 @@ namespace hnswlib { - public: - L2SpaceI(size_t dim) { - if(dim % 4 == 0) { -- fstdistfunc_ = L2SqrI4x; -+ fstdistfunc_ = L2SqrI4x; - } - else { -- fstdistfunc_ = L2SqrI; -+ fstdistfunc_ = L2SqrI; - } - dim_ = dim; - data_size_ = dim * sizeof(unsigned char); -diff --git a/hnswlib/visited_list_pool.h b/hnswlib/visited_list_pool.h -index 5e1a4a5..4195ebd 100644 ---- a/hnswlib/visited_list_pool.h -+++ b/hnswlib/visited_list_pool.h -@@ -3,6 +3,7 @@ - #include - #include - #include -+#include - - namespace hnswlib { - typedef unsigned short int vl_type; -@@ -14,7 +15,7 @@ namespace hnswlib { - unsigned int numelements; - - VisitedList(int numelements1) { -- curV = -1; -+ curV = std::numeric_limits::max(); - numelements = numelements1; - mass = new vl_type[numelements]; - } --- -2.43.0 - diff --git a/cpp/cmake/patches/hnswlib_override.json b/cpp/cmake/patches/hnswlib_override.json deleted file mode 100644 index d6ab8a18a5..0000000000 --- a/cpp/cmake/patches/hnswlib_override.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "packages" : { - "hnswlib" : { - "version": "0.6.2", - "git_url": "https://github.com/nmslib/hnswlib.git", - "git_tag": "v${version}", - "patches" : [ - { - "file" : "${current_json_dir}/hnswlib.diff", - "issue" : "Correct compilation issues", - "fixed_in" : "" - } - ] - } - } -} diff --git a/cpp/cmake/thirdparty/get_cutlass.cmake b/cpp/cmake/thirdparty/get_cutlass.cmake index 0123c4b07a..d5bdd4632f 100644 --- a/cpp/cmake/thirdparty/get_cutlass.cmake +++ b/cpp/cmake/thirdparty/get_cutlass.cmake @@ -13,7 +13,9 @@ # ============================================================================= function(find_and_configure_cutlass) - set(oneValueArgs VERSION REPOSITORY PINNED_TAG) + set(options) + set(oneValueArgs) + set(multiValueArgs) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) # if(RAFT_ENABLE_DIST_DEPENDENCIES OR RAFT_COMPILE_LIBRARIES) @@ -34,13 +36,22 @@ function(find_and_configure_cutlass) set(CUDART_LIBRARY "${CUDA_cudart_static_LIBRARY}" CACHE FILEPATH "fixing cutlass cmake code" FORCE) endif() + include("${rapids-cmake-dir}/cpm/package_override.cmake") + rapids_cpm_package_override("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../patches/cutlass_override.json") + + include("${rapids-cmake-dir}/cpm/detail/package_details.cmake") + rapids_cpm_package_details(cutlass version repository tag shallow exclude) + + include("${rapids-cmake-dir}/cpm/detail/generate_patch_command.cmake") + rapids_cpm_generate_patch_command(cutlass ${version} patch_command) + rapids_cpm_find( - NvidiaCutlass ${PKG_VERSION} + NvidiaCutlass ${version} GLOBAL_TARGETS nvidia::cutlass::cutlass CPM_ARGS - GIT_REPOSITORY ${PKG_REPOSITORY} - GIT_TAG ${PKG_PINNED_TAG} - GIT_SHALLOW TRUE + GIT_REPOSITORY ${repository} + GIT_TAG ${tag} + GIT_SHALLOW ${shallow} ${patch_command} OPTIONS "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" ) @@ -79,14 +90,4 @@ function(find_and_configure_cutlass) ) endfunction() -if(NOT RAFT_CUTLASS_GIT_TAG) - set(RAFT_CUTLASS_GIT_TAG v2.10.0) -endif() - -if(NOT RAFT_CUTLASS_GIT_REPOSITORY) - set(RAFT_CUTLASS_GIT_REPOSITORY https://github.com/NVIDIA/cutlass.git) -endif() - -find_and_configure_cutlass( - VERSION 2.10.0 REPOSITORY ${RAFT_CUTLASS_GIT_REPOSITORY} PINNED_TAG ${RAFT_CUTLASS_GIT_TAG} -) +find_and_configure_cutlass() diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake deleted file mode 100644 index 706b0c2f11..0000000000 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ /dev/null @@ -1,119 +0,0 @@ -#============================================================================= -# Copyright (c) 2021-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_faiss) - set(oneValueArgs VERSION REPOSITORY PINNED_TAG BUILD_STATIC_LIBS EXCLUDE_FROM_ALL ENABLE_GPU) - cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) - - rapids_find_generate_module(faiss - HEADER_NAMES faiss/IndexFlat.h - LIBRARY_NAMES faiss - ) - - set(patch_dir "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../patches") - rapids_cpm_package_override("${patch_dir}/faiss_override.json") - - include("${rapids-cmake-dir}/cpm/detail/package_details.cmake") - rapids_cpm_package_details(faiss version repository tag shallow exclude) - - include("${rapids-cmake-dir}/cpm/detail/generate_patch_command.cmake") - rapids_cpm_generate_patch_command(faiss ${version} patch_command) - - set(BUILD_SHARED_LIBS ON) - if (PKG_BUILD_STATIC_LIBS) - set(BUILD_SHARED_LIBS OFF) - set(CPM_DOWNLOAD_faiss ON) - endif() - - include(cmake/modules/FindAVX) - # Link against AVX CPU lib if it exists - set(RAFT_FAISS_OPT_LEVEL "generic") - if(CXX_AVX2_FOUND) - set(RAFT_FAISS_OPT_LEVEL "avx2") - endif() - - rapids_cpm_find(faiss ${version} - GLOBAL_TARGETS faiss faiss_avx2 faiss_gpu faiss::faiss faiss::faiss_avx2 - CPM_ARGS - GIT_REPOSITORY ${repository} - GIT_TAG ${tag} - GIT_SHALLOW ${shallow} ${patch_command} - EXCLUDE_FROM_ALL ${exclude} - OPTIONS - "FAISS_ENABLE_GPU ${PKG_ENABLE_GPU}" - "FAISS_ENABLE_RAFT ${PKG_ENABLE_GPU}" - "FAISS_ENABLE_PYTHON OFF" - "FAISS_OPT_LEVEL ${RAFT_FAISS_OPT_LEVEL}" - "FAISS_USE_CUDA_TOOLKIT_STATIC ${CUDA_STATIC_RUNTIME}" - "BUILD_TESTING OFF" - "CMAKE_MESSAGE_LOG_LEVEL VERBOSE" - ) - - include("${rapids-cmake-dir}/cpm/detail/display_patch_status.cmake") - rapids_cpm_display_patch_status(hnswlib) - - if(TARGET faiss AND NOT TARGET faiss::faiss) - add_library(faiss::faiss ALIAS faiss) - # We need to ensure that faiss has all the conda information. So we use this approach so that - # faiss will have the conda includes/link dirs - target_link_libraries(faiss PRIVATE $) - endif() - if(TARGET faiss_avx2 AND NOT TARGET faiss::faiss_avx2) - add_library(faiss::faiss_avx2 ALIAS faiss_avx2) - # We need to ensure that faiss has all the conda information. So we use this approach so that - # faiss will have the conda includes/link dirs - target_link_libraries(faiss_avx2 PRIVATE $) - endif() - if(TARGET faiss_gpu AND NOT TARGET faiss::faiss_gpu) - add_library(faiss::faiss_gpu ALIAS faiss_gpu) - # We need to ensure that faiss has all the conda information. So we use this approach so that - # faiss will have the conda includes/link dirs - target_link_libraries(faiss_gpu PRIVATE $) - endif() - - if(faiss_ADDED) - rapids_export(BUILD faiss - EXPORT_SET faiss-targets - GLOBAL_TARGETS ${RAFT_FAISS_EXPORT_GLOBAL_TARGETS} - NAMESPACE faiss::) - endif() - - # Need to tell CMake to rescan the link group of faiss::faiss_gpu and faiss - # so that we get proper link order when they are static - # - # We don't look at the existence of `faiss_avx2` as it will always exist - # even when CXX_AVX2_FOUND is false. In addition for arm builds the - # faiss_avx2 is marked as `EXCLUDE_FROM_ALL` so we don't want to add - # a dependency to it. Adding a dependency will cause it to compile, - # and fail due to invalid compiler flags. - if(PKG_ENABLE_GPU AND PKG_BUILD_STATIC_LIBS AND CXX_AVX2_FOUND) - set(RAFT_FAISS_TARGETS "$,faiss::faiss_avx2>" PARENT_SCOPE) - elseif(PKG_ENABLE_GPU AND PKG_BUILD_STATIC_LIBS) - set(RAFT_FAISS_TARGETS "$,faiss::faiss>" PARENT_SCOPE) - elseif(CXX_AVX2_FOUND) - set(RAFT_FAISS_TARGETS faiss::faiss_avx2 PARENT_SCOPE) - else() - set(RAFT_FAISS_TARGETS faiss::faiss PARENT_SCOPE) - endif() - -endfunction() - - -find_and_configure_faiss( - BUILD_STATIC_LIBS ${RAFT_USE_FAISS_STATIC} - ENABLE_GPU ${RAFT_FAISS_ENABLE_GPU} -) \ No newline at end of file diff --git a/cpp/cmake/thirdparty/get_ggnn.cmake b/cpp/cmake/thirdparty/get_ggnn.cmake deleted file mode 100644 index d8af4971a7..0000000000 --- a/cpp/cmake/thirdparty/get_ggnn.cmake +++ /dev/null @@ -1,50 +0,0 @@ -#============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_ggnn) - - include(${rapids-cmake-dir}/cpm/package_override.cmake) - set(patch_dir "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../patches") - rapids_cpm_package_override("${patch_dir}/ggnn_override.json") - - include("${rapids-cmake-dir}/cpm/detail/package_details.cmake") - rapids_cpm_package_details(ggnn version repository tag shallow exclude) - - include("${rapids-cmake-dir}/cpm/detail/generate_patch_command.cmake") - rapids_cpm_generate_patch_command(ggnn ${version} patch_command) - - rapids_cpm_find( - ggnn ${version} - GLOBAL_TARGETS ggnn::ggnn - CPM_ARGS - GIT_REPOSITORY ${repository} - GIT_TAG ${tag} - GIT_SHALLOW ${shallow} ${patch_command} - EXCLUDE_FROM_ALL ${exclude} - DOWNLOAD_ONLY ON - ) - - include("${rapids-cmake-dir}/cpm/detail/display_patch_status.cmake") - rapids_cpm_display_patch_status(ggnn) - - if(NOT TARGET ggnn::ggnn) - add_library(ggnn INTERFACE) - target_include_directories(ggnn INTERFACE "$") - add_library(ggnn::ggnn ALIAS ggnn) - endif() - -endfunction() -find_and_configure_ggnn() diff --git a/cpp/cmake/thirdparty/get_glog.cmake b/cpp/cmake/thirdparty/get_glog.cmake deleted file mode 100644 index 35a9170f99..0000000000 --- a/cpp/cmake/thirdparty/get_glog.cmake +++ /dev/null @@ -1,48 +0,0 @@ -#============================================================================= -# Copyright (c) 2021-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_glog) - set(oneValueArgs VERSION FORK PINNED_TAG EXCLUDE_FROM_ALL) - cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) - - rapids_cpm_find(glog ${PKG_VERSION} - GLOBAL_TARGETS glog::glog - BUILD_EXPORT_SET raft-exports - INSTALL_EXPORT_SET raft-exports - CPM_ARGS - GIT_REPOSITORY https://github.com/${PKG_FORK}/glog.git - GIT_TAG ${PKG_PINNED_TAG} - EXCLUDE_FROM_ALL ${PKG_EXCLUDE_FROM_ALL} - ) - - if(glog_ADDED) - message(VERBOSE "RAFT: Using glog located in ${glog_SOURCE_DIR}") - else() - message(VERBOSE "RAFT: Using glog located in ${glog_DIR}") - endif() - - -endfunction() - -# Change pinned tag here to test a commit in CI -# To use a different RAFT locally, set the CMake variable -# CPM_glog_SOURCE=/path/to/local/glog -find_and_configure_glog(VERSION 0.6.0 - FORK google - PINNED_TAG v0.6.0 - EXCLUDE_FROM_ALL ON - ) diff --git a/cpp/cmake/thirdparty/get_hnswlib.cmake b/cpp/cmake/thirdparty/get_hnswlib.cmake deleted file mode 100644 index 6ef493336f..0000000000 --- a/cpp/cmake/thirdparty/get_hnswlib.cmake +++ /dev/null @@ -1,88 +0,0 @@ -#============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_hnswlib) - set(oneValueArgs) - - include(${rapids-cmake-dir}/cpm/package_override.cmake) - set(patch_dir "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../patches") - rapids_cpm_package_override("${patch_dir}/hnswlib_override.json") - - include("${rapids-cmake-dir}/cpm/detail/package_details.cmake") - rapids_cpm_package_details(hnswlib version repository tag shallow exclude) - - include("${rapids-cmake-dir}/cpm/detail/generate_patch_command.cmake") - rapids_cpm_generate_patch_command(hnswlib ${version} patch_command) - - rapids_cpm_find( - hnswlib ${version} - GLOBAL_TARGETS hnswlib hnswlib::hnswlib - CPM_ARGS - GIT_REPOSITORY ${repository} - GIT_TAG ${tag} - GIT_SHALLOW ${shallow} ${patch_command} - EXCLUDE_FROM_ALL ${exclude} - DOWNLOAD_ONLY ON - ) - - include("${rapids-cmake-dir}/cpm/detail/display_patch_status.cmake") - rapids_cpm_display_patch_status(hnswlib) - - if(NOT TARGET hnswlib::hnswlib) - add_library(hnswlib INTERFACE ) - add_library(hnswlib::hnswlib ALIAS hnswlib) - target_include_directories(hnswlib INTERFACE - "$" - "$") - endif() - - if(hnswlib_ADDED) - # write build export rules - install(TARGETS hnswlib EXPORT hnswlib-exports) - if(NOT exclude) - install(DIRECTORY "${hnswlib_SOURCE_DIR}/hnswlib/" DESTINATION include/hnswlib) - - # write install export rules - rapids_export( - INSTALL hnswlib - VERSION ${version} - EXPORT_SET hnswlib-exports - GLOBAL_TARGETS hnswlib - NAMESPACE hnswlib::) - endif() - - rapids_export( - BUILD hnswlib - VERSION ${version} - EXPORT_SET hnswlib-exports - GLOBAL_TARGETS hnswlib - NAMESPACE hnswlib::) - - include("${rapids-cmake-dir}/export/package.cmake") - rapids_export_package(INSTALL hnswlib raft-exports VERSION ${version} GLOBAL_TARGETS hnswlib hnswlib::hnswlib) - rapids_export_package(BUILD hnswlib raft-exports VERSION ${version} GLOBAL_TARGETS hnswlib hnswlib::hnswlib) - - - # When using RAFT from the build dir, ensure hnswlib is also found in RAFT's build dir. This - # line adds `set(hnswlib_ROOT "${CMAKE_CURRENT_LIST_DIR}")` to build/raft-dependencies.cmake - include("${rapids-cmake-dir}/export/find_package_root.cmake") - rapids_export_find_package_root( - BUILD hnswlib [=[${CMAKE_CURRENT_LIST_DIR}]=] EXPORT_SET raft-exports - ) - endif() -endfunction() - -find_and_configure_hnswlib() diff --git a/cpp/cmake/thirdparty/get_nlohmann_json.cmake b/cpp/cmake/thirdparty/get_nlohmann_json.cmake deleted file mode 100644 index 5de98a47ce..0000000000 --- a/cpp/cmake/thirdparty/get_nlohmann_json.cmake +++ /dev/null @@ -1,39 +0,0 @@ -#============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_nlohmann_json) - set(oneValueArgs VERSION FORK PINNED_TAG EXCLUDE_FROM_ALL) - cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) - - rapids_cpm_find(nlohmann_json ${PKG_VERSION} - GLOBAL_TARGETS nlohmann_json::nlohmann_json - BUILD_EXPORT_SET raft-bench-ann-exports - INSTALL_EXPORT_SET raft-bench-ann-exports - CPM_ARGS - GIT_REPOSITORY https://github.com/${PKG_FORK}/json.git - GIT_TAG ${PKG_PINNED_TAG} - EXCLUDE_FROM_ALL ${PKG_EXCLUDE_FROM_ALL}) - -endfunction() - -# Change pinned tag here to test a commit in CI -# To use a different RAFT locally, set the CMake variable -# CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_nlohmann_json(VERSION 3.11.2 - FORK nlohmann - PINNED_TAG v3.11.2 - EXCLUDE_FROM_ALL YES) diff --git a/cpp/include/raft/cluster/specializations.cuh b/cpp/include/raft/cluster/specializations.cuh deleted file mode 100644 index e85b05575f..0000000000 --- a/cpp/include/raft/cluster/specializations.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/core/device_mdspan.hpp b/cpp/include/raft/core/device_mdspan.hpp index b4e9f8d1d7..c5241e831b 100644 --- a/cpp/include/raft/core/device_mdspan.hpp +++ b/cpp/include/raft/core/device_mdspan.hpp @@ -210,7 +210,7 @@ auto constexpr make_device_strided_matrix_view(ElementType* ptr, constexpr auto is_row_major = std::is_same_v; constexpr auto is_col_major = std::is_same_v; - assert(is_row_major || is_col_major); + static_assert(is_row_major || is_col_major, "Unsupported layout policy for strided matrix view"); IndexType stride0 = is_row_major ? (stride > 0 ? stride : n_cols) : 1; IndexType stride1 = is_row_major ? 1 : (stride > 0 ? stride : n_rows); diff --git a/cpp/include/raft/core/pinned_container_policy.hpp b/cpp/include/raft/core/pinned_container_policy.hpp index 4870e2c5dc..b661fa8860 100644 --- a/cpp/include/raft/core/pinned_container_policy.hpp +++ b/cpp/include/raft/core/pinned_container_policy.hpp @@ -19,9 +19,9 @@ #include #ifndef RAFT_DISABLE_CUDA -#include -#include -#include +#include + +#include #else #include #endif @@ -30,20 +30,16 @@ namespace raft { #ifndef RAFT_DISABLE_CUDA /** - * @brief A thin wrapper over thrust::host_vector for implementing the pinned mdarray container - * policy. + * @brief A thin wrapper over cudaMallocHost/cudaFreeHost for implementing the pinned mdarray + * container policy. * */ template struct pinned_container { - using value_type = T; - using allocator_type = - thrust::mr::stateless_resource_allocator; + using value_type = std::remove_cv_t; private: - using underlying_container_type = thrust::host_vector; - underlying_container_type data_; + value_type* data_ = nullptr; public: using size_type = std::size_t; @@ -57,21 +53,24 @@ struct pinned_container { using iterator = pointer; using const_iterator = const_pointer; - ~pinned_container() = default; - pinned_container(pinned_container&&) noexcept = default; - pinned_container(pinned_container const& that) : data_{that.data_} {} + explicit pinned_container(std::size_t size) + { + RAFT_CUDA_TRY(cudaMallocHost(&data_, size * sizeof(value_type))); + } + ~pinned_container() noexcept + { + if (data_ != nullptr) { RAFT_CUDA_TRY_NO_THROW(cudaFreeHost(data_)); } + } - auto operator=(pinned_container const& that) -> pinned_container& + pinned_container(pinned_container&& other) { std::swap(this->data_, other.data_); } + pinned_container& operator=(pinned_container&& other) { - data_ = underlying_container_type{that.data_}; + std::swap(this->data_, other.data_); return *this; } - auto operator=(pinned_container&& that) noexcept -> pinned_container& = default; + pinned_container(pinned_container const&) = delete; // Copying disallowed: one array one owner + pinned_container& operator=(pinned_container const&) = delete; - /** - * @brief Ctor that accepts a size. - */ - explicit pinned_container(std::size_t size, allocator_type const& alloc) : data_{size, alloc} {} /** * @brief Index operator that returns a reference to the actual data. */ @@ -84,15 +83,13 @@ struct pinned_container { * @brief Index operator that returns a reference to the actual data. */ template - auto operator[](Index i) const noexcept + auto operator[](Index i) const noexcept -> const_reference { return data_[i]; } - void resize(size_type size) { data_.resize(size, data_.stream()); } - - [[nodiscard]] auto data() noexcept -> pointer { return data_.data().get(); } - [[nodiscard]] auto data() const noexcept -> const_pointer { return data_.data().get(); } + [[nodiscard]] auto data() noexcept -> pointer { return data_; } + [[nodiscard]] auto data() const noexcept -> const_pointer { return data_; } }; /** @@ -102,7 +99,6 @@ template struct pinned_vector_policy { using element_type = ElementType; using container_type = pinned_container; - using allocator_type = typename container_type::allocator_type; using pointer = typename container_type::pointer; using const_pointer = typename container_type::const_pointer; using reference = typename container_type::reference; @@ -110,15 +106,7 @@ struct pinned_vector_policy { using accessor_policy = std::experimental::default_accessor; using const_accessor_policy = std::experimental::default_accessor; - auto create(raft::resources const&, size_t n) -> container_type - { - return container_type(n, allocator_); - } - - constexpr pinned_vector_policy() noexcept(std::is_nothrow_default_constructible_v) - : allocator_{} - { - } + auto create(raft::resources const&, size_t n) -> container_type { return container_type(n); } [[nodiscard]] constexpr auto access(container_type& c, size_t n) const noexcept -> reference { @@ -132,9 +120,6 @@ struct pinned_vector_policy { [[nodiscard]] auto make_accessor_policy() noexcept { return accessor_policy{}; } [[nodiscard]] auto make_accessor_policy() const noexcept { return const_accessor_policy{}; } - - private: - allocator_type allocator_; }; #else template diff --git a/cpp/include/raft/distance/detail/pairwise_distance_epilogue_elementwise.h b/cpp/include/raft/distance/detail/pairwise_distance_epilogue_elementwise.h index 2b2c04b9d3..f6dea987e5 100644 --- a/cpp/include/raft/distance/detail/pairwise_distance_epilogue_elementwise.h +++ b/cpp/include/raft/distance/detail/pairwise_distance_epilogue_elementwise.h @@ -61,6 +61,7 @@ class PairwiseDistanceEpilogueElementwise { using ElementT = ElementT_; static int const kElementsPerAccess = ElementsPerAccess; static int const kCount = kElementsPerAccess; + static bool const kIsSingleSource = true; using DistanceOp = DistanceOp_; using FinalOp = FinalOp_; diff --git a/cpp/include/raft/distance/detail/pairwise_matrix/dispatch-ext.cuh b/cpp/include/raft/distance/detail/pairwise_matrix/dispatch-ext.cuh deleted file mode 100644 index bced721ec8..0000000000 --- a/cpp/include/raft/distance/detail/pairwise_matrix/dispatch-ext.cuh +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include // raft::identity_op -#include // ops::* -#include // ops::has_cutlass_op -#include // rbf_fin_op -#include // pairwise_matrix_params -#include // RAFT_EXPLICIT - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::distance::detail { - -template -void pairwise_matrix_dispatch(OpT distance_op, - IdxT m, - IdxT n, - IdxT k, - const DataT* x, - const DataT* y, - const DataT* x_norm, - const DataT* y_norm, - OutT* out, - FinOpT fin_op, - cudaStream_t stream, - bool is_row_major) RAFT_EXPLICIT; - -}; // namespace raft::distance::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - extern template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -/* - * Hierarchy of instantiations: - * - * This file defines extern template instantiations of the distance kernels. The - * instantiation of the public API is handled in raft/distance/distance-ext.cuh. - * - * After adding an instance here, make sure to also add the instance there. - */ - -// The following two instances are used in the RBF kernel object. Note the use of int64_t for the -// index type. -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, - float, - float, - float, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, - double, - double, - double, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); - -// Rest of instances -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::canberra_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::canberra_distance_op, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::correlation_distance_op, - float, - float, - float, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::correlation_distance_op, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::cosine_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::cosine_distance_op, double, double, double, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::dice_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::dice_distance_op, double, double, double, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hamming_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hamming_distance_op, double, double, double, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hellinger_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hellinger_distance_op, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::jensen_shannon_distance_op, - float, - float, - float, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::jensen_shannon_distance_op, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::kl_divergence_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::kl_divergence_op, double, double, double, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l1_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l1_distance_op, double, double, double, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_exp_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_exp_distance_op, double, double, double, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l_inf_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l_inf_distance_op, double, double, double, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::lp_unexp_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::lp_unexp_distance_op, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::russel_rao_distance_op, float, float, float, raft::identity_op, int); -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::russel_rao_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/include/raft/distance/detail/pairwise_matrix/dispatch.cuh b/cpp/include/raft/distance/detail/pairwise_matrix/dispatch.cuh index 4a52b7ebe7..5bc70ca4aa 100644 --- a/cpp/include/raft/distance/detail/pairwise_matrix/dispatch.cuh +++ b/cpp/include/raft/distance/detail/pairwise_matrix/dispatch.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "dispatch-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "dispatch-ext.cuh" -#endif diff --git a/cpp/include/raft/distance/distance-ext.cuh b/cpp/include/raft/distance/distance-ext.cuh deleted file mode 100644 index dcbfbfdbc3..0000000000 --- a/cpp/include/raft/distance/distance-ext.cuh +++ /dev/null @@ -1,1117 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include // raft::device_matrix_view -#include // raft::identity_op -#include // raft::resources -#include // rbf_fin_op -#include // raft::distance::DistanceType -#include // RAFT_EXPLICIT - -#include // rmm::device_uvector - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft { -namespace distance { - -template -[[deprecated("Use cuVS instead")]] void distance(raft::resources const& handle, - const DataT* x, - const DataT* y, - OutT* dist, - IdxT m, - IdxT n, - IdxT k, - void* workspace, - size_t worksize, - FinalLambda fin_op, - bool isRowMajor = true, - DataT metric_arg = 2.0f) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] void distance(raft::resources const& handle, - const DataT* x, - const DataT* y, - OutT* dist, - IdxT m, - IdxT n, - IdxT k, - void* workspace, - size_t worksize, - bool isRowMajor = true, - DataT metric_arg = 2.0f) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] size_t getWorkspaceSize( - const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) RAFT_EXPLICIT; - -template -size_t getWorkspaceSize(raft::device_matrix_view const& x, - raft::device_matrix_view const& y) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] void distance(raft::resources const& handle, - const DataT* x, - const DataT* y, - OutT* dist, - IdxT m, - IdxT n, - IdxT k, - bool isRowMajor = true, - DataT metric_arg = 2.0f) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] void pairwise_distance(raft::resources const& handle, - const Type* x, - const Type* y, - Type* dist, - IdxT m, - IdxT n, - IdxT k, - rmm::device_uvector& workspace, - raft::distance::DistanceType metric, - bool isRowMajor = true, - Type metric_arg = 2.0f) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] void pairwise_distance(raft::resources const& handle, - const Type* x, - const Type* y, - Type* dist, - IdxT m, - IdxT n, - IdxT k, - raft::distance::DistanceType metric, - bool isRowMajor = true, - Type metric_arg = 2.0f) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] void distance( - raft::resources const& handle, - raft::device_matrix_view const x, - raft::device_matrix_view const y, - raft::device_matrix_view dist, - DataT metric_arg = 2.0f) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] void pairwise_distance( - raft::resources const& handle, - device_matrix_view const x, - device_matrix_view const y, - device_matrix_view dist, - raft::distance::DistanceType metric, - Type metric_arg = 2.0f) RAFT_EXPLICIT; - -}; // namespace distance -}; // namespace raft - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -/* - * Hierarchy of instantiations: - * - * This file defines the extern template instantiations for the public API of - * raft::distance. To improve compile times, the extern template instantiation - * of the distance kernels is handled in - * distance/detail/pairwise_matrix/dispatch-ext.cuh. - * - * After adding an instance here, make sure to also add the instance to - * dispatch-ext.cuh and the corresponding .cu files. - */ - -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, FinalLambda, IdxT) \ - extern template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - size_t worksize, \ - FinalLambda fin_op, \ - bool isRowMajor, \ - DataT metric_arg) - -// The following two instances are used in test/distance/gram.cu. Note the use -// of int64_t for the index type. -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - float, - float, - float, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); - -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_distance - -// Same, but without raft::identity_op -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, IdxT) \ - extern template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - size_t worksize, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, int); - -#undef instantiate_raft_distance_distance - -// Same, but without workspace -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, IdxT) \ - extern template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, int); - -#undef instantiate_raft_distance_distance - -#define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT) \ - extern template size_t raft::distance::getWorkspaceSize( \ - const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) - -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CorrelationExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CosineExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CosineExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HellingerExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HellingerExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Linf, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Linf, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::LpUnexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::LpUnexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, int); - -#undef instantiate_raft_distance_getWorkspaceSize - -#define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT, layout) \ - extern template size_t raft::distance::getWorkspaceSize( \ - raft::device_matrix_view const& x, \ - raft::device_matrix_view const& y) - -// We could consider not taking template parameters for this function. The -// number of instantiations seems a bit excessive.. -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int, raft::layout_f_contiguous); - -#undef instantiate_raft_distance_getWorkspaceSize - -#define instantiate_raft_distance_pairwise_distance(DataT, IdxT) \ - extern template void raft::distance::pairwise_distance(raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - DataT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - rmm::device_uvector& workspace, \ - raft::distance::DistanceType metric, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_pairwise_distance(float, int); -instantiate_raft_distance_pairwise_distance(double, int); - -#undef instantiate_raft_distance_pairwise_distance - -// Same, but without workspace -#define instantiate_raft_distance_pairwise_distance(DataT, IdxT) \ - extern template void raft::distance::pairwise_distance(raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - DataT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - raft::distance::DistanceType metric, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_pairwise_distance(float, int); -instantiate_raft_distance_pairwise_distance(double, int); - -#undef instantiate_raft_distance_pairwise_distance - -// Version with mdspan -#define instantiate_raft_distance_distance(DistT, DataT, AccT, OutT, layout, IdxT) \ - extern template void raft::distance::distance( \ - raft::resources const& handle, \ - raft::device_matrix_view const x, \ - raft::device_matrix_view const y, \ - raft::device_matrix_view dist, \ - DataT metric_arg) - -// Again, we might want to consider reigning in the number of instantiations... -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::LpUnexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::LpUnexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); - -#undef instantiate_raft_distance_distance - -#define instantiate_raft_distance_pairwise_distance(DataT, layout, IdxT) \ - extern template void raft::distance::pairwise_distance( \ - raft::resources const& handle, \ - raft::device_matrix_view const x, \ - raft::device_matrix_view const y, \ - raft::device_matrix_view dist, \ - raft::distance::DistanceType metric, \ - DataT metric_arg) - -instantiate_raft_distance_pairwise_distance(float, raft::layout_c_contiguous, int); -instantiate_raft_distance_pairwise_distance(float, raft::layout_f_contiguous, int); -instantiate_raft_distance_pairwise_distance(double, raft::layout_c_contiguous, int); -instantiate_raft_distance_pairwise_distance(double, raft::layout_f_contiguous, int); - -#undef instantiate_raft_distance_pairwise_distance diff --git a/cpp/include/raft/distance/distance.cuh b/cpp/include/raft/distance/distance.cuh index de70cd4691..dc81e22b18 100644 --- a/cpp/include/raft/distance/distance.cuh +++ b/cpp/include/raft/distance/distance.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "distance-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "distance-ext.cuh" -#endif diff --git a/cpp/include/raft/distance/fused_distance_nn-ext.cuh b/cpp/include/raft/distance/fused_distance_nn-ext.cuh deleted file mode 100644 index 263bbcea81..0000000000 --- a/cpp/include/raft/distance/fused_distance_nn-ext.cuh +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::KeyValuePair -#include // raft::resources -#include // include initialize and reduce operations -#include // RAFT_EXPLICIT - -#include // int64_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft { -namespace distance { - -template -void fusedDistanceNNMinReduce(OutT* min, - const DataT* x, - const DataT* y, - const DataT* xn, - const DataT* yn, - IdxT m, - IdxT n, - IdxT k, - void* workspace, - bool sqrt, - bool initOutBuffer, - bool isRowMajor, - raft::distance::DistanceType metric, - float metric_arg, - cudaStream_t stream) RAFT_EXPLICIT; - -} // namespace distance -} // namespace raft - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_distance_fusedDistanceNNMinReduce(DataT, OutT, IdxT) \ - extern template void raft::distance::fusedDistanceNNMinReduce( \ - OutT * min, \ - const DataT* x, \ - const DataT* y, \ - const DataT* xn, \ - const DataT* yn, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - bool sqrt, \ - bool initOutBuffer, \ - bool isRowMajor, \ - raft::distance::DistanceType metric, \ - float metric_arg, \ - cudaStream_t stream) - -instantiate_raft_distance_fusedDistanceNNMinReduce(float, float, int); -instantiate_raft_distance_fusedDistanceNNMinReduce(float, float, int64_t); - -// We can't have comma's in the macro expansion, so we use the COMMA macro: -#define COMMA , - -instantiate_raft_distance_fusedDistanceNNMinReduce(float, raft::KeyValuePair, int); -instantiate_raft_distance_fusedDistanceNNMinReduce(float, - raft::KeyValuePair, - int64_t); - -#undef COMMA - -#undef instantiate_raft_distance_fusedDistanceNNMinReduce diff --git a/cpp/include/raft/distance/fused_distance_nn.cuh b/cpp/include/raft/distance/fused_distance_nn.cuh index 04c42e49a1..25b1ae01ea 100755 --- a/cpp/include/raft/distance/fused_distance_nn.cuh +++ b/cpp/include/raft/distance/fused_distance_nn.cuh @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY -#include "fused_distance_nn-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "fused_distance_nn-ext.cuh" -#endif +#include "fused_distance_nn-inl.cuh" \ No newline at end of file diff --git a/cpp/include/raft/distance/fused_l2_nn-ext.cuh b/cpp/include/raft/distance/fused_l2_nn-ext.cuh deleted file mode 100644 index d0ac83cd51..0000000000 --- a/cpp/include/raft/distance/fused_l2_nn-ext.cuh +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::KeyValuePair -#include // raft::resources -#include // include initialize and reduce operations -#include // RAFT_EXPLICIT - -#include // int64_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft { -namespace distance { - -template -void fusedL2NNMinReduce(OutT* min, - const DataT* x, - const DataT* y, - const DataT* xn, - const DataT* yn, - IdxT m, - IdxT n, - IdxT k, - void* workspace, - bool sqrt, - bool initOutBuffer, - cudaStream_t stream) RAFT_EXPLICIT; - -} // namespace distance -} // namespace raft - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_distance_fusedL2NNMinReduce(DataT, OutT, IdxT) \ - extern template void raft::distance::fusedL2NNMinReduce(OutT * min, \ - const DataT* x, \ - const DataT* y, \ - const DataT* xn, \ - const DataT* yn, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - bool sqrt, \ - bool initOutBuffer, \ - cudaStream_t stream) - -instantiate_raft_distance_fusedL2NNMinReduce(double, double, int); -instantiate_raft_distance_fusedL2NNMinReduce(double, double, int64_t); -instantiate_raft_distance_fusedL2NNMinReduce(float, float, int); -instantiate_raft_distance_fusedL2NNMinReduce(float, float, int64_t); - -// We can't have comma's in the macro expansion, so we use the COMMA macro: -#define COMMA , - -instantiate_raft_distance_fusedL2NNMinReduce(double, raft::KeyValuePair, int); -instantiate_raft_distance_fusedL2NNMinReduce(double, - raft::KeyValuePair, - int64_t); -instantiate_raft_distance_fusedL2NNMinReduce(float, raft::KeyValuePair, int); -instantiate_raft_distance_fusedL2NNMinReduce(float, - raft::KeyValuePair, - int64_t); - -#undef COMMA - -#undef instantiate_raft_distance_fusedL2NNMinReduce diff --git a/cpp/include/raft/distance/fused_l2_nn.cuh b/cpp/include/raft/distance/fused_l2_nn.cuh index b1a3551323..c17439f942 100644 --- a/cpp/include/raft/distance/fused_l2_nn.cuh +++ b/cpp/include/raft/distance/fused_l2_nn.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "fused_l2_nn-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "fused_l2_nn-ext.cuh" -#endif diff --git a/cpp/include/raft/distance/specializations.cuh b/cpp/include/raft/distance/specializations.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/distance/specializations.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/distance/specializations/distance.cuh b/cpp/include/raft/distance/specializations/distance.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/distance/specializations/distance.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/distance/specializations/fused_l2_nn_min.cuh b/cpp/include/raft/distance/specializations/fused_l2_nn_min.cuh deleted file mode 100644 index e85b05575f..0000000000 --- a/cpp/include/raft/distance/specializations/fused_l2_nn_min.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/linalg/detail/coalesced_reduction-ext.cuh b/cpp/include/raft/linalg/detail/coalesced_reduction-ext.cuh deleted file mode 100644 index 4800f2e3cf..0000000000 --- a/cpp/include/raft/linalg/detail/coalesced_reduction-ext.cuh +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -// The explicit instantiation of raft::linalg::detail::coalescedReduction is not -// forced because there would be too many instances. Instead, we cover the most -// common instantiations with extern template instantiations below. - -#define instantiate_raft_linalg_detail_coalescedReduction( \ - InType, OutType, IdxType, MainLambda, ReduceLambda, FinalLambda) \ - extern template void raft::linalg::detail::coalescedReduction(OutType* dots, \ - const InType* data, \ - IdxType D, \ - IdxType N, \ - OutType init, \ - cudaStream_t stream, \ - bool inplace, \ - MainLambda main_op, \ - ReduceLambda reduce_op, \ - FinalLambda final_op) - -instantiate_raft_linalg_detail_coalescedReduction( - double, double, int, raft::identity_op, raft::min_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - double, double, int, raft::sq_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - double, double, int, raft::sq_op, raft::add_op, raft::sqrt_op); -instantiate_raft_linalg_detail_coalescedReduction( - double, double, int, raft::abs_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - double, double, int, raft::abs_op, raft::max_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, size_t, raft::abs_op, raft::add_op, raft::sqrt_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, int, raft::abs_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, int, raft::identity_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, int, raft::identity_op, raft::min_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, int, raft::sq_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, int, raft::sq_op, raft::add_op, raft::sqrt_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, long, raft::sq_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, size_t, raft::identity_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, size_t, raft::sq_op, raft::add_op, raft::identity_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, size_t, raft::abs_op, raft::max_op, raft::sqrt_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, size_t, raft::sq_op, raft::add_op, raft::sqrt_op); -instantiate_raft_linalg_detail_coalescedReduction( - float, float, unsigned int, raft::sq_op, raft::add_op, raft::identity_op); - -#undef instantiate_raft_linalg_detail_coalescedReduction diff --git a/cpp/include/raft/linalg/detail/coalesced_reduction.cuh b/cpp/include/raft/linalg/detail/coalesced_reduction.cuh index 3e6b17978b..d24c2a7444 100644 --- a/cpp/include/raft/linalg/detail/coalesced_reduction.cuh +++ b/cpp/include/raft/linalg/detail/coalesced_reduction.cuh @@ -16,11 +16,4 @@ #pragma once -// Always include inline definitions of coalesced reduction, because we do not -// force explicit instantion. #include "coalesced_reduction-inl.cuh" - -// Do include the extern template instantiations when possible. -#ifdef RAFT_COMPILED -#include "coalesced_reduction-ext.cuh" -#endif diff --git a/cpp/include/raft/matrix/detail/columnWiseSort.cuh b/cpp/include/raft/matrix/detail/columnWiseSort.cuh index a02621d054..9e94724d3d 100644 --- a/cpp/include/raft/matrix/detail/columnWiseSort.cuh +++ b/cpp/include/raft/matrix/detail/columnWiseSort.cuh @@ -72,12 +72,11 @@ RAFT_KERNEL devOffsetKernel(T* in, T value, int n_times) } // block level radix sort - can only sort as much data we can fit within shared memory -template < - typename InType, - typename OutType, - int BLOCK_SIZE, - int ITEMS_PER_THREAD, - typename std::enable_if::IsValid, InType>::type* = nullptr> +template ::IsValid, bool> = true> RAFT_KERNEL __launch_bounds__(1024, 1) devKeyValSortColumnPerRow(const InType* inputKeys, InType* outputKeys, OutType* inputVals, @@ -120,12 +119,11 @@ RAFT_KERNEL __launch_bounds__(1024, 1) devKeyValSortColumnPerRow(const InType* i } } -template < - typename InType, - typename OutType, - int BLOCK_SIZE, - int ITEMS_PER_THREAD, - typename std::enable_if::IsValid), InType>::type* = nullptr> +template ::IsValid, bool> = true> RAFT_KERNEL devKeyValSortColumnPerRow(const InType* inputKeys, InType* outputKeys, OutType* inputVals, diff --git a/cpp/include/raft/matrix/detail/select_k-ext.cuh b/cpp/include/raft/matrix/detail/select_k-ext.cuh deleted file mode 100644 index 6db1a5acac..0000000000 --- a/cpp/include/raft/matrix/detail/select_k-ext.cuh +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include // RAFT_EXPLICIT - -#include // __half - -#include // uint32_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::matrix::detail { - -template -void select_k(raft::resources const& handle, - const T* in_val, - const IdxT* in_idx, - size_t batch_size, - size_t len, - int k, - T* out_val, - IdxT* out_idx, - bool select_min, - bool sorted = false, - SelectAlgo algo = SelectAlgo::kAuto, - const IdxT* len_i = nullptr) RAFT_EXPLICIT; -} // namespace raft::matrix::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - extern template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) -instantiate_raft_matrix_detail_select_k(__half, uint32_t); -instantiate_raft_matrix_detail_select_k(__half, int64_t); -instantiate_raft_matrix_detail_select_k(float, int64_t); -instantiate_raft_matrix_detail_select_k(float, uint32_t); -// needed for brute force knn -instantiate_raft_matrix_detail_select_k(float, int); -// We did not have these two for double before, but there are tests for them. We -// therefore include them here. -instantiate_raft_matrix_detail_select_k(double, int64_t); -instantiate_raft_matrix_detail_select_k(double, uint32_t); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/include/raft/matrix/detail/select_k.cuh b/cpp/include/raft/matrix/detail/select_k.cuh index 711169984b..31a4b54a94 100644 --- a/cpp/include/raft/matrix/detail/select_k.cuh +++ b/cpp/include/raft/matrix/detail/select_k.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "select_k-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "select_k-ext.cuh" -#endif diff --git a/cpp/include/raft/matrix/specializations/detail/select_k.cuh b/cpp/include/raft/matrix/specializations/detail/select_k.cuh deleted file mode 100644 index c61d65dcaf..0000000000 --- a/cpp/include/raft/matrix/specializations/detail/select_k.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/ball_cover-ext.cuh b/cpp/include/raft/neighbors/ball_cover-ext.cuh deleted file mode 100644 index 2a83a57749..0000000000 --- a/cpp/include/raft/neighbors/ball_cover-ext.cuh +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include // raft::distance::DistanceType -#include // BallCoverIndex -#include // RAFT_EXPLICIT - -#include // uint32_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors::ball_cover { - -template -void build_index(raft::resources const& handle, - BallCoverIndex& index) RAFT_EXPLICIT; - -template -void all_knn_query(raft::resources const& handle, - BallCoverIndex& index, - int_t k, - idx_t* inds, - value_t* dists, - bool perform_post_filtering = true, - float weight = 1.0) RAFT_EXPLICIT; - -template -void all_knn_query(raft::resources const& handle, - BallCoverIndex& index, - raft::device_matrix_view inds, - raft::device_matrix_view dists, - int_t k, - bool perform_post_filtering = true, - float weight = 1.0) RAFT_EXPLICIT; - -template -void knn_query(raft::resources const& handle, - const BallCoverIndex& index, - int_t k, - const value_t* query, - int_t n_query_pts, - idx_t* inds, - value_t* dists, - bool perform_post_filtering = true, - float weight = 1.0) RAFT_EXPLICIT; - -template -void knn_query(raft::resources const& handle, - const BallCoverIndex& index, - raft::device_matrix_view query, - raft::device_matrix_view inds, - raft::device_matrix_view dists, - int_t k, - bool perform_post_filtering = true, - float weight = 1.0) RAFT_EXPLICIT; - -template -void eps_nn(raft::resources const& handle, - const BallCoverIndex& index, - raft::device_matrix_view adj, - raft::device_vector_view vd, - raft::device_matrix_view query, - value_t eps) RAFT_EXPLICIT; - -template -void eps_nn(raft::resources const& handle, - const BallCoverIndex& index, - raft::device_vector_view adj_ia, - raft::device_vector_view adj_ja, - raft::device_vector_view vd, - raft::device_matrix_view query, - value_t eps, - std::optional> max_k = std::nullopt) - RAFT_EXPLICIT; - -} // namespace raft::neighbors::ball_cover - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_neighbors_ball_cover(idx_t, value_t, int_t, matrix_idx_t) \ - extern template void \ - raft::neighbors::ball_cover::build_index( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& index); \ - \ - extern template void \ - raft::neighbors::ball_cover::all_knn_query( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& index, \ - int_t k, \ - idx_t* inds, \ - value_t* dists, \ - bool perform_post_filtering, \ - float weight); \ - \ - extern template void raft::neighbors::ball_cover::eps_nn( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_matrix_view adj, \ - raft::device_vector_view vd, \ - raft::device_matrix_view query, \ - value_t eps); \ - \ - extern template void raft::neighbors::ball_cover::eps_nn( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_vector_view adj_ia, \ - raft::device_vector_view adj_ja, \ - raft::device_vector_view vd, \ - raft::device_matrix_view query, \ - value_t eps, \ - std::optional> max_k); \ - \ - extern template void \ - raft::neighbors::ball_cover::all_knn_query( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_matrix_view inds, \ - raft::device_matrix_view dists, \ - int_t k, \ - bool perform_post_filtering, \ - float weight); \ - \ - extern template void raft::neighbors::ball_cover::knn_query( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - int_t k, \ - const value_t* query, \ - int_t n_query_pts, \ - idx_t* inds, \ - value_t* dists, \ - bool perform_post_filtering, \ - float weight); \ - \ - extern template void \ - raft::neighbors::ball_cover::knn_query( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_matrix_view query, \ - raft::device_matrix_view inds, \ - raft::device_matrix_view dists, \ - int_t k, \ - bool perform_post_filtering, \ - float weight); - -instantiate_raft_neighbors_ball_cover(int64_t, float, int64_t, int64_t); - -#undef instantiate_raft_neighbors_ball_cover diff --git a/cpp/include/raft/neighbors/ball_cover.cuh b/cpp/include/raft/neighbors/ball_cover.cuh index 09938020b9..cc74113f7e 100644 --- a/cpp/include/raft/neighbors/ball_cover.cuh +++ b/cpp/include/raft/neighbors/ball_cover.cuh @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "ball_cover-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "ball_cover-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/brute_force-ext.cuh b/cpp/include/raft/neighbors/brute_force-ext.cuh deleted file mode 100644 index 4055c253c8..0000000000 --- a/cpp/include/raft/neighbors/brute_force-ext.cuh +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Copyright (c) 2020-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::device_matrix_view -#include // raft::identity_op -#include // raft::resources -#include // raft::distance::DistanceType -#include -#include // RAFT_EXPLICIT - -#include - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors::brute_force { - -template -inline void knn_merge_parts( - raft::resources const& handle, - raft::device_matrix_view in_keys, - raft::device_matrix_view in_values, - raft::device_matrix_view out_keys, - raft::device_matrix_view out_values, - size_t n_samples, - std::optional> translations = std::nullopt) RAFT_EXPLICIT; - -template -index build(raft::resources const& res, - mdspan, row_major, Accessor> dataset, - raft::distance::DistanceType metric = distance::DistanceType::L2Unexpanded, - T metric_arg = 0.0) RAFT_EXPLICIT; - -template -index build(raft::resources const& res, - index_params const& params, - mdspan, row_major, Accessor> dataset) RAFT_EXPLICIT; - -template -void search(raft::resources const& res, - const index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances) RAFT_EXPLICIT; - -template -void search(raft::resources const& res, - search_params const& params, - const index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances) RAFT_EXPLICIT; - -template -void knn(raft::resources const& handle, - std::vector> index, - raft::device_matrix_view search, - raft::device_matrix_view indices, - raft::device_matrix_view distances, - distance::DistanceType metric = distance::DistanceType::L2Unexpanded, - std::optional metric_arg = std::make_optional(2.0f), - std::optional global_id_offset = std::nullopt, - epilogue_op distance_epilogue = raft::identity_op()) RAFT_EXPLICIT; - -template -void fused_l2_knn(raft::resources const& handle, - raft::device_matrix_view index, - raft::device_matrix_view query, - raft::device_matrix_view out_inds, - raft::device_matrix_view out_dists, - raft::distance::DistanceType metric) RAFT_EXPLICIT; - -} // namespace raft::neighbors::brute_force - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -// No extern template for raft::neighbors::brute_force::knn_merge_parts - -#define instantiate_raft_neighbors_brute_force_knn( \ - idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \ - extern template void raft::neighbors::brute_force:: \ - knn( \ - raft::resources const& handle, \ - std::vector> index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric, \ - std::optional metric_arg, \ - std::optional global_id_offset, \ - epilogue_op distance_epilogue); - -instantiate_raft_neighbors_brute_force_knn( - int64_t, float, uint32_t, raft::row_major, raft::row_major, raft::identity_op); -instantiate_raft_neighbors_brute_force_knn( - int64_t, float, int64_t, raft::row_major, raft::row_major, raft::identity_op); -instantiate_raft_neighbors_brute_force_knn( - int, float, int, raft::row_major, raft::row_major, raft::identity_op); -instantiate_raft_neighbors_brute_force_knn( - uint32_t, float, uint32_t, raft::row_major, raft::row_major, raft::identity_op); - -#undef instantiate_raft_neighbors_brute_force_knn - -namespace raft::neighbors::brute_force { - -extern template void search( - raft::resources const& res, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -extern template void search( - raft::resources const& res, - search_params const& params, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -extern template void search( - raft::resources const& res, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -extern template void search( - raft::resources const& res, - search_params const& params, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -extern template raft::neighbors::brute_force::index build( - raft::resources const& res, - raft::device_matrix_view dataset, - raft::distance::DistanceType metric, - float metric_arg); - -extern template raft::neighbors::brute_force::index build( - raft::resources const& res, - index_params const& params, - raft::device_matrix_view dataset); - -extern template raft::neighbors::brute_force::index build( - raft::resources const& res, - raft::host_matrix_view dataset, - raft::distance::DistanceType metric, - float metric_arg); - -extern template raft::neighbors::brute_force::index build( - raft::resources const& res, - index_params const& params, - raft::host_matrix_view dataset); -} // namespace raft::neighbors::brute_force - -#define instantiate_raft_neighbors_brute_force_fused_l2_knn( \ - value_t, idx_t, idx_layout, query_layout) \ - extern template void raft::neighbors::brute_force::fused_l2_knn( \ - raft::resources const& handle, \ - raft::device_matrix_view index, \ - raft::device_matrix_view query, \ - raft::device_matrix_view out_inds, \ - raft::device_matrix_view out_dists, \ - raft::distance::DistanceType metric); - -instantiate_raft_neighbors_brute_force_fused_l2_knn(float, - int64_t, - raft::row_major, - raft::row_major) - -#undef instantiate_raft_neighbors_brute_force_fused_l2_knn diff --git a/cpp/include/raft/neighbors/brute_force.cuh b/cpp/include/raft/neighbors/brute_force.cuh index 331ea55540..d01b7052cb 100644 --- a/cpp/include/raft/neighbors/brute_force.cuh +++ b/cpp/include/raft/neighbors/brute_force.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,12 @@ * limitations under the License. */ #pragma once -#include - -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "brute_force-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "brute_force-ext.cuh" -#endif #include +#include + namespace raft::neighbors::brute_force { /** * @brief Make a brute force query over batches of k diff --git a/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel-ext.cuh b/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel-ext.cuh deleted file mode 100644 index 35f4f0e1c9..0000000000 --- a/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel-ext.cuh +++ /dev/null @@ -1,418 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include // none_cagra_sample_filter -#include // RAFT_EXPLICIT - -#include - -namespace raft::neighbors::cagra::detail { -namespace multi_cta_search { - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -template -void select_and_run( - DATASET_DESCRIPTOR_T dataset_desc, - raft::device_matrix_view graph, - typename DATASET_DESCRIPTOR_T::INDEX_T* const topk_indices_ptr, - typename DATASET_DESCRIPTOR_T::DISTANCE_T* const topk_distances_ptr, - const typename DATASET_DESCRIPTOR_T::DATA_T* const queries_ptr, - const uint32_t num_queries, - const typename DATASET_DESCRIPTOR_T::INDEX_T* dev_seed_ptr, - uint32_t* const num_executed_iterations, - uint32_t topk, - uint32_t block_size, - uint32_t result_buffer_size, - uint32_t smem_size, - int64_t hash_bitlen, - typename DATASET_DESCRIPTOR_T::INDEX_T* hashmap_ptr, - uint32_t num_cta_per_query, - uint32_t num_random_samplings, - uint64_t rand_xor_mask, - uint32_t num_seeds, - size_t itopk_size, - size_t search_width, - size_t min_iterations, - size_t max_iterations, - SAMPLE_FILTER_T sample_filter, - raft::distance::DistanceType metric, - cudaStream_t stream) RAFT_EXPLICIT; -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_kernel_selection( \ - TEAM_SIZE, MAX_DATASET_DIM, DATA_T, INDEX_T, DISTANCE_T, SAMPLE_FILTER_T) \ - extern template void select_and_run< \ - TEAM_SIZE, \ - MAX_DATASET_DIM, \ - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, \ - SAMPLE_FILTER_T>( \ - raft::neighbors::cagra::detail::standard_dataset_descriptor_t \ - dataset_desc, \ - raft::device_matrix_view graph, \ - INDEX_T* const topk_indices_ptr, \ - DISTANCE_T* const topk_distances_ptr, \ - const DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t block_size, \ - uint32_t result_buffer_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - INDEX_T* hashmap_ptr, \ - uint32_t num_cta_per_query, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -instantiate_kernel_selection( - 32, 1024, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 8, 128, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 16, 256, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 512, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 1024, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 8, 128, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 16, 256, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 512, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 1024, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 8, 128, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 16, 256, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 512, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 1024, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 8, 128, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 16, 256, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection( - 32, 512, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); - -#undef instantiate_kernel_selection - -#define instantiate_q_kernel_selection(TEAM_SIZE, \ - MAX_DATASET_DIM, \ - CODE_BOOK_T, \ - PQ_BITS, \ - PQ_CODE_BOOK_DIM, \ - DATA_T, \ - INDEX_T, \ - DISTANCE_T, \ - SAMPLE_FILTER_T) \ - extern template void \ - select_and_run, \ - SAMPLE_FILTER_T>( \ - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t dataset_desc, \ - raft::device_matrix_view graph, \ - INDEX_T* const topk_indices_ptr, \ - DISTANCE_T* const topk_distances_ptr, \ - const DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t block_size, \ - uint32_t result_buffer_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - INDEX_T* hashmap_ptr, \ - uint32_t num_cta_per_query, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -instantiate_q_kernel_selection( - 8, 128, half, 8, 2, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection( - 16, 256, half, 8, 2, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection( - 32, 512, half, 8, 2, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 2, - half, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection( - 8, 128, half, 8, 4, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection( - 16, 256, half, 8, 4, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection( - 32, 512, half, 8, 4, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 4, - half, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_kernel_selection( - 8, 128, half, 8, 2, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(16, - 256, - half, - 8, - 2, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 512, - half, - 8, - 2, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 2, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection( - 8, 128, half, 8, 4, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(16, - 256, - half, - 8, - 4, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 512, - half, - 8, - 4, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 4, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_kernel_selection(8, - 128, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(16, - 256, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 512, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(8, - 128, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(16, - 256, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 512, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_kernel_selection(8, - 128, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(16, - 256, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 512, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(8, - 128, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(16, - 256, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 512, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_kernel_selection(32, - 1024, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -#undef instantiate_q_kernel_selection -} // namespace multi_cta_search -} // namespace raft::neighbors::cagra::detail diff --git a/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel.cuh b/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel.cuh index e003907292..3dc0745e6d 100644 --- a/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/search_multi_cta_kernel.cuh @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "search_multi_cta_kernel-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "search_multi_cta_kernel-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel-ext.cuh b/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel-ext.cuh deleted file mode 100644 index 510219ab5d..0000000000 --- a/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel-ext.cuh +++ /dev/null @@ -1,602 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include // RAFT_EXPLICIT - -#include - -namespace raft::neighbors::cagra::detail { -namespace single_cta_search { - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -template -void select_and_run( // raft::resources const& res, - DATASET_DESCRIPTOR_T dataset_desc, - raft::device_matrix_view graph, - typename DATASET_DESCRIPTOR_T::INDEX_T* const topk_indices_ptr, // [num_queries, topk] - typename DATASET_DESCRIPTOR_T::DISTANCE_T* const topk_distances_ptr, // [num_queries, topk] - const typename DATASET_DESCRIPTOR_T::DATA_T* const queries_ptr, // [num_queries, dataset_dim] - const uint32_t num_queries, - const typename DATASET_DESCRIPTOR_T::INDEX_T* dev_seed_ptr, // [num_queries, num_seeds] - uint32_t* const num_executed_iterations, // [num_queries,] - uint32_t topk, - uint32_t num_itopk_candidates, - uint32_t block_size, - uint32_t smem_size, - int64_t hash_bitlen, - typename DATASET_DESCRIPTOR_T::INDEX_T* hashmap_ptr, - size_t small_hash_bitlen, - size_t small_hash_reset_interval, - uint32_t num_random_samplings, - uint64_t rand_xor_mask, - uint32_t num_seeds, - size_t itopk_size, - size_t search_width, - size_t min_iterations, - size_t max_iterations, - SAMPLE_FILTER_T sample_filter, - raft::distance::DistanceType metric, - cudaStream_t stream) RAFT_EXPLICIT; - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_single_cta_select_and_run( \ - TEAM_SIZE, MAX_DATASET_DIM, DATA_T, INDEX_T, DISTANCE_T, SAMPLE_FILTER_T) \ - extern template void select_and_run< \ - TEAM_SIZE, \ - MAX_DATASET_DIM, \ - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, \ - SAMPLE_FILTER_T>( \ - raft::neighbors::cagra::detail::standard_dataset_descriptor_t \ - dataset, \ - raft::device_matrix_view graph, \ - INDEX_T* const topk_indices_ptr, \ - DISTANCE_T* const topk_distances_ptr, \ - const DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t num_itopk_candidates, \ - uint32_t block_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - INDEX_T* hashmap_ptr, \ - size_t small_hash_bitlen, \ - size_t small_hash_reset_interval, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -instantiate_single_cta_select_and_run( - 32, 1024, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 8, 128, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 16, 256, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 512, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 1024, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 8, 128, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 16, 256, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 512, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 1024, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 8, 128, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 16, 256, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 512, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 1024, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 8, 128, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 16, 256, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run( - 32, 512, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); - -#undef instantiate_single_cta_select_and_run - -#define instantiate_q_single_cta_select_and_run(TEAM_SIZE, \ - MAX_DATASET_DIM, \ - CODE_BOOK_T, \ - PQ_BITS, \ - PQ_CODE_BOOK_DIM, \ - DATA_T, \ - INDEX_T, \ - DISTANCE_T, \ - SAMPLE_FILTER_T) \ - extern template void \ - select_and_run, \ - SAMPLE_FILTER_T>( \ - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t dataset, \ - raft::device_matrix_view graph, \ - INDEX_T* const topk_indices_ptr, \ - DISTANCE_T* const topk_distances_ptr, \ - const DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t num_itopk_candidates, \ - uint32_t block_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - INDEX_T* hashmap_ptr, \ - size_t small_hash_bitlen, \ - size_t small_hash_reset_interval, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 2, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 16, 256, half, 8, 2, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 512, half, 8, 2, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 2, - half, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 4, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 16, 256, half, 8, 4, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 512, half, 8, 4, half, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 4, - half, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 2, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 2, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 2, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 2, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 4, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 4, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 4, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 4, - float, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 2, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 16, 256, half, 8, 2, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 512, half, 8, 2, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 1024, half, 8, 2, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 4, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 16, 256, half, 8, 4, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 512, half, 8, 4, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 1024, half, 8, 4, half, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 2, float, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 16, 256, half, 8, 2, float, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 512, half, 8, 2, float, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 2, - float, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 4, float, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 16, 256, half, 8, 4, float, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 32, 512, half, 8, 4, float, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 4, - float, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_single_cta_select_and_run(8, - 128, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 2, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(8, - 128, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 4, - uint8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_single_cta_select_and_run(8, - 128, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 2, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(8, - 128, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 4, - int8_t, - uint32_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_single_cta_select_and_run(8, - 128, - half, - 8, - 2, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 2, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 2, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 2, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(8, - 128, - half, - 8, - 4, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 4, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 4, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 4, - uint8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 2, int8_t, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 2, - int8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 2, - int8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 2, - int8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run( - 8, 128, half, 8, 4, int8_t, int64_t, float, raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(16, - 256, - half, - 8, - 4, - int8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 512, - half, - 8, - 4, - int8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_q_single_cta_select_and_run(32, - 1024, - half, - 8, - 4, - int8_t, - int64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -#undef instantiate_q_single_cta_select_and_run - -} // namespace single_cta_search -} // namespace raft::neighbors::cagra::detail diff --git a/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel.cuh b/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel.cuh index 1d8fd8e30a..3e72fbf8e8 100644 --- a/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/search_single_cta_kernel.cuh @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "search_single_cta_kernel-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "search_single_cta_kernel-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan-ext.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan-ext.cuh deleted file mode 100644 index 140a9f17c8..0000000000 --- a/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan-ext.cuh +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::neighbors::ivf_flat::index -#include // none_ivf_sample_filter -#include // RAFT_EXPLICIT - -#include // rmm:cuda_stream_view - -#include - -#include // uintX_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors::ivf_flat::detail { - -auto RAFT_WEAK_FUNCTION is_local_topk_feasible(uint32_t k) -> bool; - -template -void ivfflat_interleaved_scan(const raft::neighbors::ivf_flat::index& index, - const T* queries, - const uint32_t* coarse_query_results, - const uint32_t n_queries, - const uint32_t queries_offset, - const raft::distance::DistanceType metric, - const uint32_t n_probes, - const uint32_t k, - const uint32_t max_samples, - const uint32_t* chunk_indices, - const bool select_min, - IvfSampleFilterT sample_filter, - uint32_t* neighbors, - float* distances, - uint32_t& grid_dim_x, - rmm::cuda_stream_view stream) RAFT_EXPLICIT; - -} // namespace raft::neighbors::ivf_flat::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( \ - T, AccT, IdxT, IvfSampleFilterT) \ - extern template void \ - raft::neighbors::ivf_flat::detail::ivfflat_interleaved_scan( \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - const uint32_t* coarse_query_results, \ - const uint32_t n_queries, \ - const uint32_t queries_offset, \ - const raft::distance::DistanceType metric, \ - const uint32_t n_probes, \ - const uint32_t k, \ - const uint32_t max_samples, \ - const uint32_t* chunk_indices, \ - const bool select_min, \ - IvfSampleFilterT sample_filter, \ - uint32_t* neighbors, \ - float* distances, \ - uint32_t& grid_dim_x, \ - rmm::cuda_stream_view stream) - -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - float, float, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - half, half, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - int8_t, int32_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - uint8_t, uint32_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan.cuh index 63f341dd9a..11d7da851a 100644 --- a/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_flat_interleaved_scan.cuh @@ -16,10 +16,4 @@ #pragma once -#if !defined(RAFT_EXPLICIT_INSTANTIATE_ONLY) #include "ivf_flat_interleaved_scan-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "ivf_flat_interleaved_scan-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_search-ext.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_search-ext.cuh deleted file mode 100644 index c14b0e810f..0000000000 --- a/cpp/include/raft/neighbors/detail/ivf_flat_search-ext.cuh +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::neighbors::ivf_flat::index -#include // none_ivf_sample_filter -#include // RAFT_EXPLICIT - -#include - -#include - -#include // uintX_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors::ivf_flat::detail { - -template -void search(raft::resources const& handle, - const search_params& params, - const raft::neighbors::ivf_flat::index& index, - const T* queries, - uint32_t n_queries, - uint32_t k, - IdxT* neighbors, - float* distances, - rmm::device_async_resource_ref mr, - IvfSampleFilterT sample_filter = IvfSampleFilterT()) RAFT_EXPLICIT; - -} // namespace raft::neighbors::ivf_flat::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_neighbors_ivf_flat_detail_search(T, IdxT, IvfSampleFilterT) \ - extern template void raft::neighbors::ivf_flat::detail::search( \ - raft::resources const& handle, \ - const search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances, \ - rmm::device_async_resource_ref mr, \ - IvfSampleFilterT sample_filter) - -instantiate_raft_neighbors_ivf_flat_detail_search( - float, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_search( - half, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_search( - int8_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_search( - uint8_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef instantiate_raft_neighbors_ivf_flat_detail_search diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_search.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_search.cuh index 7b03ebeab6..56e58bac27 100644 --- a/cpp/include/raft/neighbors/detail/ivf_flat_search.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_flat_search.cuh @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "ivf_flat_search-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "ivf_flat_search-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity-ext.cuh b/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity-ext.cuh deleted file mode 100644 index 5e1a9b46d6..0000000000 --- a/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity-ext.cuh +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // RAFT_WEAK_FUNCTION -#include // raft::distance::DistanceType -#include // raft::neighbors::ivf_pq::detail::fp_8bit -#include // raft::neighbors::ivf_pq::codebook_gen -#include // none_ivf_sample_filter -#include // RAFT_EXPLICIT - -#include // rmm::cuda_stream_view - -#include // __half - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors::ivf_pq::detail { - -// is_local_topk_feasible is not inline here, because we would have to define it -// here as well. That would run the risk of the definitions here and in the -// -inl.cuh header diverging. -auto RAFT_WEAK_FUNCTION is_local_topk_feasible(uint32_t k, uint32_t n_probes, uint32_t n_queries) - -> bool; - -template -RAFT_KERNEL compute_similarity_kernel(uint32_t dim, - uint32_t n_probes, - uint32_t pq_dim, - uint32_t n_queries, - uint32_t queries_offset, - distance::DistanceType metric, - codebook_gen codebook_kind, - uint32_t topk, - uint32_t max_samples, - const float* cluster_centers, - const float* pq_centers, - const uint8_t* const* pq_dataset, - const uint32_t* cluster_labels, - const uint32_t* _chunk_indices, - const float* queries, - const uint32_t* index_list, - float* query_kths, - IvfSampleFilterT sample_filter, - LutT* lut_scores, - OutT* _out_scores, - uint32_t* _out_indices) RAFT_EXPLICIT; - -// The signature of the kernel defined by a minimal set of template parameters -template -using compute_similarity_kernel_t = - decltype(&compute_similarity_kernel); - -template -struct selected { - compute_similarity_kernel_t kernel; - dim3 grid_dim; - dim3 block_dim; - size_t smem_size; - size_t device_lut_size; -}; - -template -void compute_similarity_run(selected s, - rmm::cuda_stream_view stream, - uint32_t dim, - uint32_t n_probes, - uint32_t pq_dim, - uint32_t n_queries, - uint32_t queries_offset, - distance::DistanceType metric, - codebook_gen codebook_kind, - uint32_t topk, - uint32_t max_samples, - const float* cluster_centers, - const float* pq_centers, - const uint8_t* const* pq_dataset, - const uint32_t* cluster_labels, - const uint32_t* _chunk_indices, - const float* queries, - const uint32_t* index_list, - float* query_kths, - IvfSampleFilterT sample_filter, - LutT* lut_scores, - OutT* _out_scores, - uint32_t* _out_indices) RAFT_EXPLICIT; - -/** - * Use heuristics to choose an optimal instance of the search kernel. - * It selects among a few kernel variants (with/out using shared mem for - * lookup tables / precomputed distances) and tries to choose the block size - * to maximize kernel occupancy. - * - * @param manage_local_topk - * whether use the fused calculate+select or just calculate the distances for each - * query and probed cluster. - * - * @param locality_hint - * beyond this limit do not consider increasing the number of active blocks per SM - * would improve locality anymore. - */ -template -auto compute_similarity_select(const cudaDeviceProp& dev_props, - bool manage_local_topk, - int locality_hint, - double preferred_shmem_carveout, - uint32_t pq_bits, - uint32_t pq_dim, - uint32_t precomp_data_count, - uint32_t n_queries, - uint32_t n_probes, - uint32_t topk) - -> selected RAFT_EXPLICIT; - -} // namespace raft::neighbors::ivf_pq::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( \ - OutT, LutT, IvfSampleFilterT) \ - extern template auto \ - raft::neighbors::ivf_pq::detail::compute_similarity_select( \ - const cudaDeviceProp& dev_props, \ - bool manage_local_topk, \ - int locality_hint, \ - double preferred_shmem_carveout, \ - uint32_t pq_bits, \ - uint32_t pq_dim, \ - uint32_t precomp_data_count, \ - uint32_t n_queries, \ - uint32_t n_probes, \ - uint32_t topk) \ - ->raft::neighbors::ivf_pq::detail::selected; \ - \ - extern template void \ - raft::neighbors::ivf_pq::detail::compute_similarity_run( \ - raft::neighbors::ivf_pq::detail::selected s, \ - rmm::cuda_stream_view stream, \ - uint32_t dim, \ - uint32_t n_probes, \ - uint32_t pq_dim, \ - uint32_t n_queries, \ - uint32_t queries_offset, \ - raft::distance::DistanceType metric, \ - raft::neighbors::ivf_pq::codebook_gen codebook_kind, \ - uint32_t topk, \ - uint32_t max_samples, \ - const float* cluster_centers, \ - const float* pq_centers, \ - const uint8_t* const* pq_dataset, \ - const uint32_t* cluster_labels, \ - const uint32_t* _chunk_indices, \ - const float* queries, \ - const uint32_t* index_list, \ - float* query_kths, \ - IvfSampleFilterT sample_filter, \ - LutT* lut_scores, \ - OutT* _out_scores, \ - uint32_t* _out_indices); - -#define COMMA , -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); - -#undef COMMA - -#undef instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select diff --git a/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity.cuh b/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity.cuh index d987c0d4ed..467d389d38 100644 --- a/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_pq_compute_similarity.cuh @@ -16,10 +16,4 @@ #pragma once -#if !defined(RAFT_EXPLICIT_INSTANTIATE_ONLY) #include "ivf_pq_compute_similarity-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "ivf_pq_compute_similarity-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/ivf_flat-ext.cuh b/cpp/include/raft/neighbors/ivf_flat-ext.cuh deleted file mode 100644 index 12ab0dc3a6..0000000000 --- a/cpp/include/raft/neighbors/ivf_flat-ext.cuh +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::device_matrix_view -#include // raft::resources -#include -#include // raft::neighbors::ivf_flat::index -#include // RAFT_EXPLICIT - -#include - -#include // int64_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors::ivf_flat { - -template -auto build(raft::resources const& handle, - const index_params& params, - const T* dataset, - IdxT n_rows, - uint32_t dim) -> index RAFT_EXPLICIT; - -template -auto build(raft::resources const& handle, - const index_params& params, - raft::device_matrix_view dataset) - -> index RAFT_EXPLICIT; - -template -void build(raft::resources const& handle, - const index_params& params, - raft::device_matrix_view dataset, - raft::neighbors::ivf_flat::index& idx) RAFT_EXPLICIT; - -template -auto build(raft::resources const& handle, - const index_params& params, - raft::host_matrix_view dataset) - -> index RAFT_EXPLICIT; - -template -void build(raft::resources const& handle, - const index_params& params, - raft::host_matrix_view dataset, - raft::neighbors::ivf_flat::index& idx) RAFT_EXPLICIT; - -template -auto extend(raft::resources const& handle, - const index& orig_index, - const T* new_vectors, - const IdxT* new_indices, - IdxT n_rows) -> index RAFT_EXPLICIT; - -template -auto extend(raft::resources const& handle, - raft::device_matrix_view new_vectors, - std::optional> new_indices, - const index& orig_index) -> index RAFT_EXPLICIT; - -template -void extend(raft::resources const& handle, - index* index, - const T* new_vectors, - const IdxT* new_indices, - IdxT n_rows) RAFT_EXPLICIT; - -template -void extend(raft::resources const& handle, - raft::device_matrix_view new_vectors, - std::optional> new_indices, - index* index) RAFT_EXPLICIT; - -template -auto extend(raft::resources const& handle, - raft::host_matrix_view new_vectors, - std::optional> new_indices, - const raft::neighbors::ivf_flat::index& orig_index) - -> raft::neighbors::ivf_flat::index RAFT_EXPLICIT; - -template -void extend(raft::resources const& handle, - raft::host_matrix_view new_vectors, - std::optional> new_indices, - index* index) RAFT_EXPLICIT; - -template -void search_with_filtering(raft::resources const& handle, - const search_params& params, - const index& index, - const T* queries, - uint32_t n_queries, - uint32_t k, - IdxT* neighbors, - float* distances, - rmm::device_async_resource_ref mr, - IvfSampleFilterT sample_filter = IvfSampleFilterT()) RAFT_EXPLICIT; - -template -void search(raft::resources const& handle, - const search_params& params, - const index& index, - const T* queries, - uint32_t n_queries, - uint32_t k, - IdxT* neighbors, - float* distances, - rmm::device_async_resource_ref mr) RAFT_EXPLICIT; - -template -void search_with_filtering(raft::resources const& handle, - const search_params& params, - const index& index, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances, - IvfSampleFilterT sample_filter = IvfSampleFilterT()) RAFT_EXPLICIT; - -template -void search(raft::resources const& handle, - const search_params& params, - const index& index, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances) RAFT_EXPLICIT; - -} // namespace raft::neighbors::ivf_flat - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_neighbors_ivf_flat_build(T, IdxT) \ - extern template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_flat::index; \ - \ - extern template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - extern template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); \ - \ - extern template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - extern template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); - -instantiate_raft_neighbors_ivf_flat_build(float, int64_t); -instantiate_raft_neighbors_ivf_flat_build(int8_t, int64_t); -instantiate_raft_neighbors_ivf_flat_build(uint8_t, int64_t); -#undef instantiate_raft_neighbors_ivf_flat_build - -#define instantiate_raft_neighbors_ivf_flat_extend(T, IdxT) \ - extern template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index& orig_index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_flat::index; \ - \ - extern template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& orig_index) \ - ->raft::neighbors::ivf_flat::index; \ - \ - extern template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_flat::index* index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); \ - \ - extern template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); \ - \ - extern template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); \ - \ - extern template auto raft::neighbors::ivf_flat::extend( \ - const raft::resources& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& idx) \ - ->raft::neighbors::ivf_flat::index; - -instantiate_raft_neighbors_ivf_flat_extend(float, int64_t); -instantiate_raft_neighbors_ivf_flat_extend(int8_t, int64_t); -instantiate_raft_neighbors_ivf_flat_extend(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_extend - -#define instantiate_raft_neighbors_ivf_flat_search(T, IdxT) \ - extern template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances, \ - rmm::device_async_resource_ref mr); \ - \ - extern template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); - -instantiate_raft_neighbors_ivf_flat_search(float, int64_t); -instantiate_raft_neighbors_ivf_flat_search(int8_t, int64_t); -instantiate_raft_neighbors_ivf_flat_search(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_search diff --git a/cpp/include/raft/neighbors/ivf_flat.cuh b/cpp/include/raft/neighbors/ivf_flat.cuh index 8fd9628a41..d5c8f26fb3 100644 --- a/cpp/include/raft/neighbors/ivf_flat.cuh +++ b/cpp/include/raft/neighbors/ivf_flat.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "ivf_flat-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "ivf_flat-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/ivf_pq-ext.cuh b/cpp/include/raft/neighbors/ivf_pq-ext.cuh deleted file mode 100644 index 620f4a244f..0000000000 --- a/cpp/include/raft/neighbors/ivf_pq-ext.cuh +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::device_matrix_view -#include // raft::resources -#include // raft::neighbors::ivf_pq::index -#include // RAFT_EXPLICIT - -#include // int64_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors::ivf_pq { - -template -index build(raft::resources const& handle, - const index_params& params, - raft::device_matrix_view dataset) RAFT_EXPLICIT; - -template -index extend(raft::resources const& handle, - raft::device_matrix_view new_vectors, - std::optional> new_indices, - const index& idx) RAFT_EXPLICIT; - -template -void extend(raft::resources const& handle, - raft::device_matrix_view new_vectors, - std::optional> new_indices, - index* idx) RAFT_EXPLICIT; - -template -void search_with_filtering(raft::resources const& handle, - const search_params& params, - const index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances, - IvfSampleFilterT sample_filter) RAFT_EXPLICIT; - -template -void search(raft::resources const& handle, - const search_params& params, - const index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances) RAFT_EXPLICIT; - -template -auto build(raft::resources const& handle, - const index_params& params, - const T* dataset, - IdxT n_rows, - uint32_t dim) -> index RAFT_EXPLICIT; - -template -auto extend(raft::resources const& handle, - const index& idx, - const T* new_vectors, - const IdxT* new_indices, - IdxT n_rows) -> index RAFT_EXPLICIT; - -template -void extend(raft::resources const& handle, - index* idx, - const T* new_vectors, - const IdxT* new_indices, - IdxT n_rows) RAFT_EXPLICIT; - -template -void search_with_filtering(raft::resources const& handle, - const raft::neighbors::ivf_pq::search_params& params, - const index& idx, - const T* queries, - uint32_t n_queries, - uint32_t k, - IdxT* neighbors, - float* distances, - IvfSampleFilterT sample_filter = IvfSampleFilterT{}) RAFT_EXPLICIT; - -template -void search(raft::resources const& handle, - const raft::neighbors::ivf_pq::search_params& params, - const index& idx, - const T* queries, - uint32_t n_queries, - uint32_t k, - IdxT* neighbors, - float* distances) RAFT_EXPLICIT; - -} // namespace raft::neighbors::ivf_pq - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_neighbors_ivf_pq_build(T, IdxT) \ - extern template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset); \ - \ - extern template auto raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_pq::index; - -instantiate_raft_neighbors_ivf_pq_build(float, int64_t); -instantiate_raft_neighbors_ivf_pq_build(half, int64_t); -instantiate_raft_neighbors_ivf_pq_build(int8_t, int64_t); -instantiate_raft_neighbors_ivf_pq_build(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_build - -#define instantiate_raft_neighbors_ivf_pq_extend(T, IdxT) \ - extern template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_pq::index& idx); \ - \ - extern template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_pq::index* idx); \ - \ - extern template auto raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_pq::index; \ - \ - extern template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_pq::index* idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); - -instantiate_raft_neighbors_ivf_pq_extend(float, int64_t); -instantiate_raft_neighbors_ivf_pq_extend(half, int64_t); -instantiate_raft_neighbors_ivf_pq_extend(int8_t, int64_t); -instantiate_raft_neighbors_ivf_pq_extend(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_extend - -#define instantiate_raft_neighbors_ivf_pq_search(T, IdxT) \ - extern template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); \ - \ - extern template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances); \ - \ - extern template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances) - -instantiate_raft_neighbors_ivf_pq_search(float, int64_t); -instantiate_raft_neighbors_ivf_pq_search(half, int64_t); -instantiate_raft_neighbors_ivf_pq_search(int8_t, int64_t); -instantiate_raft_neighbors_ivf_pq_search(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_search diff --git a/cpp/include/raft/neighbors/ivf_pq.cuh b/cpp/include/raft/neighbors/ivf_pq.cuh index 2d20638f00..6608912c50 100644 --- a/cpp/include/raft/neighbors/ivf_pq.cuh +++ b/cpp/include/raft/neighbors/ivf_pq.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "ivf_pq-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "ivf_pq-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/refine-ext.cuh b/cpp/include/raft/neighbors/refine-ext.cuh deleted file mode 100644 index 216e1b9ab5..0000000000 --- a/cpp/include/raft/neighbors/refine-ext.cuh +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::device_matrix_view -#include // // raft::host_matrix_view -#include // raft::resources -#include // raft::distance::DistanceType -#include // RAFT_EXPLICIT - -#include // int64_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::neighbors { - -template -[[deprecated("Use cuVS instead")]] void refine( - raft::resources const& handle, - raft::device_matrix_view dataset, - raft::device_matrix_view queries, - raft::device_matrix_view neighbor_candidates, - raft::device_matrix_view indices, - raft::device_matrix_view distances, - raft::distance::DistanceType metric = distance::DistanceType::L2Unexpanded) RAFT_EXPLICIT; - -template -[[deprecated("Use cuVS instead")]] void refine( - raft::resources const& handle, - raft::host_matrix_view dataset, - raft::host_matrix_view queries, - raft::host_matrix_view neighbor_candidates, - raft::host_matrix_view indices, - raft::host_matrix_view distances, - raft::distance::DistanceType metric = distance::DistanceType::L2Unexpanded) RAFT_EXPLICIT; - -} // namespace raft::neighbors - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_neighbors_refine_d(idx_t, data_t, distance_t, matrix_idx) \ - extern template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::device_matrix_view dataset, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbor_candidates, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric); - -#define instantiate_raft_neighbors_refine_h(idx_t, data_t, distance_t, matrix_idx) \ - extern template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - raft::distance::DistanceType metric); - -instantiate_raft_neighbors_refine_d(int64_t, float, float, int64_t); -instantiate_raft_neighbors_refine_d(int64_t, int8_t, float, int64_t); -instantiate_raft_neighbors_refine_d(int64_t, uint8_t, float, int64_t); - -instantiate_raft_neighbors_refine_h(int64_t, float, float, int64_t); -instantiate_raft_neighbors_refine_h(uint32_t, float, float, int64_t); -instantiate_raft_neighbors_refine_h(int64_t, int8_t, float, int64_t); -instantiate_raft_neighbors_refine_h(int64_t, uint8_t, float, int64_t); - -#undef instantiate_raft_neighbors_refine_d -#undef instantiate_raft_neighbors_refine_h diff --git a/cpp/include/raft/neighbors/refine.cuh b/cpp/include/raft/neighbors/refine.cuh index 15f2b02928..83a98097e0 100644 --- a/cpp/include/raft/neighbors/refine.cuh +++ b/cpp/include/raft/neighbors/refine.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "refine-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "refine-ext.cuh" -#endif diff --git a/cpp/include/raft/neighbors/specializations.cuh b/cpp/include/raft/neighbors/specializations.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/neighbors/specializations.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/specializations/ball_cover.cuh b/cpp/include/raft/neighbors/specializations/ball_cover.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/neighbors/specializations/ball_cover.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/specializations/brute_force.cuh b/cpp/include/raft/neighbors/specializations/brute_force.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/neighbors/specializations/brute_force.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/specializations/detail/ball_cover_lowdim.hpp b/cpp/include/raft/neighbors/specializations/detail/ball_cover_lowdim.hpp deleted file mode 100644 index fbe38fd261..0000000000 --- a/cpp/include/raft/neighbors/specializations/detail/ball_cover_lowdim.hpp +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include - -namespace raft { -namespace spatial { -namespace knn { -namespace detail { - -extern template void rbc_low_dim_pass_one( - raft::resources const& handle, - const BallCoverIndex& index, - const float* query, - const std::uint32_t n_query_rows, - std::uint32_t k, - const std::int64_t* R_knn_inds, - const float* R_knn_dists, - DistFunc& dfunc, - std::int64_t* inds, - float* dists, - float weight, - std::uint32_t* dists_counter); - -extern template void rbc_low_dim_pass_two( - raft::resources const& handle, - const BallCoverIndex& index, - const float* query, - const std::uint32_t n_query_rows, - std::uint32_t k, - const std::int64_t* R_knn_inds, - const float* R_knn_dists, - DistFunc& dfunc, - std::int64_t* inds, - float* dists, - float weight, - std::uint32_t* post_dists_counter); - -extern template void rbc_low_dim_pass_one( - raft::resources const& handle, - const BallCoverIndex& index, - const float* query, - const std::uint32_t n_query_rows, - std::uint32_t k, - const std::int64_t* R_knn_inds, - const float* R_knn_dists, - DistFunc& dfunc, - std::int64_t* inds, - float* dists, - float weight, - std::uint32_t* dists_counter); - -extern template void rbc_low_dim_pass_two( - raft::resources const& handle, - const BallCoverIndex& index, - const float* query, - const std::uint32_t n_query_rows, - std::uint32_t k, - const std::int64_t* R_knn_inds, - const float* R_knn_dists, - DistFunc& dfunc, - std::int64_t* inds, - float* dists, - float weight, - std::uint32_t* post_dists_counter); - -}; // namespace detail -}; // namespace knn -}; // namespace spatial -}; // namespace raft \ No newline at end of file diff --git a/cpp/include/raft/neighbors/specializations/detail/ivf_pq_compute_similarity.cuh b/cpp/include/raft/neighbors/specializations/detail/ivf_pq_compute_similarity.cuh deleted file mode 100644 index e85b05575f..0000000000 --- a/cpp/include/raft/neighbors/specializations/detail/ivf_pq_compute_similarity.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/specializations/fused_l2_knn.cuh b/cpp/include/raft/neighbors/specializations/fused_l2_knn.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/neighbors/specializations/fused_l2_knn.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/specializations/ivf_flat.cuh b/cpp/include/raft/neighbors/specializations/ivf_flat.cuh deleted file mode 100644 index c61d65dcaf..0000000000 --- a/cpp/include/raft/neighbors/specializations/ivf_flat.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/specializations/ivf_pq.cuh b/cpp/include/raft/neighbors/specializations/ivf_pq.cuh deleted file mode 100644 index e85b05575f..0000000000 --- a/cpp/include/raft/neighbors/specializations/ivf_pq.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/neighbors/specializations/refine.cuh b/cpp/include/raft/neighbors/specializations/refine.cuh deleted file mode 100644 index e85b05575f..0000000000 --- a/cpp/include/raft/neighbors/specializations/refine.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/sparse/linalg/detail/cusparse_utils.hpp b/cpp/include/raft/sparse/linalg/detail/cusparse_utils.hpp index c10c0de426..97ac7c45f4 100644 --- a/cpp/include/raft/sparse/linalg/detail/cusparse_utils.hpp +++ b/cpp/include/raft/sparse/linalg/detail/cusparse_utils.hpp @@ -30,7 +30,25 @@ namespace linalg { namespace detail { /** - * @brief create a cuSparse dense descriptor + * @brief create a cuSparse dense descriptor for a vector + * @tparam ValueType Data type of vector_view (float/double) + * @tparam IndexType Type of vector_view + * @param[in] vector_view input raft::device_vector_view + * @returns dense vector descriptor to be used by cuSparse API + */ +template +cusparseDnVecDescr_t create_descriptor(raft::device_vector_view vector_view) +{ + cusparseDnVecDescr_t descr; + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsecreatednvec( + &descr, + vector_view.extent(0), + const_cast*>(vector_view.data_handle()))); + return descr; +} + +/** + * @brief create a cuSparse dense descriptor for a matrix * @tparam ValueType Data type of dense_view (float/double) * @tparam IndexType Type of dense_view * @tparam LayoutPolicy layout of dense_view diff --git a/cpp/include/raft/sparse/matrix/detail/select_k-ext.cuh b/cpp/include/raft/sparse/matrix/detail/select_k-ext.cuh deleted file mode 100644 index 01625a0ce8..0000000000 --- a/cpp/include/raft/sparse/matrix/detail/select_k-ext.cuh +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include // RAFT_EXPLICIT - -#include // __half - -#include // uint32_t - -#ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY - -namespace raft::sparse::matrix::detail { - -template -void select_k(raft::resources const& handle, - raft::device_csr_matrix_view in_val, - std::optional> in_idx, - raft::device_matrix_view out_val, - raft::device_matrix_view out_idx, - bool select_min, - bool sorted = false, - raft::matrix::SelectAlgo algo = raft::matrix::SelectAlgo::kAuto) RAFT_EXPLICIT; -} // namespace raft::sparse::matrix::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_sparse_matrix_detail_select_k(T, IdxT) \ - extern template void raft::sparse::matrix::detail::select_k( \ - raft::resources const& handle, \ - raft::device_csr_matrix_view in_val, \ - std::optional> in_idx, \ - raft::device_matrix_view out_val, \ - raft::device_matrix_view out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo) - -instantiate_raft_sparse_matrix_detail_select_k(__half, uint32_t); -instantiate_raft_sparse_matrix_detail_select_k(__half, int64_t); -instantiate_raft_sparse_matrix_detail_select_k(float, int64_t); -instantiate_raft_sparse_matrix_detail_select_k(float, uint32_t); -instantiate_raft_sparse_matrix_detail_select_k(float, int); -instantiate_raft_sparse_matrix_detail_select_k(double, int64_t); -instantiate_raft_sparse_matrix_detail_select_k(double, uint32_t); - -#undef instantiate_raft_sparse_matrix_detail_select_k diff --git a/cpp/include/raft/sparse/matrix/detail/select_k.cuh b/cpp/include/raft/sparse/matrix/detail/select_k.cuh index 5d52b94b2f..31a4b54a94 100644 --- a/cpp/include/raft/sparse/matrix/detail/select_k.cuh +++ b/cpp/include/raft/sparse/matrix/detail/select_k.cuh @@ -15,11 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "select_k-inl.cuh" - -#endif - -#ifdef RAFT_COMPILED -#include "select_k-ext.cuh" -#endif diff --git a/cpp/include/raft/sparse/neighbors/specializations.cuh b/cpp/include/raft/sparse/neighbors/specializations.cuh deleted file mode 100644 index e85b05575f..0000000000 --- a/cpp/include/raft/sparse/neighbors/specializations.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/sparse/solver/detail/lanczos.cuh b/cpp/include/raft/sparse/solver/detail/lanczos.cuh index 9ecb4b729f..02a77a0d99 100644 --- a/cpp/include/raft/sparse/solver/detail/lanczos.cuh +++ b/cpp/include/raft/sparse/solver/detail/lanczos.cuh @@ -19,10 +19,43 @@ // for cmath: #define _USE_MATH_DEFINES +#include +#include +#include +#include +#include +#include +#include #include #include #include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -30,9 +63,17 @@ #include +#include #include +#include +#include +#include #include +#include +#include +#include +#include #include namespace raft::sparse::solver::detail { @@ -1396,4 +1437,674 @@ int computeLargestEigenvectors( return status; } +template +RAFT_KERNEL kernel_triangular_populate(T* M, const T* beta, int n) +{ + int row = blockIdx.x * blockDim.x + threadIdx.x; + + if (row < n) { + // Upper diagonal: M[row + 1, row] in column-major + if (row < n - 1) { M[(row + 1) * n + row] = beta[row]; } + + // Lower diagonal: M[row - 1, row] in column-major + if (row > 0) { M[(row - 1) * n + row] = beta[row - 1]; } + } +} + +template +RAFT_KERNEL kernel_triangular_beta_k(T* t, const T* beta_k, int k, int n) +{ + int tid = threadIdx.x + blockIdx.x * blockDim.x; + + if (tid < k) { + // Update the k-th column: t[i, k] -> t[k * n + i] in column-major + t[tid * n + k] = beta_k[tid]; + + // Update the k-th row: t[k, j] -> t[j * n + k] in column-major + t[k * n + tid] = beta_k[tid]; + } +} + +template +RAFT_KERNEL kernel_normalize(const T* u, const T* beta, int j, int n, T* v, T* V, int size) +{ + int i = blockIdx.x * blockDim.x + threadIdx.x; + + if (i < size) { + if (beta[j] == 0) { + v[i] = u[i] / 1; + } else { + v[i] = u[i] / beta[j]; + } + V[i + (j + 1) * n] = v[i]; + } +} + +template +RAFT_KERNEL kernel_clamp_down(T* value, T threshold) +{ + *value = (fabs(*value) < threshold) ? 0 : *value; +} + +template +RAFT_KERNEL kernel_clamp_down_vector(T* vec, T threshold, int size) +{ + int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx < size) { vec[idx] = (fabs(vec[idx]) < threshold) ? 0 : vec[idx]; } +} + +template +void lanczos_solve_ritz( + raft::resources const& handle, + raft::device_matrix_view alpha, + raft::device_matrix_view beta, + std::optional> beta_k, + IndexTypeT k, + int which, + int ncv, + raft::device_matrix_view eigenvectors, + raft::device_vector_view eigenvalues) +{ + auto stream = resource::get_cuda_stream(handle); + + ValueTypeT zero = 0; + auto triangular_matrix = + raft::make_device_matrix(handle, ncv, ncv); + raft::matrix::fill(handle, triangular_matrix.view(), zero); + + raft::device_vector_view alphaVec = + raft::make_device_vector_view(alpha.data_handle(), ncv); + raft::matrix::set_diagonal(handle, alphaVec, triangular_matrix.view()); + + // raft::matrix::initializeDiagonalMatrix( + // alpha.data_handle(), triangular_matrix.data_handle(), ncv, ncv, stream); + + int blockSize = 256; + int numBlocks = (ncv + blockSize - 1) / blockSize; + kernel_triangular_populate + <<>>(triangular_matrix.data_handle(), beta.data_handle(), ncv); + + if (beta_k) { + int threadsPerBlock = 256; + int blocksPerGrid = (k + threadsPerBlock - 1) / threadsPerBlock; + kernel_triangular_beta_k<<>>( + triangular_matrix.data_handle(), beta_k.value().data_handle(), (int)k, ncv); + } + + auto triangular_matrix_view = + raft::make_device_matrix_view( + triangular_matrix.data_handle(), ncv, ncv); + + raft::linalg::eig_dc(handle, triangular_matrix_view, eigenvectors, eigenvalues); +} + +template +void lanczos_aux(raft::resources const& handle, + raft::device_csr_matrix_view A, + raft::device_matrix_view V, + raft::device_matrix_view u, + raft::device_matrix_view alpha, + raft::device_matrix_view beta, + int start_idx, + int end_idx, + int ncv, + raft::device_matrix_view v, + raft::device_matrix_view uu, + raft::device_matrix_view vv) +{ + auto stream = resource::get_cuda_stream(handle); + + IndexTypeT n = A.structure_view().get_n_rows(); + auto v_vector = raft::make_device_vector_view(v.data_handle(), n); + auto u_vector = raft::make_device_vector_view(u.data_handle(), n); + + raft::copy( + v.data_handle(), V.data_handle() + start_idx * V.stride(0), n, stream); // V(start_idx, 0) + + auto cusparse_h = resource::get_cusparse_handle(handle); + cusparseSpMatDescr_t cusparse_A = raft::sparse::linalg::detail::create_descriptor(A); + + cusparseDnVecDescr_t cusparse_v = raft::sparse::linalg::detail::create_descriptor(v_vector); + cusparseDnVecDescr_t cusparse_u = raft::sparse::linalg::detail::create_descriptor(u_vector); + + ValueTypeT one = 1; + ValueTypeT zero = 0; + size_t bufferSize; + raft::sparse::detail::cusparsespmv_buffersize(cusparse_h, + CUSPARSE_OPERATION_NON_TRANSPOSE, + &one, + cusparse_A, + cusparse_v, + &zero, + cusparse_u, + CUSPARSE_SPMV_ALG_DEFAULT, + &bufferSize, + stream); + auto cusparse_spmv_buffer = raft::make_device_vector(handle, bufferSize); + + for (int i = start_idx; i < end_idx; i++) { + raft::sparse::detail::cusparsespmv(cusparse_h, + CUSPARSE_OPERATION_NON_TRANSPOSE, + &one, + cusparse_A, + cusparse_v, + &zero, + cusparse_u, + CUSPARSE_SPMV_ALG_DEFAULT, + cusparse_spmv_buffer.data_handle(), + stream); + + auto alpha_i = + raft::make_device_scalar_view(alpha.data_handle() + i * alpha.stride(1)); // alpha(0, i) + raft::linalg::dot(handle, v_vector, u_vector, alpha_i); + + raft::matrix::fill(handle, vv, zero); + + auto cublas_h = resource::get_cublas_handle(handle); + + ValueTypeT alpha_i_host = 0; + ValueTypeT b = 0; + ValueTypeT mone = -1; + + raft::copy( + &b, beta.data_handle() + ((i - 1 + ncv) % ncv) * beta.stride(1), 1, stream); + raft::copy( + &alpha_i_host, alpha.data_handle() + i * alpha.stride(1), 1, stream); // alpha(0, i) + + raft::linalg::axpy(handle, n, &alpha_i_host, v.data_handle(), 1, vv.data_handle(), 1, stream); + raft::linalg::axpy(handle, + n, + &b, + V.data_handle() + (((i - 1 + ncv) % ncv) * V.stride(0)), + 1, + vv.data_handle(), + 1, + stream); + raft::linalg::axpy(handle, n, &mone, vv.data_handle(), 1, u.data_handle(), 1, stream); + + raft::linalg::gemv(handle, + CUBLAS_OP_T, + n, + i + 1, + &one, + V.data_handle(), + n, + u.data_handle(), + 1, + &zero, + uu.data_handle(), + 1, + stream); + + raft::linalg::gemv(handle, + CUBLAS_OP_N, + n, + i + 1, + &mone, + V.data_handle(), + n, + uu.data_handle(), + 1, + &one, + u.data_handle(), + 1, + stream); + + auto uu_i = raft::make_device_scalar_view(uu.data_handle() + uu.stride(1) * i); // uu(0, i) + raft::linalg::add(handle, make_const_mdspan(alpha_i), make_const_mdspan(uu_i), alpha_i); + + kernel_clamp_down<<<1, 1, 0, stream>>>(alpha_i.data_handle(), static_cast(1e-9)); + + auto output = raft::make_device_vector_view( + beta.data_handle() + beta.stride(1) * i, 1); + auto input = raft::make_device_matrix_view(u.data_handle(), 1, n); + raft::linalg::norm(handle, + input, + output, + raft::linalg::L2Norm, + raft::linalg::Apply::ALONG_ROWS, + raft::sqrt_op()); + + int blockSize = 256; + int numBlocks = (n + blockSize - 1) / blockSize; + + kernel_clamp_down_vector<<>>( + u.data_handle(), static_cast(1e-7), n); + + kernel_clamp_down<<<1, 1, 0, stream>>>(beta.data_handle() + beta.stride(1) * i, + static_cast(1e-6)); + + if (i >= end_idx - 1) { break; } + + int threadsPerBlock = 256; + int blocksPerGrid = (n + threadsPerBlock - 1) / threadsPerBlock; + + kernel_normalize<<>>( + u.data_handle(), beta.data_handle(), i, n, v.data_handle(), V.data_handle(), n); + } +} + +template +auto lanczos_smallest( + raft::resources const& handle, + raft::device_csr_matrix_view A, + int nEigVecs, + int maxIter, + int restartIter, + ValueTypeT tol, + ValueTypeT* eigVals_dev, + ValueTypeT* eigVecs_dev, + ValueTypeT* v0, + uint64_t seed) -> int +{ + int n = A.structure_view().get_n_rows(); + int ncv = restartIter; + auto stream = resource::get_cuda_stream(handle); + + auto V = raft::make_device_matrix(handle, ncv, n); + auto V_0_view = + raft::make_device_matrix_view(V.data_handle(), 1, n); // First Row V[0] + auto v0_view = raft::make_device_matrix_view(v0, 1, n); + + auto u = raft::make_device_matrix(handle, 1, n); + auto u_vector = raft::make_device_vector_view(u.data_handle(), n); + raft::copy(u.data_handle(), v0, n, stream); + + auto cublas_h = resource::get_cublas_handle(handle); + auto v0nrm = raft::make_device_vector(handle, 1); + raft::linalg::norm(handle, + v0_view, + v0nrm.view(), + raft::linalg::L2Norm, + raft::linalg::Apply::ALONG_ROWS, + raft::sqrt_op()); + + auto v0_vector_const = raft::make_device_vector_view(v0, n); + + raft::linalg::unary_op( + handle, v0_vector_const, V_0_view, [device_scalar = v0nrm.data_handle()] __device__(auto y) { + return y / *device_scalar; + }); + + auto alpha = raft::make_device_matrix(handle, 1, ncv); + auto beta = raft::make_device_matrix(handle, 1, ncv); + ValueTypeT zero = 0; + raft::matrix::fill(handle, alpha.view(), zero); + raft::matrix::fill(handle, beta.view(), zero); + + auto v = raft::make_device_matrix(handle, 1, n); + auto aux_uu = raft::make_device_matrix(handle, 1, ncv); + auto vv = raft::make_device_matrix(handle, 1, n); + + lanczos_aux(handle, + A, + V.view(), + u.view(), + alpha.view(), + beta.view(), + 0, + ncv, + ncv, + v.view(), + aux_uu.view(), + vv.view()); + + auto eigenvectors = + raft::make_device_matrix(handle, ncv, ncv); + auto eigenvalues = raft::make_device_vector(handle, ncv); + + lanczos_solve_ritz(handle, + alpha.view(), + beta.view(), + std::nullopt, + nEigVecs, + 0, + ncv, + eigenvectors.view(), + eigenvalues.view()); + + auto eigenvectors_k = raft::make_device_matrix_view( + eigenvectors.data_handle(), ncv, nEigVecs); + auto eigenvalues_k = + raft::make_device_vector_view(eigenvalues.data_handle(), nEigVecs); + + auto ritz_eigenvectors = + raft::make_device_matrix_view(eigVecs_dev, n, nEigVecs); + + auto V_T = + raft::make_device_matrix_view(V.data_handle(), n, ncv); + raft::linalg::gemm( + handle, V_T, eigenvectors_k, ritz_eigenvectors); + + auto s = raft::make_device_vector(handle, nEigVecs); + + auto eigenvectors_k_slice = + raft::make_device_matrix_view( + eigenvectors.data_handle(), ncv, nEigVecs); + auto S_matrix = raft::make_device_matrix_view( + s.data_handle(), 1, nEigVecs); + + raft::matrix::slice_coordinates coords(ncv - 1, 0, ncv, nEigVecs); + raft::matrix::slice(handle, make_const_mdspan(eigenvectors_k_slice), S_matrix, coords); + + auto beta_k = raft::make_device_vector(handle, nEigVecs); + raft::matrix::fill(handle, beta_k.view(), zero); + auto beta_scalar = raft::make_device_scalar_view(beta.data_handle() + + (ncv - 1) * beta.stride(1)); + + raft::linalg::axpy(handle, beta_scalar, raft::make_const_mdspan(s.view()), beta_k.view()); + + ValueTypeT res = 0; + + raft::device_vector output = + raft::make_device_vector(handle, 1); + raft::device_matrix_view input = + raft::make_device_matrix_view(beta_k.data_handle(), 1, nEigVecs); + raft::linalg::norm(handle, + input, + output.view(), + raft::linalg::L2Norm, + raft::linalg::Apply::ALONG_ROWS, + raft::sqrt_op()); + raft::copy(&res, output.data_handle(), 1, stream); + resource::sync_stream(handle, stream); + + auto uu = raft::make_device_matrix(handle, 0, nEigVecs); + int iter = ncv; + while (res > tol && iter < maxIter) { + auto beta_view = raft::make_device_matrix_view( + beta.data_handle(), 1, nEigVecs); + raft::matrix::fill(handle, beta_view, zero); + + raft::copy(alpha.data_handle(), eigenvalues_k.data_handle(), nEigVecs, stream); + + auto x_T = + raft::make_device_matrix_view(ritz_eigenvectors.data_handle(), nEigVecs, n); + + raft::copy(V.data_handle(), x_T.data_handle(), nEigVecs * n, stream); + + ValueTypeT one = 1; + ValueTypeT mone = -1; + + // Using raft::linalg::gemv leads to Reason=7:CUBLAS_STATUS_INVALID_VALUE (issue raft#2484) + raft::linalg::detail::cublasgemv(cublas_h, + CUBLAS_OP_T, + nEigVecs, + n, + &one, + V.data_handle(), + nEigVecs, + u.data_handle(), + 1, + &zero, + uu.data_handle(), + 1, + stream); + + raft::linalg::detail::cublasgemv(cublas_h, + CUBLAS_OP_N, + nEigVecs, + n, + &mone, + V.data_handle(), + nEigVecs, + uu.data_handle(), + 1, + &one, + u.data_handle(), + 1, + stream); + + auto V_0_view = + raft::make_device_matrix_view(V.data_handle() + (nEigVecs * n), 1, n); + auto V_0_view_vector = + raft::make_device_vector_view(V_0_view.data_handle(), n); + auto unrm = raft::make_device_vector(handle, 1); + raft::linalg::norm(handle, + raft::make_const_mdspan(u.view()), + unrm.view(), + raft::linalg::L2Norm, + raft::linalg::Apply::ALONG_ROWS, + raft::sqrt_op()); + + raft::linalg::unary_op( + handle, + raft::make_const_mdspan(u_vector), + V_0_view, + [device_scalar = unrm.data_handle()] __device__(auto y) { return y / *device_scalar; }); + + auto cusparse_h = resource::get_cusparse_handle(handle); + cusparseSpMatDescr_t cusparse_A = raft::sparse::linalg::detail::create_descriptor(A); + + cusparseDnVecDescr_t cusparse_v = + raft::sparse::linalg::detail::create_descriptor(V_0_view_vector); + cusparseDnVecDescr_t cusparse_u = raft::sparse::linalg::detail::create_descriptor(u_vector); + + ValueTypeT zero = 0; + size_t bufferSize; + raft::sparse::detail::cusparsespmv_buffersize(cusparse_h, + CUSPARSE_OPERATION_NON_TRANSPOSE, + &one, + cusparse_A, + cusparse_v, + &zero, + cusparse_u, + CUSPARSE_SPMV_ALG_DEFAULT, + &bufferSize, + stream); + auto cusparse_spmv_buffer = raft::make_device_vector(handle, bufferSize); + + raft::sparse::detail::cusparsespmv(cusparse_h, + CUSPARSE_OPERATION_NON_TRANSPOSE, + &one, + cusparse_A, + cusparse_v, + &zero, + cusparse_u, + CUSPARSE_SPMV_ALG_DEFAULT, + cusparse_spmv_buffer.data_handle(), + stream); + + auto alpha_k = raft::make_device_scalar_view(alpha.data_handle() + nEigVecs); + + raft::linalg::dot( + handle, make_const_mdspan(V_0_view_vector), make_const_mdspan(u_vector), alpha_k); + + raft::linalg::binary_op(handle, + make_const_mdspan(u_vector), + make_const_mdspan(V_0_view_vector), + u_vector, + [device_scalar_ptr = alpha_k.data_handle()] __device__( + ValueTypeT u_element, ValueTypeT V_0_element) { + return u_element - (*device_scalar_ptr) * V_0_element; + }); + + auto temp = raft::make_device_vector(handle, n); + + auto V_k = raft::make_device_matrix_view( + V.data_handle(), nEigVecs, n); + auto V_k_T = + raft::make_device_matrix(handle, n, nEigVecs); + + raft::linalg::transpose(handle, V_k, V_k_T.view()); + + ValueTypeT three = 3; + ValueTypeT two = 2; + + std::vector M = {1, 2, 3, 4, 5, 6}; + std::vector vec = {1, 1}; + + auto M_dev = raft::make_device_matrix(handle, 2, 3); + auto vec_dev = raft::make_device_vector(handle, 2); + auto out = raft::make_device_vector(handle, 3); + raft::copy(M_dev.data_handle(), M.data(), 6, stream); + raft::copy(vec_dev.data_handle(), vec.data(), 2, stream); + + raft::linalg::gemv(handle, + CUBLAS_OP_N, + three, + two, + &one, + M_dev.data_handle(), + three, + vec_dev.data_handle(), + 1, + &zero, + out.data_handle(), + 1, + stream); + + raft::linalg::gemv(handle, + CUBLAS_OP_N, + n, + nEigVecs, + &one, + V_k.data_handle(), + n, + beta_k.data_handle(), + 1, + &zero, + temp.data_handle(), + 1, + stream); + + auto one_scalar = raft::make_device_scalar(handle, 1); + raft::linalg::binary_op(handle, + make_const_mdspan(u_vector), + make_const_mdspan(temp.view()), + u_vector, + [device_scalar_ptr = one_scalar.data_handle()] __device__( + ValueTypeT u_element, ValueTypeT temp_element) { + return u_element - (*device_scalar_ptr) * temp_element; + }); + + auto output1 = raft::make_device_vector_view( + beta.data_handle() + beta.stride(1) * nEigVecs, 1); + raft::linalg::norm(handle, + raft::make_const_mdspan(u.view()), + output1, + raft::linalg::L2Norm, + raft::linalg::Apply::ALONG_ROWS, + raft::sqrt_op()); + + auto V_kplus1 = + raft::make_device_vector_view(V.data_handle() + V.stride(0) * (nEigVecs + 1), n); + + raft::linalg::unary_op( + handle, + make_const_mdspan(u_vector), + V_kplus1, + [device_scalar = (beta.data_handle() + beta.stride(1) * nEigVecs)] __device__(auto y) { + return y / *device_scalar; + }); + + lanczos_aux(handle, + A, + V.view(), + u.view(), + alpha.view(), + beta.view(), + nEigVecs + 1, + ncv, + ncv, + v.view(), + aux_uu.view(), + vv.view()); + iter += ncv - nEigVecs; + lanczos_solve_ritz(handle, + alpha.view(), + beta.view(), + beta_k.view(), + nEigVecs, + 0, + ncv, + eigenvectors.view(), + eigenvalues.view()); + auto eigenvectors_k = raft::make_device_matrix_view( + eigenvectors.data_handle(), ncv, nEigVecs); + + auto ritz_eigenvectors = raft::make_device_matrix_view( + eigVecs_dev, n, nEigVecs); + + auto V_T = + raft::make_device_matrix_view(V.data_handle(), n, ncv); + raft::linalg::gemm( + handle, V_T, eigenvectors_k, ritz_eigenvectors); + + auto eigenvectors_k_slice = + raft::make_device_matrix_view( + eigenvectors.data_handle(), ncv, nEigVecs); + auto S_matrix = raft::make_device_matrix_view( + s.data_handle(), 1, nEigVecs); + + raft::matrix::slice_coordinates coords(ncv - 1, 0, ncv, nEigVecs); + raft::matrix::slice(handle, make_const_mdspan(eigenvectors_k_slice), S_matrix, coords); + + raft::matrix::fill(handle, beta_k.view(), zero); + + auto beta_scalar = raft::make_device_scalar_view( + beta.data_handle() + beta.stride(1) * (ncv - 1)); // &((beta.view())(0, ncv - 1)) + + raft::linalg::axpy(handle, beta_scalar, raft::make_const_mdspan(s.view()), beta_k.view()); + + raft::device_vector output2 = + raft::make_device_vector(handle, 1); + raft::device_matrix_view input2 = + raft::make_device_matrix_view(beta_k.data_handle(), 1, nEigVecs); + raft::linalg::norm(handle, + input2, + output2.view(), + raft::linalg::L2Norm, + raft::linalg::Apply::ALONG_ROWS, + raft::sqrt_op()); + raft::copy(&res, output2.data_handle(), 1, stream); + resource::sync_stream(handle, stream); + RAFT_LOG_TRACE("Iteration %f: residual (tolerance) %d", iter, res); + } + + raft::copy(eigVals_dev, eigenvalues_k.data_handle(), nEigVecs, stream); + raft::copy(eigVecs_dev, ritz_eigenvectors.data_handle(), n * nEigVecs, stream); + + return 0; +} + +template +auto lanczos_compute_smallest_eigenvectors( + raft::resources const& handle, + lanczos_solver_config const& config, + raft::device_csr_matrix_view A, + std::optional> v0, + raft::device_vector_view eigenvalues, + raft::device_matrix_view eigenvectors) -> int +{ + if (v0.has_value()) { + return lanczos_smallest(handle, + A, + config.n_components, + config.max_iterations, + config.ncv, + config.tolerance, + eigenvalues.data_handle(), + eigenvectors.data_handle(), + v0->data_handle(), + config.seed); + } else { + // Handle the optional v0 initial Lanczos vector if nullopt is used + auto n = A.structure_view().get_n_rows(); + auto temp_v0 = raft::make_device_vector(handle, n); + raft::random::RngState rng_state(config.seed); + raft::random::uniform(handle, rng_state, temp_v0.view(), ValueTypeT{0.0}, ValueTypeT{1.0}); + return lanczos_smallest(handle, + A, + config.n_components, + config.max_iterations, + config.ncv, + config.tolerance, + eigenvalues.data_handle(), + eigenvectors.data_handle(), + temp_v0.data_handle(), + config.seed); + } +} + } // namespace raft::sparse::solver::detail diff --git a/cpp/include/raft/sparse/solver/lanczos.cuh b/cpp/include/raft/sparse/solver/lanczos.cuh index 1aa56d6ba2..fed31e6a9c 100644 --- a/cpp/include/raft/sparse/solver/lanczos.cuh +++ b/cpp/include/raft/sparse/solver/lanczos.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ #pragma once #include +#include #include namespace raft::sparse::solver { @@ -27,6 +28,78 @@ namespace raft::sparse::solver { // Eigensolver // ========================================================= +/** + * @brief Find the smallest eigenpairs using lanczos solver + * @tparam index_type_t the type of data used for indexing. + * @tparam value_type_t the type of data used for weights, distances. + * @param handle the raft handle. + * @param config lanczos config used to set hyperparameters + * @param A Sparse matrix in CSR format. + * @param v0 Optional Initial lanczos vector + * @param eigenvalues output eigenvalues + * @param eigenvectors output eigenvectors + * @todo Add largest eigenvalues computation (issue #2483) + * @return Zero if successful. Otherwise non-zero. + */ +template +auto lanczos_compute_smallest_eigenvectors( + raft::resources const& handle, + lanczos_solver_config const& config, + raft::device_csr_matrix_view A, + std::optional> v0, + raft::device_vector_view eigenvalues, + raft::device_matrix_view eigenvectors) -> int +{ + return detail::lanczos_compute_smallest_eigenvectors( + handle, config, A, v0, eigenvalues, eigenvectors); +} + +/** + * @brief Find the smallest eigenpairs using lanczos solver + * @tparam index_type_t the type of data used for indexing. + * @tparam value_type_t the type of data used for weights, distances. + * @param handle the raft handle. + * @param config lanczos config used to set hyperparameters + * @param rows Vector view of the rows of the sparse matrix. + * @param cols Vector view of the cols of the sparse matrix. + * @param vals Vector view of the vals of the sparse matrix. + * @param v0 Optional Initial lanczos vector + * @param eigenvalues output eigenvalues + * @param eigenvectors output eigenvectors + * @todo Add largest eigenvalues computation (issue #2483) + * @return Zero if successful. Otherwise non-zero. + */ +template +auto lanczos_compute_smallest_eigenvectors( + raft::resources const& handle, + lanczos_solver_config const& config, + raft::device_vector_view rows, + raft::device_vector_view cols, + raft::device_vector_view vals, + std::optional> v0, + raft::device_vector_view eigenvalues, + raft::device_matrix_view eigenvectors) -> int +{ + IndexTypeT ncols = rows.extent(0) - 1; + IndexTypeT nrows = rows.extent(0) - 1; + IndexTypeT nnz = cols.extent(0); + + auto csr_structure = + raft::make_device_compressed_structure_view( + const_cast(rows.data_handle()), + const_cast(cols.data_handle()), + ncols, + nrows, + nnz); + + auto csr_matrix = + raft::make_device_csr_matrix_view( + const_cast(vals.data_handle()), csr_structure); + + return lanczos_compute_smallest_eigenvectors( + handle, config, csr_matrix, v0, eigenvalues, eigenvectors); +} + /** * @brief Compute smallest eigenvectors of symmetric matrix * Computes eigenvalues and eigenvectors that are least diff --git a/cpp/test/neighbors/ann_cagra_vpq/test_float_int64_t.cu b/cpp/include/raft/sparse/solver/lanczos_types.hpp similarity index 53% rename from cpp/test/neighbors/ann_cagra_vpq/test_float_int64_t.cu rename to cpp/include/raft/sparse/solver/lanczos_types.hpp index f60edb5ed6..edd5548079 100644 --- a/cpp/test/neighbors/ann_cagra_vpq/test_float_int64_t.cu +++ b/cpp/include/raft/sparse/solver/lanczos_types.hpp @@ -14,16 +14,24 @@ * limitations under the License. */ -#undef RAFT_EXPLICIT_INSTANTIATE_ONLY -#include "../ann_cagra_vpq.cuh" +#pragma once -#include +#include -namespace raft::neighbors::cagra { +namespace raft::sparse::solver { -typedef AnnCagraVpqTest AnnCagraVpqTestF_I64; -TEST_P(AnnCagraVpqTestF_I64, AnnCagraVpq) { this->testCagra(); } +template +struct lanczos_solver_config { + /** The number of eigenvalues and eigenvectors to compute. Must be 1 <= k < n.*/ + int n_components; + /** Maximum number of iteration. */ + int max_iterations; + /** The number of Lanczos vectors generated. Must be k + 1 < ncv < n. */ + int ncv; + /** Tolerance for residuals ``||Ax - wx||`` */ + ValueTypeT tolerance; + /** random seed */ + uint64_t seed; +}; -INSTANTIATE_TEST_CASE_P(AnnCagraVpqTest, AnnCagraVpqTestF_I64, ::testing::ValuesIn(vpq_inputs)); - -} // namespace raft::neighbors::cagra +} // namespace raft::sparse::solver diff --git a/cpp/include/raft/spatial/knn/detail/ann_utils.cuh b/cpp/include/raft/spatial/knn/detail/ann_utils.cuh index 920249172f..e795a02d46 100644 --- a/cpp/include/raft/spatial/knn/detail/ann_utils.cuh +++ b/cpp/include/raft/spatial/knn/detail/ann_utils.cuh @@ -61,14 +61,9 @@ struct pointer_residency_count { auto [on_device, on_host] = pointer_residency_count::run(ptrs...); cudaPointerAttributes attr; RAFT_CUDA_TRY(cudaPointerGetAttributes(&attr, ptr)); - switch (attr.type) { - case cudaMemoryTypeUnregistered: return std::make_tuple(on_device, on_host + 1); - case cudaMemoryTypeHost: - return std::make_tuple(on_device + int(attr.devicePointer == ptr), on_host + 1); - case cudaMemoryTypeDevice: return std::make_tuple(on_device + 1, on_host); - case cudaMemoryTypeManaged: return std::make_tuple(on_device + 1, on_host + 1); - default: return std::make_tuple(on_device, on_host); - } + if (attr.devicePointer || attr.type == cudaMemoryTypeDevice) { ++on_device; } + if (attr.hostPointer || attr.type == cudaMemoryTypeUnregistered) { ++on_host; } + return std::make_tuple(on_device, on_host); } }; diff --git a/cpp/include/raft/spatial/knn/detail/ball_cover/registers-ext.cuh b/cpp/include/raft/spatial/knn/detail/ball_cover/registers-ext.cuh deleted file mode 100644 index 70df6d0165..0000000000 --- a/cpp/include/raft/spatial/knn/detail/ball_cover/registers-ext.cuh +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "../../ball_cover_types.hpp" // BallCoverIndex -#include "registers_types.cuh" // DistFunc - -#include //RAFT_EXPLICIT - -#include // uint32_t - -#if defined(RAFT_EXPLICIT_INSTANTIATE_ONLY) - -namespace raft::spatial::knn::detail { - -template -void rbc_low_dim_pass_one(raft::resources const& handle, - const BallCoverIndex& index, - const value_t* query, - const value_int n_query_rows, - value_int k, - const value_idx* R_knn_inds, - const value_t* R_knn_dists, - dist_func& dfunc, - value_idx* inds, - value_t* dists, - float weight, - value_int* dists_counter) RAFT_EXPLICIT; - -template -void rbc_low_dim_pass_two(raft::resources const& handle, - const BallCoverIndex& index, - const value_t* query, - const value_int n_query_rows, - value_int k, - const value_idx* R_knn_inds, - const value_t* R_knn_dists, - dist_func& dfunc, - value_idx* inds, - value_t* dists, - float weight, - value_int* post_dists_counter) RAFT_EXPLICIT; - -template -void rbc_eps_pass(raft::resources const& handle, - const BallCoverIndex& index, - const value_t* query, - const value_int n_query_rows, - value_t eps, - const value_t* R_dists, - dist_func& dfunc, - bool* adj, - value_idx* vd) RAFT_EXPLICIT; - -template -void rbc_eps_pass(raft::resources const& handle, - const BallCoverIndex& index, - const value_t* query, - const value_int n_query_rows, - value_t eps, - value_int* max_k, - const value_t* R_dists, - dist_func& dfunc, - value_idx* adj_ia, - value_idx* adj_ja, - value_idx* vd) RAFT_EXPLICIT; - -}; // namespace raft::spatial::knn::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - extern template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - extern template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -#define instantiate_raft_spatial_knn_detail_rbc_eps_pass( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdist_func) \ - extern template void \ - raft::spatial::knn::detail::rbc_eps_pass( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_t eps, \ - const Mvalue_t* R_dists, \ - Mdist_func& dfunc, \ - bool* adj, \ - Mvalue_idx* vd); \ - \ - extern template void \ - raft::spatial::knn::detail::rbc_eps_pass( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_t eps, \ - Mvalue_int* max_k, \ - const Mvalue_t* R_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* adj_ia, \ - Mvalue_idx* adj_ja, \ - Mvalue_idx* vd); - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::HaversineFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::HaversineFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::EuclideanFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::EuclideanFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::DistFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::DistFunc); - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::HaversineFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::HaversineFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::EuclideanFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::EuclideanFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::DistFunc); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::DistFunc); - -instantiate_raft_spatial_knn_detail_rbc_eps_pass( - std::int64_t, float, std::int64_t, std::int64_t, raft::spatial::knn::detail::EuclideanSqFunc); - -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one -#undef instantiate_raft_spatial_knn_detail_rbc_eps_pass diff --git a/cpp/include/raft/spatial/knn/detail/ball_cover/registers.cuh b/cpp/include/raft/spatial/knn/detail/ball_cover/registers.cuh index 8bd57b47cc..1c13b6b59f 100644 --- a/cpp/include/raft/spatial/knn/detail/ball_cover/registers.cuh +++ b/cpp/include/raft/spatial/knn/detail/ball_cover/registers.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,4 @@ * limitations under the License. */ #pragma once - -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "registers-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "registers-ext.cuh" -#endif diff --git a/cpp/include/raft/spatial/knn/detail/fused_l2_knn-ext.cuh b/cpp/include/raft/spatial/knn/detail/fused_l2_knn-ext.cuh deleted file mode 100644 index 1968bdb205..0000000000 --- a/cpp/include/raft/spatial/knn/detail/fused_l2_knn-ext.cuh +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include // DistanceType -#include // RAFT_EXPLICIT - -#include // size_t -#include // uint32_t - -#if defined(RAFT_EXPLICIT_INSTANTIATE_ONLY) - -namespace raft::spatial::knn::detail { - -template -void fusedL2Knn(size_t D, - value_idx* out_inds, - value_t* out_dists, - const value_t* index, - const value_t* query, - size_t n_index_rows, - size_t n_query_rows, - int k, - bool rowMajorIndex, - bool rowMajorQuery, - cudaStream_t stream, - raft::distance::DistanceType metric, - const value_t* index_norms = NULL, - const value_t* query_norms = NULL) RAFT_EXPLICIT; - -} // namespace raft::spatial::knn::detail - -#endif // RAFT_EXPLICIT_INSTANTIATE_ONLY - -#define instantiate_raft_spatial_knn_detail_fusedL2Knn(Mvalue_idx, Mvalue_t, MusePrevTopKs) \ - extern template void \ - raft::spatial::knn::detail::fusedL2Knn( \ - size_t D, \ - Mvalue_idx * out_inds, \ - Mvalue_t * out_dists, \ - const Mvalue_t* index, \ - const Mvalue_t* query, \ - size_t n_index_rows, \ - size_t n_query_rows, \ - int k, \ - bool rowMajorIndex, \ - bool rowMajorQuery, \ - cudaStream_t stream, \ - raft::distance::DistanceType metric, \ - const Mvalue_t* index_norms, \ - const Mvalue_t* query_norms); - -instantiate_raft_spatial_knn_detail_fusedL2Knn(int32_t, float, true); -instantiate_raft_spatial_knn_detail_fusedL2Knn(int32_t, float, false); -instantiate_raft_spatial_knn_detail_fusedL2Knn(int64_t, float, true); -instantiate_raft_spatial_knn_detail_fusedL2Knn(int64_t, float, false); - -// These are used by brute_force_knn: -instantiate_raft_spatial_knn_detail_fusedL2Knn(uint32_t, float, true); -instantiate_raft_spatial_knn_detail_fusedL2Knn(uint32_t, float, false); - -#undef instantiate_raft_spatial_knn_detail_fusedL2Knn diff --git a/cpp/include/raft/spatial/knn/detail/fused_l2_knn.cuh b/cpp/include/raft/spatial/knn/detail/fused_l2_knn.cuh index 8cc02c7c78..a45a00730c 100644 --- a/cpp/include/raft/spatial/knn/detail/fused_l2_knn.cuh +++ b/cpp/include/raft/spatial/knn/detail/fused_l2_knn.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,4 @@ */ #pragma once -#ifndef RAFT_EXPLICIT_INSTANTIATE_ONLY #include "fused_l2_knn-inl.cuh" -#endif - -#ifdef RAFT_COMPILED -#include "fused_l2_knn-ext.cuh" -#endif diff --git a/cpp/include/raft/spatial/knn/specializations.cuh b/cpp/include/raft/spatial/knn/specializations.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/spatial/knn/specializations.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/spatial/knn/specializations/knn.cuh b/cpp/include/raft/spatial/knn/specializations/knn.cuh deleted file mode 100644 index cba059154f..0000000000 --- a/cpp/include/raft/spatial/knn/specializations/knn.cuh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#ifndef RAFT_HIDE_DEPRECATION_WARNINGS -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") -#endif diff --git a/cpp/include/raft/spectral/eigen_solvers.cuh b/cpp/include/raft/spectral/eigen_solvers.cuh index 4774d8b8ae..324f16ac7b 100644 --- a/cpp/include/raft/spectral/eigen_solvers.cuh +++ b/cpp/include/raft/spectral/eigen_solvers.cuh @@ -69,6 +69,7 @@ struct lanczos_solver_t { eigVals, eigVecs, config_.seed); + return iters; } diff --git a/cpp/include/raft/util/cudart_utils.hpp b/cpp/include/raft/util/cudart_utils.hpp index f9e7f521be..48b8525cad 100644 --- a/cpp/include/raft/util/cudart_utils.hpp +++ b/cpp/include/raft/util/cudart_utils.hpp @@ -223,12 +223,13 @@ void print_vector(const char* variable_name, const T* ptr, size_t componentsCoun { cudaPointerAttributes attr; RAFT_CUDA_TRY(cudaPointerGetAttributes(&attr, ptr)); - if (attr.hostPointer != nullptr) { + if (attr.hostPointer) { print_host_vector(variable_name, reinterpret_cast(attr.hostPointer), componentsCount, out); } else if (attr.type == cudaMemoryTypeUnregistered) { print_host_vector(variable_name, ptr, componentsCount, out); } else { - print_device_vector(variable_name, ptr, componentsCount, out); + print_device_vector( + variable_name, reinterpret_cast(attr.devicePointer), componentsCount, out); } } /** @} */ diff --git a/cpp/include/raft_runtime/solver/lanczos.hpp b/cpp/include/raft_runtime/solver/lanczos.hpp new file mode 100644 index 0000000000..6c9d901bf1 --- /dev/null +++ b/cpp/include/raft_runtime/solver/lanczos.hpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include + +namespace raft::runtime::solver { + +/** + * @defgroup lanczos_runtime lanczos Runtime API + * @{ + */ + +#define FUNC_DECL(IndexType, ValueType) \ + void lanczos_solver( \ + const raft::resources& handle, \ + raft::sparse::solver::lanczos_solver_config config, \ + raft::device_vector_view rows, \ + raft::device_vector_view cols, \ + raft::device_vector_view vals, \ + std::optional> v0, \ + raft::device_vector_view eigenvalues, \ + raft::device_matrix_view eigenvectors) + +FUNC_DECL(int, float); +FUNC_DECL(int64_t, float); +FUNC_DECL(int, double); +FUNC_DECL(int64_t, double); + +#undef FUNC_DECL + +/** @} */ // end group lanczos_runtime + +} // namespace raft::runtime::solver diff --git a/cpp/internal/raft_internal/neighbors/ivf_pq_compute_similarity_filters_test-ext.cuh b/cpp/internal/raft_internal/neighbors/ivf_pq_compute_similarity_filters_test-ext.cuh deleted file mode 100644 index aa14ab19b8..0000000000 --- a/cpp/internal/raft_internal/neighbors/ivf_pq_compute_similarity_filters_test-ext.cuh +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // RAFT_WEAK_FUNCTION -#include // raft::distance::DistanceType -#include -#include // raft::neighbors::ivf_pq::detail::fp_8bit -#include // none_ivf_sample_filter -#include // none_ivf_sample_filter - -#include // rmm::cuda_stream_view - -#include // __half - -#define instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( \ - OutT, LutT, IvfSampleFilterT) \ - extern template auto \ - raft::neighbors::ivf_pq::detail::compute_similarity_select( \ - const cudaDeviceProp& dev_props, \ - bool manage_local_topk, \ - int locality_hint, \ - double preferred_shmem_carveout, \ - uint32_t pq_bits, \ - uint32_t pq_dim, \ - uint32_t precomp_data_count, \ - uint32_t n_queries, \ - uint32_t n_probes, \ - uint32_t topk) \ - ->raft::neighbors::ivf_pq::detail::selected; \ - \ - extern template void \ - raft::neighbors::ivf_pq::detail::compute_similarity_run( \ - raft::neighbors::ivf_pq::detail::selected s, \ - rmm::cuda_stream_view stream, \ - uint32_t dim, \ - uint32_t n_probes, \ - uint32_t pq_dim, \ - uint32_t n_queries, \ - uint32_t queries_offset, \ - raft::distance::DistanceType metric, \ - raft::neighbors::ivf_pq::codebook_gen codebook_kind, \ - uint32_t topk, \ - uint32_t max_samples, \ - const float* cluster_centers, \ - const float* pq_centers, \ - const uint8_t* const* pq_dataset, \ - const uint32_t* cluster_labels, \ - const uint32_t* _chunk_indices, \ - const float* queries, \ - const uint32_t* index_list, \ - float* query_kths, \ - IvfSampleFilterT sample_filter, \ - LutT* lut_scores, \ - OutT* _out_scores, \ - uint32_t* _out_indices); - -#define COMMA , -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); - -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); -#undef COMMA - -#undef instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select diff --git a/cpp/internal/raft_internal/neighbors/ivf_pq_search_test-ext.cuh b/cpp/internal/raft_internal/neighbors/ivf_pq_search_test-ext.cuh deleted file mode 100644 index 1e6f4f9976..0000000000 --- a/cpp/internal/raft_internal/neighbors/ivf_pq_search_test-ext.cuh +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include // raft::device_matrix_view -#include // raft::resources -#include -#include // raft::neighbors::ivf_pq::index -#include -#include - -#include - -#include - -#include // int64_t - -#define instantiate_raft_neighbors_ivf_pq_search(T, IdxT) \ - extern template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); \ - \ - extern template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances); \ - \ - extern template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances) - -instantiate_raft_neighbors_ivf_pq_search(float, uint32_t); - -#undef instantiate_raft_neighbors_ivf_pq_search - -#define instantiate_raft_neighbors_ivf_pq_search_with_filtering(T, IdxT, FilterT) \ - extern template void raft::neighbors::ivf_pq::search_with_filtering( \ - raft::resources const& handle, \ - const search_params& params, \ - const index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances, \ - FilterT sample_filter) - -#define COMMA , -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - float, uint32_t, raft::neighbors::filtering::bitset_filter); - -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - float, uint32_t, raft::neighbors::filtering::none_ivf_sample_filter); - -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - float, int64_t, raft::neighbors::filtering::bitset_filter); - -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - int8_t, int64_t, raft::neighbors::filtering::bitset_filter); - -#undef COMMA -#undef instantiate_raft_neighbors_ivf_pq_search_with_filtering diff --git a/cpp/internal/raft_internal/neighbors/naive_knn.cuh b/cpp/internal/raft_internal/neighbors/naive_knn.cuh deleted file mode 100644 index c14a8e3e9f..0000000000 --- a/cpp/internal/raft_internal/neighbors/naive_knn.cuh +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include - -namespace raft::neighbors { - -template -RAFT_KERNEL naive_distance_kernel(EvalT* dist, - const DataT* x, - const DataT* y, - IdxT m, - IdxT n, - IdxT k, - raft::distance::DistanceType metric) -{ - IdxT midx = IdxT(threadIdx.x) + IdxT(blockIdx.x) * IdxT(blockDim.x); - if (midx >= m) return; - IdxT grid_size = IdxT(blockDim.y) * IdxT(gridDim.y); - for (IdxT nidx = threadIdx.y + blockIdx.y * blockDim.y; nidx < n; nidx += grid_size) { - EvalT acc = EvalT(0); - for (IdxT i = 0; i < k; ++i) { - IdxT xidx = i + midx * k; - IdxT yidx = i + nidx * k; - auto xv = EvalT(x[xidx]); - auto yv = EvalT(y[yidx]); - switch (metric) { - case raft::distance::DistanceType::InnerProduct: { - acc += xv * yv; - } break; - case raft::distance::DistanceType::L2SqrtExpanded: - case raft::distance::DistanceType::L2SqrtUnexpanded: - case raft::distance::DistanceType::L2Expanded: - case raft::distance::DistanceType::L2Unexpanded: { - auto diff = xv - yv; - acc += diff * diff; - } break; - default: break; - } - } - switch (metric) { - case raft::distance::DistanceType::L2SqrtExpanded: - case raft::distance::DistanceType::L2SqrtUnexpanded: { - acc = raft::sqrt(acc); - } break; - default: break; - } - dist[midx * n + nidx] = acc; - } -} - -/** - * Naive, but flexible bruteforce KNN search. - * - * TODO: either replace this with brute_force_knn or with distance+select_k - * when either distance or brute_force_knn support 8-bit int inputs. - */ -template -void naive_knn(raft::resources const& handle, - EvalT* dist_topk, - IdxT* indices_topk, - const DataT* x, - const DataT* y, - size_t n_inputs, - size_t input_len, - size_t dim, - uint32_t k, - raft::distance::DistanceType type) -{ - auto mr = resource::get_workspace_resource(handle); - auto stream = raft::resource::get_cuda_stream(handle); - dim3 block_dim(16, 32, 1); - // maximum reasonable grid size in `y` direction - auto grid_y = - static_cast(std::min(raft::ceildiv(input_len, block_dim.y), 32768)); - - // bound the memory used by this function - size_t max_batch_size = - std::min(n_inputs, raft::ceildiv(size_t(1) << size_t(27), input_len)); - rmm::device_uvector dist(max_batch_size * input_len, stream, mr); - - for (size_t offset = 0; offset < n_inputs; offset += max_batch_size) { - size_t batch_size = std::min(max_batch_size, n_inputs - offset); - dim3 grid_dim(raft::ceildiv(batch_size, block_dim.x), grid_y, 1); - - naive_distance_kernel<<>>( - dist.data(), x + offset * dim, y, batch_size, input_len, dim, type); - - matrix::detail::select_k(handle, - dist.data(), - nullptr, - batch_size, - input_len, - static_cast(k), - dist_topk + offset * k, - indices_topk + offset * k, - type != raft::distance::DistanceType::InnerProduct); - } - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); -} - -} // namespace raft::neighbors diff --git a/cpp/internal/raft_internal/neighbors/refine_helper.cuh b/cpp/internal/raft_internal/neighbors/refine_helper.cuh deleted file mode 100644 index 665ec23d8e..0000000000 --- a/cpp/internal/raft_internal/neighbors/refine_helper.cuh +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -namespace raft::neighbors { - -template -struct RefineInputs { - IdxT n_queries; - IdxT n_rows; - IdxT dim; - IdxT k; // after refinement - IdxT k0; // initial k before refinement (k0 >= k). - raft::distance::DistanceType metric; - bool host_data; -}; - -/** Helper class to allocate arrays and generate input data for refinement test and benchmark. */ -template -class RefineHelper { - public: - RefineHelper(const raft::resources& handle, RefineInputs params) - : handle_(handle), - stream_(resource::get_cuda_stream(handle)), - p(params), - dataset(handle), - queries(handle), - refined_distances(handle), - refined_indices(handle), - candidates(handle), - dataset_host(handle), - queries_host(handle), - candidates_host(handle), - refined_distances_host(handle), - refined_indices_host(handle) - { - raft::random::RngState rng(1234ULL); - - dataset = raft::make_device_matrix(handle_, p.n_rows, p.dim); - queries = raft::make_device_matrix(handle_, p.n_queries, p.dim); - if constexpr (std::is_same{}) { - raft::random::uniform( - handle, rng, dataset.data_handle(), dataset.size(), DataT(-10.0), DataT(10.0)); - raft::random::uniform( - handle, rng, queries.data_handle(), queries.size(), DataT(-10.0), DataT(10.0)); - } else { - raft::random::uniformInt( - handle, rng, dataset.data_handle(), dataset.size(), DataT(1), DataT(20)); - raft::random::uniformInt( - handle, rng, queries.data_handle(), queries.size(), DataT(1), DataT(20)); - } - - refined_distances = raft::make_device_matrix(handle_, p.n_queries, p.k); - refined_indices = raft::make_device_matrix(handle_, p.n_queries, p.k); - - // Generate candidate vectors - { - candidates = raft::make_device_matrix(handle_, p.n_queries, p.k0); - rmm::device_uvector distances_tmp(p.n_queries * p.k0, stream_); - naive_knn(handle_, - distances_tmp.data(), - candidates.data_handle(), - queries.data_handle(), - dataset.data_handle(), - p.n_queries, - p.n_rows, - p.dim, - p.k0, - p.metric); - resource::sync_stream(handle_, stream_); - } - - if (p.host_data) { - dataset_host = raft::make_host_matrix(p.n_rows, p.dim); - queries_host = raft::make_host_matrix(p.n_queries, p.dim); - candidates_host = raft::make_host_matrix(p.n_queries, p.k0); - - raft::copy(dataset_host.data_handle(), dataset.data_handle(), dataset.size(), stream_); - raft::copy(queries_host.data_handle(), queries.data_handle(), queries.size(), stream_); - raft::copy( - candidates_host.data_handle(), candidates.data_handle(), candidates.size(), stream_); - - refined_distances_host = raft::make_host_matrix(p.n_queries, p.k); - refined_indices_host = raft::make_host_matrix(p.n_queries, p.k); - resource::sync_stream(handle_, stream_); - } - - // Generate ground thruth for testing. - { - rmm::device_uvector distances_dev(p.n_queries * p.k, stream_); - rmm::device_uvector indices_dev(p.n_queries * p.k, stream_); - naive_knn(handle_, - distances_dev.data(), - indices_dev.data(), - queries.data_handle(), - dataset.data_handle(), - p.n_queries, - p.n_rows, - p.dim, - p.k, - p.metric); - true_refined_distances_host.resize(p.n_queries * p.k); - true_refined_indices_host.resize(p.n_queries * p.k); - raft::copy(true_refined_indices_host.data(), indices_dev.data(), indices_dev.size(), stream_); - raft::copy( - true_refined_distances_host.data(), distances_dev.data(), distances_dev.size(), stream_); - resource::sync_stream(handle_, stream_); - } - } - - public: - RefineInputs p; - const raft::resources& handle_; - rmm::cuda_stream_view stream_; - - raft::device_matrix dataset; - raft::device_matrix queries; - raft::device_matrix candidates; // Neighbor candidate indices - raft::device_matrix refined_indices; - raft::device_matrix refined_distances; - - raft::host_matrix dataset_host; - raft::host_matrix queries_host; - raft::host_matrix candidates_host; - raft::host_matrix refined_indices_host; - raft::host_matrix refined_distances_host; - - std::vector true_refined_indices_host; - std::vector true_refined_distances_host; -}; -} // namespace raft::neighbors diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_00_generate.py b/cpp/src/distance/detail/pairwise_matrix/dispatch_00_generate.py deleted file mode 100644 index 6adff0eee1..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_00_generate.py +++ /dev/null @@ -1,199 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# NOTE: this template is not perfectly formatted. Use pre-commit to get -# everything in shape again. -header = """/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -""" - - -macro = """ -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \\ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \\ - template void raft::distance::detail:: \\ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \\ - OpT distance_op, \\ - IdxT m, \\ - IdxT n, \\ - IdxT k, \\ - const DataT* x, \\ - const DataT* y, \\ - const DataT* x_norm, \\ - const DataT* y_norm, \\ - OutT* out, \\ - FinOpT fin_op, \\ - cudaStream_t stream, \\ - bool is_row_major) -""" - -data_type_instances = [ - dict( - DataT="float", - AccT="float", - OutT="float", - IdxT="int", - ), - dict( - DataT="double", - AccT="double", - OutT="double", - IdxT="int", - ), -] - -op_instances = [ - dict( - path_prefix="canberra", - OpT="raft::distance::detail::ops::canberra_distance_op", - archs = [60], - ), - dict( - path_prefix="correlation", - OpT="raft::distance::detail::ops::correlation_distance_op", - archs = [60], - ), - dict( - path_prefix="cosine", - OpT="raft::distance::detail::ops::cosine_distance_op", - archs = [60, 80], - ), - dict( - path_prefix="dice", - OpT="raft::distance::detail::ops::dice_distance_op", - archs = [60, 80], - ), - dict( - path_prefix="hamming_unexpanded", - OpT="raft::distance::detail::ops::hamming_distance_op", - archs = [60], - ), - dict( - path_prefix="hellinger_expanded", - OpT="raft::distance::detail::ops::hellinger_distance_op", - archs = [60], - ), - # inner product is handled by cublas. - dict( - path_prefix="jensen_shannon", - OpT="raft::distance::detail::ops::jensen_shannon_distance_op", - archs = [60], - ), - dict( - path_prefix="kl_divergence", - OpT="raft::distance::detail::ops::kl_divergence_op", - archs = [60], - ), - dict( - path_prefix="l1", - OpT="raft::distance::detail::ops::l1_distance_op", - archs = [60], - ), - dict( - path_prefix="l2_expanded", - OpT="raft::distance::detail::ops::l2_exp_distance_op", - archs = [60, 80], - ), - dict( - path_prefix="l2_unexpanded", - OpT="raft::distance::detail::ops::l2_unexp_distance_op", - archs = [60], - ), - dict( - path_prefix="l_inf", - OpT="raft::distance::detail::ops::l_inf_distance_op", - archs = [60], - ), - dict( - path_prefix="lp_unexpanded", - OpT="raft::distance::detail::ops::lp_unexp_distance_op", - archs = [60], - ), - dict( - path_prefix="russel_rao", - OpT="raft::distance::detail::ops::russel_rao_distance_op", - archs = [60], - ), -] - -def arch_headers(archs): - include_headers ="\n".join([ - f"#include " - for arch in archs - ]) - return include_headers - - - -for op in op_instances: - for dt in data_type_instances: - DataT, AccT, OutT, IdxT = (dt[k] for k in ["DataT", "AccT", "OutT", "IdxT"]); - path = f"dispatch_{op['path_prefix']}_{DataT}_{AccT}_{OutT}_{IdxT}.cu" - with open(path, "w") as f: - f.write(header) - f.write(arch_headers(op["archs"])) - f.write(macro) - - OpT = op['OpT'] - FinOpT = "raft::identity_op" - f.write(f"\ninstantiate_raft_distance_detail_pairwise_matrix_dispatch({OpT}, {DataT}, {AccT}, {OutT}, {FinOpT}, {IdxT});\n") - f.write("\n#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch\n") - print(f"src/distance/detail/pairwise_matrix/{path}") - -# Dispatch kernels for with the RBF fin op. -with open("dispatch_rbf.cu", "w") as f: - OpT="raft::distance::detail::ops::l2_unexp_distance_op" - archs = [60] - - f.write(header) - f.write("#include // rbf_fin_op\n") - f.write(arch_headers(archs)) - f.write(macro) - - for dt in data_type_instances: - DataT, AccT, OutT, IdxT = (dt[k] for k in ["DataT", "AccT", "OutT", "IdxT"]); - IdxT = "int64_t" # overwrite IdxT - - FinOpT = f"raft::distance::kernels::detail::rbf_fin_op<{DataT}>" - f.write(f"\ninstantiate_raft_distance_detail_pairwise_matrix_dispatch({OpT}, {DataT}, {AccT}, {OutT}, {FinOpT}, {IdxT});\n") - - f.write("\n#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch\n") - -print("src/distance/detail/pairwise_matrix/dispatch_rbf.cu") diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_canberra_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_canberra_double_double_double_int.cu deleted file mode 100644 index 41db12e9ae..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_canberra_double_double_double_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::canberra_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_canberra_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_canberra_float_float_float_int.cu deleted file mode 100644 index f038e53381..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_canberra_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::canberra_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_correlation_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_correlation_double_double_double_int.cu deleted file mode 100644 index 52e4cc02d8..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_correlation_double_double_double_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::correlation_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_correlation_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_correlation_float_float_float_int.cu deleted file mode 100644 index c9481d6c22..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_correlation_float_float_float_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::correlation_distance_op, - float, - float, - float, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_cosine_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_cosine_double_double_double_int.cu deleted file mode 100644 index 517858125b..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_cosine_double_double_double_int.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::cosine_distance_op, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_cosine_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_cosine_float_float_float_int.cu deleted file mode 100644 index 62f1d9874b..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_cosine_float_float_float_int.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::cosine_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_dice_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_dice_double_double_double_int.cu deleted file mode 100644 index a259f8b3b0..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_dice_double_double_double_int.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::dice_distance_op, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_dice_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_dice_float_float_float_int.cu deleted file mode 100644 index e89f8b422c..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_dice_float_float_float_int.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::dice_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_double_double_double_int.cu deleted file mode 100644 index 500f7b4a9c..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_double_double_double_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hamming_distance_op, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_float_float_float_int.cu deleted file mode 100644 index 3be7586b43..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hamming_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_double_double_double_int.cu deleted file mode 100644 index 023134ddff..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_double_double_double_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hellinger_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_float_float_float_int.cu deleted file mode 100644 index e438f121f2..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hellinger_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_double_double_double_int.cu deleted file mode 100644 index 31c5003ad6..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_double_double_double_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::jensen_shannon_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_float_float_float_int.cu deleted file mode 100644 index e78c1c320a..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_float_float_float_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::jensen_shannon_distance_op, - float, - float, - float, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_kl_divergence_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_kl_divergence_double_double_double_int.cu deleted file mode 100644 index 5b95df9614..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_kl_divergence_double_double_double_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::kl_divergence_op, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_kl_divergence_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_kl_divergence_float_float_float_int.cu deleted file mode 100644 index fb72c91b73..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_kl_divergence_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::kl_divergence_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l1_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l1_double_double_double_int.cu deleted file mode 100644 index cac5acad92..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l1_double_double_double_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l1_distance_op, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l1_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l1_float_float_float_int.cu deleted file mode 100644 index 78aa097961..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l1_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l1_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_expanded_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_expanded_double_double_double_int.cu deleted file mode 100644 index c8d922f6fa..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_expanded_double_double_double_int.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_exp_distance_op, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_expanded_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_expanded_float_float_float_int.cu deleted file mode 100644 index 20cf57f898..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_expanded_float_float_float_int.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_exp_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_double_double_double_int.cu deleted file mode 100644 index eadd0d2c2b..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_double_double_double_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_float_float_float_int.cu deleted file mode 100644 index e4b5dd3a86..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l_inf_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l_inf_double_double_double_int.cu deleted file mode 100644 index 45d021bce9..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l_inf_double_double_double_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l_inf_distance_op, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_l_inf_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_l_inf_float_float_float_int.cu deleted file mode 100644 index ba48e52a18..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_l_inf_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l_inf_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_double_double_double_int.cu deleted file mode 100644 index ffa58793d9..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_double_double_double_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::lp_unexp_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_float_float_float_int.cu deleted file mode 100644 index 915c68f05f..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::lp_unexp_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_rbf.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_rbf.cu deleted file mode 100644 index 15855cea0a..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_rbf.cu +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // rbf_fin_op -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, - float, - float, - float, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, - double, - double, - double, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_russel_rao_double_double_double_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_russel_rao_double_double_double_int.cu deleted file mode 100644 index db45dc8b94..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_russel_rao_double_double_double_int.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::russel_rao_distance_op, - double, - double, - double, - raft::identity_op, - int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/detail/pairwise_matrix/dispatch_russel_rao_float_float_float_int.cu b/cpp/src/distance/detail/pairwise_matrix/dispatch_russel_rao_float_float_float_int.cu deleted file mode 100644 index a2a5a9fafe..0000000000 --- a/cpp/src/distance/detail/pairwise_matrix/dispatch_russel_rao_float_float_float_int.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by dispatch_00_generate.py - * - * Make changes there and run in this directory: - * - * > python dispatch_00_generate.py - * - */ - -#include // raft::identity_op -#include // ops::* -#include // dispatch -#include -#define instantiate_raft_distance_detail_pairwise_matrix_dispatch( \ - OpT, DataT, AccT, OutT, FinOpT, IdxT) \ - template void raft::distance::detail:: \ - pairwise_matrix_dispatch, DataT, AccT, OutT, FinOpT, IdxT>( \ - OpT distance_op, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - const DataT* x, \ - const DataT* y, \ - const DataT* x_norm, \ - const DataT* y_norm, \ - OutT* out, \ - FinOpT fin_op, \ - cudaStream_t stream, \ - bool is_row_major) - -instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::russel_rao_distance_op, float, float, float, raft::identity_op, int); - -#undef instantiate_raft_distance_detail_pairwise_matrix_dispatch diff --git a/cpp/src/distance/distance.cu b/cpp/src/distance/distance.cu deleted file mode 100644 index 8fe0bf2007..0000000000 --- a/cpp/src/distance/distance.cu +++ /dev/null @@ -1,982 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // rbf_fin_op -#include - -/* - * Hierarchy of instantiations: - * - * This file defines the template instantiations for the public API of - * raft::distance. To improve compile times, the compilation of the distance - * kernels is handled in distance/detail/pairwise_matrix/dispatch_*.cu. - * - */ - -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, FinalLambda, IdxT) \ - template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - size_t worksize, \ - FinalLambda fin_op, \ - bool isRowMajor, \ - DataT metric_arg) - -// The following two instances are used in test/distance/gram.cu. Note the use -// of int64_t for the index type. -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - float, - float, - float, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - raft::distance::kernels::detail::rbf_fin_op, - int64_t); - -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - raft::identity_op, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, double, double, double, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, raft::identity_op, int); - -#undef instantiate_raft_distance_distance - -// Same, but without raft::identity_op -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, IdxT) \ - template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - size_t worksize, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, int); - -#undef instantiate_raft_distance_distance - -// Same, but without workspace -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, IdxT) \ - template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, double, double, double, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, int); - -#undef instantiate_raft_distance_distance - -#define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT) \ - template size_t raft::distance::getWorkspaceSize( \ - const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) - -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CorrelationExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CosineExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CosineExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HellingerExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HellingerExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Linf, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Linf, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::LpUnexpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::LpUnexpanded, double, double, double, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, int); - -#undef instantiate_raft_distance_getWorkspaceSize - -#define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT, layout) \ - template size_t raft::distance::getWorkspaceSize( \ - raft::device_matrix_view const& x, \ - raft::device_matrix_view const& y) - -// We could consider not taking template parameters for this function. The -// number of instantiations seems a bit excessive.. -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::DiceExpanded, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - int, - raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - int, - raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int, raft::layout_f_contiguous); - -#undef instantiate_raft_distance_getWorkspaceSize - -#define instantiate_raft_distance_pairwise_distance(DataT, IdxT) \ - template void raft::distance::pairwise_distance(raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - DataT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - rmm::device_uvector& workspace, \ - raft::distance::DistanceType metric, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_pairwise_distance(float, int); -instantiate_raft_distance_pairwise_distance(double, int); - -#undef instantiate_raft_distance_pairwise_distance - -// Same, but without workspace -#define instantiate_raft_distance_pairwise_distance(DataT, IdxT) \ - template void raft::distance::pairwise_distance(raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - DataT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - raft::distance::DistanceType metric, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_pairwise_distance(float, int); -instantiate_raft_distance_pairwise_distance(double, int); - -#undef instantiate_raft_distance_pairwise_distance - -// Version with mdspan -#define instantiate_raft_distance_distance(DistT, DataT, AccT, OutT, layout, IdxT) \ - template void raft::distance::distance( \ - raft::resources const& handle, \ - raft::device_matrix_view const x, \ - raft::device_matrix_view const y, \ - raft::device_matrix_view dist, \ - DataT metric_arg) - -// Again, we might want to consider reigning in the number of instantiations... -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::DiceExpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::DiceExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::InnerProduct, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::JensenShannon, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::KLDivergence, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::LpUnexpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::LpUnexpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - float, - float, - float, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - double, - double, - double, - raft::layout_c_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - float, - float, - float, - raft::layout_f_contiguous, - int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, - double, - double, - double, - raft::layout_f_contiguous, - int); - -#undef instantiate_raft_distance_distance - -#define instantiate_raft_distance_pairwise_distance(DataT, layout, IdxT) \ - template void raft::distance::pairwise_distance( \ - raft::resources const& handle, \ - raft::device_matrix_view const x, \ - raft::device_matrix_view const y, \ - raft::device_matrix_view dist, \ - raft::distance::DistanceType metric, \ - DataT metric_arg) - -instantiate_raft_distance_pairwise_distance(float, raft::layout_c_contiguous, int); -instantiate_raft_distance_pairwise_distance(float, raft::layout_f_contiguous, int); -instantiate_raft_distance_pairwise_distance(double, raft::layout_c_contiguous, int); -instantiate_raft_distance_pairwise_distance(double, raft::layout_f_contiguous, int); - -#undef instantiate_raft_distance_pairwise_distance diff --git a/cpp/src/distance/fused_distance_nn.cu b/cpp/src/distance/fused_distance_nn.cu deleted file mode 100644 index dc722d929c..0000000000 --- a/cpp/src/distance/fused_distance_nn.cu +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // raft::KeyValuePair -#include - -#include // int64_t - -#define instantiate_raft_distance_fusedDistanceNNMinReduce(DataT, OutT, IdxT) \ - template void raft::distance::fusedDistanceNNMinReduce( \ - OutT * min, \ - const DataT* x, \ - const DataT* y, \ - const DataT* xn, \ - const DataT* yn, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - bool sqrt, \ - bool initOutBuffer, \ - bool isRowMajor, \ - raft::distance::DistanceType metric, \ - float metric_arg, \ - cudaStream_t stream) - -instantiate_raft_distance_fusedDistanceNNMinReduce(float, float, int); -instantiate_raft_distance_fusedDistanceNNMinReduce(float, float, int64_t); - -// We can't have comma's in the macro expansion, so we use the COMMA macro: -#define COMMA , - -instantiate_raft_distance_fusedDistanceNNMinReduce(float, raft::KeyValuePair, int); -instantiate_raft_distance_fusedDistanceNNMinReduce(float, - raft::KeyValuePair, - int64_t); - -#undef COMMA - -#undef instantiate_raft_distance_fusedDistanceNNMinReduce diff --git a/cpp/src/distance/fused_l2_nn.cu b/cpp/src/distance/fused_l2_nn.cu deleted file mode 100644 index f29ab08dc1..0000000000 --- a/cpp/src/distance/fused_l2_nn.cu +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // raft::KeyValuePair -#include - -#include // int64_t - -#define instantiate_raft_distance_fusedL2NNMinReduce(DataT, OutT, IdxT) \ - template void raft::distance::fusedL2NNMinReduce(OutT * min, \ - const DataT* x, \ - const DataT* y, \ - const DataT* xn, \ - const DataT* yn, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - bool sqrt, \ - bool initOutBuffer, \ - cudaStream_t stream) - -instantiate_raft_distance_fusedL2NNMinReduce(double, double, int); -instantiate_raft_distance_fusedL2NNMinReduce(double, double, int64_t); -instantiate_raft_distance_fusedL2NNMinReduce(float, float, int); -instantiate_raft_distance_fusedL2NNMinReduce(float, float, int64_t); - -// We can't have comma's in the macro expansion, so we use the COMMA macro: -#define COMMA , - -instantiate_raft_distance_fusedL2NNMinReduce(double, raft::KeyValuePair, int); -instantiate_raft_distance_fusedL2NNMinReduce(double, - raft::KeyValuePair, - int64_t); -instantiate_raft_distance_fusedL2NNMinReduce(float, raft::KeyValuePair, int); -instantiate_raft_distance_fusedL2NNMinReduce(float, - raft::KeyValuePair, - int64_t); - -#undef COMMA - -#undef instantiate_raft_distance_fusedL2NNMinReduce diff --git a/cpp/src/matrix/detail/select_k_double_int64_t.cu b/cpp/src/matrix/detail/select_k_double_int64_t.cu deleted file mode 100644 index bf234aacbf..0000000000 --- a/cpp/src/matrix/detail/select_k_double_int64_t.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) - -instantiate_raft_matrix_detail_select_k(double, int64_t); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/src/matrix/detail/select_k_double_uint32_t.cu b/cpp/src/matrix/detail/select_k_double_uint32_t.cu deleted file mode 100644 index 7f0511a76a..0000000000 --- a/cpp/src/matrix/detail/select_k_double_uint32_t.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include // uint32_t - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) - -instantiate_raft_matrix_detail_select_k(double, uint32_t); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/src/matrix/detail/select_k_float_int32.cu b/cpp/src/matrix/detail/select_k_float_int32.cu deleted file mode 100644 index e68b1e32df..0000000000 --- a/cpp/src/matrix/detail/select_k_float_int32.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) - -instantiate_raft_matrix_detail_select_k(float, int); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/src/matrix/detail/select_k_float_int64_t.cu b/cpp/src/matrix/detail/select_k_float_int64_t.cu deleted file mode 100644 index 5aa40d8c9d..0000000000 --- a/cpp/src/matrix/detail/select_k_float_int64_t.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) - -instantiate_raft_matrix_detail_select_k(float, int64_t); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/src/matrix/detail/select_k_float_uint32_t.cu b/cpp/src/matrix/detail/select_k_float_uint32_t.cu deleted file mode 100644 index 9aba147edf..0000000000 --- a/cpp/src/matrix/detail/select_k_float_uint32_t.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) - -instantiate_raft_matrix_detail_select_k(float, uint32_t); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/src/matrix/detail/select_k_half_int64_t.cu b/cpp/src/matrix/detail/select_k_half_int64_t.cu deleted file mode 100644 index bc513e4aeb..0000000000 --- a/cpp/src/matrix/detail/select_k_half_int64_t.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) - -instantiate_raft_matrix_detail_select_k(__half, int64_t); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/src/matrix/detail/select_k_half_uint32_t.cu b/cpp/src/matrix/detail/select_k_half_uint32_t.cu deleted file mode 100644 index e46c7d46bb..0000000000 --- a/cpp/src/matrix/detail/select_k_half_uint32_t.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_matrix_detail_select_k(T, IdxT) \ - template void raft::matrix::detail::select_k(raft::resources const& handle, \ - const T* in_val, \ - const IdxT* in_idx, \ - size_t batch_size, \ - size_t len, \ - int k, \ - T* out_val, \ - IdxT* out_idx, \ - bool select_min, \ - bool sorted, \ - raft::matrix::SelectAlgo algo, \ - const IdxT* len_i) - -instantiate_raft_matrix_detail_select_k(__half, uint32_t); - -#undef instantiate_raft_matrix_detail_select_k diff --git a/cpp/src/neighbors/ball_cover.cu b/cpp/src/neighbors/ball_cover.cu deleted file mode 100644 index e9c78f8e7c..0000000000 --- a/cpp/src/neighbors/ball_cover.cu +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -#define instantiate_raft_neighbors_ball_cover(idx_t, value_t, int_t, matrix_idx_t) \ - template void raft::neighbors::ball_cover::build_index( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& index); \ - \ - template void raft::neighbors::ball_cover::eps_nn( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_matrix_view adj, \ - raft::device_vector_view vd, \ - raft::device_matrix_view query, \ - value_t eps); \ - \ - template void raft::neighbors::ball_cover::eps_nn( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_vector_view ia, \ - raft::device_vector_view ja, \ - raft::device_vector_view vd, \ - raft::device_matrix_view query, \ - value_t eps, \ - std::optional> max_k); \ - \ - template void raft::neighbors::ball_cover::all_knn_query( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& index, \ - int_t k, \ - idx_t* inds, \ - value_t* dists, \ - bool perform_post_filtering, \ - float weight); \ - \ - template void raft::neighbors::ball_cover::all_knn_query( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_matrix_view inds, \ - raft::device_matrix_view dists, \ - int_t k, \ - bool perform_post_filtering, \ - float weight); \ - \ - template void raft::neighbors::ball_cover::knn_query( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - int_t k, \ - const value_t* query, \ - int_t n_query_pts, \ - idx_t* inds, \ - value_t* dists, \ - bool perform_post_filtering, \ - float weight); \ - \ - template void raft::neighbors::ball_cover::knn_query( \ - raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ - raft::device_matrix_view query, \ - raft::device_matrix_view inds, \ - raft::device_matrix_view dists, \ - int_t k, \ - bool perform_post_filtering, \ - float weight); - -instantiate_raft_neighbors_ball_cover(int64_t, float, int64_t, int64_t); - -#undef instantiate_raft_neighbors_ball_cover diff --git a/cpp/src/neighbors/brute_force_00_generate.py b/cpp/src/neighbors/brute_force_00_generate.py deleted file mode 100644 index 8ed05dc4c2..0000000000 --- a/cpp/src/neighbors/brute_force_00_generate.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by brute_force_00_generate.py - * - * Make changes there and run in this directory: - * - * > python brute_force_00_generate.py - * - */ - -#include -#include - -""" - -knn_macro = """ -#define instantiate_raft_neighbors_brute_force_knn(idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \\ - template void raft::neighbors::brute_force::knn( \\ - raft::resources const& handle, \\ - std::vector> index, \\ - raft::device_matrix_view search, \\ - raft::device_matrix_view indices, \\ - raft::device_matrix_view distances, \\ - raft::distance::DistanceType metric, \\ - std::optional metric_arg, \\ - std::optional global_id_offset, \\ - epilogue_op distance_epilogue); - -""" - -fused_l2_knn_macro = """ -#define instantiate_raft_neighbors_brute_force_fused_l2_knn(value_t, idx_t, idx_layout, query_layout) \\ - template void raft::neighbors::brute_force::fused_l2_knn( \\ - raft::resources const& handle, \\ - raft::device_matrix_view index, \\ - raft::device_matrix_view query, \\ - raft::device_matrix_view out_inds, \\ - raft::device_matrix_view out_dists, \\ - raft::distance::DistanceType metric); - -""" - -knn_types = dict( - int64_t_float_uint32_t=("int64_t","float","uint32_t"), - int64_t_float_int64_t=("int64_t","float","int64_t"), - int_float_int=("int","float","int"), - uint32_t_float_uint32_t=("uint32_t","float","uint32_t"), -) - -fused_l2_knn_types = dict( - float_int64_t=("float", "int64_t"), -) - -# knn -for type_path, (idx_t, value_t, matrix_idx) in knn_types.items(): - path = f"brute_force_knn_{type_path}.cu" - with open(path, "w") as f: - f.write(header) - f.write(knn_macro) - f.write(f"instantiate_raft_neighbors_brute_force_knn({idx_t},{value_t},{matrix_idx},raft::row_major,raft::row_major,raft::identity_op);\n\n") - f.write("#undef instantiate_raft_neighbors_brute_force_knn\n") - - # For pasting into CMakeLists.txt - print(f"src/neighbors/{path}") - -#fused_l2_knn -for type_path, (value_t, idx_t) in fused_l2_knn_types.items(): - path = f"brute_force_fused_l2_knn_{type_path}.cu" - with open(path, "w") as f: - f.write(header) - f.write(fused_l2_knn_macro) - f.write(f"instantiate_raft_neighbors_brute_force_fused_l2_knn({value_t},{idx_t},raft::row_major,raft::row_major);\n\n") - f.write("#undef instantiate_raft_neighbors_brute_force_fused_l2_knn\n") - - # For pasting into CMakeLists.txt - print(f"src/neighbors/{path}") diff --git a/cpp/src/neighbors/brute_force_fused_l2_knn_float_int64_t.cu b/cpp/src/neighbors/brute_force_fused_l2_knn_float_int64_t.cu deleted file mode 100644 index 4269d27225..0000000000 --- a/cpp/src/neighbors/brute_force_fused_l2_knn_float_int64_t.cu +++ /dev/null @@ -1,46 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by brute_force_00_generate.py - * - * Make changes there and run in this directory: - * - * > python brute_force_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_brute_force_fused_l2_knn( \ - value_t, idx_t, idx_layout, query_layout) \ - template void raft::neighbors::brute_force::fused_l2_knn( \ - raft::resources const& handle, \ - raft::device_matrix_view index, \ - raft::device_matrix_view query, \ - raft::device_matrix_view out_inds, \ - raft::device_matrix_view out_dists, \ - raft::distance::DistanceType metric); - -instantiate_raft_neighbors_brute_force_fused_l2_knn(float, - int64_t, - raft::row_major, - raft::row_major); - -#undef instantiate_raft_neighbors_brute_force_fused_l2_knn diff --git a/cpp/src/neighbors/brute_force_knn_index_float.cu b/cpp/src/neighbors/brute_force_knn_index_float.cu deleted file mode 100644 index de94be4c09..0000000000 --- a/cpp/src/neighbors/brute_force_knn_index_float.cu +++ /dev/null @@ -1,79 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include - -template void raft::neighbors::brute_force::search( - raft::resources const& res, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -template void raft::neighbors::brute_force::search( - raft::resources const& res, - raft::neighbors::brute_force::search_params const& params, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -template void raft::neighbors::brute_force::search( - raft::resources const& res, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -template void raft::neighbors::brute_force::search( - raft::resources const& res, - raft::neighbors::brute_force::search_params const& params, - const raft::neighbors::brute_force::index& idx, - raft::device_matrix_view queries, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances); - -template raft::neighbors::brute_force::index raft::neighbors::brute_force:: - build::accessor_type>( - raft::resources const& res, - raft::host_matrix_view dataset, - raft::distance::DistanceType metric, - float metric_arg); - -template raft::neighbors::brute_force::index raft::neighbors::brute_force:: - build::accessor_type>( - raft::resources const& res, - raft::device_matrix_view dataset, - raft::distance::DistanceType metric, - float metric_arg); - -template raft::neighbors::brute_force::index raft::neighbors::brute_force:: - build::accessor_type>( - raft::resources const& res, - raft::neighbors::brute_force::index_params const& params, - raft::host_matrix_view dataset); - -template raft::neighbors::brute_force::index raft::neighbors::brute_force:: - build::accessor_type>( - raft::resources const& res, - raft::neighbors::brute_force::index_params const& params, - raft::device_matrix_view dataset); diff --git a/cpp/src/neighbors/brute_force_knn_int64_t_float_int64_t.cu b/cpp/src/neighbors/brute_force_knn_int64_t_float_int64_t.cu deleted file mode 100644 index 1c08cf8e82..0000000000 --- a/cpp/src/neighbors/brute_force_knn_int64_t_float_int64_t.cu +++ /dev/null @@ -1,48 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by brute_force_00_generate.py - * - * Make changes there and run in this directory: - * - * > python brute_force_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_brute_force_knn( \ - idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \ - template void raft::neighbors::brute_force:: \ - knn( \ - raft::resources const& handle, \ - std::vector> index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric, \ - std::optional metric_arg, \ - std::optional global_id_offset, \ - epilogue_op distance_epilogue); - -instantiate_raft_neighbors_brute_force_knn( - int64_t, float, int64_t, raft::row_major, raft::row_major, raft::identity_op); - -#undef instantiate_raft_neighbors_brute_force_knn diff --git a/cpp/src/neighbors/brute_force_knn_int64_t_float_uint32_t.cu b/cpp/src/neighbors/brute_force_knn_int64_t_float_uint32_t.cu deleted file mode 100644 index 809cf6eec0..0000000000 --- a/cpp/src/neighbors/brute_force_knn_int64_t_float_uint32_t.cu +++ /dev/null @@ -1,48 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by brute_force_00_generate.py - * - * Make changes there and run in this directory: - * - * > python brute_force_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_brute_force_knn( \ - idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \ - template void raft::neighbors::brute_force:: \ - knn( \ - raft::resources const& handle, \ - std::vector> index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric, \ - std::optional metric_arg, \ - std::optional global_id_offset, \ - epilogue_op distance_epilogue); - -instantiate_raft_neighbors_brute_force_knn( - int64_t, float, uint32_t, raft::row_major, raft::row_major, raft::identity_op); - -#undef instantiate_raft_neighbors_brute_force_knn diff --git a/cpp/src/neighbors/brute_force_knn_int_float_int.cu b/cpp/src/neighbors/brute_force_knn_int_float_int.cu deleted file mode 100644 index 2ffa864dea..0000000000 --- a/cpp/src/neighbors/brute_force_knn_int_float_int.cu +++ /dev/null @@ -1,48 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by brute_force_00_generate.py - * - * Make changes there and run in this directory: - * - * > python brute_force_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_brute_force_knn( \ - idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \ - template void raft::neighbors::brute_force:: \ - knn( \ - raft::resources const& handle, \ - std::vector> index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric, \ - std::optional metric_arg, \ - std::optional global_id_offset, \ - epilogue_op distance_epilogue); - -instantiate_raft_neighbors_brute_force_knn( - int, float, int, raft::row_major, raft::row_major, raft::identity_op); - -#undef instantiate_raft_neighbors_brute_force_knn diff --git a/cpp/src/neighbors/brute_force_knn_uint32_t_float_uint32_t.cu b/cpp/src/neighbors/brute_force_knn_uint32_t_float_uint32_t.cu deleted file mode 100644 index dde92765b5..0000000000 --- a/cpp/src/neighbors/brute_force_knn_uint32_t_float_uint32_t.cu +++ /dev/null @@ -1,48 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by brute_force_00_generate.py - * - * Make changes there and run in this directory: - * - * > python brute_force_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_brute_force_knn( \ - idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \ - template void raft::neighbors::brute_force:: \ - knn( \ - raft::resources const& handle, \ - std::vector> index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric, \ - std::optional metric_arg, \ - std::optional global_id_offset, \ - epilogue_op distance_epilogue); - -instantiate_raft_neighbors_brute_force_knn( - uint32_t, float, uint32_t, raft::row_major, raft::row_major, raft::identity_op); - -#undef instantiate_raft_neighbors_brute_force_knn diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_00_generate.py b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_00_generate.py deleted file mode 100644 index e827c06be5..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_00_generate.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -""" - -trailer = """ -} // namespace raft::neighbors::cagra::detail::multi_cta_search -""" - -mxdim_team = [(128, 8), (256, 16), (512, 32), (1024, 32)] -pq_bits = [8] -subspace_dims = [2, 4] -# block = [(64, 16), (128, 8), (256, 4), (512, 2), (1024, 1)] -# mxelem = [64, 128, 256] -load_types = ["uint4"] -code_book_types = ["half"] -search_types = dict( - float_uint32=( - "float", - "uint32_t", - "float", - ), # data_t, vec_idx_t, distance_t - half_uint32=("half", "uint32_t", "float"), - int8_uint32=("int8_t", "uint32_t", "float"), - uint8_uint32=("uint8_t", "uint32_t", "float"), - float_uint64=("float", "uint64_t", "float"), - half_uint64=("half", "uint64_t", "float"), -) -# knn -for type_path, (data_t, idx_t, distance_t) in search_types.items(): - for (mxdim, team) in mxdim_team: - for code_book_t in code_book_types: - for subspace_dim in subspace_dims: - for pq_bit in pq_bits: - path = f"q_search_multi_cta_{type_path}_dim{mxdim}_t{team}_{pq_bit}pq_{subspace_dim}subd_{code_book_t}.cu" - with open(path, "w") as f: - f.write(header) - f.write( - f"instantiate_kernel_selection(\n {team}, {mxdim}, raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t<{data_t} COMMA {code_book_t} COMMA {pq_bit} COMMA {subspace_dim} COMMA {distance_t} COMMA {idx_t}>, raft::neighbors::filtering::none_cagra_sample_filter);\n" - ) - f.write(trailer) - # For pasting into CMakeLists.txt - print(f"src/neighbors/detail/cagra/{path}") diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 0bd386144c..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index cd891b8e97..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 66e8357498..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index eb84983f9e..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index c66f8a0ae3..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 2a1783944c..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 9fa74f1134..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 8fc91b5a10..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 4e68c00525..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 5fe526ae47..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 64c89a880a..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index c3e2427f57..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 0a8826df1c..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 8019bec3e3..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 1a2a364037..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 2f661538e6..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_float_uint64_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index aec486769f..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 03f27085d8..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 119d1f2921..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 666c676e87..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index e53b456a54..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 2aee739141..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index daa442b514..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index a19346d19b..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 1c1d5381c9..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index b7402a3c38..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index f493b83bee..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 8efcbe0650..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index cb770f44ba..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 0fd8ab809c..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 50cf198883..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 1548ed831e..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_half_uint64_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index c60ea7c87d..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 4a68e1e43c..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index df9fabd6a5..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 77075b0a44..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 374af8b56b..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index ddb80458fd..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 14e5c5d3dc..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 3c1776760a..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index e5a0a8882c..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index cee80390e8..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 88678bf4ff..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index baa7ee358a..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 5c44f052f2..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 127a065fb5..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index fcf6985f97..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index f361e771b5..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_multi_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_00_generate.py b/cpp/src/neighbors/detail/cagra/q_search_single_cta_00_generate.py deleted file mode 100644 index 418d528a82..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_00_generate.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -""" - -trailer = """ -} // namespace raft::neighbors::cagra::detail::single_cta_search -""" - -mxdim_team = [(128, 8), (256, 16), (512, 32), (1024, 32)] -# block = [(64, 16), (128, 8), (256, 4), (512, 2), (1024, 1)] -# itopk_candidates = [64, 128, 256] -# itopk_size = [64, 128, 256, 512] -# mxelem = [64, 128, 256] - -pq_bits = [8] -subspace_dims = [2, 4] - -# rblock = [(256, 4), (512, 2), (1024, 1)] -# rcandidates = [32] -# rsize = [256, 512] -code_book_types = ["half"] - -search_types = dict( - float_uint32=("float", "uint32_t", "float"), # data_t, idx_t, distance_t - half_uint32=("half", "uint32_t", "float"), - int8_uint32=("int8_t", "uint32_t", "float"), - uint8_uint32=("uint8_t", "uint32_t", "float"), - float_uint64=("float", "uint64_t", "float"), - half_uint64=("half", "uint64_t", "float"), -) - -# knn -for type_path, (data_t, idx_t, distance_t) in search_types.items(): - for (mxdim, team) in mxdim_team: - for code_book_t in code_book_types: - for subspace_dim in subspace_dims: - for pq_bit in pq_bits: - path = f"q_search_single_cta_{type_path}_dim{mxdim}_t{team}_{pq_bit}pq_{subspace_dim}subd_{code_book_t}.cu" - with open(path, "w") as f: - f.write(header) - f.write( - f"instantiate_kernel_selection(\n {team}, {mxdim}, raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t<{data_t} COMMA {code_book_t} COMMA {pq_bit} COMMA {subspace_dim} COMMA {distance_t} COMMA {idx_t}>, raft::neighbors::filtering::none_cagra_sample_filter);\n" - ) - - f.write(trailer) - # For pasting into CMakeLists.txt - print(f"src/neighbors/detail/cagra/{path}") diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index d61ad0ce15..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 410d2377ec..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 60cd58bab9..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index dfe5e6f14e..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 9a5d862276..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index d92ab50a58..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index aac197d590..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index f38a10e6d0..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 5523e63038..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index b06ef3d4fd..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 1fddee0e06..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 2aee442186..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 7a15e85280..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index efba46c248..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 990582f18b..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index a55907c66f..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_float_uint64_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - float COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 55fd749720..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 4b4063652a..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index bae83dc0fa..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 99492db344..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 797142e317..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 9a36c35ae0..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index e0a01e84cc..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 14de1b8941..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index b1d50fb445..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index c189a91764..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 8693ee3716..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 216ffd1ec5..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 36985d218b..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 8d55fe2b09..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 2fdb1cbc20..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index 6dc3dc2ca8..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_half_uint64_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - half COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint64_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index 21f8633033..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 1a3867e06f..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 9cbb16188a..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 305a1754bc..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index 900e1b69d9..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index a0bb2259f0..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 09d36a39a0..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index dc9cbb2b56..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_int8_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - int8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu deleted file mode 100644 index c5508a38e2..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu deleted file mode 100644 index 7024425155..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim1024_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 1024, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu deleted file mode 100644 index 68687bc9cf..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu deleted file mode 100644 index 60efc55a30..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim128_t8_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(8, - 128, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu deleted file mode 100644 index b2dfaac5fe..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu deleted file mode 100644 index 891e9ef7cc..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim256_t16_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(16, - 256, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu deleted file mode 100644 index 91e617204c..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_2subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 2 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu b/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu deleted file mode 100644 index a01d497676..0000000000 --- a/cpp/src/neighbors/detail/cagra/q_search_single_cta_uint8_uint32_dim512_t32_8pq_4subd_half.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by q_search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python q_search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection(32, - 512, - raft::neighbors::cagra::detail::cagra_q_dataset_descriptor_t< - uint8_t COMMA half COMMA 8 COMMA 4 COMMA float COMMA uint32_t>, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta.cuh b/cpp/src/neighbors/detail/cagra/search_multi_cta.cuh deleted file mode 100644 index 542fdaad1f..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta.cuh +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { - -#define instantiate_kernel_selection(TEAM_SIZE, MAX_DATASET_DIM, DATASET_DESC_T, SAMPLE_FILTER_T) \ - template void select_and_run( \ - DATASET_DESC_T dataset_desc, \ - raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t block_size, \ - uint32_t result_buffer_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - typename DATASET_DESC_T::INDEX_T* hashmap_ptr, \ - uint32_t num_cta_per_query, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -#define COMMA , - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_00_generate.py b/cpp/src/neighbors/detail/cagra/search_multi_cta_00_generate.py deleted file mode 100644 index 6f023c39f1..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_00_generate.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -""" - -trailer = """ -} // namespace raft::neighbors::cagra::detail::multi_cta_search -""" - -mxdim_team = [(128, 8), (256, 16), (512, 32), (1024, 32)] -# block = [(64, 16), (128, 8), (256, 4), (512, 2), (1024, 1)] -# mxelem = [64, 128, 256] -load_types = ["uint4"] -search_types = dict( - float_uint32=( - "float", - "uint32_t", - "float", - ), # data_t, vec_idx_t, distance_t - half_uint32=("half", "uint32_t", "float"), - int8_uint32=("int8_t", "uint32_t", "float"), - uint8_uint32=("uint8_t", "uint32_t", "float"), - float_uint64=("float", "uint64_t", "float"), - half_uint64=("half", "uint64_t", "float"), -) -# knn -for type_path, (data_t, idx_t, distance_t) in search_types.items(): - for (mxdim, team) in mxdim_team: - path = f"search_multi_cta_{type_path}_dim{mxdim}_t{team}.cu" - with open(path, "w") as f: - f.write(header) - f.write( - f"instantiate_kernel_selection(\n {team}, {mxdim}, raft::neighbors::cagra::detail::standard_dataset_descriptor_t<{data_t} COMMA {idx_t} COMMA {distance_t}>, raft::neighbors::filtering::none_cagra_sample_filter);\n" - ) - f.write(trailer) - # For pasting into CMakeLists.txt - print(f"src/neighbors/detail/cagra/{path}") diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim1024_t32.cu deleted file mode 100644 index 0e28d7a876..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim128_t8.cu deleted file mode 100644 index 5e5e80a5de..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim256_t16.cu deleted file mode 100644 index 9039496968..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim512_t32.cu deleted file mode 100644 index fe1c7e77e5..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim1024_t32.cu deleted file mode 100644 index 7ef36baf7d..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim128_t8.cu deleted file mode 100644 index da51c16314..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim256_t16.cu deleted file mode 100644 index 99a4f7feb7..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim512_t32.cu deleted file mode 100644 index 50cdc97dd7..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim1024_t32.cu deleted file mode 100644 index b2d9cdb600..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim128_t8.cu deleted file mode 100644 index d756b295b7..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim256_t16.cu deleted file mode 100644 index b1e998762c..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim512_t32.cu deleted file mode 100644 index e712de6390..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim1024_t32.cu deleted file mode 100644 index 282de4a851..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim128_t8.cu deleted file mode 100644 index 71ef968575..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim256_t16.cu deleted file mode 100644 index 7c88406d71..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim512_t32.cu deleted file mode 100644 index 360635dddb..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim1024_t32.cu deleted file mode 100644 index 3f129bd7cf..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim128_t8.cu deleted file mode 100644 index 053b73275e..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim256_t16.cu deleted file mode 100644 index a1bb20369a..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim512_t32.cu deleted file mode 100644 index dbbc8bdd21..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_int8_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim1024_t32.cu deleted file mode 100644 index 125499e319..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim128_t8.cu deleted file mode 100644 index f2117c4f80..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim256_t16.cu deleted file mode 100644 index 8e5ba0f98f..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim512_t32.cu deleted file mode 100644 index bea7d25392..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_uint8_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_multi_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_multi_cta_00_generate.py - * - */ - -#include "search_multi_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::multi_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::multi_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta.cuh b/cpp/src/neighbors/detail/cagra/search_single_cta.cuh deleted file mode 100644 index 855b104670..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta.cuh +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { - -#define instantiate_kernel_selection(TEAM_SIZE, MAX_DATASET_DIM, DATASET_DESC_T, SAMPLE_FILTER_T) \ - template void select_and_run( \ - DATASET_DESC_T dataset_desc, \ - raft::device_matrix_view graph, \ - typename DATASET_DESC_T::INDEX_T* const topk_indices_ptr, \ - typename DATASET_DESC_T::DISTANCE_T* const topk_distances_ptr, \ - const typename DATASET_DESC_T::DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const typename DATASET_DESC_T::INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t num_itopk_candidates, \ - uint32_t block_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - typename DATASET_DESC_T::INDEX_T* hashmap_ptr, \ - size_t small_hash_bitlen, \ - size_t small_hash_reset_interval, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -#define COMMA , - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_00_generate.py b/cpp/src/neighbors/detail/cagra/search_single_cta_00_generate.py deleted file mode 100644 index 0e809e4dc3..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_00_generate.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -""" - -trailer = """ -} // namespace raft::neighbors::cagra::detail::single_cta_search -""" - -mxdim_team = [(128, 8), (256, 16), (512, 32), (1024, 32)] -# block = [(64, 16), (128, 8), (256, 4), (512, 2), (1024, 1)] -# itopk_candidates = [64, 128, 256] -# itopk_size = [64, 128, 256, 512] -# mxelem = [64, 128, 256] - -# rblock = [(256, 4), (512, 2), (1024, 1)] -# rcandidates = [32] -# rsize = [256, 512] - -search_types = dict( - float_uint32=("float", "uint32_t", "float"), # data_t, idx_t, distance_t - half_uint32=("half", "uint32_t", "float"), - int8_uint32=("int8_t", "uint32_t", "float"), - uint8_uint32=("uint8_t", "uint32_t", "float"), - float_uint64=("float", "uint64_t", "float"), - half_uint64=("half", "uint64_t", "float"), -) - -# knn -for type_path, (data_t, idx_t, distance_t) in search_types.items(): - for (mxdim, team) in mxdim_team: - path = f"search_single_cta_{type_path}_dim{mxdim}_t{team}.cu" - with open(path, "w") as f: - f.write(header) - f.write( - f"instantiate_kernel_selection(\n {team}, {mxdim}, raft::neighbors::cagra::detail::standard_dataset_descriptor_t<{data_t} COMMA {idx_t} COMMA {distance_t}>, raft::neighbors::filtering::none_cagra_sample_filter);\n" - ) - - f.write(trailer) - # For pasting into CMakeLists.txt - print(f"src/neighbors/detail/cagra/{path}") diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim1024_t32.cu deleted file mode 100644 index 8a9fc408ee..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim128_t8.cu deleted file mode 100644 index c6f7c90c69..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim256_t16.cu deleted file mode 100644 index 2766286673..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim512_t32.cu deleted file mode 100644 index 98ee189766..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim1024_t32.cu deleted file mode 100644 index c3ea39a729..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim128_t8.cu deleted file mode 100644 index a53457656c..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim256_t16.cu deleted file mode 100644 index 52318efb85..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim512_t32.cu deleted file mode 100644 index 6451fdc7f3..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim1024_t32.cu deleted file mode 100644 index e927fd0878..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim128_t8.cu deleted file mode 100644 index 3f3d22ee08..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim256_t16.cu deleted file mode 100644 index a84e5b8bd7..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim512_t32.cu deleted file mode 100644 index af4248865b..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim1024_t32.cu deleted file mode 100644 index 16bd0cb647..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim128_t8.cu deleted file mode 100644 index afc59c8a59..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim256_t16.cu deleted file mode 100644 index 147d31cf85..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim512_t32.cu deleted file mode 100644 index 5624a71c3c..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim1024_t32.cu deleted file mode 100644 index 761fb705ba..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim128_t8.cu deleted file mode 100644 index 84b76cba53..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim256_t16.cu deleted file mode 100644 index 598fff9cdf..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim512_t32.cu deleted file mode 100644 index e7a1a9d9c6..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_int8_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim1024_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim1024_t32.cu deleted file mode 100644 index d40b9285fc..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim1024_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 1024, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim128_t8.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim128_t8.cu deleted file mode 100644 index 073bb350da..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim128_t8.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 8, - 128, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim256_t16.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim256_t16.cu deleted file mode 100644 index 29b0224b4d..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim256_t16.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 16, - 256, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim512_t32.cu b/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim512_t32.cu deleted file mode 100644 index d9601de2ad..0000000000 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_uint8_uint32_dim512_t32.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by search_single_cta_00_generate.py - * - * Make changes there and run in this directory: - * - * > python search_single_cta_00_generate.py - * - */ - -#include "search_single_cta.cuh" - -#include - -namespace raft::neighbors::cagra::detail::single_cta_search { -instantiate_kernel_selection( - 32, - 512, - raft::neighbors::cagra::detail::standard_dataset_descriptor_t, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace raft::neighbors::cagra::detail::single_cta_search diff --git a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_float_float_int64_t.cu b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_float_float_int64_t.cu deleted file mode 100644 index 5ac820e0dd..0000000000 --- a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_float_float_int64_t.cu +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#define instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( \ - T, AccT, IdxT, IvfSampleFilterT) \ - template void \ - raft::neighbors::ivf_flat::detail::ivfflat_interleaved_scan( \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - const uint32_t* coarse_query_results, \ - const uint32_t n_queries, \ - const uint32_t queries_offset, \ - const raft::distance::DistanceType metric, \ - const uint32_t n_probes, \ - const uint32_t k, \ - const uint32_t max_samples, \ - const uint32_t* chunk_indices, \ - const bool select_min, \ - IvfSampleFilterT sample_filter, \ - uint32_t* neighbors, \ - float* distances, \ - uint32_t& grid_dim_x, \ - rmm::cuda_stream_view stream) - -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - float, float, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan diff --git a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_half_half_int64_t.cu b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_half_half_int64_t.cu deleted file mode 100644 index 4d847cdeb1..0000000000 --- a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_half_half_int64_t.cu +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include - -#define instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( \ - T, AccT, IdxT, IvfSampleFilterT) \ - template void \ - raft::neighbors::ivf_flat::detail::ivfflat_interleaved_scan( \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - const uint32_t* coarse_query_results, \ - const uint32_t n_queries, \ - const uint32_t queries_offset, \ - const raft::distance::DistanceType metric, \ - const uint32_t n_probes, \ - const uint32_t k, \ - const uint32_t max_samples, \ - const uint32_t* chunk_indices, \ - const bool select_min, \ - IvfSampleFilterT sample_filter, \ - uint32_t* neighbors, \ - float* distances, \ - uint32_t& grid_dim_x, \ - rmm::cuda_stream_view stream) - -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - half, half, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan diff --git a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_int8_t_int32_t_int64_t.cu b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_int8_t_int32_t_int64_t.cu deleted file mode 100644 index 8a0e8f0118..0000000000 --- a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_int8_t_int32_t_int64_t.cu +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#define instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( \ - T, AccT, IdxT, IvfSampleFilterT) \ - template void \ - raft::neighbors::ivf_flat::detail::ivfflat_interleaved_scan( \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - const uint32_t* coarse_query_results, \ - const uint32_t n_queries, \ - const uint32_t queries_offset, \ - const raft::distance::DistanceType metric, \ - const uint32_t n_probes, \ - const uint32_t k, \ - const uint32_t max_samples, \ - const uint32_t* chunk_indices, \ - const bool select_min, \ - IvfSampleFilterT sample_filter, \ - uint32_t* neighbors, \ - float* distances, \ - uint32_t& grid_dim_x, \ - rmm::cuda_stream_view stream) - -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - int8_t, int32_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan diff --git a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_uint8_t_uint32_t_int64_t.cu b/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_uint8_t_uint32_t_int64_t.cu deleted file mode 100644 index 7cad992e2b..0000000000 --- a/cpp/src/neighbors/detail/ivf_flat_interleaved_scan_uint8_t_uint32_t_int64_t.cu +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#define instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( \ - T, AccT, IdxT, IvfSampleFilterT) \ - template void \ - raft::neighbors::ivf_flat::detail::ivfflat_interleaved_scan( \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - const uint32_t* coarse_query_results, \ - const uint32_t n_queries, \ - const uint32_t queries_offset, \ - const raft::distance::DistanceType metric, \ - const uint32_t n_probes, \ - const uint32_t k, \ - const uint32_t max_samples, \ - const uint32_t* chunk_indices, \ - const bool select_min, \ - IvfSampleFilterT sample_filter, \ - uint32_t* neighbors, \ - float* distances, \ - uint32_t& grid_dim_x, \ - rmm::cuda_stream_view stream) - -instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - uint8_t, uint32_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan diff --git a/cpp/src/neighbors/detail/ivf_flat_search.cu b/cpp/src/neighbors/detail/ivf_flat_search.cu deleted file mode 100644 index 336bea19b6..0000000000 --- a/cpp/src/neighbors/detail/ivf_flat_search.cu +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include - -#define instantiate_raft_neighbors_ivf_flat_detail_search(T, IdxT, IvfSampleFilterT) \ - template void raft::neighbors::ivf_flat::detail::search( \ - raft::resources const& handle, \ - const search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances, \ - rmm::device_async_resource_ref mr, \ - IvfSampleFilterT sample_filter) - -instantiate_raft_neighbors_ivf_flat_detail_search( - float, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_search( - int8_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); -instantiate_raft_neighbors_ivf_flat_detail_search( - uint8_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef instantiate_raft_neighbors_ivf_flat_detail_search diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_00_generate.py b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_00_generate.py deleted file mode 100644 index 9825a48f81..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_00_generate.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -""" - -none_filter_int64 = "raft::neighbors::filtering::ivf_to_sample_filter" \ - "" -none_filter_int32 = "raft::neighbors::filtering::ivf_to_sample_filter" \ - "" -bitset_filter32 = "raft::neighbors::filtering::ivf_to_sample_filter" \ - ">" -bitset_filter64 = "raft::neighbors::filtering::ivf_to_sample_filter" \ - ">" - -types = dict( - half_fp8_false=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", none_filter_int64), - half_fp8_true=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", none_filter_int64), - half_half=("half", "half", none_filter_int64), - float_half=("float", "half", none_filter_int64), - float_float= ("float", "float", none_filter_int64), - float_fp8_false=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", none_filter_int64), - float_fp8_true=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", none_filter_int64), - half_fp8_false_filt32=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", none_filter_int32), - half_fp8_true_filt32=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", none_filter_int32), - half_half_filt32=("half", "half", none_filter_int32), - float_half_filt32=("float", "half", none_filter_int32), - float_float_filt32= ("float", "float", none_filter_int32), - float_fp8_false_filt32=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", none_filter_int32), - float_fp8_true_filt32=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", none_filter_int32), - half_fp8_false_bitset32=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", bitset_filter32), - half_fp8_true_bitset32=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", bitset_filter32), - half_half_bitset32=("half", "half", bitset_filter32), - float_half_bitset32=("float", "half", bitset_filter32), - float_float_bitset32= ("float", "float", bitset_filter32), - float_fp8_false_bitset32=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", bitset_filter32), - float_fp8_true_bitset32=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", bitset_filter32), - half_fp8_false_bitset64=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", bitset_filter64), - half_fp8_true_bitset64=("half", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", bitset_filter64), - half_half_bitset64=("half", "half", bitset_filter64), - float_half_bitset64=("float", "half", bitset_filter64), - float_float_bitset64= ("float", "float", bitset_filter64), - float_fp8_false_bitset64=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>", bitset_filter64), - float_fp8_true_bitset64=("float", "raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>", bitset_filter64) -) - -for path_key, (OutT, LutT, FilterT) in types.items(): - path = f"ivf_pq_compute_similarity_{path_key}.cu" - with open(path, "w") as f: - f.write(header) - f.write(f"instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select({OutT}, {LutT}, {FilterT});\n") - print(f"src/neighbors/detail/{path}") diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float.cu deleted file mode 100644 index db51608ae1..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset32.cu deleted file mode 100644 index caaf40abdf..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset64.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset64.cu deleted file mode 100644 index 7801c25e9f..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset64.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_filt32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_filt32.cu deleted file mode 100644 index 45ae348849..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_float_filt32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - float, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false.cu deleted file mode 100644 index 2f5bcf8f92..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset32.cu deleted file mode 100644 index e7f2c44254..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset64.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset64.cu deleted file mode 100644 index 01b6900bb8..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset64.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_filt32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_filt32.cu deleted file mode 100644 index 9f8d453364..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_filt32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true.cu deleted file mode 100644 index 06d21bcd50..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset32.cu deleted file mode 100644 index 8b733a23c1..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset64.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset64.cu deleted file mode 100644 index 77e4f9a023..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset64.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_filt32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_filt32.cu deleted file mode 100644 index 3e036e3df4..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_filt32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half.cu deleted file mode 100644 index ff42f5e041..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset32.cu deleted file mode 100644 index 40b6313865..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset64.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset64.cu deleted file mode 100644 index 9cedabdb11..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset64.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_filt32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_filt32.cu deleted file mode 100644 index 61422bbc36..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_float_half_filt32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - float, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false.cu deleted file mode 100644 index d2064cfe97..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset32.cu deleted file mode 100644 index 1127f39f71..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset64.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset64.cu deleted file mode 100644 index 0330bf58d6..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset64.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_filt32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_filt32.cu deleted file mode 100644 index d20f7921d5..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_filt32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true.cu deleted file mode 100644 index 9dc954406e..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset32.cu deleted file mode 100644 index 9131fa25a8..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset64.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset64.cu deleted file mode 100644 index 8b4521b31b..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset64.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_filt32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_filt32.cu deleted file mode 100644 index 71b63cf4a0..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_filt32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half.cu deleted file mode 100644 index f527d879be..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset32.cu deleted file mode 100644 index 8e1962e2bb..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset64.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset64.cu deleted file mode 100644 index e9671703e7..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset64.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::bitset_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_filt32.cu b/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_filt32.cu deleted file mode 100644 index b66a07d1a9..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_compute_similarity_half_half_filt32.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_pq_compute_similarity_00_generate.py - * Make changes there and run in this directory: - * > python ivf_pq_compute_similarity_00_generate.py - */ - -#include -instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( - half, - half, - raft::neighbors::filtering::ivf_to_sample_filter< - uint32_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); diff --git a/cpp/src/neighbors/detail/ivf_pq_search_filtering_float_int64_t.cu b/cpp/src/neighbors/detail/ivf_pq_search_filtering_float_int64_t.cu deleted file mode 100644 index 39af78f12e..0000000000 --- a/cpp/src/neighbors/detail/ivf_pq_search_filtering_float_int64_t.cu +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // raft::device_matrix_view -#include // raft::resources -#include -#include // raft::neighbors::ivf_pq::index -#include -#include - -#include - -#include // int64_t - -#define instantiate_raft_neighbors_ivf_pq_search_with_filtering(T, IdxT, FilterT) \ - template void raft::neighbors::ivf_pq::search_with_filtering( \ - raft::resources const& handle, \ - const search_params& params, \ - const index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances, \ - FilterT sample_filter) - -#define COMMA , -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - float, int64_t, raft::neighbors::filtering::bitset_filter); - -#undef COMMA -#undef instantiate_raft_neighbors_ivf_pq_search_with_filtering diff --git a/cpp/src/neighbors/detail/refine_host_float_float.cpp b/cpp/src/neighbors/detail/refine_host_float_float.cpp deleted file mode 100644 index 09dcae9c3a..0000000000 --- a/cpp/src/neighbors/detail/refine_host_float_float.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include - -#define instantiate_raft_neighbors_refine(IdxT, DataT, DistanceT, ExtentsT) \ - template void raft::neighbors::detail::refine_host( \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - distance::DistanceType metric); - -instantiate_raft_neighbors_refine(int64_t, float, float, int64_t); -instantiate_raft_neighbors_refine(uint32_t, float, float, int64_t); - -#undef instantiate_raft_neighbors_refine diff --git a/cpp/src/neighbors/detail/refine_host_half_float.cpp b/cpp/src/neighbors/detail/refine_host_half_float.cpp deleted file mode 100644 index d9fb2864fe..0000000000 --- a/cpp/src/neighbors/detail/refine_host_half_float.cpp +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include - -#include - -#define instantiate_raft_neighbors_refine(IdxT, DataT, DistanceT, ExtentsT) \ - template void raft::neighbors::detail::refine_host( \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - distance::DistanceType metric); - -instantiate_raft_neighbors_refine(int64_t, half, float, int64_t); - -#undef instantiate_raft_neighbors_refine diff --git a/cpp/src/neighbors/detail/refine_host_int8_t_float.cpp b/cpp/src/neighbors/detail/refine_host_int8_t_float.cpp deleted file mode 100644 index 334a3e8cb6..0000000000 --- a/cpp/src/neighbors/detail/refine_host_int8_t_float.cpp +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_neighbors_refine(IdxT, DataT, DistanceT, ExtentsT) \ - template void raft::neighbors::detail::refine_host( \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - distance::DistanceType metric); -instantiate_raft_neighbors_refine(int64_t, int8_t, float, int64_t); - -#undef instantiate_raft_neighbors_refine diff --git a/cpp/src/neighbors/detail/refine_host_uint8_t_float.cpp b/cpp/src/neighbors/detail/refine_host_uint8_t_float.cpp deleted file mode 100644 index 43d93e5f2e..0000000000 --- a/cpp/src/neighbors/detail/refine_host_uint8_t_float.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_neighbors_refine(IdxT, DataT, DistanceT, ExtentsT) \ - template void raft::neighbors::detail::refine_host( \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - distance::DistanceType metric); - -instantiate_raft_neighbors_refine(int64_t, uint8_t, float, int64_t); - -#undef instantiate_raft_neighbors_refine diff --git a/cpp/src/neighbors/ivf_flat_00_generate.py b/cpp/src/neighbors/ivf_flat_00_generate.py deleted file mode 100644 index 7b55cad4de..0000000000 --- a/cpp/src/neighbors/ivf_flat_00_generate.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include -""" - -types = dict( - float_int64_t=("float", "int64_t"), - int8_t_int64_t=("int8_t", "int64_t"), - uint8_t_int64_t=("uint8_t", "int64_t"), -) - -build_macro = """ -#define instantiate_raft_neighbors_ivf_flat_build(T, IdxT) \\ - template auto raft::neighbors::ivf_flat::build( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::index_params& params, \\ - const T* dataset, \\ - IdxT n_rows, \\ - uint32_t dim) \\ - ->raft::neighbors::ivf_flat::index; \\ - \\ - template auto raft::neighbors::ivf_flat::build( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::index_params& params, \\ - raft::device_matrix_view dataset) \\ - ->raft::neighbors::ivf_flat::index; \\ - \\ - template void raft::neighbors::ivf_flat::build( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::index_params& params, \\ - raft::device_matrix_view dataset, \\ - raft::neighbors::ivf_flat::index& idx); \\ - \\ - template auto raft::neighbors::ivf_flat::build( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::index_params& params, \\ - raft::host_matrix_view dataset) \\ - ->raft::neighbors::ivf_flat::index; \\ - \\ - template void raft::neighbors::ivf_flat::build( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::index_params& params, \\ - raft::host_matrix_view dataset, \\ - raft::neighbors::ivf_flat::index& idx); -""" - -extend_macro = """ -#define instantiate_raft_neighbors_ivf_flat_extend(T, IdxT) \\ - template auto raft::neighbors::ivf_flat::extend( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::index& orig_index, \\ - const T* new_vectors, \\ - const IdxT* new_indices, \\ - IdxT n_rows) \\ - ->raft::neighbors::ivf_flat::index; \\ - \\ - template auto raft::neighbors::ivf_flat::extend( \\ - raft::resources const& handle, \\ - raft::device_matrix_view new_vectors, \\ - std::optional> new_indices, \\ - const raft::neighbors::ivf_flat::index& orig_index) \\ - ->raft::neighbors::ivf_flat::index; \\ - \\ - template void raft::neighbors::ivf_flat::extend( \\ - raft::resources const& handle, \\ - raft::neighbors::ivf_flat::index* index, \\ - const T* new_vectors, \\ - const IdxT* new_indices, \\ - IdxT n_rows); \\ - \\ - template void raft::neighbors::ivf_flat::extend( \\ - raft::resources const& handle, \\ - raft::device_matrix_view new_vectors, \\ - std::optional> new_indices, \\ - raft::neighbors::ivf_flat::index* index); \\ - \\ - template auto raft::neighbors::ivf_flat::extend( \\ - const raft::resources& handle, \\ - raft::host_matrix_view new_vectors, \\ - std::optional> new_indices, \\ - const raft::neighbors::ivf_flat::index& idx) \\ - -> raft::neighbors::ivf_flat::index; \\ - \\ - template void raft::neighbors::ivf_flat::extend( \\ - raft::resources const& handle, \\ - raft::host_matrix_view new_vectors, \\ - std::optional> new_indices, \\ - raft::neighbors::ivf_flat::index* index); -""" - -search_macro = """ -#define instantiate_raft_neighbors_ivf_flat_search(T, IdxT) \\ - template void raft::neighbors::ivf_flat::search( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::search_params& params, \\ - const raft::neighbors::ivf_flat::index& index, \\ - const T* queries, \\ - uint32_t n_queries, \\ - uint32_t k, \\ - IdxT* neighbors, \\ - float* distances, \\ - rmm::device_async_resource_ref mr); \\ - \\ - template void raft::neighbors::ivf_flat::search( \\ - raft::resources const& handle, \\ - const raft::neighbors::ivf_flat::search_params& params, \\ - const raft::neighbors::ivf_flat::index& index, \\ - raft::device_matrix_view queries, \\ - raft::device_matrix_view neighbors, \\ - raft::device_matrix_view distances); -""" - -macros = dict( - build=dict( - definition=build_macro, - name="instantiate_raft_neighbors_ivf_flat_build", - ), - extend=dict( - definition=extend_macro, - name="instantiate_raft_neighbors_ivf_flat_extend", - ), - search=dict( - definition=search_macro, - name="instantiate_raft_neighbors_ivf_flat_search", - ), -) - -for type_path, (T, IdxT) in types.items(): - for macro_path, macro in macros.items(): - path = f"ivf_flat_{macro_path}_{type_path}.cu" - with open(path, "w") as f: - f.write(header) - f.write(macro["definition"]) - - f.write(f"{macro['name']}({T}, {IdxT});\n\n") - f.write(f"#undef {macro['name']}\n") - - print(f"src/neighbors/{path}") diff --git a/cpp/src/neighbors/ivf_flat_build_float_int64_t.cu b/cpp/src/neighbors/ivf_flat_build_float_int64_t.cu deleted file mode 100644 index cf3cb6b1b2..0000000000 --- a/cpp/src/neighbors/ivf_flat_build_float_int64_t.cu +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_ivf_flat_build(T, IdxT) \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); \ - \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); -instantiate_raft_neighbors_ivf_flat_build(float, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_build diff --git a/cpp/src/neighbors/ivf_flat_build_int8_t_int64_t.cu b/cpp/src/neighbors/ivf_flat_build_int8_t_int64_t.cu deleted file mode 100644 index e1cf64907e..0000000000 --- a/cpp/src/neighbors/ivf_flat_build_int8_t_int64_t.cu +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_ivf_flat_build(T, IdxT) \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); \ - \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); -instantiate_raft_neighbors_ivf_flat_build(int8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_build diff --git a/cpp/src/neighbors/ivf_flat_build_uint8_t_int64_t.cu b/cpp/src/neighbors/ivf_flat_build_uint8_t_int64_t.cu deleted file mode 100644 index 26d1647954..0000000000 --- a/cpp/src/neighbors/ivf_flat_build_uint8_t_int64_t.cu +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_ivf_flat_build(T, IdxT) \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); \ - \ - template auto raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::host_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); -instantiate_raft_neighbors_ivf_flat_build(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_build diff --git a/cpp/src/neighbors/ivf_flat_extend_float_int64_t.cu b/cpp/src/neighbors/ivf_flat_extend_float_int64_t.cu deleted file mode 100644 index 16472c6692..0000000000 --- a/cpp/src/neighbors/ivf_flat_extend_float_int64_t.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_ivf_flat_extend(T, IdxT) \ - template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index& orig_index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& orig_index) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_flat::index* index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); \ - \ - template auto raft::neighbors::ivf_flat::extend( \ - const raft::resources& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& idx) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); -instantiate_raft_neighbors_ivf_flat_extend(float, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_extend diff --git a/cpp/src/neighbors/ivf_flat_extend_int8_t_int64_t.cu b/cpp/src/neighbors/ivf_flat_extend_int8_t_int64_t.cu deleted file mode 100644 index d98b5225c3..0000000000 --- a/cpp/src/neighbors/ivf_flat_extend_int8_t_int64_t.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_ivf_flat_extend(T, IdxT) \ - template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index& orig_index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& orig_index) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_flat::index* index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); \ - \ - template auto raft::neighbors::ivf_flat::extend( \ - const raft::resources& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& idx) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); -instantiate_raft_neighbors_ivf_flat_extend(int8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_extend diff --git a/cpp/src/neighbors/ivf_flat_extend_uint8_t_int64_t.cu b/cpp/src/neighbors/ivf_flat_extend_uint8_t_int64_t.cu deleted file mode 100644 index 520c3be536..0000000000 --- a/cpp/src/neighbors/ivf_flat_extend_uint8_t_int64_t.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_ivf_flat_extend(T, IdxT) \ - template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index& orig_index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template auto raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& orig_index) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_flat::index* index, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); \ - \ - template auto raft::neighbors::ivf_flat::extend( \ - const raft::resources& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& idx) \ - ->raft::neighbors::ivf_flat::index; \ - \ - template void raft::neighbors::ivf_flat::extend( \ - raft::resources const& handle, \ - raft::host_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); -instantiate_raft_neighbors_ivf_flat_extend(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_extend diff --git a/cpp/src/neighbors/ivf_flat_search_float_int64_t.cu b/cpp/src/neighbors/ivf_flat_search_float_int64_t.cu deleted file mode 100644 index e5cfe14e3f..0000000000 --- a/cpp/src/neighbors/ivf_flat_search_float_int64_t.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_ivf_flat_search(T, IdxT) \ - template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances, \ - rmm::device_async_resource_ref mr); \ - \ - template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); -instantiate_raft_neighbors_ivf_flat_search(float, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_search diff --git a/cpp/src/neighbors/ivf_flat_search_int8_t_int64_t.cu b/cpp/src/neighbors/ivf_flat_search_int8_t_int64_t.cu deleted file mode 100644 index 35792a78a8..0000000000 --- a/cpp/src/neighbors/ivf_flat_search_int8_t_int64_t.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_ivf_flat_search(T, IdxT) \ - template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances, \ - rmm::device_async_resource_ref mr); \ - \ - template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); -instantiate_raft_neighbors_ivf_flat_search(int8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_search diff --git a/cpp/src/neighbors/ivf_flat_search_uint8_t_int64_t.cu b/cpp/src/neighbors/ivf_flat_search_uint8_t_int64_t.cu deleted file mode 100644 index 663e52cb99..0000000000 --- a/cpp/src/neighbors/ivf_flat_search_uint8_t_int64_t.cu +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by ivf_flat_00_generate.py - * - * Make changes there and run in this directory: - * - * > python ivf_flat_00_generate.py - * - */ - -#include - -#include - -#define instantiate_raft_neighbors_ivf_flat_search(T, IdxT) \ - template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances, \ - rmm::device_async_resource_ref mr); \ - \ - template void raft::neighbors::ivf_flat::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); -instantiate_raft_neighbors_ivf_flat_search(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_flat_search diff --git a/cpp/src/neighbors/ivfpq_build_float_int64_t.cu b/cpp/src/neighbors/ivfpq_build_float_int64_t.cu deleted file mode 100644 index 8281abb62e..0000000000 --- a/cpp/src/neighbors/ivfpq_build_float_int64_t.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#define instantiate_raft_neighbors_ivf_pq_build(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset); \ - \ - template auto raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_pq::index; - -instantiate_raft_neighbors_ivf_pq_build(float, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_build diff --git a/cpp/src/neighbors/ivfpq_build_half_int64_t.cu b/cpp/src/neighbors/ivfpq_build_half_int64_t.cu deleted file mode 100644 index aacb2d7198..0000000000 --- a/cpp/src/neighbors/ivfpq_build_half_int64_t.cu +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#include - -#define instantiate_raft_neighbors_ivf_pq_build(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset); \ - \ - template auto raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_pq::index; - -instantiate_raft_neighbors_ivf_pq_build(half, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_build diff --git a/cpp/src/neighbors/ivfpq_build_int8_t_int64_t.cu b/cpp/src/neighbors/ivfpq_build_int8_t_int64_t.cu deleted file mode 100644 index 5f79ee3033..0000000000 --- a/cpp/src/neighbors/ivfpq_build_int8_t_int64_t.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#define instantiate_raft_neighbors_ivf_pq_build(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset); \ - \ - template auto raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_pq::index; - -instantiate_raft_neighbors_ivf_pq_build(int8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_build diff --git a/cpp/src/neighbors/ivfpq_build_uint8_t_int64_t.cu b/cpp/src/neighbors/ivfpq_build_uint8_t_int64_t.cu deleted file mode 100644 index 49866ba09a..0000000000 --- a/cpp/src/neighbors/ivfpq_build_uint8_t_int64_t.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#define instantiate_raft_neighbors_ivf_pq_build(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset); \ - \ - template auto raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_pq::index; - -instantiate_raft_neighbors_ivf_pq_build(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_build diff --git a/cpp/src/neighbors/ivfpq_extend_float_int64_t.cu b/cpp/src/neighbors/ivfpq_extend_float_int64_t.cu deleted file mode 100644 index 6ee6cb3879..0000000000 --- a/cpp/src/neighbors/ivfpq_extend_float_int64_t.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#define instantiate_raft_neighbors_ivf_pq_extend(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_pq::index& idx); \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_pq::index* idx); \ - \ - template auto raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_pq::index; \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_pq::index* idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); - -instantiate_raft_neighbors_ivf_pq_extend(float, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_extend diff --git a/cpp/src/neighbors/ivfpq_extend_half_int64_t.cu b/cpp/src/neighbors/ivfpq_extend_half_int64_t.cu deleted file mode 100644 index 85477ca4a0..0000000000 --- a/cpp/src/neighbors/ivfpq_extend_half_int64_t.cu +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#include - -#define instantiate_raft_neighbors_ivf_pq_extend(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_pq::index& idx); \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_pq::index* idx); \ - \ - template auto raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_pq::index; \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_pq::index* idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); - -instantiate_raft_neighbors_ivf_pq_extend(half, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_extend diff --git a/cpp/src/neighbors/ivfpq_extend_int8_t_int64_t.cu b/cpp/src/neighbors/ivfpq_extend_int8_t_int64_t.cu deleted file mode 100644 index aefeba2aa6..0000000000 --- a/cpp/src/neighbors/ivfpq_extend_int8_t_int64_t.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#define instantiate_raft_neighbors_ivf_pq_extend(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_pq::index& idx); \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_pq::index* idx); \ - \ - template auto raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_pq::index; \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_pq::index* idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); - -instantiate_raft_neighbors_ivf_pq_extend(int8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_extend diff --git a/cpp/src/neighbors/ivfpq_extend_uint8_t_int64_t.cu b/cpp/src/neighbors/ivfpq_extend_uint8_t_int64_t.cu deleted file mode 100644 index e3a6dd365b..0000000000 --- a/cpp/src/neighbors/ivfpq_extend_uint8_t_int64_t.cu +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#define instantiate_raft_neighbors_ivf_pq_extend(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_pq::index& idx); \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_pq::index* idx); \ - \ - template auto raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows) \ - ->raft::neighbors::ivf_pq::index; \ - \ - template void raft::neighbors::ivf_pq::extend( \ - raft::resources const& handle, \ - raft::neighbors::ivf_pq::index* idx, \ - const T* new_vectors, \ - const IdxT* new_indices, \ - IdxT n_rows); - -instantiate_raft_neighbors_ivf_pq_extend(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_extend diff --git a/cpp/src/neighbors/ivfpq_search_float_int64_t.cu b/cpp/src/neighbors/ivfpq_search_float_int64_t.cu deleted file mode 100644 index 2d15167099..0000000000 --- a/cpp/src/neighbors/ivfpq_search_float_int64_t.cu +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#include - -#define instantiate_raft_neighbors_ivf_pq_search(T, IdxT) \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); \ - \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances) - -instantiate_raft_neighbors_ivf_pq_search(float, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_search diff --git a/cpp/src/neighbors/ivfpq_search_half_int64_t.cu b/cpp/src/neighbors/ivfpq_search_half_int64_t.cu deleted file mode 100644 index c9a380e21f..0000000000 --- a/cpp/src/neighbors/ivfpq_search_half_int64_t.cu +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#include - -#include - -#define instantiate_raft_neighbors_ivf_pq_search(T, IdxT) \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); \ - \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances) - -instantiate_raft_neighbors_ivf_pq_search(half, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_search diff --git a/cpp/src/neighbors/ivfpq_search_int8_t_int64_t.cu b/cpp/src/neighbors/ivfpq_search_int8_t_int64_t.cu deleted file mode 100644 index e85c98d8dd..0000000000 --- a/cpp/src/neighbors/ivfpq_search_int8_t_int64_t.cu +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#include - -#define instantiate_raft_neighbors_ivf_pq_search(T, IdxT) \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); \ - \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances) - -instantiate_raft_neighbors_ivf_pq_search(int8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_search diff --git a/cpp/src/neighbors/ivfpq_search_uint8_t_int64_t.cu b/cpp/src/neighbors/ivfpq_search_uint8_t_int64_t.cu deleted file mode 100644 index 42653254e9..0000000000 --- a/cpp/src/neighbors/ivfpq_search_uint8_t_int64_t.cu +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index - -#include - -#define instantiate_raft_neighbors_ivf_pq_search(T, IdxT) \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); \ - \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances) - -instantiate_raft_neighbors_ivf_pq_search(uint8_t, int64_t); - -#undef instantiate_raft_neighbors_ivf_pq_search diff --git a/cpp/src/neighbors/refine_00_generate.py b/cpp/src/neighbors/refine_00_generate.py deleted file mode 100644 index fd11f4d5c3..0000000000 --- a/cpp/src/neighbors/refine_00_generate.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by refine_00_generate.py - * - * Make changes there and run in this directory: - * - * > python refine_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_refine(idx_t, data_t, distance_t, matrix_idx) \\ - template void raft::neighbors::refine( \\ - raft::resources const& handle, \\ - raft::device_matrix_view dataset, \\ - raft::device_matrix_view queries, \\ - raft::device_matrix_view neighbor_candidates, \\ - raft::device_matrix_view indices, \\ - raft::device_matrix_view distances, \\ - raft::distance::DistanceType metric); \\ - \\ - template void raft::neighbors::refine( \\ - raft::resources const& handle, \\ - raft::host_matrix_view dataset, \\ - raft::host_matrix_view queries, \\ - raft::host_matrix_view neighbor_candidates, \\ - raft::host_matrix_view indices, \\ - raft::host_matrix_view distances, \\ - raft::distance::DistanceType metric); - -""" - -types = dict( - float_float= ("float", "float"), - half_float= ("half", "float"), - int8_t_float=("int8_t", "float"), - uint8_t_float=("uint8_t", "float"), -) - -for type_path, (data_t, distance_t) in types.items(): - path = f"refine_{type_path}.cu" - with open(path, "w") as f: - f.write(header) - f.write(f"instantiate_raft_neighbors_refine(int64_t, {data_t}, {distance_t}, int64_t);\n\n") - f.write(f"#undef instantiate_raft_neighbors_refine\n") - - # for pasting into CMakeLists.txt - print(f"src/neighbors/{path}") diff --git a/cpp/src/neighbors/refine_float_float.cu b/cpp/src/neighbors/refine_float_float.cu deleted file mode 100644 index 75851eeedb..0000000000 --- a/cpp/src/neighbors/refine_float_float.cu +++ /dev/null @@ -1,54 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by refine_00_generate.py - * - * Make changes there and run in this directory: - * - * > python refine_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_refine_d(idx_t, data_t, distance_t, matrix_idx) \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::device_matrix_view dataset, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbor_candidates, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric); - -#define instantiate_raft_neighbors_refine_h(idx_t, data_t, distance_t, matrix_idx) \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - raft::distance::DistanceType metric); - -instantiate_raft_neighbors_refine_d(int64_t, float, float, int64_t); -instantiate_raft_neighbors_refine_h(int64_t, float, float, int64_t); -instantiate_raft_neighbors_refine_h(uint32_t, float, float, int64_t); - -#undef instantiate_raft_neighbors_refine_d -#undef instantiate_raft_neighbors_refine_h diff --git a/cpp/src/neighbors/refine_half_float.cu b/cpp/src/neighbors/refine_half_float.cu deleted file mode 100644 index c323951b82..0000000000 --- a/cpp/src/neighbors/refine_half_float.cu +++ /dev/null @@ -1,50 +0,0 @@ - -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by refine_00_generate.py - * - * Make changes there and run in this directory: - * - * > python refine_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_refine(idx_t, data_t, distance_t, matrix_idx) \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::device_matrix_view dataset, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbor_candidates, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric); \ - \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - raft::distance::DistanceType metric); - -instantiate_raft_neighbors_refine(int64_t, half, float, int64_t); - -#undef instantiate_raft_neighbors_refine diff --git a/cpp/src/neighbors/refine_int8_t_float.cu b/cpp/src/neighbors/refine_int8_t_float.cu deleted file mode 100644 index 6ed1f86db3..0000000000 --- a/cpp/src/neighbors/refine_int8_t_float.cu +++ /dev/null @@ -1,50 +0,0 @@ - -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by refine_00_generate.py - * - * Make changes there and run in this directory: - * - * > python refine_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_refine(idx_t, data_t, distance_t, matrix_idx) \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::device_matrix_view dataset, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbor_candidates, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric); \ - \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - raft::distance::DistanceType metric); - -instantiate_raft_neighbors_refine(int64_t, int8_t, float, int64_t); - -#undef instantiate_raft_neighbors_refine diff --git a/cpp/src/neighbors/refine_uint8_t_float.cu b/cpp/src/neighbors/refine_uint8_t_float.cu deleted file mode 100644 index dac3c68b9f..0000000000 --- a/cpp/src/neighbors/refine_uint8_t_float.cu +++ /dev/null @@ -1,50 +0,0 @@ - -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by refine_00_generate.py - * - * Make changes there and run in this directory: - * - * > python refine_00_generate.py - * - */ - -#include - -#define instantiate_raft_neighbors_refine(idx_t, data_t, distance_t, matrix_idx) \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::device_matrix_view dataset, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbor_candidates, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - raft::distance::DistanceType metric); \ - \ - template void raft::neighbors::refine( \ - raft::resources const& handle, \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbor_candidates, \ - raft::host_matrix_view indices, \ - raft::host_matrix_view distances, \ - raft::distance::DistanceType metric); - -instantiate_raft_neighbors_refine(int64_t, uint8_t, float, int64_t); - -#undef instantiate_raft_neighbors_refine diff --git a/cpp/src/raft_runtime/cluster/cluster_cost.cuh b/cpp/src/raft_runtime/cluster/cluster_cost.cuh deleted file mode 100644 index 325a460ab9..0000000000 --- a/cpp/src/raft_runtime/cluster/cluster_cost.cuh +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace raft::runtime::cluster::kmeans { -template -void cluster_cost(raft::resources const& handle, - const ElementType* X, - IndexType n_samples, - IndexType n_features, - IndexType n_clusters, - const ElementType* centroids, - ElementType* cost) -{ - rmm::device_uvector workspace(n_samples * sizeof(IndexType), - resource::get_cuda_stream(handle)); - - rmm::device_uvector x_norms(n_samples, resource::get_cuda_stream(handle)); - rmm::device_uvector centroid_norms(n_clusters, resource::get_cuda_stream(handle)); - raft::linalg::rowNorm(x_norms.data(), - X, - n_features, - n_samples, - raft::linalg::L2Norm, - true, - resource::get_cuda_stream(handle)); - raft::linalg::rowNorm(centroid_norms.data(), - centroids, - n_features, - n_clusters, - raft::linalg::L2Norm, - true, - resource::get_cuda_stream(handle)); - - auto min_cluster_distance = - raft::make_device_vector>(handle, n_samples); - raft::distance::fusedL2NNMinReduce(min_cluster_distance.data_handle(), - X, - centroids, - x_norms.data(), - centroid_norms.data(), - n_samples, - n_clusters, - n_features, - (void*)workspace.data(), - false, - true, - resource::get_cuda_stream(handle)); - - auto distances = raft::make_device_vector(handle, n_samples); - thrust::transform(resource::get_thrust_policy(handle), - min_cluster_distance.data_handle(), - min_cluster_distance.data_handle() + n_samples, - distances.data_handle(), - raft::value_op{}); - - rmm::device_scalar device_cost(0, resource::get_cuda_stream(handle)); - raft::cluster::kmeans::cluster_cost(handle, - distances.view(), - workspace, - make_device_scalar_view(device_cost.data()), - raft::add_op{}); - - raft::update_host(cost, device_cost.data(), 1, resource::get_cuda_stream(handle)); -} -} // namespace raft::runtime::cluster::kmeans diff --git a/cpp/src/raft_runtime/cluster/cluster_cost_double.cu b/cpp/src/raft_runtime/cluster/cluster_cost_double.cu deleted file mode 100644 index abe07c50e7..0000000000 --- a/cpp/src/raft_runtime/cluster/cluster_cost_double.cu +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "cluster_cost.cuh" - -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void cluster_cost(raft::resources const& handle, - const double* X, - int n_samples, - int n_features, - int n_clusters, - const double* centroids, - double* cost) -{ - cluster_cost(handle, X, n_samples, n_features, n_clusters, centroids, cost); -} -} // namespace raft::runtime::cluster::kmeans diff --git a/cpp/src/raft_runtime/cluster/cluster_cost_float.cu b/cpp/src/raft_runtime/cluster/cluster_cost_float.cu deleted file mode 100644 index 1634788b62..0000000000 --- a/cpp/src/raft_runtime/cluster/cluster_cost_float.cu +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "cluster_cost.cuh" - -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void cluster_cost(raft::resources const& handle, - const float* X, - int n_samples, - int n_features, - int n_clusters, - const float* centroids, - float* cost) -{ - cluster_cost(handle, X, n_samples, n_features, n_clusters, centroids, cost); -} -} // namespace raft::runtime::cluster::kmeans diff --git a/cpp/src/raft_runtime/cluster/kmeans_fit_double.cu b/cpp/src/raft_runtime/cluster/kmeans_fit_double.cu deleted file mode 100644 index 0711f6c974..0000000000 --- a/cpp/src/raft_runtime/cluster/kmeans_fit_double.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void fit(raft::resources const& handle, - const raft::cluster::kmeans::KMeansParams& params, - raft::device_matrix_view X, - std::optional> sample_weight, - raft::device_matrix_view centroids, - raft::host_scalar_view inertia, - raft::host_scalar_view n_iter) -{ - raft::cluster::kmeans::fit( - handle, params, X, sample_weight, centroids, inertia, n_iter); -} -} // namespace raft::runtime::cluster::kmeans diff --git a/cpp/src/raft_runtime/cluster/kmeans_fit_float.cu b/cpp/src/raft_runtime/cluster/kmeans_fit_float.cu deleted file mode 100644 index f98a87d906..0000000000 --- a/cpp/src/raft_runtime/cluster/kmeans_fit_float.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void fit(raft::resources const& handle, - const raft::cluster::kmeans::KMeansParams& params, - raft::device_matrix_view X, - std::optional> sample_weight, - raft::device_matrix_view centroids, - raft::host_scalar_view inertia, - raft::host_scalar_view n_iter) -{ - raft::cluster::kmeans::fit( - handle, params, X, sample_weight, centroids, inertia, n_iter); -} -} // namespace raft::runtime::cluster::kmeans diff --git a/cpp/src/raft_runtime/cluster/kmeans_init_plus_plus_double.cu b/cpp/src/raft_runtime/cluster/kmeans_init_plus_plus_double.cu deleted file mode 100644 index 6c7563e457..0000000000 --- a/cpp/src/raft_runtime/cluster/kmeans_init_plus_plus_double.cu +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void init_plus_plus(raft::resources const& handle, - const raft::cluster::kmeans::KMeansParams& params, - raft::device_matrix_view X, - raft::device_matrix_view centroids) -{ - rmm::device_uvector workspace(0, resource::get_cuda_stream(handle)); - raft::cluster::kmeans::init_plus_plus(handle, params, X, centroids, workspace); -} -} // namespace raft::runtime::cluster::kmeans diff --git a/cpp/src/raft_runtime/cluster/kmeans_init_plus_plus_float.cu b/cpp/src/raft_runtime/cluster/kmeans_init_plus_plus_float.cu deleted file mode 100644 index 99894f4ef7..0000000000 --- a/cpp/src/raft_runtime/cluster/kmeans_init_plus_plus_float.cu +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void init_plus_plus(raft::resources const& handle, - const raft::cluster::kmeans::KMeansParams& params, - raft::device_matrix_view X, - raft::device_matrix_view centroids) -{ - rmm::device_uvector workspace(0, resource::get_cuda_stream(handle)); - raft::cluster::kmeans::init_plus_plus(handle, params, X, centroids, workspace); -} -} // namespace raft::runtime::cluster::kmeans diff --git a/cpp/src/raft_runtime/cluster/update_centroids.cuh b/cpp/src/raft_runtime/cluster/update_centroids.cuh deleted file mode 100644 index e0dec4bdcf..0000000000 --- a/cpp/src/raft_runtime/cluster/update_centroids.cuh +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -namespace raft::runtime::cluster::kmeans { - -template -void update_centroids(raft::resources const& handle, - const DataT* X, - int n_samples, - int n_features, - int n_clusters, - const DataT* sample_weights, - const DataT* centroids, - const IndexT* labels, - DataT* new_centroids, - DataT* weight_per_cluster) -{ - auto X_view = raft::make_device_matrix_view(X, n_samples, n_features); - auto centroids_view = - raft::make_device_matrix_view(centroids, n_clusters, n_features); - - rmm::device_uvector sample_weights_uvec(0, resource::get_cuda_stream(handle)); - if (sample_weights == nullptr) { - sample_weights_uvec.resize(n_samples, resource::get_cuda_stream(handle)); - DataT weight = 1.0 / n_samples; - thrust::fill(resource::get_thrust_policy(handle), - sample_weights_uvec.data(), - sample_weights_uvec.data() + n_samples, - weight); - } - auto sample_weights_view = raft::make_device_vector_view( - sample_weights == nullptr ? sample_weights_uvec.data() : sample_weights, n_samples); - - auto new_centroids_view = - raft::make_device_matrix_view(new_centroids, n_clusters, n_features); - rmm::device_uvector weight_per_cluster_uvec(0, resource::get_cuda_stream(handle)); - if (weight_per_cluster == nullptr) { - weight_per_cluster_uvec.resize(n_clusters, resource::get_cuda_stream(handle)); - } - auto weight_per_cluster_view = raft::make_device_vector_view( - weight_per_cluster == nullptr ? weight_per_cluster_uvec.data() : weight_per_cluster, - n_clusters); - - raft::cluster::kmeans::update_centroids(handle, - X_view, - sample_weights_view, - centroids_view, - labels, - weight_per_cluster_view, - new_centroids_view); -} -} // namespace raft::runtime::cluster::kmeans \ No newline at end of file diff --git a/cpp/src/raft_runtime/cluster/update_centroids_double.cu b/cpp/src/raft_runtime/cluster/update_centroids_double.cu deleted file mode 100644 index 0bc6e33273..0000000000 --- a/cpp/src/raft_runtime/cluster/update_centroids_double.cu +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "update_centroids.cuh" - -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void update_centroids(raft::resources const& handle, - const double* X, - int n_samples, - int n_features, - int n_clusters, - const double* sample_weights, - const double* centroids, - const int* labels, - double* new_centroids, - double* weight_per_cluster) -{ - update_centroids(handle, - X, - n_samples, - n_features, - n_clusters, - sample_weights, - centroids, - labels, - new_centroids, - weight_per_cluster); -} - -} // namespace raft::runtime::cluster::kmeans \ No newline at end of file diff --git a/cpp/src/raft_runtime/cluster/update_centroids_float.cu b/cpp/src/raft_runtime/cluster/update_centroids_float.cu deleted file mode 100644 index 3f98e3fb38..0000000000 --- a/cpp/src/raft_runtime/cluster/update_centroids_float.cu +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "update_centroids.cuh" - -#include -#include - -namespace raft::runtime::cluster::kmeans { - -void update_centroids(raft::resources const& handle, - const float* X, - int n_samples, - int n_features, - int n_clusters, - const float* sample_weights, - const float* centroids, - const int* labels, - float* new_centroids, - float* weight_per_cluster) -{ - update_centroids(handle, - X, - n_samples, - n_features, - n_clusters, - sample_weights, - centroids, - labels, - new_centroids, - weight_per_cluster); -} - -} // namespace raft::runtime::cluster::kmeans \ No newline at end of file diff --git a/cpp/src/raft_runtime/distance/fused_distance_min_arg.cu b/cpp/src/raft_runtime/distance/fused_distance_min_arg.cu deleted file mode 100644 index dfdff4e94b..0000000000 --- a/cpp/src/raft_runtime/distance/fused_distance_min_arg.cu +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fused_distance_min_arg.hpp" - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace raft::runtime::distance { - -void fused_distance_nn_min_arg(raft::resources const& handle, - int* min, - const float* x, - const float* y, - int m, - int n, - int k, - bool sqrt, - raft::distance::DistanceType metric, - bool isRowMajor, - float metric_arg) -{ - switch (metric) { - case raft::distance::DistanceType::CosineExpanded: - compute_fused_cosine_nn_min_arg(handle, min, x, y, m, n, k, sqrt); - break; - case raft::distance::DistanceType::L2Expanded: - case raft::distance::DistanceType::L2SqrtExpanded: - compute_fused_l2_nn_min_arg(handle, min, x, y, m, n, k, sqrt); - break; - default: assert("only Cosine/L2 metric is supported with fusedDistanceNN\n"); break; - } -} - -} // end namespace raft::runtime::distance diff --git a/cpp/src/raft_runtime/distance/fused_distance_min_arg.hpp b/cpp/src/raft_runtime/distance/fused_distance_min_arg.hpp deleted file mode 100644 index 6452752a79..0000000000 --- a/cpp/src/raft_runtime/distance/fused_distance_min_arg.hpp +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace raft::runtime::distance { - -template -struct KeyValueIndexOp { - __host__ __device__ __forceinline__ IndexT - operator()(const raft::KeyValuePair& a) const - { - return a.key; - } -}; - -template -void compute_fused_l2_nn_min_arg(raft::resources const& handle, - idx_t* min, - const value_t* x, - const value_t* y, - idx_t m, - idx_t n, - idx_t k, - bool sqrt) -{ - rmm::device_uvector workspace(m, resource::get_cuda_stream(handle)); - auto kvp = raft::make_device_vector>(handle, m); - constexpr bool is_row_major = true; - - rmm::device_uvector x_norms(m, resource::get_cuda_stream(handle)); - rmm::device_uvector y_norms(n, resource::get_cuda_stream(handle)); - raft::linalg::rowNorm( - x_norms.data(), x, k, m, raft::linalg::L2Norm, is_row_major, resource::get_cuda_stream(handle)); - raft::linalg::rowNorm( - y_norms.data(), y, k, n, raft::linalg::L2Norm, is_row_major, resource::get_cuda_stream(handle)); - - raft::distance::fusedL2NNMinReduce(kvp.data_handle(), - x, - y, - x_norms.data(), - y_norms.data(), - m, - n, - k, - (void*)workspace.data(), - sqrt, - true, - resource::get_cuda_stream(handle)); - - KeyValueIndexOp conversion_op; - thrust::transform(resource::get_thrust_policy(handle), - kvp.data_handle(), - kvp.data_handle() + m, - min, - conversion_op); - resource::sync_stream(handle); -} - -template -void compute_fused_cosine_nn_min_arg(raft::resources const& handle, - idx_t* min, - const value_t* x, - const value_t* y, - idx_t m, - idx_t n, - idx_t k, - bool sqrt) -{ - rmm::device_uvector workspace(m, resource::get_cuda_stream(handle)); - auto kvp = raft::make_device_vector>(handle, m); - - rmm::device_uvector x_norms(m, resource::get_cuda_stream(handle)); - rmm::device_uvector y_norms(n, resource::get_cuda_stream(handle)); - constexpr bool is_row_major = true; - raft::linalg::rowNorm(x_norms.data(), - x, - k, - m, - raft::linalg::L2Norm, - is_row_major, - resource::get_cuda_stream(handle), - raft::sqrt_op{}); - raft::linalg::rowNorm(y_norms.data(), - y, - k, - n, - raft::linalg::L2Norm, - is_row_major, - resource::get_cuda_stream(handle), - raft::sqrt_op{}); - - raft::distance::fusedDistanceNNMinReduce(kvp.data_handle(), - x, - y, - x_norms.data(), - y_norms.data(), - m, - n, - k, - (void*)workspace.data(), - sqrt, - true, - is_row_major, - raft::distance::DistanceType::CosineExpanded, - 0.0f, - resource::get_cuda_stream(handle)); - - KeyValueIndexOp conversion_op; - thrust::transform(resource::get_thrust_policy(handle), - kvp.data_handle(), - kvp.data_handle() + m, - min, - conversion_op); - resource::sync_stream(handle); -} - -} // end namespace raft::runtime::distance diff --git a/cpp/src/raft_runtime/distance/fused_l2_min_arg.cu b/cpp/src/raft_runtime/distance/fused_l2_min_arg.cu deleted file mode 100644 index 870757dca1..0000000000 --- a/cpp/src/raft_runtime/distance/fused_l2_min_arg.cu +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fused_distance_min_arg.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace raft::runtime::distance { - -[[deprecated("use fused_distance_nn_min_arg instead")]] void fused_l2_nn_min_arg( - raft::resources const& handle, - int* min, - const float* x, - const float* y, - int m, - int n, - int k, - bool sqrt) -{ - compute_fused_l2_nn_min_arg(handle, min, x, y, m, n, k, sqrt); -} - -[[deprecated("use fused_distance_nn_min_arg instead")]] void fused_l2_nn_min_arg( - raft::resources const& handle, - int* min, - const double* x, - const double* y, - int m, - int n, - int k, - bool sqrt) -{ - compute_fused_l2_nn_min_arg(handle, min, x, y, m, n, k, sqrt); -} - -} // end namespace raft::runtime::distance diff --git a/cpp/src/raft_runtime/distance/pairwise_distance.cu b/cpp/src/raft_runtime/distance/pairwise_distance.cu deleted file mode 100644 index 868a243b02..0000000000 --- a/cpp/src/raft_runtime/distance/pairwise_distance.cu +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -namespace raft::runtime::distance { - -void pairwise_distance(raft::resources const& handle, - float* x, - float* y, - float* dists, - int m, - int n, - int k, - raft::distance::DistanceType metric, - bool isRowMajor, - float metric_arg) -{ - raft::distance::pairwise_distance( - handle, x, y, dists, m, n, k, metric, isRowMajor, metric_arg); -} - -void pairwise_distance(raft::resources const& handle, - double* x, - double* y, - double* dists, - int m, - int n, - int k, - raft::distance::DistanceType metric, - bool isRowMajor, - float metric_arg) -{ - raft::distance::pairwise_distance( - handle, x, y, dists, m, n, k, metric, isRowMajor, metric_arg); -} -} // namespace raft::runtime::distance \ No newline at end of file diff --git a/cpp/src/raft_runtime/matrix/select_k_float_int64_t.cu b/cpp/src/raft_runtime/matrix/select_k_float_int64_t.cu deleted file mode 100644 index 551a51f6b6..0000000000 --- a/cpp/src/raft_runtime/matrix/select_k_float_int64_t.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include - -namespace raft::runtime::matrix { - -void select_k(const resources& handle, - raft::device_matrix_view in_val, - std::optional> in_idx, - raft::device_matrix_view out_val, - raft::device_matrix_view out_idx, - bool select_min) -{ - raft::matrix::select_k(handle, in_val, in_idx, out_val, out_idx, select_min); -} -} // namespace raft::runtime::matrix diff --git a/cpp/src/raft_runtime/neighbors/brute_force_knn_int64_t_float.cu b/cpp/src/raft_runtime/neighbors/brute_force_knn_int64_t_float.cu deleted file mode 100644 index 3752e9218e..0000000000 --- a/cpp/src/raft_runtime/neighbors/brute_force_knn_int64_t_float.cu +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include - -namespace raft::runtime::neighbors::brute_force { - -#define RAFT_INST_BFKNN(IDX_T, DATA_T, MATRIX_IDX_T, INDEX_LAYOUT, SEARCH_LAYOUT) \ - void knn(raft::resources const& handle, \ - raft::device_matrix_view index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view indices, \ - raft::device_matrix_view distances, \ - distance::DistanceType metric, \ - std::optional metric_arg, \ - std::optional global_id_offset) \ - { \ - std::vector> vec; \ - vec.push_back(index); \ - raft::neighbors::brute_force::knn( \ - handle, vec, search, indices, distances, metric, metric_arg, global_id_offset); \ - } - -RAFT_INST_BFKNN(int64_t, float, int64_t, raft::row_major, raft::row_major); - -#undef RAFT_INST_BFKNN - -} // namespace raft::runtime::neighbors::brute_force diff --git a/cpp/src/raft_runtime/neighbors/cagra_build.cu b/cpp/src/raft_runtime/neighbors/cagra_build.cu deleted file mode 100644 index 9268551b8c..0000000000 --- a/cpp/src/raft_runtime/neighbors/cagra_build.cu +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include - -namespace raft::runtime::neighbors::cagra { - -#define RAFT_INST_CAGRA_BUILD(T, IdxT) \ - auto build(raft::resources const& handle, \ - const raft::neighbors::cagra::index_params& params, \ - raft::device_matrix_view dataset) \ - ->raft::neighbors::cagra::index \ - { \ - return raft::neighbors::cagra::build(handle, params, dataset); \ - } \ - \ - auto build(raft::resources const& handle, \ - const raft::neighbors::cagra::index_params& params, \ - raft::host_matrix_view dataset) \ - ->raft::neighbors::cagra::index \ - { \ - return raft::neighbors::cagra::build(handle, params, dataset); \ - } \ - \ - void build_device(raft::resources const& handle, \ - const raft::neighbors::cagra::index_params& params, \ - raft::device_matrix_view dataset, \ - raft::neighbors::cagra::index& idx) \ - { \ - idx = build(handle, params, dataset); \ - } \ - \ - void build_host(raft::resources const& handle, \ - const raft::neighbors::cagra::index_params& params, \ - raft::host_matrix_view dataset, \ - raft::neighbors::cagra::index& idx) \ - { \ - idx = build(handle, params, dataset); \ - } - -RAFT_INST_CAGRA_BUILD(float, uint32_t); -RAFT_INST_CAGRA_BUILD(half, uint32_t); -RAFT_INST_CAGRA_BUILD(int8_t, uint32_t); -RAFT_INST_CAGRA_BUILD(uint8_t, uint32_t); - -#undef RAFT_INST_CAGRA_BUILD - -#define RAFT_INST_CAGRA_OPTIMIZE(IdxT) \ - void optimize_device(raft::resources const& handle, \ - raft::device_matrix_view knn_graph, \ - raft::host_matrix_view new_graph) \ - { \ - raft::neighbors::cagra::optimize(handle, knn_graph, new_graph); \ - } \ - void optimize_host(raft::resources const& handle, \ - raft::host_matrix_view knn_graph, \ - raft::host_matrix_view new_graph) \ - { \ - raft::neighbors::cagra::optimize(handle, knn_graph, new_graph); \ - } - -RAFT_INST_CAGRA_OPTIMIZE(uint32_t); - -#undef RAFT_INST_CAGRA_OPTIMIZE - -} // namespace raft::runtime::neighbors::cagra diff --git a/cpp/src/raft_runtime/neighbors/cagra_search.cu b/cpp/src/raft_runtime/neighbors/cagra_search.cu deleted file mode 100644 index b550d2e521..0000000000 --- a/cpp/src/raft_runtime/neighbors/cagra_search.cu +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -#include - -namespace raft::runtime::neighbors::cagra { - -#define RAFT_INST_CAGRA_SEARCH(T, IdxT) \ - void search(raft::resources const& handle, \ - raft::neighbors::cagra::search_params const& params, \ - const raft::neighbors::cagra::index& index, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances) \ - { \ - raft::neighbors::cagra::search(handle, params, index, queries, neighbors, distances); \ - } - -RAFT_INST_CAGRA_SEARCH(float, uint32_t); -RAFT_INST_CAGRA_SEARCH(half, uint32_t); -RAFT_INST_CAGRA_SEARCH(int8_t, uint32_t); -RAFT_INST_CAGRA_SEARCH(uint8_t, uint32_t); - -#undef RAFT_INST_CAGRA_SEARCH - -} // namespace raft::runtime::neighbors::cagra diff --git a/cpp/src/raft_runtime/neighbors/cagra_serialize.cu b/cpp/src/raft_runtime/neighbors/cagra_serialize.cu deleted file mode 100644 index 0677575e23..0000000000 --- a/cpp/src/raft_runtime/neighbors/cagra_serialize.cu +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include - -#include -#include - -namespace raft::runtime::neighbors::cagra { - -#define RAFT_INST_CAGRA_SERIALIZE(DTYPE) \ - void serialize_file(raft::resources const& handle, \ - const std::string& filename, \ - const raft::neighbors::cagra::index& index, \ - bool include_dataset) \ - { \ - raft::neighbors::cagra::serialize(handle, filename, index, include_dataset); \ - }; \ - \ - void deserialize_file(raft::resources const& handle, \ - const std::string& filename, \ - raft::neighbors::cagra::index* index) \ - { \ - if (!index) { RAFT_FAIL("Invalid index pointer"); } \ - *index = raft::neighbors::cagra::deserialize(handle, filename); \ - }; \ - void serialize(raft::resources const& handle, \ - std::string& str, \ - const raft::neighbors::cagra::index& index, \ - bool include_dataset) \ - { \ - std::stringstream os; \ - raft::neighbors::cagra::serialize(handle, os, index, include_dataset); \ - str = os.str(); \ - } \ - \ - void serialize_to_hnswlib_file(raft::resources const& handle, \ - const std::string& filename, \ - const raft::neighbors::cagra::index& index) \ - { \ - raft::neighbors::cagra::serialize_to_hnswlib(handle, filename, index); \ - }; \ - void serialize_to_hnswlib(raft::resources const& handle, \ - std::string& str, \ - const raft::neighbors::cagra::index& index) \ - { \ - std::stringstream os; \ - raft::neighbors::cagra::serialize_to_hnswlib(handle, os, index); \ - str = os.str(); \ - } \ - \ - void deserialize(raft::resources const& handle, \ - const std::string& str, \ - raft::neighbors::cagra::index* index) \ - { \ - std::istringstream is(str); \ - if (!index) { RAFT_FAIL("Invalid index pointer"); } \ - *index = raft::neighbors::cagra::deserialize(handle, is); \ - } - -RAFT_INST_CAGRA_SERIALIZE(float); -RAFT_INST_CAGRA_SERIALIZE(half); -RAFT_INST_CAGRA_SERIALIZE(int8_t); -RAFT_INST_CAGRA_SERIALIZE(uint8_t); - -#undef RAFT_INST_CAGRA_SERIALIZE -} // namespace raft::runtime::neighbors::cagra diff --git a/cpp/src/raft_runtime/neighbors/eps_neighborhood.cu b/cpp/src/raft_runtime/neighbors/eps_neighborhood.cu deleted file mode 100644 index 23cb6fd790..0000000000 --- a/cpp/src/raft_runtime/neighbors/eps_neighborhood.cu +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include - -#include - -namespace raft::runtime::neighbors::epsilon_neighborhood { - -#define RAFT_INST_BFEPSN(IDX_T, DATA_T, MATRIX_IDX_T, INDEX_LAYOUT, SEARCH_LAYOUT) \ - void eps_neighbors(raft::resources const& handle, \ - raft::device_matrix_view index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view adj, \ - raft::device_vector_view vd, \ - DATA_T eps) \ - { \ - raft::neighbors::epsilon_neighborhood::eps_neighbors_l2sq( \ - handle, search, index, adj, vd, eps* eps); \ - } - -RAFT_INST_BFEPSN(int64_t, float, int64_t, raft::row_major, raft::row_major); - -#undef RAFT_INST_BFEPSN - -#define RAFT_INST_RBCEPSN(IDX_T, DATA_T, INT_T, MATRIX_IDX_T, INDEX_LAYOUT, SEARCH_LAYOUT) \ - void eps_neighbors_rbc( \ - raft::resources const& handle, \ - raft::device_matrix_view index, \ - raft::device_matrix_view search, \ - raft::device_matrix_view adj, \ - raft::device_vector_view vd, \ - DATA_T eps) \ - { \ - raft::neighbors::ball_cover::BallCoverIndex rbc_index( \ - handle, \ - index.data_handle(), \ - index.extent(0), \ - index.extent(1), \ - raft::distance::DistanceType::L2SqrtUnexpanded); \ - raft::neighbors::ball_cover::build_index(handle, rbc_index); \ - raft::neighbors::ball_cover::eps_nn(handle, rbc_index, adj, vd, search, eps); \ - } \ - void build_rbc_index( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& rbc_index) \ - { \ - raft::neighbors::ball_cover::build_index(handle, rbc_index); \ - } \ - void eps_neighbors_rbc_pass1( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex rbc_index, \ - raft::device_matrix_view search, \ - raft::device_vector_view adj_ia, \ - raft::device_vector_view vd, \ - DATA_T eps) \ - { \ - raft::neighbors::ball_cover::eps_nn( \ - handle, \ - rbc_index, \ - adj_ia, \ - raft::make_device_vector_view(nullptr, 0), \ - vd, \ - search, \ - eps); \ - } \ - void eps_neighbors_rbc_pass2( \ - raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex rbc_index, \ - raft::device_matrix_view search, \ - raft::device_vector_view adj_ia, \ - raft::device_vector_view adj_ja, \ - raft::device_vector_view vd, \ - DATA_T eps) \ - { \ - raft::neighbors::ball_cover::eps_nn(handle, rbc_index, adj_ia, adj_ja, vd, search, eps); \ - } - -RAFT_INST_RBCEPSN(int64_t, float, int64_t, int64_t, raft::row_major, raft::row_major); - -#undef RAFT_INST_RBCEPSN - -} // namespace raft::runtime::neighbors::epsilon_neighborhood diff --git a/cpp/src/raft_runtime/neighbors/hnsw.cpp b/cpp/src/raft_runtime/neighbors/hnsw.cpp deleted file mode 100644 index 5356e708d2..0000000000 --- a/cpp/src/raft_runtime/neighbors/hnsw.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include -#include - -#include -#include -#include - -namespace raft::neighbors::hnsw { -#define RAFT_INST_HNSW(T) \ - template <> \ - std::unique_ptr> from_cagra( \ - raft::resources const& res, raft::neighbors::cagra::index cagra_index) \ - { \ - std::random_device dev; \ - std::mt19937 rng(dev()); \ - std::uniform_int_distribution dist(0); \ - auto uuid = std::to_string(dist(rng)); \ - std::string filepath = "/tmp/" + uuid + ".bin"; \ - raft::runtime::neighbors::cagra::serialize_to_hnswlib(res, filepath, cagra_index); \ - auto hnsw_index = raft::runtime::neighbors::hnsw::deserialize_file( \ - res, filepath, cagra_index.dim(), cagra_index.metric()); \ - std::filesystem::remove(filepath); \ - return hnsw_index; \ - } - -RAFT_INST_HNSW(float); -RAFT_INST_HNSW(int8_t); -RAFT_INST_HNSW(uint8_t); -#undef RAFT_INST_HNSW -} // namespace raft::neighbors::hnsw - -namespace raft::runtime::neighbors::hnsw { - -#define RAFT_INST_HNSW(T) \ - void search(raft::resources const& handle, \ - raft::neighbors::hnsw::search_params const& params, \ - const raft::neighbors::hnsw::index& index, \ - raft::host_matrix_view queries, \ - raft::host_matrix_view neighbors, \ - raft::host_matrix_view distances) \ - { \ - raft::neighbors::hnsw::search(handle, params, index, queries, neighbors, distances); \ - } \ - \ - template <> \ - std::unique_ptr> deserialize_file( \ - raft::resources const& handle, \ - const std::string& filename, \ - int dim, \ - raft::distance::DistanceType metric) \ - { \ - return raft::neighbors::hnsw::deserialize(handle, filename, dim, metric); \ - } - -RAFT_INST_HNSW(float); -RAFT_INST_HNSW(int8_t); -RAFT_INST_HNSW(uint8_t); - -#undef RAFT_INST_HNSW - -} // namespace raft::runtime::neighbors::hnsw diff --git a/cpp/src/raft_runtime/neighbors/ivf_flat_build.cu b/cpp/src/raft_runtime/neighbors/ivf_flat_build.cu deleted file mode 100644 index b0c4ec3a49..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivf_flat_build.cu +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -namespace raft::runtime::neighbors::ivf_flat { - -#define RAFT_INST_BUILD_EXTEND(T, IdxT) \ - auto build(raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index \ - { \ - return raft::neighbors::ivf_flat::build(handle, params, dataset); \ - } \ - auto extend(raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& orig_index) \ - ->raft::neighbors::ivf_flat::index \ - { \ - return raft::neighbors::ivf_flat::extend( \ - handle, new_vectors, new_indices, orig_index); \ - } \ - \ - void build(raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ - raft::device_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx) \ - { \ - idx = build(handle, params, dataset); \ - } \ - \ - void extend(raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* idx) \ - { \ - raft::neighbors::ivf_flat::extend(handle, new_vectors, new_indices, idx); \ - } - -RAFT_INST_BUILD_EXTEND(float, int64_t); -RAFT_INST_BUILD_EXTEND(int8_t, int64_t); -RAFT_INST_BUILD_EXTEND(uint8_t, int64_t); - -#undef RAFT_INST_BUILD_EXTEND - -} // namespace raft::runtime::neighbors::ivf_flat diff --git a/cpp/src/raft_runtime/neighbors/ivf_flat_search.cu b/cpp/src/raft_runtime/neighbors/ivf_flat_search.cu deleted file mode 100644 index 6f26e8fc5f..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivf_flat_search.cu +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -namespace raft::runtime::neighbors::ivf_flat { - -#define RAFT_INST_SEARCH(T, IdxT) \ - void search(raft::resources const& handle, \ - raft::neighbors::ivf_flat::search_params const& params, \ - const raft::neighbors::ivf_flat::index& index, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances) \ - { \ - raft::neighbors::ivf_flat::search( \ - handle, params, index, queries, neighbors, distances); \ - } - -RAFT_INST_SEARCH(float, int64_t); -RAFT_INST_SEARCH(int8_t, int64_t); -RAFT_INST_SEARCH(uint8_t, int64_t); - -#undef RAFT_INST_SEARCH - -} // namespace raft::runtime::neighbors::ivf_flat diff --git a/cpp/src/raft_runtime/neighbors/ivf_flat_serialize.cu b/cpp/src/raft_runtime/neighbors/ivf_flat_serialize.cu deleted file mode 100644 index 576112f0a4..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivf_flat_serialize.cu +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include -#include - -namespace raft::runtime::neighbors::ivf_flat { - -#define RAFT_IVF_FLAT_SERIALIZE_INST(DTYPE) \ - void serialize_file(raft::resources const& handle, \ - const std::string& filename, \ - const raft::neighbors::ivf_flat::index& index) \ - { \ - raft::neighbors::ivf_flat::serialize(handle, filename, index); \ - }; \ - \ - void deserialize_file(raft::resources const& handle, \ - const std::string& filename, \ - raft::neighbors::ivf_flat::index* index) \ - { \ - if (!index) { RAFT_FAIL("Invalid index pointer"); } \ - *index = raft::neighbors::ivf_flat::deserialize(handle, filename); \ - }; \ - void serialize(raft::resources const& handle, \ - std::string& str, \ - const raft::neighbors::ivf_flat::index& index) \ - { \ - std::stringstream os; \ - raft::neighbors::ivf_flat::serialize(handle, os, index); \ - str = os.str(); \ - } \ - \ - void deserialize(raft::resources const& handle, \ - const std::string& str, \ - raft::neighbors::ivf_flat::index* index) \ - { \ - std::istringstream is(str); \ - if (!index) { RAFT_FAIL("Invalid index pointer"); } \ - *index = raft::neighbors::ivf_flat::deserialize(handle, is); \ - } - -RAFT_IVF_FLAT_SERIALIZE_INST(float); -RAFT_IVF_FLAT_SERIALIZE_INST(int8_t); -RAFT_IVF_FLAT_SERIALIZE_INST(uint8_t); - -#undef RAFT_IVF_FLAT_SERIALIZE_INST -} // namespace raft::runtime::neighbors::ivf_flat diff --git a/cpp/src/raft_runtime/neighbors/ivfpq_build.cu b/cpp/src/raft_runtime/neighbors/ivfpq_build.cu deleted file mode 100644 index d5c6cd4d28..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivfpq_build.cu +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -namespace raft::runtime::neighbors::ivf_pq { - -#define RAFT_INST_BUILD_EXTEND(T, IdxT) \ - raft::neighbors::ivf_pq::index build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset) \ - { \ - return raft::neighbors::ivf_pq::build(handle, params, dataset); \ - } \ - void build(raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset, \ - raft::neighbors::ivf_pq::index* idx) \ - { \ - *idx = raft::neighbors::ivf_pq::build(handle, params, dataset); \ - } \ - raft::neighbors::ivf_pq::index extend( \ - raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - const raft::neighbors::ivf_pq::index& idx) \ - { \ - return raft::neighbors::ivf_pq::extend(handle, new_vectors, new_indices, idx); \ - } \ - void extend(raft::resources const& handle, \ - raft::device_matrix_view new_vectors, \ - std::optional> new_indices, \ - raft::neighbors::ivf_pq::index* idx) \ - { \ - raft::neighbors::ivf_pq::extend(handle, new_vectors, new_indices, idx); \ - } - -RAFT_INST_BUILD_EXTEND(float, int64_t); -RAFT_INST_BUILD_EXTEND(int8_t, int64_t); -RAFT_INST_BUILD_EXTEND(uint8_t, int64_t); - -#undef RAFT_INST_BUILD_EXTEND - -} // namespace raft::runtime::neighbors::ivf_pq diff --git a/cpp/src/raft_runtime/neighbors/ivfpq_deserialize.cu b/cpp/src/raft_runtime/neighbors/ivfpq_deserialize.cu deleted file mode 100644 index 7a2383281e..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivfpq_deserialize.cu +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include - -namespace raft::runtime::neighbors::ivf_pq { - -void deserialize(raft::resources const& handle, - const std::string& filename, - raft::neighbors::ivf_pq::index* index) -{ - if (!index) { RAFT_FAIL("Invalid index pointer"); } - *index = raft::neighbors::ivf_pq::deserialize(handle, filename); -}; -} // namespace raft::runtime::neighbors::ivf_pq diff --git a/cpp/src/raft_runtime/neighbors/ivfpq_search_float_int64_t.cu b/cpp/src/raft_runtime/neighbors/ivfpq_search_float_int64_t.cu deleted file mode 100644 index 22e4b64387..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivfpq_search_float_int64_t.cu +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -namespace raft::runtime::neighbors::ivf_pq { - -#define RAFT_SEARCH_INST(T, IdxT) \ - void search(raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances) \ - { \ - raft::neighbors::ivf_pq::search(handle, params, idx, queries, neighbors, distances); \ - } - -RAFT_SEARCH_INST(float, int64_t); - -#undef RAFT_INST_SEARCH - -} // namespace raft::runtime::neighbors::ivf_pq diff --git a/cpp/src/raft_runtime/neighbors/ivfpq_search_int8_t_int64_t.cu b/cpp/src/raft_runtime/neighbors/ivfpq_search_int8_t_int64_t.cu deleted file mode 100644 index db7b3ce209..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivfpq_search_int8_t_int64_t.cu +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -namespace raft::runtime::neighbors::ivf_pq { - -#define RAFT_SEARCH_INST(T, IdxT) \ - void search(raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances) \ - { \ - raft::neighbors::ivf_pq::search(handle, params, idx, queries, neighbors, distances); \ - } - -RAFT_SEARCH_INST(int8_t, int64_t); - -#undef RAFT_INST_SEARCH - -} // namespace raft::runtime::neighbors::ivf_pq diff --git a/cpp/src/raft_runtime/neighbors/ivfpq_search_uint8_t_int64_t.cu b/cpp/src/raft_runtime/neighbors/ivfpq_search_uint8_t_int64_t.cu deleted file mode 100644 index 6a9a2888e0..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivfpq_search_uint8_t_int64_t.cu +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -namespace raft::runtime::neighbors::ivf_pq { - -#define RAFT_SEARCH_INST(T, IdxT) \ - void search(raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances) \ - { \ - raft::neighbors::ivf_pq::search(handle, params, idx, queries, neighbors, distances); \ - } - -RAFT_SEARCH_INST(uint8_t, int64_t); - -#undef RAFT_INST_SEARCH - -} // namespace raft::runtime::neighbors::ivf_pq diff --git a/cpp/src/raft_runtime/neighbors/ivfpq_serialize.cu b/cpp/src/raft_runtime/neighbors/ivfpq_serialize.cu deleted file mode 100644 index 9dea8a3b60..0000000000 --- a/cpp/src/raft_runtime/neighbors/ivfpq_serialize.cu +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include - -namespace raft::runtime::neighbors::ivf_pq { - -void serialize(raft::resources const& handle, - const std::string& filename, - const raft::neighbors::ivf_pq::index& index) -{ - raft::neighbors::ivf_pq::serialize(handle, filename, index); -}; - -} // namespace raft::runtime::neighbors::ivf_pq diff --git a/cpp/src/raft_runtime/neighbors/refine_d_int64_t_float.cu b/cpp/src/raft_runtime/neighbors/refine_d_int64_t_float.cu deleted file mode 100644 index a146eba875..0000000000 --- a/cpp/src/raft_runtime/neighbors/refine_d_int64_t_float.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -namespace raft::runtime::neighbors { - -void refine(raft::resources const& handle, - raft::device_matrix_view dataset, - raft::device_matrix_view queries, - raft::device_matrix_view neighbor_candidates, - raft::device_matrix_view indices, - raft::device_matrix_view distances, - distance::DistanceType metric) -{ - raft::neighbors::refine( - handle, dataset, queries, neighbor_candidates, indices, distances, metric); -} - -} // namespace raft::runtime::neighbors diff --git a/cpp/src/raft_runtime/neighbors/refine_d_int64_t_int8_t.cu b/cpp/src/raft_runtime/neighbors/refine_d_int64_t_int8_t.cu deleted file mode 100644 index c840acf3df..0000000000 --- a/cpp/src/raft_runtime/neighbors/refine_d_int64_t_int8_t.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -namespace raft::runtime::neighbors { - -void refine(raft::resources const& handle, - raft::device_matrix_view dataset, - raft::device_matrix_view queries, - raft::device_matrix_view neighbor_candidates, - raft::device_matrix_view indices, - raft::device_matrix_view distances, - distance::DistanceType metric) -{ - raft::neighbors::refine( - handle, dataset, queries, neighbor_candidates, indices, distances, metric); -} - -} // namespace raft::runtime::neighbors diff --git a/cpp/src/raft_runtime/neighbors/refine_d_int64_t_uint8_t.cu b/cpp/src/raft_runtime/neighbors/refine_d_int64_t_uint8_t.cu deleted file mode 100644 index 6ad8d9a38c..0000000000 --- a/cpp/src/raft_runtime/neighbors/refine_d_int64_t_uint8_t.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -namespace raft::runtime::neighbors { - -void refine(raft::resources const& handle, - raft::device_matrix_view dataset, - raft::device_matrix_view queries, - raft::device_matrix_view neighbor_candidates, - raft::device_matrix_view indices, - raft::device_matrix_view distances, - distance::DistanceType metric) -{ - raft::neighbors::refine( - handle, dataset, queries, neighbor_candidates, indices, distances, metric); -} - -} // namespace raft::runtime::neighbors diff --git a/cpp/src/raft_runtime/neighbors/refine_h_int64_t_float.cu b/cpp/src/raft_runtime/neighbors/refine_h_int64_t_float.cu deleted file mode 100644 index 3d186c017c..0000000000 --- a/cpp/src/raft_runtime/neighbors/refine_h_int64_t_float.cu +++ /dev/null @@ -1,34 +0,0 @@ - -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -namespace raft::runtime::neighbors { - -void refine(raft::resources const& handle, - raft::host_matrix_view dataset, - raft::host_matrix_view queries, - raft::host_matrix_view neighbor_candidates, - raft::host_matrix_view indices, - raft::host_matrix_view distances, - distance::DistanceType metric) -{ - raft::neighbors::refine( - handle, dataset, queries, neighbor_candidates, indices, distances, metric); -} - -} // namespace raft::runtime::neighbors diff --git a/cpp/src/raft_runtime/neighbors/refine_h_int64_t_int8_t.cu b/cpp/src/raft_runtime/neighbors/refine_h_int64_t_int8_t.cu deleted file mode 100644 index 93237d11d5..0000000000 --- a/cpp/src/raft_runtime/neighbors/refine_h_int64_t_int8_t.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -namespace raft::runtime::neighbors { - -void refine(raft::resources const& handle, - raft::host_matrix_view dataset, - raft::host_matrix_view queries, - raft::host_matrix_view neighbor_candidates, - raft::host_matrix_view indices, - raft::host_matrix_view distances, - distance::DistanceType metric) -{ - raft::neighbors::refine( - handle, dataset, queries, neighbor_candidates, indices, distances, metric); -} - -} // namespace raft::runtime::neighbors diff --git a/cpp/src/raft_runtime/neighbors/refine_h_int64_t_uint8_t.cu b/cpp/src/raft_runtime/neighbors/refine_h_int64_t_uint8_t.cu deleted file mode 100644 index 91771e171f..0000000000 --- a/cpp/src/raft_runtime/neighbors/refine_h_int64_t_uint8_t.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -namespace raft::runtime::neighbors { - -void refine(raft::resources const& handle, - raft::host_matrix_view dataset, - raft::host_matrix_view queries, - raft::host_matrix_view neighbor_candidates, - raft::host_matrix_view indices, - raft::host_matrix_view distances, - distance::DistanceType metric) -{ - raft::neighbors::refine( - handle, dataset, queries, neighbor_candidates, indices, distances, metric); -} - -} // namespace raft::runtime::neighbors diff --git a/cpp/src/raft_runtime/solver/lanczos_solver.cuh b/cpp/src/raft_runtime/solver/lanczos_solver.cuh new file mode 100644 index 0000000000..0c851ef13a --- /dev/null +++ b/cpp/src/raft_runtime/solver/lanczos_solver.cuh @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#define FUNC_DEF(IndexType, ValueType) \ + void lanczos_solver( \ + const raft::resources& handle, \ + raft::sparse::solver::lanczos_solver_config config, \ + raft::device_vector_view rows, \ + raft::device_vector_view cols, \ + raft::device_vector_view vals, \ + std::optional> v0, \ + raft::device_vector_view eigenvalues, \ + raft::device_matrix_view eigenvectors) \ + { \ + raft::sparse::solver::lanczos_compute_smallest_eigenvectors( \ + handle, config, rows, cols, vals, v0, eigenvalues, eigenvectors); \ + } diff --git a/cpp/bench/ann/src/raft/raft_cagra_half.cu b/cpp/src/raft_runtime/solver/lanczos_solver_int64_double.cu similarity index 75% rename from cpp/bench/ann/src/raft/raft_cagra_half.cu rename to cpp/src/raft_runtime/solver/lanczos_solver_int64_double.cu index a015819ec5..f772a8a0d1 100644 --- a/cpp/bench/ann/src/raft/raft_cagra_half.cu +++ b/cpp/src/raft_runtime/solver/lanczos_solver_int64_double.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * Copyright (c) 2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,8 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "raft_cagra_wrapper.h" -namespace raft::bench::ann { -template class RaftCagra; -} // namespace raft::bench::ann +#include "lanczos_solver.cuh" + +namespace raft::runtime::solver { + +FUNC_DEF(int64_t, double); + +} // namespace raft::runtime::solver diff --git a/cpp/bench/ann/src/raft/raft_cagra_float.cu b/cpp/src/raft_runtime/solver/lanczos_solver_int64_float.cu similarity index 75% rename from cpp/bench/ann/src/raft/raft_cagra_float.cu rename to cpp/src/raft_runtime/solver/lanczos_solver_int64_float.cu index 058f5bf34a..efaf3be565 100644 --- a/cpp/bench/ann/src/raft/raft_cagra_float.cu +++ b/cpp/src/raft_runtime/solver/lanczos_solver_int64_float.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * Copyright (c) 2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,8 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "raft_cagra_wrapper.h" -namespace raft::bench::ann { -template class RaftCagra; -} // namespace raft::bench::ann +#include "lanczos_solver.cuh" + +namespace raft::runtime::solver { + +FUNC_DEF(int64_t, float); + +} // namespace raft::runtime::solver diff --git a/cpp/bench/ann/src/raft/raft_cagra_int8_t.cu b/cpp/src/raft_runtime/solver/lanczos_solver_int_double.cu similarity index 75% rename from cpp/bench/ann/src/raft/raft_cagra_int8_t.cu rename to cpp/src/raft_runtime/solver/lanczos_solver_int_double.cu index be3b83ee60..9bbc00e78a 100644 --- a/cpp/bench/ann/src/raft/raft_cagra_int8_t.cu +++ b/cpp/src/raft_runtime/solver/lanczos_solver_int_double.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * Copyright (c) 2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,8 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "raft_cagra_wrapper.h" -namespace raft::bench::ann { -template class RaftCagra; -} // namespace raft::bench::ann +#include "lanczos_solver.cuh" + +namespace raft::runtime::solver { + +FUNC_DEF(int, double); + +} // namespace raft::runtime::solver diff --git a/cpp/bench/ann/src/raft/raft_cagra_uint8_t.cu b/cpp/src/raft_runtime/solver/lanczos_solver_int_float.cu similarity index 75% rename from cpp/bench/ann/src/raft/raft_cagra_uint8_t.cu rename to cpp/src/raft_runtime/solver/lanczos_solver_int_float.cu index c9679e404d..316a9fb7e1 100644 --- a/cpp/bench/ann/src/raft/raft_cagra_uint8_t.cu +++ b/cpp/src/raft_runtime/solver/lanczos_solver_int_float.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * Copyright (c) 2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,8 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "raft_cagra_wrapper.h" -namespace raft::bench::ann { -template class RaftCagra; -} // namespace raft::bench::ann +#include "lanczos_solver.cuh" + +namespace raft::runtime::solver { + +FUNC_DEF(int, float); + +} // namespace raft::runtime::solver diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers.cu b/cpp/src/spatial/knn/detail/ball_cover/registers.cu deleted file mode 100644 index 31595272b6..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers.cu +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - raft::spatial::knn::detail::DistFunc& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - raft::spatial::knn::detail::DistFunc& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -#define instantiate_raft_spatial_knn_detail_rbc_eps_pass( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx) \ - template void \ - raft::spatial::knn::detail::rbc_eps_pass( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_t eps, \ - const Mvalue_t* R_dists, \ - raft::spatial::knn::detail::DistFunc& dfunc, \ - bool* adj, \ - Mvalue_idx* vd); \ - \ - template void \ - raft::spatial::knn::detail::rbc_eps_pass( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_t eps, \ - const Mvalue_t* R_dists, \ - raft::spatial::knn::detail::DistFunc& dfunc, \ - Mvalue_idx* ia, \ - Mvalue_idx* ja, \ - Mvalue_idx* vd) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 2); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 3); - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 2); -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 3); - -instantiate_raft_spatial_knn_detail_rbc_eps_pass(std::int64_t, float, std::int64_t, std::int64_t); - -#undef instantiate_raft_spatial_knn_detail_rbc_eps_pass -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_00_generate.py b/cpp/src/spatial/knn/detail/ball_cover/registers_00_generate.py deleted file mode 100644 index 10d9c95ece..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_00_generate.py +++ /dev/null @@ -1,164 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -header = """/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include // int64_t -#include - -""" - - -macro_pass_one = """ -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \\ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \\ - template void \\ - raft::spatial::knn::detail::rbc_low_dim_pass_one( \\ - raft::resources const& handle, \\ - const BallCoverIndex& index, \\ - const Mvalue_t* query, \\ - const Mvalue_int n_query_rows, \\ - Mvalue_int k, \\ - const Mvalue_idx* R_knn_inds, \\ - const Mvalue_t* R_knn_dists, \\ - Mdist_func& dfunc, \\ - Mvalue_idx* inds, \\ - Mvalue_t* dists, \\ - float weight, \\ - Mvalue_int* dists_counter) - -""" - -macro_pass_two = """ -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \\ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \\ - template void \\ - raft::spatial::knn::detail::rbc_low_dim_pass_two( \\ - raft::resources const& handle, \\ - const BallCoverIndex& index, \\ - const Mvalue_t* query, \\ - const Mvalue_int n_query_rows, \\ - Mvalue_int k, \\ - const Mvalue_idx* R_knn_inds, \\ - const Mvalue_t* R_knn_dists, \\ - Mdist_func& dfunc, \\ - Mvalue_idx* inds, \\ - Mvalue_t* dists, \\ - float weight, \\ - Mvalue_int* dists_counter) - -""" - -macro_pass_eps = """ -#define instantiate_raft_spatial_knn_detail_rbc_eps_pass( \\ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdist_func) \\ - template void \\ - raft::spatial::knn::detail::rbc_eps_pass( \\ - raft::resources const& handle, \\ - const BallCoverIndex& index, \\ - const Mvalue_t* query, \\ - const Mvalue_int n_query_rows, \\ - Mvalue_t eps, \\ - const Mvalue_t* R_dists, \\ - Mdist_func& dfunc, \\ - bool* adj, \\ - Mvalue_idx* vd); \\ - \\ - template void \\ - raft::spatial::knn::detail::rbc_eps_pass( \\ - raft::resources const& handle, \\ - const BallCoverIndex& index, \\ - const Mvalue_t* query, \\ - const Mvalue_int n_query_rows, \\ - Mvalue_t eps, \\ - Mvalue_int* max_k, \\ - const Mvalue_t* R_dists, \\ - Mdist_func& dfunc, \\ - Mvalue_idx* adj_ia, \\ - Mvalue_idx* adj_ja, \\ - Mvalue_idx* vd) - -""" - - -distances = dict( - haversine="raft::spatial::knn::detail::HaversineFunc", - euclidean="raft::spatial::knn::detail::EuclideanFunc", - dist="raft::spatial::knn::detail::DistFunc", -) - -euclideanSq="raft::spatial::knn::detail::EuclideanSqFunc", - -types = dict( - int64_float=("std::int64_t", "float"), - #int64_double=("std::int64_t", "double"), -) - -for k, v in distances.items(): - for dim in [2, 3]: - path = f"registers_pass_one_{dim}d_{k}.cu" - with open(path, "w") as f: - f.write(header) - f.write(macro_pass_one) - for type_path, (int_t, data_t) in types.items(): - f.write(f"instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one(\n") - f.write(f" {int_t}, {data_t}, {int_t}, {int_t}, {dim}, {v});\n") - f.write("#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one\n") - print(f"src/spatial/knn/detail/ball_cover/{path}") - -for k, v in distances.items(): - for dim in [2, 3]: - path = f"registers_pass_two_{dim}d_{k}.cu" - with open(path, "w") as f: - f.write(header) - f.write(macro_pass_two) - for type_path, (int_t, data_t) in types.items(): - f.write(f"instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two(\n") - f.write(f" {int_t}, {data_t}, {int_t}, {int_t}, {dim}, {v});\n") - f.write("#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two\n") - print(f"src/spatial/knn/detail/ball_cover/{path}") - -path="registers_eps_pass_euclidean.cu" -with open(path, "w") as f: - f.write(header) - f.write(macro_pass_eps) - for type_path, (int_t, data_t) in types.items(): - f.write(f"instantiate_raft_spatial_knn_detail_rbc_eps_pass(\n") - f.write(f" {int_t}, {data_t}, {int_t}, {int_t}, {euclideanSq});\n") - f.write("#undef instantiate_raft_spatial_knn_detail_rbc_eps_pass\n") - print(f"src/spatial/knn/detail/ball_cover/{path}") - diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_eps_pass_euclidean.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_eps_pass_euclidean.cu deleted file mode 100644 index 2a88862b2c..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_eps_pass_euclidean.cu +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_eps_pass( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdist_func) \ - template void \ - raft::spatial::knn::detail::rbc_eps_pass( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_t eps, \ - const Mvalue_t* R_dists, \ - Mdist_func& dfunc, \ - bool* adj, \ - Mvalue_idx* vd); \ - \ - template void \ - raft::spatial::knn::detail::rbc_eps_pass( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_t eps, \ - Mvalue_int* max_k, \ - const Mvalue_t* R_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* adj_ia, \ - Mvalue_idx* adj_ja, \ - Mvalue_idx* vd) - -instantiate_raft_spatial_knn_detail_rbc_eps_pass( - std::int64_t, float, std::int64_t, std::int64_t, raft::spatial::knn::detail::EuclideanSqFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_eps_pass diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_dist.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_dist.cu deleted file mode 100644 index 8c46888e05..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_dist.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::DistFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_euclidean.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_euclidean.cu deleted file mode 100644 index c5a6970a94..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_euclidean.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::EuclideanFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_haversine.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_haversine.cu deleted file mode 100644 index 19eb4b681b..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_2d_haversine.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::HaversineFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_dist.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_dist.cu deleted file mode 100644 index 6f878e1296..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_dist.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::DistFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_euclidean.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_euclidean.cu deleted file mode 100644 index e8fbcd0528..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_euclidean.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::EuclideanFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_haversine.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_haversine.cu deleted file mode 100644 index 57b8c790ef..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_one_3d_haversine.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_one( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::HaversineFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_one diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_dist.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_dist.cu deleted file mode 100644 index 6e1cc1e35e..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_dist.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::DistFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_euclidean.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_euclidean.cu deleted file mode 100644 index d1123aff7d..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_euclidean.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::EuclideanFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_haversine.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_haversine.cu deleted file mode 100644 index 3e118e2f8f..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_2d_haversine.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 2, raft::spatial::knn::detail::HaversineFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_dist.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_dist.cu deleted file mode 100644 index 674f532e6c..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_dist.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::DistFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_euclidean.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_euclidean.cu deleted file mode 100644 index dd3a422baa..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_euclidean.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::EuclideanFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two diff --git a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_haversine.cu b/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_haversine.cu deleted file mode 100644 index 0b05ca91a9..0000000000 --- a/cpp/src/spatial/knn/detail/ball_cover/registers_pass_two_3d_haversine.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * NOTE: this file is generated by registers_00_generate.py - * - * Make changes there and run in this directory: - * - * > python registers_00_generate.py - * - */ - -#include - -#include // int64_t - -#define instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( \ - Mvalue_idx, Mvalue_t, Mvalue_int, Mmatrix_idx, Mdims, Mdist_func) \ - template void raft::spatial::knn::detail:: \ - rbc_low_dim_pass_two( \ - raft::resources const& handle, \ - const BallCoverIndex& index, \ - const Mvalue_t* query, \ - const Mvalue_int n_query_rows, \ - Mvalue_int k, \ - const Mvalue_idx* R_knn_inds, \ - const Mvalue_t* R_knn_dists, \ - Mdist_func& dfunc, \ - Mvalue_idx* inds, \ - Mvalue_t* dists, \ - float weight, \ - Mvalue_int* dists_counter) - -instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two( - std::int64_t, float, std::int64_t, std::int64_t, 3, raft::spatial::knn::detail::HaversineFunc); -#undef instantiate_raft_spatial_knn_detail_rbc_low_dim_pass_two diff --git a/cpp/src/spatial/knn/detail/fused_l2_knn_int32_t_float.cu b/cpp/src/spatial/knn/detail/fused_l2_knn_int32_t_float.cu deleted file mode 100644 index aba67b8a3d..0000000000 --- a/cpp/src/spatial/knn/detail/fused_l2_knn_int32_t_float.cu +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // DistanceType -#include - -#include // size_t -#include // int_Xt - -#define instantiate_raft_spatial_knn_detail_fusedL2Knn(Mvalue_idx, Mvalue_t, MusePrevTopKs) \ - template void raft::spatial::knn::detail::fusedL2Knn( \ - size_t D, \ - Mvalue_idx * out_inds, \ - Mvalue_t * out_dists, \ - const Mvalue_t* index, \ - const Mvalue_t* query, \ - size_t n_index_rows, \ - size_t n_query_rows, \ - int k, \ - bool rowMajorIndex, \ - bool rowMajorQuery, \ - cudaStream_t stream, \ - raft::distance::DistanceType metric, \ - const Mvalue_t* index_norms, \ - const Mvalue_t* query_norms) - -instantiate_raft_spatial_knn_detail_fusedL2Knn(int32_t, float, true); -instantiate_raft_spatial_knn_detail_fusedL2Knn(int32_t, float, false); - -#undef instantiate_raft_spatial_knn_detail_fusedL2Knn diff --git a/cpp/src/spatial/knn/detail/fused_l2_knn_int64_t_float.cu b/cpp/src/spatial/knn/detail/fused_l2_knn_int64_t_float.cu deleted file mode 100644 index c9d0c49727..0000000000 --- a/cpp/src/spatial/knn/detail/fused_l2_knn_int64_t_float.cu +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // DistanceType -#include - -#include // size_t -#include // int_Xt - -#define instantiate_raft_spatial_knn_detail_fusedL2Knn(Mvalue_idx, Mvalue_t, MusePrevTopKs) \ - template void raft::spatial::knn::detail::fusedL2Knn( \ - size_t D, \ - Mvalue_idx * out_inds, \ - Mvalue_t * out_dists, \ - const Mvalue_t* index, \ - const Mvalue_t* query, \ - size_t n_index_rows, \ - size_t n_query_rows, \ - int k, \ - bool rowMajorIndex, \ - bool rowMajorQuery, \ - cudaStream_t stream, \ - raft::distance::DistanceType metric, \ - const Mvalue_t* index_norms, \ - const Mvalue_t* query_norms) - -instantiate_raft_spatial_knn_detail_fusedL2Knn(int64_t, float, true); -instantiate_raft_spatial_knn_detail_fusedL2Knn(int64_t, float, false); - -#undef instantiate_raft_spatial_knn_detail_fusedL2Knn diff --git a/cpp/src/spatial/knn/detail/fused_l2_knn_uint32_t_float.cu b/cpp/src/spatial/knn/detail/fused_l2_knn_uint32_t_float.cu deleted file mode 100644 index e6e9765151..0000000000 --- a/cpp/src/spatial/knn/detail/fused_l2_knn_uint32_t_float.cu +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // DistanceType -#include - -#include // size_t -#include // int_Xt - -#define instantiate_raft_spatial_knn_detail_fusedL2Knn(Mvalue_idx, Mvalue_t, MusePrevTopKs) \ - template void raft::spatial::knn::detail::fusedL2Knn( \ - size_t D, \ - Mvalue_idx * out_inds, \ - Mvalue_t * out_dists, \ - const Mvalue_t* index, \ - const Mvalue_t* query, \ - size_t n_index_rows, \ - size_t n_query_rows, \ - int k, \ - bool rowMajorIndex, \ - bool rowMajorQuery, \ - cudaStream_t stream, \ - raft::distance::DistanceType metric, \ - const Mvalue_t* index_norms, \ - const Mvalue_t* query_norms) - -// These are used by brute_force_knn: -instantiate_raft_spatial_knn_detail_fusedL2Knn(uint32_t, float, true); -instantiate_raft_spatial_knn_detail_fusedL2Knn(uint32_t, float, false); - -#undef instantiate_raft_spatial_knn_detail_fusedL2Knn diff --git a/cpp/template/CMakeLists.txt b/cpp/template/CMakeLists.txt deleted file mode 100644 index 40a3795ed1..0000000000 --- a/cpp/template/CMakeLists.txt +++ /dev/null @@ -1,44 +0,0 @@ -# ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. - -cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) - -# ------------- configure rapids-cmake --------------# - -include(cmake/thirdparty/fetch_rapids.cmake) -include(rapids-cmake) -include(rapids-cpm) -include(rapids-cuda) -include(rapids-export) -include(rapids-find) - -# ------------- configure project --------------# - -rapids_cuda_init_architectures(test_raft) - -project(test_raft LANGUAGES CXX CUDA) - -# ------------- configure raft -----------------# - -rapids_cpm_init() -include(cmake/thirdparty/get_raft.cmake) - -# -------------- compile tasks ----------------- # -add_executable(CAGRA_EXAMPLE src/cagra_example.cu) -target_link_libraries(CAGRA_EXAMPLE PRIVATE raft::raft raft::compiled) - -add_executable(IVF_FLAT_EXAMPLE src/ivf_flat_example.cu) -target_link_libraries(IVF_FLAT_EXAMPLE PRIVATE raft::raft raft::compiled) - -add_executable(IVF_PQ_EXAMPLE src/ivf_pq_example.cu) -target_link_libraries(IVF_PQ_EXAMPLE PRIVATE raft::raft raft::compiled) diff --git a/cpp/template/README.md b/cpp/template/README.md deleted file mode 100644 index 05ec48964f..0000000000 --- a/cpp/template/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Example RAFT Project Template - -This template project provides a drop-in sample to either start building a new application with, or using RAFT in an existing CMake project. - -First, please refer to our [installation docs](https://docs.rapids.ai/api/raft/stable/build.html#cuda-gpu-requirements) for the minimum requirements to use RAFT. - -Once the minimum requirements are satisfied, this example template application can be built with the provided `build.sh` script. This is a bash script that calls the appropriate CMake commands, so you can look into it to see the typical CMake based build workflow. - -This directory (`RAFT_SOURCE/cpp/template`) can be copied directly in order to build a new application with RAFT. - -RAFT can be integrated into an existing CMake project by copying the contents in the `configure rapids-cmake` and `configure raft` sections of the provided `CMakeLists.txt` into your project, along with `cmake/thirdparty/get_raft.cmake`. - -Make sure to link against the appropriate Cmake targets. Use `raft::raft`to add make the headers available and `raft::compiled` when utilizing the shared library. - -```cmake -target_link_libraries(your_app_target PRIVATE raft::raft raft::compiled) -``` - diff --git a/cpp/template/build.sh b/cpp/template/build.sh deleted file mode 100755 index 49c17f7499..0000000000 --- a/cpp/template/build.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2023-2024, NVIDIA CORPORATION. - -# raft empty project template build script - -# Abort script on first error -set -e - -PARALLEL_LEVEL=${PARALLEL_LEVEL:=`nproc`} - -BUILD_TYPE=Release -BUILD_DIR=build/ - -RAFT_REPO_REL="" -EXTRA_CMAKE_ARGS="" -set -e - - -if [[ ${RAFT_REPO_REL} != "" ]]; then - RAFT_REPO_PATH="`readlink -f \"${RAFT_REPO_REL}\"`" - EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCPM_raft_SOURCE=${RAFT_REPO_PATH}" -fi - -if [ "$1" == "clean" ]; then - rm -rf build - exit 0 -fi - -mkdir -p $BUILD_DIR -cd $BUILD_DIR - -cmake \ - -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ - -DRAFT_NVTX=OFF \ - -DCMAKE_CUDA_ARCHITECTURES="NATIVE" \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - ${EXTRA_CMAKE_ARGS} \ - ../ - -cmake --build . -j${PARALLEL_LEVEL} diff --git a/cpp/template/cmake/thirdparty/fetch_rapids.cmake b/cpp/template/cmake/thirdparty/fetch_rapids.cmake deleted file mode 100644 index f64a924cf5..0000000000 --- a/cpp/template/cmake/thirdparty/fetch_rapids.cmake +++ /dev/null @@ -1,21 +0,0 @@ -# ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. - -# Use this variable to update RAPIDS and RAFT versions -set(RAPIDS_VERSION "24.10") - -if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) - file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION}/RAPIDS.cmake - ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) -endif() -include(${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) diff --git a/cpp/template/cmake/thirdparty/get_raft.cmake b/cpp/template/cmake/thirdparty/get_raft.cmake deleted file mode 100644 index 07b0897be0..0000000000 --- a/cpp/template/cmake/thirdparty/get_raft.cmake +++ /dev/null @@ -1,68 +0,0 @@ -# ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. - -# Use RAPIDS_VERSION from cmake/thirdparty/fetch_rapids.cmake -set(RAFT_VERSION "${RAPIDS_VERSION}") -set(RAFT_FORK "rapidsai") -set(RAFT_PINNED_TAG "branch-${RAPIDS_VERSION}") - -function(find_and_configure_raft) - set(oneValueArgs VERSION FORK PINNED_TAG COMPILE_LIBRARY ENABLE_NVTX ENABLE_MNMG_DEPENDENCIES) - cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) - - set(RAFT_COMPONENTS "") - if(PKG_COMPILE_LIBRARY) - string(APPEND RAFT_COMPONENTS " compiled") - endif() - - if(PKG_ENABLE_MNMG_DEPENDENCIES) - string(APPEND RAFT_COMPONENTS " distributed") - endif() - - #----------------------------------------------------- - # Invoke CPM find_package() - #----------------------------------------------------- - # Since the RAFT_NVTX option is used by targets generated by - # find_package(RAFT_NVTX) and when building from source we want to - # make `RAFT_NVTX` a cache variable so we get consistent - # behavior - # - set(RAFT_NVTX ${PKG_ENABLE_NVTX} CACHE BOOL "Enable raft nvtx logging" FORCE) - rapids_cpm_find(raft ${PKG_VERSION} - GLOBAL_TARGETS raft::raft - BUILD_EXPORT_SET raft-template-exports - INSTALL_EXPORT_SET raft-template-exports - COMPONENTS ${RAFT_COMPONENTS} - CPM_ARGS - GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git - GIT_TAG ${PKG_PINNED_TAG} - SOURCE_SUBDIR cpp - OPTIONS - "BUILD_TESTS OFF" - "BUILD_PRIMS_BENCH OFF" - "BUILD_ANN_BENCH OFF" - "RAFT_COMPILE_LIBRARY ${PKG_COMPILE_LIBRARY}" - ) -endfunction() - -# Change pinned tag here to test a commit in CI -# To use a different RAFT locally, set the CMake variable -# CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_raft(VERSION ${RAFT_VERSION}.00 - FORK ${RAFT_FORK} - PINNED_TAG ${RAFT_PINNED_TAG} - COMPILE_LIBRARY ON - ENABLE_MNMG_DEPENDENCIES OFF - ENABLE_NVTX OFF -) diff --git a/cpp/template/src/cagra_example.cu b/cpp/template/src/cagra_example.cu deleted file mode 100644 index 3c1be8b4f8..0000000000 --- a/cpp/template/src/cagra_example.cu +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "common.cuh" - -#include -#include -#include -#include - -#include -#include - -#include - -void cagra_build_search_simple(raft::device_resources const& dev_resources, - raft::device_matrix_view dataset, - raft::device_matrix_view queries) -{ - using namespace raft::neighbors; - - int64_t topk = 12; - int64_t n_queries = queries.extent(0); - - // create output arrays - auto neighbors = raft::make_device_matrix(dev_resources, n_queries, topk); - auto distances = raft::make_device_matrix(dev_resources, n_queries, topk); - - // use default index parameters - cagra::index_params index_params; - - std::cout << "Building CAGRA index (search graph)" << std::endl; - auto index = cagra::build(dev_resources, index_params, dataset); - - std::cout << "CAGRA index has " << index.size() << " vectors" << std::endl; - std::cout << "CAGRA graph has degree " << index.graph_degree() << ", graph size [" - << index.graph().extent(0) << ", " << index.graph().extent(1) << "]" << std::endl; - - // use default search parameters - cagra::search_params search_params; - // search K nearest neighbors - cagra::search( - dev_resources, search_params, index, queries, neighbors.view(), distances.view()); - - // The call to ivf_flat::search is asynchronous. Before accessing the data, sync by calling - // raft::resource::sync_stream(dev_resources); - - print_results(dev_resources, neighbors.view(), distances.view()); -} - -int main() -{ - raft::device_resources dev_resources; - - // Set pool memory resource with 1 GiB initial pool size. All allocations use the same pool. - rmm::mr::pool_memory_resource pool_mr( - rmm::mr::get_current_device_resource(), 1024 * 1024 * 1024ull); - rmm::mr::set_current_device_resource(&pool_mr); - - // Alternatively, one could define a pool allocator for temporary arrays (used within RAFT - // algorithms). In that case only the internal arrays would use the pool, any other allocation - // uses the default RMM memory resource. Here is how to change the workspace memory resource to - // a pool with 2 GiB upper limit. - // raft::resource::set_workspace_to_pool_resource(dev_resources, 2 * 1024 * 1024 * 1024ull); - - // Create input arrays. - int64_t n_samples = 10000; - int64_t n_dim = 90; - int64_t n_queries = 10; - auto dataset = raft::make_device_matrix(dev_resources, n_samples, n_dim); - auto queries = raft::make_device_matrix(dev_resources, n_queries, n_dim); - generate_dataset(dev_resources, dataset.view(), queries.view()); - - // Simple build and search example. - cagra_build_search_simple(dev_resources, - raft::make_const_mdspan(dataset.view()), - raft::make_const_mdspan(queries.view())); -} diff --git a/cpp/template/src/common.cuh b/cpp/template/src/common.cuh deleted file mode 100644 index 3057257537..0000000000 --- a/cpp/template/src/common.cuh +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -// Fill dataset and queries with synthetic data. -void generate_dataset(raft::device_resources const& dev_resources, - raft::device_matrix_view dataset, - raft::device_matrix_view queries) -{ - auto labels = raft::make_device_vector(dev_resources, dataset.extent(0)); - raft::random::make_blobs(dev_resources, dataset, labels.view()); - raft::random::RngState r(1234ULL); - raft::random::uniform(dev_resources, - r, - raft::make_device_vector_view(queries.data_handle(), queries.size()), - -1.0f, - 1.0f); -} - -// Copy the results to host and print them -template -void print_results(raft::device_resources const& dev_resources, - raft::device_matrix_view neighbors, - raft::device_matrix_view distances) -{ - int64_t topk = neighbors.extent(1); - auto neighbors_host = raft::make_host_matrix(neighbors.extent(0), topk); - auto distances_host = raft::make_host_matrix(distances.extent(0), topk); - - cudaStream_t stream = raft::resource::get_cuda_stream(dev_resources); - - raft::copy(neighbors_host.data_handle(), neighbors.data_handle(), neighbors.size(), stream); - raft::copy(distances_host.data_handle(), distances.data_handle(), distances.size(), stream); - - // The calls to RAFT algorithms and raft::copy is asynchronous. - // We need to sync the stream before accessing the data. - raft::resource::sync_stream(dev_resources, stream); - - for (int query_id = 0; query_id < neighbors.extent(0); query_id++) { - std::cout << "Query " << query_id << " neighbor indices: "; - raft::print_host_vector("", &neighbors_host(query_id, 0), topk, std::cout); - std::cout << "Query " << query_id << " neighbor distances: "; - raft::print_host_vector("", &distances_host(query_id, 0), topk, std::cout); - } -} - -/** Subsample the dataset to create a training set*/ -raft::device_matrix subsample( - raft::device_resources const& dev_resources, - raft::device_matrix_view dataset, - raft::device_vector_view data_indices, - float fraction) -{ - int64_t n_samples = dataset.extent(0); - int64_t n_dim = dataset.extent(1); - int64_t n_train = n_samples * fraction; - auto trainset = raft::make_device_matrix(dev_resources, n_train, n_dim); - - int seed = 137; - raft::random::RngState rng(seed); - auto train_indices = raft::make_device_vector(dev_resources, n_train); - - raft::random::sample_without_replacement( - dev_resources, rng, data_indices, std::nullopt, train_indices.view(), std::nullopt); - - raft::matrix::copy_rows( - dev_resources, dataset, trainset.view(), raft::make_const_mdspan(train_indices.view())); - - return trainset; -} diff --git a/cpp/template/src/ivf_flat_example.cu b/cpp/template/src/ivf_flat_example.cu deleted file mode 100644 index 60694aea0f..0000000000 --- a/cpp/template/src/ivf_flat_example.cu +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "common.cuh" - -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include - -#include -#include - -void ivf_flat_build_search_simple(raft::device_resources const& dev_resources, - raft::device_matrix_view dataset, - raft::device_matrix_view queries) -{ - using namespace raft::neighbors; - - ivf_flat::index_params index_params; - index_params.n_lists = 1024; - index_params.kmeans_trainset_fraction = 0.1; - index_params.metric = raft::distance::DistanceType::L2Expanded; - - std::cout << "Building IVF-Flat index" << std::endl; - auto index = ivf_flat::build(dev_resources, index_params, dataset); - - std::cout << "Number of clusters " << index.n_lists() << ", number of vectors added to index " - << index.size() << std::endl; - - // Create output arrays. - int64_t topk = 10; - int64_t n_queries = queries.extent(0); - auto neighbors = raft::make_device_matrix(dev_resources, n_queries, topk); - auto distances = raft::make_device_matrix(dev_resources, n_queries, topk); - - // Set search parameters. - ivf_flat::search_params search_params; - search_params.n_probes = 50; - - // Search K nearest neighbors for each of the queries. - ivf_flat::search( - dev_resources, search_params, index, queries, neighbors.view(), distances.view()); - - // The call to ivf_flat::search is asynchronous. Before accessing the data, sync by calling - // raft::resource::sync_stream(dev_resources); - - print_results(dev_resources, neighbors.view(), distances.view()); -} - -void ivf_flat_build_extend_search(raft::device_resources const& dev_resources, - raft::device_matrix_view dataset, - raft::device_matrix_view queries) -{ - using namespace raft::neighbors; - - // Define dataset indices. - auto data_indices = raft::make_device_vector(dev_resources, dataset.extent(0)); - thrust::counting_iterator first(0); - thrust::device_ptr ptr(data_indices.data_handle()); - thrust::copy( - raft::resource::get_thrust_policy(dev_resources), first, first + dataset.extent(0), ptr); - - // Sub-sample the dataset to create a training set. - auto trainset = - subsample(dev_resources, dataset, raft::make_const_mdspan(data_indices.view()), 0.1); - - ivf_flat::index_params index_params; - index_params.n_lists = 100; - index_params.metric = raft::distance::DistanceType::L2Expanded; - index_params.add_data_on_build = false; - - std::cout << "\nRun k-means clustering using the training set" << std::endl; - auto index = - ivf_flat::build(dev_resources, index_params, raft::make_const_mdspan(trainset.view())); - - std::cout << "Number of clusters " << index.n_lists() << ", number of vectors added to index " - << index.size() << std::endl; - - std::cout << "Filling index with the dataset vectors" << std::endl; - index = ivf_flat::extend(dev_resources, - dataset, - std::make_optional(raft::make_const_mdspan(data_indices.view())), - index); - - std::cout << "Index size after addin dataset vectors " << index.size() << std::endl; - - // Set search parameters. - ivf_flat::search_params search_params; - search_params.n_probes = 10; - - // Create output arrays. - int64_t topk = 10; - int64_t n_queries = queries.extent(0); - auto neighbors = raft::make_device_matrix(dev_resources, n_queries, topk); - auto distances = raft::make_device_matrix(dev_resources, n_queries, topk); - - // Search K nearest neighbors for each queries. - ivf_flat::search( - dev_resources, search_params, index, queries, neighbors.view(), distances.view()); - - // The call to ivf_flat::search is asynchronous. Before accessing the data, sync using: - // raft::resource::sync_stream(dev_resources); - - print_results(dev_resources, neighbors.view(), distances.view()); -} - -int main() -{ - raft::device_resources dev_resources; - - // Set pool memory resource with 1 GiB initial pool size. All allocations use the same pool. - rmm::mr::pool_memory_resource pool_mr( - rmm::mr::get_current_device_resource(), 1024 * 1024 * 1024ull); - rmm::mr::set_current_device_resource(&pool_mr); - - // Alternatively, one could define a pool allocator for temporary arrays (used within RAFT - // algorithms). In that case only the internal arrays would use the pool, any other allocation - // uses the default RMM memory resource. Here is how to change the workspace memory resource to - // a pool with 2 GiB upper limit. - // raft::resource::set_workspace_to_pool_resource(dev_resources, 2 * 1024 * 1024 * 1024ull); - - // Create input arrays. - int64_t n_samples = 10000; - int64_t n_dim = 3; - int64_t n_queries = 10; - auto dataset = raft::make_device_matrix(dev_resources, n_samples, n_dim); - auto queries = raft::make_device_matrix(dev_resources, n_queries, n_dim); - generate_dataset(dev_resources, dataset.view(), queries.view()); - - // Simple build and search example. - ivf_flat_build_search_simple(dev_resources, - raft::make_const_mdspan(dataset.view()), - raft::make_const_mdspan(queries.view())); - - // Build and extend example. - ivf_flat_build_extend_search(dev_resources, - raft::make_const_mdspan(dataset.view()), - raft::make_const_mdspan(queries.view())); -} diff --git a/cpp/template/src/ivf_pq_example.cu b/cpp/template/src/ivf_pq_example.cu deleted file mode 100644 index 4bc0ba4348..0000000000 --- a/cpp/template/src/ivf_pq_example.cu +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "common.cuh" - -#include -#include -#include -#include - -#include -#include - -#include - -void ivf_pq_build_search(raft::device_resources const& dev_resources, - raft::device_matrix_view dataset, - raft::device_matrix_view queries) -{ - using namespace raft::neighbors; // NOLINT - - ivf_pq::index_params index_params; - index_params.n_lists = 1024; - index_params.kmeans_trainset_fraction = 0.1; - index_params.metric = raft::distance::DistanceType::L2Expanded; - index_params.pq_bits = 8; - index_params.pq_dim = 2; - - std::cout << "Building IVF-PQ index" << std::endl; - auto index = ivf_pq::build(dev_resources, index_params, dataset); - - std::cout << "Number of clusters " << index.n_lists() << ", number of vectors added to index " - << index.size() << std::endl; - - // Set search parameters. - ivf_pq::search_params search_params; - search_params.n_probes = 50; - // Set the internal search precision to 16-bit floats; - // usually, this improves the performance at a slight cost to the recall. - search_params.internal_distance_dtype = CUDA_R_16F; - search_params.lut_dtype = CUDA_R_16F; - - // Create output arrays. - int64_t topk = 10; - int64_t n_queries = queries.extent(0); - auto neighbors = raft::make_device_matrix(dev_resources, n_queries, topk); - auto distances = raft::make_device_matrix(dev_resources, n_queries, topk); - - // Search K nearest neighbors for each of the queries. - ivf_pq::search( - dev_resources, search_params, index, queries, neighbors.view(), distances.view()); - - // Re-ranking operation: refine the initial search results by computing exact distances - int64_t topk_refined = 7; - auto neighbors_refined = - raft::make_device_matrix(dev_resources, n_queries, topk_refined); - auto distances_refined = raft::make_device_matrix(dev_resources, n_queries, topk_refined); - - // Note, refinement requires the original dataset and the queries. - // Don't forget to specify the same distance metric as used by the index. - raft::neighbors::refine(dev_resources, - dataset, - queries, - raft::make_const_mdspan(neighbors.view()), - neighbors_refined.view(), - distances_refined.view(), - index.metric()); - - // Show both the original and the refined results - std::cout << std::endl << "Original results:" << std::endl; - print_results(dev_resources, neighbors.view(), distances.view()); - std::cout << std::endl << "Refined results:" << std::endl; - print_results(dev_resources, neighbors_refined.view(), distances_refined.view()); -} - -int main() -{ - raft::device_resources dev_resources; - - // Set pool memory resource with 1 GiB initial pool size. All allocations use the same pool. - rmm::mr::pool_memory_resource pool_mr( - rmm::mr::get_current_device_resource(), 1024 * 1024 * 1024ull); - rmm::mr::set_current_device_resource(&pool_mr); - - // Alternatively, one could define a pool allocator for temporary arrays (used within RAFT - // algorithms). In that case only the internal arrays would use the pool, any other allocation - // uses the default RMM memory resource. Here is how to change the workspace memory resource to - // a pool with 2 GiB upper limit. - // raft::resource::set_workspace_to_pool_resource(dev_resources, 2 * 1024 * 1024 * 1024ull); - - // Create input arrays. - int64_t n_samples = 10000; - int64_t n_dim = 3; - int64_t n_queries = 10; - auto dataset = raft::make_device_matrix(dev_resources, n_samples, n_dim); - auto queries = raft::make_device_matrix(dev_resources, n_queries, n_dim); - generate_dataset(dev_resources, dataset.view(), queries.view()); - - // Simple build and search example. - ivf_pq_build_search(dev_resources, - raft::make_const_mdspan(dataset.view()), - raft::make_const_mdspan(queries.view())); -} diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 5d504d2100..621ee6c160 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -90,16 +90,7 @@ endfunction() # ################################################################################################## # test sources ################################################################################## # ################################################################################################## - -# ################################################################################################## -# * distance tests ------------------------------------------------------------------------- - if(BUILD_TESTS) - ConfigureTest( - NAME CLUSTER_TEST PATH cluster/kmeans.cu cluster/kmeans_balanced.cu cluster/kmeans_find_k.cu - cluster/cluster_solvers.cu cluster/linkage.cu cluster/spectral.cu LIB EXPLICIT_INSTANTIATE_ONLY - ) - ConfigureTest( NAME CORE_TEST @@ -139,58 +130,7 @@ if(BUILD_TESTS) NOCUDA ) - ConfigureTest( - NAME - DISTANCE_TEST - PATH - distance/dist_adj.cu - distance/dist_adj_distance_instance.cu - distance/dist_canberra.cu - distance/dist_correlation.cu - distance/dist_cos.cu - distance/dist_dice.cu - distance/dist_hamming.cu - distance/dist_hellinger.cu - distance/dist_inner_product.cu - distance/dist_jensen_shannon.cu - distance/dist_kl_divergence.cu - distance/dist_l1.cu - distance/dist_l2_exp.cu - distance/dist_l2_unexp.cu - distance/dist_l2_sqrt_exp.cu - distance/dist_l_inf.cu - distance/dist_lp_unexp.cu - distance/dist_russell_rao.cu - distance/masked_nn.cu - distance/masked_nn_compress_to_bits.cu - distance/fused_l2_nn.cu - distance/fused_cosine_nn.cu - distance/gram.cu - LIB - EXPLICIT_INSTANTIATE_ONLY - ) - - list( - APPEND - EXT_HEADER_TEST_SOURCES - ext_headers/raft_neighbors_brute_force.cu - ext_headers/raft_distance_distance.cu - ext_headers/raft_distance_detail_pairwise_matrix_dispatch.cu - ext_headers/raft_matrix_detail_select_k.cu - ext_headers/raft_neighbors_ball_cover.cu - ext_headers/raft_spatial_knn_detail_fused_l2_knn.cu - ext_headers/raft_distance_fused_l2_nn.cu - ext_headers/raft_neighbors_ivf_pq.cu - ext_headers/raft_neighbors_ivf_flat.cu - ext_headers/raft_core_logger.cpp - ext_headers/raft_neighbors_refine.cu - ext_headers/raft_neighbors_detail_ivf_flat_search.cu - ext_headers/raft_linalg_detail_coalesced_reduction.cu - ext_headers/raft_sparse_matrix_detail_select_k.cu - ext_headers/raft_spatial_knn_detail_ball_cover_registers.cu - ext_headers/raft_neighbors_detail_ivf_flat_interleaved_scan.cu - ext_headers/raft_neighbors_detail_ivf_pq_compute_similarity.cu - ) + list(APPEND EXT_HEADER_TEST_SOURCES ext_headers/raft_core_logger.cpp) # Test that the split headers compile in isolation with: # @@ -292,8 +232,8 @@ if(BUILD_TESTS) ) ConfigureTest( - NAME SOLVERS_TEST PATH cluster/cluster_solvers_deprecated.cu linalg/eigen_solvers.cu lap/lap.cu - sparse/mst.cu LIB EXPLICIT_INSTANTIATE_ONLY + NAME SOLVERS_TEST PATH linalg/eigen_solvers.cu lap/lap.cu sparse/mst.cu + sparse/solver/lanczos.cu LIB EXPLICIT_INSTANTIATE_ONLY ) ConfigureTest( @@ -322,133 +262,8 @@ if(BUILD_TESTS) ) ConfigureTest( - NAME SPARSE_DIST_TEST PATH sparse/dist_coo_spmv.cu sparse/distance.cu sparse/gram.cu LIB - EXPLICIT_INSTANTIATE_ONLY - ) - - ConfigureTest( - NAME SPARSE_NEIGHBORS_TEST PATH sparse/neighbors/cross_component_nn.cu - sparse/neighbors/brute_force.cu sparse/neighbors/knn_graph.cu LIB EXPLICIT_INSTANTIATE_ONLY - ) - - ConfigureTest( - NAME - NEIGHBORS_TEST - PATH - neighbors/knn.cu - neighbors/fused_l2_knn.cu - neighbors/tiled_knn.cu - neighbors/haversine.cu - neighbors/ball_cover.cu - neighbors/epsilon_neighborhood.cu - neighbors/refine.cu - LIB - EXPLICIT_INSTANTIATE_ONLY - ) - - ConfigureTest( - NAME NEIGHBORS_ANN_BRUTE_FORCE_TEST PATH neighbors/ann_brute_force/test_float.cu LIB - EXPLICIT_INSTANTIATE_ONLY GPUS 1 PERCENT 100 - ) - - ConfigureTest( - NAME - NEIGHBORS_ANN_CAGRA_TEST - PATH - neighbors/ann_cagra/test_float_uint32_t.cu - neighbors/ann_cagra/test_half_uint32_t.cu - neighbors/ann_cagra/test_int8_t_uint32_t.cu - neighbors/ann_cagra/test_uint8_t_uint32_t.cu - neighbors/ann_cagra/test_float_int64_t.cu - neighbors/ann_cagra/test_half_int64_t.cu - neighbors/ann_cagra_vpq/test_float_int64_t.cu - neighbors/ann_cagra_vpq/test_float_uint32_t.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim128_t8.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim256_t16.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim512_t32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_float_uint64_dim1024_t32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim128_t8.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim256_t16.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim512_t32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_float_uint64_dim1024_t32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim128_t8.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim256_t16.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim512_t32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_multi_cta_half_uint64_dim1024_t32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim128_t8.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim256_t16.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim512_t32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/cagra/search_single_cta_half_uint64_dim1024_t32.cu - LIB - EXPLICIT_INSTANTIATE_ONLY - GPUS - 1 - PERCENT - 100 - ) - - ConfigureTest( - NAME - NEIGHBORS_ANN_IVF_TEST - PATH - neighbors/ann_ivf_flat/test_filter_float_int64_t.cu - neighbors/ann_ivf_flat/test_float_int64_t.cu - neighbors/ann_ivf_flat/test_int8_t_int64_t.cu - neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu - neighbors/ann_ivf_pq/ivf_pq_build_float_uint32_t.cu - neighbors/ann_ivf_pq/ivf_pq_search_float_uint32_t.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_search_filtering_float_int64_t.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_float_filt32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_filt32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_filt32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_half_filt32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_filt32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_filt32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_half_filt32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset32.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_float_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_false_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_fp8_true_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_float_half_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_false_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_fp8_true_bitset64.cu - ${RAFT_SOURCE_DIR}/src/neighbors/detail/ivf_pq_compute_similarity_half_half_bitset64.cu - neighbors/ann_ivf_pq/test_float_uint32_t.cu - neighbors/ann_ivf_pq/test_float_int64_t.cu - neighbors/ann_ivf_pq/test_int8_t_int64_t.cu - neighbors/ann_ivf_pq/test_uint8_t_int64_t.cu - neighbors/ann_ivf_pq/test_filter_float_int64_t.cu - neighbors/ann_ivf_pq/test_filter_int8_t_int64_t.cu - LIB - EXPLICIT_INSTANTIATE_ONLY - GPUS - 1 - PERCENT - 100 - ) - - ConfigureTest( - NAME - NEIGHBORS_ANN_NN_DESCENT_TEST - PATH - neighbors/ann_nn_descent/test_float_uint32_t.cu - neighbors/ann_nn_descent/test_int8_t_uint32_t.cu - neighbors/ann_nn_descent/test_uint8_t_uint32_t.cu - # TODO: Investigate why this test is failing Reference issue - # https://github.com/rapidsai/raft/issues/2450 - # neighbors/ann_nn_descent/test_batch_float_uint32_t.cu - LIB - EXPLICIT_INSTANTIATE_ONLY - GPUS - 1 - PERCENT - 100 + NAME NEIGHBORS_TEST PATH neighbors/haversine.cu neighbors/ball_cover.cu + neighbors/epsilon_neighborhood.cu LIB EXPLICIT_INSTANTIATE_ONLY ) ConfigureTest( @@ -471,14 +286,11 @@ if(BUILD_TESTS) stats/mean_center.cu stats/minmax.cu stats/mutual_info_score.cu - stats/neighborhood_recall.cu stats/r2_score.cu stats/rand_index.cu stats/regression_metrics.cu - stats/silhouette_score.cu stats/stddev.cu stats/sum.cu - stats/trustworthiness.cu stats/weighted_mean.cu stats/v_measure.cu LIB diff --git a/cpp/test/cluster/cluster_solvers.cu b/cpp/test/cluster/cluster_solvers.cu deleted file mode 100644 index cc0a381bbf..0000000000 --- a/cpp/test/cluster/cluster_solvers.cu +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (c) 2020-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include - -#include -#include - -namespace raft { -namespace spectral { - -TEST(Raft, ClusterSolvers) -{ - using namespace matrix; - using index_type = int; - using value_type = double; - - raft::resources h; - - index_type maxiter{100}; - value_type tol{1.0e-10}; - unsigned long long seed{100110021003}; - - auto stream = resource::get_cuda_stream(h); - - index_type n{100}; - index_type d{10}; - index_type k{5}; - - // nullptr expected to trigger exceptions: - // - value_type* eigvecs{nullptr}; - index_type* codes{nullptr}; - - cluster_solver_config_t cfg{k, maxiter, tol, seed}; - - kmeans_solver_t cluster_solver{cfg}; - - EXPECT_ANY_THROW(cluster_solver.solve(h, n, d, eigvecs, codes)); -} - -TEST(Raft, ModularitySolvers) -{ - using namespace matrix; - using index_type = int; - using value_type = double; - - raft::resources h; - ASSERT_EQ(0, resource::get_device_id(h)); - - index_type neigvs{10}; - index_type maxiter{100}; - index_type restart_iter{10}; - value_type tol{1.0e-10}; - bool reorthog{true}; - - // nullptr expected to trigger exceptions: - // - index_type* clusters{nullptr}; - value_type* eigvals{nullptr}; - value_type* eigvecs{nullptr}; - - unsigned long long seed{100110021003}; - - eigen_solver_config_t eig_cfg{ - neigvs, maxiter, restart_iter, tol, reorthog, seed}; - lanczos_solver_t eig_solver{eig_cfg}; - - index_type k{5}; - - cluster_solver_config_t clust_cfg{k, maxiter, tol, seed}; - kmeans_solver_t cluster_solver{clust_cfg}; - - auto stream = resource::get_cuda_stream(h); - sparse_matrix_t sm{h, nullptr, nullptr, nullptr, 0, 0}; - - EXPECT_ANY_THROW(spectral::modularity_maximization( - h, sm, eig_solver, cluster_solver, clusters, eigvals, eigvecs)); - - value_type modularity{0}; - EXPECT_ANY_THROW(spectral::analyzeModularity(h, sm, k, clusters, modularity)); -} - -} // namespace spectral -} // namespace raft diff --git a/cpp/test/cluster/cluster_solvers_deprecated.cu b/cpp/test/cluster/cluster_solvers_deprecated.cu deleted file mode 100644 index 954e3e5bb6..0000000000 --- a/cpp/test/cluster/cluster_solvers_deprecated.cu +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2020-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include - -#include -#include - -namespace raft { -namespace spectral { - -TEST(Raft, ClusterSolvers) -{ - using namespace matrix; - using index_type = int; - using value_type = double; - - raft::resources h; - - index_type maxiter{100}; - value_type tol{1.0e-10}; - unsigned long long seed{100110021003}; - - auto stream = resource::get_cuda_stream(h); - - index_type n{100}; - index_type d{10}; - index_type k{5}; - - // nullptr expected to trigger exceptions: - // - value_type* eigvecs{nullptr}; - index_type* codes{nullptr}; - - cluster_solver_config_deprecated_t cfg{k, maxiter, tol, seed}; - kmeans_solver_deprecated_t cluster_solver{cfg}; - - EXPECT_ANY_THROW(cluster_solver.solve(h, n, d, eigvecs, codes)); -} - -} // namespace spectral -} // namespace raft diff --git a/cpp/test/cluster/kmeans.cu b/cpp/test/cluster/kmeans.cu deleted file mode 100644 index 33c17c527d..0000000000 --- a/cpp/test/cluster/kmeans.cu +++ /dev/null @@ -1,363 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include - -#include -#include - -namespace raft { - -template -struct KmeansInputs { - int n_row; - int n_col; - int n_clusters; - T tol; - bool weighted; -}; - -template -void run_cluster_cost(const raft::resources& handle, - raft::device_vector_view minClusterDistance, - rmm::device_uvector& workspace, - raft::device_scalar_view clusterCost) -{ - raft::cluster::kmeans::cluster_cost( - handle, minClusterDistance, workspace, clusterCost, raft::add_op{}); -} - -template -class KmeansTest : public ::testing::TestWithParam> { - protected: - KmeansTest() - : d_labels(0, resource::get_cuda_stream(handle)), - d_labels_ref(0, resource::get_cuda_stream(handle)), - d_centroids(0, resource::get_cuda_stream(handle)), - d_sample_weight(0, resource::get_cuda_stream(handle)) - { - } - - void apiTest() - { - testparams = ::testing::TestWithParam>::GetParam(); - - auto stream = resource::get_cuda_stream(handle); - int n_samples = testparams.n_row; - int n_features = testparams.n_col; - params.n_clusters = testparams.n_clusters; - params.tol = testparams.tol; - params.n_init = 1; - params.rng_state.seed = 1; - params.oversampling_factor = 0; - - raft::random::RngState rng(params.rng_state.seed, params.rng_state.type); - - auto X = raft::make_device_matrix(handle, n_samples, n_features); - auto labels = raft::make_device_vector(handle, n_samples); - - raft::random::make_blobs(X.data_handle(), - labels.data_handle(), - n_samples, - n_features, - params.n_clusters, - stream, - true, - nullptr, - nullptr, - T(1.0), - false, - (T)-10.0f, - (T)10.0f, - (uint64_t)1234); - d_labels.resize(n_samples, stream); - d_labels_ref.resize(n_samples, stream); - d_centroids.resize(params.n_clusters * n_features, stream); - raft::copy(d_labels_ref.data(), labels.data_handle(), n_samples, stream); - rmm::device_uvector d_sample_weight(n_samples, stream); - thrust::fill( - thrust::cuda::par.on(stream), d_sample_weight.data(), d_sample_weight.data() + n_samples, 1); - auto weight_view = - raft::make_device_vector_view(d_sample_weight.data(), n_samples); - - T inertia = 0; - int n_iter = 0; - rmm::device_uvector workspace(0, stream); - rmm::device_uvector L2NormBuf_OR_DistBuf(0, stream); - rmm::device_uvector inRankCp(0, stream); - auto X_view = raft::make_const_mdspan(X.view()); - auto centroids_view = - raft::make_device_matrix_view(d_centroids.data(), params.n_clusters, n_features); - auto miniX = raft::make_device_matrix(handle, n_samples / 4, n_features); - - // Initialize kmeans on a portion of X - raft::cluster::kmeans::shuffle_and_gather( - handle, - X_view, - raft::make_device_matrix_view(miniX.data_handle(), miniX.extent(0), miniX.extent(1)), - miniX.extent(0), - params.rng_state.seed); - - raft::cluster::kmeans::init_plus_plus( - handle, params, raft::make_const_mdspan(miniX.view()), centroids_view, workspace); - - auto minClusterDistance = raft::make_device_vector(handle, n_samples); - auto minClusterAndDistance = - raft::make_device_vector, int>(handle, n_samples); - auto L2NormX = raft::make_device_vector(handle, n_samples); - auto clusterCostBefore = raft::make_device_scalar(handle, 0); - auto clusterCostAfter = raft::make_device_scalar(handle, 0); - - raft::linalg::rowNorm(L2NormX.data_handle(), - X.data_handle(), - X.extent(1), - X.extent(0), - raft::linalg::L2Norm, - true, - stream); - - raft::cluster::kmeans::min_cluster_distance(handle, - X_view, - centroids_view, - minClusterDistance.view(), - L2NormX.view(), - L2NormBuf_OR_DistBuf, - params.metric, - params.batch_samples, - params.batch_centroids, - workspace); - - run_cluster_cost(handle, minClusterDistance.view(), workspace, clusterCostBefore.view()); - - // Run a fit of kmeans - raft::cluster::kmeans::fit_main(handle, - params, - X_view, - weight_view, - centroids_view, - raft::make_host_scalar_view(&inertia), - raft::make_host_scalar_view(&n_iter), - workspace); - - // Check that the cluster cost decreased - raft::cluster::kmeans::min_cluster_distance(handle, - X_view, - centroids_view, - minClusterDistance.view(), - L2NormX.view(), - L2NormBuf_OR_DistBuf, - params.metric, - params.batch_samples, - params.batch_centroids, - workspace); - - run_cluster_cost(handle, minClusterDistance.view(), workspace, clusterCostAfter.view()); - T h_clusterCostBefore = T(0); - T h_clusterCostAfter = T(0); - raft::update_host(&h_clusterCostBefore, clusterCostBefore.data_handle(), 1, stream); - raft::update_host(&h_clusterCostAfter, clusterCostAfter.data_handle(), 1, stream); - ASSERT_TRUE(h_clusterCostAfter < h_clusterCostBefore); - - // Count samples in clusters using 2 methods and compare them - // Fill minClusterAndDistance - raft::cluster::kmeans::min_cluster_and_distance( - handle, - X_view, - raft::make_device_matrix_view( - d_centroids.data(), params.n_clusters, n_features), - minClusterAndDistance.view(), - L2NormX.view(), - L2NormBuf_OR_DistBuf, - params.metric, - params.batch_samples, - params.batch_centroids, - workspace); - raft::cluster::kmeans::KeyValueIndexOp conversion_op; - cub::TransformInputIterator, - raft::KeyValuePair*> - itr(minClusterAndDistance.data_handle(), conversion_op); - - auto sampleCountInCluster = raft::make_device_vector(handle, params.n_clusters); - auto weigthInCluster = raft::make_device_vector(handle, params.n_clusters); - auto newCentroids = raft::make_device_matrix(handle, params.n_clusters, n_features); - raft::cluster::kmeans::update_centroids(handle, - X_view, - weight_view, - raft::make_device_matrix_view( - d_centroids.data(), params.n_clusters, n_features), - itr, - weigthInCluster.view(), - newCentroids.view()); - raft::cluster::kmeans::count_samples_in_cluster(handle, - params, - X_view, - L2NormX.view(), - newCentroids.view(), - workspace, - sampleCountInCluster.view()); - - ASSERT_TRUE(devArrMatch(sampleCountInCluster.data_handle(), - weigthInCluster.data_handle(), - params.n_clusters, - CompareApprox(params.tol))); - } - - void basicTest() - { - testparams = ::testing::TestWithParam>::GetParam(); - - int n_samples = testparams.n_row; - int n_features = testparams.n_col; - params.n_clusters = testparams.n_clusters; - params.tol = testparams.tol; - params.n_init = 5; - params.rng_state.seed = 1; - params.oversampling_factor = 0; - - auto X = raft::make_device_matrix(handle, n_samples, n_features); - auto labels = raft::make_device_vector(handle, n_samples); - auto stream = resource::get_cuda_stream(handle); - - raft::random::make_blobs(X.data_handle(), - labels.data_handle(), - n_samples, - n_features, - params.n_clusters, - stream, - true, - nullptr, - nullptr, - T(1.0), - false, - (T)-10.0f, - (T)10.0f, - (uint64_t)1234); - - d_labels.resize(n_samples, stream); - d_labels_ref.resize(n_samples, stream); - d_centroids.resize(params.n_clusters * n_features, stream); - - std::optional> d_sw = std::nullopt; - auto d_centroids_view = - raft::make_device_matrix_view(d_centroids.data(), params.n_clusters, n_features); - if (testparams.weighted) { - d_sample_weight.resize(n_samples, stream); - d_sw = std::make_optional( - raft::make_device_vector_view(d_sample_weight.data(), n_samples)); - thrust::fill(thrust::cuda::par.on(stream), - d_sample_weight.data(), - d_sample_weight.data() + n_samples, - 1); - } - - raft::copy(d_labels_ref.data(), labels.data_handle(), n_samples, stream); - - T inertia = 0; - int n_iter = 0; - auto X_view = raft::make_const_mdspan(X.view()); - - raft::cluster::kmeans_fit_predict( - handle, - params, - X_view, - d_sw, - d_centroids_view, - raft::make_device_vector_view(d_labels.data(), n_samples), - raft::make_host_scalar_view(&inertia), - raft::make_host_scalar_view(&n_iter)); - - resource::sync_stream(handle, stream); - - score = raft::stats::adjusted_rand_index( - d_labels_ref.data(), d_labels.data(), n_samples, resource::get_cuda_stream(handle)); - - if (score < 1.0) { - std::stringstream ss; - ss << "Expected: " << raft::arr2Str(d_labels_ref.data(), 25, "d_labels_ref", stream); - std::cout << (ss.str().c_str()) << '\n'; - ss.str(std::string()); - ss << "Actual: " << raft::arr2Str(d_labels.data(), 25, "d_labels", stream); - std::cout << (ss.str().c_str()) << '\n'; - std::cout << "Score = " << score << '\n'; - } - } - - void SetUp() override - { - basicTest(); - apiTest(); - } - - protected: - raft::resources handle; - KmeansInputs testparams; - rmm::device_uvector d_labels; - rmm::device_uvector d_labels_ref; - rmm::device_uvector d_centroids; - rmm::device_uvector d_sample_weight; - double score; - raft::cluster::KMeansParams params; -}; - -const std::vector> inputsf2 = {{1000, 32, 5, 0.0001f, true}, - {1000, 32, 5, 0.0001f, false}, - {1000, 100, 20, 0.0001f, true}, - {1000, 100, 20, 0.0001f, false}, - {10000, 32, 10, 0.0001f, true}, - {10000, 32, 10, 0.0001f, false}, - {10000, 100, 50, 0.0001f, true}, - {10000, 100, 50, 0.0001f, false}, - {10000, 500, 100, 0.0001f, true}, - {10000, 500, 100, 0.0001f, false}}; - -const std::vector> inputsd2 = {{1000, 32, 5, 0.0001, true}, - {1000, 32, 5, 0.0001, false}, - {1000, 100, 20, 0.0001, true}, - {1000, 100, 20, 0.0001, false}, - {10000, 32, 10, 0.0001, true}, - {10000, 32, 10, 0.0001, false}, - {10000, 100, 50, 0.0001, true}, - {10000, 100, 50, 0.0001, false}, - {10000, 500, 100, 0.0001, true}, - {10000, 500, 100, 0.0001, false}}; - -typedef KmeansTest KmeansTestF; -TEST_P(KmeansTestF, Result) { ASSERT_TRUE(score == 1.0); } - -typedef KmeansTest KmeansTestD; -TEST_P(KmeansTestD, Result) { ASSERT_TRUE(score == 1.0); } - -INSTANTIATE_TEST_CASE_P(KmeansTests, KmeansTestF, ::testing::ValuesIn(inputsf2)); - -INSTANTIATE_TEST_CASE_P(KmeansTests, KmeansTestD, ::testing::ValuesIn(inputsd2)); - -} // namespace raft diff --git a/cpp/test/cluster/kmeans_balanced.cu b/cpp/test/cluster/kmeans_balanced.cu deleted file mode 100644 index 5009eaf122..0000000000 --- a/cpp/test/cluster/kmeans_balanced.cu +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include - -#include -#include - -/* This test takes advantage of the fact that make_blobs generates balanced clusters. - * It doesn't currently test whether the algorithm can make balanced clusters with an imbalanced - * dataset. - */ - -namespace raft { - -template -struct KmeansBalancedInputs { - IdxT n_rows; - IdxT n_cols; - IdxT n_clusters; - raft::cluster::kmeans_balanced_params kb_params; - MathT tol; -}; - -template -::std::ostream& operator<<(::std::ostream& os, const KmeansBalancedInputs& p) -{ - os << "{ " << p.n_rows << ", " << p.n_cols << ", " << p.n_clusters << ", " << p.kb_params.n_iters - << static_cast(p.kb_params.metric) << '}' << std::endl; - return os; -} - -template -class KmeansBalancedTest : public ::testing::TestWithParam> { - protected: - KmeansBalancedTest() - : stream(resource::get_cuda_stream(handle)), - d_labels(0, stream), - d_labels_ref(0, stream), - d_centroids(0, stream) - { - } - - void basicTest() - { - MappingOpT op{}; - - auto p = ::testing::TestWithParam>::GetParam(); - - auto X = raft::make_device_matrix(handle, p.n_rows, p.n_cols); - auto blob_labels = raft::make_device_vector(handle, p.n_rows); - - MathT* blobs_ptr; - rmm::device_uvector blobs(0, stream); - if constexpr (!std::is_same_v) { - blobs.resize(p.n_rows * p.n_cols, stream); - blobs_ptr = blobs.data(); - } else { - blobs_ptr = X.data_handle(); - } - - raft::random::make_blobs(blobs_ptr, - blob_labels.data_handle(), - p.n_rows, - p.n_cols, - p.n_clusters, - stream, - true, - nullptr, - nullptr, - MathT{0.1}, - true, - MathT{-1}, - MathT{1}, - (uint64_t)1234); - - // Convert blobs dataset to DataT if necessary - if constexpr (!std::is_same_v) { - raft::linalg::unaryOp( - X.data_handle(), blobs.data(), p.n_rows * p.n_cols, op.reverse_op, stream); - } - - d_labels.resize(p.n_rows, stream); - d_labels_ref.resize(p.n_rows, stream); - d_centroids.resize(p.n_clusters * p.n_cols, stream); - - raft::linalg::unaryOp( - d_labels_ref.data(), blob_labels.data_handle(), p.n_rows, raft::cast_op(), stream); - - auto X_view = - raft::make_device_matrix_view(X.data_handle(), X.extent(0), X.extent(1)); - auto d_centroids_view = - raft::make_device_matrix_view(d_centroids.data(), p.n_clusters, p.n_cols); - auto d_labels_view = raft::make_device_vector_view(d_labels.data(), p.n_rows); - - raft::cluster::kmeans_balanced::fit_predict( - handle, p.kb_params, X_view, d_centroids_view, d_labels_view, op); - - resource::sync_stream(handle, stream); - - score = raft::stats::adjusted_rand_index( - d_labels_ref.data(), d_labels.data(), p.n_rows, resource::get_cuda_stream(handle)); - - if (score < 1.0) { - std::stringstream ss; - ss << "Expected: " << raft::arr2Str(d_labels_ref.data(), 25, "d_labels_ref", stream); - std::cout << (ss.str().c_str()) << '\n'; - ss.str(std::string()); - ss << "Actual: " << raft::arr2Str(d_labels.data(), 25, "d_labels", stream); - std::cout << (ss.str().c_str()) << '\n'; - std::cout << "Score = " << score << '\n'; - } - } - - void SetUp() override { basicTest(); } - - protected: - raft::handle_t handle; - cudaStream_t stream; - rmm::device_uvector d_labels; - rmm::device_uvector d_labels_ref; - rmm::device_uvector d_centroids; - double score; -}; - -template -std::vector> get_kmeans_balanced_inputs() -{ - std::vector> out; - KmeansBalancedInputs p; - p.kb_params.n_iters = 20; - p.kb_params.metric = raft::distance::DistanceType::L2Expanded; - p.tol = MathT{0.0001}; - std::vector> row_cols_k = {{1000, 32, 5}, - {1000, 100, 20}, - {10000, 32, 10}, - {10000, 100, 50}, - {10000, 500, 100}, - {1000000, 128, 10}}; - for (auto& rck : row_cols_k) { - p.n_rows = static_cast(std::get<0>(rck)); - p.n_cols = static_cast(std::get<1>(rck)); - p.n_clusters = static_cast(std::get<2>(rck)); - out.push_back(p); - } - return out; -} - -const auto inputsf_i32 = get_kmeans_balanced_inputs(); -const auto inputsd_i32 = get_kmeans_balanced_inputs(); -const auto inputsf_i64 = get_kmeans_balanced_inputs(); -const auto inputsd_i64 = get_kmeans_balanced_inputs(); - -#define KB_TEST(test_type, test_name, test_inputs) \ - typedef RAFT_DEPAREN(test_type) test_name; \ - TEST_P(test_name, Result) { ASSERT_TRUE(score == 1.0); } \ - INSTANTIATE_TEST_CASE_P(KmeansBalancedTests, test_name, ::testing::ValuesIn(test_inputs)) - -/* - * First set of tests: no conversion - */ - -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestFFU32I32, - inputsf_i32); -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestDDU32I32, - inputsd_i32); -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestFFU32I64, - inputsf_i64); -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestDDU32I64, - inputsd_i64); -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestFFI32I32, - inputsf_i32); -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestFFI32I64, - inputsf_i64); -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestFFI64I32, - inputsf_i32); -KB_TEST((KmeansBalancedTest), - KmeansBalancedTestFFI64I64, - inputsf_i64); - -/* - * Second set of tests: integer dataset with conversion - */ - -template -struct i2f_scaler { - // Note: with a scaling factor of 42, and generating blobs with centers between -1 and 1 with a - // standard deviation of 0.1, it's statistically very unlikely that we'd overflow - const raft::compose_op, raft::cast_op> op{ - raft::div_const_op{42}, raft::cast_op{}}; - const raft::compose_op, raft::mul_const_op> reverse_op{ - raft::cast_op{}, raft::mul_const_op{42}}; - - RAFT_INLINE_FUNCTION auto operator()(const DataT& x) const { return op(x); }; -}; - -KB_TEST((KmeansBalancedTest>), - KmeansBalancedTestFI8U32I32, - inputsf_i32); -KB_TEST((KmeansBalancedTest>), - KmeansBalancedTestDI8U32I32, - inputsd_i32); - -} // namespace raft diff --git a/cpp/test/cluster/kmeans_find_k.cu b/cpp/test/cluster/kmeans_find_k.cu deleted file mode 100644 index 8e05ad3695..0000000000 --- a/cpp/test/cluster/kmeans_find_k.cu +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.h" - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -namespace raft { - -template -struct KmeansFindKInputs { - int n_row; - int n_col; - int n_clusters; - T tol; - bool weighted; -}; - -template -class KmeansFindKTest : public ::testing::TestWithParam> { - protected: - KmeansFindKTest() - : stream(resource::get_cuda_stream(handle)), best_k(raft::make_host_scalar(0)) - { - } - - void basicTest() - { - testparams = ::testing::TestWithParam>::GetParam(); - - int n_samples = testparams.n_row; - int n_features = testparams.n_col; - int n_clusters = testparams.n_clusters; - - auto X = raft::make_device_matrix(handle, n_samples, n_features); - auto labels = raft::make_device_vector(handle, n_samples); - - raft::random::make_blobs(X.data_handle(), - labels.data_handle(), - n_samples, - n_features, - n_clusters, - stream, - true, - nullptr, - nullptr, - T(.001), - false, - (T)-10.0f, - (T)10.0f, - (uint64_t)1234); - - auto inertia = raft::make_host_scalar(0); - auto n_iter = raft::make_host_scalar(0); - - auto X_view = - raft::make_device_matrix_view(X.data_handle(), X.extent(0), X.extent(1)); - - raft::cluster::kmeans::find_k( - handle, X_view, best_k.view(), inertia.view(), n_iter.view(), n_clusters); - - resource::sync_stream(handle, stream); - } - - void SetUp() override { basicTest(); } - - protected: - raft::resources handle; - cudaStream_t stream; - KmeansFindKInputs testparams; - raft::host_scalar best_k; -}; - -const std::vector> inputsf2 = {{1000, 32, 8, 0.001f, true}, - {1000, 32, 8, 0.001f, false}, - {1000, 100, 20, 0.001f, true}, - {1000, 100, 20, 0.001f, false}, - {10000, 32, 10, 0.001f, true}, - {10000, 32, 10, 0.001f, false}, - {10000, 100, 50, 0.001f, true}, - {10000, 100, 50, 0.001f, false}, - {10000, 500, 100, 0.001f, true}, - {10000, 500, 100, 0.001f, false}}; - -const std::vector> inputsd2 = {{1000, 32, 5, 0.0001, true}, - {1000, 32, 5, 0.0001, false}, - {1000, 100, 20, 0.0001, true}, - {1000, 100, 20, 0.0001, false}, - {10000, 32, 10, 0.0001, true}, - {10000, 32, 10, 0.0001, false}, - {10000, 100, 50, 0.0001, true}, - {10000, 100, 50, 0.0001, false}, - {10000, 500, 100, 0.0001, true}, - {10000, 500, 100, 0.0001, false}}; - -typedef KmeansFindKTest KmeansFindKTestF; -TEST_P(KmeansFindKTestF, Result) -{ - if (best_k.view()[0] != testparams.n_clusters) { - std::cout << best_k.view()[0] << " " << testparams.n_clusters << std::endl; - } - ASSERT_TRUE(best_k.view()[0] == testparams.n_clusters); -} - -typedef KmeansFindKTest KmeansFindKTestD; -TEST_P(KmeansFindKTestD, Result) -{ - if (best_k.view()[0] != testparams.n_clusters) { - std::cout << best_k.view()[0] << " " << testparams.n_clusters << std::endl; - } - - ASSERT_TRUE(best_k.view()[0] == testparams.n_clusters); -} - -INSTANTIATE_TEST_CASE_P(KmeansFindKTests, KmeansFindKTestF, ::testing::ValuesIn(inputsf2)); - -INSTANTIATE_TEST_CASE_P(KmeansFindKTests, KmeansFindKTestD, ::testing::ValuesIn(inputsd2)); - -} // namespace raft diff --git a/cpp/test/cluster/linkage.cu b/cpp/test/cluster/linkage.cu deleted file mode 100644 index ba7ed4254e..0000000000 --- a/cpp/test/cluster/linkage.cu +++ /dev/null @@ -1,674 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// XXX: We allow the instantiation of masked_l2_nn here: -// raft::linkage::FixConnectivitiesRedOp red_op(params.n_row); -// raft::linkage::cross_component_nn( -// handle, out_edges, data.data(), colors.data(), params.n_row, params.n_col, red_op); -// -// TODO: consider adding this to libraft.so or creating an instance in a -// separate translation unit for this test. -#undef RAFT_EXPLICIT_INSTANTIATE_ONLY - -#include "../test_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include - -namespace raft { - -using namespace std; - -template -struct LinkageInputs { - IdxT n_row; - IdxT n_col; - - std::vector data; - - std::vector expected_labels; - - int n_clusters; - - bool use_knn; - - int c; -}; - -/** - * @brief kernel to calculate the values of a and b - * @param firstClusterArray: the array of classes of type T - * @param secondClusterArray: the array of classes of type T - * @param size: the size of the data points - * @param a: number of pairs of points that both the clusters have classified the same - * @param b: number of pairs of points that both the clusters have classified differently - */ -template -RAFT_KERNEL computeTheNumerator( - const T* firstClusterArray, const T* secondClusterArray, uint64_t size, uint64_t* a, uint64_t* b) -{ - // calculating the indices of pairs of datapoints compared by the current thread - uint64_t j = threadIdx.x + blockIdx.x * blockDim.x; - uint64_t i = threadIdx.y + blockIdx.y * blockDim.y; - - // thread-local variables to count a and b - uint64_t myA = 0, myB = 0; - - if (i < size && j < size && j < i) { - // checking if the pair have been classified the same by both the clusters - if (firstClusterArray[i] == firstClusterArray[j] && - secondClusterArray[i] == secondClusterArray[j]) { - ++myA; - } - - // checking if the pair have been classified differently by both the clusters - else if (firstClusterArray[i] != firstClusterArray[j] && - secondClusterArray[i] != secondClusterArray[j]) { - ++myB; - } - } - - // specialize blockReduce for a 2D block of 1024 threads of type uint64_t - typedef cub::BlockReduce - BlockReduce; - - // Allocate shared memory for blockReduce - __shared__ typename BlockReduce::TempStorage temp_storage; - - // summing up thread-local counts specific to a block - myA = BlockReduce(temp_storage).Sum(myA); - __syncthreads(); - myB = BlockReduce(temp_storage).Sum(myB); - __syncthreads(); - - // executed once per block - if (threadIdx.x == 0 && threadIdx.y == 0) { - raft::myAtomicAdd((unsigned long long int*)a, myA); - raft::myAtomicAdd((unsigned long long int*)b, myB); - } -} - -/** - * @brief Function to calculate RandIndex - * more info on rand index - * @param firstClusterArray: the array of classes of type T - * @param secondClusterArray: the array of classes of type T - * @param size: the size of the data points of type uint64_t - * @param stream: the cudaStream object - */ -template -double compute_rand_index(T* firstClusterArray, - T* secondClusterArray, - uint64_t size, - cudaStream_t stream) -{ - // rand index for size less than 2 is not defined - ASSERT(size >= 2, "Rand Index for size less than 2 not defined!"); - - // allocating and initializing memory for a and b in the GPU - rmm::device_uvector arr_buf(2, stream); - RAFT_CUDA_TRY(cudaMemsetAsync(arr_buf.data(), 0, 2 * sizeof(uint64_t), stream)); - - // kernel configuration - static const int BLOCK_DIM_Y = 16, BLOCK_DIM_X = 16; - dim3 numThreadsPerBlock(BLOCK_DIM_X, BLOCK_DIM_Y); - dim3 numBlocks(raft::ceildiv(size, numThreadsPerBlock.x), - raft::ceildiv(size, numThreadsPerBlock.y)); - - // calling the kernel - computeTheNumerator<<>>( - firstClusterArray, secondClusterArray, size, arr_buf.data(), arr_buf.data() + 1); - - // synchronizing and updating the calculated values of a and b from device to host - uint64_t ab_host[2] = {0}; - raft::update_host(ab_host, arr_buf.data(), 2, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - - // error handling - RAFT_CUDA_TRY(cudaGetLastError()); - - // denominator - uint64_t nChooseTwo = size * (size - 1) / 2; - - // calculating the rand_index - return (double)(((double)(ab_host[0] + ab_host[1])) / (double)nChooseTwo); -} - -template -::std::ostream& operator<<(::std::ostream& os, const LinkageInputs& dims) -{ - return os; -} - -template -class LinkageTest : public ::testing::TestWithParam> { - public: - LinkageTest() - : params(::testing::TestWithParam>::GetParam()), - labels(0, resource::get_cuda_stream(handle)), - labels_ref(0, resource::get_cuda_stream(handle)) - { - } - - protected: - void basicTest() - { - auto stream = resource::get_cuda_stream(handle); - - labels.resize(params.n_row, stream); - labels_ref.resize(params.n_row, stream); - rmm::device_uvector data(params.n_row * params.n_col, stream); - - raft::copy(data.data(), params.data.data(), data.size(), stream); - raft::copy(labels_ref.data(), params.expected_labels.data(), params.n_row, stream); - - rmm::device_uvector out_children(params.n_row * 2, stream); - - auto data_view = raft::make_device_matrix_view( - data.data(), params.n_row, params.n_col); - auto dendrogram_view = - raft::make_device_matrix_view(out_children.data(), params.n_row, 2); - auto labels_view = raft::make_device_vector_view(labels.data(), params.n_row); - - if (params.use_knn) { - raft::cluster::hierarchy:: - single_linkage( - handle, - data_view, - dendrogram_view, - labels_view, - raft::distance::DistanceType::L2SqrtExpanded, - params.n_clusters, - std::make_optional(params.c)); - - } else { - raft::cluster::hierarchy:: - single_linkage( - handle, - data_view, - dendrogram_view, - labels_view, - raft::distance::DistanceType::L2SqrtExpanded, - params.n_clusters, - std::make_optional(params.c)); - } - - resource::sync_stream(handle, stream); - - score = compute_rand_index(labels.data(), labels_ref.data(), params.n_row, stream); - } - - void SetUp() override { basicTest(); } - - protected: - raft::resources handle; - - LinkageInputs params; - rmm::device_uvector labels, labels_ref; - double score; -}; - -const std::vector> linkage_inputsf2 = { - // Test n_clusters == n_points - {10, - 5, - {0.21390334, 0.50261639, 0.91036676, 0.59166485, 0.71162682, 0.10248392, 0.77782677, 0.43772379, - 0.4035871, 0.3282796, 0.47544681, 0.59862974, 0.12319357, 0.06239463, 0.28200272, 0.1345717, - 0.50498218, 0.5113505, 0.16233086, 0.62165332, 0.42281548, 0.933117, 0.41386077, 0.23264562, - 0.73325968, 0.37537541, 0.70719873, 0.14522645, 0.73279625, 0.9126674, 0.84854131, 0.28890216, - 0.85267903, 0.74703138, 0.83842071, 0.34942792, 0.27864171, 0.70911132, 0.21338564, 0.32035554, - 0.73788331, 0.46926692, 0.57570162, 0.42559178, 0.87120209, 0.22734951, 0.01847905, 0.75549396, - 0.76166195, 0.66613745}, - {9, 8, 7, 6, 5, 4, 3, 2, 1, 0}, - 10, - true, - -1}, - // // Test outlier points - {9, - 2, - {-1, -50, 3, 4, 5000, 10000, 1, 3, 4, 5, 0.000005, 0.00002, 2000000, 500000, 10, 50, 30, 5}, - {6, 0, 5, 0, 0, 4, 3, 2, 1}, - 7, - true, - -1}, - - // Test n_clusters == (n_points / 2) - {10, - 5, - {0.21390334, 0.50261639, 0.91036676, 0.59166485, 0.71162682, 0.10248392, 0.77782677, 0.43772379, - 0.4035871, 0.3282796, 0.47544681, 0.59862974, 0.12319357, 0.06239463, 0.28200272, 0.1345717, - 0.50498218, 0.5113505, 0.16233086, 0.62165332, 0.42281548, 0.933117, 0.41386077, 0.23264562, - 0.73325968, 0.37537541, 0.70719873, 0.14522645, 0.73279625, 0.9126674, 0.84854131, 0.28890216, - 0.85267903, 0.74703138, 0.83842071, 0.34942792, 0.27864171, 0.70911132, 0.21338564, 0.32035554, - 0.73788331, 0.46926692, 0.57570162, 0.42559178, 0.87120209, 0.22734951, 0.01847905, 0.75549396, - 0.76166195, 0.66613745}, - {1, 0, 4, 0, 0, 3, 2, 0, 2, 1}, - 5, - true, - -1}, - - // Test n_points == 100 - {100, - 10, - {6.26168372e-01, 9.30437651e-01, 6.02450208e-01, 2.73025296e-01, 9.53050619e-01, 3.32164396e-01, - 6.88942598e-01, 5.79163537e-01, 6.70341547e-01, 2.70140602e-02, 9.30429671e-01, 7.17721157e-01, - 9.89948537e-01, 7.75253347e-01, 1.34491522e-02, 2.48522428e-02, 3.51413378e-01, 7.64405834e-01, - 7.86373507e-01, 7.18748577e-01, 8.66998621e-01, 6.80316582e-01, 2.51288712e-01, 4.91078420e-01, - 3.76246281e-01, 4.86828710e-01, 5.67464772e-01, 5.30734742e-01, 8.99478296e-01, 7.66699088e-01, - 9.49339111e-01, 3.55248484e-01, 9.06046929e-01, 4.48407772e-01, 6.96395305e-01, 2.44277335e-01, - 7.74840000e-01, 5.21046603e-01, 4.66423971e-02, 5.12019638e-02, 8.95019614e-01, 5.28956953e-01, - 4.31536306e-01, 5.83857744e-01, 4.41787364e-01, 4.68656523e-01, 5.73971433e-01, 6.79989654e-01, - 3.19650588e-01, 6.12579596e-01, 6.49126442e-02, 8.39131142e-01, 2.85252117e-01, 5.84848929e-01, - 9.46507115e-01, 8.58440748e-01, 3.61528940e-01, 2.44215959e-01, 3.80101125e-01, 4.57128957e-02, - 8.82216988e-01, 8.31498633e-01, 7.23474381e-01, 7.75788607e-01, 1.40864146e-01, 6.62092382e-01, - 5.13985168e-01, 3.00686418e-01, 8.70109949e-01, 2.43187753e-01, 2.89391938e-01, 2.84214238e-01, - 8.70985521e-01, 8.77491176e-01, 6.72537226e-01, 3.30929686e-01, 1.85934324e-01, 9.16222614e-01, - 6.18239142e-01, 2.64768597e-01, 5.76145451e-01, 8.62961369e-01, 6.84757925e-01, 7.60549082e-01, - 1.27645356e-01, 4.51004673e-01, 3.92292980e-01, 4.63170803e-01, 4.35449330e-02, 2.17583404e-01, - 5.71832605e-02, 2.06763039e-01, 3.70116249e-01, 2.09750028e-01, 6.17283019e-01, 8.62549231e-01, - 9.84156240e-02, 2.66249156e-01, 3.87635103e-01, 2.85591012e-02, 4.24826068e-01, 4.45795088e-01, - 6.86227676e-01, 1.08848960e-01, 5.96731841e-02, 3.71770228e-01, 1.91548833e-01, 6.95136078e-01, - 9.00700636e-01, 8.76363105e-01, 2.67334632e-01, 1.80619709e-01, 7.94060419e-01, 1.42854171e-02, - 1.09372387e-01, 8.74028108e-01, 6.46403232e-01, 4.86588834e-01, 5.93446175e-02, 6.11886291e-01, - 8.83865057e-01, 3.15879821e-01, 2.27043992e-01, 9.76764951e-01, 6.15620336e-01, 9.76199360e-01, - 2.40548962e-01, 3.21795663e-01, 8.75087904e-02, 8.11234663e-01, 6.96070480e-01, 8.12062321e-01, - 1.21958818e-01, 3.44348628e-02, 8.72630414e-01, 3.06162776e-01, 1.76043529e-02, 9.45894971e-01, - 5.33896401e-01, 6.21642973e-01, 4.93062535e-01, 4.48984262e-01, 2.24560379e-01, 4.24052195e-02, - 4.43447610e-01, 8.95646149e-01, 6.05220676e-01, 1.81840491e-01, 9.70831206e-01, 2.12563586e-02, - 6.92582693e-01, 7.55946922e-01, 7.95086143e-01, 6.05328941e-01, 3.99350764e-01, 4.32846636e-01, - 9.81114529e-01, 4.98266428e-01, 6.37127930e-03, 1.59085889e-01, 6.34682067e-05, 5.59429440e-01, - 7.38827633e-01, 8.93214770e-01, 2.16494306e-01, 9.35430573e-02, 4.75665868e-02, 7.80503518e-01, - 7.86240041e-01, 7.06854594e-01, 2.13725879e-02, 7.68246091e-01, 4.50234808e-01, 5.21231104e-01, - 5.01989826e-03, 4.22081572e-02, 1.65337732e-01, 8.54134740e-01, 4.99430262e-01, 8.94525601e-01, - 1.14028379e-01, 3.69739861e-01, 1.32955599e-01, 2.65563824e-01, 2.52811151e-01, 1.44792843e-01, - 6.88449594e-01, 4.44921417e-01, 8.23296587e-01, 1.93266317e-01, 1.19033309e-01, 1.36368966e-01, - 3.42600285e-01, 5.64505195e-01, 5.57594559e-01, 7.44257892e-01, 8.38231569e-02, 4.11548847e-01, - 3.21010077e-01, 8.55081359e-01, 4.30105779e-01, 1.16229135e-01, 9.87731964e-02, 3.14712335e-01, - 4.50880592e-01, 2.72289598e-01, 6.31615256e-01, 8.97432958e-01, 4.44764250e-01, 8.03776440e-01, - 2.68767748e-02, 2.43374608e-01, 4.02141103e-01, 4.98881209e-01, 5.33173003e-01, 8.82890436e-01, - 7.16149148e-01, 4.19664401e-01, 2.29335357e-01, 2.88637806e-01, 3.44696803e-01, 6.78171906e-01, - 5.69849716e-01, 5.86454477e-01, 3.54474989e-01, 9.03876540e-01, 6.45980000e-01, 6.34887593e-01, - 7.88039746e-02, 2.04814126e-01, 7.82251754e-01, 2.43147074e-01, 7.50951808e-01, 1.72799092e-02, - 2.95349590e-01, 6.57991826e-01, 8.81214312e-01, 5.73970708e-01, 2.77610881e-01, 1.82155097e-01, - 7.69797417e-02, 6.44792402e-01, 9.46950998e-01, 7.73064845e-01, 6.04733624e-01, 5.80094567e-01, - 1.67498426e-01, 2.66514296e-01, 6.50140368e-01, 1.91170299e-01, 2.08752199e-01, 3.01664091e-01, - 9.85033484e-01, 2.92909152e-01, 8.65816607e-01, 1.85222119e-01, 2.28814559e-01, 1.34286382e-02, - 2.89234322e-01, 8.18668708e-01, 4.71706924e-01, 9.23199803e-01, 2.80879188e-01, 1.47319284e-01, - 4.13915748e-01, 9.31274932e-02, 6.66322195e-01, 9.66953974e-01, 3.19405786e-01, 6.69486551e-01, - 5.03096313e-02, 6.95225201e-01, 5.78469859e-01, 6.29481655e-01, 1.39252534e-01, 1.22564968e-01, - 6.80663678e-01, 6.34607157e-01, 6.42765834e-01, 1.57127410e-02, 2.92132086e-01, 5.24423878e-01, - 4.68676824e-01, 2.86003928e-01, 7.18608322e-01, 8.95617933e-01, 5.48844309e-01, 1.74517278e-01, - 5.24379196e-01, 2.13526524e-01, 5.88375435e-01, 9.88560185e-01, 4.17435771e-01, 6.14438688e-01, - 9.53760881e-01, 5.27151288e-01, 7.03017278e-01, 3.44448559e-01, 4.47059676e-01, 2.83414901e-01, - 1.98979011e-01, 4.24917361e-01, 5.73172761e-01, 2.32398853e-02, 1.65887230e-01, 4.05552785e-01, - 9.29665524e-01, 2.26135696e-01, 9.20563384e-01, 7.65259963e-01, 4.54820075e-01, 8.97710267e-01, - 3.78559302e-03, 9.15219382e-01, 3.55705698e-01, 6.94905124e-01, 8.58540202e-01, 3.89790666e-01, - 2.49478206e-01, 7.93679304e-01, 4.75830027e-01, 4.40425353e-01, 3.70579459e-01, 1.40578049e-01, - 1.70386675e-01, 7.04056121e-01, 4.85963102e-01, 9.68450060e-01, 6.77178001e-01, 2.65934654e-01, - 2.58915007e-01, 6.70052890e-01, 2.61945109e-01, 8.46207759e-01, 1.01928951e-01, 2.85611334e-01, - 2.45776933e-01, 2.66658783e-01, 3.71724077e-01, 4.34319025e-01, 4.24407347e-01, 7.15417683e-01, - 8.07997684e-01, 1.64296275e-01, 6.01638065e-01, 8.60606804e-02, 2.68719187e-01, 5.11764101e-01, - 9.75844338e-01, 7.81226782e-01, 2.20925515e-01, 7.18135040e-01, 9.82395577e-01, 8.39160243e-01, - 9.08058083e-01, 6.88010677e-01, 8.14271847e-01, 5.12460821e-01, 1.17311345e-01, 5.96075228e-01, - 9.17455497e-01, 2.12052706e-01, 7.04074603e-01, 8.72872565e-02, 8.76047818e-01, 6.96235046e-01, - 8.54801557e-01, 2.49729159e-01, 9.76594604e-01, 2.87386363e-01, 2.36461559e-02, 9.94075254e-01, - 4.25193986e-01, 7.61869994e-01, 5.13334255e-01, 6.44711165e-02, 8.92156689e-01, 3.55235167e-01, - 1.08154647e-01, 8.78446825e-01, 2.43833016e-01, 9.23071293e-01, 2.72724115e-01, 9.46631338e-01, - 3.74510294e-01, 4.08451278e-02, 9.78392777e-01, 3.65079221e-01, 6.37199516e-01, 5.51144906e-01, - 5.25978080e-01, 1.42803678e-01, 4.05451674e-01, 7.79788219e-01, 6.26009784e-01, 3.35249497e-01, - 1.43159543e-02, 1.80363779e-01, 5.05096904e-01, 2.82619947e-01, 5.83561392e-01, 3.10951324e-01, - 8.73223968e-01, 4.38545619e-01, 4.81348800e-01, 6.68497085e-01, 3.79345401e-01, 9.58832501e-01, - 1.89869550e-01, 2.34083070e-01, 2.94066207e-01, 5.74892667e-02, 6.92106828e-02, 9.61127686e-02, - 6.72650672e-02, 8.47345378e-01, 2.80916761e-01, 7.32177357e-03, 9.80785961e-01, 5.73192225e-02, - 8.48781331e-01, 8.83225408e-01, 7.34398275e-01, 7.70381941e-01, 6.20778343e-01, 8.96822048e-01, - 5.40732486e-01, 3.69704071e-01, 5.77305837e-01, 2.08221827e-01, 7.34275341e-01, 1.06110900e-01, - 3.49496706e-01, 8.34948910e-01, 1.56403291e-02, 6.78576376e-01, 8.96141268e-01, 5.94835119e-01, - 1.43943153e-01, 3.49618530e-01, 2.10440392e-01, 3.46585620e-01, 1.05153093e-01, 3.45446174e-01, - 2.72177079e-01, 7.07946300e-01, 4.33717726e-02, 3.31232203e-01, 3.91874320e-01, 4.76338141e-01, - 6.22777789e-01, 2.95989228e-02, 4.32855769e-01, 7.61049310e-01, 3.63279149e-01, 9.47210350e-01, - 6.43721247e-01, 6.58025802e-01, 1.05247633e-02, 5.29974442e-01, 7.30675767e-01, 4.30041079e-01, - 6.62634841e-01, 8.25936616e-01, 9.91253704e-01, 6.79399281e-01, 5.44177006e-01, 7.52876048e-01, - 3.32139049e-01, 7.98732398e-01, 7.38865223e-01, 9.16055132e-01, 6.11736493e-01, 9.63672879e-01, - 1.83778839e-01, 7.27558919e-02, 5.91602822e-01, 3.25235484e-01, 2.34741217e-01, 9.52346277e-01, - 9.18556407e-01, 9.35373324e-01, 6.89209070e-01, 2.56049054e-01, 6.17975395e-01, 7.82285691e-01, - 9.84983432e-01, 6.62322741e-01, 2.04144457e-01, 3.98446577e-01, 1.38918297e-01, 3.05919921e-01, - 3.14043787e-01, 5.91072666e-01, 7.44703771e-01, 8.92272567e-01, 9.78017873e-01, 9.01203161e-01, - 1.41526372e-01, 4.14878484e-01, 6.80683651e-01, 5.01733152e-02, 8.14635389e-01, 2.27926375e-01, - 9.03269815e-01, 8.68443745e-01, 9.86939190e-01, 7.40779486e-01, 2.61005311e-01, 3.19276232e-01, - 9.69509248e-01, 1.11908818e-01, 4.49198556e-01, 1.27056715e-01, 3.84064823e-01, 5.14591811e-01, - 2.10747488e-01, 9.53884090e-01, 8.43167950e-01, 4.51187972e-01, 3.75331782e-01, 6.23566461e-01, - 3.55290379e-01, 2.95705968e-01, 1.69622690e-01, 1.42981830e-01, 2.72180991e-01, 9.46468040e-01, - 3.70932500e-01, 9.94292830e-01, 4.62587505e-01, 7.14817405e-01, 2.45370540e-02, 3.00906377e-01, - 5.75768304e-01, 9.71448393e-01, 6.95574827e-02, 3.93693854e-01, 5.29306116e-01, 5.04694554e-01, - 6.73797120e-02, 6.76596969e-01, 5.50948898e-01, 3.24909641e-01, 7.70337719e-01, 6.51842631e-03, - 3.03264879e-01, 7.61037886e-03, 2.72289601e-01, 1.50502041e-01, 6.71103888e-02, 7.41503703e-01, - 1.92088941e-01, 2.19043977e-01, 9.09320161e-01, 2.37993569e-01, 6.18107973e-02, 8.31447852e-01, - 2.23355609e-01, 1.84789435e-01, 4.16104518e-01, 4.21573859e-01, 8.72446305e-02, 2.97294197e-01, - 4.50328256e-01, 8.72199917e-01, 2.51279916e-01, 4.86219272e-01, 7.57071329e-01, 4.85655942e-01, - 1.06187277e-01, 4.92341327e-01, 1.46017513e-01, 5.25421017e-01, 4.22637906e-01, 2.24685018e-01, - 8.72648431e-01, 5.54051490e-01, 1.80745062e-01, 2.12756336e-01, 5.20883169e-01, 7.60363654e-01, - 8.30254678e-01, 5.00003328e-01, 4.69017439e-01, 6.38105527e-01, 3.50638261e-02, 5.22217353e-02, - 9.06516882e-02, 8.52975842e-01, 1.19985883e-01, 3.74926753e-01, 6.50302066e-01, 1.98875727e-01, - 6.28362507e-02, 4.32693501e-01, 3.10500685e-01, 6.20732833e-01, 4.58503272e-01, 3.20790034e-01, - 7.91284868e-01, 7.93054570e-01, 2.93406765e-01, 8.95399023e-01, 1.06441034e-01, 7.53085241e-02, - 8.67523104e-01, 1.47963482e-01, 1.25584706e-01, 3.81545040e-02, 6.34338619e-01, 1.76368938e-02, - 5.75553531e-02, 5.31607516e-01, 2.63869588e-01, 9.41945823e-01, 9.24028838e-02, 5.21496463e-01, - 7.74866558e-01, 5.65210610e-01, 7.28015327e-02, 6.51963790e-01, 8.94727453e-01, 4.49571590e-01, - 1.29932405e-01, 8.64026259e-01, 9.92599934e-01, 7.43721560e-01, 8.87300215e-01, 1.06369925e-01, - 8.11335531e-01, 7.87734900e-01, 9.87344678e-01, 5.32502820e-01, 4.42612382e-01, 9.64041183e-01, - 1.66085871e-01, 1.12937664e-01, 5.24423470e-01, 6.54689333e-01, 4.59119726e-01, 5.22774091e-01, - 3.08722276e-02, 6.26979315e-01, 4.49754105e-01, 8.07495757e-01, 2.34199499e-01, 1.67765675e-01, - 9.22168418e-01, 3.73210378e-01, 8.04432575e-01, 5.61890354e-01, 4.47025593e-01, 6.43155678e-01, - 2.40407640e-01, 5.91631279e-01, 1.59369206e-01, 7.75799090e-01, 8.32067212e-01, 5.59791576e-02, - 6.39105224e-01, 4.85274738e-01, 2.12630838e-01, 2.81431312e-02, 7.16205363e-01, 6.83885011e-01, - 5.23869697e-01, 9.99418314e-01, 8.35331599e-01, 4.69877463e-02, 6.74712562e-01, 7.99273684e-01, - 2.77001890e-02, 5.75809742e-01, 2.78513031e-01, 8.36209905e-01, 7.25472379e-01, 4.87173943e-01, - 7.88311357e-01, 9.64676177e-01, 1.75752651e-01, 4.98112580e-01, 8.08850418e-02, 6.40981131e-01, - 4.06647450e-01, 8.46539387e-01, 2.12620694e-01, 9.11012851e-01, 8.25041445e-01, 8.90065575e-01, - 9.63626055e-01, 5.96689242e-01, 1.63372670e-01, 4.51640148e-01, 3.43026542e-01, 5.80658851e-01, - 2.82327625e-01, 4.75535418e-01, 6.27760926e-01, 8.46314115e-01, 9.61961932e-01, 3.19806094e-01, - 5.05508062e-01, 5.28102944e-01, 6.13045057e-01, 7.44714938e-01, 1.50586073e-01, 7.91878033e-01, - 4.89839179e-01, 3.10496849e-01, 8.82309038e-01, 2.86922314e-01, 4.84687559e-01, 5.20838630e-01, - 4.62955493e-01, 2.38185305e-01, 5.47259907e-02, 7.10916137e-01, 7.31887202e-01, 6.25602317e-01, - 8.77741168e-01, 4.19881322e-01, 4.81222328e-01, 1.28224501e-01, 2.46034010e-01, 3.34971854e-01, - 7.37216484e-01, 5.62134821e-02, 7.14089724e-01, 9.85549393e-01, 4.66295827e-01, 3.08722434e-03, - 4.70237690e-01, 2.66524167e-01, 7.93875484e-01, 4.54795911e-02, 8.09702944e-01, 1.47709735e-02, - 1.70082405e-01, 6.35905179e-01, 3.75379109e-01, 4.30315011e-01, 3.15788760e-01, 5.58065230e-01, - 2.24643800e-01, 2.42142981e-01, 6.57283636e-01, 3.34921891e-01, 1.26588975e-01, 7.68064155e-01, - 9.43856291e-01, 4.47518596e-01, 5.44453573e-01, 9.95764932e-01, 7.16444391e-01, 8.51019765e-01, - 1.01179183e-01, 4.45473958e-01, 4.60327322e-01, 4.96895844e-02, 4.72907738e-01, 5.58987444e-01, - 3.41027487e-01, 1.56175026e-01, 7.58283148e-01, 6.83600909e-01, 2.14623396e-01, 3.27348880e-01, - 3.92517893e-01, 6.70418431e-01, 5.16440832e-01, 8.63140348e-01, 5.73277464e-01, 3.46608058e-01, - 7.39396341e-01, 7.20852434e-01, 2.35653246e-02, 3.89935659e-01, 7.53783745e-01, 6.34563528e-01, - 8.79339335e-01, 7.41599159e-02, 5.62433904e-01, 6.15553852e-01, 4.56956324e-01, 5.20047447e-01, - 5.26845015e-02, 5.58471266e-01, 1.63632233e-01, 5.38936665e-02, 6.49593683e-01, 2.56838748e-01, - 8.99035326e-01, 7.20847756e-01, 5.68954684e-01, 7.43684755e-01, 5.70924238e-01, 3.82318724e-01, - 4.89328290e-01, 5.62208561e-01, 4.97540804e-02, 4.18011085e-01, 6.88041565e-01, 2.16234653e-01, - 7.89548214e-01, 8.46136387e-01, 8.46816189e-01, 1.73842353e-01, 6.11627842e-02, 8.44440559e-01, - 4.50646654e-01, 3.74785037e-01, 4.87196697e-01, 4.56276448e-01, 9.13284391e-01, 4.15715464e-01, - 7.13597697e-01, 1.23641270e-02, 5.10031271e-01, 4.74601930e-02, 2.55731159e-01, 3.22090006e-01, - 1.91165703e-01, 4.51170940e-01, 7.50843157e-01, 4.42420576e-01, 4.25380660e-01, 4.50667257e-01, - 6.55689206e-01, 9.68257670e-02, 1.96528793e-01, 8.97343028e-01, 4.99940904e-01, 6.65504083e-01, - 9.41828079e-01, 4.54397338e-01, 5.61893331e-01, 5.09839880e-01, 4.53117514e-01, 8.96804127e-02, - 1.74888861e-01, 6.65641378e-01, 2.81668336e-01, 1.89532742e-01, 5.61668382e-01, 8.68330157e-02, - 8.25092797e-01, 5.18106324e-01, 1.71904024e-01, 3.68385523e-01, 1.62005436e-01, 7.48507399e-01, - 9.30274827e-01, 2.38198517e-01, 9.52222901e-01, 5.23587800e-01, 6.94384557e-01, 1.09338652e-01, - 4.83356794e-01, 2.73050402e-01, 3.68027050e-01, 5.92366466e-01, 1.83192289e-01, 8.60376029e-01, - 7.13926203e-01, 8.16750052e-01, 1.57890291e-01, 6.25691951e-01, 5.24831646e-01, 1.73873797e-01, - 1.02429784e-01, 9.17488471e-01, 4.03584434e-01, 9.31170884e-01, 2.79386137e-01, 8.77745206e-01, - 2.45200576e-01, 1.28896951e-01, 3.15713052e-01, 5.27874291e-01, 2.16444335e-01, 7.03883817e-01, - 7.74738919e-02, 8.42422142e-01, 3.75598924e-01, 3.51002411e-01, 6.22752776e-01, 4.82407943e-01, - 7.43107867e-01, 9.46182666e-01, 9.44344819e-01, 3.28124763e-01, 1.06147431e-01, 1.65102684e-01, - 3.84060507e-01, 2.91057722e-01, 7.68173662e-02, 1.03543651e-01, 6.76698940e-01, 1.43141994e-01, - 7.21342202e-01, 6.69471294e-03, 9.07298311e-01, 5.57080171e-01, 8.10954489e-01, 4.11120526e-01, - 2.06407453e-01, 2.59590556e-01, 7.58512718e-01, 5.79873897e-01, 2.92875650e-01, 2.83686529e-01, - 2.42829343e-01, 9.19323719e-01, 3.46832864e-01, 3.58238858e-01, 7.42827585e-01, 2.05760059e-01, - 9.58438860e-01, 5.66326411e-01, 6.60292846e-01, 5.61095078e-02, 6.79465531e-01, 7.05118513e-01, - 4.44713264e-01, 2.09732933e-01, 5.22732436e-01, 1.74396512e-01, 5.29356748e-01, 4.38475687e-01, - 4.94036404e-01, 4.09785794e-01, 6.40025507e-01, 5.79371821e-01, 1.57726118e-01, 6.04572263e-01, - 5.41072639e-01, 5.18847173e-01, 1.97093284e-01, 8.91767002e-01, 4.29050835e-01, 8.25490570e-01, - 3.87699807e-01, 4.50705808e-01, 2.49371643e-01, 3.36074898e-01, 9.29925118e-01, 6.65393649e-01, - 9.07275994e-01, 3.73075859e-01, 4.14044139e-03, 2.37463702e-01, 2.25893784e-01, 2.46900245e-01, - 4.50350196e-01, 3.48618117e-01, 5.07193932e-01, 5.23435142e-01, 8.13611417e-01, 8.92715622e-01, - 1.02623450e-01, 3.06088345e-01, 7.80461650e-01, 2.21453645e-01, 2.01419652e-01, 2.84254457e-01, - 3.68286735e-01, 7.39358243e-01, 8.97879394e-01, 9.81599566e-01, 7.56526442e-01, 7.37645545e-01, - 4.23976657e-02, 8.25922012e-01, 2.60956996e-01, 2.90702065e-01, 8.98388344e-01, 3.03733299e-01, - 8.49071471e-01, 3.45835425e-01, 7.65458276e-01, 5.68094872e-01, 8.93770930e-01, 9.93161641e-01, - 5.63368667e-02, 4.26548945e-01, 5.46745780e-01, 5.75674571e-01, 7.94599487e-01, 7.18935553e-02, - 4.46492976e-01, 6.40240123e-01, 2.73246969e-01, 2.00465968e-01, 1.30718835e-01, 1.92492005e-01, - 1.96617189e-01, 6.61271644e-01, 8.12687657e-01, 8.66342445e-01 - - }, - {0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - 10, - true, - -4}, - {10, - 5, - {0.21390334, 0.50261639, 0.91036676, 0.59166485, 0.71162682, 0.10248392, 0.77782677, 0.43772379, - 0.4035871, 0.3282796, 0.47544681, 0.59862974, 0.12319357, 0.06239463, 0.28200272, 0.1345717, - 0.50498218, 0.5113505, 0.16233086, 0.62165332, 0.42281548, 0.933117, 0.41386077, 0.23264562, - 0.73325968, 0.37537541, 0.70719873, 0.14522645, 0.73279625, 0.9126674, 0.84854131, 0.28890216, - 0.85267903, 0.74703138, 0.83842071, 0.34942792, 0.27864171, 0.70911132, 0.21338564, 0.32035554, - 0.73788331, 0.46926692, 0.57570162, 0.42559178, 0.87120209, 0.22734951, 0.01847905, 0.75549396, - 0.76166195, 0.66613745}, - {9, 8, 7, 6, 5, 4, 3, 2, 1, 0}, - 10, - false, - 5}, - // Test outlier points - {9, - 2, - {-1, -50, 3, 4, 5000, 10000, 1, 3, 4, 5, 0.000005, 0.00002, 2000000, 500000, 10, 50, 30, 5}, - {6, 0, 5, 0, 0, 4, 3, 2, 1}, - 7, - false, - 5}, - - // Test n_clusters == (n_points / 2) - {10, - 5, - {0.21390334, 0.50261639, 0.91036676, 0.59166485, 0.71162682, 0.10248392, 0.77782677, 0.43772379, - 0.4035871, 0.3282796, 0.47544681, 0.59862974, 0.12319357, 0.06239463, 0.28200272, 0.1345717, - 0.50498218, 0.5113505, 0.16233086, 0.62165332, 0.42281548, 0.933117, 0.41386077, 0.23264562, - 0.73325968, 0.37537541, 0.70719873, 0.14522645, 0.73279625, 0.9126674, 0.84854131, 0.28890216, - 0.85267903, 0.74703138, 0.83842071, 0.34942792, 0.27864171, 0.70911132, 0.21338564, 0.32035554, - 0.73788331, 0.46926692, 0.57570162, 0.42559178, 0.87120209, 0.22734951, 0.01847905, 0.75549396, - 0.76166195, 0.66613745}, - {1, 0, 4, 0, 0, 3, 2, 0, 2, 1}, - 5, - false, - 10}, - - // Test n_points == 100 - {100, - 10, - {6.26168372e-01, 9.30437651e-01, 6.02450208e-01, 2.73025296e-01, 9.53050619e-01, 3.32164396e-01, - 6.88942598e-01, 5.79163537e-01, 6.70341547e-01, 2.70140602e-02, 9.30429671e-01, 7.17721157e-01, - 9.89948537e-01, 7.75253347e-01, 1.34491522e-02, 2.48522428e-02, 3.51413378e-01, 7.64405834e-01, - 7.86373507e-01, 7.18748577e-01, 8.66998621e-01, 6.80316582e-01, 2.51288712e-01, 4.91078420e-01, - 3.76246281e-01, 4.86828710e-01, 5.67464772e-01, 5.30734742e-01, 8.99478296e-01, 7.66699088e-01, - 9.49339111e-01, 3.55248484e-01, 9.06046929e-01, 4.48407772e-01, 6.96395305e-01, 2.44277335e-01, - 7.74840000e-01, 5.21046603e-01, 4.66423971e-02, 5.12019638e-02, 8.95019614e-01, 5.28956953e-01, - 4.31536306e-01, 5.83857744e-01, 4.41787364e-01, 4.68656523e-01, 5.73971433e-01, 6.79989654e-01, - 3.19650588e-01, 6.12579596e-01, 6.49126442e-02, 8.39131142e-01, 2.85252117e-01, 5.84848929e-01, - 9.46507115e-01, 8.58440748e-01, 3.61528940e-01, 2.44215959e-01, 3.80101125e-01, 4.57128957e-02, - 8.82216988e-01, 8.31498633e-01, 7.23474381e-01, 7.75788607e-01, 1.40864146e-01, 6.62092382e-01, - 5.13985168e-01, 3.00686418e-01, 8.70109949e-01, 2.43187753e-01, 2.89391938e-01, 2.84214238e-01, - 8.70985521e-01, 8.77491176e-01, 6.72537226e-01, 3.30929686e-01, 1.85934324e-01, 9.16222614e-01, - 6.18239142e-01, 2.64768597e-01, 5.76145451e-01, 8.62961369e-01, 6.84757925e-01, 7.60549082e-01, - 1.27645356e-01, 4.51004673e-01, 3.92292980e-01, 4.63170803e-01, 4.35449330e-02, 2.17583404e-01, - 5.71832605e-02, 2.06763039e-01, 3.70116249e-01, 2.09750028e-01, 6.17283019e-01, 8.62549231e-01, - 9.84156240e-02, 2.66249156e-01, 3.87635103e-01, 2.85591012e-02, 4.24826068e-01, 4.45795088e-01, - 6.86227676e-01, 1.08848960e-01, 5.96731841e-02, 3.71770228e-01, 1.91548833e-01, 6.95136078e-01, - 9.00700636e-01, 8.76363105e-01, 2.67334632e-01, 1.80619709e-01, 7.94060419e-01, 1.42854171e-02, - 1.09372387e-01, 8.74028108e-01, 6.46403232e-01, 4.86588834e-01, 5.93446175e-02, 6.11886291e-01, - 8.83865057e-01, 3.15879821e-01, 2.27043992e-01, 9.76764951e-01, 6.15620336e-01, 9.76199360e-01, - 2.40548962e-01, 3.21795663e-01, 8.75087904e-02, 8.11234663e-01, 6.96070480e-01, 8.12062321e-01, - 1.21958818e-01, 3.44348628e-02, 8.72630414e-01, 3.06162776e-01, 1.76043529e-02, 9.45894971e-01, - 5.33896401e-01, 6.21642973e-01, 4.93062535e-01, 4.48984262e-01, 2.24560379e-01, 4.24052195e-02, - 4.43447610e-01, 8.95646149e-01, 6.05220676e-01, 1.81840491e-01, 9.70831206e-01, 2.12563586e-02, - 6.92582693e-01, 7.55946922e-01, 7.95086143e-01, 6.05328941e-01, 3.99350764e-01, 4.32846636e-01, - 9.81114529e-01, 4.98266428e-01, 6.37127930e-03, 1.59085889e-01, 6.34682067e-05, 5.59429440e-01, - 7.38827633e-01, 8.93214770e-01, 2.16494306e-01, 9.35430573e-02, 4.75665868e-02, 7.80503518e-01, - 7.86240041e-01, 7.06854594e-01, 2.13725879e-02, 7.68246091e-01, 4.50234808e-01, 5.21231104e-01, - 5.01989826e-03, 4.22081572e-02, 1.65337732e-01, 8.54134740e-01, 4.99430262e-01, 8.94525601e-01, - 1.14028379e-01, 3.69739861e-01, 1.32955599e-01, 2.65563824e-01, 2.52811151e-01, 1.44792843e-01, - 6.88449594e-01, 4.44921417e-01, 8.23296587e-01, 1.93266317e-01, 1.19033309e-01, 1.36368966e-01, - 3.42600285e-01, 5.64505195e-01, 5.57594559e-01, 7.44257892e-01, 8.38231569e-02, 4.11548847e-01, - 3.21010077e-01, 8.55081359e-01, 4.30105779e-01, 1.16229135e-01, 9.87731964e-02, 3.14712335e-01, - 4.50880592e-01, 2.72289598e-01, 6.31615256e-01, 8.97432958e-01, 4.44764250e-01, 8.03776440e-01, - 2.68767748e-02, 2.43374608e-01, 4.02141103e-01, 4.98881209e-01, 5.33173003e-01, 8.82890436e-01, - 7.16149148e-01, 4.19664401e-01, 2.29335357e-01, 2.88637806e-01, 3.44696803e-01, 6.78171906e-01, - 5.69849716e-01, 5.86454477e-01, 3.54474989e-01, 9.03876540e-01, 6.45980000e-01, 6.34887593e-01, - 7.88039746e-02, 2.04814126e-01, 7.82251754e-01, 2.43147074e-01, 7.50951808e-01, 1.72799092e-02, - 2.95349590e-01, 6.57991826e-01, 8.81214312e-01, 5.73970708e-01, 2.77610881e-01, 1.82155097e-01, - 7.69797417e-02, 6.44792402e-01, 9.46950998e-01, 7.73064845e-01, 6.04733624e-01, 5.80094567e-01, - 1.67498426e-01, 2.66514296e-01, 6.50140368e-01, 1.91170299e-01, 2.08752199e-01, 3.01664091e-01, - 9.85033484e-01, 2.92909152e-01, 8.65816607e-01, 1.85222119e-01, 2.28814559e-01, 1.34286382e-02, - 2.89234322e-01, 8.18668708e-01, 4.71706924e-01, 9.23199803e-01, 2.80879188e-01, 1.47319284e-01, - 4.13915748e-01, 9.31274932e-02, 6.66322195e-01, 9.66953974e-01, 3.19405786e-01, 6.69486551e-01, - 5.03096313e-02, 6.95225201e-01, 5.78469859e-01, 6.29481655e-01, 1.39252534e-01, 1.22564968e-01, - 6.80663678e-01, 6.34607157e-01, 6.42765834e-01, 1.57127410e-02, 2.92132086e-01, 5.24423878e-01, - 4.68676824e-01, 2.86003928e-01, 7.18608322e-01, 8.95617933e-01, 5.48844309e-01, 1.74517278e-01, - 5.24379196e-01, 2.13526524e-01, 5.88375435e-01, 9.88560185e-01, 4.17435771e-01, 6.14438688e-01, - 9.53760881e-01, 5.27151288e-01, 7.03017278e-01, 3.44448559e-01, 4.47059676e-01, 2.83414901e-01, - 1.98979011e-01, 4.24917361e-01, 5.73172761e-01, 2.32398853e-02, 1.65887230e-01, 4.05552785e-01, - 9.29665524e-01, 2.26135696e-01, 9.20563384e-01, 7.65259963e-01, 4.54820075e-01, 8.97710267e-01, - 3.78559302e-03, 9.15219382e-01, 3.55705698e-01, 6.94905124e-01, 8.58540202e-01, 3.89790666e-01, - 2.49478206e-01, 7.93679304e-01, 4.75830027e-01, 4.40425353e-01, 3.70579459e-01, 1.40578049e-01, - 1.70386675e-01, 7.04056121e-01, 4.85963102e-01, 9.68450060e-01, 6.77178001e-01, 2.65934654e-01, - 2.58915007e-01, 6.70052890e-01, 2.61945109e-01, 8.46207759e-01, 1.01928951e-01, 2.85611334e-01, - 2.45776933e-01, 2.66658783e-01, 3.71724077e-01, 4.34319025e-01, 4.24407347e-01, 7.15417683e-01, - 8.07997684e-01, 1.64296275e-01, 6.01638065e-01, 8.60606804e-02, 2.68719187e-01, 5.11764101e-01, - 9.75844338e-01, 7.81226782e-01, 2.20925515e-01, 7.18135040e-01, 9.82395577e-01, 8.39160243e-01, - 9.08058083e-01, 6.88010677e-01, 8.14271847e-01, 5.12460821e-01, 1.17311345e-01, 5.96075228e-01, - 9.17455497e-01, 2.12052706e-01, 7.04074603e-01, 8.72872565e-02, 8.76047818e-01, 6.96235046e-01, - 8.54801557e-01, 2.49729159e-01, 9.76594604e-01, 2.87386363e-01, 2.36461559e-02, 9.94075254e-01, - 4.25193986e-01, 7.61869994e-01, 5.13334255e-01, 6.44711165e-02, 8.92156689e-01, 3.55235167e-01, - 1.08154647e-01, 8.78446825e-01, 2.43833016e-01, 9.23071293e-01, 2.72724115e-01, 9.46631338e-01, - 3.74510294e-01, 4.08451278e-02, 9.78392777e-01, 3.65079221e-01, 6.37199516e-01, 5.51144906e-01, - 5.25978080e-01, 1.42803678e-01, 4.05451674e-01, 7.79788219e-01, 6.26009784e-01, 3.35249497e-01, - 1.43159543e-02, 1.80363779e-01, 5.05096904e-01, 2.82619947e-01, 5.83561392e-01, 3.10951324e-01, - 8.73223968e-01, 4.38545619e-01, 4.81348800e-01, 6.68497085e-01, 3.79345401e-01, 9.58832501e-01, - 1.89869550e-01, 2.34083070e-01, 2.94066207e-01, 5.74892667e-02, 6.92106828e-02, 9.61127686e-02, - 6.72650672e-02, 8.47345378e-01, 2.80916761e-01, 7.32177357e-03, 9.80785961e-01, 5.73192225e-02, - 8.48781331e-01, 8.83225408e-01, 7.34398275e-01, 7.70381941e-01, 6.20778343e-01, 8.96822048e-01, - 5.40732486e-01, 3.69704071e-01, 5.77305837e-01, 2.08221827e-01, 7.34275341e-01, 1.06110900e-01, - 3.49496706e-01, 8.34948910e-01, 1.56403291e-02, 6.78576376e-01, 8.96141268e-01, 5.94835119e-01, - 1.43943153e-01, 3.49618530e-01, 2.10440392e-01, 3.46585620e-01, 1.05153093e-01, 3.45446174e-01, - 2.72177079e-01, 7.07946300e-01, 4.33717726e-02, 3.31232203e-01, 3.91874320e-01, 4.76338141e-01, - 6.22777789e-01, 2.95989228e-02, 4.32855769e-01, 7.61049310e-01, 3.63279149e-01, 9.47210350e-01, - 6.43721247e-01, 6.58025802e-01, 1.05247633e-02, 5.29974442e-01, 7.30675767e-01, 4.30041079e-01, - 6.62634841e-01, 8.25936616e-01, 9.91253704e-01, 6.79399281e-01, 5.44177006e-01, 7.52876048e-01, - 3.32139049e-01, 7.98732398e-01, 7.38865223e-01, 9.16055132e-01, 6.11736493e-01, 9.63672879e-01, - 1.83778839e-01, 7.27558919e-02, 5.91602822e-01, 3.25235484e-01, 2.34741217e-01, 9.52346277e-01, - 9.18556407e-01, 9.35373324e-01, 6.89209070e-01, 2.56049054e-01, 6.17975395e-01, 7.82285691e-01, - 9.84983432e-01, 6.62322741e-01, 2.04144457e-01, 3.98446577e-01, 1.38918297e-01, 3.05919921e-01, - 3.14043787e-01, 5.91072666e-01, 7.44703771e-01, 8.92272567e-01, 9.78017873e-01, 9.01203161e-01, - 1.41526372e-01, 4.14878484e-01, 6.80683651e-01, 5.01733152e-02, 8.14635389e-01, 2.27926375e-01, - 9.03269815e-01, 8.68443745e-01, 9.86939190e-01, 7.40779486e-01, 2.61005311e-01, 3.19276232e-01, - 9.69509248e-01, 1.11908818e-01, 4.49198556e-01, 1.27056715e-01, 3.84064823e-01, 5.14591811e-01, - 2.10747488e-01, 9.53884090e-01, 8.43167950e-01, 4.51187972e-01, 3.75331782e-01, 6.23566461e-01, - 3.55290379e-01, 2.95705968e-01, 1.69622690e-01, 1.42981830e-01, 2.72180991e-01, 9.46468040e-01, - 3.70932500e-01, 9.94292830e-01, 4.62587505e-01, 7.14817405e-01, 2.45370540e-02, 3.00906377e-01, - 5.75768304e-01, 9.71448393e-01, 6.95574827e-02, 3.93693854e-01, 5.29306116e-01, 5.04694554e-01, - 6.73797120e-02, 6.76596969e-01, 5.50948898e-01, 3.24909641e-01, 7.70337719e-01, 6.51842631e-03, - 3.03264879e-01, 7.61037886e-03, 2.72289601e-01, 1.50502041e-01, 6.71103888e-02, 7.41503703e-01, - 1.92088941e-01, 2.19043977e-01, 9.09320161e-01, 2.37993569e-01, 6.18107973e-02, 8.31447852e-01, - 2.23355609e-01, 1.84789435e-01, 4.16104518e-01, 4.21573859e-01, 8.72446305e-02, 2.97294197e-01, - 4.50328256e-01, 8.72199917e-01, 2.51279916e-01, 4.86219272e-01, 7.57071329e-01, 4.85655942e-01, - 1.06187277e-01, 4.92341327e-01, 1.46017513e-01, 5.25421017e-01, 4.22637906e-01, 2.24685018e-01, - 8.72648431e-01, 5.54051490e-01, 1.80745062e-01, 2.12756336e-01, 5.20883169e-01, 7.60363654e-01, - 8.30254678e-01, 5.00003328e-01, 4.69017439e-01, 6.38105527e-01, 3.50638261e-02, 5.22217353e-02, - 9.06516882e-02, 8.52975842e-01, 1.19985883e-01, 3.74926753e-01, 6.50302066e-01, 1.98875727e-01, - 6.28362507e-02, 4.32693501e-01, 3.10500685e-01, 6.20732833e-01, 4.58503272e-01, 3.20790034e-01, - 7.91284868e-01, 7.93054570e-01, 2.93406765e-01, 8.95399023e-01, 1.06441034e-01, 7.53085241e-02, - 8.67523104e-01, 1.47963482e-01, 1.25584706e-01, 3.81545040e-02, 6.34338619e-01, 1.76368938e-02, - 5.75553531e-02, 5.31607516e-01, 2.63869588e-01, 9.41945823e-01, 9.24028838e-02, 5.21496463e-01, - 7.74866558e-01, 5.65210610e-01, 7.28015327e-02, 6.51963790e-01, 8.94727453e-01, 4.49571590e-01, - 1.29932405e-01, 8.64026259e-01, 9.92599934e-01, 7.43721560e-01, 8.87300215e-01, 1.06369925e-01, - 8.11335531e-01, 7.87734900e-01, 9.87344678e-01, 5.32502820e-01, 4.42612382e-01, 9.64041183e-01, - 1.66085871e-01, 1.12937664e-01, 5.24423470e-01, 6.54689333e-01, 4.59119726e-01, 5.22774091e-01, - 3.08722276e-02, 6.26979315e-01, 4.49754105e-01, 8.07495757e-01, 2.34199499e-01, 1.67765675e-01, - 9.22168418e-01, 3.73210378e-01, 8.04432575e-01, 5.61890354e-01, 4.47025593e-01, 6.43155678e-01, - 2.40407640e-01, 5.91631279e-01, 1.59369206e-01, 7.75799090e-01, 8.32067212e-01, 5.59791576e-02, - 6.39105224e-01, 4.85274738e-01, 2.12630838e-01, 2.81431312e-02, 7.16205363e-01, 6.83885011e-01, - 5.23869697e-01, 9.99418314e-01, 8.35331599e-01, 4.69877463e-02, 6.74712562e-01, 7.99273684e-01, - 2.77001890e-02, 5.75809742e-01, 2.78513031e-01, 8.36209905e-01, 7.25472379e-01, 4.87173943e-01, - 7.88311357e-01, 9.64676177e-01, 1.75752651e-01, 4.98112580e-01, 8.08850418e-02, 6.40981131e-01, - 4.06647450e-01, 8.46539387e-01, 2.12620694e-01, 9.11012851e-01, 8.25041445e-01, 8.90065575e-01, - 9.63626055e-01, 5.96689242e-01, 1.63372670e-01, 4.51640148e-01, 3.43026542e-01, 5.80658851e-01, - 2.82327625e-01, 4.75535418e-01, 6.27760926e-01, 8.46314115e-01, 9.61961932e-01, 3.19806094e-01, - 5.05508062e-01, 5.28102944e-01, 6.13045057e-01, 7.44714938e-01, 1.50586073e-01, 7.91878033e-01, - 4.89839179e-01, 3.10496849e-01, 8.82309038e-01, 2.86922314e-01, 4.84687559e-01, 5.20838630e-01, - 4.62955493e-01, 2.38185305e-01, 5.47259907e-02, 7.10916137e-01, 7.31887202e-01, 6.25602317e-01, - 8.77741168e-01, 4.19881322e-01, 4.81222328e-01, 1.28224501e-01, 2.46034010e-01, 3.34971854e-01, - 7.37216484e-01, 5.62134821e-02, 7.14089724e-01, 9.85549393e-01, 4.66295827e-01, 3.08722434e-03, - 4.70237690e-01, 2.66524167e-01, 7.93875484e-01, 4.54795911e-02, 8.09702944e-01, 1.47709735e-02, - 1.70082405e-01, 6.35905179e-01, 3.75379109e-01, 4.30315011e-01, 3.15788760e-01, 5.58065230e-01, - 2.24643800e-01, 2.42142981e-01, 6.57283636e-01, 3.34921891e-01, 1.26588975e-01, 7.68064155e-01, - 9.43856291e-01, 4.47518596e-01, 5.44453573e-01, 9.95764932e-01, 7.16444391e-01, 8.51019765e-01, - 1.01179183e-01, 4.45473958e-01, 4.60327322e-01, 4.96895844e-02, 4.72907738e-01, 5.58987444e-01, - 3.41027487e-01, 1.56175026e-01, 7.58283148e-01, 6.83600909e-01, 2.14623396e-01, 3.27348880e-01, - 3.92517893e-01, 6.70418431e-01, 5.16440832e-01, 8.63140348e-01, 5.73277464e-01, 3.46608058e-01, - 7.39396341e-01, 7.20852434e-01, 2.35653246e-02, 3.89935659e-01, 7.53783745e-01, 6.34563528e-01, - 8.79339335e-01, 7.41599159e-02, 5.62433904e-01, 6.15553852e-01, 4.56956324e-01, 5.20047447e-01, - 5.26845015e-02, 5.58471266e-01, 1.63632233e-01, 5.38936665e-02, 6.49593683e-01, 2.56838748e-01, - 8.99035326e-01, 7.20847756e-01, 5.68954684e-01, 7.43684755e-01, 5.70924238e-01, 3.82318724e-01, - 4.89328290e-01, 5.62208561e-01, 4.97540804e-02, 4.18011085e-01, 6.88041565e-01, 2.16234653e-01, - 7.89548214e-01, 8.46136387e-01, 8.46816189e-01, 1.73842353e-01, 6.11627842e-02, 8.44440559e-01, - 4.50646654e-01, 3.74785037e-01, 4.87196697e-01, 4.56276448e-01, 9.13284391e-01, 4.15715464e-01, - 7.13597697e-01, 1.23641270e-02, 5.10031271e-01, 4.74601930e-02, 2.55731159e-01, 3.22090006e-01, - 1.91165703e-01, 4.51170940e-01, 7.50843157e-01, 4.42420576e-01, 4.25380660e-01, 4.50667257e-01, - 6.55689206e-01, 9.68257670e-02, 1.96528793e-01, 8.97343028e-01, 4.99940904e-01, 6.65504083e-01, - 9.41828079e-01, 4.54397338e-01, 5.61893331e-01, 5.09839880e-01, 4.53117514e-01, 8.96804127e-02, - 1.74888861e-01, 6.65641378e-01, 2.81668336e-01, 1.89532742e-01, 5.61668382e-01, 8.68330157e-02, - 8.25092797e-01, 5.18106324e-01, 1.71904024e-01, 3.68385523e-01, 1.62005436e-01, 7.48507399e-01, - 9.30274827e-01, 2.38198517e-01, 9.52222901e-01, 5.23587800e-01, 6.94384557e-01, 1.09338652e-01, - 4.83356794e-01, 2.73050402e-01, 3.68027050e-01, 5.92366466e-01, 1.83192289e-01, 8.60376029e-01, - 7.13926203e-01, 8.16750052e-01, 1.57890291e-01, 6.25691951e-01, 5.24831646e-01, 1.73873797e-01, - 1.02429784e-01, 9.17488471e-01, 4.03584434e-01, 9.31170884e-01, 2.79386137e-01, 8.77745206e-01, - 2.45200576e-01, 1.28896951e-01, 3.15713052e-01, 5.27874291e-01, 2.16444335e-01, 7.03883817e-01, - 7.74738919e-02, 8.42422142e-01, 3.75598924e-01, 3.51002411e-01, 6.22752776e-01, 4.82407943e-01, - 7.43107867e-01, 9.46182666e-01, 9.44344819e-01, 3.28124763e-01, 1.06147431e-01, 1.65102684e-01, - 3.84060507e-01, 2.91057722e-01, 7.68173662e-02, 1.03543651e-01, 6.76698940e-01, 1.43141994e-01, - 7.21342202e-01, 6.69471294e-03, 9.07298311e-01, 5.57080171e-01, 8.10954489e-01, 4.11120526e-01, - 2.06407453e-01, 2.59590556e-01, 7.58512718e-01, 5.79873897e-01, 2.92875650e-01, 2.83686529e-01, - 2.42829343e-01, 9.19323719e-01, 3.46832864e-01, 3.58238858e-01, 7.42827585e-01, 2.05760059e-01, - 9.58438860e-01, 5.66326411e-01, 6.60292846e-01, 5.61095078e-02, 6.79465531e-01, 7.05118513e-01, - 4.44713264e-01, 2.09732933e-01, 5.22732436e-01, 1.74396512e-01, 5.29356748e-01, 4.38475687e-01, - 4.94036404e-01, 4.09785794e-01, 6.40025507e-01, 5.79371821e-01, 1.57726118e-01, 6.04572263e-01, - 5.41072639e-01, 5.18847173e-01, 1.97093284e-01, 8.91767002e-01, 4.29050835e-01, 8.25490570e-01, - 3.87699807e-01, 4.50705808e-01, 2.49371643e-01, 3.36074898e-01, 9.29925118e-01, 6.65393649e-01, - 9.07275994e-01, 3.73075859e-01, 4.14044139e-03, 2.37463702e-01, 2.25893784e-01, 2.46900245e-01, - 4.50350196e-01, 3.48618117e-01, 5.07193932e-01, 5.23435142e-01, 8.13611417e-01, 8.92715622e-01, - 1.02623450e-01, 3.06088345e-01, 7.80461650e-01, 2.21453645e-01, 2.01419652e-01, 2.84254457e-01, - 3.68286735e-01, 7.39358243e-01, 8.97879394e-01, 9.81599566e-01, 7.56526442e-01, 7.37645545e-01, - 4.23976657e-02, 8.25922012e-01, 2.60956996e-01, 2.90702065e-01, 8.98388344e-01, 3.03733299e-01, - 8.49071471e-01, 3.45835425e-01, 7.65458276e-01, 5.68094872e-01, 8.93770930e-01, 9.93161641e-01, - 5.63368667e-02, 4.26548945e-01, 5.46745780e-01, 5.75674571e-01, 7.94599487e-01, 7.18935553e-02, - 4.46492976e-01, 6.40240123e-01, 2.73246969e-01, 2.00465968e-01, 1.30718835e-01, 1.92492005e-01, - 1.96617189e-01, 6.61271644e-01, 8.12687657e-01, 8.66342445e-01 - - }, - {0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - 10, - false, - 5}}; - -typedef LinkageTest LinkageTestF_Int; -TEST_P(LinkageTestF_Int, Result) { EXPECT_TRUE(score == 1.0); } - -INSTANTIATE_TEST_CASE_P(LinkageTest, LinkageTestF_Int, ::testing::ValuesIn(linkage_inputsf2)); -} // end namespace raft diff --git a/cpp/test/cluster/spectral.cu b/cpp/test/cluster/spectral.cu deleted file mode 100644 index b8ee611f29..0000000000 --- a/cpp/test/cluster/spectral.cu +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2020-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" - -#include -#include -#include - -#include - -#include -#include - -namespace raft { -namespace cluster { - -/** - * Warning: There appears to be a CUDA 12.2 bug in cusparse that causes an - * alignment issue. We've fixed the bug in our code through a workaround - * (see raft/sparse/linalg/spmm.hpp for fix). This test is meant to fail - * in the case where the fix is accidentally reverted, so that it doesn't - * break any downstream libraries that depend on RAFT - */ -TEST(Raft, Spectral) -{ - raft::handle_t handle; - - std::vector h_offsets({0, 2, 4, 7, 10, 12, 14}); - std::vector h_indices({1, 2, 0, 2, 0, 1, 3, 2, 4, 5, 3, 5, 3, 4}); - std::vector h_values( - {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}); - std::vector expected_clustering({1, 1, 1, 0, 0, 0}); - - int32_t n_clusters{2}; - int32_t n_eigenvectors{2}; - int32_t evs_max_it{100}; - int32_t kmean_max_it{100}; - int32_t restartIter_lanczos = 15 + n_eigenvectors; - float evs_tol{0.001}; - float kmean_tol{0.001}; - unsigned long long seed1{1234567}; - unsigned long long seed2{12345678}; - bool reorthog{false}; - - rmm::device_uvector offsets(h_offsets.size(), handle.get_stream()); - rmm::device_uvector indices(h_indices.size(), handle.get_stream()); - rmm::device_uvector values(h_indices.size(), handle.get_stream()); - rmm::device_uvector clustering(expected_clustering.size(), handle.get_stream()); - rmm::device_uvector eigenvalues(n_eigenvectors, handle.get_stream()); - rmm::device_uvector eigenvectors(n_eigenvectors * expected_clustering.size(), - handle.get_stream()); - - rmm::device_uvector exp_dev(expected_clustering.size(), handle.get_stream()); - - raft::update_device( - exp_dev.data(), expected_clustering.data(), expected_clustering.size(), handle.get_stream()); - - raft::update_device(offsets.data(), h_offsets.data(), h_offsets.size(), handle.get_stream()); - raft::update_device(indices.data(), h_indices.data(), h_indices.size(), handle.get_stream()); - raft::update_device(values.data(), h_values.data(), h_values.size(), handle.get_stream()); - - raft::spectral::matrix::sparse_matrix_t const matrix{ - handle, - offsets.data(), - indices.data(), - values.data(), - static_cast(offsets.size() - 1), - static_cast(indices.size())}; - - raft::spectral::eigen_solver_config_t eig_cfg{ - n_eigenvectors, evs_max_it, restartIter_lanczos, evs_tol, reorthog, seed1}; - raft::spectral::lanczos_solver_t eig_solver{eig_cfg}; - - raft::spectral::cluster_solver_config_t clust_cfg{ - n_clusters, kmean_max_it, kmean_tol, seed2}; - raft::spectral::kmeans_solver_t cluster_solver{clust_cfg}; - - raft::spectral::partition(handle, - matrix, - eig_solver, - cluster_solver, - clustering.data(), - eigenvalues.data(), - eigenvectors.data()); - - ASSERT_TRUE(devArrMatch(expected_clustering.data(), - exp_dev.data(), - exp_dev.size(), - 1, - raft::Compare(), - handle.get_stream())); -} - -} // namespace cluster -} // namespace raft \ No newline at end of file diff --git a/cpp/test/distance/dist_adj.cu b/cpp/test/distance/dist_adj.cu deleted file mode 100644 index a22fb7b1f9..0000000000 --- a/cpp/test/distance/dist_adj.cu +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "dist_adj.cuh" - -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace raft { -namespace distance { - -template -RAFT_KERNEL naiveDistanceAdjKernel(uint8_t* dist, - const DataType* x, - const DataType* y, - int m, - int n, - int k, - DataType eps, - bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - DataType acc = DataType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto diff = x[xidx] - y[yidx]; - acc += diff * diff; - } - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc <= eps; -} - -template -void naiveDistanceAdj(uint8_t* dist, - const DataType* x, - const DataType* y, - int m, - int n, - int k, - DataType eps, - bool isRowMajor, - cudaStream_t stream) -{ - static const dim3 TPB(16, 32, 1); - dim3 nblks(raft::ceildiv(m, (int)TPB.x), raft::ceildiv(n, (int)TPB.y), 1); - naiveDistanceAdjKernel<<>>(dist, x, y, m, n, k, eps, isRowMajor); - RAFT_CUDA_TRY(cudaPeekAtLastError()); -} - -template -struct DistanceAdjInputs { - DataType eps; - int m, n, k; - bool isRowMajor; - unsigned long long int seed; -}; - -template -::std::ostream& operator<<(::std::ostream& os, const DistanceAdjInputs& dims) -{ - return os; -} - -template -class DistanceAdjTest : public ::testing::TestWithParam> { - public: - DistanceAdjTest() - : params(::testing::TestWithParam>::GetParam()), - stream(resource::get_cuda_stream(handle)), - dist(params.m * params.n, stream), - dist_ref(params.m * params.n, stream) - { - } - - void SetUp() override - { - raft::random::RngState r(params.seed); - int m = params.m; - int n = params.n; - int k = params.k; - bool isRowMajor = params.isRowMajor; - - rmm::device_uvector x(m * k, stream); - rmm::device_uvector y(n * k, stream); - - uniform(handle, r, x.data(), m * k, DataType(-1.0), DataType(1.0)); - uniform(handle, r, y.data(), n * k, DataType(-1.0), DataType(1.0)); - - DataType threshold = params.eps; - - naiveDistanceAdj(dist_ref.data(), x.data(), y.data(), m, n, k, threshold, isRowMajor, stream); - size_t worksize = raft::distance:: - getWorkspaceSize( - x.data(), y.data(), m, n, k); - rmm::device_uvector workspace(worksize, stream); - - using threshold_final_op_ = threshold_final_op; - threshold_final_op_ threshold_op(threshold); - - raft::distance::distance(handle, - x.data(), - y.data(), - dist.data(), - m, - n, - k, - workspace.data(), - worksize, - threshold_op, - isRowMajor); - resource::sync_stream(handle, stream); - } - - void TearDown() override {} - - protected: - DistanceAdjInputs params; - // We use uint8_t even if the output in this test is a bool because - // cutlass doesn't support bool as output buffer yet. In cuda - // sizeof(bool) is 1 byte hence it doesn't increase - // memory consumption if we use uint8_t instead of bool. - rmm::device_uvector dist_ref; - rmm::device_uvector dist; - raft::resources handle; - cudaStream_t stream; -}; - -const std::vector> inputsf = { - {0.01f, 1024, 1024, 32, true, 1234ULL}, - {0.1f, 1024, 1024, 32, true, 1234ULL}, - {1.0f, 1024, 1024, 32, true, 1234ULL}, - {10.0f, 1024, 1024, 32, true, 1234ULL}, - {0.01f, 1024, 1024, 32, false, 1234ULL}, - {0.1f, 1024, 1024, 32, false, 1234ULL}, - {1.0f, 1024, 1024, 32, false, 1234ULL}, - {10.0f, 1024, 1024, 32, false, 1234ULL}, -}; -typedef DistanceAdjTest DistanceAdjTestF; -TEST_P(DistanceAdjTestF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch(dist_ref.data(), dist.data(), m, n, raft::Compare(), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceAdjTests, DistanceAdjTestF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.01, 1024, 1024, 32, true, 1234ULL}, - {0.1, 1024, 1024, 32, true, 1234ULL}, - {1.0, 1024, 1024, 32, true, 1234ULL}, - {10.0, 1024, 1024, 32, true, 1234ULL}, - {0.01, 1024, 1024, 32, false, 1234ULL}, - {0.1, 1024, 1024, 32, false, 1234ULL}, - {1.0, 1024, 1024, 32, false, 1234ULL}, - {10.0, 1024, 1024, 32, false, 1234ULL}, -}; -typedef DistanceAdjTest DistanceAdjTestD; -TEST_P(DistanceAdjTestD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch(dist_ref.data(), dist.data(), m, n, raft::Compare(), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceAdjTests, DistanceAdjTestD, ::testing::ValuesIn(inputsd)); - -} // namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_adj.cuh b/cpp/test/distance/dist_adj.cuh deleted file mode 100644 index 2861cb33de..0000000000 --- a/cpp/test/distance/dist_adj.cuh +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "dist_adj_threshold.cuh" - -#include - -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, FinalLambda, IdxT) \ - extern template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - size_t worksize, \ - FinalLambda fin_op, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Expanded, - float, - float, - uint8_t, - raft::distance::threshold_float, - int); - -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Expanded, - double, - double, - uint8_t, - raft::distance::threshold_double, - int); - -#undef instantiate_raft_distance_distance - -#define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT) \ - extern template size_t raft::distance::getWorkspaceSize( \ - const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) - -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, uint8_t, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, uint8_t, int); - -#undef instantiate_raft_distance_getWorkspaceSize - -#define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT) \ - extern template size_t raft::distance::getWorkspaceSize( \ - const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) - -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, uint8_t, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, uint8_t, int); - -#undef instantiate_raft_distance_getWorkspaceSize diff --git a/cpp/test/distance/dist_adj_distance_instance.cu b/cpp/test/distance/dist_adj_distance_instance.cu deleted file mode 100644 index 158a5986c2..0000000000 --- a/cpp/test/distance/dist_adj_distance_instance.cu +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#undef RAFT_EXPLICIT_INSTANTIATE_ONLY - -#include "dist_adj_threshold.cuh" - -#include - -#include - -#define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, FinalLambda, IdxT) \ - template void raft::distance::distance( \ - raft::resources const& handle, \ - const DataT* x, \ - const DataT* y, \ - OutT* dist, \ - IdxT m, \ - IdxT n, \ - IdxT k, \ - void* workspace, \ - size_t worksize, \ - FinalLambda fin_op, \ - bool isRowMajor, \ - DataT metric_arg) - -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Expanded, - float, - float, - uint8_t, - raft::distance::threshold_float, - int); - -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Expanded, - double, - double, - uint8_t, - raft::distance::threshold_double, - int); - -#undef instantiate_raft_distance_distance - -#define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT) \ - template size_t raft::distance::getWorkspaceSize( \ - const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) - -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, uint8_t, int); -instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, uint8_t, int); - -#undef instantiate_raft_distance_getWorkspaceSize diff --git a/cpp/test/distance/dist_adj_threshold.cuh b/cpp/test/distance/dist_adj_threshold.cuh deleted file mode 100644 index 78663b3cd1..0000000000 --- a/cpp/test/distance/dist_adj_threshold.cuh +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include // uint8_t - -namespace raft::distance { - -template -struct threshold_final_op { - DataT threshold_val; - - __device__ __host__ threshold_final_op() noexcept : threshold_val(0.0) {} - __device__ __host__ threshold_final_op(DataT val) noexcept : threshold_val(val) {} - __device__ __host__ OutT operator()(AccT d_val, Index g_idx) const noexcept - { - return d_val <= threshold_val; - } -}; - -using threshold_float = threshold_final_op; -using threshold_double = threshold_final_op; - -} // namespace raft::distance diff --git a/cpp/test/distance/dist_canberra.cu b/cpp/test/distance/dist_canberra.cu deleted file mode 100644 index 9b8b6c016b..0000000000 --- a/cpp/test/distance/dist_canberra.cu +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceCanberra : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceCanberra DistanceCanberraF; -TEST_P(DistanceCanberraF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceCanberraF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceCanberra DistanceCanberraD; -TEST_P(DistanceCanberraD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceCanberraD, ::testing::ValuesIn(inputsd)); - -class BigMatrixCanberra : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixCanberra, Result) {} - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_correlation.cu b/cpp/test/distance/dist_correlation.cu deleted file mode 100644 index aa2866483a..0000000000 --- a/cpp/test/distance/dist_correlation.cu +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceCorrelation - : public DistanceTest {}; - -template -class DistanceCorrelationXequalY - : public DistanceTestSameBuffer {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceCorrelation DistanceCorrelationF; -TEST_P(DistanceCorrelationF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceCorrelationF, ::testing::ValuesIn(inputsf)); - -typedef DistanceCorrelationXequalY DistanceCorrelationXequalYF; -TEST_P(DistanceCorrelationXequalYF, Result) -{ - int m = params.m; - ASSERT_TRUE(raft::devArrMatch(dist_ref[0].data(), - dist[0].data(), - m, - m, - raft::CompareApprox(params.tolerance), - stream)); - ASSERT_TRUE(raft::devArrMatch(dist_ref[1].data(), - dist[1].data(), - m / 2, - m, - raft::CompareApprox(params.tolerance), - stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceCorrelationXequalYF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceCorrelation DistanceCorrelationD; -TEST_P(DistanceCorrelationD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceCorrelationD, ::testing::ValuesIn(inputsd)); - -class BigMatrixCorrelation - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixCorrelation, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_cos.cu b/cpp/test/distance/dist_cos.cu deleted file mode 100644 index b792ec4039..0000000000 --- a/cpp/test/distance/dist_cos.cu +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceExpCos : public DistanceTest { -}; - -template -class DistanceExpCosXequalY - : public DistanceTestSameBuffer {}; - -const std::vector> inputsf = { - {0.001f, 128, (65536 + 128) * 128, 8, true, 1234ULL}, - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, (65536 + 128) * 128, 128, 8, false, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; - -const std::vector> inputsXeqYf = { - {0.01f, 1024, 1024, 32, true, 1234ULL}, - {0.01f, 1024, 32, 1024, true, 1234ULL}, - {0.01f, 32, 1024, 1024, true, 1234ULL}, - {0.03f, 1024, 1024, 1024, true, 1234ULL}, - {0.01f, 1024, 1024, 32, false, 1234ULL}, - {0.01f, 1024, 32, 1024, false, 1234ULL}, - {0.01f, 32, 1024, 1024, false, 1234ULL}, - {0.03f, 1024, 1024, 1024, false, 1234ULL}, -}; - -typedef DistanceExpCos DistanceExpCosF; -TEST_P(DistanceExpCosF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceExpCosF, ::testing::ValuesIn(inputsf)); - -typedef DistanceExpCosXequalY DistanceExpCosXequalYF; -TEST_P(DistanceExpCosXequalYF, Result) -{ - int m = params.m; - int n = params.m; - ASSERT_TRUE(raft::devArrMatch(dist_ref[0].data(), - dist[0].data(), - m, - n, - raft::CompareApprox(params.tolerance), - stream)); - n = params.isRowMajor ? m : m / 2; - m = params.isRowMajor ? m / 2 : m; - - ASSERT_TRUE(raft::devArrMatch(dist_ref[1].data(), - dist[1].data(), - m, - n, - raft::CompareApprox(params.tolerance), - stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceExpCosXequalYF, ::testing::ValuesIn(inputsXeqYf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceExpCos DistanceExpCosD; -TEST_P(DistanceExpCosD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceExpCosD, ::testing::ValuesIn(inputsd)); - -class BigMatrixCos : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixCos, Result) {} - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_dice.cu b/cpp/test/distance/dist_dice.cu deleted file mode 100644 index e127659dc6..0000000000 --- a/cpp/test/distance/dist_dice.cu +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceExpDice : public DistanceTest { -}; - -template -class DistanceExpDiceXequalY - : public DistanceTestSameBuffer {}; - -const std::vector> inputsf = { - {0.001f, 128, (65536 + 128) * 128, 8, true, 1234ULL}, - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, (65536 + 128) * 128, 128, 8, false, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; - -const std::vector> inputsXeqYf = { - {0.01f, 1024, 1024, 32, true, 1234ULL}, - {0.01f, 1024, 32, 1024, true, 1234ULL}, - {0.01f, 32, 1024, 1024, true, 1234ULL}, - {0.03f, 1024, 1024, 1024, true, 1234ULL}, - {0.01f, 1024, 1024, 32, false, 1234ULL}, - {0.01f, 1024, 32, 1024, false, 1234ULL}, - {0.01f, 32, 1024, 1024, false, 1234ULL}, - {0.03f, 1024, 1024, 1024, false, 1234ULL}, -}; - -typedef DistanceExpDice DistanceExpDiceF; -TEST_P(DistanceExpDiceF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApproxNaN(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceExpDiceF, ::testing::ValuesIn(inputsf)); - -typedef DistanceExpDiceXequalY DistanceExpDiceXequalYF; -TEST_P(DistanceExpDiceXequalYF, Result) -{ - int m = params.m; - int n = params.m; - ASSERT_TRUE(raft::devArrMatch(dist_ref[0].data(), - dist[0].data(), - m, - n, - raft::CompareApproxNaN(params.tolerance), - stream)); - n = params.isRowMajor ? m : m / 2; - m = params.isRowMajor ? m / 2 : m; - - ASSERT_TRUE(raft::devArrMatch(dist_ref[1].data(), - dist[1].data(), - m, - n, - raft::CompareApproxNaN(params.tolerance), - stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceExpDiceXequalYF, ::testing::ValuesIn(inputsXeqYf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceExpDice DistanceExpDiceD; -TEST_P(DistanceExpDiceD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApproxNaN(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceExpDiceD, ::testing::ValuesIn(inputsd)); - -class BigMatrixDice : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixDice, Result) {} - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_hamming.cu b/cpp/test/distance/dist_hamming.cu deleted file mode 100644 index 9529ec2eaa..0000000000 --- a/cpp/test/distance/dist_hamming.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceHamming - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceHamming DistanceHammingF; -TEST_P(DistanceHammingF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceHammingF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceHamming DistanceHammingD; -TEST_P(DistanceHammingD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceHammingD, ::testing::ValuesIn(inputsd)); - -class BigMatrixHamming - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixHamming, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_hellinger.cu b/cpp/test/distance/dist_hellinger.cu deleted file mode 100644 index 93d6101a18..0000000000 --- a/cpp/test/distance/dist_hellinger.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceHellingerExp - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceHellingerExp DistanceHellingerExpF; -TEST_P(DistanceHellingerExpF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceHellingerExpF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceHellingerExp DistanceHellingerExpD; -TEST_P(DistanceHellingerExpD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceHellingerExpD, ::testing::ValuesIn(inputsd)); - -class BigMatrixHellingerExp - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixHellingerExp, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_inner_product.cu b/cpp/test/distance/dist_inner_product.cu deleted file mode 100644 index 8dd7ef0874..0000000000 --- a/cpp/test/distance/dist_inner_product.cu +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceInnerProduct - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 10, 5, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceInnerProduct DistanceInnerProductF; -TEST_P(DistanceInnerProductF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceInnerProductF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceInnerProduct DistanceInnerProductD; -TEST_P(DistanceInnerProductD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceInnerProductD, ::testing::ValuesIn(inputsd)); - -class BigMatrixInnerProduct - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixInnerProduct, Result) {} - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_jensen_shannon.cu b/cpp/test/distance/dist_jensen_shannon.cu deleted file mode 100644 index e0e256c925..0000000000 --- a/cpp/test/distance/dist_jensen_shannon.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceJensenShannon - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceJensenShannon DistanceJensenShannonF; -TEST_P(DistanceJensenShannonF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceJensenShannonF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceJensenShannon DistanceJensenShannonD; -TEST_P(DistanceJensenShannonD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceJensenShannonD, ::testing::ValuesIn(inputsd)); - -class BigMatrixJensenShannon - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixJensenShannon, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_kl_divergence.cu b/cpp/test/distance/dist_kl_divergence.cu deleted file mode 100644 index 1f79ebcad4..0000000000 --- a/cpp/test/distance/dist_kl_divergence.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceKLDivergence - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceKLDivergence DistanceKLDivergenceF; -TEST_P(DistanceKLDivergenceF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceKLDivergenceF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceKLDivergence DistanceKLDivergenceD; -TEST_P(DistanceKLDivergenceD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceKLDivergenceD, ::testing::ValuesIn(inputsd)); - -class BigMatrixKLDivergence - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixKLDivergence, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_l1.cu b/cpp/test/distance/dist_l1.cu deleted file mode 100644 index ce62a4aeec..0000000000 --- a/cpp/test/distance/dist_l1.cu +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceUnexpL1 : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceUnexpL1 DistanceUnexpL1F; -TEST_P(DistanceUnexpL1F, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceUnexpL1F, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceUnexpL1 DistanceUnexpL1D; -TEST_P(DistanceUnexpL1D, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceUnexpL1D, ::testing::ValuesIn(inputsd)); - -class BigMatrixUnexpL1 : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixUnexpL1, Result) {} - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_l2_exp.cu b/cpp/test/distance/dist_l2_exp.cu deleted file mode 100644 index 0203d9ed9d..0000000000 --- a/cpp/test/distance/dist_l2_exp.cu +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceEucExpTest : public DistanceTest { -}; - -template -class DistanceEucExpTestXequalY - : public DistanceTestSameBuffer {}; - -const std::vector> inputsf = { - {0.001f, 128, (65536 + 128) * 128, 8, true, 1234ULL}, - {0.001f, 2048, 4096, 128, true, 1234ULL}, - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.003f, 1021, 1021, 1021, true, 1234ULL}, - {0.001f, (65536 + 128) * 128, 128, 8, false, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, - {0.003f, 1021, 1021, 1021, false, 1234ULL}, -}; - -const std::vector> inputsXeqYf = { - {0.01f, 2048, 4096, 128, true, 1234ULL}, - {0.01f, 1024, 1024, 32, true, 1234ULL}, - {0.01f, 1024, 32, 1024, true, 1234ULL}, - {0.01f, 32, 1024, 1024, true, 1234ULL}, - {0.03f, 1024, 1024, 1024, true, 1234ULL}, - {0.03f, 1021, 1021, 1021, true, 1234ULL}, - {0.01f, 1024, 1024, 32, false, 1234ULL}, - {0.01f, 1024, 32, 1024, false, 1234ULL}, - {0.01f, 32, 1024, 1024, false, 1234ULL}, - {0.03f, 1024, 1024, 1024, false, 1234ULL}, - {0.03f, 1021, 1021, 1021, false, 1234ULL}, -}; - -typedef DistanceEucExpTest DistanceEucExpTestF; -TEST_P(DistanceEucExpTestF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceEucExpTestF, ::testing::ValuesIn(inputsf)); - -typedef DistanceEucExpTestXequalY DistanceEucExpTestXequalYF; -TEST_P(DistanceEucExpTestXequalYF, Result) -{ - int m = params.m; - ASSERT_TRUE(raft::devArrMatch(dist_ref[0].data(), - dist[0].data(), - m, - m, - raft::CompareApprox(params.tolerance), - stream)); - ASSERT_TRUE(raft::devArrMatch(dist_ref[1].data(), - dist[1].data(), - m / 2, - m, - raft::CompareApprox(params.tolerance), - stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, - DistanceEucExpTestXequalYF, - ::testing::ValuesIn(inputsXeqYf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceEucExpTest DistanceEucExpTestD; -TEST_P(DistanceEucExpTestD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceEucExpTestD, ::testing::ValuesIn(inputsd)); - -class BigMatrixEucExp : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixEucExp, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_l2_sqrt_exp.cu b/cpp/test/distance/dist_l2_sqrt_exp.cu deleted file mode 100644 index 5bccabcc3f..0000000000 --- a/cpp/test/distance/dist_l2_sqrt_exp.cu +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceEucSqrtExpTest - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 2048, 4096, 128, true, 1234ULL}, - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.003f, 1021, 1021, 1021, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, - {0.003f, 1021, 1021, 1021, false, 1234ULL}, -}; -typedef DistanceEucSqrtExpTest DistanceEucSqrtExpTestF; -TEST_P(DistanceEucSqrtExpTestF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceEucSqrtExpTestF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceEucSqrtExpTest DistanceEucSqrtExpTestD; -TEST_P(DistanceEucSqrtExpTestD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceEucSqrtExpTestD, ::testing::ValuesIn(inputsd)); - -class BigMatrixEucSqrtExp - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixEucSqrtExp, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_l2_unexp.cu b/cpp/test/distance/dist_l2_unexp.cu deleted file mode 100644 index 19b0ff6dbf..0000000000 --- a/cpp/test/distance/dist_l2_unexp.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceEucUnexpTest - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceEucUnexpTest DistanceEucUnexpTestF; -TEST_P(DistanceEucUnexpTestF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceEucUnexpTestF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceEucUnexpTest DistanceEucUnexpTestD; -TEST_P(DistanceEucUnexpTestD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceEucUnexpTestD, ::testing::ValuesIn(inputsd)); - -class BigMatrixEucUnexp : public BigMatrixDistanceTest { -}; -TEST_F(BigMatrixEucUnexp, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_l_inf.cu b/cpp/test/distance/dist_l_inf.cu deleted file mode 100644 index 223d186a8d..0000000000 --- a/cpp/test/distance/dist_l_inf.cu +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceLinf : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceLinf DistanceLinfF; -TEST_P(DistanceLinfF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceLinfF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceLinf DistanceLinfD; -TEST_P(DistanceLinfD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceLinfD, ::testing::ValuesIn(inputsd)); - -class BigMatrixLinf : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixLinf, Result) {} - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_lp_unexp.cu b/cpp/test/distance/dist_lp_unexp.cu deleted file mode 100644 index 9d6f5921a7..0000000000 --- a/cpp/test/distance/dist_lp_unexp.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceLpUnexp : public DistanceTest { -}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL, 4.0f}, - {0.001f, 1024, 32, 1024, true, 1234ULL, 3.0f}, - {0.001f, 32, 1024, 1024, true, 1234ULL, 4.0f}, - {0.003f, 1024, 1024, 1024, true, 1234ULL, 3.0f}, - {0.001f, 1024, 1024, 32, false, 1234ULL, 4.0f}, - {0.001f, 1024, 32, 1024, false, 1234ULL, 3.0f}, - {0.001f, 32, 1024, 1024, false, 1234ULL, 4.0f}, - {0.003f, 1024, 1024, 1024, false, 1234ULL, 3.0f}, -}; -typedef DistanceLpUnexp DistanceLpUnexpF; -TEST_P(DistanceLpUnexpF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceLpUnexpF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL, 4.0}, - {0.001, 1024, 32, 1024, true, 1234ULL, 3.0}, - {0.001, 32, 1024, 1024, true, 1234ULL, 4.0}, - {0.003, 1024, 1024, 1024, true, 1234ULL, 3.0}, - {0.001, 1024, 1024, 32, false, 1234ULL, 4.0}, - {0.001, 1024, 32, 1024, false, 1234ULL, 3.0}, - {0.001, 32, 1024, 1024, false, 1234ULL, 4.0}, - {0.003, 1024, 1024, 1024, false, 1234ULL, 3.0}, -}; -typedef DistanceLpUnexp DistanceLpUnexpD; -TEST_P(DistanceLpUnexpD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceLpUnexpD, ::testing::ValuesIn(inputsd)); - -class BigMatrixLpUnexp : public BigMatrixDistanceTest { -}; -TEST_F(BigMatrixLpUnexp, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/dist_russell_rao.cu b/cpp/test/distance/dist_russell_rao.cu deleted file mode 100644 index 73cf4b33a4..0000000000 --- a/cpp/test/distance/dist_russell_rao.cu +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "distance_base.cuh" - -namespace raft { -namespace distance { - -template -class DistanceRussellRao - : public DistanceTest {}; - -const std::vector> inputsf = { - {0.001f, 1024, 1024, 32, true, 1234ULL}, - {0.001f, 1024, 32, 1024, true, 1234ULL}, - {0.001f, 32, 1024, 1024, true, 1234ULL}, - {0.003f, 1024, 1024, 1024, true, 1234ULL}, - {0.001f, 1024, 1024, 32, false, 1234ULL}, - {0.001f, 1024, 32, 1024, false, 1234ULL}, - {0.001f, 32, 1024, 1024, false, 1234ULL}, - {0.003f, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceRussellRao DistanceRussellRaoF; -TEST_P(DistanceRussellRaoF, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceRussellRaoF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.001, 1024, 1024, 32, true, 1234ULL}, - {0.001, 1024, 32, 1024, true, 1234ULL}, - {0.001, 32, 1024, 1024, true, 1234ULL}, - {0.003, 1024, 1024, 1024, true, 1234ULL}, - {0.001, 1024, 1024, 32, false, 1234ULL}, - {0.001, 1024, 32, 1024, false, 1234ULL}, - {0.001, 32, 1024, 1024, false, 1234ULL}, - {0.003, 1024, 1024, 1024, false, 1234ULL}, -}; -typedef DistanceRussellRao DistanceRussellRaoD; -TEST_P(DistanceRussellRaoD, Result) -{ - int m = params.isRowMajor ? params.m : params.n; - int n = params.isRowMajor ? params.n : params.m; - ASSERT_TRUE(raft::devArrMatch( - dist_ref.data(), dist.data(), m, n, raft::CompareApprox(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(DistanceTests, DistanceRussellRaoD, ::testing::ValuesIn(inputsd)); - -class BigMatrixRussellRao - : public BigMatrixDistanceTest {}; -TEST_F(BigMatrixRussellRao, Result) {} -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/distance_base.cuh b/cpp/test/distance/distance_base.cuh deleted file mode 100644 index f44fb18519..0000000000 --- a/cpp/test/distance/distance_base.cuh +++ /dev/null @@ -1,708 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" - -#include // common::nvtx::range -#include // make_device_matrix_view -#include // raft::sqrt -#include -#include // raft::resources -#include -#include // raft::distance::DistanceType -#include - -#include // rmm::device_uvector - -#include - -namespace raft { -namespace distance { - -template -RAFT_KERNEL naiveDistanceKernel(DataType* dist, - const DataType* x, - const DataType* y, - int m, - int n, - int k, - raft::distance::DistanceType type, - bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - DataType acc = DataType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto diff = x[xidx] - y[yidx]; - acc += diff * diff; - } - if (type == raft::distance::DistanceType::L2SqrtExpanded || - type == raft::distance::DistanceType::L2SqrtUnexpanded) - acc = raft::sqrt(acc); - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -RAFT_KERNEL naiveL1_Linf_CanberraDistanceKernel(DataType* dist, - const DataType* x, - const DataType* y, - int m, - int n, - int k, - raft::distance::DistanceType type, - bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) { return; } - - DataType acc = DataType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - auto diff = (a > b) ? (a - b) : (b - a); - if (type == raft::distance::DistanceType::Linf) { - acc = raft::max(acc, diff); - } else if (type == raft::distance::DistanceType::Canberra) { - const auto add = raft::abs(a) + raft::abs(b); - // deal with potential for 0 in denominator by - // forcing 1/0 instead - acc += ((add != 0) * diff / (add + (add == 0))); - } else { - acc += diff; - } - } - - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -RAFT_KERNEL naiveDiceDistanceKernel( - DataType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) { return; } - - DataType acc_a = DataType(0); - DataType acc_b = DataType(0); - DataType acc_ab = DataType(0); - - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - acc_a += a; - acc_b += b; - acc_ab += a * b; - } - - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - - // Use 1.0 - (dice dissimilarity) to calc the distance - dist[outidx] = (DataType)1.0 - (2 * acc_ab / ((acc_a) + (acc_b))); -} - -template -RAFT_KERNEL naiveCosineDistanceKernel( - DataType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) { return; } - - DataType acc_a = DataType(0); - DataType acc_b = DataType(0); - DataType acc_ab = DataType(0); - - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - acc_a += a * a; - acc_b += b * b; - acc_ab += a * b; - } - - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - - // Use 1.0 - (cosine similarity) to calc the distance - dist[outidx] = (DataType)1.0 - acc_ab / (raft::sqrt(acc_a) * raft::sqrt(acc_b)); -} - -template -RAFT_KERNEL naiveInnerProductKernel( - DataType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) { return; } - - DataType acc_ab = DataType(0); - - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - acc_ab += a * b; - } - - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc_ab; -} - -template -RAFT_KERNEL naiveHellingerDistanceKernel( - DataType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) { return; } - - DataType acc_ab = DataType(0); - - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - acc_ab += raft::sqrt(a) * raft::sqrt(b); - } - - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - - // Adjust to replace NaN in sqrt with 0 if input to sqrt is negative - acc_ab = 1 - acc_ab; - auto rectifier = (!signbit(acc_ab)); - dist[outidx] = raft::sqrt(rectifier * acc_ab); -} - -template -RAFT_KERNEL naiveLpUnexpDistanceKernel(DataType* dist, - const DataType* x, - const DataType* y, - int m, - int n, - int k, - bool isRowMajor, - DataType p) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - DataType acc = DataType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - auto diff = raft::abs(a - b); - acc += raft::pow(diff, p); - } - auto one_over_p = 1 / p; - acc = raft::pow(acc, one_over_p); - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -RAFT_KERNEL naiveHammingDistanceKernel( - DataType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - DataType acc = DataType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - acc += (a != b); - } - acc = acc / k; - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -RAFT_KERNEL naiveJensenShannonDistanceKernel( - DataType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - DataType acc = DataType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - - DataType m = 0.5f * (a + b); - bool a_zero = a == 0; - bool b_zero = b == 0; - - DataType p = (!a_zero * m) / (a_zero + a); - DataType q = (!b_zero * m) / (b_zero + b); - - bool p_zero = p == 0; - bool q_zero = q == 0; - - acc += (-a * (!p_zero * log(p + p_zero))) + (-b * (!q_zero * log(q + q_zero))); - } - acc = raft::sqrt(0.5f * acc); - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -RAFT_KERNEL naiveRussellRaoDistanceKernel( - OutType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - OutType acc = OutType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - acc += (a * b); - } - acc = (k - acc) / k; - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -RAFT_KERNEL naiveKLDivergenceDistanceKernel( - OutType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - OutType acc = OutType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - bool b_zero = (b == 0); - bool a_zero = (a == 0); - acc += a * (log(a + a_zero) - log(b + b_zero)); - } - acc = 0.5f * acc; - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -RAFT_KERNEL naiveCorrelationDistanceKernel( - OutType* dist, const DataType* x, const DataType* y, int m, int n, int k, bool isRowMajor) -{ - int midx = threadIdx.x + blockIdx.x * blockDim.x; - int nidx = threadIdx.y + blockIdx.y * blockDim.y; - if (midx >= m || nidx >= n) return; - OutType acc = OutType(0); - auto a_norm = DataType(0); - auto b_norm = DataType(0); - auto a_sq_norm = DataType(0); - auto b_sq_norm = DataType(0); - for (int i = 0; i < k; ++i) { - int xidx = isRowMajor ? i + midx * k : i * m + midx; - int yidx = isRowMajor ? i + nidx * k : i * n + nidx; - auto a = x[xidx]; - auto b = y[yidx]; - a_norm += a; - b_norm += b; - a_sq_norm += (a * a); - b_sq_norm += (b * b); - acc += (a * b); - } - - auto numer = k * acc - (a_norm * b_norm); - auto Q_denom = k * a_sq_norm - (a_norm * a_norm); - auto R_denom = k * b_sq_norm - (b_norm * b_norm); - - acc = 1 - (numer / raft::sqrt(Q_denom * R_denom)); - - int outidx = isRowMajor ? midx * n + nidx : midx + m * nidx; - dist[outidx] = acc; -} - -template -void naiveDistance(DataType* dist, - const DataType* x, - const DataType* y, - int m, - int n, - int k, - raft::distance::DistanceType type, - bool isRowMajor, - DataType metric_arg = 2.0f, - cudaStream_t stream = 0) -{ - static const dim3 TPB(4, 256, 1); - dim3 nblks(raft::ceildiv(m, (int)TPB.x), raft::ceildiv(n, (int)TPB.y), 1); - - switch (type) { - case raft::distance::DistanceType::Canberra: - case raft::distance::DistanceType::Linf: - case raft::distance::DistanceType::L1: - naiveL1_Linf_CanberraDistanceKernel - <<>>(dist, x, y, m, n, k, type, isRowMajor); - break; - case raft::distance::DistanceType::L2SqrtUnexpanded: - case raft::distance::DistanceType::L2Unexpanded: - case raft::distance::DistanceType::L2SqrtExpanded: - case raft::distance::DistanceType::L2Expanded: - naiveDistanceKernel - <<>>(dist, x, y, m, n, k, type, isRowMajor); - break; - case raft::distance::DistanceType::CosineExpanded: - naiveCosineDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::HellingerExpanded: - naiveHellingerDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::LpUnexpanded: - naiveLpUnexpDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor, metric_arg); - break; - case raft::distance::DistanceType::HammingUnexpanded: - naiveHammingDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::InnerProduct: - naiveInnerProductKernel<<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::JensenShannon: - naiveJensenShannonDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::RusselRaoExpanded: - naiveRussellRaoDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::KLDivergence: - naiveKLDivergenceDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::CorrelationExpanded: - naiveCorrelationDistanceKernel - <<>>(dist, x, y, m, n, k, isRowMajor); - break; - case raft::distance::DistanceType::DiceExpanded: - naiveDiceDistanceKernel<<>>(dist, x, y, m, n, k, isRowMajor); - break; - default: FAIL() << "should be here\n"; - } - RAFT_CUDA_TRY(cudaPeekAtLastError()); -} - -template -struct DistanceInputs { - DataType tolerance; - int m, n, k; - bool isRowMajor; - unsigned long long int seed; - DataType metric_arg = 2.0f; -}; - -template -::std::ostream& operator<<(::std::ostream& os, const DistanceInputs& dims) -{ - return os; -} - -// TODO: Remove when mdspan-based raft::runtime::distance::pairwise_distance is -// implemented. -// -// Context: -// https://github.com/rapidsai/raft/issues/1338 -template -constexpr bool layout_to_row_major(); - -template <> -constexpr bool layout_to_row_major() -{ - return true; -} -template <> -constexpr bool layout_to_row_major() -{ - return false; -} - -template -void distanceLauncher(raft::resources const& handle, - DataType* x, - DataType* y, - DataType* dist, - DataType* dist2, - int m, - int n, - int k, - DistanceInputs& params, - DataType threshold, - DataType metric_arg = 2.0f) -{ - auto x_v = make_device_matrix_view(x, m, k); - auto y_v = make_device_matrix_view(y, n, k); - auto dist_v = make_device_matrix_view(dist, m, n); - - raft::distance::distance( - handle, x_v, y_v, dist_v, metric_arg); -} - -template -class DistanceTest : public ::testing::TestWithParam> { - public: - DistanceTest() - : params(::testing::TestWithParam>::GetParam()), - stream(resource::get_cuda_stream(handle)), - x(params.m * params.k, stream), - y(params.n * params.k, stream), - dist_ref(params.m * params.n, stream), - dist(params.m * params.n, stream), - dist2(params.m * params.n, stream) - { - } - - void SetUp() override - { - auto testInfo = testing::UnitTest::GetInstance()->current_test_info(); - common::nvtx::range fun_scope("test::%s/%s", testInfo->test_suite_name(), testInfo->name()); - - raft::random::RngState r(params.seed); - int m = params.m; - int n = params.n; - int k = params.k; - DataType metric_arg = params.metric_arg; - bool isRowMajor = params.isRowMajor; - if (distanceType == raft::distance::DistanceType::HellingerExpanded || - distanceType == raft::distance::DistanceType::JensenShannon || - distanceType == raft::distance::DistanceType::KLDivergence) { - // Hellinger works only on positive numbers - uniform(handle, r, x.data(), m * k, DataType(0.0), DataType(1.0)); - uniform(handle, r, y.data(), n * k, DataType(0.0), DataType(1.0)); - } else if (distanceType == raft::distance::DistanceType::RusselRaoExpanded || - distanceType == raft::distance::DistanceType::DiceExpanded) { - uniform(handle, r, x.data(), m * k, DataType(0.0), DataType(1.0)); - uniform(handle, r, y.data(), n * k, DataType(0.0), DataType(1.0)); - // Russel rao works on boolean values. - bernoulli(handle, r, x.data(), m * k, 0.5f); - bernoulli(handle, r, y.data(), n * k, 0.5f); - } else { - uniform(handle, r, x.data(), m * k, DataType(-1.0), DataType(1.0)); - uniform(handle, r, y.data(), n * k, DataType(-1.0), DataType(1.0)); - } - naiveDistance( - dist_ref.data(), x.data(), y.data(), m, n, k, distanceType, isRowMajor, metric_arg, stream); - - DataType threshold = -10000.f; - - if (isRowMajor) { - distanceLauncher(handle, - x.data(), - y.data(), - dist.data(), - dist2.data(), - m, - n, - k, - params, - threshold, - metric_arg); - - } else { - distanceLauncher(handle, - x.data(), - y.data(), - dist.data(), - dist2.data(), - m, - n, - k, - params, - threshold, - metric_arg); - } - resource::sync_stream(handle, stream); - } - - protected: - raft::resources handle; - cudaStream_t stream; - - DistanceInputs params; - rmm::device_uvector x, y, dist_ref, dist, dist2; -}; - -/* - * This test suite verifies the path when X and Y are same buffer, - * distance metrics which requires norms like L2 expanded/cosine/correlation - * takes a more optimal path in such case to skip norm calculation for Y buffer. - * It may happen that though both X and Y are same buffer but user passes - * different dimensions for them like in case of tiled_brute_force_knn. - */ -template -class DistanceTestSameBuffer : public ::testing::TestWithParam> { - public: - using dev_vector = rmm::device_uvector; - DistanceTestSameBuffer() - : params(::testing::TestWithParam>::GetParam()), - stream(resource::get_cuda_stream(handle)), - x(params.m * params.k, stream), - dist_ref({dev_vector(params.m * params.m, stream), dev_vector(params.m * params.m, stream)}), - dist({dev_vector(params.m * params.m, stream), dev_vector(params.m * params.m, stream)}), - dist2({dev_vector(params.m * params.m, stream), dev_vector(params.m * params.m, stream)}) - { - } - - void SetUp() override - { - auto testInfo = testing::UnitTest::GetInstance()->current_test_info(); - common::nvtx::range fun_scope("test::%s/%s", testInfo->test_suite_name(), testInfo->name()); - - raft::random::RngState r(params.seed); - int m = params.m; - int n = params.m; - int k = params.k; - DataType metric_arg = params.metric_arg; - bool isRowMajor = params.isRowMajor; - if (distanceType == raft::distance::DistanceType::HellingerExpanded || - distanceType == raft::distance::DistanceType::JensenShannon || - distanceType == raft::distance::DistanceType::KLDivergence) { - // Hellinger works only on positive numbers - uniform(handle, r, x.data(), m * k, DataType(0.0), DataType(1.0)); - } else if (distanceType == raft::distance::DistanceType::RusselRaoExpanded || - distanceType == raft::distance::DistanceType::DiceExpanded) { - uniform(handle, r, x.data(), m * k, DataType(0.0), DataType(1.0)); - // Russel rao works on boolean values. - bernoulli(handle, r, x.data(), m * k, 0.5f); - } else { - uniform(handle, r, x.data(), m * k, DataType(-1.0), DataType(1.0)); - } - - for (int i = 0; i < 2; i++) { - // both X and Y are same buffer but when i = 1 - // different dimensions for x & y is passed. - m = m / (i + 1); - naiveDistance(dist_ref[i].data(), - x.data(), - x.data(), - m, - n, - k, - distanceType, - isRowMajor, - metric_arg, - stream); - - DataType threshold = -10000.f; - - if (isRowMajor) { - distanceLauncher(handle, - x.data(), - x.data(), - dist[i].data(), - dist2[i].data(), - m, - n, - k, - params, - threshold, - metric_arg); - - } else { - distanceLauncher(handle, - x.data(), - x.data(), - dist[i].data(), - dist2[i].data(), - m, - n, - k, - params, - threshold, - metric_arg); - } - } - resource::sync_stream(handle, stream); - } - - protected: - raft::resources handle; - cudaStream_t stream; - - DistanceInputs params; - dev_vector x; - static const int N = 2; - std::array dist_ref, dist, dist2; -}; - -template -class BigMatrixDistanceTest : public ::testing::Test { - public: - BigMatrixDistanceTest() - : x(m * k, resource::get_cuda_stream(handle)), - dist(std::size_t(m) * m, resource::get_cuda_stream(handle)){}; - void SetUp() override - { - auto testInfo = testing::UnitTest::GetInstance()->current_test_info(); - common::nvtx::range fun_scope("test::%s/%s", testInfo->test_suite_name(), testInfo->name()); - - void pairwise_distance(raft::resources const& handle, - float* x, - float* y, - float* dists, - int m, - int n, - int k, - raft::distance::DistanceType metric, - bool isRowMajor, - float metric_arg); - constexpr bool row_major = true; - constexpr float metric_arg = 0.0f; - raft::distance::distance( - handle, x.data(), x.data(), dist.data(), m, n, k, row_major, metric_arg); - RAFT_CUDA_TRY(cudaStreamSynchronize(resource::get_cuda_stream(handle))); - } - - protected: - raft::resources handle; - int m = 48000; - int n = 48000; - int k = 1; - rmm::device_uvector x, dist; -}; -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/fused_cosine_nn.cu b/cpp/test/distance/fused_cosine_nn.cu deleted file mode 100644 index d4d632e1dc..0000000000 --- a/cpp/test/distance/fused_cosine_nn.cu +++ /dev/null @@ -1,420 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#undef RAFT_EXPLICIT_INSTANTIATE_ONLY // Search with filter instantiation - -#include "../test_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace raft { -namespace distance { - -template -struct RaftKVPMinReduce { - typedef raft::KeyValuePair KVP; - - DI KVP operator()(LabelT rit, const KVP& a, const KVP& b) { return b.value < a.value ? b : a; } - - DI KVP operator()(const KVP& a, const KVP& b) { return b.value < a.value ? b : a; } - -}; // KVPMinReduce - -template -__global__ void naiveCosKernel(raft::KeyValuePair* min, - DataT* x, - DataT* y, - int m, - int n, - int k, - int* workspace, - DataT maxVal) -{ - int midx = threadIdx.y + blockIdx.y * blockDim.y; - int nidx = threadIdx.x + blockIdx.x * blockDim.x; - DataT acc_a = DataT(0); - DataT acc_b = DataT(0); - DataT acc_ab = DataT(0); - // if (midx >= m || nidx >= n) { return; } - - for (int i = 0; i < k; ++i) { - int xidx = i + midx * k; - int yidx = i + nidx * k; - auto a = x[xidx]; - auto b = y[yidx]; - acc_a += a * a; - acc_b += b * b; - acc_ab += a * b; - } - - // Use 1.0 - (cosine similarity) to calc the distance - DataT acc = maxVal; - if (midx < m || nidx < n) { acc = (DataT)1.0 - acc_ab / (raft::sqrt(acc_a) * raft::sqrt(acc_b)); } - - ReduceOpT redOp; - typedef cub::WarpReduce> WarpReduce; - __shared__ typename WarpReduce::TempStorage temp[NWARPS]; - int warpId = threadIdx.x / raft::WarpSize; - raft::KeyValuePair tmp; - tmp.key = nidx; - tmp.value = midx >= m || nidx >= n ? maxVal : acc; - tmp = WarpReduce(temp[warpId]).Reduce(tmp, RaftKVPMinReduce()); - if (threadIdx.x % raft::WarpSize == 0 && midx < m) { - while (atomicCAS(workspace + midx, 0, 1) == 1) - ; - __threadfence(); - redOp(midx, min + midx, tmp); - __threadfence(); - atomicCAS(workspace + midx, 1, 0); - } -} - -template -void naive(raft::KeyValuePair* min, - DataT* x, - DataT* y, - int m, - int n, - int k, - int* workspace, - cudaStream_t stream) -{ - static const dim3 TPB(32, 16, 1); - dim3 nblks(raft::ceildiv(n, (int)TPB.x), raft::ceildiv(m, (int)TPB.y), 1); - RAFT_CUDA_TRY(cudaMemsetAsync(workspace, 0, sizeof(int) * m, stream)); - auto blks = raft::ceildiv(m, 256); - MinAndDistanceReduceOp op; - detail::initKernel, int> - <<>>(min, m, std::numeric_limits::max(), op); - RAFT_CUDA_TRY(cudaGetLastError()); - naiveCosKernel, 16> - <<>>(min, x, y, m, n, k, workspace, std::numeric_limits::max()); - RAFT_CUDA_TRY(cudaGetLastError()); -} - -template -struct Inputs { - DataT tolerance; - int m, n, k; - unsigned long long int seed; - - friend std::ostream& operator<<(std::ostream& os, const Inputs& p) - { - return os << "m: " << p.m - << ", " - "n: " - << p.n - << ", " - "k: " - << p.k - << ", " - "seed: " - << p.seed - << ", " - "tol: " - << p.tolerance; - } -}; - -template -class FusedCosineNNTest : public ::testing::TestWithParam> { - public: - FusedCosineNNTest() - : params(::testing::TestWithParam>::GetParam()), - stream(resource::get_cuda_stream(handle)), - x(params.m * params.k, stream), - y(params.n * params.k, stream), - xn(params.m, stream), - yn(params.n, stream), - min(params.m, stream), - min_ref(params.m, stream), - workspace(params.m * sizeof(int), stream) - { - } - - protected: - void SetUp() override - { - raft::random::RngState r(params.seed); - int m = params.m; - int n = params.n; - int k = params.k; - uniform(handle, r, x.data(), m * k, DataT(-1.0), DataT(1.0)); - uniform(handle, r, y.data(), n * k, DataT(-1.0), DataT(1.0)); - generateGoldenResult(); - raft::linalg::rowNorm( - xn.data(), x.data(), k, m, raft::linalg::L2Norm, true, stream, raft::sqrt_op{}); - raft::linalg::rowNorm( - yn.data(), y.data(), k, n, raft::linalg::L2Norm, true, stream, raft::sqrt_op{}); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - } - - protected: - raft::resources handle; - cudaStream_t stream; - Inputs params; - rmm::device_uvector x; - rmm::device_uvector y; - rmm::device_uvector xn; - rmm::device_uvector yn; - rmm::device_uvector> min; - rmm::device_uvector> min_ref; - rmm::device_uvector workspace; - - virtual void generateGoldenResult() - { - int m = params.m; - int n = params.n; - int k = params.k; - naive(min_ref.data(), x.data(), y.data(), m, n, k, (int*)workspace.data(), stream); - } - - void runTest(raft::KeyValuePair* out) - { - int m = params.m; - int n = params.n; - int k = params.k; - raft::distance::DistanceType metric = raft::distance::DistanceType::CosineExpanded; - constexpr bool init_out_buffer = true; - fusedDistanceNNMinReduce, int>(out, - x.data(), - y.data(), - xn.data(), - yn.data(), - m, - n, - k, - (void*)workspace.data(), - false, - init_out_buffer, - true, - metric, - 0.0f, - stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - } -}; - -template -struct CompareApproxAbsKVP { - typedef typename raft::KeyValuePair KVP; - CompareApproxAbsKVP(T eps_) : eps(eps_) {} - bool operator()(const KVP& a, const KVP& b) const - { - T diff = std::abs(std::abs(a.value) - std::abs(b.value)); - T m = std::max(std::abs(a.value), std::abs(b.value)); - T ratio = m >= eps ? diff / m : diff; - return (ratio <= eps); - } - - private: - T eps; -}; - -template -struct CompareExactKVP { - typedef typename raft::KeyValuePair KVP; - bool operator()(const KVP& a, const KVP& b) const - { - if (a.value != b.value) return false; - return true; - } -}; - -template -::testing::AssertionResult devArrMatch(const raft::KeyValuePair* expected, - const raft::KeyValuePair* actual, - size_t size, - L eq_compare, - cudaStream_t stream = 0) -{ - typedef typename raft::KeyValuePair KVP; - std::shared_ptr exp_h(new KVP[size]); - std::shared_ptr act_h(new KVP[size]); - raft::update_host(exp_h.get(), expected, size, stream); - raft::update_host(act_h.get(), actual, size, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - for (size_t i(0); i < size; ++i) { - auto exp = exp_h.get()[i]; - auto act = act_h.get()[i]; - if (!eq_compare(exp, act)) { - return ::testing::AssertionFailure() - << "actual=" << act.key << "," << act.value << " != expected=" << exp.key << "," - << exp.value << " @" << i; - } - } - return ::testing::AssertionSuccess(); -} - -const std::vector> inputsf = { - {0.001f, 32, 32, 32, 1234ULL}, - {0.001f, 32, 64, 32, 1234ULL}, - {0.001f, 64, 32, 32, 1234ULL}, - {0.001f, 64, 64, 32, 1234ULL}, - {0.001f, 128, 32, 32, 1234ULL}, - {0.001f, 128, 64, 32, 1234ULL}, - {0.001f, 128, 128, 64, 1234ULL}, - {0.001f, 64, 128, 128, 1234ULL}, - - {0.001f, 32, 32, 34, 1234ULL}, - {0.001f, 32, 64, 34, 1234ULL}, - {0.001f, 64, 32, 34, 1234ULL}, - {0.001f, 64, 64, 34, 1234ULL}, - {0.001f, 128, 32, 34, 1234ULL}, - {0.001f, 128, 64, 34, 1234ULL}, - {0.001f, 128, 128, 66, 1234ULL}, - {0.001f, 64, 128, 130, 1234ULL}, - - {0.001f, 32, 32, 33, 1234ULL}, - {0.001f, 32, 64, 33, 1234ULL}, - {0.001f, 64, 32, 33, 1234ULL}, - {0.001f, 64, 64, 33, 1234ULL}, - {0.001f, 128, 32, 33, 1234ULL}, - {0.001f, 128, 64, 33, 1234ULL}, - {0.001f, 128, 128, 65, 1234ULL}, - {0.001f, 64, 128, 129, 1234ULL}, - {0.006f, 1805, 134, 2, 1234ULL}, - {0.006f, 8192, 1024, 64, 1234ULL}, - {0.006f, 8192, 1025, 64, 1234ULL}, - - // Repeat with smaller values of k - {0.006f, 32, 32, 1, 1234ULL}, - {0.001f, 32, 64, 2, 1234ULL}, - {0.001f, 64, 32, 3, 1234ULL}, - {0.001f, 64, 64, 4, 1234ULL}, - {0.001f, 128, 32, 5, 1234ULL}, - {0.001f, 128, 64, 6, 1234ULL}, - {0.001f, 128, 128, 7, 1234ULL}, - {0.001f, 64, 128, 8, 1234ULL}, - - {0.001f, 32, 32, 9, 1234ULL}, - {0.001f, 32, 64, 10, 1234ULL}, - {0.001f, 64, 32, 11, 1234ULL}, - {0.001f, 64, 64, 12, 1234ULL}, - {0.001f, 128, 32, 13, 1234ULL}, - {0.001f, 128, 64, 14, 1234ULL}, - {0.001f, 128, 128, 15, 1234ULL}, - {0.001f, 64, 128, 16, 1234ULL}, - - {0.001f, 32, 32, 17, 1234ULL}, - {0.001f, 32, 64, 18, 1234ULL}, - {0.001f, 64, 32, 19, 1234ULL}, - {0.001f, 64, 64, 20, 1234ULL}, - {0.001f, 128, 32, 21, 1234ULL}, - {0.001f, 128, 64, 22, 1234ULL}, - {0.001f, 128, 128, 23, 1234ULL}, - {0.00001, 64, 128, 24, 1234ULL}, - {0.001f, 1805, 134, 25, 1234ULL}, - {0.006f, 8192, 1024, 25, 1234ULL}, - {0.006f, 8192, 1024, 66, 1234ULL}, -}; -typedef FusedCosineNNTest FusedCosineNNTestF; -TEST_P(FusedCosineNNTestF, Result) -{ - runTest(min.data()); - ASSERT_TRUE(devArrMatch( - min_ref.data(), min.data(), params.m, CompareApproxAbsKVP(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(FusedCosineNNTests, FusedCosineNNTestF, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.00001, 32, 32, 32, 1234ULL}, {0.00001, 32, 64, 32, 1234ULL}, - {0.00001, 64, 32, 32, 1234ULL}, {0.00001, 64, 64, 32, 1234ULL}, - {0.00001, 128, 32, 32, 1234ULL}, {0.00001, 128, 64, 32, 1234ULL}, - {0.00001, 128, 128, 64, 1234ULL}, {0.00001, 64, 128, 128, 1234ULL}, - - {0.00001, 32, 32, 34, 1234ULL}, {0.00001, 32, 64, 34, 1234ULL}, - {0.00001, 64, 32, 34, 1234ULL}, {0.00001, 64, 64, 34, 1234ULL}, - {0.00001, 128, 32, 34, 1234ULL}, {0.00001, 128, 64, 34, 1234ULL}, - {0.00001, 128, 128, 66, 1234ULL}, {0.00001, 64, 128, 130, 1234ULL}, - - {0.00001, 32, 32, 33, 1234ULL}, {0.00001, 32, 64, 33, 1234ULL}, - {0.00001, 64, 32, 33, 1234ULL}, {0.00001, 64, 64, 33, 1234ULL}, - {0.00001, 128, 32, 33, 1234ULL}, {0.00001, 128, 64, 33, 1234ULL}, - {0.00001, 128, 128, 65, 1234ULL}, {0.00001, 64, 128, 129, 1234ULL}, - - {0.00001, 1805, 134, 2, 1234ULL}, {0.00001, 8192, 1024, 25, 1234ULL}, -}; -typedef FusedCosineNNTest FusedCosineNNTestD; -TEST_P(FusedCosineNNTestD, Result) -{ - runTest(min.data()); - ASSERT_TRUE(devArrMatch( - min_ref.data(), min.data(), params.m, CompareApproxAbsKVP(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(FusedCosineNNTests, FusedCosineNNTestD, ::testing::ValuesIn(inputsd)); - -/// This is to test output determinism of the prim -template -class FusedCosineNNDetTest : public FusedCosineNNTest { - public: - FusedCosineNNDetTest() : stream(resource::get_cuda_stream(handle)), min1(0, stream) {} - - void SetUp() override - { - FusedCosineNNTest::SetUp(); - int m = this->params.m; - min1.resize(m, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - } - - void TearDown() override { FusedCosineNNTest::TearDown(); } - - protected: - raft::resources handle; - cudaStream_t stream; - - rmm::device_uvector> min1; - - static const int NumRepeats = 3; - - void generateGoldenResult() override {} -}; - -typedef FusedCosineNNDetTest FusedCosineNNDetTestF; -TEST_P(FusedCosineNNDetTestF, Result) -{ - runTest(min.data()); // assumed to be golden - for (int i = 0; i < NumRepeats; ++i) { - runTest(min1.data()); - ASSERT_TRUE(devArrMatch(min.data(), min1.data(), params.m, CompareExactKVP(), stream)); - cudaMemsetAsync(min1.data(), 0, sizeof(*min.data()) * params.m, stream); - } -} -INSTANTIATE_TEST_CASE_P(FusedCosineNNDetTests, FusedCosineNNDetTestF, ::testing::ValuesIn(inputsf)); - -typedef FusedCosineNNDetTest FusedCosineNNDetTestD; -TEST_P(FusedCosineNNDetTestD, Result) -{ - runTest(min.data()); // assumed to be golden - for (int i = 0; i < NumRepeats; ++i) { - runTest(min1.data()); - ASSERT_TRUE(devArrMatch(min.data(), min1.data(), params.m, CompareExactKVP(), stream)); - } -} -INSTANTIATE_TEST_CASE_P(FusedCosineNNDetTests, FusedCosineNNDetTestD, ::testing::ValuesIn(inputsd)); - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/fused_l2_nn.cu b/cpp/test/distance/fused_l2_nn.cu deleted file mode 100644 index 6fd8f15808..0000000000 --- a/cpp/test/distance/fused_l2_nn.cu +++ /dev/null @@ -1,437 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace raft { -namespace distance { - -template -struct RaftKVPMinReduce { - typedef raft::KeyValuePair KVP; - - DI KVP operator()(LabelT rit, const KVP& a, const KVP& b) { return b.value < a.value ? b : a; } - - DI KVP operator()(const KVP& a, const KVP& b) { return b.value < a.value ? b : a; } - -}; // KVPMinReduce - -template -RAFT_KERNEL naiveKernel(raft::KeyValuePair* min, - DataT* x, - DataT* y, - int m, - int n, - int k, - int* workspace, - DataT maxVal) -{ - int midx = threadIdx.y + blockIdx.y * blockDim.y; - int nidx = threadIdx.x + blockIdx.x * blockDim.x; - DataT acc = DataT(0); - for (int i = 0; i < k; ++i) { - int xidx = i + midx * k; - int yidx = i + nidx * k; - auto diff = midx >= m || nidx >= n ? DataT(0) : x[xidx] - y[yidx]; - acc += diff * diff; - } - - if (Sqrt) { acc = raft::sqrt(acc); } - ReduceOpT redOp; - typedef cub::WarpReduce> WarpReduce; - __shared__ typename WarpReduce::TempStorage temp[NWARPS]; - int warpId = threadIdx.x / raft::WarpSize; - raft::KeyValuePair tmp; - tmp.key = nidx; - tmp.value = midx >= m || nidx >= n ? maxVal : acc; - tmp = WarpReduce(temp[warpId]).Reduce(tmp, RaftKVPMinReduce()); - if (threadIdx.x % raft::WarpSize == 0 && midx < m) { - while (atomicCAS(workspace + midx, 0, 1) == 1) - ; - __threadfence(); - redOp(midx, min + midx, tmp); - __threadfence(); - atomicCAS(workspace + midx, 1, 0); - } -} - -template -void naive(raft::KeyValuePair* min, - DataT* x, - DataT* y, - int m, - int n, - int k, - int* workspace, - cudaStream_t stream) -{ - static const dim3 TPB(32, 16, 1); - dim3 nblks(raft::ceildiv(n, (int)TPB.x), raft::ceildiv(m, (int)TPB.y), 1); - RAFT_CUDA_TRY(cudaMemsetAsync(workspace, 0, sizeof(int) * m, stream)); - auto blks = raft::ceildiv(m, 256); - MinAndDistanceReduceOp op; - detail::initKernel, int> - <<>>(min, m, std::numeric_limits::max(), op); - RAFT_CUDA_TRY(cudaGetLastError()); - naiveKernel, 16> - <<>>(min, x, y, m, n, k, workspace, std::numeric_limits::max()); - RAFT_CUDA_TRY(cudaGetLastError()); -} - -template -struct Inputs { - DataT tolerance; - int m, n, k; - unsigned long long int seed; - - friend std::ostream& operator<<(std::ostream& os, const Inputs& p) - { - return os << "m: " << p.m - << ", " - "n: " - << p.n - << ", " - "k: " - << p.k - << ", " - "seed: " - << p.seed - << ", " - "tol: " - << p.tolerance; - } -}; - -template -class FusedL2NNTest : public ::testing::TestWithParam> { - public: - FusedL2NNTest() - : params(::testing::TestWithParam>::GetParam()), - stream(resource::get_cuda_stream(handle)), - x(params.m * params.k, stream), - y(params.n * params.k, stream), - xn(params.m, stream), - yn(params.n, stream), - min(params.m, stream), - min_ref(params.m, stream), - workspace(params.m * sizeof(int), stream) - { - } - - protected: - void SetUp() override - { - raft::random::RngState r(params.seed); - int m = params.m; - int n = params.n; - int k = params.k; - uniform(handle, r, x.data(), m * k, DataT(-1.0), DataT(1.0)); - uniform(handle, r, y.data(), n * k, DataT(-1.0), DataT(1.0)); - generateGoldenResult(); - raft::linalg::rowNorm(xn.data(), x.data(), k, m, raft::linalg::L2Norm, true, stream); - raft::linalg::rowNorm(yn.data(), y.data(), k, n, raft::linalg::L2Norm, true, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - } - - protected: - raft::resources handle; - cudaStream_t stream; - Inputs params; - rmm::device_uvector x; - rmm::device_uvector y; - rmm::device_uvector xn; - rmm::device_uvector yn; - rmm::device_uvector> min; - rmm::device_uvector> min_ref; - rmm::device_uvector workspace; - - virtual void generateGoldenResult() - { - int m = params.m; - int n = params.n; - int k = params.k; - naive(min_ref.data(), x.data(), y.data(), m, n, k, (int*)workspace.data(), stream); - } - - void runTest(raft::KeyValuePair* out) - { - int m = params.m; - int n = params.n; - int k = params.k; - - const bool init_out_buffer = true; - fusedL2NNMinReduce, int>(out, - x.data(), - y.data(), - xn.data(), - yn.data(), - m, - n, - k, - (void*)workspace.data(), - Sqrt, - init_out_buffer, - stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - } -}; - -template -struct CompareApproxAbsKVP { - typedef typename raft::KeyValuePair KVP; - CompareApproxAbsKVP(T eps_) : eps(eps_) {} - bool operator()(const KVP& a, const KVP& b) const - { - T diff = std::abs(std::abs(a.value) - std::abs(b.value)); - T m = std::max(std::abs(a.value), std::abs(b.value)); - T ratio = m >= eps ? diff / m : diff; - return (ratio <= eps); - } - - private: - T eps; -}; - -template -struct CompareExactKVP { - typedef typename raft::KeyValuePair KVP; - bool operator()(const KVP& a, const KVP& b) const - { - if (a.value != b.value) return false; - return true; - } -}; - -template -::testing::AssertionResult devArrMatch(const raft::KeyValuePair* expected, - const raft::KeyValuePair* actual, - size_t size, - L eq_compare, - cudaStream_t stream = 0) -{ - typedef typename raft::KeyValuePair KVP; - std::shared_ptr exp_h(new KVP[size]); - std::shared_ptr act_h(new KVP[size]); - raft::update_host(exp_h.get(), expected, size, stream); - raft::update_host(act_h.get(), actual, size, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - for (size_t i(0); i < size; ++i) { - auto exp = exp_h.get()[i]; - auto act = act_h.get()[i]; - if (!eq_compare(exp, act)) { - return ::testing::AssertionFailure() - << "actual=" << act.key << "," << act.value << " != expected=" << exp.key << "," - << exp.value << " @" << i; - } - } - return ::testing::AssertionSuccess(); -} - -const std::vector> inputsf = { - {0.001f, 32, 32, 32, 1234ULL}, - {0.001f, 32, 64, 32, 1234ULL}, - {0.001f, 64, 32, 32, 1234ULL}, - {0.001f, 64, 64, 32, 1234ULL}, - {0.001f, 128, 32, 32, 1234ULL}, - {0.001f, 128, 64, 32, 1234ULL}, - {0.001f, 128, 128, 64, 1234ULL}, - {0.001f, 64, 128, 128, 1234ULL}, - - {0.001f, 32, 32, 34, 1234ULL}, - {0.001f, 32, 64, 34, 1234ULL}, - {0.001f, 64, 32, 34, 1234ULL}, - {0.001f, 64, 64, 34, 1234ULL}, - {0.001f, 128, 32, 34, 1234ULL}, - {0.001f, 128, 64, 34, 1234ULL}, - {0.001f, 128, 128, 66, 1234ULL}, - {0.001f, 64, 128, 130, 1234ULL}, - - {0.001f, 32, 32, 33, 1234ULL}, - {0.001f, 32, 64, 33, 1234ULL}, - {0.001f, 64, 32, 33, 1234ULL}, - {0.001f, 64, 64, 33, 1234ULL}, - {0.001f, 128, 32, 33, 1234ULL}, - {0.001f, 128, 64, 33, 1234ULL}, - {0.001f, 128, 128, 65, 1234ULL}, - {0.001f, 64, 128, 129, 1234ULL}, - {0.006f, 1805, 134, 2, 1234ULL}, - {0.006f, 8192, 1024, 64, 1234ULL}, - {0.006f, 8192, 1025, 64, 1234ULL}, - - // Repeat with smaller values of k - {0.006f, 32, 32, 1, 1234ULL}, - {0.001f, 32, 64, 2, 1234ULL}, - {0.001f, 64, 32, 3, 1234ULL}, - {0.001f, 64, 64, 4, 1234ULL}, - {0.001f, 128, 32, 5, 1234ULL}, - {0.001f, 128, 64, 6, 1234ULL}, - {0.001f, 128, 128, 7, 1234ULL}, - {0.001f, 64, 128, 8, 1234ULL}, - - {0.001f, 32, 32, 9, 1234ULL}, - {0.001f, 32, 64, 10, 1234ULL}, - {0.001f, 64, 32, 11, 1234ULL}, - {0.001f, 64, 64, 12, 1234ULL}, - {0.001f, 128, 32, 13, 1234ULL}, - {0.001f, 128, 64, 14, 1234ULL}, - {0.001f, 128, 128, 15, 1234ULL}, - {0.001f, 64, 128, 16, 1234ULL}, - - {0.001f, 32, 32, 17, 1234ULL}, - {0.001f, 32, 64, 18, 1234ULL}, - {0.001f, 64, 32, 19, 1234ULL}, - {0.001f, 64, 64, 20, 1234ULL}, - {0.001f, 128, 32, 21, 1234ULL}, - {0.001f, 128, 64, 22, 1234ULL}, - {0.001f, 128, 128, 23, 1234ULL}, - {0.00001, 64, 128, 24, 1234ULL}, - {0.001f, 1805, 134, 25, 1234ULL}, - {0.006f, 8192, 1024, 25, 1234ULL}, - {0.006f, 8192, 1024, 66, 1234ULL}, -}; -typedef FusedL2NNTest FusedL2NNTestF_Sq; -TEST_P(FusedL2NNTestF_Sq, Result) -{ - runTest(min.data()); - ASSERT_TRUE(devArrMatch( - min_ref.data(), min.data(), params.m, CompareApproxAbsKVP(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(FusedL2NNTests, FusedL2NNTestF_Sq, ::testing::ValuesIn(inputsf)); -typedef FusedL2NNTest FusedL2NNTestF_Sqrt; -TEST_P(FusedL2NNTestF_Sqrt, Result) -{ - runTest(min.data()); - ASSERT_TRUE(devArrMatch( - min_ref.data(), min.data(), params.m, CompareApproxAbsKVP(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(FusedL2NNTests, FusedL2NNTestF_Sqrt, ::testing::ValuesIn(inputsf)); - -const std::vector> inputsd = { - {0.00001, 32, 32, 32, 1234ULL}, {0.00001, 32, 64, 32, 1234ULL}, - {0.00001, 64, 32, 32, 1234ULL}, {0.00001, 64, 64, 32, 1234ULL}, - {0.00001, 128, 32, 32, 1234ULL}, {0.00001, 128, 64, 32, 1234ULL}, - {0.00001, 128, 128, 64, 1234ULL}, {0.00001, 64, 128, 128, 1234ULL}, - - {0.00001, 32, 32, 34, 1234ULL}, {0.00001, 32, 64, 34, 1234ULL}, - {0.00001, 64, 32, 34, 1234ULL}, {0.00001, 64, 64, 34, 1234ULL}, - {0.00001, 128, 32, 34, 1234ULL}, {0.00001, 128, 64, 34, 1234ULL}, - {0.00001, 128, 128, 66, 1234ULL}, {0.00001, 64, 128, 130, 1234ULL}, - - {0.00001, 32, 32, 33, 1234ULL}, {0.00001, 32, 64, 33, 1234ULL}, - {0.00001, 64, 32, 33, 1234ULL}, {0.00001, 64, 64, 33, 1234ULL}, - {0.00001, 128, 32, 33, 1234ULL}, {0.00001, 128, 64, 33, 1234ULL}, - {0.00001, 128, 128, 65, 1234ULL}, {0.00001, 64, 128, 129, 1234ULL}, - - {0.00001, 1805, 134, 2, 1234ULL}, //{0.00001, 8192, 1024, 25, 1234ULL}, -}; -typedef FusedL2NNTest FusedL2NNTestD_Sq; -TEST_P(FusedL2NNTestD_Sq, Result) -{ - runTest(min.data()); - ASSERT_TRUE(devArrMatch( - min_ref.data(), min.data(), params.m, CompareApproxAbsKVP(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(FusedL2NNTests, FusedL2NNTestD_Sq, ::testing::ValuesIn(inputsd)); -typedef FusedL2NNTest FusedL2NNTestD_Sqrt; -TEST_P(FusedL2NNTestD_Sqrt, Result) -{ - runTest(min.data()); - ASSERT_TRUE(devArrMatch( - min_ref.data(), min.data(), params.m, CompareApproxAbsKVP(params.tolerance), stream)); -} -INSTANTIATE_TEST_CASE_P(FusedL2NNTests, FusedL2NNTestD_Sqrt, ::testing::ValuesIn(inputsd)); - -/// This is to test output determinism of the prim -template -class FusedL2NNDetTest : public FusedL2NNTest { - public: - FusedL2NNDetTest() : stream(resource::get_cuda_stream(handle)), min1(0, stream) {} - - void SetUp() override - { - FusedL2NNTest::SetUp(); - int m = this->params.m; - min1.resize(m, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - } - - void TearDown() override { FusedL2NNTest::TearDown(); } - - protected: - raft::resources handle; - cudaStream_t stream; - - rmm::device_uvector> min1; - - static const int NumRepeats = 3; - - void generateGoldenResult() override {} -}; - -typedef FusedL2NNDetTest FusedL2NNDetTestF_Sq; -TEST_P(FusedL2NNDetTestF_Sq, Result) -{ - runTest(min.data()); // assumed to be golden - for (int i = 0; i < NumRepeats; ++i) { - runTest(min1.data()); - ASSERT_TRUE(devArrMatch(min.data(), min1.data(), params.m, CompareExactKVP(), stream)); - } -} -INSTANTIATE_TEST_CASE_P(FusedL2NNDetTests, FusedL2NNDetTestF_Sq, ::testing::ValuesIn(inputsf)); -typedef FusedL2NNDetTest FusedL2NNDetTestF_Sqrt; -TEST_P(FusedL2NNDetTestF_Sqrt, Result) -{ - runTest(min.data()); // assumed to be golden - for (int i = 0; i < NumRepeats; ++i) { - runTest(min1.data()); - ASSERT_TRUE(devArrMatch(min.data(), min1.data(), params.m, CompareExactKVP(), stream)); - } -} -INSTANTIATE_TEST_CASE_P(FusedL2NNDetTests, FusedL2NNDetTestF_Sqrt, ::testing::ValuesIn(inputsf)); - -typedef FusedL2NNDetTest FusedL2NNDetTestD_Sq; -TEST_P(FusedL2NNDetTestD_Sq, Result) -{ - runTest(min.data()); // assumed to be golden - for (int i = 0; i < NumRepeats; ++i) { - runTest(min1.data()); - ASSERT_TRUE(devArrMatch(min.data(), min1.data(), params.m, CompareExactKVP(), stream)); - } -} -INSTANTIATE_TEST_CASE_P(FusedL2NNDetTests, FusedL2NNDetTestD_Sq, ::testing::ValuesIn(inputsd)); -typedef FusedL2NNDetTest FusedL2NNDetTestD_Sqrt; -TEST_P(FusedL2NNDetTestD_Sqrt, Result) -{ - runTest(min.data()); // assumed to be golden - for (int i = 0; i < NumRepeats; ++i) { - runTest(min1.data()); - ASSERT_TRUE(devArrMatch(min.data(), min1.data(), params.m, CompareExactKVP(), stream)); - } -} -INSTANTIATE_TEST_CASE_P(FusedL2NNDetTests, FusedL2NNDetTestD_Sqrt, ::testing::ValuesIn(inputsd)); - -} // end namespace distance -} // end namespace raft diff --git a/cpp/test/distance/gram.cu b/cpp/test/distance/gram.cu deleted file mode 100644 index e911a25ff1..0000000000 --- a/cpp/test/distance/gram.cu +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright (c) 2019-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "gram_base.cuh" - -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include - -namespace raft::distance::kernels { - -struct GramMatrixInputs { - int n1; // feature vectors in matrix 1 - int n2; // featuer vectors in matrix 2 - int n_cols; // number of elements in a feature vector - bool is_row_major; - KernelParams kernel; - int ld1; - int ld2; - int ld_out; - // We will generate random input using the dimensions given here. - // The reference output is calculated by a custom kernel. -}; - -std::ostream& operator<<(std::ostream& os, const GramMatrixInputs& p) -{ - std::vector kernel_names{"linear", "poly", "rbf", "tanh"}; - os << "/" << p.n1 << "x" << p.n2 << "x" << p.n_cols << "/" - << (p.is_row_major ? "RowMajor/" : "ColMajor/") << kernel_names[p.kernel.kernel] << "/ld_" - << p.ld1 << "x" << p.ld2 << "x" << p.ld_out; - return os; -} - -const std::vector inputs = { - {42, 137, 2, false, {KernelType::LINEAR}}, - {42, 137, 2, true, {KernelType::LINEAR}}, - {42, 137, 2, false, {KernelType::LINEAR}, 64, 179, 181}, - {42, 137, 2, true, {KernelType::LINEAR}, 64, 179, 181}, - {137, 42, 2, false, {KernelType::POLYNOMIAL, 2, 0.5, 2.4}}, - {137, 42, 2, true, {KernelType::POLYNOMIAL, 2, 0.5, 2.4}}, - {137, 42, 2, false, {KernelType::POLYNOMIAL, 2, 0.5, 2.4}, 159, 73, 144}, - {137, 42, 2, true, {KernelType::POLYNOMIAL, 2, 0.5, 2.4}, 159, 73, 144}, - {42, 137, 2, false, {KernelType::TANH, 0, 0.5, 2.4}}, - {42, 137, 2, true, {KernelType::TANH, 0, 0.5, 2.4}}, - {42, 137, 2, false, {KernelType::TANH, 0, 0.5, 2.4}, 64, 155, 49}, - {42, 137, 2, true, {KernelType::TANH, 0, 0.5, 2.4}, 64, 155, 143}, - {3, 4, 2, false, {KernelType::RBF, 0, 0.5}}, - {42, 137, 2, false, {KernelType::RBF, 0, 0.5}}, - {42, 137, 2, true, {KernelType::RBF, 0, 0.5}}, - // Distance kernel does not support LD parameter yet. - //{42, 137, 2, false, {KernelType::RBF, 0, 0.5}, 64, 155, 49}, - // {42, 137, 2, true, {KernelType::RBF, 0, 0.5}, 64, 155, 143}, -}; - -template -class GramMatrixTest : public ::testing::TestWithParam { - protected: - GramMatrixTest() - : params(GetParam()), - handle(), - x1(0, resource::get_cuda_stream(handle)), - x2(0, resource::get_cuda_stream(handle)), - gram(0, resource::get_cuda_stream(handle)), - gram_host(0) - { - auto stream = resource::get_cuda_stream(handle); - - if (params.ld1 == 0) { params.ld1 = params.is_row_major ? params.n_cols : params.n1; } - if (params.ld2 == 0) { params.ld2 = params.is_row_major ? params.n_cols : params.n2; } - if (params.ld_out == 0) { params.ld_out = params.is_row_major ? params.n2 : params.n1; } - // Derive the size of the output from the offset of the last element. - size_t size = get_offset(params.n1 - 1, params.n_cols - 1, params.ld1, params.is_row_major) + 1; - x1.resize(size, stream); - size = get_offset(params.n2 - 1, params.n_cols - 1, params.ld2, params.is_row_major) + 1; - x2.resize(size, stream); - size = get_offset(params.n1 - 1, params.n2 - 1, params.ld_out, params.is_row_major) + 1; - - gram.resize(size, stream); - RAFT_CUDA_TRY(cudaMemsetAsync(gram.data(), 0, gram.size() * sizeof(math_t), stream)); - gram_host.resize(gram.size()); - std::fill(gram_host.begin(), gram_host.end(), 0); - - raft::random::RngState rng(42137ULL); - raft::random::uniform(handle, rng, x1.data(), x1.size(), math_t(0), math_t(1)); - raft::random::uniform(handle, rng, x2.data(), x2.size(), math_t(0), math_t(1)); - } - - ~GramMatrixTest() override {} - - void runTest() - { - std::unique_ptr> kernel = - std::unique_ptr>(KernelFactory::create(params.kernel)); - - auto x1_span = - params.is_row_major - ? raft::make_device_strided_matrix_view( - x1.data(), params.n1, params.n_cols, params.ld1) - : raft::make_device_strided_matrix_view( - x1.data(), params.n1, params.n_cols, params.ld1); - auto x2_span = - params.is_row_major - ? raft::make_device_strided_matrix_view( - x2.data(), params.n2, params.n_cols, params.ld2) - : raft::make_device_strided_matrix_view( - x2.data(), params.n2, params.n_cols, params.ld2); - auto out_span = - params.is_row_major - ? raft::make_device_strided_matrix_view( - gram.data(), params.n1, params.n2, params.ld_out) - : raft::make_device_strided_matrix_view( - gram.data(), params.n1, params.n2, params.ld_out); - - (*kernel)(handle, x1_span, x2_span, out_span); - - auto stream = resource::get_cuda_stream(handle); - naiveGramMatrixKernel(params.n1, - params.n2, - params.n_cols, - x1, - x2, - gram_host.data(), - params.ld1, - params.ld2, - params.ld_out, - params.is_row_major, - params.kernel, - stream, - handle); - - ASSERT_TRUE(raft::devArrMatchHost( - gram_host.data(), gram.data(), gram.size(), raft::CompareApprox(1e-6f), stream)); - } - - GramMatrixInputs params; - raft::resources handle; - - rmm::device_uvector x1; - rmm::device_uvector x2; - rmm::device_uvector gram; - - std::vector gram_host; -}; - -typedef GramMatrixTest GramMatrixTestFloat; -typedef GramMatrixTest GramMatrixTestDouble; - -TEST_P(GramMatrixTestFloat, Gram) { runTest(); } - -INSTANTIATE_TEST_SUITE_P(GramMatrixTests, GramMatrixTestFloat, ::testing::ValuesIn(inputs)); -}; // end namespace raft::distance::kernels diff --git a/cpp/test/distance/gram_base.cuh b/cpp/test/distance/gram_base.cuh deleted file mode 100644 index 170bcb76c1..0000000000 --- a/cpp/test/distance/gram_base.cuh +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include - -#include -#include - -namespace raft { -namespace distance { -namespace kernels { - -// Get the offset of element [i,k]. -HDI int get_offset(int i, int k, int ld, bool is_row_major) -{ - return is_row_major ? i * ld + k : i + k * ld; -} - -// Calculate the Gram matrix on the host. -template -void naiveGramMatrixKernel(int n1, - int n2, - int n_cols, - const rmm::device_uvector& x1, - const rmm::device_uvector& x2, - math_t* gram_host, - int ld1, - int ld2, - int ld_out, - bool is_row_major, - KernelParams kernel, - cudaStream_t stream, - const raft::resources& handle) -{ - std::vector x1_host(x1.size()); - raft::update_host(x1_host.data(), x1.data(), x1.size(), stream); - std::vector x2_host(x2.size()); - raft::update_host(x2_host.data(), x2.data(), x2.size(), stream); - resource::sync_stream(handle, stream); - - for (int i = 0; i < n1; i++) { - for (int j = 0; j < n2; j++) { - float d = 0; - for (int k = 0; k < n_cols; k++) { - if (kernel.kernel == KernelType::RBF) { - math_t diff = x1_host[get_offset(i, k, ld1, is_row_major)] - - x2_host[get_offset(j, k, ld2, is_row_major)]; - d += diff * diff; - } else { - d += x1_host[get_offset(i, k, ld1, is_row_major)] * - x2_host[get_offset(j, k, ld2, is_row_major)]; - } - } - int idx = get_offset(i, j, ld_out, is_row_major); - math_t v = 0; - switch (kernel.kernel) { - case (KernelType::LINEAR): gram_host[idx] = d; break; - case (KernelType::POLYNOMIAL): - v = kernel.gamma * d + kernel.coef0; - gram_host[idx] = std::pow(v, kernel.degree); - break; - case (KernelType::TANH): gram_host[idx] = std::tanh(kernel.gamma * d + kernel.coef0); break; - case (KernelType::RBF): gram_host[idx] = exp(-kernel.gamma * d); break; - } - } - } -} - -} // namespace kernels -} // namespace distance -} // namespace raft diff --git a/cpp/test/distance/masked_nn.cu b/cpp/test/distance/masked_nn.cu deleted file mode 100644 index 34fa07c45b..0000000000 --- a/cpp/test/distance/masked_nn.cu +++ /dev/null @@ -1,438 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace raft::distance::masked_nn { - -// The adjacency pattern determines what distances get computed. -enum AdjacencyPattern { - checkerboard = 0, // adjacency matrix looks like a checkerboard (half the distances are computed) - checkerboard_4 = 1, // checkerboard with tiles of size 4x4 - checkerboard_64 = 2, // checkerboard with tiles of size 64x64 - all_true = 3, // no distance computations can be skipped - all_false = 4 // all distance computations can be skipped -}; - -// Kernels: -// - init_adj: to initialize the adjacency kernel with a specific adjacency pattern -// - referenceKernel: to produce the ground-truth output - -RAFT_KERNEL init_adj(AdjacencyPattern pattern, - int n, - raft::device_matrix_view adj, - raft::device_vector_view group_idxs) -{ - int m = adj.extent(0); - int num_groups = adj.extent(1); - - for (int idx_m = blockIdx.y * blockDim.y + threadIdx.y; idx_m < m; - idx_m += blockDim.y * gridDim.y) { - for (int idx_g = blockIdx.x * blockDim.x + threadIdx.x; idx_g < num_groups; - idx_g += blockDim.x * gridDim.x) { - switch (pattern) { - case checkerboard: adj(idx_m, idx_g) = (idx_m + idx_g) % 2; break; - case checkerboard_4: adj(idx_m, idx_g) = (idx_m / 4 + idx_g) % 2; break; - case checkerboard_64: adj(idx_m, idx_g) = (idx_m / 64 + idx_g) % 2; break; - case all_true: adj(idx_m, idx_g) = true; break; - case all_false: adj(idx_m, idx_g) = false; break; - default: assert(false && "unknown pattern"); - } - } - } - // Each group is of size n / num_groups. - // - // - group_idxs[j] indicates the start of group j + 1 (i.e. is the inclusive - // scan of the group lengths) - // - // - The first group always starts at index zero, so we do not store it. - // - // - The group_idxs[num_groups - 1] should always equal n. - - if (blockIdx.y == 0 && threadIdx.y == 0) { - const int g_stride = blockDim.x * gridDim.x; - for (int idx_g = blockIdx.x * blockDim.x + threadIdx.x; idx_g < num_groups; idx_g += g_stride) { - group_idxs(idx_g) = (idx_g + 1) * (n / num_groups); - } - group_idxs(num_groups - 1) = n; - } -} - -template -__launch_bounds__(32 * NWARPS, 2) RAFT_KERNEL referenceKernel(raft::KeyValuePair* min, - DataT* x, - DataT* y, - bool* adj, - int* group_idxs, - int m, - int n, - int k, - int num_groups, - bool sqrt, - int* workspace, - DataT maxVal) -{ - const int m_stride = blockDim.y * gridDim.y; - const int m_offset = threadIdx.y + blockIdx.y * blockDim.y; - const int n_stride = blockDim.x * gridDim.x; - const int n_offset = threadIdx.x + blockIdx.x * blockDim.x; - - for (int m_grid = 0; m_grid < m; m_grid += m_stride) { - for (int n_grid = 0; n_grid < n; n_grid += n_stride) { - int midx = m_grid + m_offset; - int nidx = n_grid + n_offset; - - // Do a reverse linear search to determine the group index. - int group_idx = 0; - for (int i = num_groups; 0 <= i; --i) { - if (nidx < group_idxs[i]) { group_idx = i; } - } - const bool include_dist = adj[midx * num_groups + group_idx] && midx < m && nidx < n; - - // Compute L2 metric. - DataT acc = DataT(0); - for (int i = 0; i < k; ++i) { - int xidx = i + midx * k; - int yidx = i + nidx * k; - auto diff = x[xidx] - y[yidx]; - acc += diff * diff; - } - if (sqrt) { acc = raft::sqrt(acc); } - ReduceOpT redOp; - typedef cub::WarpReduce> WarpReduce; - __shared__ typename WarpReduce::TempStorage temp[NWARPS]; - int warpId = threadIdx.x / raft::WarpSize; - raft::KeyValuePair tmp; - tmp.key = include_dist ? nidx : -1; - tmp.value = include_dist ? acc : maxVal; - tmp = WarpReduce(temp[warpId]).Reduce(tmp, raft::distance::KVPMinReduce{}); - if (threadIdx.x % raft::WarpSize == 0 && midx < m) { - while (atomicCAS(workspace + midx, 0, 1) == 1) - ; - __threadfence(); - redOp(midx, min + midx, tmp); - __threadfence(); - atomicCAS(workspace + midx, 1, 0); - } - __syncthreads(); - } - } -} - -// Structs -// - Params: holds parameters for test case -// - Inputs: holds the inputs to the functions under test (x, y, adj, group_idxs). Is generated from -// the inputs. -struct Params { - double tolerance; - int m, n, k, num_groups; - bool sqrt; - unsigned long long int seed; - AdjacencyPattern pattern; -}; - -inline auto operator<<(std::ostream& os, const Params& p) -> std::ostream& -{ - os << "m: " << p.m << ", n: " << p.n << ", k: " << p.k << ", num_groups: " << p.num_groups - << ", sqrt: " << p.sqrt << ", seed: " << p.seed << ", tol: " << p.tolerance; - return os; -} - -template -struct Inputs { - using IdxT = int; - - raft::device_matrix x, y; - raft::device_matrix adj; - raft::device_vector group_idxs; - - Inputs(const raft::handle_t& handle, const Params& p) - : x{raft::make_device_matrix(handle, p.m, p.k)}, - y{raft::make_device_matrix(handle, p.n, p.k)}, - adj{raft::make_device_matrix(handle, p.m, p.num_groups)}, - group_idxs{raft::make_device_vector(handle, p.num_groups)} - { - // Initialize x, y - raft::random::RngState r(p.seed); - uniform(handle, r, x.data_handle(), p.m * p.k, DataT(-1.0), DataT(1.0)); - uniform(handle, r, y.data_handle(), p.n * p.k, DataT(-1.0), DataT(1.0)); - - // Initialize adj, group_idxs. - dim3 block(32, 32); - dim3 grid(10, 10); - init_adj<<>>( - p.pattern, p.n, adj.view(), group_idxs.view()); - RAFT_CUDA_TRY(cudaGetLastError()); - } -}; - -template > -auto reference(const raft::handle_t& handle, Inputs inp, const Params& p) - -> raft::device_vector -{ - int m = inp.x.extent(0); - int n = inp.y.extent(0); - int k = inp.x.extent(1); - int num_groups = inp.group_idxs.extent(0); - - if (m == 0 || n == 0 || k == 0 || num_groups == 0) { - return raft::make_device_vector(handle, 0); - } - - // Initialize workspace - auto stream = resource::get_cuda_stream(handle); - rmm::device_uvector workspace(p.m * sizeof(int), stream); - RAFT_CUDA_TRY(cudaMemsetAsync(workspace.data(), 0, sizeof(int) * m, stream)); - - // Initialize output - auto out = raft::make_device_vector(handle, m); - auto blks = raft::ceildiv(m, 256); - MinAndDistanceReduceOp op; - raft::distance::detail::initKernel, int> - <<>>(out.data_handle(), m, std::numeric_limits::max(), op); - RAFT_CUDA_TRY(cudaGetLastError()); - - // Launch reference kernel - const int nwarps = 16; - static const dim3 TPB(32, nwarps, 1); - dim3 nblks(1, 200, 1); - referenceKernel - <<>>(out.data_handle(), - inp.x.data_handle(), - inp.y.data_handle(), - inp.adj.data_handle(), - inp.group_idxs.data_handle(), - m, - n, - k, - num_groups, - p.sqrt, - (int*)workspace.data(), - std::numeric_limits::max()); - RAFT_CUDA_TRY(cudaGetLastError()); - - return out; -} - -template > -auto run_masked_nn(const raft::handle_t& handle, Inputs inp, const Params& p) - -> raft::device_vector -{ - // Compute norms: - auto x_norm = raft::make_device_vector(handle, p.m); - auto y_norm = raft::make_device_vector(handle, p.n); - - raft::linalg::norm(handle, - std::as_const(inp.x).view(), - x_norm.view(), - raft::linalg::L2Norm, - raft::linalg::Apply::ALONG_ROWS); - raft::linalg::norm(handle, - std::as_const(inp.y).view(), - y_norm.view(), - raft::linalg::L2Norm, - raft::linalg::Apply::ALONG_ROWS); - - // Create parameters for masked_l2_nn - using IdxT = int; - using RedOpT = MinAndDistanceReduceOp; - using PairRedOpT = raft::distance::KVPMinReduce; - using ParamT = raft::distance::masked_l2_nn_params; - - bool init_out = true; - ParamT masked_l2_params{RedOpT{}, PairRedOpT{}, p.sqrt, init_out}; - - // Create output - auto out = raft::make_device_vector(handle, p.m); - - // Launch kernel - raft::distance::masked_l2_nn(handle, - masked_l2_params, - inp.x.view(), - inp.y.view(), - x_norm.view(), - y_norm.view(), - inp.adj.view(), - inp.group_idxs.view(), - out.view()); - - resource::sync_stream(handle); - - return out; -} - -template -struct CompareApproxAbsKVP { - typedef typename raft::KeyValuePair KVP; - CompareApproxAbsKVP(T eps_) : eps(eps_) {} - bool operator()(const KVP& a, const KVP& b) const - { - T diff = raft::abs(raft::abs(a.value) - raft::abs(b.value)); - T m = std::max(raft::abs(a.value), raft::abs(b.value)); - T ratio = m >= eps ? diff / m : diff; - return (ratio <= eps); - } - - private: - T eps; -}; - -template -::testing::AssertionResult devArrMatch(const raft::KeyValuePair* expected, - const raft::KeyValuePair* actual, - size_t size, - L eq_compare, - cudaStream_t stream = 0) -{ - typedef typename raft::KeyValuePair KVP; - std::shared_ptr exp_h(new KVP[size]); - std::shared_ptr act_h(new KVP[size]); - raft::update_host(exp_h.get(), expected, size, stream); - raft::update_host(act_h.get(), actual, size, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - for (size_t i(0); i < size; ++i) { - auto exp = exp_h.get()[i]; - auto act = act_h.get()[i]; - if (!eq_compare(exp, act)) { - return ::testing::AssertionFailure() - << "actual=" << act.key << "," << act.value << " != expected=" << exp.key << "," - << exp.value << " @" << i; - } - } - return ::testing::AssertionSuccess(); -} - -inline auto gen_params() -> std::vector -{ - // Regular powers of two - auto regular = raft::util::itertools::product({0.001f}, // tolerance - {32, 64, 512}, // m - {32, 64, 512}, // n - {8, 32}, // k - {2, 32}, // num_groups - {true, false}, // sqrt - {1234ULL}, // seed - {AdjacencyPattern::all_true, - AdjacencyPattern::checkerboard, - AdjacencyPattern::checkerboard_64, - AdjacencyPattern::all_false}); - - // Irregular sizes to check tiling and bounds checking - auto irregular = raft::util::itertools::product({0.001f}, // tolerance - {511, 512, 513}, // m - {127, 128, 129}, // n - {5}, // k - {3, 9}, // num_groups - {true, false}, // sqrt - {1234ULL}, // seed - {AdjacencyPattern::all_true, - AdjacencyPattern::checkerboard, - AdjacencyPattern::checkerboard_64}); - - regular.insert(regular.end(), irregular.begin(), irregular.end()); - - return regular; -} - -class MaskedL2NNTest : public ::testing::TestWithParam { - // Empty. -}; - -// -TEST_P(MaskedL2NNTest, ReferenceCheckFloat) -{ - using DataT = float; - - // Get parameters; create handle and input data. - Params p = GetParam(); - raft::handle_t handle{}; - Inputs inputs{handle, p}; - - // Calculate reference and test output - auto out_reference = reference(handle, inputs, p); - auto out_fast = run_masked_nn(handle, inputs, p); - - // Check for differences. - ASSERT_TRUE(devArrMatch(out_reference.data_handle(), - out_fast.data_handle(), - p.m, - CompareApproxAbsKVP(p.tolerance), - resource::get_cuda_stream(handle))); -} - -// This test checks whether running the masked_l2_nn twice returns the same -// output. -TEST_P(MaskedL2NNTest, DeterminismCheck) -{ - using DataT = float; - - // Get parameters; create handle and input data. - Params p = GetParam(); - raft::handle_t handle{}; - Inputs inputs{handle, p}; - - // Calculate reference and test output - auto out1 = run_masked_nn(handle, inputs, p); - auto out2 = run_masked_nn(handle, inputs, p); - - // Check for differences. - ASSERT_TRUE(devArrMatch(out1.data_handle(), - out2.data_handle(), - p.m, - CompareApproxAbsKVP(p.tolerance), - resource::get_cuda_stream(handle))); -} - -TEST_P(MaskedL2NNTest, ReferenceCheckDouble) -{ - using DataT = double; - - // Get parameters; create handle and input data. - Params p = GetParam(); - raft::handle_t handle{}; - Inputs inputs{handle, p}; - - // Calculate reference and test output - auto out_reference = reference(handle, inputs, p); - auto out_fast = run_masked_nn(handle, inputs, p); - - // Check for differences. - ASSERT_TRUE(devArrMatch(out_reference.data_handle(), - out_fast.data_handle(), - p.m, - CompareApproxAbsKVP(p.tolerance), - resource::get_cuda_stream(handle))); -} - -INSTANTIATE_TEST_CASE_P(MaskedL2NNTests, MaskedL2NNTest, ::testing::ValuesIn(gen_params())); - -} // end namespace raft::distance::masked_nn diff --git a/cpp/test/distance/masked_nn_compress_to_bits.cu b/cpp/test/distance/masked_nn_compress_to_bits.cu deleted file mode 100644 index 2512af5e4f..0000000000 --- a/cpp/test/distance/masked_nn_compress_to_bits.cu +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "../test_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -namespace raft::distance::masked_nn::compress_to_bits { - -/** - * @brief Transpose and decompress 2D bitfield to boolean matrix - * - * Inverse operation of compress_to_bits - * - * @tparam T - * - * @parameter[in] in An `m x n` bitfield matrix. Row major. - * @parameter in_rows The number of rows of `in`, i.e. `m`. - * @parameter in_cols The number of cols of `in`, i.e. `n`. - * - * @parameter[out] out An `(m * bits_per_elem) x n` boolean matrix. - */ -template ::value>> -RAFT_KERNEL decompress_bits_kernel(const T* in, int in_rows, int in_cols, bool* out) -{ - constexpr int bits_per_element = 8 * sizeof(T); - - const size_t i = threadIdx.y + blockIdx.y * blockDim.y; - const size_t j = threadIdx.x + blockIdx.x * blockDim.x; - - if (in_rows <= i || in_cols <= j) { return; } - - const size_t out_rows = in_rows * bits_per_element; - const size_t out_cols = in_cols; - const size_t out_i = i * bits_per_element; - const size_t out_j = j; - - if (out_rows <= out_i && out_cols <= out_j) { return; } - - T bitfield = in[i * in_cols + j]; - for (int bitpos = 0; bitpos < bits_per_element; ++bitpos) { - bool bit = ((T(1) << bitpos) & bitfield) != 0; - out[(out_i + bitpos) * out_cols + out_j] = bit; - } -} - -/** - * @brief Transpose and decompress 2D bitfield to boolean matrix - * - * Inverse operation of compress_to_bits - * - * @tparam T - * - * @parameter[in] in An `m x n` bitfield matrix. Row major. - * @parameter in_rows The number of rows of `in`, i.e. `m`. - * @parameter in_cols The number of cols of `in`, i.e. `n`. - * - * @parameter[out] out An `n x (m * bits_per_elem)` boolean matrix. - */ -template ::value>> -void decompress_bits(const raft::handle_t& handle, const T* in, int in_rows, int in_cols, bool* out) -{ - auto stream = resource::get_cuda_stream(handle); - dim3 grid(raft::ceildiv(in_cols, 32), raft::ceildiv(in_rows, 32)); - dim3 block(32, 32); - decompress_bits_kernel<<>>(in, in_rows, in_cols, out); - RAFT_CUDA_TRY(cudaGetLastError()); -} - -// Params holds parameters for test case -struct Params { - int m, n; -}; - -inline auto operator<<(std::ostream& os, const Params& p) -> std::ostream& -{ - return os << "m: " << p.m << ", n: " << p.n; -} - -// Check that the following holds -// -// decompress(compress(x)) == x -// -// for 2D boolean matrices x. -template -void check_invertible(const Params& p) -{ - using raft::distance::detail::compress_to_bits; - constexpr int bits_per_elem = sizeof(T) * 8; - - // Make m and n that are safe to ceildiv. - int m = raft::round_up_safe(p.m, bits_per_elem); - int n = p.n; - - // Generate random input - raft::handle_t handle{}; - raft::random::RngState r(1ULL); - auto in = raft::make_device_matrix(handle, m, n); - raft::random::bernoulli(handle, r, in.data_handle(), m * n, 0.5f); - - int tmp_m = raft::ceildiv(m, bits_per_elem); - int out_m = tmp_m * bits_per_elem; - - auto tmp = raft::make_device_matrix(handle, tmp_m, n); - auto out = raft::make_device_matrix(handle, out_m, n); - - resource::sync_stream(handle); - RAFT_CUDA_TRY(cudaGetLastError()); - - ASSERT_EQ(in.extent(0), out.extent(0)) << "M does not match"; - ASSERT_EQ(in.extent(1), out.extent(1)) << "N does not match"; - - compress_to_bits(handle, in.view(), tmp.view()); - resource::sync_stream(handle); - RAFT_CUDA_TRY(cudaGetLastError()); - - decompress_bits(handle, tmp.data_handle(), tmp.extent(0), tmp.extent(1), out.data_handle()); - resource::sync_stream(handle); - RAFT_CUDA_TRY(cudaGetLastError()); - - // Check for differences. - ASSERT_TRUE(raft::devArrMatch(in.data_handle(), - out.data_handle(), - in.extent(0) * in.extent(1), - raft::Compare(), - resource::get_cuda_stream(handle))); - resource::sync_stream(handle); - RAFT_CUDA_TRY(cudaGetLastError()); -} - -void check_all_true(const Params& p) -{ - using raft::distance::detail::compress_to_bits; - using T = uint64_t; - constexpr int bits_per_elem = sizeof(T) * 8; - - // Make m and n that are safe to ceildiv. - int m = raft::round_up_safe(p.m, bits_per_elem); - int n = p.n; - - raft::handle_t handle{}; - raft::random::RngState r(1ULL); - auto in = raft::make_device_matrix(handle, m, n); - raft::matrix::fill(handle, in.view(), true); - - int tmp_m = raft::ceildiv(m, bits_per_elem); - auto tmp = raft::make_device_matrix(handle, tmp_m, n); - resource::sync_stream(handle); - RAFT_CUDA_TRY(cudaGetLastError()); - - compress_to_bits(handle, in.view(), tmp.view()); - resource::sync_stream(handle); - RAFT_CUDA_TRY(cudaGetLastError()); - - auto expected = raft::make_device_matrix(handle, tmp_m, n); - raft::matrix::fill(handle, expected.view(), ~T(0)); - - // Check for differences. - ASSERT_TRUE(raft::devArrMatch(expected.data_handle(), - tmp.data_handle(), - tmp.extent(0) * tmp.extent(1), - raft::Compare(), - resource::get_cuda_stream(handle))); - resource::sync_stream(handle); - RAFT_CUDA_TRY(cudaGetLastError()); -} - -class CompressToBitsTest : public ::testing::TestWithParam { - // Empty. -}; - -TEST_P(CompressToBitsTest, CheckTrue64) { check_all_true(GetParam()); } - -TEST_P(CompressToBitsTest, CheckInvertible64) -{ - using T = uint64_t; - check_invertible(GetParam()); -} - -TEST_P(CompressToBitsTest, CheckInvertible32) -{ - using T = uint32_t; - check_invertible(GetParam()); -} - -std::vector params = raft::util::itertools::product( - {1, 3, 32, 33, 63, 64, 65, 128, 10013}, {1, 3, 32, 33, 63, 64, 65, 13001}); - -INSTANTIATE_TEST_CASE_P(CompressToBits, CompressToBitsTest, ::testing::ValuesIn(params)); - -} // namespace raft::distance::masked_nn::compress_to_bits \ No newline at end of file diff --git a/cpp/test/neighbors/ann_brute_force.cuh b/cpp/test/neighbors/ann_brute_force.cuh deleted file mode 100644 index 6370c5ee83..0000000000 --- a/cpp/test/neighbors/ann_brute_force.cuh +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../test_utils.cuh" -#include "ann_utils.cuh" -#include "knn_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -#include -#include - -#include - -#include -#include -#include - -namespace raft::neighbors::brute_force { - -template -struct AnnBruteForceInputs { - IdxT num_queries; - IdxT num_db_vecs; - IdxT dim; - IdxT k; - raft::distance::DistanceType metric; - bool host_dataset; -}; - -template -::std::ostream& operator<<(::std::ostream& os, const AnnBruteForceInputs& p) -{ - os << "{ " << p.num_queries << ", " << p.num_db_vecs << ", " << p.dim << ", " << p.k << ", " - << static_cast(p.metric) << ", " << p.host_dataset << '}' << std::endl; - return os; -} - -template -class AnnBruteForceTest : public ::testing::TestWithParam> { - public: - AnnBruteForceTest() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam>::GetParam()), - database(0, stream_), - search_queries(0, stream_) - { - } - - void testBruteForce() - { - size_t queries_size = ps.num_queries * ps.k; - - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database.data(), - ps.num_queries, - ps.num_db_vecs, - ps.dim, - ps.k, - ps.metric); - resource::sync_stream(handle_); - - { - // Require exact result for brute force - rmm::device_uvector distances_bruteforce_dev(queries_size, stream_); - rmm::device_uvector indices_bruteforce_dev(queries_size, stream_); - brute_force::index_params index_params{}; - brute_force::search_params search_params{}; - index_params.metric = ps.metric; - index_params.metric_arg = 0; - - auto device_dataset = std::optional>{}; - auto idx = [this, &index_params]() { - if (ps.host_dataset) { - auto host_database = raft::make_host_matrix(ps.num_db_vecs, ps.dim); - raft::copy( - host_database.data_handle(), database.data(), ps.num_db_vecs * ps.dim, stream_); - return brute_force::build( - handle_, index_params, raft::make_const_mdspan(host_database.view())); - } else { - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.num_db_vecs, ps.dim); - return brute_force::build(handle_, index_params, database_view); - } - }(); - - auto search_queries_view = raft::make_device_matrix_view( - search_queries.data(), ps.num_queries, ps.dim); - auto indices_out_view = raft::make_device_matrix_view( - indices_bruteforce_dev.data(), ps.num_queries, ps.k); - auto dists_out_view = raft::make_device_matrix_view( - distances_bruteforce_dev.data(), ps.num_queries, ps.k); - brute_force::serialize(handle_, std::string{"brute_force_index"}, idx); - - auto index_loaded = - brute_force::deserialize(handle_, std::string{"brute_force_index"}); - ASSERT_EQ(idx.size(), index_loaded.size()); - - brute_force::search(handle_, - search_params, - index_loaded, - search_queries_view, - indices_out_view, - dists_out_view); - - resource::sync_stream(handle_); - - ASSERT_TRUE(raft::spatial::knn::devArrMatchKnnPair(indices_naive_dev.data(), - indices_bruteforce_dev.data(), - distances_naive_dev.data(), - distances_bruteforce_dev.data(), - ps.num_queries, - ps.k, - 0.001f, - stream_, - true)); - brute_force::serialize(handle_, std::string{"brute_force_index"}, idx, false); - index_loaded = brute_force::deserialize(handle_, std::string{"brute_force_index"}); - index_loaded.update_dataset(handle_, idx.dataset()); - ASSERT_EQ(idx.size(), index_loaded.size()); - - brute_force::search(handle_, - search_params, - index_loaded, - search_queries_view, - indices_out_view, - dists_out_view); - - resource::sync_stream(handle_); - - ASSERT_TRUE(raft::spatial::knn::devArrMatchKnnPair(indices_naive_dev.data(), - indices_bruteforce_dev.data(), - distances_naive_dev.data(), - distances_bruteforce_dev.data(), - ps.num_queries, - ps.k, - 0.001f, - stream_, - true)); - } - } - - void SetUp() override - { - database.resize(ps.num_db_vecs * ps.dim, stream_); - search_queries.resize(ps.num_queries * ps.dim, stream_); - - raft::random::RngState r(1234ULL); - if constexpr (std::is_same{}) { - raft::random::uniform( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(0.1), DataT(2.0)); - raft::random::uniform( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(0.1), DataT(2.0)); - } else { - raft::random::uniformInt( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(1), DataT(20)); - raft::random::uniformInt( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(1), DataT(20)); - } - resource::sync_stream(handle_); - } - - void TearDown() override - { - resource::sync_stream(handle_); - database.resize(0, stream_); - search_queries.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - AnnBruteForceInputs ps; - rmm::device_uvector database; - rmm::device_uvector search_queries; -}; - -const std::vector> inputs = { - // test various dims (aligned and not aligned to vector sizes) - {1000, 10000, 1, 16, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 2, 16, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 3, 16, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 4, 16, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 5, 16, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 8, 16, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 5, 16, raft::distance::DistanceType::L2SqrtExpanded, true}, - {1000, 10000, 8, 16, raft::distance::DistanceType::L2SqrtExpanded, true}, - - // test dims that do not fit into kernel shared memory limits - {1000, 10000, 2048, 16, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 2049, 16, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 2050, 16, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 2051, 16, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 2052, 16, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 2053, 16, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 2056, 16, raft::distance::DistanceType::L2Expanded, true}, - - // host input data - {1000, 10000, 16, 10, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 16, 10, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 16, 10, raft::distance::DistanceType::L2Expanded, false}, - {100, 10000, 16, 10, raft::distance::DistanceType::L2Expanded, false}, - {20, 100000, 16, 10, raft::distance::DistanceType::L2Expanded, false}, - {1000, 100000, 16, 10, raft::distance::DistanceType::L2Expanded, false}, - {10000, 131072, 8, 10, raft::distance::DistanceType::L2Expanded, false}, - - {1000, 10000, 16, 10, raft::distance::DistanceType::InnerProduct, false}}; -} // namespace raft::neighbors::brute_force diff --git a/cpp/test/neighbors/ann_brute_force/test_float.cu b/cpp/test/neighbors/ann_brute_force/test_float.cu deleted file mode 100644 index f157b5f65c..0000000000 --- a/cpp/test/neighbors/ann_brute_force/test_float.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_brute_force.cuh" - -#include - -namespace raft::neighbors::brute_force { - -using AnnBruteForceTest_float = AnnBruteForceTest; -TEST_P(AnnBruteForceTest_float, AnnBruteForce) { this->testBruteForce(); } - -INSTANTIATE_TEST_CASE_P(AnnBruteForceTest, AnnBruteForceTest_float, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::brute_force diff --git a/cpp/test/neighbors/ann_cagra.cuh b/cpp/test/neighbors/ann_cagra.cuh deleted file mode 100644 index cc787d3e57..0000000000 --- a/cpp/test/neighbors/ann_cagra.cuh +++ /dev/null @@ -1,949 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#undef RAFT_EXPLICIT_INSTANTIATE_ONLY // Search with filter instantiation - -#include "../test_utils.cuh" -#include "ann_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include - -#include - -#include -#include -#include -#include - -namespace raft::neighbors::cagra { -namespace { - -/* A filter that excludes all indices below `offset`. */ -struct test_cagra_sample_filter { - static constexpr unsigned offset = 300; - inline _RAFT_HOST_DEVICE auto operator()( - // query index - const uint32_t query_ix, - // the index of the current sample inside the current inverted list - const uint32_t sample_ix) const - { - return sample_ix >= offset; - } -}; - -// For sort_knn_graph test -template -void RandomSuffle(raft::host_matrix_view index) -{ - for (IdxT i = 0; i < index.extent(0); i++) { - uint64_t rand = i; - IdxT* const row_ptr = index.data_handle() + i * index.extent(1); - for (unsigned j = 0; j < index.extent(1); j++) { - // Swap two indices at random - rand = raft::neighbors::cagra::detail::device::xorshift64(rand); - const auto i0 = rand % index.extent(1); - rand = raft::neighbors::cagra::detail::device::xorshift64(rand); - const auto i1 = rand % index.extent(1); - - const auto tmp = row_ptr[i0]; - row_ptr[i0] = row_ptr[i1]; - row_ptr[i1] = tmp; - } - } -} - -template -testing::AssertionResult CheckOrder(raft::host_matrix_view index_test, - raft::host_matrix_view dataset, - raft::distance::DistanceType metric) -{ - for (IdxT i = 0; i < index_test.extent(0); i++) { - const DatatT* const base_vec = dataset.data_handle() + i * dataset.extent(1); - const IdxT* const index_row = index_test.data_handle() + i * index_test.extent(1); - DistanceT prev_distance = metric == raft::distance::DistanceType::L2Expanded - ? 0 - : std::numeric_limits::max(); - for (unsigned j = 0; j < index_test.extent(1) - 1; j++) { - const DatatT* const target_vec = dataset.data_handle() + index_row[j] * dataset.extent(1); - DistanceT distance = 0; - switch (metric) { - case raft::distance::DistanceType::L2Expanded: - for (unsigned l = 0; l < dataset.extent(1); l++) { - const auto diff = - static_cast(target_vec[l]) - static_cast(base_vec[l]); - distance += diff * diff; - } - if (prev_distance > distance) { - return testing::AssertionFailure() - << "Wrong index order (row = " << i << ", neighbor_id = " << j - << "). (distance[neighbor_id-1] = " << prev_distance - << "should be lesser than distance[neighbor_id] = " << distance << ")"; - } - break; - case raft::distance::DistanceType::InnerProduct: - for (unsigned l = 0; l < dataset.extent(1); l++) { - const auto prod = - static_cast(target_vec[l]) * static_cast(base_vec[l]); - distance += prod; - } - if (prev_distance < distance) { - return testing::AssertionFailure() - << "Wrong index order (row = " << i << ", neighbor_id = " << j - << "). (distance[neighbor_id-1] = " << prev_distance - << "should be greater than distance[neighbor_id] = " << distance << ")"; - } - break; - default: - return testing::AssertionFailure() - << "Distance metric " << metric - << " not supported. Only L2Expanded and InnerProduct are supported"; - } - prev_distance = distance; - } - } - return testing::AssertionSuccess(); -} - -template -struct fpi_mapper {}; - -template <> -struct fpi_mapper { - using type = int64_t; - static constexpr int kBitshiftBase = 53; -}; - -template <> -struct fpi_mapper { - using type = int32_t; - static constexpr int kBitshiftBase = 24; -}; - -template <> -struct fpi_mapper { - using type = int16_t; - static constexpr int kBitshiftBase = 11; -}; - -// Generate dataset to ensure no rounding error occurs in the norm computation of any two vectors. -// When testing the CAGRA index sorting function, rounding errors can affect the norm and alter the -// order of the index. To ensure the accuracy of the test, we utilize the dataset. The generation -// method is based on the error-free transformation (EFT) method. -template -RAFT_KERNEL GenerateRoundingErrorFreeDataset_kernel(T* const ptr, - const uint32_t size, - const typename fpi_mapper::type resolution) -{ - const auto tid = threadIdx.x + blockIdx.x * blockDim.x; - if (tid >= size) { return; } - - const float u32 = *reinterpret_cast::type*>(ptr + tid); - ptr[tid] = u32 / resolution; -} - -template -void GenerateRoundingErrorFreeDataset( - const raft::resources& handle, - T* const ptr, - const uint32_t n_row, - const uint32_t dim, - raft::random::RngState& rng, - const bool diff_flag // true if compute the norm between two vectors -) -{ - using mapper_type = fpi_mapper; - using int_type = typename mapper_type::type; - auto cuda_stream = resource::get_cuda_stream(handle); - const uint32_t size = n_row * dim; - const uint32_t block_size = 256; - const uint32_t grid_size = (size + block_size - 1) / block_size; - - const auto bitshift = (mapper_type::kBitshiftBase - std::log2(dim) - (diff_flag ? 1 : 0)) / 2; - // Skip the test when `dim` is too big for type `T` to allow rounding error-free test. - if (bitshift <= 1) { GTEST_SKIP(); } - const int_type resolution = int_type{1} << static_cast(std::floor(bitshift)); - raft::random::uniformInt( - handle, rng, reinterpret_cast(ptr), size, -resolution, resolution - 1); - - GenerateRoundingErrorFreeDataset_kernel - <<>>(ptr, size, resolution); -} - -template -void InitDataset(const raft::resources& handle, - DataT* const datatset_ptr, - std::uint32_t size, - std::uint32_t dim, - raft::distance::DistanceType metric, - raft::random::RngState& r) -{ - if constexpr (std::is_same_v || std::is_same_v) { - GenerateRoundingErrorFreeDataset(handle, datatset_ptr, size, dim, r, true); - - if (metric == raft::distance::InnerProduct) { - auto dataset_view = raft::make_device_matrix_view(datatset_ptr, size, dim); - raft::linalg::row_normalize( - handle, raft::make_const_mdspan(dataset_view), dataset_view, raft::linalg::L2Norm); - } - } else if constexpr (std::is_same_v || std::is_same_v) { - if constexpr (std::is_same_v) { - raft::random::uniformInt(handle, r, datatset_ptr, size * dim, DataT(-10), DataT(10)); - } else { - raft::random::uniformInt(handle, r, datatset_ptr, size * dim, DataT(1), DataT(20)); - } - - if (metric == raft::distance::InnerProduct) { - // TODO (enp1s0): Change this once row_normalize supports (u)int8 matrices. - // https://github.com/rapidsai/raft/issues/2291 - - using ComputeT = float; - auto dataset_view = raft::make_device_matrix_view(datatset_ptr, size, dim); - auto dev_row_norm = raft::make_device_vector(handle, size); - const auto normalized_norm = - (std::is_same_v ? 40 : 20) * std::sqrt(static_cast(dim)); - - raft::linalg::reduce(dev_row_norm.data_handle(), - datatset_ptr, - dim, - size, - 0.f, - true, - true, - resource::get_cuda_stream(handle), - false, - raft::sq_op(), - raft::add_op(), - raft::sqrt_op()); - raft::linalg::matrix_vector_op( - handle, - raft::make_const_mdspan(dataset_view), - raft::make_const_mdspan(dev_row_norm.view()), - dataset_view, - raft::linalg::Apply::ALONG_COLUMNS, - [normalized_norm] __device__(DataT elm, ComputeT norm) { - const ComputeT v = elm / norm * normalized_norm; - const ComputeT max_v_range = std::numeric_limits::max(); - const ComputeT min_v_range = std::numeric_limits::min(); - return static_cast(std::min(max_v_range, std::max(min_v_range, v))); - }); - } - } -} -} // namespace - -struct AnnCagraInputs { - int n_queries; - int n_rows; - int dim; - int k; - graph_build_algo build_algo; - search_algo algo; - int max_queries; - int team_size; - int itopk_size; - int search_width; - raft::distance::DistanceType metric; - bool host_dataset; - bool include_serialized_dataset; - // std::optional - double min_recall; // = std::nullopt; -}; - -inline ::std::ostream& operator<<(::std::ostream& os, const AnnCagraInputs& p) -{ - std::vector algo = {"single-cta", "multi_cta", "multi_kernel", "auto"}; - std::vector build_algo = {"IVF_PQ", "NN_DESCENT"}; - os << "{n_queries=" << p.n_queries << ", dataset shape=" << p.n_rows << "x" << p.dim - << ", k=" << p.k << ", " << algo.at((int)p.algo) << ", max_queries=" << p.max_queries - << ", itopk_size=" << p.itopk_size << ", search_width=" << p.search_width - << ", metric=" << static_cast(p.metric) << (p.host_dataset ? ", host" : ", device") - << ", build_algo=" << build_algo.at((int)p.build_algo) << '}' << std::endl; - return os; -} - -template -class AnnCagraTest : public ::testing::TestWithParam { - public: - AnnCagraTest() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam::GetParam()), - database(0, stream_), - search_queries(0, stream_) - { - } - - protected: - void testCagra() - { - // TODO (tarang-jain): remove when NN Descent index building support InnerProduct. Reference - // issue: https://github.com/rapidsai/raft/issues/2276 - if (ps.metric == distance::InnerProduct && ps.build_algo == graph_build_algo::NN_DESCENT) - GTEST_SKIP(); - - size_t queries_size = ps.n_queries * ps.k; - std::vector indices_Cagra(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_Cagra(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database.data(), - ps.n_queries, - ps.n_rows, - ps.dim, - ps.k, - ps.metric); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - { - rmm::device_uvector distances_dev(queries_size, stream_); - rmm::device_uvector indices_dev(queries_size, stream_); - - { - cagra::index_params index_params; - index_params.metric = ps.metric; // Note: currently ony the cagra::index_params metric is - // not used for knn_graph building. - index_params.build_algo = ps.build_algo; - cagra::search_params search_params; - search_params.algo = ps.algo; - search_params.max_queries = ps.max_queries; - search_params.team_size = ps.team_size; - search_params.itopk_size = ps.itopk_size; - - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.n_rows, ps.dim); - - { - cagra::index index(handle_); - if (ps.host_dataset) { - auto database_host = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy(database_host.data_handle(), database.data(), database.size(), stream_); - auto database_host_view = raft::make_host_matrix_view( - (const DataT*)database_host.data_handle(), ps.n_rows, ps.dim); - index = cagra::build(handle_, index_params, database_host_view); - } else { - index = cagra::build(handle_, index_params, database_view); - }; - cagra::serialize(handle_, "cagra_index", index, ps.include_serialized_dataset); - } - - auto index = cagra::deserialize(handle_, "cagra_index"); - if (!ps.include_serialized_dataset) { index.update_dataset(handle_, database_view); } - - auto search_queries_view = raft::make_device_matrix_view( - search_queries.data(), ps.n_queries, ps.dim); - auto indices_out_view = - raft::make_device_matrix_view(indices_dev.data(), ps.n_queries, ps.k); - auto dists_out_view = raft::make_device_matrix_view( - distances_dev.data(), ps.n_queries, ps.k); - - cagra::search( - handle_, search_params, index, search_queries_view, indices_out_view, dists_out_view); - update_host(distances_Cagra.data(), distances_dev.data(), queries_size, stream_); - update_host(indices_Cagra.data(), indices_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - // for (int i = 0; i < min(ps.n_queries, 10); i++) { - // // std::cout << "query " << i << std::end; - // print_vector("T", indices_naive.data() + i * ps.k, ps.k, std::cout); - // print_vector("C", indices_Cagra.data() + i * ps.k, ps.k, std::cout); - // print_vector("T", distances_naive.data() + i * ps.k, ps.k, std::cout); - // print_vector("C", distances_Cagra.data() + i * ps.k, ps.k, std::cout); - // } - - double min_recall = ps.min_recall; - EXPECT_TRUE(eval_neighbours(indices_naive, - indices_Cagra, - distances_naive, - distances_Cagra, - ps.n_queries, - ps.k, - 0.003, - min_recall)); - EXPECT_TRUE(eval_distances(handle_, - database.data(), - search_queries.data(), - indices_dev.data(), - distances_dev.data(), - ps.n_rows, - ps.dim, - ps.n_queries, - ps.k, - ps.metric, - 1.0e-4)); - } - } - - void SetUp() override - { - database.resize(((size_t)ps.n_rows) * ps.dim, stream_); - search_queries.resize(ps.n_queries * ps.dim, stream_); - raft::random::RngState r(1234ULL); - InitDataset(handle_, database.data(), ps.n_rows, ps.dim, ps.metric, r); - InitDataset(handle_, search_queries.data(), ps.n_queries, ps.dim, ps.metric, r); - resource::sync_stream(handle_); - } - - void TearDown() override - { - resource::sync_stream(handle_); - database.resize(0, stream_); - search_queries.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - AnnCagraInputs ps; - rmm::device_uvector database; - rmm::device_uvector search_queries; -}; - -template -class AnnCagraSortTest : public ::testing::TestWithParam { - public: - AnnCagraSortTest() - : ps(::testing::TestWithParam::GetParam()), database(0, handle_.get_stream()) - { - } - - protected: - void testCagraSort() - { - if (ps.metric == distance::InnerProduct && ps.build_algo == graph_build_algo::NN_DESCENT) - GTEST_SKIP(); - - { - // Step 1: Build a sorted KNN graph by CAGRA knn build - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.n_rows, ps.dim); - auto database_host = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy( - database_host.data_handle(), database.data(), database.size(), handle_.get_stream()); - auto database_host_view = raft::make_host_matrix_view( - (const DataT*)database_host.data_handle(), ps.n_rows, ps.dim); - - cagra::index_params index_params; - auto knn_graph = - raft::make_host_matrix(ps.n_rows, index_params.intermediate_graph_degree); - - if (ps.build_algo == graph_build_algo::IVF_PQ) { - auto build_params = ivf_pq::index_params::from_dataset(database_view, ps.metric); - if (ps.host_dataset) { - cagra::build_knn_graph( - handle_, database_host_view, knn_graph.view(), 2, build_params); - } else { - cagra::build_knn_graph( - handle_, database_view, knn_graph.view(), 2, build_params); - } - } else { - auto nn_descent_idx_params = experimental::nn_descent::index_params{}; - nn_descent_idx_params.graph_degree = index_params.intermediate_graph_degree; - nn_descent_idx_params.intermediate_graph_degree = index_params.intermediate_graph_degree; - - if (ps.host_dataset) { - cagra::build_knn_graph( - handle_, database_host_view, knn_graph.view(), nn_descent_idx_params); - } else { - cagra::build_knn_graph( - handle_, database_host_view, knn_graph.view(), nn_descent_idx_params); - } - } - - handle_.sync_stream(); - ASSERT_TRUE(CheckOrder(knn_graph.view(), database_host.view(), ps.metric)); - - if (ps.metric != raft::distance::DistanceType::InnerProduct) { - RandomSuffle(knn_graph.view()); - - cagra::sort_knn_graph(handle_, database_view, knn_graph.view()); - handle_.sync_stream(); - - ASSERT_TRUE(CheckOrder(knn_graph.view(), database_host.view(), ps.metric)); - } - } - } - - void SetUp() override - { - database.resize(((size_t)ps.n_rows) * ps.dim, handle_.get_stream()); - raft::random::RngState r(1234ULL); - if constexpr (std::is_same_v || std::is_same_v) { - GenerateRoundingErrorFreeDataset(handle_, database.data(), ps.n_rows, ps.dim, r, false); - } else { - raft::random::uniformInt( - handle_, r, database.data(), ps.n_rows * ps.dim, DataT(1), DataT(20)); - } - handle_.sync_stream(); - } - - void TearDown() override - { - handle_.sync_stream(); - database.resize(0, handle_.get_stream()); - } - - private: - raft::device_resources handle_; - AnnCagraInputs ps; - rmm::device_uvector database; -}; - -template -class AnnCagraFilterTest : public ::testing::TestWithParam { - public: - AnnCagraFilterTest() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam::GetParam()), - database(0, stream_), - search_queries(0, stream_) - { - } - - protected: - void testCagraFilter() - { - if (ps.metric == distance::InnerProduct && ps.build_algo == graph_build_algo::NN_DESCENT) - GTEST_SKIP(); - - size_t queries_size = ps.n_queries * ps.k; - std::vector indices_Cagra(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_Cagra(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - auto* database_filtered_ptr = database.data() + test_cagra_sample_filter::offset * ps.dim; - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database_filtered_ptr, - ps.n_queries, - ps.n_rows - test_cagra_sample_filter::offset, - ps.dim, - ps.k, - ps.metric); - raft::linalg::addScalar(indices_naive_dev.data(), - indices_naive_dev.data(), - IdxT(test_cagra_sample_filter::offset), - queries_size, - stream_); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - { - rmm::device_uvector distances_dev(queries_size, stream_); - rmm::device_uvector indices_dev(queries_size, stream_); - - { - cagra::index_params index_params; - index_params.metric = ps.metric; // Note: currently ony the cagra::index_params metric is - // not used for knn_graph building. - index_params.nn_descent_niter = 50; - cagra::search_params search_params; - search_params.algo = ps.algo; - search_params.max_queries = ps.max_queries; - search_params.team_size = ps.team_size; - search_params.hashmap_mode = cagra::hash_mode::HASH; - - // TODO: setting search_params.itopk_size here breaks the filter tests, but is required for - // k>1024 skip these tests until fixed - if (ps.k >= 1024) { GTEST_SKIP(); } - // search_params.itopk_size = ps.itopk_size; - - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.n_rows, ps.dim); - - cagra::index index(handle_); - if (ps.host_dataset) { - auto database_host = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy(database_host.data_handle(), database.data(), database.size(), stream_); - auto database_host_view = raft::make_host_matrix_view( - (const DataT*)database_host.data_handle(), ps.n_rows, ps.dim); - index = cagra::build(handle_, index_params, database_host_view); - } else { - index = cagra::build(handle_, index_params, database_view); - } - - if (!ps.include_serialized_dataset) { index.update_dataset(handle_, database_view); } - - auto search_queries_view = raft::make_device_matrix_view( - search_queries.data(), ps.n_queries, ps.dim); - auto indices_out_view = - raft::make_device_matrix_view(indices_dev.data(), ps.n_queries, ps.k); - auto dists_out_view = raft::make_device_matrix_view( - distances_dev.data(), ps.n_queries, ps.k); - - cagra::search_with_filtering(handle_, - search_params, - index, - search_queries_view, - indices_out_view, - dists_out_view, - test_cagra_sample_filter()); - update_host(distances_Cagra.data(), distances_dev.data(), queries_size, stream_); - update_host(indices_Cagra.data(), indices_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - // Test filter - bool unacceptable_node = false; - for (int q = 0; q < ps.n_queries; q++) { - for (int i = 0; i < ps.k; i++) { - const auto n = indices_Cagra[q * ps.k + i]; - unacceptable_node = unacceptable_node | !test_cagra_sample_filter()(q, n); - } - } - EXPECT_FALSE(unacceptable_node); - - double min_recall = ps.min_recall; - // TODO(mfoerster): re-enable uniquenes test - EXPECT_TRUE(eval_neighbours(indices_naive, - indices_Cagra, - distances_naive, - distances_Cagra, - ps.n_queries, - ps.k, - 0.003, - min_recall, - false)); - EXPECT_TRUE(eval_distances(handle_, - database.data(), - search_queries.data(), - indices_dev.data(), - distances_dev.data(), - ps.n_rows, - ps.dim, - ps.n_queries, - ps.k, - ps.metric, - 1.0e-4)); - } - } - - void testCagraRemoved() - { - if (ps.metric == distance::InnerProduct && ps.build_algo == graph_build_algo::NN_DESCENT) - GTEST_SKIP(); - - size_t queries_size = ps.n_queries * ps.k; - std::vector indices_Cagra(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_Cagra(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - auto* database_filtered_ptr = database.data() + test_cagra_sample_filter::offset * ps.dim; - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database_filtered_ptr, - ps.n_queries, - ps.n_rows - test_cagra_sample_filter::offset, - ps.dim, - ps.k, - ps.metric); - raft::linalg::addScalar(indices_naive_dev.data(), - indices_naive_dev.data(), - IdxT(test_cagra_sample_filter::offset), - queries_size, - stream_); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - { - rmm::device_uvector distances_dev(queries_size, stream_); - rmm::device_uvector indices_dev(queries_size, stream_); - - { - cagra::index_params index_params; - index_params.metric = ps.metric; // Note: currently ony the cagra::index_params metric is - // not used for knn_graph building. - index_params.nn_descent_niter = 50; - cagra::search_params search_params; - search_params.algo = ps.algo; - search_params.max_queries = ps.max_queries; - search_params.team_size = ps.team_size; - search_params.hashmap_mode = cagra::hash_mode::HASH; - - // TODO: setting search_params.itopk_size here breaks the filter tests, but is required for - // k>1024 skip these tests until fixed - if (ps.k >= 1024) { GTEST_SKIP(); } - // search_params.itopk_size = ps.itopk_size; - - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.n_rows, ps.dim); - - cagra::index index(handle_); - if (ps.host_dataset) { - auto database_host = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy(database_host.data_handle(), database.data(), database.size(), stream_); - auto database_host_view = raft::make_host_matrix_view( - (const DataT*)database_host.data_handle(), ps.n_rows, ps.dim); - index = cagra::build(handle_, index_params, database_host_view); - } else { - index = cagra::build(handle_, index_params, database_view); - } - - if (!ps.include_serialized_dataset) { index.update_dataset(handle_, database_view); } - - auto search_queries_view = raft::make_device_matrix_view( - search_queries.data(), ps.n_queries, ps.dim); - auto indices_out_view = - raft::make_device_matrix_view(indices_dev.data(), ps.n_queries, ps.k); - auto dists_out_view = raft::make_device_matrix_view( - distances_dev.data(), ps.n_queries, ps.k); - auto removed_indices = - raft::make_device_vector(handle_, test_cagra_sample_filter::offset); - thrust::sequence( - resource::get_thrust_policy(handle_), - thrust::device_pointer_cast(removed_indices.data_handle()), - thrust::device_pointer_cast(removed_indices.data_handle() + removed_indices.extent(0))); - resource::sync_stream(handle_); - raft::core::bitset removed_indices_bitset( - handle_, removed_indices.view(), ps.n_rows); - cagra::search_with_filtering( - handle_, - search_params, - index, - search_queries_view, - indices_out_view, - dists_out_view, - raft::neighbors::filtering::bitset_filter(removed_indices_bitset.view())); - update_host(distances_Cagra.data(), distances_dev.data(), queries_size, stream_); - update_host(indices_Cagra.data(), indices_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - double min_recall = ps.min_recall; - // TODO(mfoerster): re-enable uniquenes test - EXPECT_TRUE(eval_neighbours(indices_naive, - indices_Cagra, - distances_naive, - distances_Cagra, - ps.n_queries, - ps.k, - 0.003, - min_recall, - false)); - EXPECT_TRUE(eval_distances(handle_, - database.data(), - search_queries.data(), - indices_dev.data(), - distances_dev.data(), - ps.n_rows, - ps.dim, - ps.n_queries, - ps.k, - ps.metric, - 1.0e-4)); - } - } - - void SetUp() override - { - database.resize(((size_t)ps.n_rows) * ps.dim, stream_); - search_queries.resize(ps.n_queries * ps.dim, stream_); - raft::random::RngState r(1234ULL); - InitDataset(handle_, database.data(), ps.n_rows, ps.dim, ps.metric, r); - InitDataset(handle_, search_queries.data(), ps.n_queries, ps.dim, ps.metric, r); - resource::sync_stream(handle_); - } - - void TearDown() override - { - resource::sync_stream(handle_); - database.resize(0, stream_); - search_queries.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - AnnCagraInputs ps; - rmm::device_uvector database; - rmm::device_uvector search_queries; -}; - -inline std::vector generate_inputs() -{ - // TODO(tfeher): test MULTI_CTA kernel with search_width > 1 to allow multiple CTA per queries - std::vector inputs = raft::util::itertools::product( - {100}, - {1000}, - {1, 8, 17, 1599}, - {16}, // k - {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, - {search_algo::SINGLE_CTA, search_algo::MULTI_CTA, search_algo::MULTI_KERNEL}, - {0, 1, 10, 100}, // query size - {0}, - {256}, - {1}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false}, - {true}, - {0.995}); - - auto inputs2 = raft::util::itertools::product( - {100}, - {1000}, - {1, 8, 17, 1599}, - {1}, // k - {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, - {search_algo::SINGLE_CTA, search_algo::MULTI_CTA, search_algo::MULTI_KERNEL}, - {0, 1, 10, 100}, // query size - {0}, - {256}, - {1}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false}, - {true}, - {99. / 100} - // smaller threshould than the other test cases because it is too strict for Top-1 search - ); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - - inputs2 = raft::util::itertools::product( - {100}, - {1000}, - {1, 3, 5, 7, 8, 17, 64, 128, 137, 192, 256, 512, 619, 1024}, // dim - {16}, // k - {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, - {search_algo::AUTO}, - {10}, - {0}, - {64}, - {1}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false}, - {true}, - {0.995}); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - inputs2 = raft::util::itertools::product( - {100}, - {1000}, - {64}, - {16}, - {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, - {search_algo::AUTO}, - {10}, - {0, 4, 8, 16, 32}, // team_size - {64}, - {1}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false}, - {false}, - {0.995}); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - - inputs2 = raft::util::itertools::product( - {100}, - {1000}, - {64}, - {16}, - {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, - {search_algo::AUTO}, - {10}, - {0}, // team_size - {32, 64, 128, 256, 512, 768}, - {1}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false}, - {true}, - {0.995}); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - - inputs2 = raft::util::itertools::product( - {100}, - {10000, 20000}, - {32}, - {10}, - {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, - {search_algo::AUTO}, - {10}, - {0}, // team_size - {64}, - {1}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false, true}, - {false}, - {0.995}); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - - inputs2 = raft::util::itertools::product( - {100}, - {20000}, - {32}, - {2048}, // k - {graph_build_algo::NN_DESCENT}, - {search_algo::AUTO}, - {10}, - {0}, - {4096}, // itopk_size - {1}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false}, - {false}, - {0.995}); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - - return inputs; -} - -const std::vector inputs = generate_inputs(); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra/search_kernel_uint64_t.cuh b/cpp/test/neighbors/ann_cagra/search_kernel_uint64_t.cuh deleted file mode 100644 index 412e71bff1..0000000000 --- a/cpp/test/neighbors/ann_cagra/search_kernel_uint64_t.cuh +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include // none_cagra_sample_filter -#include // RAFT_EXPLICIT - -namespace raft::neighbors::cagra::detail { - -namespace multi_cta_search { -#define instantiate_kernel_selection( \ - DATASET_DESCRIPTOR, TEAM_SIZE, MAX_DATASET_DIM, DATA_T, INDEX_T, DISTANCE_T, SAMPLE_FILTER_T) \ - extern template void \ - select_and_run, \ - SAMPLE_FILTER_T>( \ - raft::neighbors::cagra::detail::DATASET_DESCRIPTOR dataset_desc, \ - raft::device_matrix_view graph, \ - INDEX_T* const topk_indices_ptr, \ - DISTANCE_T* const topk_distances_ptr, \ - const DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t block_size, \ - uint32_t result_buffer_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - INDEX_T* hashmap_ptr, \ - uint32_t num_cta_per_query, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -instantiate_kernel_selection(standard_dataset_descriptor_t, - 32, - 1024, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection(standard_dataset_descriptor_t, - 8, - 128, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection(standard_dataset_descriptor_t, - 16, - 256, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_kernel_selection(standard_dataset_descriptor_t, - 32, - 512, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -#undef instantiate_kernel_selection -} // namespace multi_cta_search - -namespace single_cta_search { - -#define instantiate_single_cta_select_and_run( \ - DATASET_DESCRIPTOR, TEAM_SIZE, MAX_DATASET_DIM, DATA_T, INDEX_T, DISTANCE_T, SAMPLE_FILTER_T) \ - extern template void \ - select_and_run, \ - SAMPLE_FILTER_T>( \ - raft::neighbors::cagra::detail::DATASET_DESCRIPTOR dataset_desc, \ - raft::device_matrix_view graph, \ - INDEX_T* const topk_indices_ptr, \ - DISTANCE_T* const topk_distances_ptr, \ - const DATA_T* const queries_ptr, \ - const uint32_t num_queries, \ - const INDEX_T* dev_seed_ptr, \ - uint32_t* const num_executed_iterations, \ - uint32_t topk, \ - uint32_t num_itopk_candidates, \ - uint32_t block_size, \ - uint32_t smem_size, \ - int64_t hash_bitlen, \ - INDEX_T* hashmap_ptr, \ - size_t small_hash_bitlen, \ - size_t small_hash_reset_interval, \ - uint32_t num_random_samplings, \ - uint64_t rand_xor_mask, \ - uint32_t num_seeds, \ - size_t itopk_size, \ - size_t search_width, \ - size_t min_iterations, \ - size_t max_iterations, \ - SAMPLE_FILTER_T sample_filter, \ - raft::distance::DistanceType metric, \ - cudaStream_t stream); - -instantiate_single_cta_select_and_run(standard_dataset_descriptor_t, - 32, - 1024, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run(standard_dataset_descriptor_t, - 8, - 128, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run(standard_dataset_descriptor_t, - 16, - 256, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); -instantiate_single_cta_select_and_run(standard_dataset_descriptor_t, - 32, - 512, - float, - uint64_t, - float, - raft::neighbors::filtering::none_cagra_sample_filter); - -} // namespace single_cta_search -} // namespace raft::neighbors::cagra::detail diff --git a/cpp/test/neighbors/ann_cagra/test_float_int64_t.cu b/cpp/test/neighbors/ann_cagra/test_float_int64_t.cu deleted file mode 100644 index ff7e839abf..0000000000 --- a/cpp/test/neighbors/ann_cagra/test_float_int64_t.cu +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_cagra.cuh" -#include "search_kernel_uint64_t.cuh" - -#include - -namespace raft::neighbors::cagra { - -typedef AnnCagraTest AnnCagraTestF_I64; -TEST_P(AnnCagraTestF_I64, AnnCagra) { this->testCagra(); } - -INSTANTIATE_TEST_CASE_P(AnnCagraTest, AnnCagraTestF_I64, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra/test_float_uint32_t.cu b/cpp/test/neighbors/ann_cagra/test_float_uint32_t.cu deleted file mode 100644 index 7d29ce4f99..0000000000 --- a/cpp/test/neighbors/ann_cagra/test_float_uint32_t.cu +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_cagra.cuh" - -#include - -namespace raft::neighbors::cagra { - -typedef AnnCagraTest AnnCagraTestF_U32; -TEST_P(AnnCagraTestF_U32, AnnCagra) { this->testCagra(); } - -typedef AnnCagraSortTest AnnCagraSortTestF_U32; -TEST_P(AnnCagraSortTestF_U32, AnnCagraSort) { this->testCagraSort(); } - -typedef AnnCagraFilterTest AnnCagraFilterTestF_U32; -TEST_P(AnnCagraFilterTestF_U32, AnnCagraFilter) -{ - this->testCagraFilter(); - this->testCagraRemoved(); -} - -INSTANTIATE_TEST_CASE_P(AnnCagraTest, AnnCagraTestF_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraSortTest, AnnCagraSortTestF_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraFilterTest, AnnCagraFilterTestF_U32, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra/test_half_int64_t.cu b/cpp/test/neighbors/ann_cagra/test_half_int64_t.cu deleted file mode 100644 index bcdd95bece..0000000000 --- a/cpp/test/neighbors/ann_cagra/test_half_int64_t.cu +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_cagra.cuh" -#include "search_kernel_uint64_t.cuh" - -#include - -namespace raft::neighbors::cagra { - -typedef AnnCagraTest AnnCagraTestH_I64; -TEST_P(AnnCagraTestH_I64, AnnCagra) { this->testCagra(); } - -INSTANTIATE_TEST_CASE_P(AnnCagraTest, AnnCagraTestH_I64, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra/test_half_uint32_t.cu b/cpp/test/neighbors/ann_cagra/test_half_uint32_t.cu deleted file mode 100644 index ec7144f8d0..0000000000 --- a/cpp/test/neighbors/ann_cagra/test_half_uint32_t.cu +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_cagra.cuh" - -#include - -namespace raft::neighbors::cagra { - -typedef AnnCagraTest AnnCagraTestH_U32; -TEST_P(AnnCagraTestH_U32, AnnCagra) { this->testCagra(); } - -typedef AnnCagraSortTest AnnCagraSortTestH_U32; -TEST_P(AnnCagraSortTestH_U32, AnnCagraSort) { this->testCagraSort(); } - -typedef AnnCagraFilterTest AnnCagraFilterTestH_U32; -TEST_P(AnnCagraFilterTestH_U32, AnnCagraFilter) -{ - this->testCagraFilter(); - this->testCagraRemoved(); -} - -INSTANTIATE_TEST_CASE_P(AnnCagraTest, AnnCagraTestH_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraSortTest, AnnCagraSortTestH_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraFilterTest, AnnCagraFilterTestH_U32, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra/test_int8_t_uint32_t.cu b/cpp/test/neighbors/ann_cagra/test_int8_t_uint32_t.cu deleted file mode 100644 index b2242d89b1..0000000000 --- a/cpp/test/neighbors/ann_cagra/test_int8_t_uint32_t.cu +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_cagra.cuh" - -#include - -namespace raft::neighbors::cagra { - -typedef AnnCagraTest AnnCagraTestI8_U32; -TEST_P(AnnCagraTestI8_U32, AnnCagra) { this->testCagra(); } -typedef AnnCagraSortTest AnnCagraSortTestI8_U32; -TEST_P(AnnCagraSortTestI8_U32, AnnCagraSort) { this->testCagraSort(); } -typedef AnnCagraFilterTest AnnCagraFilterTestI8_U32; -TEST_P(AnnCagraFilterTestI8_U32, AnnCagraFilter) -{ - this->testCagraFilter(); - this->testCagraRemoved(); -} - -INSTANTIATE_TEST_CASE_P(AnnCagraTest, AnnCagraTestI8_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraSortTest, AnnCagraSortTestI8_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraFilterTest, AnnCagraFilterTestI8_U32, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra/test_uint8_t_uint32_t.cu b/cpp/test/neighbors/ann_cagra/test_uint8_t_uint32_t.cu deleted file mode 100644 index 302b2bec18..0000000000 --- a/cpp/test/neighbors/ann_cagra/test_uint8_t_uint32_t.cu +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_cagra.cuh" - -#include - -namespace raft::neighbors::cagra { - -typedef AnnCagraTest AnnCagraTestU8_U32; -TEST_P(AnnCagraTestU8_U32, AnnCagra) { this->testCagra(); } - -typedef AnnCagraSortTest AnnCagraSortTestU8_U32; -TEST_P(AnnCagraSortTestU8_U32, AnnCagraSort) { this->testCagraSort(); } - -typedef AnnCagraFilterTest AnnCagraFilterTestU8_U32; -TEST_P(AnnCagraFilterTestU8_U32, AnnCagraSort) -{ - this->testCagraFilter(); - this->testCagraRemoved(); -} - -INSTANTIATE_TEST_CASE_P(AnnCagraTest, AnnCagraTestU8_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraSortTest, AnnCagraSortTestU8_U32, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(AnnCagraFilterTest, AnnCagraFilterTestU8_U32, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra_vpq.cuh b/cpp/test/neighbors/ann_cagra_vpq.cuh deleted file mode 100644 index 6b24bca921..0000000000 --- a/cpp/test/neighbors/ann_cagra_vpq.cuh +++ /dev/null @@ -1,336 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../test_utils.cuh" -#include "ann_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include - -#include - -#include -#include -#include -#include -#include - -namespace { -template -void GenerateDataset(T* const dataset_ptr, - T* const query_ptr, - const std::size_t dataset_size, - const std::size_t query_size, - const std::size_t dim, - const std::size_t num_centers, - cudaStream_t cuda_stream) -{ - auto center_list = raft::make_host_matrix(num_centers, dim); - auto host_dataset = raft::make_host_matrix(std::max(dataset_size, query_size), dim); - - std::normal_distribution dist(0, 1); - std::mt19937 mt(0); - for (std::size_t i = 0; i < center_list.size(); i++) { - center_list.data_handle()[i] = dist(mt); - } - - std::uniform_int_distribution i_dist(0, num_centers - 1); - for (std::size_t i = 0; i < dataset_size; i++) { - const auto center_index = i_dist(mt); - for (std::size_t j = 0; j < dim; j++) { - host_dataset.data_handle()[i * dim + j] = - center_list.data_handle()[center_index + j] + dist(mt) * 1e-1; - } - } - raft::copy(dataset_ptr, host_dataset.data_handle(), dataset_size * dim, cuda_stream); - - for (std::size_t i = 0; i < query_size; i++) { - const auto center_index = i_dist(mt); - for (std::size_t j = 0; j < dim; j++) { - host_dataset.data_handle()[i * dim + j] = - center_list.data_handle()[center_index + j] + dist(mt) * 1e-1; - } - } - raft::copy(query_ptr, host_dataset.data_handle(), query_size * dim, cuda_stream); -} -} // namespace - -namespace raft::neighbors::cagra { -struct AnnCagraVpqInputs { - int n_queries; - int n_rows; - int dim; - int k; - int pq_len; - int pq_bits; - graph_build_algo build_algo; - search_algo algo; - int max_queries; - int team_size; - int itopk_size; - int search_width; - raft::distance::DistanceType metric; - bool host_dataset; - bool include_serialized_dataset; - // std::optional - double min_recall; // = std::nullopt; -}; - -inline ::std::ostream& operator<<(::std::ostream& os, const AnnCagraVpqInputs& p) -{ - std::vector algo = {"single-cta", "multi_cta", "multi_kernel", "auto"}; - std::vector build_algo = {"IVF_PQ", "NN_DESCENT"}; - os << "{n_queries=" << p.n_queries << ", dataset shape=" << p.n_rows << "x" << p.dim - << ", k=" << p.k << ", pq_bits=" << p.pq_bits << ", pq_len=" << p.pq_len << ", " - << algo.at((int)p.algo) << ", max_queries=" << p.max_queries << ", itopk_size=" << p.itopk_size - << ", search_width=" << p.search_width << ", metric=" << static_cast(p.metric) - << (p.host_dataset ? ", host" : ", device") - << ", build_algo=" << build_algo.at((int)p.build_algo) << '}' << std::endl; - return os; -} - -template -class AnnCagraVpqTest : public ::testing::TestWithParam { - public: - AnnCagraVpqTest() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam::GetParam()), - database(0, stream_), - search_queries(0, stream_) - { - } - - protected: - void testCagra() - { - size_t queries_size = ps.n_queries * ps.k; - std::vector indices_Cagra(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_Cagra(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database.data(), - ps.n_queries, - ps.n_rows, - ps.dim, - ps.k, - ps.metric); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - const auto vpq_k = ps.k * 4; - { - rmm::device_uvector distances_dev(vpq_k * ps.n_queries, stream_); - rmm::device_uvector indices_dev(vpq_k * ps.n_queries, stream_); - - { - if ((ps.dim % ps.pq_len) != 0) { - // TODO: remove this requirement in the algorithm. - GTEST_SKIP() << "(TODO) At the moment dim, (" << ps.dim - << ") must be a multiple of pq_len (" << ps.pq_len << ")"; - } - cagra::index_params index_params; - index_params.compression = vpq_params{.pq_bits = static_cast(ps.pq_bits), - .pq_dim = static_cast(ps.dim / ps.pq_len)}; - index_params.metric = ps.metric; // Note: currently ony the cagra::index_params metric is - // not used for knn_graph building. - index_params.build_algo = ps.build_algo; - cagra::search_params search_params; - search_params.algo = ps.algo; - search_params.max_queries = ps.max_queries; - search_params.team_size = ps.team_size; - search_params.itopk_size = ps.itopk_size; - - auto database_view = - raft::make_device_matrix_view(database.data(), ps.n_rows, ps.dim); - - { - cagra::index index(handle_); - if (ps.host_dataset) { - auto database_host = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy(database_host.data_handle(), database.data(), database.size(), stream_); - auto database_host_view = raft::make_host_matrix_view( - database_host.data_handle(), ps.n_rows, ps.dim); - index = cagra::build(handle_, index_params, database_host_view); - } else { - index = cagra::build(handle_, index_params, database_view); - }; - cagra::serialize(handle_, "cagra_index", index, ps.include_serialized_dataset); - } - - auto index = cagra::deserialize(handle_, "cagra_index"); - if (!ps.include_serialized_dataset) { index.update_dataset(handle_, database_view); } - - // CAGRA-Q sanity check: we've built the right index type - auto* vpq_dataset = - dynamic_cast*>(&index.data()); - EXPECT_NE(vpq_dataset, nullptr) - << "Expected VPQ dataset, because we're testing CAGRA-Q here."; - - auto search_queries_view = raft::make_device_matrix_view( - search_queries.data(), ps.n_queries, ps.dim); - auto indices_out_view = - raft::make_device_matrix_view(indices_dev.data(), ps.n_queries, vpq_k); - auto dists_out_view = raft::make_device_matrix_view( - distances_dev.data(), ps.n_queries, vpq_k); - - cagra::search( - handle_, search_params, index, search_queries_view, indices_out_view, dists_out_view); - - { - auto host_dataset = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy( - host_dataset.data_handle(), (const DataT*)database.data(), ps.n_rows * ps.dim, stream_); - - auto host_queries = raft::make_host_matrix(ps.n_queries, ps.dim); - raft::copy(host_queries.data_handle(), - (const DataT*)search_queries_view.data_handle(), - ps.n_queries * ps.dim, - stream_); - - auto host_index_candidate = raft::make_host_matrix(ps.n_queries, vpq_k); - raft::copy(host_index_candidate.data_handle(), - indices_out_view.data_handle(), - ps.n_queries * vpq_k, - stream_); - - auto host_indices_Cagra_view = - raft::make_host_matrix_view(indices_Cagra.data(), ps.n_queries, ps.k); - - auto host_dists_Cagra_view = - raft::make_host_matrix_view(distances_Cagra.data(), ps.n_queries, ps.k); - - resource::sync_stream(handle_); - - raft::neighbors::refine(handle_, - raft::make_const_mdspan(host_dataset.view()), - raft::make_const_mdspan(host_queries.view()), - raft::make_const_mdspan(host_index_candidate.view()), - host_indices_Cagra_view, - host_dists_Cagra_view, - ps.metric); - - raft::copy(indices_dev.data(), - host_indices_Cagra_view.data_handle(), - ps.k * ps.n_queries, - stream_); - raft::copy(distances_dev.data(), - host_dists_Cagra_view.data_handle(), - ps.k * ps.n_queries, - stream_); - resource::sync_stream(handle_); - } - } - - double min_recall = ps.min_recall; - EXPECT_TRUE(eval_neighbours(indices_naive, - indices_Cagra, - distances_naive, - distances_Cagra, - ps.n_queries, - ps.k, - 0.003, - min_recall)); - EXPECT_TRUE(eval_distances(handle_, - database.data(), - search_queries.data(), - indices_dev.data(), - distances_dev.data(), - ps.n_rows, - ps.dim, - ps.n_queries, - ps.k, - ps.metric, - 1.0e-4)); - } - } - - void SetUp() override - { - database.resize(((size_t)ps.n_rows) * ps.dim, stream_); - search_queries.resize(ps.n_queries * ps.dim, stream_); - GenerateDataset(database.data(), - search_queries.data(), - ps.n_rows, - ps.n_queries, - ps.dim, - static_cast(std::sqrt(ps.n_rows)), - stream_); - resource::sync_stream(handle_); - } - - void TearDown() override - { - resource::sync_stream(handle_); - database.resize(0, stream_); - search_queries.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - AnnCagraVpqInputs ps; - rmm::device_uvector database; - rmm::device_uvector search_queries; -}; - -const std::vector vpq_inputs = raft::util::itertools::product( - {100}, // n_queries - {1000, 10000}, // n_rows - {128, 132, 192, 256, 512, 768}, // dim - {8, 12}, // k - {2, 4}, // pq_len - {8}, // pq_bits - {graph_build_algo::NN_DESCENT}, // build_algo - {search_algo::SINGLE_CTA, search_algo::MULTI_CTA}, // algo - {0}, // max_queries - {0}, // team_size - {512}, // itopk_size - {1}, // search_width - {raft::distance::DistanceType::L2Expanded}, // metric - {false}, // host_dataset - {true}, // include_serialized_dataset - {0.8} // min_recall -); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_cagra_vpq/test_float_uint32_t.cu b/cpp/test/neighbors/ann_cagra_vpq/test_float_uint32_t.cu deleted file mode 100644 index 19d3f32250..0000000000 --- a/cpp/test/neighbors/ann_cagra_vpq/test_float_uint32_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_cagra_vpq.cuh" - -#include - -namespace raft::neighbors::cagra { - -typedef AnnCagraVpqTest AnnCagraVpqTestF_U32; -TEST_P(AnnCagraVpqTestF_U32, AnnCagraVpq) { this->testCagra(); } - -INSTANTIATE_TEST_CASE_P(AnnCagraVpqTest, AnnCagraVpqTestF_U32, ::testing::ValuesIn(vpq_inputs)); - -} // namespace raft::neighbors::cagra diff --git a/cpp/test/neighbors/ann_ivf_flat.cuh b/cpp/test/neighbors/ann_ivf_flat.cuh deleted file mode 100644 index de6af589fa..0000000000 --- a/cpp/test/neighbors/ann_ivf_flat.cuh +++ /dev/null @@ -1,675 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../test_utils.cuh" -#include "ann_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -#include -#include - -#include - -#include -#include -#include - -namespace raft::neighbors::ivf_flat { - -struct test_ivf_sample_filter { - static constexpr unsigned offset = 300; -}; - -template -struct AnnIvfFlatInputs { - IdxT num_queries; - IdxT num_db_vecs; - IdxT dim; - IdxT k; - IdxT nprobe; - IdxT nlist; - raft::distance::DistanceType metric; - bool adaptive_centers; - bool host_dataset; -}; - -template -::std::ostream& operator<<(::std::ostream& os, const AnnIvfFlatInputs& p) -{ - os << "{ " << p.num_queries << ", " << p.num_db_vecs << ", " << p.dim << ", " << p.k << ", " - << p.nprobe << ", " << p.nlist << ", " << static_cast(p.metric) << ", " - << p.adaptive_centers << ", " << p.host_dataset << '}' << std::endl; - return os; -} - -template -class AnnIVFFlatTest : public ::testing::TestWithParam> { - public: - AnnIVFFlatTest() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam>::GetParam()), - database(0, stream_), - search_queries(0, stream_) - { - } - - void testIVFFlat() - { - size_t queries_size = ps.num_queries * ps.k; - std::vector indices_ivfflat(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_ivfflat(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database.data(), - ps.num_queries, - ps.num_db_vecs, - ps.dim, - ps.k, - ps.metric); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - { - // unless something is really wrong with clustering, this could serve as a lower bound on - // recall - double min_recall = static_cast(ps.nprobe) / static_cast(ps.nlist); - - rmm::device_uvector distances_ivfflat_dev(queries_size, stream_); - rmm::device_uvector indices_ivfflat_dev(queries_size, stream_); - - { - // legacy interface - raft::spatial::knn::IVFFlatParam ivfParams; - ivfParams.nprobe = ps.nprobe; - ivfParams.nlist = ps.nlist; - raft::spatial::knn::knnIndex index; - - approx_knn_build_index(handle_, - &index, - dynamic_cast(&ivfParams), - ps.metric, - (IdxT)0, - database.data(), - ps.num_db_vecs, - ps.dim); - - resource::sync_stream(handle_); - approx_knn_search(handle_, - distances_ivfflat_dev.data(), - indices_ivfflat_dev.data(), - &index, - ps.k, - search_queries.data(), - ps.num_queries); - - update_host(distances_ivfflat.data(), distances_ivfflat_dev.data(), queries_size, stream_); - update_host(indices_ivfflat.data(), indices_ivfflat_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - ASSERT_TRUE(eval_neighbours(indices_naive, - indices_ivfflat, - distances_naive, - distances_ivfflat, - ps.num_queries, - ps.k, - 0.001, - min_recall)); - { - ivf_flat::index_params index_params; - ivf_flat::search_params search_params; - index_params.n_lists = ps.nlist; - index_params.metric = ps.metric; - index_params.adaptive_centers = ps.adaptive_centers; - search_params.n_probes = ps.nprobe; - - index_params.add_data_on_build = false; - index_params.kmeans_trainset_fraction = 0.5; - index_params.metric_arg = 0; - - ivf_flat::index idx(handle_, index_params, ps.dim); - ivf_flat::index index_2(handle_, index_params, ps.dim); - - if (!ps.host_dataset) { - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.num_db_vecs, ps.dim); - idx = ivf_flat::build(handle_, index_params, database_view); - rmm::device_uvector vector_indices(ps.num_db_vecs, stream_); - thrust::sequence(resource::get_thrust_policy(handle_), - thrust::device_pointer_cast(vector_indices.data()), - thrust::device_pointer_cast(vector_indices.data() + ps.num_db_vecs)); - resource::sync_stream(handle_); - - IdxT half_of_data = ps.num_db_vecs / 2; - - auto half_of_data_view = raft::make_device_matrix_view( - (const DataT*)database.data(), half_of_data, ps.dim); - - const std::optional> no_opt = std::nullopt; - index_2 = ivf_flat::extend(handle_, half_of_data_view, no_opt, idx); - - auto new_half_of_data_view = raft::make_device_matrix_view( - database.data() + half_of_data * ps.dim, IdxT(ps.num_db_vecs) - half_of_data, ps.dim); - - auto new_half_of_data_indices_view = raft::make_device_vector_view( - vector_indices.data() + half_of_data, IdxT(ps.num_db_vecs) - half_of_data); - - ivf_flat::extend(handle_, - new_half_of_data_view, - std::make_optional>( - new_half_of_data_indices_view), - &index_2); - - } else { - auto host_database = raft::make_host_matrix(ps.num_db_vecs, ps.dim); - raft::copy( - host_database.data_handle(), database.data(), ps.num_db_vecs * ps.dim, stream_); - idx = - ivf_flat::build(handle_, index_params, raft::make_const_mdspan(host_database.view())); - - auto vector_indices = raft::make_host_vector(handle_, ps.num_db_vecs); - std::iota(vector_indices.data_handle(), vector_indices.data_handle() + ps.num_db_vecs, 0); - - IdxT half_of_data = ps.num_db_vecs / 2; - - auto half_of_data_view = raft::make_host_matrix_view( - (const DataT*)host_database.data_handle(), half_of_data, ps.dim); - - const std::optional> no_opt = std::nullopt; - index_2 = ivf_flat::extend(handle_, half_of_data_view, no_opt, idx); - - auto new_half_of_data_view = raft::make_host_matrix_view( - host_database.data_handle() + half_of_data * ps.dim, - IdxT(ps.num_db_vecs) - half_of_data, - ps.dim); - auto new_half_of_data_indices_view = raft::make_host_vector_view( - vector_indices.data_handle() + half_of_data, IdxT(ps.num_db_vecs) - half_of_data); - ivf_flat::extend(handle_, - new_half_of_data_view, - std::make_optional>( - new_half_of_data_indices_view), - &index_2); - } - - auto search_queries_view = raft::make_device_matrix_view( - search_queries.data(), ps.num_queries, ps.dim); - auto indices_out_view = raft::make_device_matrix_view( - indices_ivfflat_dev.data(), ps.num_queries, ps.k); - auto dists_out_view = raft::make_device_matrix_view( - distances_ivfflat_dev.data(), ps.num_queries, ps.k); - ivf_flat::detail::serialize(handle_, "ivf_flat_index", index_2); - - auto index_loaded = ivf_flat::detail::deserialize(handle_, "ivf_flat_index"); - ASSERT_EQ(index_2.size(), index_loaded.size()); - - ivf_flat::search(handle_, - search_params, - index_loaded, - search_queries_view, - indices_out_view, - dists_out_view); - - update_host(distances_ivfflat.data(), distances_ivfflat_dev.data(), queries_size, stream_); - update_host(indices_ivfflat.data(), indices_ivfflat_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - - // Test the centroid invariants - if (index_2.adaptive_centers()) { - // The centers must be up-to-date with the corresponding data - std::vector list_sizes(index_2.n_lists()); - std::vector list_indices(index_2.n_lists()); - rmm::device_uvector centroid(ps.dim, stream_); - raft::copy( - list_sizes.data(), index_2.list_sizes().data_handle(), index_2.n_lists(), stream_); - raft::copy( - list_indices.data(), index_2.inds_ptrs().data_handle(), index_2.n_lists(), stream_); - resource::sync_stream(handle_); - for (uint32_t l = 0; l < index_2.n_lists(); l++) { - if (list_sizes[l] == 0) continue; - rmm::device_uvector cluster_data(list_sizes[l] * ps.dim, stream_); - raft::spatial::knn::detail::utils::copy_selected((IdxT)list_sizes[l], - (IdxT)ps.dim, - database.data(), - list_indices[l], - (IdxT)ps.dim, - cluster_data.data(), - (IdxT)ps.dim, - stream_); - raft::stats::mean( - centroid.data(), cluster_data.data(), ps.dim, list_sizes[l], false, true, stream_); - ASSERT_TRUE(raft::devArrMatch(index_2.centers().data_handle() + ps.dim * l, - centroid.data(), - ps.dim, - raft::CompareApprox(0.001), - stream_)); - } - } else { - // The centers must be immutable - ASSERT_TRUE(raft::devArrMatch(index_2.centers().data_handle(), - idx.centers().data_handle(), - index_2.centers().size(), - raft::Compare(), - stream_)); - } - } - ASSERT_TRUE(eval_neighbours(indices_naive, - indices_ivfflat, - distances_naive, - distances_ivfflat, - ps.num_queries, - ps.k, - 0.001, - min_recall)); - } - } - - void testPacker() - { - ivf_flat::index_params index_params; - ivf_flat::search_params search_params; - index_params.n_lists = ps.nlist; - index_params.metric = ps.metric; - index_params.adaptive_centers = false; - search_params.n_probes = ps.nprobe; - - index_params.add_data_on_build = false; - index_params.kmeans_trainset_fraction = 1.0; - index_params.metric_arg = 0; - - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.num_db_vecs, ps.dim); - - auto idx = ivf_flat::build(handle_, index_params, database_view); - - const std::optional> no_opt = std::nullopt; - index extend_index = ivf_flat::extend(handle_, database_view, no_opt, idx); - - auto list_sizes = raft::make_host_vector(idx.n_lists()); - update_host(list_sizes.data_handle(), - extend_index.list_sizes().data_handle(), - extend_index.n_lists(), - stream_); - resource::sync_stream(handle_); - - auto& lists = idx.lists(); - - // conservative memory allocation for codepacking - auto list_device_spec = list_spec{idx.dim(), false}; - - for (uint32_t label = 0; label < idx.n_lists(); label++) { - uint32_t list_size = list_sizes.data_handle()[label]; - - ivf::resize_list(handle_, lists[label], list_device_spec, list_size, 0); - } - - helpers::recompute_internal_state(handle_, &idx); - - using interleaved_group = Pow2; - - for (uint32_t label = 0; label < idx.n_lists(); label++) { - uint32_t list_size = list_sizes.data_handle()[label]; - - if (list_size > 0) { - uint32_t padded_list_size = interleaved_group::roundUp(list_size); - uint32_t n_elems = padded_list_size * idx.dim(); - auto list_data = lists[label]->data; - auto list_inds = extend_index.lists()[label]->indices; - - // fetch the flat codes - auto flat_codes = make_device_matrix(handle_, list_size, idx.dim()); - - matrix::gather( - handle_, - make_device_matrix_view( - (const DataT*)database.data(), static_cast(ps.num_db_vecs), idx.dim()), - make_device_vector_view((const IdxT*)list_inds.data_handle(), - list_size), - flat_codes.view()); - - helpers::codepacker::pack( - handle_, make_const_mdspan(flat_codes.view()), idx.veclen(), 0, list_data.view()); - - { - auto mask = make_device_vector(handle_, n_elems); - - linalg::map_offset(handle_, - mask.view(), - [dim = idx.dim(), - list_size, - padded_list_size, - chunk_size = util::FastIntDiv(idx.veclen())] __device__(auto i) { - uint32_t max_group_offset = interleaved_group::roundDown(list_size); - if (i < max_group_offset * dim) { return true; } - uint32_t surplus = (i - max_group_offset * dim); - uint32_t ingroup_id = interleaved_group::mod(surplus / chunk_size); - return ingroup_id < (list_size - max_group_offset); - }); - - // ensure that the correct number of indices are masked out - ASSERT_TRUE(thrust::reduce(resource::get_thrust_policy(handle_), - mask.data_handle(), - mask.data_handle() + n_elems, - 0) == list_size * ps.dim); - - auto packed_list_data = make_device_vector(handle_, n_elems); - - linalg::map_offset(handle_, - packed_list_data.view(), - [mask = mask.data_handle(), - list_data = list_data.data_handle()] __device__(uint32_t i) { - if (mask[i]) return list_data[i]; - return DataT{0}; - }); - - auto extend_data = extend_index.lists()[label]->data; - auto extend_data_filtered = make_device_vector(handle_, n_elems); - linalg::map_offset(handle_, - extend_data_filtered.view(), - [mask = mask.data_handle(), - extend_data = extend_data.data_handle()] __device__(uint32_t i) { - if (mask[i]) return extend_data[i]; - return DataT{0}; - }); - - ASSERT_TRUE(raft::devArrMatch(packed_list_data.data_handle(), - extend_data_filtered.data_handle(), - n_elems, - raft::Compare(), - stream_)); - } - - auto unpacked_flat_codes = - make_device_matrix(handle_, list_size, idx.dim()); - - helpers::codepacker::unpack( - handle_, list_data.view(), idx.veclen(), 0, unpacked_flat_codes.view()); - - ASSERT_TRUE(raft::devArrMatch(flat_codes.data_handle(), - unpacked_flat_codes.data_handle(), - list_size * ps.dim, - raft::Compare(), - stream_)); - } - } - } - - void testFilter() - { - size_t queries_size = ps.num_queries * ps.k; - std::vector indices_ivfflat(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_ivfflat(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - auto* database_filtered_ptr = database.data() + test_ivf_sample_filter::offset * ps.dim; - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database_filtered_ptr, - ps.num_queries, - ps.num_db_vecs - test_ivf_sample_filter::offset, - ps.dim, - ps.k, - ps.metric); - raft::linalg::addScalar(indices_naive_dev.data(), - indices_naive_dev.data(), - IdxT(test_ivf_sample_filter::offset), - queries_size, - stream_); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - { - // unless something is really wrong with clustering, this could serve as a lower bound on - // recall - double min_recall = static_cast(ps.nprobe) / static_cast(ps.nlist); - - auto distances_ivfflat_dev = raft::make_device_matrix(handle_, ps.num_queries, ps.k); - auto indices_ivfflat_dev = - raft::make_device_matrix(handle_, ps.num_queries, ps.k); - - { - ivf_flat::index_params index_params; - ivf_flat::search_params search_params; - index_params.n_lists = ps.nlist; - index_params.metric = ps.metric; - index_params.adaptive_centers = ps.adaptive_centers; - search_params.n_probes = ps.nprobe; - - index_params.add_data_on_build = true; - index_params.kmeans_trainset_fraction = 0.5; - index_params.metric_arg = 0; - - // Create IVF Flat index - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.num_db_vecs, ps.dim); - auto index = ivf_flat::build(handle_, index_params, database_view); - - // Create Bitset filter - auto removed_indices = - raft::make_device_vector(handle_, test_ivf_sample_filter::offset); - thrust::sequence(resource::get_thrust_policy(handle_), - thrust::device_pointer_cast(removed_indices.data_handle()), - thrust::device_pointer_cast(removed_indices.data_handle() + - test_ivf_sample_filter::offset)); - resource::sync_stream(handle_); - - raft::core::bitset removed_indices_bitset( - handle_, removed_indices.view(), ps.num_db_vecs); - - // Search with the filter - auto search_queries_view = raft::make_device_matrix_view( - search_queries.data(), ps.num_queries, ps.dim); - ivf_flat::search_with_filtering( - handle_, - search_params, - index, - search_queries_view, - indices_ivfflat_dev.view(), - distances_ivfflat_dev.view(), - raft::neighbors::filtering::bitset_filter(removed_indices_bitset.view())); - - update_host( - distances_ivfflat.data(), distances_ivfflat_dev.data_handle(), queries_size, stream_); - update_host( - indices_ivfflat.data(), indices_ivfflat_dev.data_handle(), queries_size, stream_); - resource::sync_stream(handle_); - } - ASSERT_TRUE(eval_neighbours(indices_naive, - indices_ivfflat, - distances_naive, - distances_ivfflat, - ps.num_queries, - ps.k, - 0.001, - min_recall)); - } - } - - void SetUp() override - { - database.resize(ps.num_db_vecs * ps.dim, stream_); - search_queries.resize(ps.num_queries * ps.dim, stream_); - - raft::random::RngState r(1234ULL); - if constexpr (std::is_same{}) { - raft::random::uniform( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(0.1), DataT(2.0)); - raft::random::uniform( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(0.1), DataT(2.0)); - } else { - raft::random::uniformInt( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(1), DataT(20)); - raft::random::uniformInt( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(1), DataT(20)); - } - resource::sync_stream(handle_); - } - - void TearDown() override - { - resource::sync_stream(handle_); - database.resize(0, stream_); - search_queries.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - AnnIvfFlatInputs ps; - rmm::device_uvector database; - rmm::device_uvector search_queries; -}; - -const std::vector> inputs = { - // test various dims (aligned and not aligned to vector sizes) - {1000, 10000, 1, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 2, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 3, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 4, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 5, 16, 40, 1024, raft::distance::DistanceType::InnerProduct, false}, - {1000, 10000, 8, 16, 40, 1024, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 5, 16, 40, 1024, raft::distance::DistanceType::L2SqrtExpanded, false}, - {1000, 10000, 8, 16, 40, 1024, raft::distance::DistanceType::L2SqrtExpanded, true}, - - // test dims that do not fit into kernel shared memory limits - {1000, 10000, 2048, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 2049, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 2050, 16, 40, 1024, raft::distance::DistanceType::InnerProduct, false}, - {1000, 10000, 2051, 16, 40, 1024, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 2052, 16, 40, 1024, raft::distance::DistanceType::InnerProduct, false}, - {1000, 10000, 2053, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, true}, - {1000, 10000, 2056, 16, 40, 1024, raft::distance::DistanceType::L2Expanded, true}, - - // various random combinations - {1000, 10000, 16, 10, 40, 1024, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 16, 10, 50, 1024, raft::distance::DistanceType::L2Expanded, false}, - {1000, 10000, 16, 10, 70, 1024, raft::distance::DistanceType::L2Expanded, false}, - {100, 10000, 16, 10, 20, 512, raft::distance::DistanceType::L2Expanded, false}, - {20, 100000, 16, 10, 20, 1024, raft::distance::DistanceType::L2Expanded, true}, - {1000, 100000, 16, 10, 20, 1024, raft::distance::DistanceType::L2Expanded, true}, - {10000, 131072, 8, 10, 20, 1024, raft::distance::DistanceType::L2Expanded, false}, - - // various combinations with k>raft::matrix::detail::select::warpsort::kMaxCapacity - {1000, 10000, 16, 1024, 40, 1024, raft::distance::DistanceType::L2SqrtExpanded, true}, - {1000, 10000, 2053, 512, 50, 1024, raft::distance::DistanceType::L2SqrtExpanded, false}, - {1000, 10000, 2049, 2048, 70, 1024, raft::distance::DistanceType::L2SqrtExpanded, false}, - {1000, 10000, 16, 4000, 100, 2048, raft::distance::DistanceType::L2SqrtExpanded, false}, - {10, 10000, 16, 4000, 100, 2048, raft::distance::DistanceType::L2SqrtExpanded, false}, - {10, 10000, 16, 4000, 120, 2048, raft::distance::DistanceType::L2SqrtExpanded, true}, - {20, 100000, 16, 257, 20, 1024, raft::distance::DistanceType::L2SqrtExpanded, true}, - {1000, 100000, 16, 259, 20, 1024, raft::distance::DistanceType::L2Expanded, true, true}, - {10000, 131072, 8, 280, 20, 1024, raft::distance::DistanceType::InnerProduct, false}, - {100000, 1024, 32, 257, 64, 64, raft::distance::DistanceType::L2Expanded, false}, - {100000, 1024, 32, 257, 64, 64, raft::distance::DistanceType::L2SqrtExpanded, false}, - {100000, 1024, 32, 257, 64, 64, raft::distance::DistanceType::InnerProduct, false}, - {100000, 1024, 16, 300, 20, 60, raft::distance::DistanceType::L2Expanded, false}, - {100000, 1024, 16, 500, 20, 60, raft::distance::DistanceType::L2SqrtExpanded, false}, - {100000, 1024, 16, 700, 20, 60, raft::distance::DistanceType::InnerProduct, false}, - - // host input data - {1000, 10000, 16, 10, 40, 1024, raft::distance::DistanceType::L2Expanded, false, true}, - {1000, 10000, 16, 10, 50, 1024, raft::distance::DistanceType::L2Expanded, false, true}, - {1000, 10000, 16, 10, 70, 1024, raft::distance::DistanceType::L2Expanded, false, true}, - {100, 10000, 16, 10, 20, 512, raft::distance::DistanceType::L2Expanded, false, true}, - {20, 100000, 16, 10, 20, 1024, raft::distance::DistanceType::L2Expanded, false, true}, - {1000, 100000, 16, 10, 20, 1024, raft::distance::DistanceType::L2Expanded, false, true}, - {10000, 131072, 8, 10, 20, 1024, raft::distance::DistanceType::L2Expanded, false, true}, - - {1000, 10000, 16, 10, 40, 1024, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 16, 10, 50, 1024, raft::distance::DistanceType::InnerProduct, true}, - {1000, 10000, 16, 10, 70, 1024, raft::distance::DistanceType::InnerProduct, false}, - {100, 10000, 16, 10, 20, 512, raft::distance::DistanceType::InnerProduct, true}, - {20, 100000, 16, 10, 20, 1024, raft::distance::DistanceType::InnerProduct, true}, - {1000, 100000, 16, 10, 20, 1024, raft::distance::DistanceType::InnerProduct, false}, - {10000, 131072, 8, 10, 50, 1024, raft::distance::DistanceType::InnerProduct, true}, - - {1000, 10000, 4096, 20, 50, 1024, raft::distance::DistanceType::InnerProduct, false}, - - // test splitting the big query batches (> max gridDim.y) into smaller batches - {100000, 1024, 32, 10, 64, 64, raft::distance::DistanceType::InnerProduct, false}, - {1000000, 1024, 32, 10, 256, 256, raft::distance::DistanceType::InnerProduct, false}, - {98306, 1024, 32, 10, 64, 64, raft::distance::DistanceType::InnerProduct, true}, - - // test radix_sort for getting the cluster selection - {1000, - 10000, - 16, - 10, - raft::matrix::detail::select::warpsort::kMaxCapacity * 2, - raft::matrix::detail::select::warpsort::kMaxCapacity * 4, - raft::distance::DistanceType::L2Expanded, - false}, - {1000, - 10000, - 16, - 10, - raft::matrix::detail::select::warpsort::kMaxCapacity * 4, - raft::matrix::detail::select::warpsort::kMaxCapacity * 4, - raft::distance::DistanceType::InnerProduct, - false}, - - // The following two test cases should show very similar recall. - // num_queries, num_db_vecs, dim, k, nprobe, nlist, metric, adaptive_centers - {20000, 8712, 3, 10, 51, 66, raft::distance::DistanceType::L2Expanded, false}, - {100000, 8712, 3, 10, 51, 66, raft::distance::DistanceType::L2Expanded, false}}; - -} // namespace raft::neighbors::ivf_flat diff --git a/cpp/test/neighbors/ann_ivf_flat/test_filter_float_int64_t.cu b/cpp/test/neighbors/ann_ivf_flat/test_filter_float_int64_t.cu deleted file mode 100644 index 0e1036e566..0000000000 --- a/cpp/test/neighbors/ann_ivf_flat/test_filter_float_int64_t.cu +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#undef RAFT_EXPLICIT_INSTANTIATE_ONLY // Enable instantiation of search with filter -#include "../ann_ivf_flat.cuh" - -namespace raft::neighbors::ivf_flat { - -typedef AnnIVFFlatTest AnnIVFFlatFilterTestF; -TEST_P(AnnIVFFlatFilterTestF, AnnIVFFlatFilter) { this->testFilter(); } - -INSTANTIATE_TEST_CASE_P(AnnIVFFlatTest, AnnIVFFlatFilterTestF, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::ivf_flat diff --git a/cpp/test/neighbors/ann_ivf_flat/test_float_int64_t.cu b/cpp/test/neighbors/ann_ivf_flat/test_float_int64_t.cu deleted file mode 100644 index 2ff17b8536..0000000000 --- a/cpp/test/neighbors/ann_ivf_flat/test_float_int64_t.cu +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_flat.cuh" - -#include - -namespace raft::neighbors::ivf_flat { - -typedef AnnIVFFlatTest AnnIVFFlatTestF; -TEST_P(AnnIVFFlatTestF, AnnIVFFlat) -{ - this->testIVFFlat(); - this->testPacker(); -} - -INSTANTIATE_TEST_CASE_P(AnnIVFFlatTest, AnnIVFFlatTestF, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::ivf_flat diff --git a/cpp/test/neighbors/ann_ivf_flat/test_int8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_flat/test_int8_t_int64_t.cu deleted file mode 100644 index 6fe12506aa..0000000000 --- a/cpp/test/neighbors/ann_ivf_flat/test_int8_t_int64_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_flat.cuh" - -#include - -namespace raft::neighbors::ivf_flat { - -typedef AnnIVFFlatTest AnnIVFFlatTestF_int8; -TEST_P(AnnIVFFlatTestF_int8, AnnIVFFlat) { this->testIVFFlat(); } - -INSTANTIATE_TEST_CASE_P(AnnIVFFlatTest, AnnIVFFlatTestF_int8, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::ivf_flat diff --git a/cpp/test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu deleted file mode 100644 index ab6001c71b..0000000000 --- a/cpp/test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_flat.cuh" - -#include - -namespace raft::neighbors::ivf_flat { - -typedef AnnIVFFlatTest AnnIVFFlatTestF_uint8; -TEST_P(AnnIVFFlatTestF_uint8, AnnIVFFlat) { this->testIVFFlat(); } - -INSTANTIATE_TEST_CASE_P(AnnIVFFlatTest, AnnIVFFlatTestF_uint8, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::ivf_flat diff --git a/cpp/test/neighbors/ann_ivf_pq.cuh b/cpp/test/neighbors/ann_ivf_pq.cuh deleted file mode 100644 index 4ebe02027f..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq.cuh +++ /dev/null @@ -1,1095 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "../test_utils.cuh" -#include "ann_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include - -#include -#include - -#include - -#include -#include -#include -#include -#include - -namespace raft::neighbors::ivf_pq { - -struct test_ivf_sample_filter { - static constexpr unsigned offset = 1500; -}; - -struct ivf_pq_inputs { - uint32_t num_db_vecs = 4096; - uint32_t num_queries = 1024; - uint32_t dim = 64; - uint32_t k = 32; - std::optional min_recall = std::nullopt; - - ivf_pq::index_params index_params; - ivf_pq::search_params search_params; - - // Set some default parameters for tests - ivf_pq_inputs() - { - index_params.n_lists = max(32u, min(1024u, num_db_vecs / 128u)); - index_params.kmeans_trainset_fraction = 1.0; - } -}; - -inline auto operator<<(std::ostream& os, const ivf_pq::codebook_gen& p) -> std::ostream& -{ - switch (p) { - case ivf_pq::codebook_gen::PER_CLUSTER: os << "codebook_gen::PER_CLUSTER"; break; - case ivf_pq::codebook_gen::PER_SUBSPACE: os << "codebook_gen::PER_SUBSPACE"; break; - default: RAFT_FAIL("unreachable code"); - } - return os; -} - -inline auto operator<<(std::ostream& os, const ivf_pq_inputs& p) -> std::ostream& -{ - ivf_pq_inputs dflt; - bool need_comma = false; -#define PRINT_DIFF_V(spec, val) \ - do { \ - if (dflt spec != p spec) { \ - if (need_comma) { os << ", "; } \ - os << #spec << " = " << val; \ - need_comma = true; \ - } \ - } while (0) -#define PRINT_DIFF(spec) PRINT_DIFF_V(spec, p spec) - - os << "ivf_pq_inputs {"; - PRINT_DIFF(.num_db_vecs); - PRINT_DIFF(.num_queries); - PRINT_DIFF(.dim); - PRINT_DIFF(.k); - PRINT_DIFF_V(.min_recall, p.min_recall.value_or(0)); - PRINT_DIFF_V(.index_params.metric, print_metric{p.index_params.metric}); - PRINT_DIFF(.index_params.metric_arg); - PRINT_DIFF(.index_params.add_data_on_build); - PRINT_DIFF(.index_params.n_lists); - PRINT_DIFF(.index_params.kmeans_n_iters); - PRINT_DIFF(.index_params.kmeans_trainset_fraction); - PRINT_DIFF(.index_params.pq_bits); - PRINT_DIFF(.index_params.pq_dim); - PRINT_DIFF(.index_params.codebook_kind); - PRINT_DIFF(.index_params.force_random_rotation); - PRINT_DIFF(.search_params.n_probes); - PRINT_DIFF_V(.search_params.lut_dtype, print_dtype{p.search_params.lut_dtype}); - PRINT_DIFF_V(.search_params.internal_distance_dtype, - print_dtype{p.search_params.internal_distance_dtype}); - os << "}"; - return os; -} - -template -void compare_vectors_l2( - const raft::resources& res, T a, T b, uint32_t label, double compression_ratio, double eps) -{ - auto n_rows = a.extent(0); - auto dim = a.extent(1); - rmm::mr::managed_memory_resource managed_memory; - auto dist = make_device_mdarray(res, &managed_memory, make_extents(n_rows)); - linalg::map_offset(res, dist.view(), [a, b, dim] __device__(uint32_t i) { - spatial::knn::detail::utils::mapping f{}; - double d = 0.0f; - for (uint32_t j = 0; j < dim; j++) { - double t = f(a(i, j)) - f(b(i, j)); - d += t * t; - } - return sqrt(d / double(dim)); - }); - resource::sync_stream(res); - for (uint32_t i = 0; i < n_rows; i++) { - double d = dist(i); - // The theoretical estimate of the error is hard to come up with, - // the estimate below is based on experimentation + curse of dimensionality - ASSERT_LE(d, 1.2 * eps * std::pow(2.0, compression_ratio)) - << " (label = " << label << ", ix = " << i << ", eps = " << eps << ")"; - } -} - -template -auto min_output_size(const raft::resources& handle, - const ivf_pq::index& index, - uint32_t n_probes) -> IdxT -{ - auto acc_sizes = index.accum_sorted_sizes(); - uint32_t last_nonzero = index.n_lists(); - while (last_nonzero > 0 && acc_sizes(last_nonzero - 1) == acc_sizes(last_nonzero)) { - last_nonzero--; - } - return acc_sizes(last_nonzero) - acc_sizes(last_nonzero - std::min(last_nonzero, n_probes)); -} - -template -class ivf_pq_test : public ::testing::TestWithParam { - public: - ivf_pq_test() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam::GetParam()), - database(0, stream_), - search_queries(0, stream_) - { - } - - void gen_data() - { - database.resize(size_t{ps.num_db_vecs} * size_t{ps.dim}, stream_); - search_queries.resize(size_t{ps.num_queries} * size_t{ps.dim}, stream_); - - raft::random::RngState r(1234ULL); - if constexpr (std::is_same{}) { - raft::random::uniform( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(0.1), DataT(2.0)); - raft::random::uniform( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(0.1), DataT(2.0)); - } else { - raft::random::uniformInt( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(1), DataT(20)); - raft::random::uniformInt( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(1), DataT(20)); - } - resource::sync_stream(handle_); - } - - void calc_ref() - { - size_t queries_size = size_t{ps.num_queries} * size_t{ps.k}; - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database.data(), - ps.num_queries, - ps.num_db_vecs, - ps.dim, - ps.k, - ps.index_params.metric); - distances_ref.resize(queries_size); - update_host(distances_ref.data(), distances_naive_dev.data(), queries_size, stream_); - indices_ref.resize(queries_size); - update_host(indices_ref.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - auto build_only() - { - auto ipams = ps.index_params; - ipams.add_data_on_build = true; - - auto index_view = - raft::make_device_matrix_view(database.data(), ps.num_db_vecs, ps.dim); - return ivf_pq::build(handle_, ipams, index_view); - } - - auto build_2_extends() - { - auto db_indices = make_device_vector(handle_, ps.num_db_vecs); - linalg::map_offset(handle_, db_indices.view(), identity_op{}); - resource::sync_stream(handle_); - auto size_1 = IdxT(ps.num_db_vecs) / 2; - auto size_2 = IdxT(ps.num_db_vecs) - size_1; - auto vecs_1 = database.data(); - auto vecs_2 = database.data() + size_t(size_1) * size_t(ps.dim); - auto inds_1 = db_indices.data_handle(); - auto inds_2 = db_indices.data_handle() + size_t(size_1); - - auto ipams = ps.index_params; - ipams.add_data_on_build = false; - - auto database_view = - raft::make_device_matrix_view(database.data(), ps.num_db_vecs, ps.dim); - auto idx = ivf_pq::build(handle_, ipams, database_view); - - auto vecs_2_view = raft::make_device_matrix_view(vecs_2, size_2, ps.dim); - auto inds_2_view = raft::make_device_vector_view(inds_2, size_2); - ivf_pq::extend(handle_, vecs_2_view, inds_2_view, &idx); - - auto vecs_1_view = - raft::make_device_matrix_view(vecs_1, size_1, ps.dim); - auto inds_1_view = raft::make_device_vector_view(inds_1, size_1); - ivf_pq::extend(handle_, vecs_1_view, inds_1_view, &idx); - return idx; - } - - auto build_serialize() - { - ivf_pq::serialize(handle_, "ivf_pq_index", build_only()); - return ivf_pq::deserialize(handle_, "ivf_pq_index"); - } - - void check_reconstruction(const index& index, - double compression_ratio, - uint32_t label, - uint32_t n_take, - uint32_t n_skip) - { - auto& rec_list = index.lists()[label]; - auto dim = index.dim(); - n_take = std::min(n_take, rec_list->size.load()); - n_skip = std::min(n_skip, rec_list->size.load() - n_take); - - if (n_take == 0) { return; } - - auto rec_data = make_device_matrix(handle_, n_take, dim); - auto orig_data = make_device_matrix(handle_, n_take, dim); - - ivf_pq::helpers::reconstruct_list_data(handle_, index, rec_data.view(), label, n_skip); - - matrix::gather(database.data(), - IdxT{dim}, - IdxT{n_take}, - rec_list->indices.data_handle() + n_skip, - IdxT{n_take}, - orig_data.data_handle(), - stream_); - - compare_vectors_l2(handle_, rec_data.view(), orig_data.view(), label, compression_ratio, 0.06); - } - - void check_reconstruct_extend(index* index, double compression_ratio, uint32_t label) - { - // NB: this is not reference, the list is retained; the index will have to create a new list on - // `erase_list` op. - auto old_list = index->lists()[label]; - auto n_rows = old_list->size.load(); - if (n_rows == 0) { return; } - - auto vectors_1 = make_device_matrix(handle_, n_rows, index->dim()); - auto indices = make_device_vector(handle_, n_rows); - copy(indices.data_handle(), old_list->indices.data_handle(), n_rows, stream_); - - ivf_pq::helpers::reconstruct_list_data(handle_, *index, vectors_1.view(), label, 0); - ivf_pq::helpers::erase_list(handle_, index, label); - // NB: passing the type parameter because const->non-const implicit conversion of the mdspans - // breaks type inference - ivf_pq::helpers::extend_list( - handle_, index, vectors_1.view(), indices.view(), label); - - auto& new_list = index->lists()[label]; - ASSERT_NE(old_list.get(), new_list.get()) - << "The old list should have been shared and retained after ivf_pq index has erased the " - "corresponding cluster."; - - auto vectors_2 = make_device_matrix(handle_, n_rows, index->dim()); - ivf_pq::helpers::reconstruct_list_data(handle_, *index, vectors_2.view(), label, 0); - // The code search is unstable, and there's high chance of repeating values of the lvl-2 codes. - // Hence, encoding-decoding chain often leads to altering both the PQ codes and the - // reconstructed data. - compare_vectors_l2( - handle_, vectors_1.view(), vectors_2.view(), label, compression_ratio, 0.04); // 0.025); - } - - void check_packing(index* index, uint32_t label) - { - auto old_list = index->lists()[label]; - auto n_rows = old_list->size.load(); - - if (n_rows == 0) { return; } - - auto codes = make_device_matrix(handle_, n_rows, index->pq_dim()); - auto indices = make_device_vector(handle_, n_rows); - copy(indices.data_handle(), old_list->indices.data_handle(), n_rows, stream_); - - ivf_pq::helpers::unpack_list_data(handle_, *index, codes.view(), label, 0); - ivf_pq::helpers::erase_list(handle_, index, label); - ivf_pq::helpers::extend_list_with_codes( - handle_, index, codes.view(), indices.view(), label); - - auto& new_list = index->lists()[label]; - ASSERT_NE(old_list.get(), new_list.get()) - << "The old list should have been shared and retained after ivf_pq index has erased the " - "corresponding cluster."; - auto list_data_size = (n_rows / ivf_pq::kIndexGroupSize) * new_list->data.extent(1) * - new_list->data.extent(2) * new_list->data.extent(3); - - ASSERT_TRUE(old_list->data.size() >= list_data_size); - ASSERT_TRUE(new_list->data.size() >= list_data_size); - ASSERT_TRUE(devArrMatch(old_list->data.data_handle(), - new_list->data.data_handle(), - list_data_size, - Compare{})); - - // Pack a few vectors back to the list. - int row_offset = 9; - int n_vec = 3; - ASSERT_TRUE(row_offset + n_vec < n_rows); - size_t offset = row_offset * index->pq_dim(); - auto codes_to_pack = make_device_matrix_view( - codes.data_handle() + offset, n_vec, index->pq_dim()); - ivf_pq::helpers::pack_list_data(handle_, index, codes_to_pack, label, row_offset); - ASSERT_TRUE(devArrMatch(old_list->data.data_handle(), - new_list->data.data_handle(), - list_data_size, - Compare{})); - - // Another test with the API that take list_data directly - auto list_data = index->lists()[label]->data.view(); - uint32_t n_take = 4; - ASSERT_TRUE(row_offset + n_take < n_rows); - auto codes2 = raft::make_device_matrix(handle_, n_take, index->pq_dim()); - ivf_pq::helpers::codepacker::unpack( - handle_, list_data, index->pq_bits(), row_offset, codes2.view()); - - // Write it back - ivf_pq::helpers::codepacker::pack( - handle_, make_const_mdspan(codes2.view()), index->pq_bits(), row_offset, list_data); - ASSERT_TRUE(devArrMatch(old_list->data.data_handle(), - new_list->data.data_handle(), - list_data_size, - Compare{})); - } - void check_packing_contiguous(index* index, uint32_t label) - { - auto old_list = index->lists()[label]; - auto n_rows = old_list->size.load(); - - if (n_rows == 0) { return; } - - auto codes = make_device_matrix(handle_, n_rows, index->pq_dim()); - auto indices = make_device_vector(handle_, n_rows); - copy(indices.data_handle(), old_list->indices.data_handle(), n_rows, stream_); - - uint32_t code_size = ceildiv(index->pq_dim() * index->pq_bits(), 8); - - auto codes_compressed = make_device_matrix(handle_, n_rows, code_size); - - ivf_pq::helpers::unpack_contiguous_list_data( - handle_, *index, codes_compressed.data_handle(), n_rows, label, 0); - ivf_pq::helpers::erase_list(handle_, index, label); - ivf_pq::detail::extend_list_prepare(handle_, index, make_const_mdspan(indices.view()), label); - ivf_pq::helpers::pack_contiguous_list_data( - handle_, index, codes_compressed.data_handle(), n_rows, label, 0); - ivf_pq::helpers::recompute_internal_state(handle_, index); - - auto& new_list = index->lists()[label]; - ASSERT_NE(old_list.get(), new_list.get()) - << "The old list should have been shared and retained after ivf_pq index has erased the " - "corresponding cluster."; - auto list_data_size = (n_rows / ivf_pq::kIndexGroupSize) * new_list->data.extent(1) * - new_list->data.extent(2) * new_list->data.extent(3); - - ASSERT_TRUE(old_list->data.size() >= list_data_size); - ASSERT_TRUE(new_list->data.size() >= list_data_size); - ASSERT_TRUE(devArrMatch(old_list->data.data_handle(), - new_list->data.data_handle(), - list_data_size, - Compare{})); - - // Pack a few vectors back to the list. - uint32_t row_offset = 9; - uint32_t n_vec = 3; - ASSERT_TRUE(row_offset + n_vec < n_rows); - size_t offset = row_offset * code_size; - auto codes_to_pack = make_device_matrix_view( - codes_compressed.data_handle() + offset, n_vec, index->pq_dim()); - ivf_pq::helpers::pack_contiguous_list_data( - handle_, index, codes_to_pack.data_handle(), n_vec, label, row_offset); - ASSERT_TRUE(devArrMatch(old_list->data.data_handle(), - new_list->data.data_handle(), - list_data_size, - Compare{})); - - // // Another test with the API that take list_data directly - auto list_data = index->lists()[label]->data.view(); - uint32_t n_take = 4; - ASSERT_TRUE(row_offset + n_take < n_rows); - auto codes2 = raft::make_device_matrix(handle_, n_take, code_size); - ivf_pq::helpers::codepacker::unpack_contiguous(handle_, - list_data, - index->pq_bits(), - row_offset, - n_take, - index->pq_dim(), - codes2.data_handle()); - - // Write it back - ivf_pq::helpers::codepacker::pack_contiguous(handle_, - codes2.data_handle(), - n_vec, - index->pq_dim(), - index->pq_bits(), - row_offset, - list_data); - ASSERT_TRUE(devArrMatch(old_list->data.data_handle(), - new_list->data.data_handle(), - list_data_size, - Compare{})); - } - - template - void run(BuildIndex build_index) - { - index index = build_index(); - - double compression_ratio = - static_cast(ps.dim * 8) / static_cast(index.pq_dim() * index.pq_bits()); - - for (uint32_t label = 0; label < index.n_lists(); label++) { - switch (label % 3) { - case 0: { - // Reconstruct and re-write vectors for one label - check_reconstruct_extend(&index, compression_ratio, label); - } break; - case 1: { - // Dump and re-write codes for one label - check_packing(&index, label); - check_packing_contiguous(&index, label); - } break; - default: { - // check a small subset of data in a randomly chosen cluster to see if the data - // reconstruction works well. - check_reconstruction(index, compression_ratio, label, 100, 7); - } - } - } - - size_t queries_size = ps.num_queries * ps.k; - std::vector indices_ivf_pq(queries_size); - std::vector distances_ivf_pq(queries_size); - - rmm::device_uvector distances_ivf_pq_dev(queries_size, stream_); - rmm::device_uvector indices_ivf_pq_dev(queries_size, stream_); - - auto query_view = - raft::make_device_matrix_view(search_queries.data(), ps.num_queries, ps.dim); - auto inds_view = raft::make_device_matrix_view( - indices_ivf_pq_dev.data(), ps.num_queries, ps.k); - auto dists_view = raft::make_device_matrix_view( - distances_ivf_pq_dev.data(), ps.num_queries, ps.k); - - ivf_pq::search( - handle_, ps.search_params, index, query_view, inds_view, dists_view); - - update_host(distances_ivf_pq.data(), distances_ivf_pq_dev.data(), queries_size, stream_); - update_host(indices_ivf_pq.data(), indices_ivf_pq_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - - // A very conservative lower bound on recall - double min_recall = - static_cast(ps.search_params.n_probes) / static_cast(ps.index_params.n_lists); - // Using a heuristic to lower the required recall due to code-packing errors - min_recall = - std::min(std::erfc(0.05 * compression_ratio / std::max(min_recall, 0.5)), min_recall); - // Use explicit per-test min recall value if provided. - min_recall = ps.min_recall.value_or(min_recall); - - ASSERT_TRUE(eval_neighbours(indices_ref, - indices_ivf_pq, - distances_ref, - distances_ivf_pq, - ps.num_queries, - ps.k, - 0.0001 * compression_ratio, - min_recall)) - << ps; - - // Test a few extra invariants - IdxT min_results = min_output_size(handle_, index, ps.search_params.n_probes); - IdxT max_oob = ps.k <= min_results ? 0 : ps.k - min_results; - IdxT found_oob = 0; - for (uint32_t query_ix = 0; query_ix < ps.num_queries; query_ix++) { - for (uint32_t k = 0; k < ps.k; k++) { - auto flat_i = query_ix * ps.k + k; - auto found_ix = indices_ivf_pq[flat_i]; - if (found_ix == ivf_pq::kOutOfBoundsRecord) { - found_oob++; - continue; - } - ASSERT_NE(found_ix, ivf::kInvalidRecord) - << "got an invalid record at query_ix = " << query_ix << ", k = " << k - << " (distance = " << distances_ivf_pq[flat_i] << ")"; - ASSERT_LT(found_ix, ps.num_db_vecs) - << "got an impossible index = " << found_ix << " at query_ix = " << query_ix - << ", k = " << k << " (distance = " << distances_ivf_pq[flat_i] << ")"; - } - } - ASSERT_LE(found_oob, max_oob) - << "got too many records out-of-bounds (see ivf_pq::kOutOfBoundsRecord)."; - if (found_oob > 0) { - RAFT_LOG_WARN( - "Got %zu results out-of-bounds because of large top-k (%zu) and small n_probes (%u) and " - "small DB size/n_lists ratio (%zu / %u)", - size_t(found_oob), - size_t(ps.k), - ps.search_params.n_probes, - size_t(ps.num_db_vecs), - ps.index_params.n_lists); - } - } - - void SetUp() override // NOLINT - { - gen_data(); - calc_ref(); - } - - void TearDown() override // NOLINT - { - cudaGetLastError(); - resource::sync_stream(handle_); - database.resize(0, stream_); - search_queries.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - ivf_pq_inputs ps; // NOLINT - rmm::device_uvector database; // NOLINT - rmm::device_uvector search_queries; // NOLINT - std::vector indices_ref; // NOLINT - std::vector distances_ref; // NOLINT -}; - -template -class ivf_pq_filter_test : public ::testing::TestWithParam { - public: - ivf_pq_filter_test() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam::GetParam()), - database(0, stream_), - search_queries(0, stream_) - { - } - - void gen_data() - { - database.resize(size_t{ps.num_db_vecs} * size_t{ps.dim}, stream_); - search_queries.resize(size_t{ps.num_queries} * size_t{ps.dim}, stream_); - - raft::random::RngState r(1234ULL); - if constexpr (std::is_same{}) { - raft::random::uniform( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(0.1), DataT(2.0)); - raft::random::uniform( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(0.1), DataT(2.0)); - } else { - raft::random::uniformInt( - handle_, r, database.data(), ps.num_db_vecs * ps.dim, DataT(1), DataT(20)); - raft::random::uniformInt( - handle_, r, search_queries.data(), ps.num_queries * ps.dim, DataT(1), DataT(20)); - } - resource::sync_stream(handle_); - } - - void calc_ref() - { - size_t queries_size = size_t{ps.num_queries} * size_t{ps.k}; - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - search_queries.data(), - database.data() + test_ivf_sample_filter::offset * ps.dim, - ps.num_queries, - ps.num_db_vecs - test_ivf_sample_filter::offset, - ps.dim, - ps.k, - ps.index_params.metric); - raft::linalg::addScalar(indices_naive_dev.data(), - indices_naive_dev.data(), - IdxT(test_ivf_sample_filter::offset), - queries_size, - stream_); - distances_ref.resize(queries_size); - update_host(distances_ref.data(), distances_naive_dev.data(), queries_size, stream_); - indices_ref.resize(queries_size); - update_host(indices_ref.data(), indices_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - auto build_only() - { - auto ipams = ps.index_params; - ipams.add_data_on_build = true; - - auto index_view = - raft::make_device_matrix_view(database.data(), ps.num_db_vecs, ps.dim); - return ivf_pq::build(handle_, ipams, index_view); - } - - template - void run(BuildIndex build_index) - { - index index = build_index(); - - double compression_ratio = - static_cast(ps.dim * 8) / static_cast(index.pq_dim() * index.pq_bits()); - size_t queries_size = ps.num_queries * ps.k; - std::vector indices_ivf_pq(queries_size); - std::vector distances_ivf_pq(queries_size); - - rmm::device_uvector distances_ivf_pq_dev(queries_size, stream_); - rmm::device_uvector indices_ivf_pq_dev(queries_size, stream_); - - auto query_view = - raft::make_device_matrix_view(search_queries.data(), ps.num_queries, ps.dim); - auto inds_view = raft::make_device_matrix_view( - indices_ivf_pq_dev.data(), ps.num_queries, ps.k); - auto dists_view = raft::make_device_matrix_view( - distances_ivf_pq_dev.data(), ps.num_queries, ps.k); - - // Create Bitset filter - auto removed_indices = - raft::make_device_vector(handle_, test_ivf_sample_filter::offset); - thrust::sequence( - resource::get_thrust_policy(handle_), - thrust::device_pointer_cast(removed_indices.data_handle()), - thrust::device_pointer_cast(removed_indices.data_handle() + test_ivf_sample_filter::offset)); - resource::sync_stream(handle_); - - raft::core::bitset removed_indices_bitset( - handle_, removed_indices.view(), ps.num_db_vecs); - ivf_pq::search_with_filtering( - handle_, - ps.search_params, - index, - query_view, - inds_view, - dists_view, - raft::neighbors::filtering::bitset_filter(removed_indices_bitset.view())); - - update_host(distances_ivf_pq.data(), distances_ivf_pq_dev.data(), queries_size, stream_); - update_host(indices_ivf_pq.data(), indices_ivf_pq_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - - // A very conservative lower bound on recall - double min_recall = - static_cast(ps.search_params.n_probes) / static_cast(ps.index_params.n_lists); - // Using a heuristic to lower the required recall due to code-packing errors - min_recall = - std::min(std::erfc(0.05 * compression_ratio / std::max(min_recall, 0.5)), min_recall); - // Use explicit per-test min recall value if provided. - min_recall = ps.min_recall.value_or(min_recall); - - ASSERT_TRUE(eval_neighbours(indices_ref, - indices_ivf_pq, - distances_ref, - distances_ivf_pq, - ps.num_queries, - ps.k, - 0.0001 * compression_ratio, - min_recall)) - << ps; - } - - void SetUp() override // NOLINT - { - gen_data(); - calc_ref(); - } - - void TearDown() override // NOLINT - { - cudaGetLastError(); - resource::sync_stream(handle_); - database.resize(0, stream_); - search_queries.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - ivf_pq_inputs ps; // NOLINT - rmm::device_uvector database; // NOLINT - rmm::device_uvector search_queries; // NOLINT - std::vector indices_ref; // NOLINT - std::vector distances_ref; // NOLINT -}; - -/* Test cases */ -using test_cases_t = std::vector; - -// concatenate parameter sets for different type -template -auto operator+(const std::vector& a, const std::vector& b) -> std::vector -{ - std::vector res = a; - res.insert(res.end(), b.begin(), b.end()); - return res; -} - -inline auto defaults() -> test_cases_t { return {ivf_pq_inputs{}}; } - -template -auto map(const std::vector& xs, F f) -> std::vector -{ - std::vector ys(xs.size()); - std::transform(xs.begin(), xs.end(), ys.begin(), f); - return ys; -} - -inline auto with_dims(const std::vector& dims) -> test_cases_t -{ - return map(dims, [](uint32_t d) { - ivf_pq_inputs x; - x.dim = d; - return x; - }); -} - -/** These will surely trigger the fastest kernel available. */ -inline auto small_dims() -> test_cases_t { return with_dims({1, 2, 3, 4, 5, 8, 15, 16, 17}); } - -inline auto small_dims_per_cluster() -> test_cases_t -{ - return map(small_dims(), [](const ivf_pq_inputs& x) { - ivf_pq_inputs y(x); - y.index_params.codebook_kind = ivf_pq::codebook_gen::PER_CLUSTER; - return y; - }); -} - -inline auto big_dims() -> test_cases_t -{ - // with_dims({512, 513, 1023, 1024, 1025, 2048, 2049, 2050, 2053, 6144, 8192, 12288, 16384}); - auto xs = with_dims({512, 513, 1023, 1024, 1025, 2048, 2049, 2050, 2053, 6144}); - return map(xs, [](const ivf_pq_inputs& x) { - ivf_pq_inputs y(x); - uint32_t pq_len = 2; - y.index_params.pq_dim = div_rounding_up_safe(x.dim, pq_len); - // This comes from pure experimentation, also the recall depens a lot on pq_len. - y.min_recall = 0.48 + 0.028 * std::log2(x.dim); - return y; - }); -} - -/** These will surely trigger no-smem-lut kernel. */ -inline auto big_dims_moderate_lut() -> test_cases_t -{ - return map(big_dims(), [](const ivf_pq_inputs& x) { - ivf_pq_inputs y(x); - uint32_t pq_len = 2; - y.index_params.pq_dim = round_up_safe(div_rounding_up_safe(x.dim, pq_len), 4u); - y.index_params.pq_bits = 6; - y.search_params.lut_dtype = CUDA_R_16F; - y.min_recall = 0.69; - return y; - }); -} - -/** Some of these should trigger no-basediff kernel. */ -inline auto big_dims_small_lut() -> test_cases_t -{ - return map(big_dims(), [](const ivf_pq_inputs& x) { - ivf_pq_inputs y(x); - uint32_t pq_len = 8; - y.index_params.pq_dim = round_up_safe(div_rounding_up_safe(x.dim, pq_len), 4u); - y.index_params.pq_bits = 6; - y.search_params.lut_dtype = CUDA_R_8U; - y.min_recall = 0.21; - return y; - }); -} - -/** - * A minimal set of tests to check various enum-like parameters. - */ -inline auto enum_variety() -> test_cases_t -{ - test_cases_t xs; -#define ADD_CASE(f) \ - do { \ - xs.push_back({}); \ - ([](ivf_pq_inputs & x) f)(xs[xs.size() - 1]); \ - } while (0); - - ADD_CASE({ - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_CLUSTER; - x.min_recall = 0.86; - }); - ADD_CASE({ - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_SUBSPACE; - x.min_recall = 0.86; - }); - ADD_CASE({ - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_CLUSTER; - x.index_params.pq_bits = 4; - x.min_recall = 0.79; - }); - ADD_CASE({ - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_CLUSTER; - x.index_params.pq_bits = 5; - x.min_recall = 0.83; - }); - - ADD_CASE({ - x.index_params.pq_bits = 6; - x.min_recall = 0.84; - }); - ADD_CASE({ - x.index_params.pq_bits = 7; - x.min_recall = 0.85; - }); - ADD_CASE({ - x.index_params.pq_bits = 8; - x.min_recall = 0.86; - }); - - ADD_CASE({ - x.index_params.force_random_rotation = true; - x.min_recall = 0.86; - }); - ADD_CASE({ - x.index_params.force_random_rotation = false; - x.min_recall = 0.86; - }); - - ADD_CASE({ - x.search_params.lut_dtype = CUDA_R_32F; - x.min_recall = 0.86; - }); - ADD_CASE({ - x.search_params.lut_dtype = CUDA_R_16F; - x.min_recall = 0.86; - }); - ADD_CASE({ - x.search_params.lut_dtype = CUDA_R_8U; - x.min_recall = 0.84; - }); - - ADD_CASE({ - x.search_params.internal_distance_dtype = CUDA_R_32F; - x.min_recall = 0.86; - }); - ADD_CASE({ - x.search_params.internal_distance_dtype = CUDA_R_16F; - x.search_params.lut_dtype = CUDA_R_16F; - x.min_recall = 0.86; - }); - - return xs; -} - -inline auto enum_variety_l2() -> test_cases_t -{ - return map(enum_variety(), [](const ivf_pq_inputs& x) { - ivf_pq_inputs y(x); - y.index_params.metric = distance::DistanceType::L2Expanded; - return y; - }); -} - -inline auto enum_variety_ip() -> test_cases_t -{ - return map(enum_variety(), [](const ivf_pq_inputs& x) { - ivf_pq_inputs y(x); - if (y.min_recall.has_value()) { - if (y.search_params.lut_dtype == CUDA_R_8U) { - // InnerProduct score is signed, - // thus we're forced to used signed 8-bit representation, - // thus we have one bit less precision - y.min_recall = y.min_recall.value() * 0.90; - } else { - // In other cases it seems to perform a little bit better, still worse than L2 - y.min_recall = y.min_recall.value() * 0.94; - } - } - y.index_params.metric = distance::DistanceType::InnerProduct; - return y; - }); -} - -inline auto enum_variety_l2sqrt() -> test_cases_t -{ - return map(enum_variety(), [](const ivf_pq_inputs& x) { - ivf_pq_inputs y(x); - y.index_params.metric = distance::DistanceType::L2SqrtExpanded; - return y; - }); -} - -/** - * Try different number of n_probes, some of which may trigger the non-fused version of the search - * kernel. - */ -inline auto var_n_probes() -> test_cases_t -{ - ivf_pq_inputs dflt; - std::vector xs; - for (auto x = dflt.index_params.n_lists; x >= 1; x /= 2) { - xs.push_back(x); - } - return map(xs, [](uint32_t n_probes) { - ivf_pq_inputs x; - x.search_params.n_probes = n_probes; - return x; - }); -} - -/** - * Try different number of nearest neighbours. - * Values smaller than 32 test if the code behaves well when Capacity (== 32) does not change, - * but `k <= Capacity` changes. - * - * Values between `32 and ivf_pq::detail::kMaxCapacity` test various instantiations of the - * main kernel (Capacity-templated) - * - * Values above ivf_pq::detail::kMaxCapacity should trigger the non-fused version of the kernel - * (manage_local_topk = false). - * - * Also we test here various values that are close-but-not-power-of-two to catch any problems - * related to rounding/alignment. - * - * Note, we cannot control explicitly which instance of the search kernel to choose, hence it's - * important to try a variety of different values of `k` to make sure all paths are triggered. - * - * Set the log level to DEBUG (5) or above to inspect the selected kernel instances. - */ -inline auto var_k() -> test_cases_t -{ - return map( - {1, 2, 3, 5, 8, 15, 16, 32, 63, 65, 127, 128, 256, 257, 1023, 2048, 2049}, [](uint32_t k) { - ivf_pq_inputs x; - x.k = k; - // when there's not enough data, try more cluster probes - x.search_params.n_probes = max(x.search_params.n_probes, min(x.index_params.n_lists, k)); - return x; - }); -} - -/** - * Cases brought up from downstream projects. - */ -inline auto special_cases() -> test_cases_t -{ - test_cases_t xs; - -#define ADD_CASE(f) \ - do { \ - xs.push_back({}); \ - ([](ivf_pq_inputs & x) f)(xs[xs.size() - 1]); \ - } while (0); - - ADD_CASE({ - x.num_db_vecs = 1183514; - x.dim = 100; - x.num_queries = 10000; - x.k = 10; - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_SUBSPACE; - x.index_params.pq_dim = 10; - x.index_params.pq_bits = 8; - x.index_params.n_lists = 1024; - x.search_params.n_probes = 50; - }); - - ADD_CASE({ - x.num_db_vecs = 10000; - x.dim = 16; - x.num_queries = 500; - x.k = 128; - x.index_params.metric = distance::DistanceType::L2Expanded; - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_SUBSPACE; - x.index_params.pq_bits = 8; - x.index_params.n_lists = 100; - x.search_params.n_probes = 100; - }); - - ADD_CASE({ - x.num_db_vecs = 10000; - x.dim = 16; - x.num_queries = 500; - x.k = 129; - x.index_params.metric = distance::DistanceType::L2Expanded; - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_SUBSPACE; - x.index_params.pq_bits = 8; - x.index_params.n_lists = 100; - x.search_params.n_probes = 100; - }); - - ADD_CASE({ - x.num_db_vecs = 4335; - x.dim = 4; - x.num_queries = 100000; - x.k = 12; - x.index_params.metric = distance::DistanceType::L2Expanded; - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_SUBSPACE; - x.index_params.pq_dim = 2; - x.index_params.pq_bits = 8; - x.index_params.n_lists = 69; - x.search_params.n_probes = 69; - }); - - ADD_CASE({ - x.num_db_vecs = 4335; - x.dim = 4; - x.num_queries = 100000; - x.k = 12; - x.index_params.metric = distance::DistanceType::L2Expanded; - x.index_params.codebook_kind = ivf_pq::codebook_gen::PER_CLUSTER; - x.index_params.pq_dim = 2; - x.index_params.pq_bits = 8; - x.index_params.n_lists = 69; - x.search_params.n_probes = 69; - }); - - return xs; -} - -/* Test instantiations */ - -#define TEST_BUILD_SEARCH(type) \ - TEST_P(type, build_search) /* NOLINT */ \ - { \ - this->run([this]() { return this->build_only(); }); \ - } - -#define TEST_BUILD_EXTEND_SEARCH(type) \ - TEST_P(type, build_extend_search) /* NOLINT */ \ - { \ - this->run([this]() { return this->build_2_extends(); }); \ - } - -#define TEST_BUILD_SERIALIZE_SEARCH(type) \ - TEST_P(type, build_serialize_search) /* NOLINT */ \ - { \ - this->run([this]() { return this->build_serialize(); }); \ - } - -#define INSTANTIATE(type, vals) \ - INSTANTIATE_TEST_SUITE_P(IvfPq, type, ::testing::ValuesIn(vals)); /* NOLINT */ - -} // namespace raft::neighbors::ivf_pq diff --git a/cpp/test/neighbors/ann_ivf_pq/ivf_pq_build_float_uint32_t.cu b/cpp/test/neighbors/ann_ivf_pq/ivf_pq_build_float_uint32_t.cu deleted file mode 100644 index 5ba21c3c2f..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/ivf_pq_build_float_uint32_t.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index -#include - -#define instantiate_raft_neighbors_ivf_pq_build(T, IdxT) \ - template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset); \ - \ - template auto raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_pq::index; - -instantiate_raft_neighbors_ivf_pq_build(float, uint32_t); - -#undef instantiate_raft_neighbors_ivf_pq_build diff --git a/cpp/test/neighbors/ann_ivf_pq/ivf_pq_build_test-ext.cuh b/cpp/test/neighbors/ann_ivf_pq/ivf_pq_build_test-ext.cuh deleted file mode 100644 index cd5435ab2e..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/ivf_pq_build_test-ext.cuh +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include // raft::neighbors::ivf_pq::index -#include - -#define instantiate_raft_neighbors_ivf_pq_build(T, IdxT) \ - extern template raft::neighbors::ivf_pq::index raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - raft::device_matrix_view dataset); \ - \ - extern template auto raft::neighbors::ivf_pq::build( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::index_params& params, \ - const T* dataset, \ - IdxT n_rows, \ - uint32_t dim) \ - ->raft::neighbors::ivf_pq::index; - -instantiate_raft_neighbors_ivf_pq_build(float, uint32_t); - -#undef instantiate_raft_neighbors_ivf_pq_build diff --git a/cpp/test/neighbors/ann_ivf_pq/ivf_pq_search_float_uint32_t.cu b/cpp/test/neighbors/ann_ivf_pq/ivf_pq_search_float_uint32_t.cu deleted file mode 100644 index 00baa59f58..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/ivf_pq_search_float_uint32_t.cu +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include // raft::neighbors::ivf_pq::index -#include - -#include - -#define instantiate_raft_neighbors_ivf_pq_search(T, IdxT) \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); \ - \ - template void raft::neighbors::ivf_pq::search( \ - raft::resources const& handle, \ - const raft::neighbors::ivf_pq::search_params& params, \ - const raft::neighbors::ivf_pq::index& idx, \ - const T* queries, \ - uint32_t n_queries, \ - uint32_t k, \ - IdxT* neighbors, \ - float* distances) - -instantiate_raft_neighbors_ivf_pq_search(float, uint32_t); - -#undef instantiate_raft_neighbors_ivf_pq_search - -#define instantiate_raft_neighbors_ivf_pq_search_with_filtering(T, IdxT, FilterT) \ - template void raft::neighbors::ivf_pq::search_with_filtering( \ - raft::resources const& handle, \ - const search_params& params, \ - const index& idx, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances, \ - FilterT sample_filter) - -#define COMMA , -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - float, uint32_t, raft::neighbors::filtering::bitset_filter); - -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - int8_t, int64_t, raft::neighbors::filtering::bitset_filter); - -instantiate_raft_neighbors_ivf_pq_search_with_filtering( - float, uint32_t, raft::neighbors::filtering::none_ivf_sample_filter); - -#undef COMMA -#undef instantiate_raft_neighbors_ivf_pq_search_with_filtering diff --git a/cpp/test/neighbors/ann_ivf_pq/test_filter_float_int64_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_filter_float_int64_t.cu deleted file mode 100644 index 70d5d8761f..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/test_filter_float_int64_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_pq.cuh" - -#include -#include - -namespace raft::neighbors::ivf_pq { - -using f32_f32_i64_filter = ivf_pq_filter_test; - -TEST_BUILD_SEARCH(f32_f32_i64_filter) -INSTANTIATE(f32_f32_i64_filter, defaults() + big_dims_moderate_lut()); -} // namespace raft::neighbors::ivf_pq diff --git a/cpp/test/neighbors/ann_ivf_pq/test_filter_int8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_filter_int8_t_int64_t.cu deleted file mode 100644 index ba96a8db0b..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/test_filter_int8_t_int64_t.cu +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_pq.cuh" - -#include -#include - -namespace raft::neighbors::ivf_pq { - -using f32_i08_i64_filter = ivf_pq_filter_test; - -TEST_BUILD_SEARCH(f32_i08_i64_filter) -INSTANTIATE(f32_i08_i64_filter, big_dims()); - -} // namespace raft::neighbors::ivf_pq diff --git a/cpp/test/neighbors/ann_ivf_pq/test_float_int64_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_float_int64_t.cu deleted file mode 100644 index 9859061d70..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/test_float_int64_t.cu +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_pq.cuh" - -namespace raft::neighbors::ivf_pq { - -using f32_f32_i64 = ivf_pq_test; - -TEST_BUILD_EXTEND_SEARCH(f32_f32_i64) -TEST_BUILD_SERIALIZE_SEARCH(f32_f32_i64) -INSTANTIATE(f32_f32_i64, defaults() + small_dims() + big_dims_moderate_lut()); - -} // namespace raft::neighbors::ivf_pq diff --git a/cpp/test/neighbors/ann_ivf_pq/test_float_uint32_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_float_uint32_t.cu deleted file mode 100644 index b8ada2249a..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/test_float_uint32_t.cu +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_pq.cuh" -#include "ivf_pq_build_test-ext.cuh" - -#include -#include - -namespace raft::neighbors::ivf_pq { - -using f32_f32_u32 = ivf_pq_test; -using f32_f32_u32_filter = ivf_pq_filter_test; - -TEST_BUILD_SEARCH(f32_f32_u32) -TEST_BUILD_SERIALIZE_SEARCH(f32_f32_u32) -INSTANTIATE(f32_f32_u32, defaults() + var_n_probes() + var_k() + special_cases()); - -TEST_BUILD_SEARCH(f32_f32_u32_filter) -INSTANTIATE(f32_f32_u32_filter, defaults()); -} // namespace raft::neighbors::ivf_pq diff --git a/cpp/test/neighbors/ann_ivf_pq/test_int8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_int8_t_int64_t.cu deleted file mode 100644 index 970bdd6a12..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/test_int8_t_int64_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_pq.cuh" - -#include -namespace raft::neighbors::ivf_pq { - -using f32_i08_i64 = ivf_pq_test; - -TEST_BUILD_SEARCH(f32_i08_i64) -TEST_BUILD_SERIALIZE_SEARCH(f32_i08_i64) -INSTANTIATE(f32_i08_i64, defaults() + big_dims() + var_k()); - -} // namespace raft::neighbors::ivf_pq diff --git a/cpp/test/neighbors/ann_ivf_pq/test_uint8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_pq/test_uint8_t_int64_t.cu deleted file mode 100644 index e949c2f7ed..0000000000 --- a/cpp/test/neighbors/ann_ivf_pq/test_uint8_t_int64_t.cu +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_ivf_pq.cuh" - -namespace raft::neighbors::ivf_pq { - -using f32_u08_i64 = ivf_pq_test; - -TEST_BUILD_SEARCH(f32_u08_i64) -TEST_BUILD_EXTEND_SEARCH(f32_u08_i64) -INSTANTIATE(f32_u08_i64, small_dims_per_cluster() + enum_variety()); - -} // namespace raft::neighbors::ivf_pq diff --git a/cpp/test/neighbors/ann_nn_descent.cuh b/cpp/test/neighbors/ann_nn_descent.cuh deleted file mode 100644 index 5070d83b15..0000000000 --- a/cpp/test/neighbors/ann_nn_descent.cuh +++ /dev/null @@ -1,332 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "ann_utils.cuh" - -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include -#include - -namespace raft::neighbors::experimental::nn_descent { - -struct AnnNNDescentInputs { - int n_rows; - int dim; - int graph_degree; - raft::distance::DistanceType metric; - bool host_dataset; - double min_recall; -}; - -struct AnnNNDescentBatchInputs { - std::pair recall_cluster; - int n_rows; - int dim; - int graph_degree; - raft::distance::DistanceType metric; - bool host_dataset; -}; - -inline ::std::ostream& operator<<(::std::ostream& os, const AnnNNDescentInputs& p) -{ - os << "dataset shape=" << p.n_rows << "x" << p.dim << ", graph_degree=" << p.graph_degree - << ", metric=" << static_cast(p.metric) << (p.host_dataset ? ", host" : ", device") - << std::endl; - return os; -} - -inline ::std::ostream& operator<<(::std::ostream& os, const AnnNNDescentBatchInputs& p) -{ - os << "dataset shape=" << p.n_rows << "x" << p.dim << ", graph_degree=" << p.graph_degree - << ", metric=" << static_cast(p.metric) << (p.host_dataset ? ", host" : ", device") - << ", clusters=" << p.recall_cluster.second << std::endl; - return os; -} - -template -class AnnNNDescentTest : public ::testing::TestWithParam { - public: - AnnNNDescentTest() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam::GetParam()), - database(0, stream_) - { - } - - protected: - void testNNDescent() - { - size_t queries_size = ps.n_rows * ps.graph_degree; - std::vector indices_NNDescent(queries_size); - std::vector distances_NNDescent(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - database.data(), - database.data(), - ps.n_rows, - ps.n_rows, - ps.dim, - ps.graph_degree, - ps.metric); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - { - { - nn_descent::index_params index_params; - index_params.metric = ps.metric; - index_params.graph_degree = ps.graph_degree; - index_params.intermediate_graph_degree = 2 * ps.graph_degree; - index_params.max_iterations = 100; - index_params.return_distances = true; - - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.n_rows, ps.dim); - - { - if (ps.host_dataset) { - auto database_host = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy(database_host.data_handle(), database.data(), database.size(), stream_); - auto database_host_view = raft::make_host_matrix_view( - (const DataT*)database_host.data_handle(), ps.n_rows, ps.dim); - index index{handle_, ps.n_rows, static_cast(ps.graph_degree), true}; - nn_descent::build( - handle_, index_params, database_host_view, index, DistEpilogue()); - raft::copy( - indices_NNDescent.data(), index.graph().data_handle(), queries_size, stream_); - if (index.distances().has_value()) { - raft::copy(distances_NNDescent.data(), - index.distances().value().data_handle(), - queries_size, - stream_); - } - - } else { - index index{handle_, ps.n_rows, static_cast(ps.graph_degree), true}; - nn_descent::build( - handle_, index_params, database_view, index, DistEpilogue()); - raft::copy( - indices_NNDescent.data(), index.graph().data_handle(), queries_size, stream_); - if (index.distances().has_value()) { - raft::copy(distances_NNDescent.data(), - index.distances().value().data_handle(), - queries_size, - stream_); - } - }; - } - resource::sync_stream(handle_); - } - - double min_recall = ps.min_recall; - EXPECT_TRUE(eval_neighbours(indices_naive, - indices_NNDescent, - distances_naive, - distances_NNDescent, - ps.n_rows, - ps.graph_degree, - 0.001, - min_recall)); - } - } - - void SetUp() override - { - database.resize(((size_t)ps.n_rows) * ps.dim, stream_); - raft::random::RngState r(1234ULL); - if constexpr (std::is_same{}) { - raft::random::normal(handle_, r, database.data(), ps.n_rows * ps.dim, DataT(0.1), DataT(2.0)); - } else { - raft::random::uniformInt( - handle_, r, database.data(), ps.n_rows * ps.dim, DataT(1), DataT(20)); - } - resource::sync_stream(handle_); - } - - void TearDown() override - { - resource::sync_stream(handle_); - database.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - AnnNNDescentInputs ps; - rmm::device_uvector database; -}; - -template -class AnnNNDescentBatchTest : public ::testing::TestWithParam { - public: - AnnNNDescentBatchTest() - : stream_(resource::get_cuda_stream(handle_)), - ps(::testing::TestWithParam::GetParam()), - database(0, stream_) - { - } - - void testNNDescentBatch() - { - size_t queries_size = ps.n_rows * ps.graph_degree; - std::vector indices_NNDescent(queries_size); - std::vector distances_NNDescent(queries_size); - std::vector indices_naive(queries_size); - std::vector distances_naive(queries_size); - - { - rmm::device_uvector distances_naive_dev(queries_size, stream_); - rmm::device_uvector indices_naive_dev(queries_size, stream_); - naive_knn(handle_, - distances_naive_dev.data(), - indices_naive_dev.data(), - database.data(), - database.data(), - ps.n_rows, - ps.n_rows, - ps.dim, - ps.graph_degree, - ps.metric); - update_host(indices_naive.data(), indices_naive_dev.data(), queries_size, stream_); - update_host(distances_naive.data(), distances_naive_dev.data(), queries_size, stream_); - resource::sync_stream(handle_); - } - - { - { - nn_descent::index_params index_params; - index_params.metric = ps.metric; - index_params.graph_degree = ps.graph_degree; - index_params.intermediate_graph_degree = 2 * ps.graph_degree; - index_params.max_iterations = 10; - index_params.return_distances = true; - index_params.n_clusters = ps.recall_cluster.second; - - auto database_view = raft::make_device_matrix_view( - (const DataT*)database.data(), ps.n_rows, ps.dim); - - { - if (ps.host_dataset) { - auto database_host = raft::make_host_matrix(ps.n_rows, ps.dim); - raft::copy(database_host.data_handle(), database.data(), database.size(), stream_); - auto database_host_view = raft::make_host_matrix_view( - (const DataT*)database_host.data_handle(), ps.n_rows, ps.dim); - auto index = nn_descent::build( - handle_, index_params, database_host_view, DistEpilogue()); - raft::copy( - indices_NNDescent.data(), index.graph().data_handle(), queries_size, stream_); - if (index.distances().has_value()) { - raft::copy(distances_NNDescent.data(), - index.distances().value().data_handle(), - queries_size, - stream_); - } - - } else { - auto index = nn_descent::build( - handle_, index_params, database_view, DistEpilogue()); - raft::copy( - indices_NNDescent.data(), index.graph().data_handle(), queries_size, stream_); - if (index.distances().has_value()) { - raft::copy(distances_NNDescent.data(), - index.distances().value().data_handle(), - queries_size, - stream_); - } - }; - } - resource::sync_stream(handle_); - } - double min_recall = ps.recall_cluster.first; - EXPECT_TRUE(eval_neighbours(indices_naive, - indices_NNDescent, - distances_naive, - distances_NNDescent, - ps.n_rows, - ps.graph_degree, - 0.01, - min_recall, - true, - static_cast(ps.graph_degree * 0.1))); - } - } - - void SetUp() override - { - database.resize(((size_t)ps.n_rows) * ps.dim, stream_); - raft::random::RngState r(1234ULL); - if constexpr (std::is_same{}) { - raft::random::normal(handle_, r, database.data(), ps.n_rows * ps.dim, DataT(0.1), DataT(2.0)); - } else { - raft::random::uniformInt( - handle_, r, database.data(), ps.n_rows * ps.dim, DataT(1), DataT(20)); - } - resource::sync_stream(handle_); - } - - void TearDown() override - { - resource::sync_stream(handle_); - database.resize(0, stream_); - } - - private: - raft::resources handle_; - rmm::cuda_stream_view stream_; - AnnNNDescentBatchInputs ps; - rmm::device_uvector database; -}; - -const std::vector inputs = raft::util::itertools::product( - {1000, 2000}, // n_rows - {3, 5, 7, 8, 17, 64, 128, 137, 192, 256, 512, 619, 1024}, // dim - {32, 64}, // graph_degree - {raft::distance::DistanceType::L2Expanded}, - {false, true}, - {0.90}); - -// TODO: Investigate why this test is failing -// Reference issue https://github.com/rapidsai/raft/issues/2450 -// const std::vector inputsBatch = -// raft::util::itertools::product( -// {std::make_pair(0.9, 3lu), std::make_pair(0.9, 2lu)}, // min_recall, n_clusters -// {4000, 5000}, // n_rows -// {192, 512}, // dim -// {32, 64}, // graph_degree -// {raft::distance::DistanceType::L2Expanded}, -// {false, true}); - -} // namespace raft::neighbors::experimental::nn_descent diff --git a/cpp/test/neighbors/ann_nn_descent/test_batch_float_uint32_t.cu b/cpp/test/neighbors/ann_nn_descent/test_batch_float_uint32_t.cu deleted file mode 100644 index c6f56e8c39..0000000000 --- a/cpp/test/neighbors/ann_nn_descent/test_batch_float_uint32_t.cu +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_nn_descent.cuh" - -#include - -namespace raft::neighbors::experimental::nn_descent { - -typedef AnnNNDescentBatchTest AnnNNDescentBatchTestF_U32; -TEST_P(AnnNNDescentBatchTestF_U32, AnnNNDescentBatch) { this->testNNDescentBatch(); } - -INSTANTIATE_TEST_CASE_P(AnnNNDescentBatchTest, - AnnNNDescentBatchTestF_U32, - ::testing::ValuesIn(inputsBatch)); - -} // namespace raft::neighbors::experimental::nn_descent diff --git a/cpp/test/neighbors/ann_nn_descent/test_float_uint32_t.cu b/cpp/test/neighbors/ann_nn_descent/test_float_uint32_t.cu deleted file mode 100644 index ec6d04ad12..0000000000 --- a/cpp/test/neighbors/ann_nn_descent/test_float_uint32_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_nn_descent.cuh" - -#include - -namespace raft::neighbors::experimental::nn_descent { - -typedef AnnNNDescentTest AnnNNDescentTestF_U32; -TEST_P(AnnNNDescentTestF_U32, AnnNNDescent) { this->testNNDescent(); } - -INSTANTIATE_TEST_CASE_P(AnnNNDescentTest, AnnNNDescentTestF_U32, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::experimental::nn_descent diff --git a/cpp/test/neighbors/ann_nn_descent/test_int8_t_uint32_t.cu b/cpp/test/neighbors/ann_nn_descent/test_int8_t_uint32_t.cu deleted file mode 100644 index 27fa42d636..0000000000 --- a/cpp/test/neighbors/ann_nn_descent/test_int8_t_uint32_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_nn_descent.cuh" - -#include - -namespace raft::neighbors::experimental::nn_descent { - -typedef AnnNNDescentTest AnnNNDescentTestI8_U32; -TEST_P(AnnNNDescentTestI8_U32, AnnNNDescent) { this->testNNDescent(); } - -INSTANTIATE_TEST_CASE_P(AnnNNDescentTest, AnnNNDescentTestI8_U32, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::experimental::nn_descent diff --git a/cpp/test/neighbors/ann_nn_descent/test_uint8_t_uint32_t.cu b/cpp/test/neighbors/ann_nn_descent/test_uint8_t_uint32_t.cu deleted file mode 100644 index 3afe79dcc4..0000000000 --- a/cpp/test/neighbors/ann_nn_descent/test_uint8_t_uint32_t.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../ann_nn_descent.cuh" - -#include - -namespace raft::neighbors::experimental::nn_descent { - -typedef AnnNNDescentTest AnnNNDescentTestUI8_U32; -TEST_P(AnnNNDescentTestUI8_U32, AnnNNDescent) { this->testNNDescent(); } - -INSTANTIATE_TEST_CASE_P(AnnNNDescentTest, AnnNNDescentTestUI8_U32, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::experimental::nn_descent diff --git a/cpp/test/neighbors/ann_utils.cuh b/cpp/test/neighbors/ann_utils.cuh deleted file mode 100644 index 82e3ace9da..0000000000 --- a/cpp/test/neighbors/ann_utils.cuh +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "../test_utils.cuh" - -#include // raft::make_device_matrix -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include - -namespace raft::neighbors { - -struct print_dtype { - cudaDataType_t value; -}; - -inline auto operator<<(std::ostream& os, const print_dtype& p) -> std::ostream& -{ - switch (p.value) { - case CUDA_R_16F: os << "CUDA_R_16F"; break; - case CUDA_C_16F: os << "CUDA_C_16F"; break; - case CUDA_R_16BF: os << "CUDA_R_16BF"; break; - case CUDA_C_16BF: os << "CUDA_C_16BF"; break; - case CUDA_R_32F: os << "CUDA_R_32F"; break; - case CUDA_C_32F: os << "CUDA_C_32F"; break; - case CUDA_R_64F: os << "CUDA_R_64F"; break; - case CUDA_C_64F: os << "CUDA_C_64F"; break; - case CUDA_R_4I: os << "CUDA_R_4I"; break; - case CUDA_C_4I: os << "CUDA_C_4I"; break; - case CUDA_R_4U: os << "CUDA_R_4U"; break; - case CUDA_C_4U: os << "CUDA_C_4U"; break; - case CUDA_R_8I: os << "CUDA_R_8I"; break; - case CUDA_C_8I: os << "CUDA_C_8I"; break; - case CUDA_R_8U: os << "CUDA_R_8U"; break; - case CUDA_C_8U: os << "CUDA_C_8U"; break; - case CUDA_R_16I: os << "CUDA_R_16I"; break; - case CUDA_C_16I: os << "CUDA_C_16I"; break; - case CUDA_R_16U: os << "CUDA_R_16U"; break; - case CUDA_C_16U: os << "CUDA_C_16U"; break; - case CUDA_R_32I: os << "CUDA_R_32I"; break; - case CUDA_C_32I: os << "CUDA_C_32I"; break; - case CUDA_R_32U: os << "CUDA_R_32U"; break; - case CUDA_C_32U: os << "CUDA_C_32U"; break; - case CUDA_R_64I: os << "CUDA_R_64I"; break; - case CUDA_C_64I: os << "CUDA_C_64I"; break; - case CUDA_R_64U: os << "CUDA_R_64U"; break; - case CUDA_C_64U: os << "CUDA_C_64U"; break; - default: RAFT_FAIL("unreachable code"); - } - return os; -} - -struct print_metric { - raft::distance::DistanceType value; -}; - -inline auto operator<<(std::ostream& os, const print_metric& p) -> std::ostream& -{ - switch (p.value) { - case raft::distance::L2Expanded: os << "distance::L2Expanded"; break; - case raft::distance::L2SqrtExpanded: os << "distance::L2SqrtExpanded"; break; - case raft::distance::CosineExpanded: os << "distance::CosineExpanded"; break; - case raft::distance::L1: os << "distance::L1"; break; - case raft::distance::L2Unexpanded: os << "distance::L2Unexpanded"; break; - case raft::distance::L2SqrtUnexpanded: os << "distance::L2SqrtUnexpanded"; break; - case raft::distance::InnerProduct: os << "distance::InnerProduct"; break; - case raft::distance::Linf: os << "distance::Linf"; break; - case raft::distance::Canberra: os << "distance::Canberra"; break; - case raft::distance::LpUnexpanded: os << "distance::LpUnexpanded"; break; - case raft::distance::CorrelationExpanded: os << "distance::CorrelationExpanded"; break; - case raft::distance::JaccardExpanded: os << "distance::JaccardExpanded"; break; - case raft::distance::HellingerExpanded: os << "distance::HellingerExpanded"; break; - case raft::distance::Haversine: os << "distance::Haversine"; break; - case raft::distance::BrayCurtis: os << "distance::BrayCurtis"; break; - case raft::distance::JensenShannon: os << "distance::JensenShannon"; break; - case raft::distance::HammingUnexpanded: os << "distance::HammingUnexpanded"; break; - case raft::distance::KLDivergence: os << "distance::KLDivergence"; break; - case raft::distance::RusselRaoExpanded: os << "distance::RusselRaoExpanded"; break; - case raft::distance::DiceExpanded: os << "distance::DiceExpanded"; break; - case raft::distance::Precomputed: os << "distance::Precomputed"; break; - default: RAFT_FAIL("unreachable code"); - } - return os; -} - -template -struct idx_dist_pair { - IdxT idx; - DistT dist; - CompareDist eq_compare; - auto operator==(const idx_dist_pair& a) const -> bool - { - if (idx == a.idx) return true; - if (eq_compare(dist, a.dist)) return true; - return false; - } - idx_dist_pair(IdxT x, DistT y, CompareDist op) : idx(x), dist(y), eq_compare(op) {} -}; - -/** Calculate recall value using only neighbor indices - */ -template -auto calc_recall(const std::vector& expected_idx, - const std::vector& actual_idx, - size_t rows, - size_t cols) -{ - size_t match_count = 0; - size_t total_count = static_cast(rows) * static_cast(cols); - for (size_t i = 0; i < rows; ++i) { - for (size_t k = 0; k < cols; ++k) { - size_t idx_k = i * cols + k; // row major assumption! - auto act_idx = actual_idx[idx_k]; - for (size_t j = 0; j < cols; ++j) { - size_t idx = i * cols + j; // row major assumption! - auto exp_idx = expected_idx[idx]; - if (act_idx == exp_idx) { - match_count++; - break; - } - } - } - } - return std::make_tuple( - static_cast(match_count) / static_cast(total_count), match_count, total_count); -} - -/** check uniqueness of indices - */ -template -auto check_unique_indices(const std::vector& actual_idx, - size_t rows, - size_t cols, - size_t max_duplicates) -{ - size_t max_count; - size_t dup_count = 0lu; - std::set unique_indices; - for (size_t i = 0; i < rows; ++i) { - unique_indices.clear(); - max_count = 0; - for (size_t k = 0; k < cols; ++k) { - size_t idx_k = i * cols + k; // row major assumption! - auto act_idx = actual_idx[idx_k]; - if (act_idx == std::numeric_limits::max()) { - max_count++; - } else if (unique_indices.find(act_idx) == unique_indices.end()) { - unique_indices.insert(act_idx); - } else { - dup_count++; - if (dup_count > max_duplicates) { - return testing::AssertionFailure() - << "Duplicated index " << act_idx << " at k " << k << " for query " << i << "! "; - } - } - } - } - return testing::AssertionSuccess(); -} - -template -auto eval_recall(const std::vector& expected_idx, - const std::vector& actual_idx, - size_t rows, - size_t cols, - double eps, - double min_recall, - bool test_unique = true) -> testing::AssertionResult -{ - auto [actual_recall, match_count, total_count] = - calc_recall(expected_idx, actual_idx, rows, cols); - double error_margin = (actual_recall - min_recall) / std::max(1.0 - min_recall, eps); - RAFT_LOG_INFO("Recall = %f (%zu/%zu), the error is %2.1f%% %s the threshold (eps = %f).", - actual_recall, - match_count, - total_count, - std::abs(error_margin * 100.0), - error_margin < 0 ? "above" : "below", - eps); - if (actual_recall < min_recall - eps) { - return testing::AssertionFailure() - << "actual recall (" << actual_recall << ") is lower than the minimum expected recall (" - << min_recall << "); eps = " << eps << ". "; - } - if (test_unique) - return check_unique_indices(actual_idx, rows, cols); - else - return testing::AssertionSuccess(); -} - -/** Overload of calc_recall to account for distances - */ -template -auto calc_recall(const std::vector& expected_idx, - const std::vector& actual_idx, - const std::vector& expected_dist, - const std::vector& actual_dist, - size_t rows, - size_t cols, - double eps) -{ - size_t match_count = 0; - size_t total_count = static_cast(rows) * static_cast(cols); - for (size_t i = 0; i < rows; ++i) { - for (size_t k = 0; k < cols; ++k) { - size_t idx_k = i * cols + k; // row major assumption! - auto act_idx = actual_idx[idx_k]; - auto act_dist = actual_dist[idx_k]; - for (size_t j = 0; j < cols; ++j) { - size_t idx = i * cols + j; // row major assumption! - auto exp_idx = expected_idx[idx]; - auto exp_dist = expected_dist[idx]; - idx_dist_pair exp_kvp(exp_idx, exp_dist, raft::CompareApprox(eps)); - idx_dist_pair act_kvp(act_idx, act_dist, raft::CompareApprox(eps)); - if (exp_kvp == act_kvp) { - match_count++; - break; - } - } - } - } - return std::make_tuple( - static_cast(match_count) / static_cast(total_count), match_count, total_count); -} - -/** same as eval_recall, but in case indices do not match, - * then check distances as well, and accept match if actual dist is equal to expected_dist */ -template -auto eval_neighbours(const std::vector& expected_idx, - const std::vector& actual_idx, - const std::vector& expected_dist, - const std::vector& actual_dist, - size_t rows, - size_t cols, - double eps, - double min_recall, - bool test_unique = true, - size_t max_duplicates = 0) -> testing::AssertionResult -{ - auto [actual_recall, match_count, total_count] = - calc_recall(expected_idx, actual_idx, expected_dist, actual_dist, rows, cols, eps); - double error_margin = (actual_recall - min_recall) / std::max(1.0 - min_recall, eps); - RAFT_LOG_INFO("Recall = %f (%zu/%zu), the error is %2.1f%% %s the threshold (eps = %f).", - actual_recall, - match_count, - total_count, - std::abs(error_margin * 100.0), - error_margin < 0 ? "above" : "below", - eps); - if (actual_recall < min_recall - eps) { - return testing::AssertionFailure() - << "actual recall (" << actual_recall << ") is lower than the minimum expected recall (" - << min_recall << "); eps = " << eps << ". "; - } - if (test_unique) - return check_unique_indices(actual_idx, rows, cols, max_duplicates); - else - return testing::AssertionSuccess(); -} - -template -auto eval_distances(raft::resources const& handle, - const T* x, // dataset, n_rows * n_cols - const T* queries, // n_queries * n_cols - const IdxT* neighbors, // n_queries * k - const DistT* distances, // n_queries *k - size_t n_rows, - size_t n_cols, - size_t n_queries, - uint32_t k, - raft::distance::DistanceType metric, - double eps) -> testing::AssertionResult -{ - // for each vector, we calculate the actual distance to the k neighbors - - for (size_t i = 0; i < n_queries; i++) { - auto y = raft::make_device_matrix(handle, k, n_cols); - auto naive_dist = raft::make_device_matrix(handle, 1, k); - - raft::matrix::copy_rows( - handle, - make_device_matrix_view(x, n_rows, n_cols), - y.view(), - make_device_vector_view(neighbors + i * k, k)); - - dim3 block_dim(16, 32, 1); - auto grid_y = - static_cast(std::min(raft::ceildiv(k, block_dim.y), 32768)); - dim3 grid_dim(raft::ceildiv(n_rows, block_dim.x), grid_y, 1); - - naive_distance_kernel - <<>>( - naive_dist.data_handle(), queries + i * n_cols, y.data_handle(), 1, k, n_cols, metric); - - if (!devArrMatch(distances + i * k, - naive_dist.data_handle(), - naive_dist.size(), - CompareApprox(eps))) { - std::cout << n_rows << "x" << n_cols << ", " << k << std::endl; - std::cout << "query " << i << std::endl; - print_vector(" indices", neighbors + i * k, k, std::cout); - print_vector("n dist", distances + i * k, k, std::cout); - print_vector("c dist", naive_dist.data_handle(), naive_dist.size(), std::cout); - - return testing::AssertionFailure(); - } - } - return testing::AssertionSuccess(); -} -} // namespace raft::neighbors diff --git a/cpp/test/neighbors/fused_l2_knn.cu b/cpp/test/neighbors/fused_l2_knn.cu deleted file mode 100644 index e4d018aff0..0000000000 --- a/cpp/test/neighbors/fused_l2_knn.cu +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "./knn_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include - -namespace raft { -namespace spatial { -namespace knn { -struct FusedL2KNNInputs { - int num_queries; - int num_db_vecs; - int dim; - int k; - raft::distance::DistanceType metric_; -}; - -template -class FusedL2KNNTest : public ::testing::TestWithParam { - public: - FusedL2KNNTest() - : stream_(resource::get_cuda_stream(handle_)), - params_(::testing::TestWithParam::GetParam()), - database(params_.num_db_vecs * params_.dim, stream_), - search_queries(params_.num_queries * params_.dim, stream_), - raft_indices_(params_.num_queries * params_.k, stream_), - raft_distances_(params_.num_queries * params_.k, stream_), - ref_indices_(params_.num_queries * params_.k, stream_), - ref_distances_(params_.num_queries * params_.k, stream_) - { - RAFT_CUDA_TRY(cudaMemsetAsync(database.data(), 0, database.size() * sizeof(T), stream_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(search_queries.data(), 0, search_queries.size() * sizeof(T), stream_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(raft_indices_.data(), 0, raft_indices_.size() * sizeof(int64_t), stream_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(raft_distances_.data(), 0, raft_distances_.size() * sizeof(T), stream_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(ref_indices_.data(), 0, ref_indices_.size() * sizeof(int64_t), stream_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(ref_distances_.data(), 0, ref_distances_.size() * sizeof(T), stream_)); - } - - protected: - void testBruteForce() - { - // calculate the naive knn, by calculating the full pairwise distances and doing a k-select - rmm::device_uvector temp_distances(num_db_vecs * num_queries, stream_); - distance::pairwise_distance( - handle_, - raft::make_device_matrix_view(search_queries.data(), num_queries, dim), - raft::make_device_matrix_view(database.data(), num_db_vecs, dim), - raft::make_device_matrix_view(temp_distances.data(), num_queries, num_db_vecs), - metric); - - matrix::select_k( - handle_, - make_device_matrix_view(temp_distances.data(), num_queries, num_db_vecs), - std::nullopt, - make_device_matrix_view(ref_distances_.data(), num_queries, k_), - make_device_matrix_view(ref_indices_.data(), num_queries, k_), - true, - true); - - auto index_view = - raft::make_device_matrix_view(database.data(), num_db_vecs, dim); - auto query_view = - raft::make_device_matrix_view(search_queries.data(), num_queries, dim); - auto out_indices_view = - raft::make_device_matrix_view(raft_indices_.data(), num_queries, k_); - auto out_dists_view = - raft::make_device_matrix_view(raft_distances_.data(), num_queries, k_); - raft::neighbors::brute_force::fused_l2_knn( - handle_, index_view, query_view, out_indices_view, out_dists_view, metric); - - // verify. - ASSERT_TRUE(devArrMatchKnnPair(ref_indices_.data(), - raft_indices_.data(), - ref_distances_.data(), - raft_distances_.data(), - num_queries, - k_, - float(0.001), - stream_)); - } - - void SetUp() override - { - num_queries = params_.num_queries; - num_db_vecs = params_.num_db_vecs; - dim = params_.dim; - k_ = params_.k; - metric = params_.metric_; - - unsigned long long int seed = 1234ULL; - raft::random::RngState r(seed); - uniform(handle_, r, database.data(), num_db_vecs * dim, T(-1.0), T(1.0)); - uniform(handle_, r, search_queries.data(), num_queries * dim, T(-1.0), T(1.0)); - } - - private: - raft::resources handle_; - cudaStream_t stream_ = 0; - FusedL2KNNInputs params_; - int num_queries; - int num_db_vecs; - int dim; - rmm::device_uvector database; - rmm::device_uvector search_queries; - rmm::device_uvector raft_indices_; - rmm::device_uvector raft_distances_; - rmm::device_uvector ref_indices_; - rmm::device_uvector ref_distances_; - int k_; - raft::distance::DistanceType metric; -}; - -const std::vector inputs = { - {100, 1000, 16, 10, raft::distance::DistanceType::L2Expanded}, - {256, 256, 30, 10, raft::distance::DistanceType::L2Expanded}, - {1000, 10000, 16, 10, raft::distance::DistanceType::L2Expanded}, - {100, 1000, 16, 50, raft::distance::DistanceType::L2Expanded}, - {20, 10000, 16, 10, raft::distance::DistanceType::L2Expanded}, - {1000, 10000, 16, 50, raft::distance::DistanceType::L2Expanded}, - {1000, 10000, 32, 50, raft::distance::DistanceType::L2Expanded}, - {10000, 40000, 32, 30, raft::distance::DistanceType::L2Expanded}, - // L2 unexpanded - {100, 1000, 16, 10, raft::distance::DistanceType::L2Unexpanded}, - {1000, 10000, 16, 10, raft::distance::DistanceType::L2Unexpanded}, - {100, 1000, 16, 50, raft::distance::DistanceType::L2Unexpanded}, - {20, 10000, 16, 50, raft::distance::DistanceType::L2Unexpanded}, - {1000, 10000, 16, 50, raft::distance::DistanceType::L2Unexpanded}, - {1000, 10000, 32, 50, raft::distance::DistanceType::L2Unexpanded}, - {10000, 40000, 32, 30, raft::distance::DistanceType::L2Unexpanded}, -}; - -typedef FusedL2KNNTest FusedL2KNNTestF; -TEST_P(FusedL2KNNTestF, FusedBruteForce) { this->testBruteForce(); } - -INSTANTIATE_TEST_CASE_P(FusedL2KNNTest, FusedL2KNNTestF, ::testing::ValuesIn(inputs)); - -} // namespace knn -} // namespace spatial -} // namespace raft diff --git a/cpp/test/neighbors/knn.cu b/cpp/test/neighbors/knn.cu deleted file mode 100644 index a98de85bda..0000000000 --- a/cpp/test/neighbors/knn.cu +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" - -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include - -namespace raft::neighbors::brute_force { -struct KNNInputs { - std::vector> input; - int k; - std::vector labels; -}; - -template -RAFT_KERNEL build_actual_output( - int* output, int n_rows, int k, const int* idx_labels, const IdxT* indices) -{ - int element = threadIdx.x + blockDim.x * blockIdx.x; - if (element >= n_rows * k) return; - - output[element] = idx_labels[indices[element]]; -} - -RAFT_KERNEL build_expected_output(int* output, int n_rows, int k, const int* labels) -{ - int row = threadIdx.x + blockDim.x * blockIdx.x; - if (row >= n_rows) return; - - int cur_label = labels[row]; - for (int i = 0; i < k; i++) { - output[row * k + i] = cur_label; - } -} - -template -class KNNTest : public ::testing::TestWithParam { - public: - KNNTest() - : params_(::testing::TestWithParam::GetParam()), - stream(resource::get_cuda_stream(handle)), - actual_labels_(0, stream), - expected_labels_(0, stream), - input_(0, stream), - search_data_(0, stream), - indices_(0, stream), - distances_(0, stream), - search_labels_(0, stream) - { - } - - protected: - void testBruteForce() - { - // #if (RAFT_ACTIVE_LEVEL >= RAFT_LEVEL_DEBUG) - raft::print_device_vector("Input array: ", input_.data(), rows_ * cols_, std::cout); - std::cout << "K: " << k_ << std::endl; - raft::print_device_vector("Labels array: ", search_labels_.data(), rows_, std::cout); - // #endif - - std::vector> index = { - make_device_matrix_view((const T*)(input_.data()), rows_, cols_)}; - auto search = raft::make_device_matrix_view( - (const T*)(search_data_.data()), rows_, cols_); - - auto indices = raft::make_device_matrix_view(indices_.data(), rows_, k_); - auto distances = - raft::make_device_matrix_view(distances_.data(), rows_, k_); - - auto metric = raft::distance::DistanceType::L2Unexpanded; - knn(handle, index, search, indices, distances, metric, std::make_optional(0)); - - build_actual_output<<>>( - actual_labels_.data(), rows_, k_, search_labels_.data(), indices_.data()); - - build_expected_output<<>>( - expected_labels_.data(), rows_, k_, search_labels_.data()); - - ASSERT_TRUE(devArrMatch( - expected_labels_.data(), actual_labels_.data(), rows_ * k_, raft::Compare(), stream)); - } - - void SetUp() override - { - rows_ = params_.input.size(); - cols_ = params_.input[0].size(); - k_ = params_.k; - - actual_labels_.resize(rows_ * k_, stream); - expected_labels_.resize(rows_ * k_, stream); - input_.resize(rows_ * cols_, stream); - search_data_.resize(rows_ * cols_, stream); - indices_.resize(rows_ * k_, stream); - distances_.resize(rows_ * k_, stream); - search_labels_.resize(rows_, stream); - - RAFT_CUDA_TRY( - cudaMemsetAsync(actual_labels_.data(), 0, actual_labels_.size() * sizeof(int), stream)); - RAFT_CUDA_TRY( - cudaMemsetAsync(expected_labels_.data(), 0, expected_labels_.size() * sizeof(int), stream)); - RAFT_CUDA_TRY(cudaMemsetAsync(input_.data(), 0, input_.size() * sizeof(float), stream)); - RAFT_CUDA_TRY( - cudaMemsetAsync(search_data_.data(), 0, search_data_.size() * sizeof(float), stream)); - RAFT_CUDA_TRY(cudaMemsetAsync(indices_.data(), 0, indices_.size() * sizeof(IdxT), stream)); - RAFT_CUDA_TRY(cudaMemsetAsync(distances_.data(), 0, distances_.size() * sizeof(float), stream)); - RAFT_CUDA_TRY( - cudaMemsetAsync(search_labels_.data(), 0, search_labels_.size() * sizeof(int), stream)); - - std::vector row_major_input; - for (std::size_t i = 0; i < params_.input.size(); ++i) { - for (std::size_t j = 0; j < params_.input[i].size(); ++j) { - row_major_input.push_back(params_.input[i][j]); - } - } - rmm::device_buffer input_d = - rmm::device_buffer(row_major_input.data(), row_major_input.size() * sizeof(float), stream); - float* input_ptr = static_cast(input_d.data()); - - rmm::device_buffer labels_d = - rmm::device_buffer(params_.labels.data(), params_.labels.size() * sizeof(int), stream); - int* labels_ptr = static_cast(labels_d.data()); - - raft::copy(input_.data(), input_ptr, rows_ * cols_, stream); - raft::copy(search_data_.data(), input_ptr, rows_ * cols_, stream); - raft::copy(search_labels_.data(), labels_ptr, rows_, stream); - resource::sync_stream(handle, stream); - } - - private: - raft::resources handle; - cudaStream_t stream; - - KNNInputs params_; - int rows_; - int cols_; - rmm::device_uvector input_; - rmm::device_uvector search_data_; - rmm::device_uvector indices_; - rmm::device_uvector distances_; - int k_; - - rmm::device_uvector search_labels_; - rmm::device_uvector actual_labels_; - rmm::device_uvector expected_labels_; -}; - -const std::vector inputs = { - // 2D - {{ - {2.7810836, 2.550537003}, - {1.465489372, 2.362125076}, - {3.396561688, 4.400293529}, - {1.38807019, 1.850220317}, - {3.06407232, 3.005305973}, - {7.627531214, 2.759262235}, - {5.332441248, 2.088626775}, - {6.922596716, 1.77106367}, - {8.675418651, -0.242068655}, - {7.673756466, 3.508563011}, - }, - 2, - {0, 0, 0, 0, 0, 1, 1, 1, 1, 1}}}; - -typedef KNNTest KNNTestFint32_t; -TEST_P(KNNTestFint32_t, BruteForce) { this->testBruteForce(); } -typedef KNNTest KNNTestFuint32_t; -TEST_P(KNNTestFuint32_t, BruteForce) { this->testBruteForce(); } - -INSTANTIATE_TEST_CASE_P(KNNTest, KNNTestFint32_t, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_CASE_P(KNNTest, KNNTestFuint32_t, ::testing::ValuesIn(inputs)); - -} // namespace raft::neighbors::brute_force diff --git a/cpp/test/neighbors/refine.cu b/cpp/test/neighbors/refine.cu deleted file mode 100644 index 05e6048e56..0000000000 --- a/cpp/test/neighbors/refine.cu +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "ann_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include - -#include - -namespace raft::neighbors { - -template -class RefineTest : public ::testing::TestWithParam> { - public: - RefineTest() - : stream_(resource::get_cuda_stream(handle_)), - data(handle_, ::testing::TestWithParam>::GetParam()) - { - } - - protected: - public: // tamas remove - void testRefine() - { - std::vector indices(data.p.n_queries * data.p.k); - std::vector distances(data.p.n_queries * data.p.k); - - if (data.p.host_data) { - raft::neighbors::refine(handle_, - data.dataset_host.view(), - data.queries_host.view(), - data.candidates_host.view(), - data.refined_indices_host.view(), - data.refined_distances_host.view(), - data.p.metric); - raft::copy(indices.data(), - data.refined_indices_host.data_handle(), - data.refined_indices_host.size(), - stream_); - raft::copy(distances.data(), - data.refined_distances_host.data_handle(), - data.refined_distances_host.size(), - stream_); - - } else { - raft::neighbors::refine(handle_, - data.dataset.view(), - data.queries.view(), - data.candidates.view(), - data.refined_indices.view(), - data.refined_distances.view(), - data.p.metric); - update_host(distances.data(), - data.refined_distances.data_handle(), - data.refined_distances.size(), - stream_); - update_host( - indices.data(), data.refined_indices.data_handle(), data.refined_indices.size(), stream_); - } - resource::sync_stream(handle_); - - double min_recall = 1; - - ASSERT_TRUE(raft::neighbors::eval_neighbours(data.true_refined_indices_host, - indices, - data.true_refined_distances_host, - distances, - data.p.n_queries, - data.p.k, - 0.001, - min_recall)); - } - - public: - raft::resources handle_; - rmm::cuda_stream_view stream_; - RefineHelper data; -}; - -const std::vector> inputs = - raft::util::itertools::product>( - {static_cast(137)}, - {static_cast(1000)}, - {static_cast(16)}, - {static_cast(1), static_cast(10), static_cast(33)}, - {static_cast(33)}, - {raft::distance::DistanceType::L2Expanded, raft::distance::DistanceType::InnerProduct}, - {false, true}); - -typedef RefineTest RefineTestF; -TEST_P(RefineTestF, AnnRefine) { this->testRefine(); } - -INSTANTIATE_TEST_CASE_P(RefineTest, RefineTestF, ::testing::ValuesIn(inputs)); - -typedef RefineTest RefineTestF_uint8; -TEST_P(RefineTestF_uint8, AnnRefine) { this->testRefine(); } -INSTANTIATE_TEST_CASE_P(RefineTest, RefineTestF_uint8, ::testing::ValuesIn(inputs)); - -typedef RefineTest RefineTestF_int8; -TEST_P(RefineTestF_int8, AnnRefine) { this->testRefine(); } -INSTANTIATE_TEST_CASE_P(RefineTest, RefineTestF_int8, ::testing::ValuesIn(inputs)); -} // namespace raft::neighbors diff --git a/cpp/test/neighbors/tiled_knn.cu b/cpp/test/neighbors/tiled_knn.cu deleted file mode 100644 index d378100711..0000000000 --- a/cpp/test/neighbors/tiled_knn.cu +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" -#include "./ann_utils.cuh" -#include "./knn_utils.cuh" - -#include -#include -#include -#include // raft::distance::pairwise_distance -#include -#include -#include -#include -#include // raft::neighbors::detail::brute_force_knn_impl - -#include - -#include - -#include -#include -#include - -namespace raft::neighbors::brute_force { - -struct TiledKNNInputs { - int num_queries; - int num_db_vecs; - int dim; - int k; - int row_tiles; - int col_tiles; - raft::distance::DistanceType metric; - bool row_major; -}; - -std::ostream& operator<<(std::ostream& os, const TiledKNNInputs& input) -{ - return os << "num_queries:" << input.num_queries << " num_vecs:" << input.num_db_vecs - << " dim:" << input.dim << " k:" << input.k << " row_tiles:" << input.row_tiles - << " col_tiles:" << input.col_tiles << " metric:" << print_metric{input.metric} - << " row_major:" << input.row_major; -} - -template -class TiledKNNTest : public ::testing::TestWithParam { - public: - TiledKNNTest() - : stream_(resource::get_cuda_stream(handle_)), - params_(::testing::TestWithParam::GetParam()), - database(params_.num_db_vecs * params_.dim, stream_), - search_queries(params_.num_queries * params_.dim, stream_), - raft_indices_(params_.num_queries * params_.k, stream_), - raft_distances_(params_.num_queries * params_.k, stream_), - ref_indices_(params_.num_queries * params_.k, stream_), - ref_distances_(params_.num_queries * params_.k, stream_) - { - raft::matrix::fill( - handle_, - raft::make_device_matrix_view(database.data(), params_.num_db_vecs, params_.dim), - T{0.0}); - raft::matrix::fill( - handle_, - raft::make_device_matrix_view(search_queries.data(), params_.num_queries, params_.dim), - T{0.0}); - raft::matrix::fill( - handle_, - raft::make_device_matrix_view(raft_indices_.data(), params_.num_queries, params_.k), - 0); - raft::matrix::fill( - handle_, - raft::make_device_matrix_view(raft_distances_.data(), params_.num_queries, params_.k), - T{0.0}); - raft::matrix::fill( - handle_, - raft::make_device_matrix_view(ref_indices_.data(), params_.num_queries, params_.k), - 0); - raft::matrix::fill( - handle_, - raft::make_device_matrix_view(ref_distances_.data(), params_.num_queries, params_.k), - T{0.0}); - } - - protected: - void testBruteForce() - { - float metric_arg = 3.0; - - // calculate the naive knn, by calculating the full pairwise distances and doing a k-select - rmm::device_uvector temp_distances(num_db_vecs * num_queries, stream_); - rmm::device_uvector workspace(0, stream_); - distance::pairwise_distance(handle_, - search_queries.data(), - database.data(), - temp_distances.data(), - num_queries, - num_db_vecs, - dim, - workspace, - metric, - params_.row_major, - metric_arg); - - // setting the 'isRowMajor' flag in the pairwise distances api, not only sets - // the inputs as colmajor - but also the output. this means we have to transpose in this - // case - auto temp_dist = temp_distances.data(); - rmm::device_uvector temp_row_major_dist(num_db_vecs * num_queries, stream_); - if (!params_.row_major) { - raft::linalg::transpose( - handle_, temp_dist, temp_row_major_dist.data(), num_queries, num_db_vecs, stream_); - temp_dist = temp_row_major_dist.data(); - } - - matrix::select_k( - handle_, - raft::make_device_matrix_view(temp_dist, num_queries, num_db_vecs), - std::nullopt, - raft::make_device_matrix_view(ref_distances_.data(), params_.num_queries, params_.k), - raft::make_device_matrix_view(ref_indices_.data(), params_.num_queries, params_.k), - raft::distance::is_min_close(metric), - true); - - if ((params_.row_tiles == 0) && (params_.col_tiles == 0)) { - std::vector input{database.data()}; - std::vector sizes{static_cast(num_db_vecs)}; - neighbors::detail::brute_force_knn_impl(handle_, - input, - sizes, - dim, - const_cast(search_queries.data()), - num_queries, - raft_indices_.data(), - raft_distances_.data(), - k_, - params_.row_major, - params_.row_major, - nullptr, - metric, - metric_arg); - } else { - neighbors::detail::tiled_brute_force_knn(handle_, - search_queries.data(), - database.data(), - num_queries, - num_db_vecs, - dim, - k_, - raft_distances_.data(), - raft_indices_.data(), - metric, - metric_arg, - params_.row_tiles, - params_.col_tiles); - } - - // verify. - ASSERT_TRUE(raft::spatial::knn::devArrMatchKnnPair(ref_indices_.data(), - raft_indices_.data(), - ref_distances_.data(), - raft_distances_.data(), - num_queries, - k_, - float(0.001), - stream_, - true)); - - // Also test out the 'index' api - where we can use precomputed norms - if (params_.row_major) { - auto idx = - raft::neighbors::brute_force::build(handle_, - raft::make_device_matrix_view( - database.data(), params_.num_db_vecs, params_.dim), - metric, - metric_arg); - - auto query_view = raft::make_device_matrix_view( - search_queries.data(), params_.num_queries, params_.dim); - - raft::neighbors::brute_force::search( - handle_, - idx, - query_view, - raft::make_device_matrix_view( - raft_indices_.data(), params_.num_queries, params_.k), - raft::make_device_matrix_view( - raft_distances_.data(), params_.num_queries, params_.k)); - - ASSERT_TRUE(raft::spatial::knn::devArrMatchKnnPair(ref_indices_.data(), - raft_indices_.data(), - ref_distances_.data(), - raft_distances_.data(), - num_queries, - k_, - float(0.001), - stream_, - true)); - // also test out the batch api. First get new reference results (all k, up to a certain - // max size) - auto all_size = std::min(params_.num_db_vecs, 1024); - auto all_indices = raft::make_device_matrix(handle_, num_queries, all_size); - auto all_distances = raft::make_device_matrix(handle_, num_queries, all_size); - raft::neighbors::brute_force::search( - handle_, idx, query_view, all_indices.view(), all_distances.view()); - - int64_t offset = 0; - auto query = make_batch_k_query(handle_, idx, query_view, k_); - for (auto batch : *query) { - auto batch_size = batch.batch_size(); - auto indices = raft::make_device_matrix(handle_, num_queries, batch_size); - auto distances = raft::make_device_matrix(handle_, num_queries, batch_size); - - matrix::slice_coordinates coords{0, offset, num_queries, offset + batch_size}; - - matrix::slice(handle_, raft::make_const_mdspan(all_indices.view()), indices.view(), coords); - matrix::slice( - handle_, raft::make_const_mdspan(all_distances.view()), distances.view(), coords); - - ASSERT_TRUE(raft::spatial::knn::devArrMatchKnnPair(indices.data_handle(), - batch.indices().data_handle(), - distances.data_handle(), - batch.distances().data_handle(), - num_queries, - batch_size, - float(0.001), - stream_, - true)); - - offset += batch_size; - if (offset + batch_size > all_size) break; - } - - // also test out with variable batch sizes - offset = 0; - int64_t batch_size = k_; - query = make_batch_k_query(handle_, idx, query_view, batch_size); - for (auto it = query->begin(); it != query->end(); it.advance(batch_size)) { - // batch_size could be less than requested (in the case of final batch). handle. - ASSERT_TRUE(it->indices().extent(1) <= batch_size); - batch_size = it->indices().extent(1); - - auto indices = raft::make_device_matrix(handle_, num_queries, batch_size); - auto distances = raft::make_device_matrix(handle_, num_queries, batch_size); - - matrix::slice_coordinates coords{0, offset, num_queries, offset + batch_size}; - matrix::slice(handle_, raft::make_const_mdspan(all_indices.view()), indices.view(), coords); - matrix::slice( - handle_, raft::make_const_mdspan(all_distances.view()), distances.view(), coords); - - ASSERT_TRUE(raft::spatial::knn::devArrMatchKnnPair(indices.data_handle(), - it->indices().data_handle(), - distances.data_handle(), - it->distances().data_handle(), - num_queries, - batch_size, - float(0.001), - stream_, - true)); - - offset += batch_size; - if (offset + batch_size > all_size) break; - - batch_size += 2; - } - } - } - - void SetUp() override - { - num_queries = params_.num_queries; - num_db_vecs = params_.num_db_vecs; - dim = params_.dim; - k_ = params_.k; - metric = params_.metric; - - unsigned long long int seed = 1234ULL; - raft::random::RngState r(seed); - - // JensenShannon distance requires positive values - T min_val = metric == raft::distance::DistanceType::JensenShannon ? T(0.0) : T(-1.0); - uniform(handle_, r, database.data(), num_db_vecs * dim, min_val, T(1.0)); - uniform(handle_, r, search_queries.data(), num_queries * dim, min_val, T(1.0)); - } - - private: - raft::resources handle_; - cudaStream_t stream_ = 0; - TiledKNNInputs params_; - int num_queries; - int num_db_vecs; - int dim; - rmm::device_uvector database; - rmm::device_uvector search_queries; - rmm::device_uvector raft_indices_; - rmm::device_uvector raft_distances_; - rmm::device_uvector ref_indices_; - rmm::device_uvector ref_distances_; - int k_; - raft::distance::DistanceType metric; -}; - -const std::vector random_inputs = { - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::L2Expanded, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::L2Unexpanded, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::L2SqrtExpanded, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::L2SqrtUnexpanded, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::L1, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::Linf, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::InnerProduct, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::CorrelationExpanded, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::CosineExpanded, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::LpUnexpanded, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::JensenShannon, true}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::L2SqrtExpanded, true}, - // BrayCurtis isn't currently supported by pairwise_distance api - // {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::BrayCurtis}, - {256, 512, 16, 8, 16, 8, raft::distance::DistanceType::Canberra, true}, - {10000, 40000, 32, 30, 512, 1024, raft::distance::DistanceType::L2Expanded, true}, - {345, 1023, 16, 128, 512, 1024, raft::distance::DistanceType::CosineExpanded, true}, - {789, 20516, 64, 256, 512, 4096, raft::distance::DistanceType::L2SqrtExpanded, true}, - // Test where the final column tile has < K items: - {4, 12, 32, 6, 4, 8, raft::distance::DistanceType::L2Expanded, true}, - // Test where passing column_tiles < K - {1, 40, 32, 30, 1, 8, raft::distance::DistanceType::L2Expanded, true}, - // Passing tile sizes of 0 means to use brute_force_knn_impl (instead of the - // tiled_brute_force_knn api). - {1000, 500000, 128, 128, 0, 0, raft::distance::DistanceType::L2Expanded, true}, - {1000, 500000, 128, 128, 0, 0, raft::distance::DistanceType::L2Expanded, false}, - {1000, 5000, 128, 128, 0, 0, raft::distance::DistanceType::LpUnexpanded, true}, - {1000, 5000, 128, 128, 0, 0, raft::distance::DistanceType::L2SqrtExpanded, false}, - {1000, 5000, 128, 128, 0, 0, raft::distance::DistanceType::InnerProduct, false}}; - -typedef TiledKNNTest TiledKNNTestF; -TEST_P(TiledKNNTestF, BruteForce) { this->testBruteForce(); } - -INSTANTIATE_TEST_CASE_P(TiledKNNTest, TiledKNNTestF, ::testing::ValuesIn(random_inputs)); -} // namespace raft::neighbors::brute_force diff --git a/cpp/test/sparse/gram.cu b/cpp/test/sparse/gram.cu deleted file mode 100644 index 3505a3ddf5..0000000000 --- a/cpp/test/sparse/gram.cu +++ /dev/null @@ -1,332 +0,0 @@ -/* - * Copyright (c) 2019-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#if defined RAFT_DISTANCE_COMPILED -#include -#include -#endif - -#include "../distance/gram_base.cuh" -#include "../test_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include - -namespace raft::distance::kernels { - -/** - * Structure to describe structure of the input matrices: - * - DENSE: dense, dense - * - MIX: CSR, dense - * - CSR: CSR, CSR - */ -enum SparseType { DENSE, MIX, CSR }; - -struct GramMatrixInputs { - int n1; // feature vectors in matrix 1 - int n2; // featuer vectors in matrix 2 - int n_cols; // number of elements in a feature vector - bool is_row_major; - SparseType sparse_input; - KernelParams kernel; - int ld1; - int ld2; - int ld_out; - // We will generate random input using the dimensions given here. - // The reference output is calculated by a custom kernel. -}; - -std::ostream& operator<<(std::ostream& os, const GramMatrixInputs& p) -{ - std::vector kernel_names{"linear", "poly", "rbf", "tanh"}; - os << "/" << p.n1 << "x" << p.n2 << "x" << p.n_cols << "/" - << (p.is_row_major ? "RowMajor/" : "ColMajor/") - << (p.sparse_input == SparseType::DENSE - ? "DenseDense/" - : (p.sparse_input == SparseType::MIX ? "CsrDense/" : "CsrCsr/")) - << kernel_names[p.kernel.kernel] << "/ld_" << p.ld1 << "x" << p.ld2 << "x" << p.ld_out; - return os; -} - -/*struct KernelParams { - // Kernel function parameters - KernelType kernel; //!< Type of the kernel function - int degree; //!< Degree of polynomial kernel (ignored by others) - double gamma; //!< multiplier in the - double coef0; //!< additive constant in poly and tanh kernels -};*/ - -// const KernelParams linear_kernel_params{.kernel=KernelType::LINEAR}; - -// {KernelType::POLYNOMIAL, 2, 0.5, 2.4}, {KernelType::TANH, 0, 0.5, 2.4}, {KernelType::RBF, 0, 0.5} -const std::vector inputs = raft::util::itertools::product( - {42}, - {137}, - {2}, - {true, false}, - {SparseType::DENSE, SparseType::MIX, SparseType::CSR}, - {KernelParams{KernelType::LINEAR}, - KernelParams{KernelType::POLYNOMIAL, 2, 0.5, 2.4}, - KernelParams{KernelType::TANH, 0, 0.5, 2.4}, - KernelParams{KernelType::RBF, 0, 0.5}}); - -// (ld_1, ld_2, ld_out) not supported by RBF and CSR -const std::vector inputs_ld = raft::util::itertools::product( - {137}, - {42}, - {2}, - {true, false}, - {SparseType::DENSE, SparseType::MIX}, - {KernelParams{KernelType::LINEAR}, - KernelParams{KernelType::POLYNOMIAL, 2, 0.5, 2.4}, - KernelParams{KernelType::TANH, 0, 0.5, 2.4}}, - {159}, - {73}, - {144}); - -// (ld_1, ld_2) are supported by CSR -const std::vector inputs_ld_csr = - raft::util::itertools::product( - {42}, - {137}, - {2}, - {true, false}, - {SparseType::CSR, SparseType::MIX}, - {KernelParams{KernelType::LINEAR}, - KernelParams{KernelType::POLYNOMIAL, 2, 0.5, 2.4}, - KernelParams{KernelType::TANH, 0, 0.5, 2.4}}, - {64}, - {155}, - {0}); - -template -class GramMatrixTest : public ::testing::TestWithParam { - protected: - GramMatrixTest() - : params(GetParam()), - stream(resource::get_cuda_stream(handle)), - x1(0, stream), - x2(0, stream), - x1_csr_indptr(0, stream), - x1_csr_indices(0, stream), - x1_csr_data(0, stream), - x2_csr_indptr(0, stream), - x2_csr_indices(0, stream), - x2_csr_data(0, stream), - gram(0, stream), - gram_host(0) - { - if (params.ld1 == 0) { params.ld1 = params.is_row_major ? params.n_cols : params.n1; } - if (params.ld2 == 0) { params.ld2 = params.is_row_major ? params.n_cols : params.n2; } - if (params.ld_out == 0) { params.ld_out = params.is_row_major ? params.n2 : params.n1; } - // Derive the size of the output from the offset of the last element. - size_t size = get_offset(params.n1 - 1, params.n_cols - 1, params.ld1, params.is_row_major) + 1; - x1.resize(size, stream); - size = get_offset(params.n2 - 1, params.n_cols - 1, params.ld2, params.is_row_major) + 1; - x2.resize(size, stream); - size = get_offset(params.n1 - 1, params.n2 - 1, params.ld_out, params.is_row_major) + 1; - - gram.resize(size, stream); - RAFT_CUDA_TRY(cudaMemsetAsync(gram.data(), 0, gram.size() * sizeof(math_t), stream)); - gram_host.resize(gram.size()); - std::fill(gram_host.begin(), gram_host.end(), 0); - - raft::random::RngState r(42137ULL); - raft::random::uniform(handle, r, x1.data(), x1.size(), math_t(0), math_t(1)); - raft::random::uniform(handle, r, x2.data(), x2.size(), math_t(0), math_t(1)); - - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - } - - ~GramMatrixTest() override {} - - int prepareCsr(math_t* dense, int n_rows, int ld, int* indptr, int* indices, math_t* data) - { - int nnz = 0; - double eps = 1e-6; - int n_cols = params.n_cols; - bool is_row_major = params.is_row_major; - size_t dense_size = get_offset(n_rows - 1, n_cols - 1, ld, is_row_major) + 1; - - std::vector dense_host(dense_size); - raft::update_host(dense_host.data(), dense, dense_size, stream); - resource::sync_stream(handle, stream); - - std::vector indptr_host(n_rows + 1); - std::vector indices_host(n_rows * n_cols); - std::vector data_host(n_rows * n_cols); - - // create csr matrix from dense (with threshold) - for (int i = 0; i < n_rows; ++i) { - indptr_host[i] = nnz; - for (int j = 0; j < n_cols; ++j) { - math_t value = dense_host[get_offset(i, j, ld, is_row_major)]; - if (value > eps) { - indices_host[nnz] = j; - data_host[nnz] = value; - nnz++; - } - } - } - indptr_host[n_rows] = nnz; - - // fill back dense matrix from CSR - std::fill(dense_host.data(), dense_host.data() + dense_size, 0); - for (int i = 0; i < n_rows; ++i) { - for (int idx = indptr_host[i]; idx < indptr_host[i + 1]; ++idx) { - dense_host[get_offset(i, indices_host[idx], ld, is_row_major)] = data_host[idx]; - } - } - - raft::update_device(dense, dense_host.data(), dense_size, stream); - raft::update_device(indptr, indptr_host.data(), n_rows + 1, stream); - raft::update_device(indices, indices_host.data(), nnz, stream); - raft::update_device(data, data_host.data(), nnz, stream); - resource::sync_stream(handle, stream); - return nnz; - } - - void runTest() - { - std::unique_ptr> kernel = - std::unique_ptr>(KernelFactory::create(params.kernel)); - - auto x1_span = - params.is_row_major - ? raft::make_device_strided_matrix_view( - x1.data(), params.n1, params.n_cols, params.ld1) - : raft::make_device_strided_matrix_view( - x1.data(), params.n1, params.n_cols, params.ld1); - auto x2_span = - params.is_row_major - ? raft::make_device_strided_matrix_view( - x2.data(), params.n2, params.n_cols, params.ld2) - : raft::make_device_strided_matrix_view( - x2.data(), params.n2, params.n_cols, params.ld2); - auto out_span = - params.is_row_major - ? raft::make_device_strided_matrix_view( - gram.data(), params.n1, params.n2, params.ld_out) - : raft::make_device_strided_matrix_view( - gram.data(), params.n1, params.n2, params.ld_out); - - if (params.sparse_input == SparseType::DENSE) { - (*kernel)(handle, x1_span, x2_span, out_span); - } else { - x1_csr_indptr.reserve(params.n1 + 1, stream); - x1_csr_indices.reserve(params.n1 * params.n_cols, stream); - x1_csr_data.reserve(params.n1 * params.n_cols, stream); - int x1_nnz = prepareCsr(x1.data(), - params.n1, - params.ld1, - x1_csr_indptr.data(), - x1_csr_indices.data(), - x1_csr_data.data()); - - auto x1_csr_structure = raft::make_device_compressed_structure_view( - x1_csr_indptr.data(), x1_csr_indices.data(), params.n1, params.n_cols, x1_nnz); - auto x1_csr = raft::device_csr_matrix_view( - raft::device_span(x1_csr_data.data(), x1_csr_structure.get_nnz()), - x1_csr_structure); - - if (params.sparse_input == SparseType::MIX) { - (*kernel)(handle, x1_csr, x2_span, out_span); - } else { - x2_csr_indptr.reserve(params.n2 + 1, stream); - x2_csr_indices.reserve(params.n2 * params.n_cols, stream); - x2_csr_data.reserve(params.n2 * params.n_cols, stream); - int x2_nnz = prepareCsr(x2.data(), - params.n2, - params.ld2, - x2_csr_indptr.data(), - x2_csr_indices.data(), - x2_csr_data.data()); - - auto x2_csr_structure = raft::make_device_compressed_structure_view( - x2_csr_indptr.data(), x2_csr_indices.data(), params.n2, params.n_cols, x2_nnz); - auto x2_csr = raft::device_csr_matrix_view( - raft::device_span(x2_csr_data.data(), x2_csr_structure.get_nnz()), - x2_csr_structure); - - (*kernel)(handle, x1_csr, x2_csr, out_span); - } - } - // Something in gram is executing not on the 'stream' and therefore - // a full device sync is required - RAFT_CUDA_TRY(cudaDeviceSynchronize()); - naiveGramMatrixKernel(params.n1, - params.n2, - params.n_cols, - x1, - x2, - gram_host.data(), - params.ld1, - params.ld2, - params.ld_out, - params.is_row_major, - params.kernel, - stream, - handle); - resource::sync_stream(handle, stream); - - ASSERT_TRUE(raft::devArrMatchHost( - gram_host.data(), gram.data(), gram.size(), raft::CompareApprox(1e-6f), stream)); - } - - raft::resources handle; - cudaStream_t stream = 0; - GramMatrixInputs params; - - rmm::device_uvector x1; - rmm::device_uvector x2; - - rmm::device_uvector x1_csr_indptr; - rmm::device_uvector x1_csr_indices; - rmm::device_uvector x1_csr_data; - rmm::device_uvector x2_csr_indptr; - rmm::device_uvector x2_csr_indices; - rmm::device_uvector x2_csr_data; - - rmm::device_uvector gram; - std::vector gram_host; -}; - -typedef GramMatrixTest GramMatrixTestFloatStandard; -typedef GramMatrixTest GramMatrixTestFloatLd; -typedef GramMatrixTest GramMatrixTestFloatLdCsr; - -TEST_P(GramMatrixTestFloatStandard, Gram) { runTest(); } -TEST_P(GramMatrixTestFloatLd, Gram) { runTest(); } -TEST_P(GramMatrixTestFloatLdCsr, Gram) { runTest(); } - -INSTANTIATE_TEST_SUITE_P(GramMatrixTests, GramMatrixTestFloatStandard, ::testing::ValuesIn(inputs)); -INSTANTIATE_TEST_SUITE_P(GramMatrixTests, GramMatrixTestFloatLd, ::testing::ValuesIn(inputs_ld)); -INSTANTIATE_TEST_SUITE_P(GramMatrixTests, - GramMatrixTestFloatLdCsr, - ::testing::ValuesIn(inputs_ld_csr)); -}; // end namespace raft::distance::kernels diff --git a/cpp/test/sparse/neighbors/brute_force.cu b/cpp/test/sparse/neighbors/brute_force.cu deleted file mode 100644 index ed2705a253..0000000000 --- a/cpp/test/sparse/neighbors/brute_force.cu +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../../test_utils.cuh" - -#include -#include -#include -#include - -#include -#include - -namespace raft { -namespace sparse { -namespace selection { - -using namespace raft; -using namespace raft::sparse; - -template -struct SparseKNNInputs { - value_idx n_cols; - - std::vector indptr_h; - std::vector indices_h; - std::vector data_h; - - std::vector out_dists_ref_h; - std::vector out_indices_ref_h; - - int k; - - int batch_size_index = 2; - int batch_size_query = 2; - - raft::distance::DistanceType metric = raft::distance::DistanceType::L2SqrtExpanded; -}; - -template -::std::ostream& operator<<(::std::ostream& os, const SparseKNNInputs& dims) -{ - return os; -} - -template -class SparseKNNTest : public ::testing::TestWithParam> { - public: - SparseKNNTest() - : params(::testing::TestWithParam>::GetParam()), - indptr(0, resource::get_cuda_stream(handle)), - indices(0, resource::get_cuda_stream(handle)), - data(0, resource::get_cuda_stream(handle)), - out_indices(0, resource::get_cuda_stream(handle)), - out_dists(0, resource::get_cuda_stream(handle)), - out_indices_ref(0, resource::get_cuda_stream(handle)), - out_dists_ref(0, resource::get_cuda_stream(handle)) - { - } - - protected: - void SetUp() override - { - n_rows = params.indptr_h.size() - 1; - nnz = params.indices_h.size(); - k = params.k; - - make_data(); - - raft::sparse::neighbors::brute_force_knn(indptr.data(), - indices.data(), - data.data(), - nnz, - n_rows, - params.n_cols, - indptr.data(), - indices.data(), - data.data(), - nnz, - n_rows, - params.n_cols, - out_indices.data(), - out_dists.data(), - k, - handle, - params.batch_size_index, - params.batch_size_query, - params.metric); - - RAFT_CUDA_TRY(cudaStreamSynchronize(resource::get_cuda_stream(handle))); - } - - void compare() - { - ASSERT_TRUE(devArrMatch( - out_dists_ref.data(), out_dists.data(), n_rows * k, CompareApprox(1e-4))); - ASSERT_TRUE( - devArrMatch(out_indices_ref.data(), out_indices.data(), n_rows * k, Compare())); - } - - protected: - void make_data() - { - std::vector indptr_h = params.indptr_h; - std::vector indices_h = params.indices_h; - std::vector data_h = params.data_h; - - auto stream = resource::get_cuda_stream(handle); - indptr.resize(indptr_h.size(), stream); - indices.resize(indices_h.size(), stream); - data.resize(data_h.size(), stream); - - update_device(indptr.data(), indptr_h.data(), indptr_h.size(), stream); - update_device(indices.data(), indices_h.data(), indices_h.size(), stream); - update_device(data.data(), data_h.data(), data_h.size(), stream); - - std::vector out_dists_ref_h = params.out_dists_ref_h; - std::vector out_indices_ref_h = params.out_indices_ref_h; - - out_indices_ref.resize(out_indices_ref_h.size(), stream); - out_dists_ref.resize(out_dists_ref_h.size(), stream); - - update_device( - out_indices_ref.data(), out_indices_ref_h.data(), out_indices_ref_h.size(), stream); - update_device(out_dists_ref.data(), out_dists_ref_h.data(), out_dists_ref_h.size(), stream); - - out_dists.resize(n_rows * k, stream); - out_indices.resize(n_rows * k, stream); - } - - raft::resources handle; - - int n_rows, nnz, k; - - // input data - rmm::device_uvector indptr, indices; - rmm::device_uvector data; - - // output data - rmm::device_uvector out_indices; - rmm::device_uvector out_dists; - - rmm::device_uvector out_indices_ref; - rmm::device_uvector out_dists_ref; - - SparseKNNInputs params; -}; - -const std::vector> inputs_i32_f = { - {9, // ncols - {0, 2, 4, 6, 8}, // indptr - {0, 4, 0, 3, 0, 2, 0, 8}, // indices - {0.0f, 1.0f, 5.0f, 6.0f, 5.0f, 6.0f, 0.0f, 1.0f}, // data - {0, 1.41421, 0, 7.87401, 0, 7.87401, 0, 1.41421}, // dists - {0, 3, 1, 0, 2, 0, 3, 0}, // inds - 2, - 2, - 2, - raft::distance::DistanceType::L2SqrtExpanded}}; -typedef SparseKNNTest SparseKNNTestF; -TEST_P(SparseKNNTestF, Result) { compare(); } -INSTANTIATE_TEST_CASE_P(SparseKNNTest, SparseKNNTestF, ::testing::ValuesIn(inputs_i32_f)); - -}; // end namespace selection -}; // end namespace sparse -}; // end namespace raft diff --git a/cpp/test/sparse/neighbors/cross_component_nn.cu b/cpp/test/sparse/neighbors/cross_component_nn.cu deleted file mode 100644 index 13f7ae9de4..0000000000 --- a/cpp/test/sparse/neighbors/cross_component_nn.cu +++ /dev/null @@ -1,1036 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// XXX: We allow the instantiation of masked_l2_nn here: -// raft::linkage::FixConnectivitiesRedOp red_op(params.n_row); -// raft::linkage::cross_component_nn( -// handle, out_edges, data.data(), colors.data(), params.n_row, params.n_col, red_op); -// -// TODO: consider adding this to libraft.so or creating an instance in a -// separate translation unit for this test. -// -// TODO: edge case testing. Reference: https://github.com/rapidsai/raft/issues/1669 - -#include "../../test_utils.cuh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include - -#include - -namespace raft { -namespace sparse { - -using namespace std; - -template -struct ConnectComponentsInputs { - value_idx n_row; - value_idx n_col; - std::vector data; - - int c; -}; - -template -class ConnectComponentsTest - : public ::testing::TestWithParam> { - protected: - void basicTest() - { - raft::resources handle; - - auto stream = resource::get_cuda_stream(handle); - - params = ::testing::TestWithParam>::GetParam(); - - raft::sparse::COO out_edges(resource::get_cuda_stream(handle)); - raft::sparse::COO out_edges_batched(resource::get_cuda_stream(handle)); - - rmm::device_uvector data(params.n_row * params.n_col, - resource::get_cuda_stream(handle)); - - raft::copy(data.data(), params.data.data(), data.size(), resource::get_cuda_stream(handle)); - - rmm::device_uvector indptr(params.n_row + 1, stream); - - /** - * 1. Construct knn graph - */ - raft::sparse::COO knn_graph_coo(stream); - - raft::sparse::neighbors::knn_graph(handle, - data.data(), - params.n_row, - params.n_col, - raft::distance::DistanceType::L2SqrtExpanded, - knn_graph_coo, - params.c); - - raft::sparse::convert::sorted_coo_to_csr( - knn_graph_coo.rows(), knn_graph_coo.nnz, indptr.data(), params.n_row + 1, stream); - - /** - * 2. Construct MST, sorted by weights - */ - rmm::device_uvector colors(params.n_row, stream); - - auto mst_coo = raft::mst::mst(handle, - indptr.data(), - knn_graph_coo.cols(), - knn_graph_coo.vals(), - params.n_row, - knn_graph_coo.nnz, - colors.data(), - stream, - false, - true); - - /** - * 3. cross_component_nn to fix connectivities - */ - raft::linkage::FixConnectivitiesRedOp red_op(params.n_row); - raft::linkage::cross_component_nn(handle, - out_edges, - data.data(), - colors.data(), - params.n_row, - params.n_col, - red_op, - params.n_row, - params.n_col); - - raft::linkage::cross_component_nn(handle, - out_edges_batched, - data.data(), - colors.data(), - params.n_row, - params.n_col, - red_op, - params.n_row / 2, - params.n_col / 2); - - ASSERT_TRUE(out_edges.nnz == out_edges_batched.nnz); - - ASSERT_TRUE( - devArrMatch(out_edges.rows(), out_edges_batched.rows(), out_edges.nnz, Compare())); - - ASSERT_TRUE( - devArrMatch(out_edges.cols(), out_edges_batched.cols(), out_edges.nnz, Compare())); - - ASSERT_TRUE(devArrMatch( - out_edges.vals(), out_edges_batched.vals(), out_edges.nnz, CompareApprox(1e-4))); - - /** - * Construct final edge list - */ - rmm::device_uvector indptr2(params.n_row + 1, stream); - - raft::sparse::convert::sorted_coo_to_csr( - out_edges.rows(), out_edges.nnz, indptr2.data(), params.n_row + 1, stream); - - auto output_mst = raft::mst::mst(handle, - indptr2.data(), - out_edges.cols(), - out_edges.vals(), - params.n_row, - out_edges.nnz, - colors.data(), - stream, - false, - false); - - resource::sync_stream(handle, stream); - - // The sum of edges for both MST runs should be n_rows - 1 - final_edges = output_mst.n_edges + mst_coo.n_edges; - } - - void SetUp() override { basicTest(); } - - void TearDown() override {} - - protected: - ConnectComponentsInputs params; - - value_idx final_edges; -}; - -const std::vector> fix_conn_inputsf2 = { - // Test n_clusters == n_points - {10, - 5, - {0.21390334, 0.50261639, 0.91036676, 0.59166485, 0.71162682, 0.10248392, 0.77782677, 0.43772379, - 0.4035871, 0.3282796, 0.47544681, 0.59862974, 0.12319357, 0.06239463, 0.28200272, 0.1345717, - 0.50498218, 0.5113505, 0.16233086, 0.62165332, 0.42281548, 0.933117, 0.41386077, 0.23264562, - 0.73325968, 0.37537541, 0.70719873, 0.14522645, 0.73279625, 0.9126674, 0.84854131, 0.28890216, - 0.85267903, 0.74703138, 0.83842071, 0.34942792, 0.27864171, 0.70911132, 0.21338564, 0.32035554, - 0.73788331, 0.46926692, 0.57570162, 0.42559178, 0.87120209, 0.22734951, 0.01847905, 0.75549396, - 0.76166195, 0.66613745}, - -1}, - // Test n_points == 100 - {100, - 10, - {6.26168372e-01, 9.30437651e-01, 6.02450208e-01, 2.73025296e-01, 9.53050619e-01, 3.32164396e-01, - 6.88942598e-01, 5.79163537e-01, 6.70341547e-01, 2.70140602e-02, 9.30429671e-01, 7.17721157e-01, - 9.89948537e-01, 7.75253347e-01, 1.34491522e-02, 2.48522428e-02, 3.51413378e-01, 7.64405834e-01, - 7.86373507e-01, 7.18748577e-01, 8.66998621e-01, 6.80316582e-01, 2.51288712e-01, 4.91078420e-01, - 3.76246281e-01, 4.86828710e-01, 5.67464772e-01, 5.30734742e-01, 8.99478296e-01, 7.66699088e-01, - 9.49339111e-01, 3.55248484e-01, 9.06046929e-01, 4.48407772e-01, 6.96395305e-01, 2.44277335e-01, - 7.74840000e-01, 5.21046603e-01, 4.66423971e-02, 5.12019638e-02, 8.95019614e-01, 5.28956953e-01, - 4.31536306e-01, 5.83857744e-01, 4.41787364e-01, 4.68656523e-01, 5.73971433e-01, 6.79989654e-01, - 3.19650588e-01, 6.12579596e-01, 6.49126442e-02, 8.39131142e-01, 2.85252117e-01, 5.84848929e-01, - 9.46507115e-01, 8.58440748e-01, 3.61528940e-01, 2.44215959e-01, 3.80101125e-01, 4.57128957e-02, - 8.82216988e-01, 8.31498633e-01, 7.23474381e-01, 7.75788607e-01, 1.40864146e-01, 6.62092382e-01, - 5.13985168e-01, 3.00686418e-01, 8.70109949e-01, 2.43187753e-01, 2.89391938e-01, 2.84214238e-01, - 8.70985521e-01, 8.77491176e-01, 6.72537226e-01, 3.30929686e-01, 1.85934324e-01, 9.16222614e-01, - 6.18239142e-01, 2.64768597e-01, 5.76145451e-01, 8.62961369e-01, 6.84757925e-01, 7.60549082e-01, - 1.27645356e-01, 4.51004673e-01, 3.92292980e-01, 4.63170803e-01, 4.35449330e-02, 2.17583404e-01, - 5.71832605e-02, 2.06763039e-01, 3.70116249e-01, 2.09750028e-01, 6.17283019e-01, 8.62549231e-01, - 9.84156240e-02, 2.66249156e-01, 3.87635103e-01, 2.85591012e-02, 4.24826068e-01, 4.45795088e-01, - 6.86227676e-01, 1.08848960e-01, 5.96731841e-02, 3.71770228e-01, 1.91548833e-01, 6.95136078e-01, - 9.00700636e-01, 8.76363105e-01, 2.67334632e-01, 1.80619709e-01, 7.94060419e-01, 1.42854171e-02, - 1.09372387e-01, 8.74028108e-01, 6.46403232e-01, 4.86588834e-01, 5.93446175e-02, 6.11886291e-01, - 8.83865057e-01, 3.15879821e-01, 2.27043992e-01, 9.76764951e-01, 6.15620336e-01, 9.76199360e-01, - 2.40548962e-01, 3.21795663e-01, 8.75087904e-02, 8.11234663e-01, 6.96070480e-01, 8.12062321e-01, - 1.21958818e-01, 3.44348628e-02, 8.72630414e-01, 3.06162776e-01, 1.76043529e-02, 9.45894971e-01, - 5.33896401e-01, 6.21642973e-01, 4.93062535e-01, 4.48984262e-01, 2.24560379e-01, 4.24052195e-02, - 4.43447610e-01, 8.95646149e-01, 6.05220676e-01, 1.81840491e-01, 9.70831206e-01, 2.12563586e-02, - 6.92582693e-01, 7.55946922e-01, 7.95086143e-01, 6.05328941e-01, 3.99350764e-01, 4.32846636e-01, - 9.81114529e-01, 4.98266428e-01, 6.37127930e-03, 1.59085889e-01, 6.34682067e-05, 5.59429440e-01, - 7.38827633e-01, 8.93214770e-01, 2.16494306e-01, 9.35430573e-02, 4.75665868e-02, 7.80503518e-01, - 7.86240041e-01, 7.06854594e-01, 2.13725879e-02, 7.68246091e-01, 4.50234808e-01, 5.21231104e-01, - 5.01989826e-03, 4.22081572e-02, 1.65337732e-01, 8.54134740e-01, 4.99430262e-01, 8.94525601e-01, - 1.14028379e-01, 3.69739861e-01, 1.32955599e-01, 2.65563824e-01, 2.52811151e-01, 1.44792843e-01, - 6.88449594e-01, 4.44921417e-01, 8.23296587e-01, 1.93266317e-01, 1.19033309e-01, 1.36368966e-01, - 3.42600285e-01, 5.64505195e-01, 5.57594559e-01, 7.44257892e-01, 8.38231569e-02, 4.11548847e-01, - 3.21010077e-01, 8.55081359e-01, 4.30105779e-01, 1.16229135e-01, 9.87731964e-02, 3.14712335e-01, - 4.50880592e-01, 2.72289598e-01, 6.31615256e-01, 8.97432958e-01, 4.44764250e-01, 8.03776440e-01, - 2.68767748e-02, 2.43374608e-01, 4.02141103e-01, 4.98881209e-01, 5.33173003e-01, 8.82890436e-01, - 7.16149148e-01, 4.19664401e-01, 2.29335357e-01, 2.88637806e-01, 3.44696803e-01, 6.78171906e-01, - 5.69849716e-01, 5.86454477e-01, 3.54474989e-01, 9.03876540e-01, 6.45980000e-01, 6.34887593e-01, - 7.88039746e-02, 2.04814126e-01, 7.82251754e-01, 2.43147074e-01, 7.50951808e-01, 1.72799092e-02, - 2.95349590e-01, 6.57991826e-01, 8.81214312e-01, 5.73970708e-01, 2.77610881e-01, 1.82155097e-01, - 7.69797417e-02, 6.44792402e-01, 9.46950998e-01, 7.73064845e-01, 6.04733624e-01, 5.80094567e-01, - 1.67498426e-01, 2.66514296e-01, 6.50140368e-01, 1.91170299e-01, 2.08752199e-01, 3.01664091e-01, - 9.85033484e-01, 2.92909152e-01, 8.65816607e-01, 1.85222119e-01, 2.28814559e-01, 1.34286382e-02, - 2.89234322e-01, 8.18668708e-01, 4.71706924e-01, 9.23199803e-01, 2.80879188e-01, 1.47319284e-01, - 4.13915748e-01, 9.31274932e-02, 6.66322195e-01, 9.66953974e-01, 3.19405786e-01, 6.69486551e-01, - 5.03096313e-02, 6.95225201e-01, 5.78469859e-01, 6.29481655e-01, 1.39252534e-01, 1.22564968e-01, - 6.80663678e-01, 6.34607157e-01, 6.42765834e-01, 1.57127410e-02, 2.92132086e-01, 5.24423878e-01, - 4.68676824e-01, 2.86003928e-01, 7.18608322e-01, 8.95617933e-01, 5.48844309e-01, 1.74517278e-01, - 5.24379196e-01, 2.13526524e-01, 5.88375435e-01, 9.88560185e-01, 4.17435771e-01, 6.14438688e-01, - 9.53760881e-01, 5.27151288e-01, 7.03017278e-01, 3.44448559e-01, 4.47059676e-01, 2.83414901e-01, - 1.98979011e-01, 4.24917361e-01, 5.73172761e-01, 2.32398853e-02, 1.65887230e-01, 4.05552785e-01, - 9.29665524e-01, 2.26135696e-01, 9.20563384e-01, 7.65259963e-01, 4.54820075e-01, 8.97710267e-01, - 3.78559302e-03, 9.15219382e-01, 3.55705698e-01, 6.94905124e-01, 8.58540202e-01, 3.89790666e-01, - 2.49478206e-01, 7.93679304e-01, 4.75830027e-01, 4.40425353e-01, 3.70579459e-01, 1.40578049e-01, - 1.70386675e-01, 7.04056121e-01, 4.85963102e-01, 9.68450060e-01, 6.77178001e-01, 2.65934654e-01, - 2.58915007e-01, 6.70052890e-01, 2.61945109e-01, 8.46207759e-01, 1.01928951e-01, 2.85611334e-01, - 2.45776933e-01, 2.66658783e-01, 3.71724077e-01, 4.34319025e-01, 4.24407347e-01, 7.15417683e-01, - 8.07997684e-01, 1.64296275e-01, 6.01638065e-01, 8.60606804e-02, 2.68719187e-01, 5.11764101e-01, - 9.75844338e-01, 7.81226782e-01, 2.20925515e-01, 7.18135040e-01, 9.82395577e-01, 8.39160243e-01, - 9.08058083e-01, 6.88010677e-01, 8.14271847e-01, 5.12460821e-01, 1.17311345e-01, 5.96075228e-01, - 9.17455497e-01, 2.12052706e-01, 7.04074603e-01, 8.72872565e-02, 8.76047818e-01, 6.96235046e-01, - 8.54801557e-01, 2.49729159e-01, 9.76594604e-01, 2.87386363e-01, 2.36461559e-02, 9.94075254e-01, - 4.25193986e-01, 7.61869994e-01, 5.13334255e-01, 6.44711165e-02, 8.92156689e-01, 3.55235167e-01, - 1.08154647e-01, 8.78446825e-01, 2.43833016e-01, 9.23071293e-01, 2.72724115e-01, 9.46631338e-01, - 3.74510294e-01, 4.08451278e-02, 9.78392777e-01, 3.65079221e-01, 6.37199516e-01, 5.51144906e-01, - 5.25978080e-01, 1.42803678e-01, 4.05451674e-01, 7.79788219e-01, 6.26009784e-01, 3.35249497e-01, - 1.43159543e-02, 1.80363779e-01, 5.05096904e-01, 2.82619947e-01, 5.83561392e-01, 3.10951324e-01, - 8.73223968e-01, 4.38545619e-01, 4.81348800e-01, 6.68497085e-01, 3.79345401e-01, 9.58832501e-01, - 1.89869550e-01, 2.34083070e-01, 2.94066207e-01, 5.74892667e-02, 6.92106828e-02, 9.61127686e-02, - 6.72650672e-02, 8.47345378e-01, 2.80916761e-01, 7.32177357e-03, 9.80785961e-01, 5.73192225e-02, - 8.48781331e-01, 8.83225408e-01, 7.34398275e-01, 7.70381941e-01, 6.20778343e-01, 8.96822048e-01, - 5.40732486e-01, 3.69704071e-01, 5.77305837e-01, 2.08221827e-01, 7.34275341e-01, 1.06110900e-01, - 3.49496706e-01, 8.34948910e-01, 1.56403291e-02, 6.78576376e-01, 8.96141268e-01, 5.94835119e-01, - 1.43943153e-01, 3.49618530e-01, 2.10440392e-01, 3.46585620e-01, 1.05153093e-01, 3.45446174e-01, - 2.72177079e-01, 7.07946300e-01, 4.33717726e-02, 3.31232203e-01, 3.91874320e-01, 4.76338141e-01, - 6.22777789e-01, 2.95989228e-02, 4.32855769e-01, 7.61049310e-01, 3.63279149e-01, 9.47210350e-01, - 6.43721247e-01, 6.58025802e-01, 1.05247633e-02, 5.29974442e-01, 7.30675767e-01, 4.30041079e-01, - 6.62634841e-01, 8.25936616e-01, 9.91253704e-01, 6.79399281e-01, 5.44177006e-01, 7.52876048e-01, - 3.32139049e-01, 7.98732398e-01, 7.38865223e-01, 9.16055132e-01, 6.11736493e-01, 9.63672879e-01, - 1.83778839e-01, 7.27558919e-02, 5.91602822e-01, 3.25235484e-01, 2.34741217e-01, 9.52346277e-01, - 9.18556407e-01, 9.35373324e-01, 6.89209070e-01, 2.56049054e-01, 6.17975395e-01, 7.82285691e-01, - 9.84983432e-01, 6.62322741e-01, 2.04144457e-01, 3.98446577e-01, 1.38918297e-01, 3.05919921e-01, - 3.14043787e-01, 5.91072666e-01, 7.44703771e-01, 8.92272567e-01, 9.78017873e-01, 9.01203161e-01, - 1.41526372e-01, 4.14878484e-01, 6.80683651e-01, 5.01733152e-02, 8.14635389e-01, 2.27926375e-01, - 9.03269815e-01, 8.68443745e-01, 9.86939190e-01, 7.40779486e-01, 2.61005311e-01, 3.19276232e-01, - 9.69509248e-01, 1.11908818e-01, 4.49198556e-01, 1.27056715e-01, 3.84064823e-01, 5.14591811e-01, - 2.10747488e-01, 9.53884090e-01, 8.43167950e-01, 4.51187972e-01, 3.75331782e-01, 6.23566461e-01, - 3.55290379e-01, 2.95705968e-01, 1.69622690e-01, 1.42981830e-01, 2.72180991e-01, 9.46468040e-01, - 3.70932500e-01, 9.94292830e-01, 4.62587505e-01, 7.14817405e-01, 2.45370540e-02, 3.00906377e-01, - 5.75768304e-01, 9.71448393e-01, 6.95574827e-02, 3.93693854e-01, 5.29306116e-01, 5.04694554e-01, - 6.73797120e-02, 6.76596969e-01, 5.50948898e-01, 3.24909641e-01, 7.70337719e-01, 6.51842631e-03, - 3.03264879e-01, 7.61037886e-03, 2.72289601e-01, 1.50502041e-01, 6.71103888e-02, 7.41503703e-01, - 1.92088941e-01, 2.19043977e-01, 9.09320161e-01, 2.37993569e-01, 6.18107973e-02, 8.31447852e-01, - 2.23355609e-01, 1.84789435e-01, 4.16104518e-01, 4.21573859e-01, 8.72446305e-02, 2.97294197e-01, - 4.50328256e-01, 8.72199917e-01, 2.51279916e-01, 4.86219272e-01, 7.57071329e-01, 4.85655942e-01, - 1.06187277e-01, 4.92341327e-01, 1.46017513e-01, 5.25421017e-01, 4.22637906e-01, 2.24685018e-01, - 8.72648431e-01, 5.54051490e-01, 1.80745062e-01, 2.12756336e-01, 5.20883169e-01, 7.60363654e-01, - 8.30254678e-01, 5.00003328e-01, 4.69017439e-01, 6.38105527e-01, 3.50638261e-02, 5.22217353e-02, - 9.06516882e-02, 8.52975842e-01, 1.19985883e-01, 3.74926753e-01, 6.50302066e-01, 1.98875727e-01, - 6.28362507e-02, 4.32693501e-01, 3.10500685e-01, 6.20732833e-01, 4.58503272e-01, 3.20790034e-01, - 7.91284868e-01, 7.93054570e-01, 2.93406765e-01, 8.95399023e-01, 1.06441034e-01, 7.53085241e-02, - 8.67523104e-01, 1.47963482e-01, 1.25584706e-01, 3.81545040e-02, 6.34338619e-01, 1.76368938e-02, - 5.75553531e-02, 5.31607516e-01, 2.63869588e-01, 9.41945823e-01, 9.24028838e-02, 5.21496463e-01, - 7.74866558e-01, 5.65210610e-01, 7.28015327e-02, 6.51963790e-01, 8.94727453e-01, 4.49571590e-01, - 1.29932405e-01, 8.64026259e-01, 9.92599934e-01, 7.43721560e-01, 8.87300215e-01, 1.06369925e-01, - 8.11335531e-01, 7.87734900e-01, 9.87344678e-01, 5.32502820e-01, 4.42612382e-01, 9.64041183e-01, - 1.66085871e-01, 1.12937664e-01, 5.24423470e-01, 6.54689333e-01, 4.59119726e-01, 5.22774091e-01, - 3.08722276e-02, 6.26979315e-01, 4.49754105e-01, 8.07495757e-01, 2.34199499e-01, 1.67765675e-01, - 9.22168418e-01, 3.73210378e-01, 8.04432575e-01, 5.61890354e-01, 4.47025593e-01, 6.43155678e-01, - 2.40407640e-01, 5.91631279e-01, 1.59369206e-01, 7.75799090e-01, 8.32067212e-01, 5.59791576e-02, - 6.39105224e-01, 4.85274738e-01, 2.12630838e-01, 2.81431312e-02, 7.16205363e-01, 6.83885011e-01, - 5.23869697e-01, 9.99418314e-01, 8.35331599e-01, 4.69877463e-02, 6.74712562e-01, 7.99273684e-01, - 2.77001890e-02, 5.75809742e-01, 2.78513031e-01, 8.36209905e-01, 7.25472379e-01, 4.87173943e-01, - 7.88311357e-01, 9.64676177e-01, 1.75752651e-01, 4.98112580e-01, 8.08850418e-02, 6.40981131e-01, - 4.06647450e-01, 8.46539387e-01, 2.12620694e-01, 9.11012851e-01, 8.25041445e-01, 8.90065575e-01, - 9.63626055e-01, 5.96689242e-01, 1.63372670e-01, 4.51640148e-01, 3.43026542e-01, 5.80658851e-01, - 2.82327625e-01, 4.75535418e-01, 6.27760926e-01, 8.46314115e-01, 9.61961932e-01, 3.19806094e-01, - 5.05508062e-01, 5.28102944e-01, 6.13045057e-01, 7.44714938e-01, 1.50586073e-01, 7.91878033e-01, - 4.89839179e-01, 3.10496849e-01, 8.82309038e-01, 2.86922314e-01, 4.84687559e-01, 5.20838630e-01, - 4.62955493e-01, 2.38185305e-01, 5.47259907e-02, 7.10916137e-01, 7.31887202e-01, 6.25602317e-01, - 8.77741168e-01, 4.19881322e-01, 4.81222328e-01, 1.28224501e-01, 2.46034010e-01, 3.34971854e-01, - 7.37216484e-01, 5.62134821e-02, 7.14089724e-01, 9.85549393e-01, 4.66295827e-01, 3.08722434e-03, - 4.70237690e-01, 2.66524167e-01, 7.93875484e-01, 4.54795911e-02, 8.09702944e-01, 1.47709735e-02, - 1.70082405e-01, 6.35905179e-01, 3.75379109e-01, 4.30315011e-01, 3.15788760e-01, 5.58065230e-01, - 2.24643800e-01, 2.42142981e-01, 6.57283636e-01, 3.34921891e-01, 1.26588975e-01, 7.68064155e-01, - 9.43856291e-01, 4.47518596e-01, 5.44453573e-01, 9.95764932e-01, 7.16444391e-01, 8.51019765e-01, - 1.01179183e-01, 4.45473958e-01, 4.60327322e-01, 4.96895844e-02, 4.72907738e-01, 5.58987444e-01, - 3.41027487e-01, 1.56175026e-01, 7.58283148e-01, 6.83600909e-01, 2.14623396e-01, 3.27348880e-01, - 3.92517893e-01, 6.70418431e-01, 5.16440832e-01, 8.63140348e-01, 5.73277464e-01, 3.46608058e-01, - 7.39396341e-01, 7.20852434e-01, 2.35653246e-02, 3.89935659e-01, 7.53783745e-01, 6.34563528e-01, - 8.79339335e-01, 7.41599159e-02, 5.62433904e-01, 6.15553852e-01, 4.56956324e-01, 5.20047447e-01, - 5.26845015e-02, 5.58471266e-01, 1.63632233e-01, 5.38936665e-02, 6.49593683e-01, 2.56838748e-01, - 8.99035326e-01, 7.20847756e-01, 5.68954684e-01, 7.43684755e-01, 5.70924238e-01, 3.82318724e-01, - 4.89328290e-01, 5.62208561e-01, 4.97540804e-02, 4.18011085e-01, 6.88041565e-01, 2.16234653e-01, - 7.89548214e-01, 8.46136387e-01, 8.46816189e-01, 1.73842353e-01, 6.11627842e-02, 8.44440559e-01, - 4.50646654e-01, 3.74785037e-01, 4.87196697e-01, 4.56276448e-01, 9.13284391e-01, 4.15715464e-01, - 7.13597697e-01, 1.23641270e-02, 5.10031271e-01, 4.74601930e-02, 2.55731159e-01, 3.22090006e-01, - 1.91165703e-01, 4.51170940e-01, 7.50843157e-01, 4.42420576e-01, 4.25380660e-01, 4.50667257e-01, - 6.55689206e-01, 9.68257670e-02, 1.96528793e-01, 8.97343028e-01, 4.99940904e-01, 6.65504083e-01, - 9.41828079e-01, 4.54397338e-01, 5.61893331e-01, 5.09839880e-01, 4.53117514e-01, 8.96804127e-02, - 1.74888861e-01, 6.65641378e-01, 2.81668336e-01, 1.89532742e-01, 5.61668382e-01, 8.68330157e-02, - 8.25092797e-01, 5.18106324e-01, 1.71904024e-01, 3.68385523e-01, 1.62005436e-01, 7.48507399e-01, - 9.30274827e-01, 2.38198517e-01, 9.52222901e-01, 5.23587800e-01, 6.94384557e-01, 1.09338652e-01, - 4.83356794e-01, 2.73050402e-01, 3.68027050e-01, 5.92366466e-01, 1.83192289e-01, 8.60376029e-01, - 7.13926203e-01, 8.16750052e-01, 1.57890291e-01, 6.25691951e-01, 5.24831646e-01, 1.73873797e-01, - 1.02429784e-01, 9.17488471e-01, 4.03584434e-01, 9.31170884e-01, 2.79386137e-01, 8.77745206e-01, - 2.45200576e-01, 1.28896951e-01, 3.15713052e-01, 5.27874291e-01, 2.16444335e-01, 7.03883817e-01, - 7.74738919e-02, 8.42422142e-01, 3.75598924e-01, 3.51002411e-01, 6.22752776e-01, 4.82407943e-01, - 7.43107867e-01, 9.46182666e-01, 9.44344819e-01, 3.28124763e-01, 1.06147431e-01, 1.65102684e-01, - 3.84060507e-01, 2.91057722e-01, 7.68173662e-02, 1.03543651e-01, 6.76698940e-01, 1.43141994e-01, - 7.21342202e-01, 6.69471294e-03, 9.07298311e-01, 5.57080171e-01, 8.10954489e-01, 4.11120526e-01, - 2.06407453e-01, 2.59590556e-01, 7.58512718e-01, 5.79873897e-01, 2.92875650e-01, 2.83686529e-01, - 2.42829343e-01, 9.19323719e-01, 3.46832864e-01, 3.58238858e-01, 7.42827585e-01, 2.05760059e-01, - 9.58438860e-01, 5.66326411e-01, 6.60292846e-01, 5.61095078e-02, 6.79465531e-01, 7.05118513e-01, - 4.44713264e-01, 2.09732933e-01, 5.22732436e-01, 1.74396512e-01, 5.29356748e-01, 4.38475687e-01, - 4.94036404e-01, 4.09785794e-01, 6.40025507e-01, 5.79371821e-01, 1.57726118e-01, 6.04572263e-01, - 5.41072639e-01, 5.18847173e-01, 1.97093284e-01, 8.91767002e-01, 4.29050835e-01, 8.25490570e-01, - 3.87699807e-01, 4.50705808e-01, 2.49371643e-01, 3.36074898e-01, 9.29925118e-01, 6.65393649e-01, - 9.07275994e-01, 3.73075859e-01, 4.14044139e-03, 2.37463702e-01, 2.25893784e-01, 2.46900245e-01, - 4.50350196e-01, 3.48618117e-01, 5.07193932e-01, 5.23435142e-01, 8.13611417e-01, 8.92715622e-01, - 1.02623450e-01, 3.06088345e-01, 7.80461650e-01, 2.21453645e-01, 2.01419652e-01, 2.84254457e-01, - 3.68286735e-01, 7.39358243e-01, 8.97879394e-01, 9.81599566e-01, 7.56526442e-01, 7.37645545e-01, - 4.23976657e-02, 8.25922012e-01, 2.60956996e-01, 2.90702065e-01, 8.98388344e-01, 3.03733299e-01, - 8.49071471e-01, 3.45835425e-01, 7.65458276e-01, 5.68094872e-01, 8.93770930e-01, 9.93161641e-01, - 5.63368667e-02, 4.26548945e-01, 5.46745780e-01, 5.75674571e-01, 7.94599487e-01, 7.18935553e-02, - 4.46492976e-01, 6.40240123e-01, 2.73246969e-01, 2.00465968e-01, 1.30718835e-01, 1.92492005e-01, - 1.96617189e-01, 6.61271644e-01, 8.12687657e-01, 8.66342445e-01 - - }, - -4}}; - -typedef ConnectComponentsTest ConnectComponentsTestF_Int; -TEST_P(ConnectComponentsTestF_Int, Result) -{ - /** - * Verify the src & dst vertices on each edge have different colors - */ - EXPECT_TRUE(final_edges == params.n_row - 1); -} - -INSTANTIATE_TEST_CASE_P(ConnectComponentsTest, - ConnectComponentsTestF_Int, - ::testing::ValuesIn(fix_conn_inputsf2)); - -template -struct MutualReachabilityFixConnectivitiesRedOp { - value_t* core_dists; - value_idx m; - - DI MutualReachabilityFixConnectivitiesRedOp() : m(0) {} - - MutualReachabilityFixConnectivitiesRedOp(value_t* core_dists_, value_idx m_) - : core_dists(core_dists_), m(m_){}; - - typedef typename raft::KeyValuePair KVP; - DI void operator()(value_idx rit, KVP* out, const KVP& other) const - { - if (rit < m && other.value < std::numeric_limits::max()) { - value_t core_dist_rit = core_dists[rit]; - value_t core_dist_other = max(core_dist_rit, max(core_dists[other.key], other.value)); - - value_t core_dist_out; - if (out->key > -1) { - core_dist_out = max(core_dist_rit, max(core_dists[out->key], out->value)); - } else { - core_dist_out = out->value; - } - - bool smaller = core_dist_other < core_dist_out; - out->key = smaller ? other.key : out->key; - out->value = smaller ? core_dist_other : core_dist_out; - } - } - - DI KVP operator()(value_idx rit, const KVP& a, const KVP& b) const - { - if (rit < m && a.key > -1) { - value_t core_dist_rit = core_dists[rit]; - value_t core_dist_a = max(core_dist_rit, max(core_dists[a.key], a.value)); - - value_t core_dist_b; - if (b.key > -1) { - core_dist_b = max(core_dist_rit, max(core_dists[b.key], b.value)); - } else { - core_dist_b = b.value; - } - - return core_dist_a < core_dist_b ? KVP(a.key, core_dist_a) : KVP(b.key, core_dist_b); - } - - return b; - } - - DI void init(value_t* out, value_t maxVal) const { *out = maxVal; } - DI void init(KVP* out, value_t maxVal) const - { - out->key = -1; - out->value = maxVal; - } - - DI void init_key(value_t& out, value_idx idx) const { return; } - DI void init_key(KVP& out, value_idx idx) const { out.key = idx; } - - DI value_t get_value(KVP& out) const { return out.value; } - DI value_t get_value(value_t& out) const { return out; } - - void gather(const raft::resources& handle, value_idx* map) - { - auto tmp_core_dists = raft::make_device_vector(handle, m); - thrust::gather(raft::resource::get_thrust_policy(handle), - map, - map + m, - core_dists, - tmp_core_dists.data_handle()); - raft::copy_async( - core_dists, tmp_core_dists.data_handle(), m, raft::resource::get_cuda_stream(handle)); - } - - void scatter(const raft::resources& handle, value_idx* map) - { - auto tmp_core_dists = raft::make_device_vector(handle, m); - thrust::scatter(raft::resource::get_thrust_policy(handle), - core_dists, - core_dists + m, - map, - tmp_core_dists.data_handle()); - raft::copy_async( - core_dists, tmp_core_dists.data_handle(), m, raft::resource::get_cuda_stream(handle)); - } -}; - -template -struct ConnectComponentsMutualReachabilityInputs { - value_idx n_row; - value_idx n_col; - std::vector data; - std::vector core_dists; - std::vector colors; - std::vector expected_rows; - std::vector expected_cols; - std::vector expected_vals; -}; - -template -class ConnectComponentsEdgesTest - : public ::testing::TestWithParam> { - protected: - void basicTest() - { - raft::resources handle; - - auto stream = resource::get_cuda_stream(handle); - - params = ::testing::TestWithParam< - ConnectComponentsMutualReachabilityInputs>::GetParam(); - - raft::sparse::COO out_edges_unbatched(resource::get_cuda_stream(handle)); - raft::sparse::COO out_edges_batched(resource::get_cuda_stream(handle)); - - rmm::device_uvector data(params.n_row * params.n_col, - resource::get_cuda_stream(handle)); - rmm::device_uvector core_dists(params.n_row, resource::get_cuda_stream(handle)); - rmm::device_uvector colors(params.n_row, resource::get_cuda_stream(handle)); - - raft::copy(data.data(), params.data.data(), data.size(), resource::get_cuda_stream(handle)); - raft::copy(core_dists.data(), - params.core_dists.data(), - core_dists.size(), - resource::get_cuda_stream(handle)); - raft::copy( - colors.data(), params.colors.data(), colors.size(), resource::get_cuda_stream(handle)); - - /** - * 3. cross_component_nn to fix connectivities - */ - MutualReachabilityFixConnectivitiesRedOp red_op(core_dists.data(), - params.n_row); - - raft::linkage::cross_component_nn(handle, - out_edges_unbatched, - data.data(), - colors.data(), - params.n_row, - params.n_col, - red_op, - params.n_row, - params.n_col); - - raft::linkage::cross_component_nn(handle, - out_edges_batched, - data.data(), - colors.data(), - params.n_row, - params.n_col, - red_op, - 11, - 1); - - ASSERT_TRUE(out_edges_unbatched.nnz == out_edges_batched.nnz && - out_edges_unbatched.nnz == params.expected_rows.size()); - - ASSERT_TRUE(devArrMatch(out_edges_unbatched.rows(), - params.expected_rows.data(), - out_edges_unbatched.nnz, - Compare())); - - ASSERT_TRUE(devArrMatch(out_edges_unbatched.cols(), - params.expected_cols.data(), - out_edges_unbatched.nnz, - Compare())); - - ASSERT_TRUE(devArrMatch(out_edges_unbatched.vals(), - params.expected_vals.data(), - out_edges_unbatched.nnz, - CompareApprox(1e-4))); - - ASSERT_TRUE(devArrMatch(out_edges_batched.rows(), - params.expected_rows.data(), - out_edges_batched.nnz, - Compare())); - - ASSERT_TRUE(devArrMatch(out_edges_batched.cols(), - params.expected_cols.data(), - out_edges_batched.nnz, - Compare())); - - ASSERT_TRUE(devArrMatch(out_edges_batched.vals(), - params.expected_vals.data(), - out_edges_batched.nnz, - CompareApprox(1e-4))); - } - - void SetUp() override { basicTest(); } - - void TearDown() override {} - - protected: - ConnectComponentsMutualReachabilityInputs params; -}; - -const std::vector> mr_fix_conn_inputsf2 = { - {100, - 2, - {-7.72642, -8.39496, 5.4534, 0.742305, -2.97867, 9.55685, 6.04267, 0.571319, -6.52184, - -6.31932, 3.64934, 1.40687, -2.17793, 9.98983, 4.42021, 2.33028, 4.73696, 2.94181, - -3.66019, 9.38998, -3.05358, 9.12521, -6.65217, -5.57297, -6.35769, -6.58313, -3.61553, - 7.81808, -1.77073, 9.18565, -7.95052, -6.39764, -6.60294, -6.05293, -2.58121, 10.0178, - -7.76348, -6.72638, -6.40639, -6.95294, -2.97262, 8.54856, -6.95673, -6.53896, -7.32614, - -6.02371, -2.1478, 10.5523, -2.54502, 10.5789, -2.96984, 10.0714, 3.22451, 1.55252, - -6.25396, -7.73727, -7.85431, -6.09303, -8.11658, -8.20057, -7.55965, -6.64786, 4.936, - 2.23423, 4.44752, 2.27472, -5.72103, -7.70079, -0.929985, 9.78172, -3.10984, 8.72259, - -2.44167, 7.58954, -2.18511, 8.6292, 5.55528, 2.30192, 4.73164, -0.0143992, -8.2573, - -7.81793, -2.98837, 8.82863, 4.60517, 0.804492, -3.83738, 9.21115, -2.62485, 8.71318, - 3.57758, 2.44676, -8.48711, -6.69548, -6.70645, -6.49479, -6.86663, -5.42658, 3.83139, - 1.47141, 2.02013, 2.79507, 4.64499, 1.73858, -1.69667, 10.3705, -6.61974, -6.09829, - -6.05757, -4.98332, -7.10309, -6.16611, -3.52203, 9.32853, -2.26724, 7.10101, 6.11777, - 1.4549, -4.23412, 8.452, -6.58655, -7.59446, 3.93783, 1.64551, -7.12502, -7.63385, - 2.72111, 1.94666, -7.14428, -4.15994, -6.66553, -8.12585, 4.70011, 4.43641, -7.76914, - -7.69592, 4.11012, 2.48644, 4.89743, 1.89872, 4.29716, 1.17089, -6.62913, -6.53366, - -8.07093, -6.22356, -2.16558, 7.25125, 4.73953, 1.46969, -5.91625, -6.46733, 5.43091, - 1.06378, -6.82142, -8.02308, 6.52606, 2.14775, 3.08922, 2.04173, -2.14756, 8.36917, - 3.85663, 1.65111, -1.68665, 7.79344, -5.01385, -6.40628, -2.52269, 7.95658, -2.30033, - 7.05462, -1.04355, 8.78851, 3.72045, 3.5231, -3.98772, 8.29444, 4.24777, 0.509655, - 4.72693, 1.67416, 5.7827, 2.7251, -3.41722, 7.60198, 5.22674, 4.16363, -3.1109, - 10.8666, -3.18612, 9.62596, -1.4782, 9.94557, 4.47859, 2.37722, -5.79658, -5.82631, - -3.34842, 8.70507}, - {0.978428, 1.01917, 0.608673, 1.45629, 0.310713, 0.689461, 0.701126, 0.63296, 0.774788, - 0.701648, 0.513282, 0.757651, 0.45638, 0.973111, 0.901396, 0.613692, 0.482497, 0.688143, - 0.72428, 0.666345, 0.58232, 0.554756, 0.710315, 0.903611, 0.694115, 0.796099, 0.639759, - 0.798998, 0.639839, 1.30727, 0.663729, 0.57476, 0.571348, 1.14662, 1.26518, 0.485068, - 0.78207, 0.791621, 1.01678, 1.28509, 1.14715, 0.381395, 0.850507, 0.788511, 0.588341, - 0.878516, 0.928669, 0.405874, 0.776421, 0.612274, 1.84963, 0.57476, 0.95226, 0.488078, - 1.24868, 0.515136, 0.589378, 0.903632, 1.01678, 1.09964, 0.666345, 0.713265, 0.877168, - 1.10053, 1.96887, 1.03574, 2.03728, 0.969553, 0.774788, 0.586338, 0.65168, 0.435472, - 0.664396, 0.790584, 0.678637, 0.715964, 0.865494, 0.978428, 1.59242, 0.861109, 0.833259, - 0.65168, 0.903632, 1.49599, 0.76347, 0.960453, 1.1848, 1.37398, 0.928957, 1.07848, - 0.661798, 1.21104, 1.04579, 1.89047, 1.24288, 0.529553, 0.903611, 0.620897, 0.882467, - 0.647189}, - {0, 1, 2, 1, 0, 1, 2, 1, 1, 2, 2, 0, 0, 2, 2, 0, 0, 2, 0, 0, 2, 0, 0, 2, 2, - 2, 1, 0, 0, 0, 0, 1, 1, 0, 2, 2, 2, 2, 1, 1, 0, 2, 1, 2, 2, 1, 0, 0, 0, 1, - 1, 1, 2, 0, 0, 0, 2, 2, 1, 2, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 2, 1, - 0, 1, 0, 1, 1, 2, 1, 2, 0, 2, 2, 2, 1, 2, 1, 1, 1, 2, 1, 2, 2, 2, 1, 0, 2}, - {50, 54, 57, 63, 82, 87}, - {57, 63, 50, 54, 87, 82}, - {6.0764, 11.1843, 6.0764, 11.1843, 6.89004, 6.89004}}, - {1000, - 2, - {-6.59634, -7.13901, -6.13753, -6.58082, 5.19821, 2.04918, -2.96856, 8.16444, - -2.76879, 7.51114, -6.82261, -6.61152, 5.02008, 2.58376, 5.55621, 2.31966, - 4.86379, 3.33731, 5.84639, 1.15623, -2.17159, 8.60241, -4.97844, -6.94077, - -2.31014, 8.41407, 5.5582, 0.402669, 5.25265, 0.919754, 5.85298, 2.11489, - -3.29245, 8.69222, -1.9621, 8.81209, -1.53408, 8.86723, -2.18227, 8.79519, - 4.60519, 2.20738, -6.4759, -6.9043, -7.18766, -6.10045, -9.00148, -7.48793, - 4.01674, 1.41769, -2.45347, 10.1085, -3.20892, 9.22827, -3.18612, 9.62596, - 4.81977, 3.36517, 4.90693, 2.8628, -6.44269, -5.68946, -8.30144, -5.37878, - 4.61485, 2.79094, -1.98726, 9.31127, -3.66019, 9.38998, -6.58607, -8.23669, - -7.46015, -6.29153, 4.08468, 3.85433, -6.36842, -5.50645, -6.83602, -5.18506, - -0.627173, 10.3597, 3.98846, 1.48928, -2.9968, 8.58173, -7.2144, -7.28376, - -0.660242, 10.1409, -4.23528, -8.38308, -3.15984, 8.52716, -2.40987, 9.76567, - -8.7548, -6.76508, 4.56971, 0.312209, -7.5487, -5.8402, -1.6096, 9.32159, - 5.04813, 0.270586, -7.6525, -6.47306, -1.79758, 7.88964, -9.0153, -3.74236, - -3.5715, 9.48788, -1.65154, 8.85435, -3.47412, 9.70034, 6.31245, 2.39219, - 4.03851, 2.29295, -3.17098, 9.86672, -6.90693, -7.81338, -6.22373, -6.68537, - -3.22204, 9.12072, -0.365254, 9.6482, -7.76712, -7.31757, 4.15669, 3.54716, - 4.1937, 0.083629, -3.03896, 9.52755, -6.29293, -7.35501, -2.95926, 9.63714, - 4.02709, 1.58547, 4.56828, 1.93595, 5.6242, 1.75918, -7.36237, -7.83344, - 5.32177, 3.81988, -2.43183, 8.153, -1.97939, 10.4559, -3.49492, 9.51833, - 3.39602, 1.28026, -2.42215, 8.71528, -3.57682, 8.87191, -2.77385, 11.7345, - 5.71351, 0.946654, -6.50253, -6.90937, 4.08239, 0.603367, -5.64134, -6.85884, - -2.76177, 7.7665, -2.25165, 8.93984, -3.49071, 9.47639, -1.06792, 7.57842, - 5.15754, 1.24743, 3.63574, 1.20537, -6.07969, -8.49642, 4.12227, 2.19696, - -7.17144, -8.4433, -1.92234, 11.2047, 3.23237, 1.19535, 3.85389, 0.641937, - 4.82665, 1.21779, -7.68923, -6.45605, -7.00816, -8.76196, -5.12894, 9.83619, - -5.66247, -5.35879, 3.05598, 2.73358, 6.06038, 1.40242, -1.69568, 7.78342, - 5.13391, 2.23384, -2.96984, 10.0714, -5.36618, -6.2493, 5.55896, 1.6829, - 3.55882, 2.58911, 5.36155, 0.844118, -0.0634456, 9.14351, 4.88368, 1.40909, - -7.04675, -6.59753, -7.78333, -6.55575, 5.39881, 2.25436, -2.85189, 8.64285, - -2.22821, 8.39159, 3.88591, 1.69249, -7.55481, -7.02463, 4.60032, 2.65467, - -6.90615, -7.76198, -6.76005, -7.85318, 4.15044, 3.01733, -7.18884, -7.63227, - 4.68874, 2.01376, 3.51716, 2.35558, -3.81367, 9.68396, 4.42644, 3.4639, - 4.81758, 0.637825, -6.20705, -4.98023, -1.68603, 9.0876, -4.99504, -5.33687, - -1.77073, 9.18565, 4.86433, 3.02027, 4.20538, 1.664, 4.59042, 2.64799, - -3.09856, 9.86389, -3.02306, 7.95507, -6.32402, -6.79053, -7.67205, -7.18807, - -8.10918, -6.38341, -1.67979, 6.80315, 4.00249, 3.16219, -2.54391, 7.84561, - -3.22764, 8.80084, -2.63712, 8.05875, -2.41744, 7.02672, -6.71117, -5.56251, - 5.18348, 1.60256, -7.40824, -6.29375, -4.22233, 10.3682, 4.8509, 1.87646, - -2.99456, 9.09616, 5.1332, 2.15801, -2.27358, 9.78515, -6.73874, -8.64855, - 4.96124, 2.39509, -3.70949, 8.67978, -4.13674, 9.06237, 2.80367, 2.48116, - -0.876786, 7.58414, -3.7005, 9.67084, 6.48652, 0.903085, 6.28189, 2.98299, - -6.07922, -6.12582, -5.67921, -7.537, 4.55014, 3.41329, -1.63688, 9.19763, - -4.02439, 10.3812, 5.23053, 3.08187, -2.2951, 7.76855, -6.24491, -5.77041, - 6.02415, 2.53708, -6.91286, -7.08823, 4.83193, 1.66405, -7.07454, -5.74634, - -2.09576, 10.8911, 3.29543, 1.05452, -3.49973, 8.44799, 5.2922, 0.396778, - -2.54502, 10.5789, -6.38865, -6.14523, -1.75221, 8.09212, -9.30387, -5.99606, - -2.98113, 10.1032, -6.2017, -7.36802, 4.63628, 0.814805, -1.81905, 8.61307, - 4.88926, 3.55062, 3.08325, 2.57918, -2.51717, 10.4942, -5.75358, -6.9315, - 6.36742, 2.40949, 5.74806, 0.933264, 4.74408, 1.91058, -7.41496, -6.97064, - -2.98414, 8.36096, 6.72825, 1.83358, -2.95349, 9.39159, -3.35599, 7.49944, - 6.18738, 3.76905, -3.17182, 9.58488, 5.17863, 1.0525, -3.0397, 8.43847, - -2.23874, 8.96405, 3.04689, 2.41364, 6.14064, 2.82339, -6.33334, -6.87369, - -7.92444, -8.84647, 3.65129, 0.86958, 5.29842, 3.98337, -2.06538, 9.78892, - -6.89494, -6.30082, -2.52144, 8.11703, -8.11398, -7.47257, 5.3381, 2.36666, - -6.93452, -6.59456, -7.50634, -6.01772, 6.23438, 1.12621, -2.15218, 8.32138, - -7.04777, -7.3522, -2.52771, 8.72563, -2.77907, 8.03552, 4.29123, 1.62391, - -8.07551, -6.43551, -3.28202, 8.77747, -2.21308, 9.27534, -8.25153, -8.49367, - -3.54644, 8.82395, -8.05867, -5.69243, 4.46681, 1.98875, 3.8362, 3.61229, - -6.96231, -7.00186, 5.18993, 1.00483, -5.35116, -6.37227, 5.23298, 1.66362, - -5.68306, -7.03864, -9.03144, -7.59926, -6.10127, -7.4313, 4.83572, 0.994797, - -7.32695, -5.59909, 0.569683, 10.1339, 3.35957, 2.84563, -2.4122, 9.60944, - 5.00855, 1.57983, -2.57528, 7.80327, 3.96349, 3.77411, 4.59429, 2.21651, - -6.54765, -6.68961, 4.76798, 1.29212, -1.67351, 7.88458, 5.63615, 1.47941, - -2.5301, 9.13161, 4.26075, 1.76959, 4.67788, 2.0932, 4.39955, 1.59835, - 3.91274, 1.72565, -4.1786, 9.55765, -7.34566, -8.47481, 4.8364, 2.68217, - -7.36848, -7.99973, -5.84708, -5.7534, 5.37252, 1.89245, -2.1707, 8.599, - -1.3299, 9.0818, -6.79122, -5.40258, 5.56391, 1.78827, -0.194539, 7.14702, - 4.60489, 3.74397, 5.50995, 2.46885, -3.98772, 8.29444, -5.21837, -7.33721, - -1.63959, 10.3699, -5.92932, -5.1695, -5.88358, -7.6369, 4.11716, 3.02218, - -6.54114, -7.17551, 3.97179, 2.96521, -6.75325, -4.94118, 5.26169, 0.402945, - 3.25031, 0.327771, -0.44845, 10.7696, -2.15141, 9.57507, 7.04329, 1.91555, - -3.74615, 7.69383, -7.52318, -5.85015, -6.80419, -8.48208, -4.57664, 8.92517, - 4.57574, 2.30193, 4.84098, 3.02382, -9.43355, -5.94579, -3.52203, 9.32853, - 3.43018, 2.5731, -6.15725, -7.25294, -6.69861, -8.17694, -2.40955, 8.51081, - -4.82342, -7.98332, -7.10611, -6.51274, 5.86755, 0.763529, -6.56045, -5.53966, - -3.61553, 7.81808, 4.3825, 0.304586, -6.52818, -5.80996, 4.59972, 0.542395, - -6.90603, -6.59995, -6.3585, -6.23489, -6.01915, -7.46319, -5.38694, -7.15123, - -7.83475, -6.45651, 5.89564, 1.07856, -5.15266, -7.27975, -6.97978, -7.08378, - 5.83493, 0.449983, -2.62374, 10.2521, -7.34494, -6.98606, -6.79719, -8.33766, - 3.54757, 1.65676, -8.40528, -5.61753, -5.85556, -6.28758, 4.66862, 3.25162, - -6.26047, -4.82261, 4.61552, 4.11544, -1.36637, 9.76622, 4.2517, 2.14359, - -2.45099, 7.87132, -0.376164, 7.0622, 4.34493, 3.22091, 6.95921, 2.36649, - -6.70319, -7.24714, -5.56932, -5.48443, -7.43149, -4.32191, -3.23956, 9.23074, - -5.77255, -7.00049, 4.96601, 0.722056, -7.88617, -5.74023, 4.18757, -0.45071, - -7.12569, -7.72336, 5.27366, 2.38697, 3.93487, 1.9174, 3.19186, -0.225636, - -3.41722, 7.60198, -3.08286, 8.46743, -5.87905, -7.55073, -5.26425, -7.20243, - -2.97867, 9.55685, -1.23153, 8.42272, -2.33602, 9.3996, -3.33819, 8.45411, - -3.58009, 9.49676, 3.78152, 2.67348, -1.54582, 9.42707, -4.04331, 10.292, - 3.3452, 3.134, -2.75494, 8.74156, -3.26555, 7.59203, -7.27139, -7.80252, - 3.5293, 3.72544, 6.11642, 3.35326, 4.01611, 3.8872, 4.89591, 2.95586, - -7.06677, -5.89438, 4.19438, 3.42655, -6.11355, -5.65318, -7.59645, -8.74665, - -5.80362, -6.8588, 3.80453, 4.11832, 5.70655, 3.14247, -4.98084, 8.21739, - -1.87642, 11.285, 4.39864, 2.32523, -3.48388, 9.80137, 4.02836, 0.566509, - -2.41212, 9.98293, -5.40846, -7.08943, 4.01506, 1.99926, -3.43613, 8.95476, - -7.24458, -7.71932, 6.02204, 2.62188, -6.29999, -6.55431, 6.19038, 0.974816, - 3.55882, 3.02632, -7.06011, -3.687, -1.55877, 8.43738, -5.14711, -4.64881, - 4.7167, 0.690177, -7.90381, -5.02602, 4.17218, 2.31967, -0.643423, 9.48812, - -7.95237, -6.64086, -4.05986, 9.08285, -6.24158, -6.37927, -6.6105, -7.2233, - -6.21675, -5.70664, -3.29967, 9.48575, 3.41775, 2.68617, -2.24948, 8.10997, - -2.24931, 9.79611, -9.0523, -6.03269, -2.2587, 9.36073, 5.20965, 2.42088, - -3.10159, 8.1503, -6.67906, -5.73147, 4.0687, 2.54575, -1.24229, 8.30662, - -2.09627, 8.45056, -7.87801, -6.57832, 4.72216, 3.03865, -0.929985, 9.78172, - -8.56307, -7.68598, -7.05257, -5.1684, -7.09076, -7.86729, 4.61432, 3.1459, - -6.34133, -5.8076, -3.82943, 10.8457, -8.46082, -5.98507, 5.34763, 1.4107, - -1.68714, 10.9111, -1.67886, 8.1582, -0.623012, 9.18886, -4.21258, 8.95874, - -2.16744, 10.8905, -6.57158, -7.27176, 2.14047, 4.26411, -8.44217, -7.40916, - 5.29008, 1.87399, 4.31824, 4.04992, -3.77008, 9.93215, -2.72688, 10.1131, - -6.14278, -7.16144, -3.92457, 8.59364, -5.92649, -6.59299, 4.68369, 1.82617, - -6.89905, -7.18329, 3.95173, 4.22561, -7.66453, -6.23183, -2.44167, 7.58954, - -6.36603, -7.41281, -6.45081, -6.187, -6.6125, -6.37138, 5.46036, 2.48044, - -2.14756, 8.36917, -2.3889, 9.52872, 3.80752, 2.44459, -3.98778, 10.158, - -6.63887, -4.27843, -8.65266, -5.61819, -7.97003, -5.46918, -5.9604, -7.54825, - -0.916011, 8.50307, -3.69246, 6.97505, -7.98533, -7.09503, -2.30033, 7.05462, - 4.76218, 2.51647, -7.04981, -7.33334, 3.66401, 3.02681, -2.50408, 8.7797, - 7.19996, 1.87711, 4.01291, 3.78562, -0.356015, 8.24694, -0.958046, 9.12996, - 4.60675, 3.76773, 6.21945, 1.45031, 4.27744, 0.8535, -4.72232, -7.48582, - 6.03923, 2.8978, -3.26833, 9.16468, -7.97059, -7.29092, -2.3998, 9.74005, - -2.66721, 8.58741, -7.36269, -6.73332, -7.87893, -7.38488, 4.65023, 0.661333, - -4.8171, -7.94764, -4.11564, 9.21775, 4.80633, 2.46562, -2.72887, 9.3714, - -5.26735, -5.5652, 4.9826, 2.42992, -6.17018, -7.3156, 4.38084, 1.77682, - 5.35084, 2.41743, -2.61796, 9.416, 5.27229, 2.94572, -7.52315, -5.95227, - -1.45077, 7.25555, -3.79916, 7.71921, -2.23251, 9.84147, 3.70054, 1.82908, - -1.93831, 10.1499, -6.18324, -5.9248, -3.33142, 9.25797, -6.08536, -8.1344, - 5.95727, 2.17077, 4.87366, 0.417274, -6.529, -6.39092, -9.24256, -7.88984, - -6.36652, -7.13966, -3.90777, 9.57726, -7.06252, -5.50523, -2.26423, 8.50734, - -2.84498, 10.6833, 5.0391, 2.62037, -2.74815, 8.10672, 3.35945, 3.72796, - -4.11668, 9.19892, 5.66903, 2.44577, -1.63807, 8.68826, -7.42587, -6.48831, - 6.17063, 3.19193, -2.28511, 9.02688, -7.10088, -7.15692, 4.46293, 1.17487, - -5.91017, -6.45292, -2.26724, 7.10101, -2.43339, 8.33712, -4.63309, 8.48853, - -3.31769, 8.51253, -2.49078, 10.6907, -1.30798, 8.60621, 6.30535, 2.98754, - -5.79384, -6.78213, -1.93213, 8.81124, 4.55773, 3.09047, 6.37584, 2.17108, - 4.3927, 1.29119, -3.2245, 9.69388, -1.69634, 9.64392, 2.799, 0.693593, - -2.1426, 8.07441, -8.4505, -8.00688, 4.736, 1.51089, -2.5863, 9.35544, - -2.94924, 9.14503, 6.2054, 1.90742, 5.67172, 0.487609, -5.69071, -6.17181, - -8.24651, -7.10488, -7.34424, -6.67895, -6.71977, -7.90778, -1.82294, 7.40157, - -9.40991, -7.16611, -4.37999, 8.66277, -1.42615, 10.0681, -2.00828, 8.03673, - -7.50228, -6.6855, -5.65859, -6.29801, -8.02335, -6.77155, -3.40761, 9.50621, - -2.82447, 9.77326, -1.5938, 9.34304, -3.5213, 7.35943, -3.36961, 8.62973, - -7.01708, -5.92724, 5.20886, 3.60157, -1.71817, 8.1049, -2.46363, 8.36269, - -2.77809, 7.90776, -2.75459, 8.26055, -2.03596, 8.94146, -4.53434, 9.20074, - -7.44387, -6.69556, -6.90099, -7.62732, 3.29169, 2.71643, 6.08686, 2.16972, - -2.31111, 8.86993, -5.75046, 7.9899, 4.69951, 1.32623, 4.71851, -0.025031, - -6.42374, -4.71511, -8.04974, -8.68209, -3.16103, 9.06168, -6.18267, -7.21393, - -7.94202, -6.4518, -7.07697, -7.03138, 3.93554, 0.564708, -1.20372, 9.03529, - -7.10611, -7.83955, -7.47529, -5.50567, -6.15453, -6.36393, -2.98024, 9.24634, - -7.75761, -7.70699, -3.08597, 9.76968, -8.04954, -9.75237, 5.2534, 0.950377, - 5.63789, -0.923086, -5.7065, -6.51047, -8.02132, -7.07377, -8.28594, -6.96322, - -7.70722, -6.79397, -2.4962, 10.4678, 5.02846, 4.46617, 4.02648, 1.6707, - -0.319395, 8.20599, 4.74525, 0.639144, -1.0313, 8.49602, 4.08766, 2.6061, - 3.63826, 1.69207, 2.55795, 3.66963, 5.2826, 3.30232, -1.04355, 8.78851, - -6.84762, -7.63353, -4.70868, -7.056, 3.53651, -0.179721, -3.38482, 7.63149, - -5.9265, -6.36702, -0.986074, 9.5532, -2.42261, 8.85861, -7.42835, -6.78726, - -4.02857, 8.53005, -8.22675, -7.85172, -5.57529, -8.5426, 6.03009, 2.53098, - -7.10448, -7.53011, -3.4988, 8.8885, -2.62485, 8.71318, -6.39489, -7.72647, - 3.93789, 1.31027, 4.27627, 1.91622, -0.923181, 7.77647, -5.16017, 10.1058, - -6.44307, -5.97617, -7.24495, -6.69543, 6.27331, 0.826824, -6.55655, -7.13246, - 5.66245, 4.41292, -2.13805, 8.4103, 5.23463, 2.82659, -4.86624, -6.74357, - -6.14082, -6.26474, -2.67048, 9.41834, -1.26311, 6.9409, -7.20231, -7.13094, - -1.35109, 9.80595, 3.9906, 0.749229, -6.75696, -5.25543, 4.84826, -0.0685652, - -7.4914, -6.91715, 4.46725, 2.85683, -2.95571, 9.87068, 6.32381, 1.51429, - -6.81177, -6.02734, -2.57188, 9.96943, -4.28792, 10.5103, 3.65025, 2.91394, - -7.11856, -7.24693, -6.98693, -6.43239, 4.7651, 1.54376, 4.00092, 0.65008, - -7.14816, -7.7713, -7.58803, -8.39382, 4.3321, 2.19232, -7.89545, -6.81843, - -2.11475, 8.5933, -0.743743, 9.41927, 3.64849, -0.18022, -1.68665, 7.79344, - 4.00214, 1.44217, -6.96799, -7.25012, -1.58302, 10.9237, -6.68524, -7.23328, - 4.65831, 2.32075, 4.62024, 2.52566, -4.23412, 8.452, -0.822056, 9.89593, - -7.19868, -7.67614, -3.32742, 11.1067, 5.27861, 0.830165, 4.48982, 2.09875, - -6.58087, -7.6319, -0.880582, 7.63418, -7.01088, -6.80326, -7.31601, -6.98972, - -6.85883, -7.60811, 6.14328, 2.85053, -7.49206, -6.51861, -2.28174, 10.3214, - 4.81074, 1.78919, -5.58987, -6.20693, 4.08096, 2.35038, -1.5029, 8.43739, - 4.11536, 2.46254, -3.28299, 7.76963, 4.31953, 2.39734, 4.91146, 0.696421, - -1.4782, 9.94557, -3.34842, 8.70507, -6.97822, -6.86126, 4.10012, 1.19486, - -2.50395, 9.06127, 4.41891, 2.00006, -2.73266, 9.72829, 3.5436, 0.533119, - 5.78864, 0.233456, -6.62589, -6.41242, -2.21942, 11.0897, -6.76636, -8.31839, - -2.71732, 8.52129, -5.20972, -6.48544, 3.26056, 1.24224, 3.45228, 2.28299, - 4.72171, 1.87428, -7.52585, -5.1048, 5.0695, 2.18086, -6.55646, -7.02771, - 3.23727, 3.72275, 3.41411, 0.508795, -7.80698, -6.64174, -5.90443, -6.37902, - -0.387041, 10.0468, -1.3506, 8.1936, -6.08614, -8.62864, -5.91478, -5.26453, - -2.61623, 7.97904, 4.45459, 1.84335, -6.66643, -7.63208, 3.6729, 1.92546, - -1.32976, 8.54511, 6.31758, 1.41958, 4.63381, 2.81166, -7.01394, -6.0693, - -2.7786, 9.73183, -2.90131, 7.55077, -7.13842, -5.28146, 6.71514, 1.28398, - -6.98408, -7.04893, -3.03946, 8.22141, -2.76417, 10.5183, -7.35347, -6.89456, - 4.19345, 2.16726, -2.02819, 9.23817, 4.97076, 2.8067, -0.544473, 9.04955, - 4.90727, 2.29487, -6.31871, -7.17559, 3.71665, 0.621485, 4.7903, 2.33813, - -6.47994, -7.53147, -6.80958, -5.71823, -8.07326, -5.96096, 4.77342, 1.8207, - 5.71856, 1.93466, -2.70156, 9.31583, -2.1478, 10.5523, 4.78855, 1.63608, - 5.53507, 2.60834, -7.00058, -6.46058, 5.4738, 2.43235, -1.34603, 9.02452, - -7.5337, -8.71074, -7.30893, -7.57253, -5.33752, -4.87402, -7.01364, -6.86542, - -7.93331, -7.94791, -5.69392, -6.16116, -7.32291, -7.76491, -6.41965, -7.55783, - -7.87996, -7.55785, -6.69005, -5.87906, 3.92147, 2.86809, -1.5552, 9.66568, - 5.07989, 1.47112, -7.48524, -5.0541, -1.82724, 8.70402, -2.00421, 9.88004, - -2.62153, 8.79332, -7.52111, -6.44819, 4.06424, 2.09518, -6.65494, -5.94752, - 6.93878, 1.61033, -3.95728, 7.60682, 5.67016, 2.21196, -7.81507, -5.79413, - -2.41152, 8.24128, -3.83738, 9.21115, 4.5516, 4.55288, -5.75551, -5.93258, - 4.56545, 2.59384, -7.45614, -9.47115, -2.39568, 9.67642, 5.57816, 1.45712, - -7.48184, -6.41134, -1.99415, 12.867, -8.35854, -6.69675, -7.52559, -7.6793, - 5.7454, 3.1602, 2.94692, 1.87483, -8.77324, -6.66682, -3.21125, 8.68662, - -6.25806, -7.24972, 5.17639, 1.0747, -2.44897, 11.4775, -3.30172, 8.89955, - -2.85191, 8.21201, -8.85893, -6.1322, 4.08957, 1.30155, -5.88132, -7.31173, - -7.10309, -7.22943, -2.46068, 8.18334, -7.01226, -7.85464, 4.75411, 2.12347, - -3.42862, 10.5642, 7.16681, 1.4423, 5.42568, 2.39863, -6.00833, -8.22609, - -1.7619, 9.62466, -2.49527, 8.99016, -2.98837, 8.82863, -2.97262, 8.54856, - -1.34142, 9.26871, -5.99652, -6.95795, -1.87061, 7.35277, -8.68277, -8.46425, - -7.01808, -8.10441, -7.04269, -7.62501, -7.69783, -6.88348, -2.19829, 10.4896, - 4.67396, 1.2032, -5.58263, -6.90298, -5.69224, -4.29055, 4.77285, 1.27305, - -3.33469, 8.6929, -2.54195, 8.47086, 4.46492, 1.21742, 5.41158, -0.875373, - -8.68069, -7.42278, -3.88687, 8.07646, 4.6682, 2.00293, -8.29799, -8.64092, - -1.86382, 10.3829, -6.51234, -5.04193, 4.54458, 2.25219, -1.93264, 9.32554, - -3.06285, 7.81641, -6.90714, -5.10786, 4.69653, 2.50286, 6.43757, 2.61401, - -1.85483, 8.9587, 4.60224, 3.07647, 4.4492, 2.1906, 5.02181, 2.40321, - -2.22923, 7.8888, 5.68943, 1.43793, -6.71097, -6.43817, -5.00633, -5.80006, - -2.43763, 8.53663, 5.72577, 2.44787, -6.57079, -5.17789, -5.77867, -4.92176, - -6.57222, -6.06437, 3.96639, 2.25216, -7.95177, -9.80146, 4.92574, 2.30763, - -7.6221, -8.20013, -6.4132, -6.91575, 4.01432, 2.36897, 3.0833, 1.54505, - -1.99416, 9.52807, -7.85128, -8.25973, -0.86423, 8.76525, -6.31412, -8.64087, - -8.07355, -6.73717, -2.52821, 8.01176, -5.82357, -6.65687, -7.08865, -7.73063, - -5.56251, -6.99818, -2.12513, 8.98159, -6.89834, -7.26863, -7.92654, -6.34346, - 4.86201, 1.49442, 4.92905, 4.42847, -5.57789, -5.3186, 4.34232, 3.34888, - 2.64614, 2.34723, -4.10363, 8.41491, -2.18648, 8.18706, -3.39871, 8.19848, - -2.66098, 9.6026, -6.95927, -6.42774, -5.61392, -7.74628, 5.60376, 4.18369, - 5.28536, 4.13642, 4.8428, 0.457426, -6.33816, -6.12095, -2.4394, 8.62897, - 4.56938, 2.45967, 4.0582, 0.958413, 5.62164, 1.64834, 5.73119, 2.58231, - 4.66806, 1.96405, -6.71905, -6.87706, -2.18503, 8.88414, -6.03901, -6.33338, - -8.38435, -6.12005, 0.0641622, 9.0735, 5.19967, 3.05395, -5.48716, -7.13016, - -6.85541, -5.46789, -1.88353, 8.15713, 4.27891, 3.1325, -2.75816, 9.98586, - -2.03022, 9.34795, -7.66741, -7.50096, -3.39305, 9.16801, -8.49476, -5.71537, - -1.68378, 9.8278, -7.41559, -6.07205, -3.15577, 7.93274, 5.22381, 1.61388, - 3.65739, 1.74854, 4.94251, 1.21889, -7.12832, -5.27276, -9.58286, -6.20223, - -2.21613, 8.29993, 5.34799, 2.92987, 4.09496, 2.37231, -7.25183, -5.79136, - -6.46981, -7.12137, -6.28607, -9.8205, 4.52865, 1.06926, -3.10984, 8.72259, - 3.61865, 2.68153, -5.96604, -7.68329, 3.11435, 1.28126, -1.1064, 7.61243, - -2.17688, 8.2658, -3.27246, 7.2094, -5.55143, -6.32388, -1.69667, 10.3705, - -2.16558, 7.25125, -6.36572, -6.70053, 4.12259, 3.38252, -4.80554, -7.79949, - -5.23966, -6.13798, 4.21969, 1.69139, -1.98985, 10.547, -2.52269, 7.95658, - -6.75642, -6.32862, -3.51521, 7.8001, 4.70435, -0.00229688, 6.25359, 2.4267, - 5.82935, 0.745562, 5.24778, 2.15978, 5.48052, 1.32055, -3.05358, 9.12521, - -3.18922, 9.24654, 4.47276, 2.11988, 5.36751, 2.02512, -2.18511, 8.6292, - -2.48469, 9.51228, 5.57556, 3.24472, -2.58121, 10.0178, -6.12629, -6.49895, - -4.54732, 8.0062, -4.20166, 10.5438, -7.61422, -7.69036, -4.42797, 8.98777, - 4.45301, 1.53344, 4.59296, 2.45021, -6.81264, -6.36417, 4.62346, 3.16156, - -5.93007, -8.36501, -2.78425, 6.71237, -6.17141, -6.64689, -5.20608, 8.95999, - -7.30598, -5.73166, 4.39572, 2.93726, -1.89503, 9.77179, -5.683, -7.48989, - 4.80924, 0.559455, -2.17793, 9.98983, 5.23728, 2.67434, -7.03976, -6.20877, - 3.90435, 3.20926, -7.78536, -7.53388, -1.00684, 9.08838, -5.26741, -5.98327, - 3.28002, 2.71942, -1.47166, 8.50427, -2.32733, 9.26251, 5.16271, 1.39947, - -6.59093, -6.61979, -2.44492, 7.93654, -1.05805, 9.97356, -3.1109, 10.8666, - 3.38834, 3.41693, 4.83098, 2.01961, -2.74013, 9.71049, -3.34892, 8.41489, - 4.94768, 0.263001, 3.57477, 1.66795, 5.78915, 1.26999, -4.81812, -5.67174, - -1.88508, 9.64263, 3.69048, 4.60555, 4.03037, 1.7862, -7.4418, -7.08933}, - {0.127717, 0.211407, 0.195547, 0.21633, 0.39671, 0.229008, 0.20839, 0.169236, 0.314314, - 0.322473, 0.169506, 0.45499, 0.147819, 0.296502, 0.15198, 0.356444, 0.0992833, 0.220833, - 0.296206, 0.178067, 0.135359, 0.189725, 0.243099, 0.519986, 0.168105, 0.273465, 0.126033, - 0.18045, 0.282832, 0.193901, 0.213704, 0.425046, 0.203191, 0.228674, 0.209267, 0.355039, - 0.212918, 0.315495, 0.294112, 0.257576, 0.5786, 0.186019, 0.171919, 0.171919, 0.449151, - 1.34947, 0.171919, 0.16341, 0.641387, 0.342115, 0.267343, 0.246125, 0.277612, 0.181462, - 0.22944, 1.95598, 0.164897, 0.235803, 0.228273, 0.314629, 0.127403, 0.241241, 0.189362, - 0.151691, 0.130085, 0.526707, 0.217069, 0.282306, 0.531523, 0.177035, 0.169776, 0.20395, - 0.177165, 0.146628, 0.280013, 0.223033, 0.50947, 0.184133, 0.295329, 0.183219, 0.28166, - 0.179348, 0.276462, 1.00283, 0.248147, 0.214453, 0.231732, 0.170672, 0.256893, 0.133271, - 0.151137, 0.500823, 0.23678, 0.376983, 0.362061, 0.140013, 0.388863, 0.398552, 0.38015, - 0.190081, 0.167115, 0.206884, 0.473849, 1.05117, 0.435665, 0.323618, 0.326201, 0.32226, - 0.201787, 0.246496, 0.28325, 0.226596, 0.238153, 0.277268, 0.674629, 0.179433, 0.175651, - 0.154778, 0.178195, 0.192796, 0.103571, 0.227621, 0.201124, 0.160525, 0.160964, 0.240099, - 0.258027, 0.134127, 0.127717, 0.341378, 0.311595, 0.282306, 0.168988, 0.40775, 0.246125, - 0.583131, 0.236804, 0.238633, 0.194824, 0.169315, 0.244227, 0.249511, 0.189725, 0.305662, - 0.301415, 0.658641, 0.250944, 0.151792, 0.141383, 0.143843, 0.563347, 0.184216, 0.204155, - 0.221764, 0.314908, 0.144518, 0.228808, 0.255785, 0.163457, 0.424705, 0.170202, 0.312598, - 0.300629, 0.532614, 0.661392, 0.228273, 0.543432, 0.257175, 0.258994, 0.281413, 0.273897, - 0.246837, 0.293489, 0.25533, 0.260492, 0.213704, 0.3091, 0.17103, 0.172285, 0.241399, - 0.35999, 0.372243, 0.269191, 0.390239, 0.31761, 0.200593, 0.22197, 0.752914, 0.266571, - 0.13102, 0.268659, 0.293723, 0.356294, 0.296258, 0.264531, 0.15468, 0.358535, 0.243711, - 0.112147, 0.121659, 0.197101, 0.515292, 0.245628, 0.279863, 0.789807, 0.195156, 0.196073, - 0.149564, 0.118675, 0.389373, 0.233821, 0.176128, 0.481088, 0.360027, 0.553152, 0.208207, - 0.171608, 0.160489, 0.334298, 0.139426, 0.168603, 0.266199, 0.326458, 0.103571, 0.171208, - 0.130961, 0.190887, 0.177229, 0.241651, 0.115152, 0.196753, 0.481088, 0.230965, 0.354631, - 0.14591, 0.328543, 0.141544, 0.195888, 0.290379, 0.245954, 0.184547, 0.575214, 0.186929, - 0.28527, 0.292213, 1.20033, 0.281528, 0.15625, 0.211524, 0.186398, 0.298061, 0.147393, - 0.245349, 0.164527, 0.224771, 0.222382, 0.251643, 0.148835, 0.135359, 0.204967, 0.193024, - 0.486309, 0.389686, 0.211921, 0.307405, 0.38666, 0.26802, 0.16605, 0.323134, 0.268397, - 0.217894, 0.974118, 0.371618, 0.156201, 0.305787, 0.339305, 0.371032, 0.381765, 0.22747, - 0.24906, 0.100884, 0.253192, 0.314253, 0.388289, 0.580947, 1.00267, 0.241998, 0.489101, - 0.341501, 0.247423, 0.328311, 0.440281, 0.14927, 0.244469, 0.846828, 0.191725, 0.217429, - 0.123403, 0.322875, 0.145373, 0.757259, 0.190086, 0.316286, 0.268397, 0.296721, 0.440472, - 0.186848, 0.232134, 0.180239, 0.219724, 0.205886, 0.250975, 0.145636, 0.312476, 0.366418, - 0.128135, 0.315235, 0.264531, 0.161815, 0.31631, 0.296489, 0.37171, 0.197217, 0.195625, - 0.479579, 0.443037, 0.323347, 0.193616, 0.160251, 0.8952, 0.256291, 0.593345, 0.177165, - 0.409514, 0.847863, 0.111448, 0.210031, 0.251347, 0.351953, 0.705204, 0.117901, 0.182343, - 0.230179, 0.83632, 0.22104, 0.145163, 0.200326, 0.23431, 0.21868, 0.253575, 0.186562, - 0.192757, 0.172716, 0.27396, 0.258581, 0.327892, 0.376138, 0.223477, 0.302375, 0.145845, - 0.436902, 0.421794, 0.328543, 0.19246, 0.238889, 0.254866, 0.284674, 0.457849, 0.202937, - 0.392568, 0.453083, 0.782713, 0.465401, 0.178623, 0.304863, 0.190081, 0.228641, 0.255135, - 0.245037, 0.217526, 0.109584, 0.276462, 0.182301, 0.38582, 0.349942, 1.3889, 0.30235, - 0.796353, 0.160168, 0.643204, 0.153752, 0.410268, 0.186439, 0.256834, 0.185783, 0.0957629, - 0.226596, 0.197951, 0.17123, 0.192836, 0.18405, 0.575784, 0.228874, 0.201787, 0.241209, - 0.217386, 0.195751, 0.291585, 0.144531, 0.14176, 0.157635, 0.410268, 0.476338, 0.308148, - 0.148077, 0.152093, 0.196791, 0.568087, 0.414026, 0.250587, 0.473463, 0.293645, 0.396768, - 0.2766, 0.38664, 0.135034, 1.50827, 0.472527, 0.268418, 0.40383, 0.375914, 0.246496, - 0.176474, 0.340405, 0.220833, 0.138782, 0.159009, 0.444219, 0.259582, 0.33638, 0.195586, - 0.210974, 0.200288, 0.148129, 0.0974216, 0.211588, 0.280081, 0.44113, 0.773921, 0.553848, - 0.448079, 0.183136, 0.380854, 0.685021, 0.308767, 0.553276, 0.181578, 0.164759, 0.313889, - 0.137886, 0.545387, 0.278449, 0.736895, 0.360054, 0.358929, 0.457315, 0.343278, 0.507662, - 0.280829, 0.113886, 0.23146, 0.160584, 0.192796, 0.147561, 0.241272, 0.168988, 0.730511, - 0.27836, 0.179847, 0.22555, 0.418069, 0.158348, 0.128965, 0.179454, 0.126366, 0.164434, - 0.273633, 0.309556, 0.500823, 0.367852, 0.192875, 0.230262, 0.32724, 0.249969, 0.142618, - 0.494229, 0.36108, 0.227931, 0.23113, 0.742825, 0.190126, 0.33741, 0.280598, 0.145268, - 0.378423, 0.211921, 0.183594, 0.59201, 0.279563, 0.195683, 0.248101, 0.199754, 0.342494, - 0.174343, 0.14149, 0.28085, 0.175781, 0.518738, 0.17223, 0.489904, 0.181167, 0.354286, - 0.297824, 0.280829, 0.219412, 0.22814, 0.195625, 0.313949, 0.294708, 0.211551, 0.236255, - 0.666933, 0.204808, 0.52591, 0.180725, 0.186889, 0.246589, 0.410575, 0.338348, 0.206219, - 0.361766, 0.158143, 0.280816, 0.4149, 0.773082, 0.340046, 0.369672, 0.256923, 0.167195, - 0.197217, 0.252339, 0.172716, 0.191526, 0.263085, 0.345698, 0.168286, 0.243099, 0.434631, - 0.22944, 0.161862, 0.206589, 0.23457, 0.181924, 0.419063, 0.183427, 0.186152, 0.236352, - 0.306336, 0.149002, 1.50086, 0.188231, 0.442757, 0.485602, 0.466662, 0.17329, 0.141329, - 0.180619, 0.160061, 0.192569, 0.270999, 0.117901, 0.362693, 0.217561, 0.208975, 0.233658, - 0.175173, 1.10307, 0.14625, 1.31124, 0.237608, 0.286784, 0.325112, 0.2485, 0.259641, - 0.553152, 0.179039, 0.780781, 0.174758, 0.297824, 0.2558, 0.235949, 0.952186, 0.356744, - 0.312646, 0.189362, 0.574524, 0.705204, 0.213168, 0.225956, 0.424165, 0.169506, 0.137109, - 0.352451, 0.454554, 0.653302, 0.31261, 0.194412, 0.23719, 0.137886, 0.31498, 0.199085, - 0.203875, 0.597248, 1.10036, 0.196869, 0.22104, 0.451345, 0.105613, 0.683928, 0.135204, - 0.25533, 0.607871, 0.219724, 0.184464, 0.725001, 0.160061, 0.333407, 0.192569, 0.234147, - 0.47178, 0.161815, 0.242455, 0.215305, 0.410575, 0.242376, 0.211335, 0.462804, 0.275065, - 0.126878, 0.170404, 0.179433, 0.147244, 0.109584, 0.352905, 0.158215, 0.197604, 0.172407, - 0.407506, 0.645446, 0.313061, 0.165602, 0.136663, 0.55444, 0.15527, 0.133128, 0.125912, - 0.340405, 0.44521, 0.122783, 0.814526, 0.243773, 0.15743, 0.266743, 0.684458, 0.22221, - 0.181294, 0.193901, 0.258802, 0.167195, 0.292056, 0.132309, 0.227671, 0.117334, 0.271758, - 0.146185, 0.225042, 0.225964, 0.194863, 0.290274, 0.138438, 0.196714, 0.266012, 0.267771, - 0.162544, 0.244258, 0.358038, 0.522617, 0.192875, 0.45066, 0.330396, 0.223477, 0.42967, - 0.350884, 0.404655, 0.123155, 0.431583, 0.191675, 0.147354, 0.609034, 0.459487, 0.187337, - 0.215128, 0.604169, 0.330165, 0.494229, 0.40775, 0.167377, 0.192648, 0.234635, 0.275578, - 0.253094, 0.420063, 0.228299, 0.206478, 0.20395, 0.377656, 0.317393, 0.478623, 0.159009, - 0.217034, 0.300933, 0.139754, 0.153901, 0.261077, 0.22834, 0.449609, 0.157672, 0.176474, - 0.285704, 0.180186, 0.212738, 0.266428, 0.388313, 0.0954637, 0.298093, 0.251643, 0.330696, - 0.159572, 0.210666, 0.149411, 0.139618, 0.338472, 0.450304, 0.208793, 0.583609, 0.185865, - 0.400576, 0.21626, 0.174867, 0.239144, 0.249113, 0.200402, 0.275065, 0.238793, 0.205784, - 0.4475, 0.231262, 0.259082, 0.20934, 0.16806, 0.193616, 0.213811, 0.395632, 0.482465, - 0.274649, 0.307405, 0.165866, 0.334275, 0.683337, 0.368825, 0.14625, 0.780742, 0.163457, - 0.226596, 0.138713, 1.79155, 0.400443, 0.233658, 0.426399, 0.623024, 0.670955, 0.123588, - 0.110899, 0.173751, 0.651068, 0.199983, 0.190887, 0.541435, 0.21324, 0.266571, 0.134638, - 0.179348, 0.145636, 0.170929, 0.623252, 0.587738, 0.109688, 0.515314, 0.217666, 0.213311, - 0.249144, 0.187947, 0.270999, 0.268311, 0.469782, 0.763609, 0.32124, 0.146315, 0.265223, - 0.298694, 0.197623, 0.21349, 0.845778, 0.175466, 0.123588, 0.17223, 0.258603, 1.17119, - 0.538142, 0.407675, 0.120288, 0.587238, 0.244664, 0.333956, 0.132812, 0.21399, 0.302375, - 0.275882, 0.134284, 0.377555, 0.228541, 0.187307, 0.143804, 0.180545, 0.222451, 0.239638, - 0.188028, 0.46334, 0.175868, 0.242392, 0.314762, 0.44473, 0.21962, 0.175966, 1.12364, - 0.138837, 0.400576, 0.18184, 0.137706, 0.409763, 0.216894, 0.466662, 0.376604, 0.487155, - 0.283143, 0.118547, 0.221591, 0.122783, 0.179007, 0.16628, 0.180999, 0.239845, 0.169607, - 0.578402, 0.396537, 0.222288, 0.563237, 0.371238, 0.138658, 0.324336, 0.191526, 0.168603, - 0.357715, 0.640905, 0.460706, 0.220902, 0.240797, 0.164062, 0.157853, 0.34457, 0.196092, - 0.289353, 0.104597, 0.259641, 0.126878, 0.175781, 0.441458, 0.820108, 0.261864, 0.23431, - 0.254506, 0.271955, 0.227529, 0.22834, 0.196753, 0.224906, 0.193783, 0.419481, 0.236933, - 0.229706, 0.29785, 0.222947, 0.177606, 0.216911, 0.305188, 0.933438, 0.116666, 0.278483, - 0.0973824, 0.271224, 0.127717, 1.28139, 0.276283, 0.180704, 0.234554, 0.285984, 0.290172, - 0.49594, 0.135879, 0.436784, 0.206219, 0.342215, 0.374165, 0.182217, 0.274864, 0.625, - 0.356925, 0.194324, 0.342215, 0.113012, 0.155123, 0.254207, 0.438919, 0.262548, 0.302299, - 0.179528, 0.312744, 0.168513, 0.142618, 0.150543, 0.231361, 0.166004, 0.186725, 0.38848, - 0.179857, 0.182301, 0.629476, 0.44113, 0.289669, 0.328543, 0.279938, 0.14625, 0.187174, - 0.157635, 0.396749, 0.798931, 0.201541, 0.778619, 0.265883, 0.258027, 0.218576, 0.266571, - 0.160168, 0.230303, 0.273633, 0.233298, 0.30175, 0.217069, 0.345145, 0.397901, 0.224499, - 0.248101, 0.241335, 0.222947, 0.237094, 0.176518, 0.380032, 0.634775, 0.426193, 0.16362, - 0.231097, 0.219898, 0.343789, 0.275578, 0.282022, 0.628542, 0.232184, 0.848367, 0.200754, - 0.179177}, - {0, 0, 2, 3, 3, 0, 2, 2, 2, 2, 3, 0, 3, 2, 2, 2, 3, 3, 3, 3, 2, 0, 0, 0, 2, 3, 3, 3, 2, 2, 0, 0, - 2, 3, 3, 0, 0, 2, 0, 0, 3, 2, 3, 0, 3, 0, 3, 3, 0, 2, 0, 3, 2, 0, 3, 0, 3, 3, 3, 2, 2, 3, 0, 0, - 3, 3, 0, 2, 2, 3, 0, 3, 2, 2, 2, 0, 2, 3, 3, 3, 2, 3, 3, 3, 2, 0, 2, 0, 3, 3, 3, 3, 2, 2, 0, 2, - 0, 3, 2, 2, 2, 0, 0, 3, 0, 2, 2, 3, 2, 3, 0, 2, 2, 2, 3, 2, 0, 0, 2, 3, 3, 2, 0, 2, 0, 0, 2, 0, - 2, 2, 3, 2, 2, 0, 3, 0, 3, 2, 2, 2, 3, 3, 0, 0, 0, 3, 2, 3, 3, 3, 3, 0, 2, 0, 3, 2, 3, 2, 3, 0, - 2, 3, 3, 2, 3, 3, 2, 2, 0, 0, 2, 3, 3, 2, 3, 0, 2, 0, 2, 0, 3, 2, 3, 2, 3, 0, 3, 0, 3, 0, 2, 3, - 2, 2, 3, 0, 2, 2, 2, 0, 3, 2, 3, 3, 2, 3, 2, 3, 3, 2, 2, 0, 0, 2, 2, 3, 0, 3, 0, 2, 0, 0, 2, 3, - 0, 3, 3, 2, 0, 3, 3, 0, 3, 0, 2, 2, 0, 2, 0, 2, 0, 0, 0, 2, 0, 3, 2, 3, 2, 3, 2, 2, 0, 2, 3, 2, - 3, 2, 2, 2, 2, 3, 0, 2, 0, 0, 2, 3, 3, 0, 2, 3, 2, 2, 3, 0, 3, 0, 0, 2, 0, 2, 0, 2, 2, 3, 3, 2, - 3, 0, 0, 3, 2, 2, 0, 3, 2, 0, 0, 3, 0, 0, 2, 0, 3, 2, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 2, 3, 0, 0, - 2, 0, 0, 2, 0, 2, 3, 2, 3, 3, 2, 2, 0, 0, 0, 3, 0, 2, 0, 2, 0, 2, 2, 2, 3, 3, 0, 0, 3, 3, 3, 3, - 3, 2, 3, 3, 2, 3, 3, 0, 2, 2, 2, 2, 0, 2, 0, 0, 0, 2, 2, 3, 3, 2, 3, 2, 3, 0, 2, 3, 0, 2, 0, 2, - 2, 0, 3, 0, 2, 0, 2, 3, 0, 3, 0, 0, 0, 3, 2, 3, 3, 0, 3, 2, 3, 0, 2, 3, 3, 0, 2, 3, 0, 0, 0, 2, - 0, 3, 0, 2, 3, 3, 3, 3, 3, 0, 2, 0, 2, 2, 3, 3, 0, 3, 0, 2, 0, 2, 0, 3, 0, 0, 0, 2, 3, 3, 2, 3, - 0, 0, 0, 0, 3, 3, 0, 3, 2, 0, 2, 3, 2, 2, 3, 3, 2, 2, 2, 0, 2, 3, 0, 3, 3, 0, 0, 2, 0, 3, 2, 3, - 0, 2, 0, 2, 2, 3, 2, 0, 3, 3, 3, 2, 3, 0, 3, 0, 2, 2, 0, 0, 0, 3, 0, 3, 3, 2, 3, 2, 3, 2, 3, 0, - 2, 3, 0, 2, 0, 3, 3, 3, 3, 3, 3, 2, 0, 3, 2, 2, 2, 3, 3, 2, 3, 0, 2, 3, 3, 2, 2, 0, 0, 0, 0, 3, - 0, 3, 3, 3, 0, 0, 0, 3, 3, 3, 3, 3, 0, 2, 3, 3, 3, 3, 3, 3, 0, 0, 2, 2, 3, 3, 2, 2, 0, 0, 3, 0, - 0, 0, 2, 3, 0, 0, 0, 3, 0, 3, 0, 2, 2, 0, 0, 0, 0, 3, 2, 2, 3, 2, 3, 2, 2, 2, 2, 3, 0, 0, 2, 3, - 0, 3, 3, 0, 3, 0, 0, 2, 0, 3, 3, 0, 2, 2, 3, 3, 0, 0, 2, 0, 2, 3, 2, 0, 0, 3, 3, 0, 3, 2, 0, 2, - 0, 2, 3, 2, 0, 3, 3, 2, 0, 0, 2, 2, 0, 0, 2, 0, 3, 3, 2, 3, 2, 0, 3, 0, 2, 2, 3, 3, 0, 3, 2, 2, - 0, 3, 0, 0, 0, 2, 0, 3, 2, 0, 2, 3, 2, 3, 2, 2, 3, 3, 0, 2, 3, 2, 3, 2, 2, 0, 3, 0, 3, 0, 2, 2, - 2, 0, 2, 0, 2, 2, 0, 0, 3, 3, 0, 0, 3, 2, 0, 2, 3, 2, 2, 0, 3, 3, 0, 2, 0, 3, 3, 0, 2, 3, 2, 3, - 2, 0, 2, 2, 0, 0, 0, 2, 2, 3, 3, 2, 2, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 2, 0, 3, 3, - 3, 0, 2, 0, 2, 3, 2, 0, 3, 3, 2, 0, 2, 0, 3, 2, 0, 3, 0, 0, 2, 2, 0, 3, 0, 2, 3, 3, 3, 0, 2, 0, - 0, 3, 0, 2, 3, 2, 2, 0, 3, 3, 3, 3, 3, 0, 3, 0, 0, 0, 0, 3, 2, 0, 0, 2, 3, 3, 2, 2, 0, 3, 2, 0, - 3, 0, 2, 3, 3, 0, 2, 2, 3, 2, 2, 2, 3, 2, 0, 0, 3, 2, 0, 0, 0, 2, 0, 2, 0, 0, 2, 2, 3, 0, 3, 0, - 0, 3, 0, 0, 0, 3, 0, 0, 2, 2, 0, 2, 2, 3, 3, 3, 3, 0, 0, 2, 2, 2, 0, 3, 2, 2, 2, 2, 2, 0, 3, 0, - 0, 3, 2, 0, 0, 3, 2, 3, 3, 0, 3, 0, 3, 0, 3, 2, 2, 2, 0, 0, 3, 2, 2, 0, 0, 0, 2, 3, 2, 0, 2, 3, - 3, 3, 0, 3, 3, 0, 2, 0, 0, 2, 3, 3, 0, 3, 2, 2, 2, 2, 2, 3, 3, 2, 2, 3, 3, 2, 3, 0, 3, 3, 0, 3, - 2, 2, 0, 2, 0, 3, 0, 3, 0, 2, 3, 0, 2, 3, 2, 0, 2, 0, 3, 0, 2, 3, 3, 2, 0, 3, 3, 3, 2, 2, 3, 3, - 2, 2, 2, 0, 3, 2, 2, 0}, - {271, 271, 329, 343, 387, 426, 426, 601}, - {426, 601, 426, 387, 343, 271, 329, 271}, - {3.70991, 4.43491, 3.76334, 9.43944, 9.43944, 3.70991, 3.76334, 4.43491}}}; - -typedef ConnectComponentsEdgesTest ConnectComponentsEdgesTestF_Int; -TEST_P(ConnectComponentsEdgesTestF_Int, Result) { EXPECT_TRUE(true); } - -INSTANTIATE_TEST_CASE_P(ConnectComponentsEdgesTest, - ConnectComponentsEdgesTestF_Int, - ::testing::ValuesIn(mr_fix_conn_inputsf2)); - -}; // namespace sparse -}; // end namespace raft diff --git a/cpp/test/sparse/neighbors/knn_graph.cu b/cpp/test/sparse/neighbors/knn_graph.cu deleted file mode 100644 index db610099e9..0000000000 --- a/cpp/test/sparse/neighbors/knn_graph.cu +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../../test_utils.cuh" - -#include -#include -#include -#include - -#include -#include - -#include - -#include - -namespace raft { -namespace sparse { - -template -RAFT_KERNEL assert_symmetry( - value_idx* rows, value_idx* cols, value_t* vals, value_idx nnz, value_idx* sum) -{ - int tid = blockDim.x * blockIdx.x + threadIdx.x; - - if (tid >= nnz) return; - - atomicAdd(sum, rows[tid]); - atomicAdd(sum, -1 * cols[tid]); -} - -template -struct KNNGraphInputs { - value_idx m; - value_idx n; - - std::vector X; - - int k = 2; -}; - -template -::std::ostream& operator<<(::std::ostream& os, const KNNGraphInputs& dims) -{ - return os; -} - -template -class KNNGraphTest : public ::testing::TestWithParam> { - public: - KNNGraphTest() - : params(::testing::TestWithParam>::GetParam()), - stream(resource::get_cuda_stream(handle)), - X(0, stream) - { - X.resize(params.X.size(), stream); - } - - protected: - void SetUp() override - { - out = new raft::sparse::COO(stream); - - update_device(X.data(), params.X.data(), params.X.size(), stream); - - raft::sparse::neighbors::knn_graph( - handle, X.data(), params.m, params.n, raft::distance::DistanceType::L2Unexpanded, *out); - - rmm::device_scalar sum(stream); - sum.set_value_to_zero_async(stream); - - /** - * Assert the knn graph is symmetric - */ - assert_symmetry<<nnz, 256), 256, 0, stream>>>( - out->rows(), out->cols(), out->vals(), out->nnz, sum.data()); - - sum_h = sum.value(stream); - resource::sync_stream(handle, stream); - } - - void TearDown() override { delete out; } - - protected: - raft::resources handle; - cudaStream_t stream; - - // input data - raft::sparse::COO* out; - - rmm::device_uvector X; - - value_idx sum_h; - - KNNGraphInputs params; -}; - -const std::vector> knn_graph_inputs_fint = { - // Test n_clusters == n_points - {4, 2, {0, 100, 0.01, 0.02, 5000, 10000, -5, -2}, 2}}; - -typedef KNNGraphTest KNNGraphTestF_int; -TEST_P(KNNGraphTestF_int, Result) -{ - // nnz should not be larger than twice m * k - ASSERT_TRUE(out->nnz <= (params.m * params.k * 2)); - ASSERT_TRUE(sum_h == 0); -} - -INSTANTIATE_TEST_CASE_P(KNNGraphTest, - KNNGraphTestF_int, - ::testing::ValuesIn(knn_graph_inputs_fint)); - -} // namespace sparse -} // namespace raft diff --git a/cpp/test/sparse/solver/lanczos.cu b/cpp/test/sparse/solver/lanczos.cu new file mode 100644 index 0000000000..128ab73747 --- /dev/null +++ b/cpp/test/sparse/solver/lanczos.cu @@ -0,0 +1,470 @@ +/* + * Copyright (c) 2019-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../../test_utils.cuh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace raft::sparse { + +template +struct lanczos_inputs { + int n_components; + int restartiter; + int maxiter; + int conv_n_iters; + float conv_eps; + float tol; + uint64_t seed; + std::vector rows; // indptr + std::vector cols; // indices + std::vector vals; // data + std::vector expected_eigenvalues; +}; + +template +struct rmat_lanczos_inputs { + int n_components; + int restartiter; + int maxiter; + int conv_n_iters; + float conv_eps; + float tol; + uint64_t seed; + int r_scale; + int c_scale; + float sparsity; + std::vector expected_eigenvalues; +}; + +template +class rmat_lanczos_tests + : public ::testing::TestWithParam> { + public: + rmat_lanczos_tests() + : params(::testing::TestWithParam>::GetParam()), + stream(resource::get_cuda_stream(handle)), + rng(params.seed), + expected_eigenvalues(raft::make_device_vector( + handle, params.n_components)), + r_scale(params.r_scale), + c_scale(params.c_scale), + sparsity(params.sparsity) + { + } + + protected: + void SetUp() override + { + raft::copy(expected_eigenvalues.data_handle(), + params.expected_eigenvalues.data(), + params.n_components, + stream); + } + + void TearDown() override {} + + void Run() + { + int runtimeVersion; + cudaError_t result = cudaRuntimeGetVersion(&runtimeVersion); + + if (result == cudaSuccess) { + int major = runtimeVersion / 1000; + int minor = (runtimeVersion % 1000) / 10; + + // Skip gtests for CUDA 11.4.x and below because hard-coded results are causing issues. + // See https://github.com/rapidsai/raft/issues/2519 for more information. + if (major == 11 && minor <= 4) { GTEST_SKIP(); } + } + + uint64_t n_edges = sparsity * ((long long)(1 << r_scale) * (long long)(1 << c_scale)); + uint64_t n_nodes = 1 << std::max(r_scale, c_scale); + uint64_t theta_len = std::max(r_scale, c_scale) * 4; + + auto theta = raft::make_device_vector(handle, theta_len); + raft::random::uniform(handle, rng, theta.view(), 0, 1); + + auto out = + raft::make_device_matrix(handle, n_edges * 2, 2); + auto out_src = raft::make_device_vector(handle, n_edges); + auto out_dst = raft::make_device_vector(handle, n_edges); + + raft::random::RngState rng1{params.seed}; + + raft::random::rmat_rectangular_gen(handle, + rng1, + make_const_mdspan(theta.view()), + out.view(), + out_src.view(), + out_dst.view(), + r_scale, + c_scale); + + raft::device_vector out_data = + raft::make_device_vector(handle, n_edges); + raft::matrix::fill(handle, out_data.view(), 1.0); + raft::sparse::COO coo(stream); + + raft::sparse::op::coo_sort(n_nodes, + n_nodes, + n_edges, + out_src.data_handle(), + out_dst.data_handle(), + out_data.data_handle(), + stream); + raft::sparse::op::max_duplicates(handle, + coo, + out_src.data_handle(), + out_dst.data_handle(), + out_data.data_handle(), + n_edges, + n_nodes, + n_nodes); + + raft::sparse::COO symmetric_coo(stream); + raft::sparse::linalg::symmetrize( + handle, coo.rows(), coo.cols(), coo.vals(), coo.n_rows, coo.n_cols, coo.nnz, symmetric_coo); + + raft::device_vector row_indices = + raft::make_device_vector(handle, + symmetric_coo.n_rows + 1); + raft::sparse::convert::sorted_coo_to_csr(symmetric_coo.rows(), + symmetric_coo.nnz, + row_indices.data_handle(), + symmetric_coo.n_rows + 1, + stream); + + int n_components = params.n_components; + + raft::device_vector v0 = + raft::make_device_vector(handle, symmetric_coo.n_rows); + + raft::random::uniform(handle, rng, v0.view(), 0, 1); + std::tuple stats; + + raft::device_vector eigenvalues = + raft::make_device_vector(handle, n_components); + raft::device_matrix eigenvectors = + raft::make_device_matrix( + handle, symmetric_coo.n_rows, n_components); + + raft::spectral::matrix::sparse_matrix_t const csr_m{ + handle, + row_indices.data_handle(), + symmetric_coo.cols(), + symmetric_coo.vals(), + symmetric_coo.n_rows, + symmetric_coo.nnz}; + raft::sparse::solver::lanczos_solver_config config{ + n_components, params.maxiter, params.restartiter, params.tol, rng.seed}; + + auto csr_structure = + raft::make_device_compressed_structure_view( + const_cast(row_indices.data_handle()), + const_cast(symmetric_coo.cols()), + symmetric_coo.n_rows, + symmetric_coo.n_rows, + symmetric_coo.nnz); + + auto csr_matrix = raft::make_device_csr_matrix_view( + const_cast(symmetric_coo.vals()), csr_structure); + + std::get<0>(stats) = + raft::sparse::solver::lanczos_compute_smallest_eigenvectors( + handle, + config, + csr_matrix, + std::make_optional(v0.view()), + eigenvalues.view(), + eigenvectors.view()); + + ASSERT_TRUE(raft::devArrMatch(eigenvalues.data_handle(), + expected_eigenvalues.data_handle(), + n_components, + raft::CompareApprox(1e-5), + stream)); + } + + protected: + rmat_lanczos_inputs params; + raft::resources handle; + cudaStream_t stream; + raft::random::RngState rng; + int r_scale; + int c_scale; + float sparsity; + raft::device_vector expected_eigenvalues; +}; + +template +class lanczos_tests : public ::testing::TestWithParam> { + public: + lanczos_tests() + : params(::testing::TestWithParam>::GetParam()), + stream(resource::get_cuda_stream(handle)), + n(params.rows.size() - 1), + nnz(params.vals.size()), + rng(params.seed), + rows(raft::make_device_vector(handle, n + 1)), + cols(raft::make_device_vector(handle, nnz)), + vals(raft::make_device_vector(handle, nnz)), + v0(raft::make_device_vector(handle, n)), + eigenvalues(raft::make_device_vector( + handle, params.n_components)), + eigenvectors(raft::make_device_matrix( + handle, n, params.n_components)), + expected_eigenvalues( + raft::make_device_vector(handle, params.n_components)) + { + } + + protected: + void SetUp() override + { + raft::copy(rows.data_handle(), params.rows.data(), n + 1, stream); + raft::copy(cols.data_handle(), params.cols.data(), nnz, stream); + raft::copy(vals.data_handle(), params.vals.data(), nnz, stream); + raft::copy(expected_eigenvalues.data_handle(), + params.expected_eigenvalues.data(), + params.n_components, + stream); + } + + void TearDown() override {} + + void Run() + { + int runtimeVersion; + cudaError_t result = cudaRuntimeGetVersion(&runtimeVersion); + + if (result == cudaSuccess) { + int major = runtimeVersion / 1000; + int minor = (runtimeVersion % 1000) / 10; + + // Skip gtests for CUDA 11.4.x and below because hard-coded results are causing issues. + // See https://github.com/rapidsai/raft/issues/2519 for more information. + if (major == 11 && minor <= 4) { GTEST_SKIP(); } + } + + raft::random::uniform(handle, rng, v0.view(), 0, 1); + std::tuple stats; + + raft::sparse::solver::lanczos_solver_config config{ + params.n_components, params.maxiter, params.restartiter, params.tol, rng.seed}; + auto csr_structure = + raft::make_device_compressed_structure_view( + const_cast(rows.data_handle()), + const_cast(cols.data_handle()), + n, + n, + nnz); + + auto csr_matrix = raft::make_device_csr_matrix_view( + const_cast(vals.data_handle()), csr_structure); + + std::get<0>(stats) = + raft::sparse::solver::lanczos_compute_smallest_eigenvectors( + handle, + config, + csr_matrix, + std::make_optional(v0.view()), + eigenvalues.view(), + eigenvectors.view()); + + ASSERT_TRUE(raft::devArrMatch(eigenvalues.data_handle(), + expected_eigenvalues.data_handle(), + params.n_components, + raft::CompareApprox(1e-5), + stream)); + } + + protected: + lanczos_inputs params; + raft::resources handle; + cudaStream_t stream; + int n; + int nnz; + raft::random::RngState rng; + raft::device_vector rows; + raft::device_vector cols; + raft::device_vector vals; + raft::device_vector v0; + raft::device_vector eigenvalues; + raft::device_matrix eigenvectors; + raft::device_vector expected_eigenvalues; +}; + +// TODO: Find a way to generate and validate test data without hardcoding them (issue #2485) +const std::vector> inputsf = { + {2, + 34, + 10000, + 0, + 0, + 1e-15, + 42, + {0, 0, 0, 0, 3, 5, 6, 8, 9, 11, 16, 16, 18, 20, 23, 24, 27, + 30, 31, 33, 37, 37, 39, 41, 43, 44, 46, 46, 47, 49, 50, 50, 51, 53, + 57, 58, 59, 66, 67, 68, 69, 71, 72, 75, 78, 83, 86, 90, 93, 94, 96, + 98, 99, 101, 101, 104, 106, 108, 109, 109, 109, 109, 111, 113, 118, 120, 121, 123, + 124, 128, 132, 134, 136, 138, 139, 141, 145, 148, 151, 152, 154, 155, 157, 160, 164, + 167, 170, 170, 170, 173, 178, 179, 182, 184, 186, 191, 192, 196, 198, 198, 198}, + {44, 68, 74, 16, 36, 85, 34, 75, 61, 51, 83, 15, 33, 55, 69, 71, 18, 84, 70, 95, 71, 83, + 97, 83, 9, 36, 54, 4, 42, 46, 52, 11, 89, 31, 37, 74, 96, 36, 88, 56, 64, 68, 94, 82, + 35, 90, 50, 82, 85, 83, 19, 47, 94, 9, 44, 56, 79, 6, 25, 4, 15, 21, 52, 75, 79, 92, + 19, 72, 94, 94, 96, 80, 16, 54, 89, 46, 48, 63, 3, 33, 67, 73, 77, 46, 47, 75, 16, 43, + 45, 81, 32, 45, 68, 43, 55, 63, 27, 89, 8, 17, 36, 15, 42, 96, 9, 49, 22, 33, 77, 7, + 75, 78, 88, 43, 49, 66, 76, 91, 22, 82, 69, 63, 84, 44, 3, 23, 47, 81, 9, 65, 76, 92, + 12, 96, 9, 13, 38, 93, 44, 3, 19, 6, 36, 45, 61, 63, 69, 89, 44, 57, 94, 62, 33, 36, + 41, 46, 68, 24, 28, 64, 8, 13, 14, 29, 11, 66, 88, 5, 28, 93, 21, 62, 84, 18, 42, 50, + 76, 91, 25, 63, 89, 97, 36, 69, 72, 85, 23, 32, 39, 40, 77, 12, 19, 40, 54, 70, 13, 91}, + {0.4734894, 0.1402491, 0.7686475, 0.0416142, 0.2559651, 0.9360436, 0.7486080, 0.5206724, + 0.0374126, 0.8082515, 0.5993828, 0.4866583, 0.8907925, 0.9251201, 0.8566143, 0.9528994, + 0.4557763, 0.4907070, 0.4158074, 0.8311127, 0.9026024, 0.3103237, 0.5876446, 0.7585195, + 0.4866583, 0.4493615, 0.5909155, 0.0416142, 0.0963910, 0.6722401, 0.3468698, 0.4557763, + 0.1445242, 0.7720124, 0.9923756, 0.1227579, 0.7194629, 0.8916773, 0.4320931, 0.5840980, + 0.0216121, 0.3709223, 0.1705930, 0.8297898, 0.2409706, 0.9585592, 0.3171389, 0.0228039, + 0.4350971, 0.4939908, 0.7720124, 0.2722416, 0.1792683, 0.8907925, 0.1085757, 0.8745620, + 0.3298612, 0.7486080, 0.2409706, 0.2559651, 0.4493615, 0.8916773, 0.5540361, 0.5150571, + 0.9160119, 0.1767728, 0.9923756, 0.5717281, 0.1077409, 0.9368132, 0.6273088, 0.6616613, + 0.0963910, 0.9378265, 0.3059566, 0.3159291, 0.0449106, 0.9085807, 0.4734894, 0.1085757, + 0.2909013, 0.7787509, 0.7168902, 0.9691764, 0.2669757, 0.4389115, 0.6722401, 0.3159291, + 0.9691764, 0.7467896, 0.2722416, 0.2669757, 0.1532843, 0.0449106, 0.2023634, 0.8934466, + 0.3171389, 0.6594226, 0.8082515, 0.3468698, 0.5540361, 0.5909155, 0.9378265, 0.2909178, + 0.9251201, 0.2023634, 0.5840980, 0.8745620, 0.2624605, 0.0374126, 0.1034030, 0.3736577, + 0.3315690, 0.9085807, 0.8934466, 0.5548525, 0.2302140, 0.7827352, 0.0216121, 0.8262919, + 0.1646078, 0.5548525, 0.2658700, 0.2909013, 0.1402491, 0.3709223, 0.1532843, 0.5792196, + 0.8566143, 0.1646078, 0.0827300, 0.5810611, 0.4158074, 0.5188584, 0.9528994, 0.9026024, + 0.5717281, 0.7269946, 0.7787509, 0.7686475, 0.1227579, 0.5206724, 0.5150571, 0.4389115, + 0.1034030, 0.2302140, 0.0827300, 0.8961608, 0.7168902, 0.2624605, 0.4823034, 0.3736577, + 0.3298612, 0.9160119, 0.6616613, 0.7467896, 0.5792196, 0.8297898, 0.0228039, 0.8262919, + 0.5993828, 0.3103237, 0.7585195, 0.4939908, 0.4907070, 0.2658700, 0.0844443, 0.9360436, + 0.4350971, 0.6997072, 0.4320931, 0.3315690, 0.0844443, 0.1445242, 0.3059566, 0.6594226, + 0.8961608, 0.6498466, 0.9585592, 0.7827352, 0.6498466, 0.2812338, 0.1767728, 0.5810611, + 0.7269946, 0.6997072, 0.1705930, 0.1792683, 0.1077409, 0.9368132, 0.4823034, 0.8311127, + 0.7194629, 0.6273088, 0.2909178, 0.5188584, 0.5876446, 0.2812338}, + {-2.0369630, -1.7673520}}}; + +const std::vector> inputsd = { + {2, + 34, + 10000, + 0, + 0, + 1e-15, + 42, + {0, 0, 0, 0, 3, 5, 6, 8, 9, 11, 16, 16, 18, 20, 23, 24, 27, + 30, 31, 33, 37, 37, 39, 41, 43, 44, 46, 46, 47, 49, 50, 50, 51, 53, + 57, 58, 59, 66, 67, 68, 69, 71, 72, 75, 78, 83, 86, 90, 93, 94, 96, + 98, 99, 101, 101, 104, 106, 108, 109, 109, 109, 109, 111, 113, 118, 120, 121, 123, + 124, 128, 132, 134, 136, 138, 139, 141, 145, 148, 151, 152, 154, 155, 157, 160, 164, + 167, 170, 170, 170, 173, 178, 179, 182, 184, 186, 191, 192, 196, 198, 198, 198}, + {44, 68, 74, 16, 36, 85, 34, 75, 61, 51, 83, 15, 33, 55, 69, 71, 18, 84, 70, 95, 71, 83, + 97, 83, 9, 36, 54, 4, 42, 46, 52, 11, 89, 31, 37, 74, 96, 36, 88, 56, 64, 68, 94, 82, + 35, 90, 50, 82, 85, 83, 19, 47, 94, 9, 44, 56, 79, 6, 25, 4, 15, 21, 52, 75, 79, 92, + 19, 72, 94, 94, 96, 80, 16, 54, 89, 46, 48, 63, 3, 33, 67, 73, 77, 46, 47, 75, 16, 43, + 45, 81, 32, 45, 68, 43, 55, 63, 27, 89, 8, 17, 36, 15, 42, 96, 9, 49, 22, 33, 77, 7, + 75, 78, 88, 43, 49, 66, 76, 91, 22, 82, 69, 63, 84, 44, 3, 23, 47, 81, 9, 65, 76, 92, + 12, 96, 9, 13, 38, 93, 44, 3, 19, 6, 36, 45, 61, 63, 69, 89, 44, 57, 94, 62, 33, 36, + 41, 46, 68, 24, 28, 64, 8, 13, 14, 29, 11, 66, 88, 5, 28, 93, 21, 62, 84, 18, 42, 50, + 76, 91, 25, 63, 89, 97, 36, 69, 72, 85, 23, 32, 39, 40, 77, 12, 19, 40, 54, 70, 13, 91}, + {0.4734894, 0.1402491, 0.7686475, 0.0416142, 0.2559651, 0.9360436, 0.7486080, 0.5206724, + 0.0374126, 0.8082515, 0.5993828, 0.4866583, 0.8907925, 0.9251201, 0.8566143, 0.9528994, + 0.4557763, 0.4907070, 0.4158074, 0.8311127, 0.9026024, 0.3103237, 0.5876446, 0.7585195, + 0.4866583, 0.4493615, 0.5909155, 0.0416142, 0.0963910, 0.6722401, 0.3468698, 0.4557763, + 0.1445242, 0.7720124, 0.9923756, 0.1227579, 0.7194629, 0.8916773, 0.4320931, 0.5840980, + 0.0216121, 0.3709223, 0.1705930, 0.8297898, 0.2409706, 0.9585592, 0.3171389, 0.0228039, + 0.4350971, 0.4939908, 0.7720124, 0.2722416, 0.1792683, 0.8907925, 0.1085757, 0.8745620, + 0.3298612, 0.7486080, 0.2409706, 0.2559651, 0.4493615, 0.8916773, 0.5540361, 0.5150571, + 0.9160119, 0.1767728, 0.9923756, 0.5717281, 0.1077409, 0.9368132, 0.6273088, 0.6616613, + 0.0963910, 0.9378265, 0.3059566, 0.3159291, 0.0449106, 0.9085807, 0.4734894, 0.1085757, + 0.2909013, 0.7787509, 0.7168902, 0.9691764, 0.2669757, 0.4389115, 0.6722401, 0.3159291, + 0.9691764, 0.7467896, 0.2722416, 0.2669757, 0.1532843, 0.0449106, 0.2023634, 0.8934466, + 0.3171389, 0.6594226, 0.8082515, 0.3468698, 0.5540361, 0.5909155, 0.9378265, 0.2909178, + 0.9251201, 0.2023634, 0.5840980, 0.8745620, 0.2624605, 0.0374126, 0.1034030, 0.3736577, + 0.3315690, 0.9085807, 0.8934466, 0.5548525, 0.2302140, 0.7827352, 0.0216121, 0.8262919, + 0.1646078, 0.5548525, 0.2658700, 0.2909013, 0.1402491, 0.3709223, 0.1532843, 0.5792196, + 0.8566143, 0.1646078, 0.0827300, 0.5810611, 0.4158074, 0.5188584, 0.9528994, 0.9026024, + 0.5717281, 0.7269946, 0.7787509, 0.7686475, 0.1227579, 0.5206724, 0.5150571, 0.4389115, + 0.1034030, 0.2302140, 0.0827300, 0.8961608, 0.7168902, 0.2624605, 0.4823034, 0.3736577, + 0.3298612, 0.9160119, 0.6616613, 0.7467896, 0.5792196, 0.8297898, 0.0228039, 0.8262919, + 0.5993828, 0.3103237, 0.7585195, 0.4939908, 0.4907070, 0.2658700, 0.0844443, 0.9360436, + 0.4350971, 0.6997072, 0.4320931, 0.3315690, 0.0844443, 0.1445242, 0.3059566, 0.6594226, + 0.8961608, 0.6498466, 0.9585592, 0.7827352, 0.6498466, 0.2812338, 0.1767728, 0.5810611, + 0.7269946, 0.6997072, 0.1705930, 0.1792683, 0.1077409, 0.9368132, 0.4823034, 0.8311127, + 0.7194629, 0.6273088, 0.2909178, 0.5188584, 0.5876446, 0.2812338}, + {-2.0369630, -1.7673520}}}; + +const std::vector> rmat_inputsf = { + {50, 100, 10000, 0, 0, 1e-9, 42, 12, 12, 1, {-122.526794, -74.00686, -59.698284, -54.68617, + -49.686813, -34.02644, -32.130703, -31.26906, + -30.32097, -22.946098, -20.497862, -20.23817, + -19.269697, -18.42496, -17.675667, -17.013401, + -16.734581, -15.820215, -15.73925, -15.448187, + -15.044634, -14.692028, -14.127425, -13.967386, + -13.6237755, -13.469393, -13.181225, -12.777589, + -12.623185, -12.55508, -12.2874565, -12.053391, + -11.677346, -11.558279, -11.163732, -10.922034, + -10.7936945, -10.558049, -10.205776, -10.005316, + -9.559181, -9.491834, -9.242631, -8.883637, + -8.765364, -8.688508, -8.458255, -8.385196, + -8.217982, -8.0442095}}}; + +using LanczosTestF = lanczos_tests; +TEST_P(LanczosTestF, Result) { Run(); } + +using LanczosTestD = lanczos_tests; +TEST_P(LanczosTestD, Result) { Run(); } + +using RmatLanczosTestF = rmat_lanczos_tests; +TEST_P(RmatLanczosTestF, Result) { Run(); } + +INSTANTIATE_TEST_CASE_P(LanczosTests, LanczosTestF, ::testing::ValuesIn(inputsf)); +INSTANTIATE_TEST_CASE_P(LanczosTests, LanczosTestD, ::testing::ValuesIn(inputsd)); +INSTANTIATE_TEST_CASE_P(LanczosTests, RmatLanczosTestF, ::testing::ValuesIn(rmat_inputsf)); + +} // namespace raft::sparse diff --git a/cpp/test/stats/neighborhood_recall.cu b/cpp/test/stats/neighborhood_recall.cu deleted file mode 100644 index 1a76154e2e..0000000000 --- a/cpp/test/stats/neighborhood_recall.cu +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../neighbors/ann_utils.cuh" -#include "../test_utils.h" - -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace raft::stats { - -struct NeighborhoodRecallInputs { - int n_rows; - int n_cols; - int k; -}; - -template -class NeighborhoodRecallTest : public ::testing::TestWithParam { - public: - NeighborhoodRecallTest() - : ps{::testing::TestWithParam::GetParam()}, - data_1{raft::make_device_matrix(res, ps.n_rows, ps.n_cols)}, - data_2{raft::make_device_matrix(res, ps.n_rows, ps.n_cols)} - { - } - - protected: - void test_recall() - { - size_t queries_size = ps.n_rows * ps.k; - - // calculate nn for dataset 1 - auto distances_1 = raft::make_device_matrix(res, ps.n_rows, ps.k); - auto indices_1 = raft::make_device_matrix(res, ps.n_rows, ps.k); - raft::neighbors::naive_knn( - res, - distances_1.data_handle(), - indices_1.data_handle(), - data_1.data_handle(), - data_1.data_handle(), - ps.n_rows, - ps.n_rows, - ps.n_cols, - ps.k, - raft::distance::DistanceType::L2Expanded); - std::vector distances_1_h(queries_size); - std::vector indices_1_h(queries_size); - raft::copy(distances_1_h.data(), - distances_1.data_handle(), - ps.n_rows * ps.k, - raft::resource::get_cuda_stream(res)); - raft::copy(indices_1_h.data(), - indices_1.data_handle(), - ps.n_rows * ps.k, - raft::resource::get_cuda_stream(res)); - - // calculate nn for dataset 2 - auto distances_2 = raft::make_device_matrix(res, ps.n_rows, ps.k); - auto indices_2 = raft::make_device_matrix(res, ps.n_rows, ps.k); - raft::neighbors::naive_knn( - res, - distances_2.data_handle(), - indices_2.data_handle(), - data_2.data_handle(), - data_2.data_handle(), - ps.n_rows, - ps.n_rows, - ps.n_cols, - ps.k, - raft::distance::DistanceType::L2Expanded); - std::vector distances_2_h(queries_size); - std::vector indices_2_h(queries_size); - raft::copy(distances_2_h.data(), - distances_2.data_handle(), - ps.n_rows * ps.k, - raft::resource::get_cuda_stream(res)); - raft::copy(indices_2_h.data(), - indices_2.data_handle(), - ps.n_rows * ps.k, - raft::resource::get_cuda_stream(res)); - - raft::resource::sync_stream(res); - - // find CPU recall scores - [[maybe_unused]] auto [indices_only_recall_h, mc1, tc1] = - raft::neighbors::calc_recall(indices_1_h, indices_2_h, ps.n_rows, ps.k); - [[maybe_unused]] auto [recall_h, mc2, tc2] = raft::neighbors::calc_recall( - indices_1_h, indices_2_h, distances_1_h, distances_2_h, ps.n_rows, ps.k, 0.001); - - // find GPU recall scores - auto s1 = 0; - auto indices_only_recall_scalar = raft::make_host_scalar(s1); - neighborhood_recall(res, - raft::make_const_mdspan(indices_1.view()), - raft::make_const_mdspan(indices_2.view()), - indices_only_recall_scalar.view()); - - auto s2 = 0; - auto recall_scalar = raft::make_host_scalar(s2); - DistanceT s3 = 0.001; - auto eps_mda = raft::make_host_scalar(s3); - - neighborhood_recall(res, - raft::make_const_mdspan(indices_1.view()), - raft::make_const_mdspan(indices_2.view()), - recall_scalar.view(), - raft::make_const_mdspan(distances_1.view()), - raft::make_const_mdspan(distances_2.view())); - - // assert correctness - ASSERT_TRUE(raft::match(indices_only_recall_h, - *indices_only_recall_scalar.data_handle(), - raft::CompareApprox(0.01))); - ASSERT_TRUE( - raft::match(recall_h, *recall_scalar.data_handle(), raft::CompareApprox(0.01))); - } - - void SetUp() override - { - // form two random datasets - raft::random::Rng r1(1234ULL); - r1.normal(data_1.data_handle(), - ps.n_rows * ps.n_cols, - DistanceT(0.1), - DistanceT(2.0), - raft::resource::get_cuda_stream(res)); - raft::random::Rng r2(21111ULL); - r2.normal(data_2.data_handle(), - ps.n_rows * ps.n_cols, - DistanceT(0.1), - DistanceT(2.0), - raft::resource::get_cuda_stream(res)); - resource::sync_stream(res); - } - - private: - raft::resources res; - NeighborhoodRecallInputs ps; - raft::device_matrix data_1; - raft::device_matrix data_2; -}; - -const std::vector inputs = - raft::util::itertools::product({10, 50, 100}, // n_rows - {80, 100}, // n_cols - {32, 64}); // k - -using NeighborhoodRecallTestF_U32 = NeighborhoodRecallTest; -TEST_P(NeighborhoodRecallTestF_U32, AnnCagra) { this->test_recall(); } - -INSTANTIATE_TEST_CASE_P(NeighborhoodRecallTest, - NeighborhoodRecallTestF_U32, - ::testing::ValuesIn(inputs)); - -} // end namespace raft::stats diff --git a/cpp/test/stats/silhouette_score.cu b/cpp/test/stats/silhouette_score.cu deleted file mode 100644 index ad080f5894..0000000000 --- a/cpp/test/stats/silhouette_score.cu +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "../test_utils.cuh" - -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include - -namespace raft { -namespace stats { - -// parameter structure definition -struct silhouetteScoreParam { - int nRows; - int nCols; - int nLabels; - raft::distance::DistanceType metric; - int chunk; - double tolerance; -}; - -// test fixture class -template -class silhouetteScoreTest : public ::testing::TestWithParam { - protected: - silhouetteScoreTest() - : d_X(0, resource::get_cuda_stream(handle)), - sampleSilScore(0, resource::get_cuda_stream(handle)), - d_labels(0, resource::get_cuda_stream(handle)) - { - } - - void host_silhouette_score() - { - // generating random value test input - std::vector h_X(nElements, 0.0); - std::vector h_labels(nRows, 0); - std::random_device rd; - std::default_random_engine dre(nElements * nLabels); - std::uniform_int_distribution intGenerator(0, nLabels - 1); - std::uniform_real_distribution realGenerator(0, 100); - - std::generate(h_X.begin(), h_X.end(), [&]() { return realGenerator(dre); }); - std::generate(h_labels.begin(), h_labels.end(), [&]() { return intGenerator(dre); }); - - // allocating and initializing memory to the GPU - auto stream = resource::get_cuda_stream(handle); - d_X.resize(nElements, stream); - d_labels.resize(nElements, stream); - RAFT_CUDA_TRY(cudaMemsetAsync(d_X.data(), 0, d_X.size() * sizeof(DataT), stream)); - RAFT_CUDA_TRY(cudaMemsetAsync(d_labels.data(), 0, d_labels.size() * sizeof(LabelT), stream)); - sampleSilScore.resize(nElements, stream); - - raft::update_device(d_X.data(), &h_X[0], (int)nElements, stream); - raft::update_device(d_labels.data(), &h_labels[0], (int)nElements, stream); - - // finding the distance matrix - - rmm::device_uvector d_distanceMatrix(nRows * nRows, stream); - double* h_distanceMatrix = (double*)malloc(nRows * nRows * sizeof(double*)); - - raft::distance::pairwise_distance( - handle, d_X.data(), d_X.data(), d_distanceMatrix.data(), nRows, nRows, nCols, params.metric); - - resource::sync_stream(handle, stream); - - raft::update_host(h_distanceMatrix, d_distanceMatrix.data(), nRows * nRows, stream); - - // finding the bincount array - - double* binCountArray = (double*)malloc(nLabels * sizeof(double*)); - memset(binCountArray, 0, nLabels * sizeof(double)); - - for (int i = 0; i < nRows; ++i) { - binCountArray[h_labels[i]] += 1; - } - - // finding the average intra cluster distance for every element - - double* a = (double*)malloc(nRows * sizeof(double*)); - - for (int i = 0; i < nRows; ++i) { - int myLabel = h_labels[i]; - double sumOfIntraClusterD = 0; - - for (int j = 0; j < nRows; ++j) { - if (h_labels[j] == myLabel) { sumOfIntraClusterD += h_distanceMatrix[i * nRows + j]; } - } - - if (binCountArray[myLabel] <= 1) - a[i] = -1; - else - a[i] = sumOfIntraClusterD / (binCountArray[myLabel] - 1); - } - - // finding the average inter cluster distance for every element - - double* b = (double*)malloc(nRows * sizeof(double*)); - - for (int i = 0; i < nRows; ++i) { - int myLabel = h_labels[i]; - double minAvgInterCD = ULLONG_MAX; - - for (int j = 0; j < nLabels; ++j) { - int curClLabel = j; - if (curClLabel == myLabel) continue; - double avgInterCD = 0; - - for (int k = 0; k < nRows; ++k) { - if (h_labels[k] == curClLabel) { avgInterCD += h_distanceMatrix[i * nRows + k]; } - } - - if (binCountArray[curClLabel]) - avgInterCD /= binCountArray[curClLabel]; - else - avgInterCD = ULLONG_MAX; - minAvgInterCD = min(minAvgInterCD, avgInterCD); - } - - b[i] = minAvgInterCD; - } - - // finding the silhouette score for every element - - double* truthSampleSilScore = (double*)malloc(nRows * sizeof(double*)); - for (int i = 0; i < nRows; ++i) { - if (a[i] == -1) - truthSampleSilScore[i] = 0; - else if (a[i] == 0 && b[i] == 0) - truthSampleSilScore[i] = 0; - else - truthSampleSilScore[i] = (b[i] - a[i]) / max(a[i], b[i]); - truthSilhouetteScore += truthSampleSilScore[i]; - } - - truthSilhouetteScore /= nRows; - } - - // the constructor - void SetUp() override - { - // getting the parameters - params = ::testing::TestWithParam::GetParam(); - - nRows = params.nRows; - nCols = params.nCols; - nLabels = params.nLabels; - chunk = params.chunk; - nElements = nRows * nCols; - - host_silhouette_score(); - - // calling the silhouette_score CUDA implementation - computedSilhouetteScore = raft::stats::silhouette_score( - handle, - raft::make_device_matrix_view(d_X.data(), nRows, nCols), - raft::make_device_vector_view(d_labels.data(), nRows), - std::make_optional(raft::make_device_vector_view(sampleSilScore.data(), nRows)), - nLabels, - params.metric); - - batchedSilhouetteScore = raft::stats::silhouette_score_batched( - handle, - raft::make_device_matrix_view(d_X.data(), nRows, nCols), - raft::make_device_vector_view(d_labels.data(), nRows), - std::make_optional(raft::make_device_vector_view(sampleSilScore.data(), nRows)), - nLabels, - chunk, - params.metric); - } - - // declaring the data values - raft::resources handle; - silhouetteScoreParam params; - int nLabels; - rmm::device_uvector d_X; - rmm::device_uvector sampleSilScore; - rmm::device_uvector d_labels; - int nRows; - int nCols; - int nElements; - double truthSilhouetteScore = 0; - double computedSilhouetteScore = 0; - double batchedSilhouetteScore = 0; - int chunk; -}; - -// setting test parameter values -const std::vector inputs = { - {4, 2, 3, raft::distance::DistanceType::L2Expanded, 4, 0.00001}, - {4, 2, 2, raft::distance::DistanceType::L2SqrtUnexpanded, 2, 0.00001}, - {8, 8, 3, raft::distance::DistanceType::L2Unexpanded, 4, 0.00001}, - {11, 2, 5, raft::distance::DistanceType::L2Expanded, 3, 0.00001}, - {40, 2, 8, raft::distance::DistanceType::L2Expanded, 10, 0.00001}, - {12, 7, 3, raft::distance::DistanceType::CosineExpanded, 8, 0.00001}, - {7, 5, 5, raft::distance::DistanceType::L1, 2, 0.00001}}; - -// writing the test suite -typedef silhouetteScoreTest silhouetteScoreTestClass; -TEST_P(silhouetteScoreTestClass, Result) -{ - ASSERT_NEAR(computedSilhouetteScore, truthSilhouetteScore, params.tolerance); - ASSERT_NEAR(batchedSilhouetteScore, truthSilhouetteScore, params.tolerance); -} -INSTANTIATE_TEST_CASE_P(silhouetteScore, silhouetteScoreTestClass, ::testing::ValuesIn(inputs)); - -} // end namespace stats -} // end namespace raft diff --git a/cpp/test/stats/trustworthiness.cu b/cpp/test/stats/trustworthiness.cu deleted file mode 100644 index 846c192022..0000000000 --- a/cpp/test/stats/trustworthiness.cu +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "../test_utils.cuh" - -#include -#include -#include -#include - -#include - -#include -#include - -namespace raft { -namespace stats { - -class TrustworthinessScoreTest : public ::testing::Test { - public: - TrustworthinessScoreTest() - : d_X(0, resource::get_cuda_stream(handle)), d_X_embedded(0, resource::get_cuda_stream(handle)) - { - } - - protected: - void basicTest() - { - std::vector X = { - 5.6142087, 8.59787, -4.382763, -3.6452143, -5.8816037, -0.6330313, 4.6920023, - -0.79210913, 0.6106314, 2.1210914, 5.919943, -8.43784, -6.4819884, 0.41001374, - -6.1052523, -4.0825715, -5.314755, -2.834671, 5.751696, -6.5012555, -0.4719201, - -7.53353, 7.6789393, -1.4959852, -5.5977287, -9.564147, 1.2902534, 3.559834, - -6.7659483, 8.265964, 4.595404, 9.133477, -6.1553917, -6.319754, -2.9039452, - 4.4150834, -3.094395, -4.426273, 9.584571, -5.64133, 6.6209483, 7.4044604, - 3.9620576, 5.639907, 10.33007, -0.8792053, 5.143776, -7.464049, 1.2448754, - -5.6300974, 5.4518576, 4.119535, 6.749645, 7.627064, -7.2298336, 1.9681473, - -6.9083176, 6.404673, 0.07186685, 9.0994835, 8.51037, -8.986389, 0.40534487, - 2.115397, 4.086756, 1.2284287, -2.6272132, 0.06527536, -9.587425, -7.206078, - 7.864875, 7.4397306, -6.9233336, -2.6643622, 3.3466153, 7.0408177, -3.6069896, - -9.971769, 4.4075623, 7.9063697, 2.559074, 4.323717, 1.6867131, -1.1576937, - -9.893141, -3.251416, -7.4889135, -4.0588717, -2.73338, -7.4852257, 3.4460473, - 9.759119, -5.4680476, -4.722435, -8.032619, -1.4598992, 4.227361, 3.135568, - 1.1950601, 1.1982028, 6.998856, -6.131138, -6.6921015, 0.5361224, -7.1213965, - -5.6104236, -7.2212887, -2.2710054, 8.544764, -6.0254574, 1.4582269, -5.5587835, - 8.031556, -0.26328218, -5.2591386, -9.262641, 2.8691363, 5.299787, -9.209455, - 8.523085, 5.180329, 10.655528, -5.7171874, -6.7739563, -3.6306462, 4.067106, - -1.5912259, -3.2345476, 8.042973, -3.6364832, 4.1242137, 9.886953, 5.4743724, - 6.3058076, 9.369645, -0.5175337, 4.9859877, -7.879498, 1.358422, -4.147944, - 3.8984218, 5.894656, 6.4903927, 8.702036, -8.023722, 2.802145, -7.748032, - 5.8461113, -0.34215945, 11.298865, 1.4107164, -9.949621, -1.6257563, -10.655836, - 2.4528909, 1.1570255, 5.170669, 2.8398793, 7.1838694, 9.088459, 2.631155, - 3.964414, 2.8769252, 0.04198391, -0.16993195, 3.6747139, -2.8377378, 6.1782537, - 10.759618, -4.5642614, -8.522967, 0.8614642, 6.623416, -1.029324, 5.5488334, - -7.804511, 2.128833, 7.9042315, 7.789576, -2.7944536, 0.72271067, -10.511495, - -0.78634536, -10.661714, 2.9376361, 1.9148129, 6.22859, 0.26264945, 8.028384, - 6.8743043, 0.9351067, 7.0690722, 4.2846055, 1.4134506, -0.18144785, 5.2778087, - -1.7140163, 9.217541, 8.602799, -2.6537218, -7.8377395, 1.1244944, 5.4540544, - -0.38506773, 3.9885726, -10.76455, 1.4440702, 9.136163, 6.664117, -5.7046547, - 8.038592, -9.229767, -0.2799413, 3.6064725, 4.187257, 1.0516582, -2.0707326, - -0.7615968, -8.561018, -3.7831352, 10.300297, 5.332594, -6.5880876, -4.2508664, - 1.7985519, 5.7226253, -4.1223383, -9.6697855, 1.4885283, 7.524974, 1.7206005, - 4.890457, 3.7264557, 0.4428284, -9.922455, -4.250455, -6.4410596, -2.107994, - -1.4109765, -6.1325397, 0.32883006, 6.0489736, 7.7257385, -8.281174, 1.0129383, - -10.792166, 8.378851, 10.802716, 9.848448, -9.188757, 1.3151443, 1.9971865, - -2.521849, 4.3268294, -7.775683, -2.2902298, 3.0824065, -7.17559, 9.6100855, - 7.3965735, -10.476525, 5.895973, -3.6974669, -7.6688933, 1.7354839, -7.4045196, - -1.7992063, -4.0394845, 5.2471714, -2.250571, 2.528036, -8.343515, -2.2374575, - -10.019771, 0.73371273, 3.1853926, 2.7994921, 2.6637669, 7.620401, 7.515571, - 0.68636256, 5.834537, 4.650282, -1.0362619, 0.4461701, 3.7870514, -4.1340904, - 7.202998, 9.736904, -3.005512, -8.920467, 1.1228397, 6.2598724, 1.2812365, - 4.5442104, -8.791537, 0.92113096, 8.464749, 8.359035, -4.3923397, 1.2252625, - -10.1986475, -1.4409319, -10.013967, 3.9071581, 1.683064, 4.877419, 1.6570637, - 9.559105, 7.3546534, 0.36635467, 5.220211, 4.6303267, 0.6601065, 0.16149978, - 3.8818731, -3.4438233, 8.42085, 8.659159, -3.0935583, -8.039611, 2.3060374, - 5.134666, 1.0458113, 6.0190983, -9.143728, 0.99048865, 9.210842, 6.670241, - -5.9614363, 0.8747396, 7.078824, 8.067469, -10.314754, 0.45977542, -9.28306, - 9.1838665, 9.318644, 7.189082, -11.092555, 1.0320464, 3.882163, 0.10953151, - 7.9029684, -6.9068265, -1.3526366, 5.3996363, -8.430931, 11.452577, 6.39663, - -11.090514, 4.6662245, -3.1268113, -8.357452, 2.2276728, -10.357126, -0.9291848, - -3.4193344, 3.1289792, -2.5030103, 6.772719, 11.457757, -4.2125936, -6.684548, - -4.7611327, 3.6960156, -2.3030636, -3.0591488, 10.452471, -4.1267314, 5.66614, - 7.501461, 5.072407, 6.636537, 8.990381, -0.2559256, 4.737867, -6.2149944, - 2.535682, -5.5484023, 5.7113924, 3.4742818, 7.9915137, 7.0052586, -7.156467, - 1.4354781, -8.286235, 5.7523417, -2.4175215, 9.678009, 0.05066403, -9.645226, - -2.2658763, -9.518178, 4.493372, 2.3232365, 2.1659086, 0.42507997, 8.360246, - 8.23535, 2.6878164, 5.236947, 3.4924245, -0.6089895, 0.8884741, 4.359464, - -4.6073823, 7.83441, 8.958755, -3.4690795, -9.182282, 1.2478025, 5.6311107, - -1.2408862, 3.6316886, -8.684654, 2.1078515, 7.2813864, 7.9265943, -3.6135032, - 0.4571511, 8.493568, 10.496853, -7.432897, 0.8625995, -9.607528, 7.2899456, - 8.83158, 8.908199, -10.300263, 1.1451302, 3.7871468, -0.97040755, 5.7664757, - -8.9688, -2.146672, 5.9641485, -6.2908535, 10.126465, 6.1553903, -12.066902, - 6.301596, -5.0419583, -8.228695, 2.4879954, -8.918582, -3.7434099, -4.1593685, - 3.7431836, -1.1704745, 0.5524103, 9.109399, 9.571567, -11.209955, 1.2462777, - -9.554555, 9.091726, 11.477966, 7.630937, -10.450911, 1.9205878, 5.358983, - -0.44546837, 6.7611346, -9.74753, -0.5939732, 3.8892255, -6.437991, 10.294727, - 5.6723895, -10.7883, 6.192348, -5.293862, -10.811491, 1.0194173, -7.074576, - -3.192368, -2.5231771, 4.2791643, -0.53309685, 0.501366, 9.636625, 7.710316, - -6.4219728, 1.0975566, -8.218886, 6.9011984, 9.873679, 8.903804, -9.316832, - 1.2404599, 4.9039655, 1.2272617, 4.541515, -5.2753224, -3.2196746, 3.1303136, - -7.285681, 9.041425, 5.6417427, -9.93667, 5.7548947, -5.113397, -8.544622, - 4.182665, -7.7709813, -3.2810235, -3.312072, 3.8900535, -2.0604856, 6.709082, - -8.461194, 1.2666026, 4.8770437, 2.6955879, 3.0340345, -1.1614609, -3.536341, - -7.090382, -5.36146, 9.072544, 6.4554095, -4.4728956, -1.88395, 3.1095037, - 8.782348, -3.316743, -8.65248, 1.6802986, 8.186188, 2.1783829, 4.931278, - 4.158475, 1.4033595, -11.320101, -3.7084908, -6.740436, -2.5555193, -1.0451177, - -6.5569925, 0.82810307, 8.505919, 8.332857, -9.488569, -0.21588463, -8.056692, - 8.493993, 7.6401625, 8.812983, -9.377281, 2.4369764, 3.1766508, 0.6300803, - 5.6666765, -7.913654, -0.42301777, 4.506412, -7.8954244, 10.904591, 5.042256, - -9.626183, 8.347351, -3.605006, -7.923387, 1.1024277, -8.705793, -2.5151258, - -2.5066147, 4.0515003, -2.060757, 6.2635093, 8.286584, -6.0509276, -6.76452, - -3.1158175, 1.6578803, -1.4608748, -1.24211, 8.151246, -4.2970877, 6.093071, - 7.4911637, 4.51018, 4.8425875, 9.211085, -2.4386222, 4.5830803, -5.6079445, - 2.3713675, -4.0707507, 3.1787417, 5.462342, 6.915912, 6.3928423, -7.2970796, - 5.0112796, -9.140893, 4.9990606, 0.38391754, 7.7088532, 1.9340848, 8.18833, - 8.16617, -9.42086, -0.3388326, -9.659727, 8.243045, 8.099073, 8.439428, - -7.038694, 2.1077902, 3.3866816, -1.9975324, 7.4972878, -7.2525196, -1.553731, - 4.08758, -6.6922374, 9.50525, 4.026735, -9.243538, 7.2740564, -3.9319072, - -6.3228955, 1.6693478, -7.923119, -3.7423058, -2.2813146, 5.3469067, -1.8285407, - 3.3118162, 8.826356, -4.4641976, -6.4751124, -9.200089, -2.519147, 4.225298, - 2.4105988, -0.4344186, 0.53441775, 5.2836394, -8.2816105, -4.996147, -1.6870759, - -7.8543897, -3.9788852, -7.0346904, -3.1289773, 7.4567637, -5.6227813, 1.0709786, - -8.866012, 8.427324, -1.1755563, -5.789216, -8.197835, 5.3342214, 6.0646234, - -6.8975716, 7.717031, 3.480355, 8.312151, -3.6645212, -3.0976524, -8.090359, - -1.9176173, 2.4257212, 1.9700835, 0.4098958, 2.1341088, 7.652741, -9.9595585, - -5.989757, 0.10119354, -7.935407, -5.792786, -5.22783, -4.318978, 5.414037, - -6.4621663, 1.670883, -6.9224787, 8.696932, -2.0214002, -6.6681314, -8.326418, - 4.9049683, 5.4442496, -6.403739, 7.5822453, 7.0972915, -9.072851, -0.23897195, - 1.7662339, 5.3096304, 1.983179, -2.222645, -0.34700772, -9.094717, -6.107907, - 9.525174, 8.1550665, -5.6940084, -4.1636486, 1.7360662, 8.528821, -3.7299833, - -9.341266, 2.608542, 9.108706, 0.7978509, 4.2488184, 2.454484, 0.9446999, - -10.106636, -3.8973773, -6.6566644, -4.5647273, -0.99837756, -6.568582, 9.324853, - -7.9020953, 2.0910501, 2.2896829, 1.6790711, 1.3159255, -3.5258796, 1.8898442, - -8.105812, -4.924962, 8.771129, 7.1202874, -5.991957, -3.4106019, 2.4450088, - 7.796387, -3.055946, -7.8971434, 1.9856719, 9.001636, 1.8511922, 3.019749, - 3.1227696, 0.4822102, -10.021213, -3.530504, -6.225959, -3.0029628, -1.7881511, - -7.3879776, 1.3925704, 9.499782, -3.7318087, -3.7074296, -7.7466836, -1.5284524, - 4.0535855, 3.112011, 0.10340207, -0.5429599, 6.67026, -9.155924, -4.924038, - 0.64248866, -10.0103655, -3.2742946, -4.850029, -3.6707063, 8.586258, -5.855605, - 4.906918, -6.7813993, 7.9938135, -2.5473144, -5.688948, -7.822478, 2.1421318, - 4.66659, -9.701272, 9.549149, 0.8998125, -8.651497, -0.56899565, -8.639817, - 2.3088377, 2.1264515, 3.2764478, 2.341989, 8.594338, 8.630639, 2.8440373, - 6.2043204, 4.433932, 0.6320018, -1.8179281, 5.09452, -1.5741565, 8.153934, - 8.744339, -3.6945698, -8.883078, 1.5329908, 5.2745943, 0.44716078, 4.8809066, - -7.9594903, 1.134374, 9.233994, 6.5528665, -4.520542, 9.477355, -8.622195, - -0.23191702, 2.0485356, 3.9379985, 1.5916302, -1.4516805, -0.0843819, -7.8554378, - -5.88308, 7.999766, 6.2572145, -5.585321, -4.0097756, 0.42382592, 6.160884, - -3.631315, -8.333449, 2.770595, 7.8495173, 3.3331623, 4.940415, 3.6207345, - -0.037517, -11.034698, -3.185103, -6.614664, -3.2177854, -2.0792234, -6.8879867, - 7.821685, -8.455084, 1.0784642, 4.0033927, 2.7343264, 2.6052725, -4.1224284, - -0.89305353, -6.8267674, -4.9715133, 8.880253, 5.6994023, -5.9695024, -4.9181266, - 1.3017995, 7.972617, -3.9452884, -10.424556, 2.4504194, 6.21529, 0.93840516, - 4.2070026, 6.159839, 0.91979957, -8.706724, -4.317946, -6.6823545, -3.0388, - -2.464262, -7.3716645, 1.3926703, 6.544412, -5.6251183, -5.122411, -8.622049, - -2.3905911, 3.9138813, 1.9779967, -0.05011125, 0.13310997, 7.229751, -9.742043, - -8.08724, 1.2426697, -7.9230795, -3.3162494, -7.129571, -3.5488048, 7.4701195, - -5.2357526, 0.5917681, -6.272206, 6.342328, -2.909731, -4.991607, -8.845513, - 3.3228495, 7.033246, -7.8180246, 8.214469, 6.3910093, 9.185153, -6.20472, - -7.713809, -3.8481297, 3.5579286, 0.7078448, -3.2893546, 7.384514, -4.448121, - 3.0104196, 9.492943, 8.024847, 4.9114385, 9.965594, -3.014036, 5.182494, - -5.8806014, 2.5312455, -5.9926524, 4.474469, 6.3717875, 6.993105, 6.493093, - -8.935534, 3.004074, -8.055647, 8.315765, -1.3026813, 8.250377, 0.02606229, - 6.8508425, 9.655665, -7.0116496, -0.41060972, -10.049198, 7.897801, 6.7791023, - 8.3362, -9.821014, 2.491157, 3.5160472, -1.6228812, 7.398063, -8.769123, - -3.1743705, 3.2827861, -6.497855, 10.831924, 5.2761307, -9.704417, 4.3817043, - -3.9841619, -8.111647, 1.1883026, -8.115312, -2.9240117, -5.8879666, 4.20928, - -0.3587938, 6.935672, -10.177582, 0.48819053, 3.1250648, 2.9306343, 3.082544, - -3.477687, -1.3768549, -7.4922366, -3.756631, 10.039836, 3.6670392, -5.9761434, - -4.4728765, 3.244255, 7.027899, -2.3806512, -10.4100685, 1.605716, 7.7953773, - 0.5408159, 1.7156523, 3.824097, -1.0604783, -10.142124, -5.246805, -6.5283823, - -4.579547, -2.42714, -6.709197, 2.7782338, 7.33353, -6.454507, -2.9929368, - -7.8362985, -2.695445, 2.4900775, 1.6682367, 0.4641757, -1.0495365, 6.9631333, - -9.291356, -8.23837, -0.34263706, -8.275113, -2.8454232, -5.0864096, -2.681942, - 7.5450225, -6.2517986, 0.06810654, -6.470652, 4.9042645, -1.8369255, -6.6937943, - -7.9625087, 2.8510258, 6.180508, -8.282598, 7.919079, 1.4897474, 6.7217417, - -4.2459426, -4.114431, -8.375707, -2.143264, 5.6972933, 1.5574739, 0.39375135, - 1.7930849, 5.1737595, -7.826241, -5.160268, -0.80433255, -7.839536, -5.2620406, - -5.4643164, -3.185536, 6.620315, -7.065227, 1.0524757, -6.125088, 5.7126627, - -1.6161644, -3.852159, -9.164279, 2.7005782, 5.946544, -8.468236, 8.2145405, - 1.1035942, 6.590157, -4.0461283, -4.8090615, -7.6702685, -2.1121511, 5.1147075, - 1.6128504, 2.0064135, 1.0544407, 6.0038295, -7.8282537, -4.801278, 0.32349443, - -8.0649805, -4.372714, -5.61336, -5.21394, 8.176595, -5.4753284, 1.7800134, - -8.267283, 7.2133374, -0.16594432, -6.317046, -9.490406, 4.1261597, 5.473317, - -7.7551675, 7.007468, 7.478628, -8.801905, 0.10975724, 3.5478222, 4.797803, - 1.3825226, -3.357369, 0.99262005, -6.94877, -5.4781394, 9.632604, 5.7492557, - -5.9014316, -3.1632116, 2.340859, 8.708098, -3.1255999, -8.848661, 4.5612836, - 8.455157, 0.73460823, 4.112301, 4.392744, -0.30759293, -6.8036823, -3.0331545, - -8.269506, -2.82415, -0.9411246, -5.993506, 2.1618164, -8.716055, -0.7432543, - -10.255819, 3.095418, 2.5131428, 4.752442, 0.9907621, 7.8279433, 7.85814, - 0.50430876, 5.2840405, 4.457291, 0.03330028, -0.40692952, 3.9244103, -2.117118, - 7.6977615, 8.759009, -4.2157164, -9.136053, 3.247858, 4.668686, 0.76162136, - 5.3833632, -9.231471, 0.44309422, 8.380872, 6.7211227, -3.091507, 2.173508, - -9.038242, -1.3666698, -9.819077, 0.37825826, 2.3898845, 4.2440815, 1.9161536, - 7.24787, 6.9124637, 1.6238527, 5.1140285, 3.1935842, 1.02845, -1.1273454, - 5.638998, -2.497932, 8.342559, 8.586319, -2.9069402, -7.6387944, 3.5975037, - 4.4115705, 0.41506064, 4.9078383, -9.68327, 1.8159529, 9.744613, 8.40622, - -4.495336, 9.244892, -8.789869, 1.3158468, 4.018167, 3.3922846, 2.652022, - -2.7495477, 0.2528986, -8.268324, -6.004913, 10.428784, 6.6580734, -5.537176, - -1.7177434, 2.7504628, 6.7735, -2.4454272, -9.998361, 2.9483433, 6.8266654, - 2.3787718, 4.472637, 2.5871701, 0.7355365, -7.7027745, -4.1879907, -7.172832, - -4.1843605, -0.03646783, -5.419406, 6.958486, 11.011111, -7.1821184, -7.956423, - -3.408451, 4.6850276, -2.348787, -4.398289, 6.9787564, -3.8324208, 5.967827, - 8.433518, 4.660108, 5.5657144, 9.964243, -1.3515275, 6.404833, -6.4805903, - 2.4379845, -6.0816774, 1.752272, 5.3771873, 6.9613523, 6.9788294, -6.3894596, - 3.7521114, -6.8034263, 6.4458385, -0.7233525, 10.512529, 4.362273, 9.231461, - -6.3382263, -7.659, -3.461823, 4.71463, 0.17817476, -3.685746, 7.2962036, - -4.6489477, 5.218017, 11.546999, 4.7218375, 6.8498397, 9.281103, -3.900459, - 6.844054, -7.0886965, -0.05019227, -8.233724, 5.5808983, 6.374517, 8.321048, - 7.969449, -7.3478637, 1.4917561, -8.003144, 4.780668, -1.1981848, 7.753739, - 2.0260844, -8.880096, -3.4258451, -7.141975, 1.9637157, 1.814725, 5.311151, - 1.4831505, 7.8483663, 7.257948, 1.395786, 6.417756, 5.376912, 0.59505713, - 0.00062552, 3.6634305, -4.159713, 7.3571978, 10.966816, -2.5419605, -8.466229, - 1.904205, 5.6338267, -0.52567476, 5.59736, -8.361799, 0.5009981, 8.460681, - 7.3891273, -3.5272243, 5.0552278, 9.921456, -7.69693, -7.286378, -1.9198836, - 3.1666567, -2.5832257, -2.2445817, 9.888111, -5.076563, 5.677401, 7.497946, - 5.662994, 5.414262, 8.566503, -2.5530663, 7.1032815, -6.0612082, 1.3419591, - -4.9595256, 4.3377542, 4.3790717, 6.793512, 8.383502, -7.1278043, 3.3240774, - -9.379446, 6.838661, -0.81241214, 8.694813, 0.79141915, 7.632467, 8.575382, - -8.533798, 0.28954387, -7.5675836, 5.8653326, 8.97235, 7.1649346, -10.575289, - 0.9359381, 5.02381, -0.5609511, 5.543464, -7.69131, -2.1792977, 2.4729247, - -6.1917787, 10.373678, 7.6549597, -8.809486, 5.5657206, -3.3169382, -8.042887, - 2.0874746, -7.079005, -3.33398, -3.6843317, 4.0172358, -2.0754814, 1.1726758, - 7.4618697, 6.9483604, -8.469206, 0.7401797, -10.318176, 8.384557, 10.5476265, - 9.146971, -9.250223, 0.6290606, 4.4941425, -0.7514017, 7.2271705, -8.309598, - -1.4761636, 4.0140634, -6.021102, 9.132852, 5.6610966, -11.249811, 8.359293, - -1.9445792, -7.7393436, -0.3931331, -8.824441, -2.5995944, -2.5714035, 4.140213, - -3.6863053, 5.517265, 9.020411, -4.9286127, -7.871219, -3.7446704, 2.5179656, - -1.4543481, -2.2703636, 7.010597, -3.6436229, 6.753862, 7.4129915, 7.1406755, - 5.653706, 9.5445175, 0.15698843, 4.761813, -7.698002, 1.6870106, -4.5410123, - 4.171763, 5.3747005, 6.341021, 7.456738, -8.231657, 2.763487, -9.208167, - 6.676799, -1.1957736, 10.062605, 4.0975976, 7.312957, -2.4981596, -2.9658387, - -8.150425, -2.1075552, 2.64375, 1.6636052, 1.1483809, 0.09276015, 5.8556347, - -7.8481026, -5.9913163, -0.02840613, -9.937289, -1.0486673, -5.2340155, -3.83912, - 7.7165728, -8.409944, 0.80863273, -6.9119215, 7.5712357, 0.36031485, -6.056131, - -8.470033, 1.8678337, 3.0121377, -7.3096333, 8.205484, 5.262654, 8.774514, - -4.7603083, -7.2096143, -4.437014, 3.6080024, -1.624254, -4.2787876, 8.880863, - -4.8984556, 5.1782074, 9.944454, 3.911282, 3.5396595, 8.867042, -1.2006199, - 5.393288, -5.6455317, 0.7829499, -4.0338907, 2.479272, 6.5080743, 8.582535, - 7.0097537, -6.9823785, 3.984318, -7.225381, 5.3135114, -1.0391048, 8.951443, - -0.70119005, -8.510742, -0.42949116, -10.9224825, 2.8176029, 1.6800792, 5.778404, - 1.7269998, 7.1975236, 7.7258267, 2.7632928, 5.3399253, 3.4650044, 0.01971426, - -1.6468811, 4.114996, -1.5110453, 6.8689218, 8.269899, -3.1568048, -7.0344677, - 1.2911975, 5.950357, 0.19028673, 4.657226, -8.199647, 2.246055, 8.989509, - 5.3101015, -4.2400866}; - - std::vector X_embedded = { - -0.41849962, -0.53906363, 0.46958843, -0.35832694, -0.23779503, -0.29751351, -0.01072748, - -0.21353109, -0.54769957, -0.55086273, 0.37093949, -0.12714292, -0.06639574, -0.36098689, - -0.13060696, -0.07362658, -1.01205945, -0.39285606, 0.2864089, -0.32031146, -0.19595343, - 0.08900568, -0.04813879, -0.06563424, -0.42655188, -0.69014251, 0.51459783, -0.1942696, - -0.07767916, -0.6119386, 0.04813685, -0.22557008, -0.56890118, -0.60293794, 0.43429622, - -0.09240723, -0.00624062, -0.25800395, -0.1886092, 0.01655941, -0.01961523, -0.14147359, - 0.41414487, -0.8512944, -0.61199242, -0.18586016, 0.14024924, -0.41635606, -0.02890144, - 0.1065347, 0.39700791, -1.14060664, -0.95313865, 0.14416681, 0.17306046, -0.53189689, - -0.98987544, -0.67918193, 0.41787854, -0.20878236, -0.06612862, 0.03502904, -0.03765266, - -0.0980606, -0.00971657, 0.29432917, 0.36575687, -1.1645509, -0.89094597, 0.03718805, - 0.2310573, -0.38345811, -0.10401925, -0.10653082, 0.38469055, -0.88302094, -0.80197543, - 0.03548668, 0.02775662, -0.54374295, 0.03379983, 0.00923623, 0.29320273, -1.05263519, - -0.93360096, 0.03778313, 0.12360487, -0.56437284, 0.0644429, 0.33432651, 0.36450726, - -1.22978747, -0.83822101, -0.18796451, 0.34888434, -0.3801491, -0.45327303, -0.59747899, - 0.39697698, -0.15616602, -0.06159166, -0.40301991, -0.11725303, -0.11913263, -0.12406619, - -0.11227967, 0.43083835, -0.90535849, -0.81646025, 0.10012121, -0.0141237, -0.63747931, - 0.04805023, 0.34190539, 0.50725192, -1.17861414, -0.74641538, -0.09333111, 0.27992678, - -0.56214809, 0.04970971, 0.36249384, 0.57705611, -1.16913795, -0.69849908, 0.10957897, - 0.27983218, -0.62088525, 0.0410459, 0.23973398, 0.40960434, -1.14183664, -0.83321381, - 0.02149482, 0.21720445, -0.49869928, -0.95655465, -0.51680422, 0.45761383, -0.08351214, - -0.12151554, 0.00819737, -0.20813803, -0.01055793, 0.25319234, 0.36154974, 0.1822421, - -1.15837133, -0.92209691, -0.0501582, 0.08535917, -0.54003763, -1.08675635, -1.04009593, - 0.09408128, 0.07009826, -0.01762833, -0.19180447, -0.18029785, -0.20342001, 0.04034991, - 0.1814747, 0.36906669, -1.13532007, -0.8852452, 0.0782818, 0.16825101, -0.50301319, - -0.29128098, -0.65341312, 0.51484352, -0.38758236, -0.22531103, -0.55021971, 0.10804344, - -0.3521522, -0.38849035, -0.74110794, 0.53761131, -0.25142813, -0.1118066, -0.47453368, - 0.06347904, -0.23796193, -1.02682328, -0.47594091, 0.39515916, -0.2782529, -0.16566519, - 0.08063579, 0.00810116, -0.06213913, -1.059654, -0.62496334, 0.53698546, -0.11806234, - 0.00356161, 0.11513405, -0.14213292, 0.04102662, -0.36622161, -0.73686272, 0.48323864, - -0.27338892, -0.14203401, -0.41736352, 0.03332564, -0.21907479, -0.06396769, 0.01831361, - 0.46263444, -1.01878166, -0.86486858, 0.17622118, -0.01249686, -0.74530888, -0.9354887, - -0.5027945, 0.38170099, -0.15547098, 0.00677824, -0.04677663, -0.13541745, 0.07253501, - -0.97933143, -0.58001202, 0.48235369, -0.18836913, -0.02430783, 0.07572441, -0.08101331, - 0.00630076, -0.16881248, -0.67989182, 0.46083611, -0.43910736, -0.29321918, -0.38735861, - 0.07669903, -0.29749861, -0.40047669, -0.56722462, 0.33168188, -0.13118173, -0.06672747, - -0.56856316, -0.26269144, -0.14236671, 0.10651901, 0.4962585, 0.38848072, -1.06653547, - -0.64079332, -0.47378591, 0.43195483, -0.04856951, -0.9840439, -0.70610428, 0.34028092, - -0.2089237, -0.05382041, 0.01625874, -0.02080803, -0.12535211, -0.04146428, -1.24533033, - 0.48944879, 0.0578458, 0.26708388, -0.90321028, 0.35377088, -0.36791429, -0.35382384, - -0.52748734, 0.42854419, -0.31744713, -0.19174226, -0.39073724, -0.03258846, -0.19978228, - -0.36185205, -0.57412046, 0.43681973, -0.25414538, -0.12904905, -0.46334973, -0.03123853, - -0.11303604, -0.87073672, -0.45441297, 0.41825858, -0.25303507, -0.21845073, 0.10248682, - -0.11045569, -0.10002795, -0.00572806, 0.16519061, 0.42651513, -1.11417019, -0.83789682, - 0.02995787, 0.16843079, -0.53874511, 0.03056994, 0.17877036, 0.49632853, -1.03276777, - -0.74778616, -0.03971953, 0.10907949, -0.67385727, -0.9523471, -0.56550741, 0.40409449, - -0.2703723, -0.10175014, 0.13605487, -0.06306008, -0.01768126, -0.4749442, -0.56964815, - 0.39389887, -0.19248079, -0.04161081, -0.38728487, -0.20341556, -0.12656988, -0.35949609, - -0.46137866, 0.28798422, -0.06603147, -0.04363992, -0.60343552, -0.23565227, -0.10242701, - -0.06792886, 0.09689897, 0.33259571, -0.98854214, -0.84444433, 0.00673901, 0.13457057, - -0.43145794, -0.51500046, -0.50821936, 0.38000089, 0.0132636, 0.0580942, -0.40157595, - -0.11967677, 0.02549113, -0.10350953, 0.22918226, 0.40411913, -1.05619383, -0.71218503, - -0.02197581, 0.26422262, -0.34765676, 0.06601537, 0.21712676, 0.34723559, -1.20982027, - -0.95646334, 0.00793948, 0.27620381, -0.43475035, -0.67326003, -0.6137197, 0.43724492, - -0.17666136, -0.06591748, -0.18937394, -0.07400128, -0.06881691, -0.5201112, -0.61088628, - 0.4225319, -0.18969463, -0.06921366, -0.33993208, -0.06990873, -0.10288513, -0.70659858, - -0.56003648, 0.46628812, -0.16090363, -0.0185108, -0.1431348, -0.1128775, -0.0078648, - -0.02323332, 0.04292452, 0.39291084, -0.94897962, -0.63863206, -0.16546988, 0.23698957, - -0.30633628}; - - auto stream = resource::get_cuda_stream(handle); - - d_X.resize(X.size(), stream); - d_X_embedded.resize(X_embedded.size(), stream); - raft::update_device(d_X.data(), X.data(), X.size(), stream); - raft::update_device(d_X_embedded.data(), X_embedded.data(), X_embedded.size(), stream); - auto n_sample = 50; - auto n_features_origin = 30; - auto n_features_embedded = 8; - - // euclidean test - score = trustworthiness_score( - handle, - raft::make_device_matrix_view(d_X.data(), n_sample, n_features_origin), - raft::make_device_matrix_view( - d_X_embedded.data(), n_sample, n_features_embedded), - 5); - } - - void SetUp() override { basicTest(); } - - void TearDown() override {} - - protected: - raft::resources handle; - - rmm::device_uvector d_X; - rmm::device_uvector d_X_embedded; - - double score; -}; - -typedef TrustworthinessScoreTest TrustworthinessScoreTestF; -TEST_F(TrustworthinessScoreTestF, Result) { ASSERT_TRUE(0.9375 < score && score < 0.9379); } -}; // namespace stats -}; // namespace raft diff --git a/dependencies.yaml b/dependencies.yaml index d0991f4d04..1772c5d539 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -22,19 +22,6 @@ files: - run_pylibraft - test_python_common - test_pylibraft - bench_ann: - output: conda - matrix: - cuda: ["11.8", "12.0"] - arch: [x86_64, aarch64] - includes: - - rapids_build - - cuda - - cuda_version - - develop - - nn_bench - - nn_bench_python - - rapids_build_skbuild test_cpp: output: none includes: @@ -127,20 +114,6 @@ files: key: test includes: - test_python_common - py_build_raft_ann_bench: - output: pyproject - pyproject_dir: python/raft-ann-bench - extras: - table: build-system - includes: - - rapids_build_setuptools - py_run_raft_ann_bench: - output: pyproject - pyproject_dir: python/raft-ann-bench - extras: - table: project - includes: - - nn_bench_python channels: - rapidsai - rapidsai-nightly @@ -164,14 +137,14 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - &cmake_ver cmake>=3.26.4,!=3.30.0 - - cython>=3.0.0 + - cython>=3.0.0,<3.1.0a0 - ninja - output_types: [conda] packages: - c-compiler - cxx-compiler - nccl>=2.19 - - libucxx==0.40.*,>=0.0.0a0 + - libucxx==0.41.*,>=0.0.0a0 specific: - output_types: conda matrices: @@ -210,7 +183,7 @@ dependencies: common: - output_types: [conda] packages: - - &rmm_unsuffixed rmm==24.10.*,>=0.0.0a0 + - &rmm_unsuffixed rmm==24.12.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -223,11 +196,11 @@ dependencies: - matrix: cuda: "12.*" packages: - - &cuda_python12 cuda-python>=12.0,<13.0a0 + - &cuda_python12 cuda-python>=12.0,<13.0a0,<=12.6.0 - matrix: cuda: "11.*" packages: - - &cuda_python11 cuda-python>=11.7.1,<12.0a0 + - &cuda_python11 cuda-python>=11.7.1,<12.0a0,<=11.8.3 - matrix: packages: - &cuda_python cuda-python @@ -237,12 +210,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - &rmm_cu12 rmm-cu12==24.10.*,>=0.0.0a0 + - &rmm_cu12 rmm-cu12==24.12.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - &rmm_cu11 rmm-cu11==24.10.*,>=0.0.0a0 + - &rmm_cu11 rmm-cu11==24.12.*,>=0.0.0a0 - {matrix: null, packages: [*rmm_unsuffixed] } checks: common: @@ -255,25 +228,6 @@ dependencies: packages: - clang==16.0.6 - clang-tools=16.0.6 - nn_bench: - common: - - output_types: [conda, pyproject, requirements] - packages: - - hnswlib=0.7.0 - - nlohmann_json>=3.11.2 - - glog>=0.6.0 - - h5py>=3.8.0 - - benchmark>=1.8.2 - - openblas - - *rmm_unsuffixed - nn_bench_python: - common: - - output_types: [conda] - packages: - - matplotlib - - pandas - - pyyaml - - pandas cuda_version: specific: - output_types: conda @@ -514,14 +468,14 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - dask-cuda==24.10.*,>=0.0.0a0 + - dask-cuda==24.12.*,>=0.0.0a0 - joblib>=0.11 - numba>=0.57 - - rapids-dask-dependency==24.10.*,>=0.0.0a0 + - rapids-dask-dependency==24.12.*,>=0.0.0a0 - output_types: conda packages: - - &pylibraft_unsuffixed pylibraft==24.10.*,>=0.0.0a0 - - &ucx_py_unsuffixed ucx-py==0.40.*,>=0.0.0a0 + - &pylibraft_unsuffixed pylibraft==24.12.*,>=0.0.0a0 + - &ucx_py_unsuffixed ucx-py==0.41.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -535,14 +489,14 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - &pylibraft_cu12 pylibraft-cu12==24.10.*,>=0.0.0a0 - - &ucx_py_cu12 ucx-py-cu12==0.40.*,>=0.0.0a0 + - &pylibraft_cu12 pylibraft-cu12==24.12.*,>=0.0.0a0 + - &ucx_py_cu12 ucx-py-cu12==0.41.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - &pylibraft_cu11 pylibraft-cu11==24.10.*,>=0.0.0a0 - - &ucx_py_cu11 ucx-py-cu11==0.40.*,>=0.0.0a0 + - &pylibraft_cu11 pylibraft-cu11==24.12.*,>=0.0.0a0 + - &ucx_py_cu11 ucx-py-cu11==0.41.*,>=0.0.0a0 - {matrix: null, packages: [*pylibraft_unsuffixed, *ucx_py_unsuffixed]} test_python_common: common: @@ -562,7 +516,7 @@ dependencies: packages: # UCXX is not currently a hard-dependency thus only installed during tests, # this will change in the future. - - &distributed_ucxx_unsuffixed distributed-ucxx==0.40.*,>=0.0.0a0 + - &distributed_ucxx_unsuffixed distributed-ucxx==0.41.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -575,12 +529,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - distributed-ucxx-cu12==0.40.*,>=0.0.0a0 + - distributed-ucxx-cu12==0.41.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - distributed-ucxx-cu11==0.40.*,>=0.0.0a0 + - distributed-ucxx-cu11==0.41.*,>=0.0.0a0 - {matrix: null, packages: [*distributed_ucxx_unsuffixed]} depends_on_ucx_build: common: diff --git a/docs/source/ann_benchmarks_build.md b/docs/source/ann_benchmarks_build.md deleted file mode 100644 index 56af8e555c..0000000000 --- a/docs/source/ann_benchmarks_build.md +++ /dev/null @@ -1,51 +0,0 @@ -### Dependencies - -CUDA 11 and a GPU with Pascal architecture or later are required to run the benchmarks. - -Please refer to the [installation docs](https://docs.rapids.ai/api/raft/stable/build.html#cuda-gpu-requirements) for the base requirements to build RAFT. - -In addition to the base requirements for building RAFT, additional dependencies needed to build the ANN benchmarks include: -1. FAISS GPU >= 1.7.1 -2. Google Logging (GLog) -3. H5Py -4. HNSWLib -5. nlohmann_json -6. GGNN - -[rapids-cmake](https://github.com/rapidsai/rapids-cmake) is used to build the ANN benchmarks so the code for dependencies not already supplied in the CUDA toolkit will be downloaded and built automatically. - -The easiest (and most reproducible) way to install the dependencies needed to build the ANN benchmarks is to use the conda environment file located in the `conda/environments` directory of the RAFT repository. The following command will use `mamba` (which is preferred over `conda`) to build and activate a new environment for compiling the benchmarks: - -```bash -mamba env create --name raft_ann_benchmarks -f conda/environments/bench_ann_cuda-118_arch-x86_64.yaml -conda activate raft_ann_benchmarks -``` - -The above conda environment will also reduce the compile times as dependencies like FAISS will already be installed and not need to be compiled with `rapids-cmake`. - -### Compiling the Benchmarks - -After the needed dependencies are satisfied, the easiest way to compile ANN benchmarks is through the `build.sh` script in the root of the RAFT source code repository. The following will build the executables for all the support algorithms: -```bash -./build.sh bench-ann -``` - -You can limit the algorithms that are built by providing a semicolon-delimited list of executable names (each algorithm is suffixed with `_ANN_BENCH`): -```bash -./build.sh bench-ann -n --limit-bench-ann=HNSWLIB_ANN_BENCH;RAFT_IVF_PQ_ANN_BENCH -``` - -Available targets to use with `--limit-bench-ann` are: -- FAISS_GPU_IVF_FLAT_ANN_BENCH -- FAISS_GPU_IVF_PQ_ANN_BENCH -- FAISS_CPU_IVF_FLAT_ANN_BENCH -- FAISS_CPU_IVF_PQ_ANN_BENCH -- FAISS_GPU_FLAT_ANN_BENCH -- FAISS_CPU_FLAT_ANN_BENCH -- GGNN_ANN_BENCH -- HNSWLIB_ANN_BENCH -- RAFT_CAGRA_ANN_BENCH -- RAFT_IVF_PQ_ANN_BENCH -- RAFT_IVF_FLAT_ANN_BENCH - -By default, the `*_ANN_BENCH` executables program infer the dataset's datatype from the filename's extension. For example, an extension of `fbin` uses a `float` datatype, `f16bin` uses a `float16` datatype, extension of `i8bin` uses `int8_t` datatype, and `u8bin` uses `uint8_t` type. Currently, only `float`, `float16`, int8_t`, and `unit8_t` are supported. \ No newline at end of file diff --git a/docs/source/ann_benchmarks_dataset.md b/docs/source/ann_benchmarks_dataset.md deleted file mode 100644 index 26c1559504..0000000000 --- a/docs/source/ann_benchmarks_dataset.md +++ /dev/null @@ -1,63 +0,0 @@ -# ANN Benchmarks Datasets - -A dataset usually has 4 binary files containing database vectors, query vectors, ground truth neighbors and their corresponding distances. For example, Glove-100 dataset has files `base.fbin` (database vectors), `query.fbin` (query vectors), `groundtruth.neighbors.ibin` (ground truth neighbors), and `groundtruth.distances.fbin` (ground truth distances). The first two files are for index building and searching, while the other two are associated with a particular distance and are used for evaluation. - -The file suffixes `.fbin`, `.f16bin`, `.ibin`, `.u8bin`, and `.i8bin` denote that the data type of vectors stored in the file are `float32`, `float16`(a.k.a `half`), `int`, `uint8`, and `int8`, respectively. -These binary files are little-endian and the format is: the first 8 bytes are `num_vectors` (`uint32_t`) and `num_dimensions` (`uint32_t`), and the following `num_vectors * num_dimensions * sizeof(type)` bytes are vectors stored in row-major order. - -Some implementation can take `float16` database and query vectors as inputs and will have better performance. Use `script/fbin_to_f16bin.py` to transform dataset from `float32` to `float16` type. - -Commonly used datasets can be downloaded from two websites: -1. Million-scale datasets can be found at the [Data sets](https://github.com/erikbern/ann-benchmarks#data-sets) section of [`ann-benchmarks`](https://github.com/erikbern/ann-benchmarks). - - However, these datasets are in HDF5 format. Use `cpp/bench/ann/scripts/hdf5_to_fbin.py` to transform the format. A few Python packages are required to run it: - ```bash - pip3 install numpy h5py - ``` - The usage of this script is: - ```bash - $ cpp/bench/ann/scripts/hdf5_to_fbin.py - usage: scripts/hdf5_to_fbin.py [-n] .hdf5 - -n: normalize base/query set - outputs: .base.fbin - .query.fbin - .groundtruth.neighbors.ibin - .groundtruth.distances.fbin - ``` - So for an input `.hdf5` file, four output binary files will be produced. See previous section for an example of prepossessing GloVe dataset. - - Most datasets provided by `ann-benchmarks` use `Angular` or `Euclidean` distance. `Angular` denotes cosine distance. However, computing cosine distance reduces to computing inner product by normalizing vectors beforehand. In practice, we can always do the normalization to decrease computation cost, so it's better to measure the performance of inner product rather than cosine distance. The `-n` option of `hdf5_to_fbin.py` can be used to normalize the dataset. - -2. Billion-scale datasets can be found at [`big-ann-benchmarks`](http://big-ann-benchmarks.com). The ground truth file contains both neighbors and distances, thus should be split. A script is provided for this: - ```bash - $ cpp/bench/ann/scripts/split_groundtruth.pl - usage: script/split_groundtruth.pl input output_prefix - ``` - Take Deep-1B dataset as an example: - ```bash - pushd - cd cpp/bench/ann - mkdir -p data/deep-1B && cd data/deep-1B - # download manually "Ground Truth" file of "Yandex DEEP" - # suppose the file name is deep_new_groundtruth.public.10K.bin - ../../scripts/split_groundtruth.pl deep_new_groundtruth.public.10K.bin groundtruth - # two files 'groundtruth.neighbors.ibin' and 'groundtruth.distances.fbin' should be produced - popd - ``` - Besides ground truth files for the whole billion-scale datasets, this site also provides ground truth files for the first 10M or 100M vectors of the base sets. This mean we can use these billion-scale datasets as million-scale datasets. To facilitate this, an optional parameter `subset_size` for dataset can be used. See the next step for further explanation. - -## Generate ground truth - -If you have a dataset, but no corresponding ground truth file, then you can generate ground trunth using the `generate_groundtruth` utility. Example usage: - -```bash -# With existing query file -python -m raft_ann_bench.generate_groundtruth --dataset /dataset/base.fbin --output=groundtruth_dir --queries=/dataset/query.public.10K.fbin - -# With randomly generated queries -python -m raft_ann_bench.generate_groundtruth --dataset /dataset/base.fbin --output=groundtruth_dir --queries=random --n_queries=10000 - -# Using only a subset of the dataset. Define queries by randomly -# selecting vectors from the (subset of the) dataset. -python -m raft_ann_bench.generate_groundtruth --dataset /dataset/base.fbin --nrows=2000000 --output=groundtruth_dir --queries=random-choice --n_queries=10000 -``` \ No newline at end of file diff --git a/docs/source/ann_benchmarks_low_level.md b/docs/source/ann_benchmarks_low_level.md deleted file mode 100644 index 7ba13dec8d..0000000000 --- a/docs/source/ann_benchmarks_low_level.md +++ /dev/null @@ -1,219 +0,0 @@ -### Low-level Scripts and Executables -#### End-to-end Example -An end-to-end example (run from the RAFT source code root directory): -```bash -# (0) get raft sources -git clone https://github.com/rapidsai/raft.git -cd raft - -# (1) prepare a dataset -export PYTHONPATH=python/raft-ann-bench/src:$PYTHONPATH -python -m raft_ann_bench.get_dataset --dataset glove-100-angular --normalize - -# option --normalize is used here to normalize vectors so cosine distance is converted -# to inner product; don't use -n for l2 distance - -# (2) build index -$CONDA_PREFIX/bin/ann/RAFT_IVF_FLAT_ANN_BENCH \ - --data_prefix=datasets \ - --build \ - --benchmark_filter="raft_ivf_flat\..*" \ - python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-inner.json - -# (3) search -$CONDA_PREFIX/bin/ann/RAFT_IVF_FLAT_ANN_BENCH\ - --data_prefix=datasets \ - --benchmark_min_time=2s \ - --benchmark_out=ivf_flat_search.csv \ - --benchmark_out_format=csv \ - --benchmark_counters_tabular \ - --search \ - --benchmark_filter="raft_ivf_flat\..*" \ - python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-inner.json - - -# optional step: plot QPS-Recall figure using data in ivf_flat_search.csv with your favorite tool -``` - -##### Step 1: Prepare Dataset -Note: the preferred way to download and process smaller (million scale) datasets is to use the `get_dataset` script as demonstrated in the example above. - -A dataset usually has 4 binary files containing database vectors, query vectors, ground truth neighbors and their corresponding distances. For example, Glove-100 dataset has files `base.fbin` (database vectors), `query.fbin` (query vectors), `groundtruth.neighbors.ibin` (ground truth neighbors), and `groundtruth.distances.fbin` (ground truth distances). The first two files are for index building and searching, while the other two are associated with a particular distance and are used for evaluation. - -The file suffixes `.fbin`, `.f16bin`, `.ibin`, `.u8bin`, and `.i8bin` denote that the data type of vectors stored in the file are `float32`, `float16`(a.k.a `half`), `int`, `uint8`, and `int8`, respectively. -These binary files are little-endian and the format is: the first 8 bytes are `num_vectors` (`uint32_t`) and `num_dimensions` (`uint32_t`), and the following `num_vectors * num_dimensions * sizeof(type)` bytes are vectors stored in row-major order. - -Some implementation can take `float16` database and query vectors as inputs and will have better performance. Use `python/raft-ann-bench/src/raft_ann_bench/get_dataset/fbin_to_f16bin.py` to transform dataset from `float32` to `float16` type. - -Commonly used datasets can be downloaded from two websites: -1. Million-scale datasets can be found at the [Data sets](https://github.com/erikbern/ann-benchmarks#data-sets) section of [`ann-benchmarks`](https://github.com/erikbern/ann-benchmarks). - - However, these datasets are in HDF5 format. Use `python/raft-ann-bench/src/raft_ann_bench/get_dataset/fbin_to_f16bin.py/hdf5_to_fbin.py` to transform the format. A few Python packages are required to run it: - ```bash - pip3 install numpy h5py - ``` - The usage of this script is: - ```bash - $ cpp/bench/ann/scripts/hdf5_to_fbin.py - usage: scripts/hdf5_to_fbin.py [-n] .hdf5 - -n: normalize base/query set - outputs: .base.fbin - .query.fbin - .groundtruth.neighbors.ibin - .groundtruth.distances.fbin - ``` - So for an input `.hdf5` file, four output binary files will be produced. See previous section for an example of prepossessing GloVe dataset. - - Most datasets provided by `ann-benchmarks` use `Angular` or `Euclidean` distance. `Angular` denotes cosine distance. However, computing cosine distance reduces to computing inner product by normalizing vectors beforehand. In practice, we can always do the normalization to decrease computation cost, so it's better to measure the performance of inner product rather than cosine distance. The `-n` option of `hdf5_to_fbin.py` can be used to normalize the dataset. - -2. Billion-scale datasets can be found at [`big-ann-benchmarks`](http://big-ann-benchmarks.com). The ground truth file contains both neighbors and distances, thus should be split. A script is provided for this: - ```bash - $ python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/split_groundtruth.pl - usage: split_groundtruth.pl input output_prefix - ``` - Take Deep-1B dataset as an example: - ```bash - pushd - cd cpp/bench/ann - mkdir -p data/deep-1B && cd data/deep-1B - # download manually "Ground Truth" file of "Yandex DEEP" - # suppose the file name is deep_new_groundtruth.public.10K.bin - /path/to/raft/python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/split_groundtruth.pl deep_new_groundtruth.public.10K.bin groundtruth - # two files 'groundtruth.neighbors.ibin' and 'groundtruth.distances.fbin' should be produced - popd - ``` - Besides ground truth files for the whole billion-scale datasets, this site also provides ground truth files for the first 10M or 100M vectors of the base sets. This mean we can use these billion-scale datasets as million-scale datasets. To facilitate this, an optional parameter `subset_size` for dataset can be used. See the next step for further explanation. - - -##### Step 2: Build Index -An index is a data structure to facilitate searching. Different algorithms may use different data structures for their index. We can use `RAFT_IVF_FLAT_ANN_BENCH --build` to build an index and save it to disk. - -To run a benchmark executable, like `RAFT_IVF_FLAT_ANN_BENCH`, a JSON configuration file is required. Refer to [`cpp/bench/ann/conf/glove-100-inner.json`](../../cpp/cpp/bench/ann/conf/glove-100-inner.json) as an example. Configuration file has 3 sections: -* `dataset` section specifies the name and files of a dataset, and also the distance in use. Since the `*_ANN_BENCH` programs are for index building and searching, only `base_file` for database vectors and `query_file` for query vectors are needed. Ground truth files are for evaluation thus not needed. - - To use only a subset of the base dataset, an optional parameter `subset_size` can be specified. It means using only the first `subset_size` vectors of `base_file` as the base dataset. -* `search_basic_param` section specifies basic parameters for searching: - - `k` is the "k" in "k-nn", that is, the number of neighbors (or results) we want from the searching. -* `index` section specifies an array of configurations for index building and searching: - - `build_param` and `search_params` are parameters for building and searching, respectively. `search_params` is an array since we will search with different parameters to get different recall values. - - `file` is the file name of index. Building will save built index to this file, while searching will load this file. - - if `refine_ratio` is specified, refinement, as a post-processing step of search, will be done. It's for algorithms that compress vectors. For example, if `"refine_ratio" : 2` is set, 2`k` results are first computed, then exact distances of them are computed using original uncompressed vectors, and finally top `k` results among them are kept. - - -The usage of `*_ANN_BENCH` can be found by running `*_ANN_BENCH --help` on one of the executables: -```bash -$ ./cpp/build/*_ANN_BENCH --help -benchmark [--benchmark_list_tests={true|false}] - [--benchmark_filter=] - [--benchmark_min_time=`x` OR `s` ] - [--benchmark_min_warmup_time=] - [--benchmark_repetitions=] - [--benchmark_enable_random_interleaving={true|false}] - [--benchmark_report_aggregates_only={true|false}] - [--benchmark_display_aggregates_only={true|false}] - [--benchmark_format=] - [--benchmark_out=] - [--benchmark_out_format=] - [--benchmark_color={auto|true|false}] - [--benchmark_counters_tabular={true|false}] - [--benchmark_context==,...] - [--benchmark_time_unit={ns|us|ms|s}] - [--v=] - [--build|--search] - [--overwrite] - [--data_prefix=] - .json - -Note the non-standard benchmark parameters: - --build: build mode, will build index - --search: search mode, will search using the built index - one and only one of --build and --search should be specified - --overwrite: force overwriting existing index files - --data_prefix=: prepend to dataset file paths specified in the .json. - --override_kv=: override a build/search key one or more times multiplying the number of configurations; you can use this parameter multiple times to get the Cartesian product of benchmark configs. -``` -* `--build`: build index. -* `--search`: do the searching with built index. -* `--overwrite`: by default, the building mode skips building an index if it find out it already exists. This is useful when adding more configurations to the config; only new indices are build without the need to specify an elaborate filtering regex. By supplying `overwrite` flag, you disable this behavior; all indices are build regardless whether they are already stored on disk. -* `--data_prefix`: prepend an arbitrary path to the data file paths. By default, it is equal to `data`. Note, this does not apply to index file paths. -* `--override_kv`: override a build/search key one or more times multiplying the number of configurations. - -In addition to these ANN-specific flags, you can use all of the standard google benchmark flags. Some of the useful flags: -* `--benchmark_filter`: specify subset of benchmarks to run -* `--benchmark_out`, `--benchmark_out_format`: store the output to a file -* `--benchmark_list_tests`: check the available configurations -* `--benchmark_min_time`: specify the minimum duration or number of iterations per case to improve accuracy of the benchmarks. - -Refer to the google benchmark [user guide](https://github.com/google/benchmark/blob/main/docs/user_guide.md#command-line) for more information about the command-line usage. - -##### Step 3: Searching -Use the `--search` flag on any of the `*_ANN_BENCH` executables. Other options are the same as in step 2. - -## Adding a new ANN algorithm -Implementation of a new algorithm should be a class that inherits `class ANN` (defined in `cpp/bench/ann/src/ann.h`) and implements all the pure virtual functions. - -In addition, it should define two `struct`s for building and searching parameters. The searching parameter class should inherit `struct ANN::AnnSearchParam`. Take `class HnswLib` as an example, its definition is: -```c++ -template -class HnswLib : public ANN { -public: - struct BuildParam { - int M; - int ef_construction; - int num_threads; - }; - - using typename ANN::AnnSearchParam; - struct SearchParam : public AnnSearchParam { - int ef; - int num_threads; - }; - - // ... -}; -``` - -The benchmark program uses JSON configuration file. To add the new algorithm to the benchmark, need be able to specify `build_param`, whose value is a JSON object, and `search_params`, whose value is an array of JSON objects, for this algorithm in configuration file. Still take the configuration for `HnswLib` as an example: -```json -{ - "name" : "...", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "/path/to/file", - "search_params" : [ - {"ef":10, "numThreads":1}, - {"ef":20, "numThreads":1}, - {"ef":40, "numThreads":1} - ] -}, -``` - -How to interpret these JSON objects is totally left to the implementation and should be specified in `cpp/bench/ann/src/factory.cuh`: -1. First, add two functions for parsing JSON object to `struct BuildParam` and `struct SearchParam`, respectively: - ```c++ - template - void parse_build_param(const nlohmann::json& conf, - typename cuann::HnswLib::BuildParam& param) { - param.ef_construction = conf.at("efConstruction"); - param.M = conf.at("M"); - if (conf.contains("numThreads")) { - param.num_threads = conf.at("numThreads"); - } - } - - template - void parse_search_param(const nlohmann::json& conf, - typename cuann::HnswLib::SearchParam& param) { - param.ef = conf.at("ef"); - if (conf.contains("numThreads")) { - param.num_threads = conf.at("numThreads"); - } - } - ``` - -2. Next, add corresponding `if` case to functions `create_algo()` and `create_search_param()` by calling parsing functions. The string literal in `if` condition statement must be the same as the value of `algo` in configuration file. For example, - ```c++ - // JSON configuration file contains a line like: "algo" : "hnswlib" - if (algo == "hnswlib") { - // ... - } - ``` diff --git a/docs/source/ann_benchmarks_param_tuning.md b/docs/source/ann_benchmarks_param_tuning.md deleted file mode 100644 index afb4ed18ea..0000000000 --- a/docs/source/ann_benchmarks_param_tuning.md +++ /dev/null @@ -1,178 +0,0 @@ -# ANN Benchmarks Parameter Tuning Guide - -This guide outlines the various parameter settings that can be specified in [RAFT ANN Benchmark](raft_ann_benchmarks.md) json configuration files and explains the impact they have on corresponding algorithms to help inform their settings for benchmarking across desired levels of recall. - - -## RAFT Indexes - -### `raft_brute_force` - -Use RAFT brute-force index for exact search. Brute-force has no further build or search parameters. - -### `raft_ivf_flat` - -IVF-flat uses an inverted-file index, which partitions the vectors into a series of clusters, or lists, storing them in an interleaved format which is optimized for fast distance computation. The searching of an IVF-flat index reduces the total vectors in the index to those within some user-specified nearest clusters called probes. - -IVF-flat is a simple algorithm which won't save any space, but it provides competitive search times even at higher levels of recall. - -| Parameter | Type | Required | Data Type | Default | Description | -|----------------------|------------------|----------|----------------------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `nlist` | `build` | Y | Positive Integer >0 | | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `niter` | `build` | N | Positive Integer >0 | 20 | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `ratio` | `build` | N | Positive Integer >0 | 2 | `1/ratio` is the number of training points which should be used to train the clusters. | -| `dataset_memory_type` | `build` | N | ["device", "host", "mmap"] | "mmap" | What memory type should the dataset reside? | -| `query_memory_type` | `search` | N | ["device", "host", "mmap"] | "device | What memory type should the queries reside? | -| `nprobe` | `search` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | - - -### `raft_ivf_pq` - -IVF-pq is an inverted-file index, which partitions the vectors into a series of clusters, or lists, in a similar way to IVF-flat above. The difference is that IVF-PQ uses product quantization to also compress the vectors, giving the index a smaller memory footprint. Unfortunately, higher levels of compression can also shrink recall, which a refinement step can improve when the original vectors are still available. - -| Parameter | Type | Required | Data Type | Default | Description | -|------------------------|----------------|---|----------------------------------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `nlist` | `build` | Y | Positive Integer >0 | | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `niter` | `build` | N | Positive Integer >0 | 20 | Number of k-means iterations to use when training the clusters. | -| `ratio` | `build` | N | Positive Integer >0 | 2 | `1/ratio` is the number of training points which should be used to train the clusters. | -| `pq_dim` | `build` | N | Positive Integer. Multiple of 8. | 0 | Dimensionality of the vector after product quantization. When 0, a heuristic is used to select this value. `pq_dim` * `pq_bits` must be a multiple of 8. | -| `pq_bits` | `build` | N | Positive Integer. [4-8] | 8 | Bit length of the vector element after quantization. | -| `codebook_kind` | `build` | N | ["cluster", "subspace"] | "subspace" | Type of codebook. See the [API docs](https://docs.rapids.ai/api/raft/nightly/cpp_api/neighbors_ivf_pq/#_CPPv412codebook_gen) for more detail | -| `dataset_memory_type` | `build` | N | ["device", "host", "mmap"] | "host" | What memory type should the dataset reside? | -| `query_memory_type` | `search` | N | ["device", "host", "mmap"] | "device | What memory type should the queries reside? | -| `nprobe` | `search` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | -| `internalDistanceDtype` | `search` | N | [`float`, `half`] | `half` | The precision to use for the distance computations. Lower precision can increase performance at the cost of accuracy. | -| `smemLutDtype` | `search` | N | [`float`, `half`, `fp8`] | `half` | The precision to use for the lookup table in shared memory. Lower precision can increase performance at the cost of accuracy. | -| `refine_ratio` | `search` | N| Positive Number >=1 | 1 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | - - -### `raft_cagra` -CAGRA uses a graph-based index, which creates an intermediate, approximate kNN graph using IVF-PQ and then further refining and optimizing to create a final kNN graph. This kNN graph is used by CAGRA as an index for search. - -| Parameter | Type | Required | Data Type | Default | Description | -|-----------------------------|----------------|----------|----------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `graph_degree` | `build` | N | Positive Integer >0 | 64 | Degree of the final kNN graph index. | -| `intermediate_graph_degree` | `build` | N | Positive Integer >0 | 128 | Degree of the intermediate kNN graph. | -| `graph_build_algo` | `build` | N | ["IVF_PQ", "NN_DESCENT"] | "IVF_PQ" | Algorithm to use for search | -| `dataset_memory_type` | `build` | N | ["device", "host", "mmap"] | "mmap" | What memory type should the dataset reside while constructing the index? | -| `query_memory_type` | `search` | N | ["device", "host", "mmap"] | "device | What memory type should the queries reside? | -| `itopk` | `search_wdith` | N | Positive Integer >0 | 64 | Number of intermediate search results retained during the search. Higher values improve search accuracy at the cost of speed. | -| `search_width` | `search` | N | Positive Integer >0 | 1 | Number of graph nodes to select as the starting point for the search in each iteration. | -| `max_iterations` | `search` | N | Integer >=0 | 0 | Upper limit of search iterations. Auto select when 0. | -| `algo` | `search` | N | string | "auto" | Algorithm to use for search. Possible values: {"auto", "single_cta", "multi_cta", "multi_kernel"} | -| `graph_memory_type` | `search` | N | string | "device" | Memory type to store gaph. Must be one of {"device", "host_pinned", "host_huge_page"}. | -| `internal_dataset_memory_type` | `search` | N | string | "device" | Memory type to store dataset in the index. Must be one of {"device", "host_pinned", "host_huge_page"}. | - -The `graph_memory_type` or `internal_dataset_memory_type` options can be useful for large datasets that do not fit the device memory. Setting `internal_dataset_memory_type` other than `device` has negative impact on search speed. Using `host_huge_page` option is only supported on systems with Heterogeneous Memory Management or on platforms that natively support GPU access to system allocated memory, for example Grace Hopper. - -To fine tune CAGRA index building we can customize IVF-PQ index builder options using the following settings. These take effect only if `graph_build_algo == "IVF_PQ"`. It is recommended to experiment using a separate IVF-PQ index to find the config that gives the largest QPS for large batch. Recall does not need to be very high, since CAGRA further optimizes the kNN neighbor graph. Some of the default values are derived from the dataset size which is assumed to be [n_vecs, dim]. - -| Parameter | Type | Required | Data Type | Default | Description | -|------------------------|----------------|---|----------------------------------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `ivf_pq_build_nlist` | `build` | N | Positive Integer >0 | n_vecs / 2500 | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `ivf_pq_build_niter` | `build` | N | Positive Integer >0 | 25 | Number of k-means iterations to use when training the clusters. | -| `ivf_pq_build_ratio` | `build` | N | Positive Integer >0 | 10 | `1/ratio` is the number of training points which should be used to train the clusters. | -| `ivf_pq_build_pq_dim` | `build` | N | Positive Integer. Multiple of 8. | dim/2 rounded up to 8 | Dimensionality of the vector after product quantization. When 0, a heuristic is used to select this value. `pq_dim` * `pq_bits` must be a multiple of 8. | -| `ivf_pq_build_pq_bits` | `build` | N | Positive Integer. [4-8] | 8 | Bit length of the vector element after quantization. | -| `ivf_pq_build_codebook_kind` | `build` | N | ["cluster", "subspace"] | "subspace" | Type of codebook. See the [API docs](https://docs.rapids.ai/api/raft/nightly/cpp_api/neighbors_ivf_pq/#_CPPv412codebook_gen) for more detail | -| `ivf_pq_search_nprobe` | `build` | N | Positive Integer >0 | min(2*dim, nlist) | The closest number of clusters to search for each query vector. | -| `ivf_pq_search_internalDistanceDtype` | `build` | N | [`float`, `half`] | `fp8` | The precision to use for the distance computations. Lower precision can increase performance at the cost of accuracy. | -| `ivf_pq_search_smemLutDtype` | `build` | N | [`float`, `half`, `fp8`] | `half` | The precision to use for the lookup table in shared memory. Lower precision can increase performance at the cost of accuracy. | -| `ivf_pq_search_refine_ratio` | `build` | N| Positive Number >=1 | 2 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | - -Alternatively, if `graph_build_algo == "NN_DESCENT"`, then we can customize the following parameters - -| Parameter | Type | Required | Data Type | Default | Description | -|-----------------------------|----------------|----------|----------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `nn_descent_niter` | `build` | N | Positive Integer>0 | 20 | Number of NN Descent iterations. | -| `nn_descent_intermediate_graph_degree` | `build` | N | Positive Integer>0 | `intermediate_graph_degree` * 1.5 | Intermadiate graph degree during NN descent iterations | -| `nn_descent_max_iterations` | `build` | N | Positive Integer>0 | 20 | Alias for `nn_descent_niter` | -| `nn_descent_termination_threshold` | `build` | N | Positive float>0 | 0.0001 | Termination threshold for NN descent. | - -### `raft_cagra_hnswlib` -This is a benchmark that enables interoperability between `CAGRA` built `HNSW` search. It uses the `CAGRA` built graph as the base layer of an `hnswlib` index to search queries only within the base layer (this is enabled with a simple patch to `hnswlib`). - -`build` : Same as `build` of [CAGRA](#raft-cagra) - -`search` : Same as `search` of [hnswlib](#hnswlib) - -## FAISS Indexes - -### `faiss_gpu_flat` - -Use FAISS flat index on the GPU, which performs an exact search using brute-force and doesn't have any further build or search parameters. - -### `faiss_gpu_ivf_flat` - -IVF-flat uses an inverted-file index, which partitions the vectors into a series of clusters, or lists, storing them in an interleaved format which is optimized for fast distance computation. The searching of an IVF-flat index reduces the total vectors in the index to those within some user-specified nearest clusters called probes. - -IVF-flat is a simple algorithm which won't save any space, but it provides competitive search times even at higher levels of recall. - -| Parameter | Type | Required | Data Type | Default | Description | -|-----------|----------------|----------|---------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `nlists` | `build` | Y | Positive Integer >0 | | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `ratio` | `build` | N | Positive Integer >0 | 2 | `1/ratio` is the number of training points which should be used to train the clusters. | -| `nprobe` | `search` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | - -### `faiss_gpu_ivf_pq` - -IVF-pq is an inverted-file index, which partitions the vectors into a series of clusters, or lists, in a similar way to IVF-flat above. The difference is that IVF-PQ uses product quantization to also compress the vectors, giving the index a smaller memory footprint. Unfortunately, higher levels of compression can also shrink recall, which a refinement step can improve when the original vectors are still available. - -| Parameter | Type | Required | Data Type | Default | Description | -|------------------|----------------|----------|----------------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `nlist` | `build` | Y | Positive Integer >0 | | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `ratio` | `build` | N | Positive Integer >0 | 2 | `1/ratio` is the number of training points which should be used to train the clusters. | -| `M_ratio` | `build` | Y | Positive Integer Power of 2 [8-64] | | Ratio of numbeer of chunks or subquantizers for each vector. Computed by `dims` / `M_ratio` | -| `usePrecomputed` | `build` | N | Boolean. Default=`false` | `false` | Use pre-computed lookup tables to speed up search at the cost of increased memory usage. | -| `useFloat16` | `build` | N | Boolean. Default=`false` | `false` | Use half-precision floats for clustering step. | -| `nprobe` | `search` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | -| `refine_ratio` | `search` | N| Positive Number >=1 | 1 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | - -### `faiss_cpu_flat` - -Use FAISS flat index on the CPU, which performs an exact search using brute-force and doesn't have any further build or search parameters. - - -| Parameter | Type | Required | Data Type | Default | Description | -|-----------|----------------|----------|---------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `numThreads` | `search` | N | Positive Integer >0 | 1 | Number of threads to use for queries. | - -### `faiss_cpu_ivf_flat` - -Use FAISS IVF-Flat index on CPU - -| Parameter | Type | Required | Data Type | Default | Description | -|----------|----------------|----------|---------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `nlist` | `build` | Y | Positive Integer >0 | | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `ratio` | `build` | N | Positive Integer >0 | 2 | `1/ratio` is the number of training points which should be used to train the clusters. | -| `nprobe` | `search` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | -| `numThreads` | `search` | N | Positive Integer >0 | 1 | Number of threads to use for queries. | - -### `faiss_cpu_ivf_pq` - -Use FAISS IVF-PQ index on CPU - -| Parameter | Type | Required | Data Type | Default | Description | -|------------------|----------------|----------|------------------------------------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `nlist` | `build` | Y | Positive Integer >0 | | Number of clusters to partition the vectors into. Larger values will put less points into each cluster but this will impact index build time as more clusters need to be trained. | -| `ratio` | `build` | N | Positive Integer >0 | 2 | `1/ratio` is the number of training points which should be used to train the clusters. | -| `M` | `build` | Y | Positive Integer Power of 2 [8-64] | | Number of chunks or subquantizers for each vector. | -| `usePrecomputed` | `build` | N | Boolean. Default=`false` | `false` | Use pre-computed lookup tables to speed up search at the cost of increased memory usage. | -| `bitsPerCode` | `build` | N | Positive Integer [4-8] | 8 | Number of bits to use for each code. | -| `nprobe` | `search` | Y | Positive Integer >0 | | The closest number of clusters to search for each query vector. Larger values will improve recall but will search more points in the index. | -| `refine_ratio` | `search` | N| Positive Number >=1 | 1 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | -| `numThreads` | `search` | N | Positive Integer >0 | 1 | Number of threads to use for queries. | - - -## HNSW - -### `hnswlib` - -| Parameter | Type | Required | Data Type | Default | Description | -|------------------|-----------|----------|--------------------------------------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `efConstruction` | `build` | Y | Positive Integer >0 | | Controls index time and accuracy. Bigger values increase the index quality. At some point, increasing this will no longer improve the quality. | -| `M` | `build` | Y | Positive Integer often between 2-100 | | Number of bi-directional links create for every new element during construction. Higher values work for higher intrinsic dimensionality and/or high recall, low values can work for datasets with low intrinsic dimensionality and/or low recalls. Also affects the algorithm's memory consumption. | -| `numThreads` | `build` | N | Positive Integer >0 | 1 | Number of threads to use to build the index. | -| `ef` | `search` | Y | Positive Integer >0 | | Size of the dynamic list for the nearest neighbors used for search. Higher value leads to more accurate but slower search. Cannot be lower than `k`. | -| `numThreads` | `search` | N | Positive Integer >0 | 1 | Number of threads to use for queries. | - -Please refer to [HNSW algorithm parameters guide](https://github.com/nmslib/hnswlib/blob/master/ALGO_PARAMS.md) from `hnswlib` to learn more about these arguments. \ No newline at end of file diff --git a/docs/source/build.md b/docs/source/build.md index 4ba087e68d..5a0dbf7e11 100644 --- a/docs/source/build.md +++ b/docs/source/build.md @@ -1,6 +1,6 @@ # Installation -RAFT currently provides libraries for C++ and Python. The C++ libraries, including the header-only and optional shared library, can be installed with Conda. +RAFT currently provides libraries for C++ and Python. The C++ libraries, including the header-only and optional shared library, can be installed with Conda. Both the C++ and Python APIs require CMake to build from source. @@ -31,11 +31,8 @@ Both the C++ and Python APIs require CMake to build from source. The easiest way to install RAFT is through conda and several packages are provided. - `libraft-headers` C++ headers -- `libraft` (optional) C++ shared library containing pre-compiled template instantiations and runtime API. - `pylibraft` (optional) Python library - `raft-dask` (optional) Python library for deployment of multi-node multi-GPU algorithms that use the RAFT `raft::comms` abstraction layer in Dask clusters. -- `raft-ann-bench` (optional) Benchmarking tool for easily producing benchmarks that compare RAFT's vector search algorithms against other state-of-the-art implementations. -- `raft-ann-bench-cpu` (optional) Reproducible benchmarking tool similar to above, but doesn't require CUDA to be installed on the machine. Can be used to test in environments with competitive CPUs. Use the following command, depending on your CUDA version, to install all of the RAFT packages with conda (replace `rapidsai` with `rapidsai-nightly` to install more up-to-date but less stable nightly packages). `mamba` is preferred over the `conda` command. ```bash @@ -50,17 +47,15 @@ mamba install -c rapidsai -c conda-forge -c nvidia raft-dask pylibraft cuda-vers Note that the above commands will also install `libraft-headers` and `libraft`. -You can also install the conda packages individually using the `mamba` command above. For example, if you'd like to install RAFT's headers and pre-compiled shared library to use in your project: +You can also install the conda packages individually using the `mamba` command above. For example, if you'd like to install RAFT's headers to use in your project: ```bash # for CUDA 12.0 -mamba install -c rapidsai -c conda-forge -c nvidia libraft libraft-headers cuda-version=12.0 +mamba install -c rapidsai -c conda-forge -c nvidia libraft-headers cuda-version=12.0 ``` -If installing the C++ APIs Please see [using libraft](https://docs.rapids.ai/api/raft/nightly/using_libraft/) for more information on using the pre-compiled shared library. You can also refer to the [example C++ template project](https://github.com/rapidsai/raft/tree/branch-24.10/cpp/template) for a ready-to-go CMake configuration that you can drop into your project and build against installed RAFT development artifacts above. - ## Installing Python through Pip -`pylibraft` and `raft-dask` both have packages that can be [installed through pip](https://rapids.ai/pip.html#install). +`pylibraft` and `raft-dask` both have packages that can be [installed through pip](https://rapids.ai/pip.html#install). For CUDA 11 packages: ```bash @@ -74,20 +69,18 @@ pip install pylibraft-cu12 --extra-index-url=https://pypi.nvidia.com pip install raft-dask-cu12 --extra-index-url=https://pypi.nvidia.com ``` -These packages statically build RAFT's pre-compiled instantiations, so the C++ headers and pre-compiled shared library won't be readily available to use in your code. - ## Building C++ and Python from source ### CUDA/GPU Requirements - cmake 3.26.4+ - GCC 9.3+ (9.5.0+ recommended) -- CUDA Toolkit 11.2+ +- CUDA Toolkit 11.8+ - NVIDIA driver 450.80.02+ -- Pascal architecture or better (compute capability >= 6.0) +- Volta architecture or better (compute capability >= 7.0) ### Build Dependencies -In addition to the libraries included with cudatoolkit 11.0+, there are some other dependencies below for building RAFT from source. Many of the dependencies are optional and depend only on the primitives being used. All of these can be installed with cmake or [rapids-cpm](https://github.com/rapidsai/rapids-cmake#cpm) and many of them can be installed with [conda](https://anaconda.org). +In addition to the libraries included with cudatoolkit 11.8+, there are some other dependencies below for building RAFT from source. Many of the dependencies are optional and depend only on the primitives being used. All of these can be installed with cmake or [rapids-cpm](https://github.com/rapidsai/rapids-cmake#cpm) and many of them can be installed with [conda](https://anaconda.org). #### Required - [RMM](https://github.com/rapidsai/rmm) corresponding to RAFT version. @@ -110,9 +103,9 @@ mamba env create --name rapids_raft -f conda/environments/all_cuda-125_arch-x86_ mamba activate rapids_raft ``` -All of RAFT's C++ APIs can be used header-only and optional pre-compiled shared libraries provide some host-accessible runtime APIs and template instantiations to accelerate compile times. +All of RAFT's C++ APIs can be used header-only. -The process for building from source with CUDA 11 differs slightly in that your host system will also need to have CUDA toolkit installed which is greater than, or equal to, the version you install into you conda environment. Installing CUDA toolkit into your host system is necessary because `nvcc` is not provided with Conda's cudatoolkit dependencies for CUDA 11. The following example will install create and install dependencies for a CUDA 11.8 conda environment +The process for building from source with CUDA 11 differs slightly in that your host system will also need to have CUDA toolkit installed which is greater than, or equal to, the version you install into you conda environment. Installing CUDA toolkit into your host system is necessary because `nvcc` is not provided with Conda's cudatoolkit dependencies for CUDA 11. The following example will install create and install dependencies for a CUDA 11.8 conda environment: ```bash mamba env create --name rapids_raft -f conda/environments/all_cuda-118_arch-x86_64.yaml mamba activate rapids_raft @@ -124,7 +117,7 @@ The recommended way to build and install RAFT from source is to use the `build.s `build.sh` uses [rapids-cmake](https://github.com/rapidsai/rapids-cmake), which will automatically download any dependencies which are not already installed. It's important to note that while all the headers will be installed and available, some parts of the RAFT API depend on libraries like CUTLASS, which will need to be explicitly enabled in `build.sh`. -The following example will download the needed dependencies and install the RAFT headers into `$INSTALL_PREFIX/include/raft`. +The following example will download the needed dependencies and install the RAFT headers into `$INSTALL_PREFIX/include/raft`. ```bash ./build.sh libraft ``` @@ -140,7 +133,7 @@ Once installed, `libraft` headers (and dependencies which were downloaded and in ### C++ Shared Library (optional) -A shared library can be built for speeding up compile times. The shared library also contains a runtime API that allows you to invoke RAFT APIs directly from C++ source files (without `nvcc`). The shared library can also significantly improve re-compile times both while developing RAFT and using its APIs to develop applications. Pass the `--compile-lib` flag to `build.sh` to build the library: +A shared library must be built in order to build `pylibraft`. Pass the `--compile-lib` flag to `build.sh` to build the library: ```bash ./build.sh libraft --compile-lib ``` @@ -169,23 +162,17 @@ Compile the tests using the `tests` target in `build.sh`. ./build.sh libraft tests ``` -Test compile times can be improved significantly by using the optional shared libraries. If installed, they will be used automatically when building the tests but `--compile-libs` can be used to add additional compilation units and compile them with the tests. - -```bash -./build.sh libraft tests --compile-lib -``` - The tests are broken apart by algorithm category, so you will find several binaries in `cpp/build/` named `*_TEST`. For example, to run the distance tests: ```bash -./cpp/build/DISTANCE_TEST +./cpp/build/MATRIX_TEST ``` It can take sometime to compile all of the tests. You can build individual tests by providing a semicolon-separated list to the `--limit-tests` option in `build.sh`: ```bash -./build.sh libraft tests -n --limit-tests=NEIGHBORS_TEST;DISTANCE_TEST;MATRIX_TEST +./build.sh libraft tests -n --limit-tests=CORE_TEST;MATRIX_TEST ``` ### C++ Primitives Microbenchmarks @@ -198,11 +185,9 @@ The benchmarks are broken apart by algorithm category, so you will find several It can take sometime to compile all of the benchmarks. You can build individual benchmarks by providing a semicolon-separated list to the `--limit-bench-prims` option in `build.sh`: ```bash -./build.sh libraft bench-prims -n --limit-bench=NEIGHBORS_PRIMS_BENCH;DISTANCE_PRIMS_BENCH;LINALG_PRIMS_BENCH +./build.sh libraft bench-prims -n --limit-bench=NEIGHBORS_PRIMS_BENCH;MATRIX_PRIMS_BENCH;LINALG_PRIMS_BENCH ``` -In addition to microbenchmarks for individual primitives, RAFT contains a reproducible benchmarking tool for evaluating the performance of RAFT's vector search algorithms against the existing state-of-the-art. Please refer to the [RAFT ANN Benchmarks](https://docs.rapids.ai/api/raft/nightly/raft_ann_benchmarks/) guide for more information on this tool. - ### Python libraries The Python libraries can be built and installed using the `build.sh` script: @@ -242,7 +227,7 @@ The Python packages can also be uninstalled using the `build.sh` script: ### Using CMake directly -When building RAFT from source, the `build.sh` script offers a nice wrapper around the `cmake` commands to ease the burdens of manually configuring the various available cmake options. When more fine-grained control over the CMake configuration is desired, the `cmake` command can be invoked directly as the below example demonstrates. +When building RAFT from source, the `build.sh` script offers a nice wrapper around the `cmake` commands to ease the burdens of manually configuring the various available cmake options. When more fine-grained control over the CMake configuration is desired, the `cmake` command can be invoked directly as the below example demonstrates. The `CMAKE_INSTALL_PREFIX` installs RAFT into a specific location. The example below installs RAFT into the current Conda environment: ```bash @@ -259,7 +244,6 @@ RAFT's CMake has the following configurable flags available: |---------------------------------|----------------------| --- |------------------------------------------------------------------------------| | BUILD_TESTS | ON, OFF | ON | Compile Googletests | | BUILD_PRIMS_BENCH | ON, OFF | OFF | Compile benchmarks | -| BUILD_ANN_BENCH | ON, OFF | OFF | Compile end-to-end ANN benchmarks | | CUDA_ENABLE_KERNELINFO | ON, OFF | OFF | Enables `kernelinfo` in nvcc. This is useful for `compute-sanitizer` | | CUDA_ENABLE_LINEINFO | ON, OFF | OFF | Enable the -lineinfo option for nvcc | | CUDA_STATIC_RUNTIME | ON, OFF | OFF | Statically link the CUDA runtime | @@ -267,10 +251,10 @@ RAFT's CMake has the following configurable flags available: | DETECT_CONDA_ENV | ON, OFF | ON | Enable detection of conda environment for dependencies | | raft_FIND_COMPONENTS | compiled distributed | | Configures the optional components as a space-separated list | | RAFT_COMPILE_LIBRARY | ON, OFF | ON if either BUILD_TESTS or BUILD_PRIMS_BENCH is ON; otherwise OFF | Compiles all `libraft` shared libraries (these are required for Googletests) | -| RAFT_ENABLE_CUBLAS_DEPENDENCY | ON, OFF | ON | Link against cublas library in `raft::raft` | -| RAFT_ENABLE_CUSOLVER_DEPENDENCY | ON, OFF | ON | Link against cusolver library in `raft::raft` | -| RAFT_ENABLE_CUSPARSE_DEPENDENCY | ON, OFF | ON | Link against cusparse library in `raft::raft` | -| RAFT_ENABLE_CUSOLVER_DEPENDENCY | ON, OFF | ON | Link against curand library in `raft::raft` | +| RAFT_ENABLE_CUBLAS_DEPENDENCY | ON, OFF | ON | Link against cublas library in `raft::raft` | +| RAFT_ENABLE_CUSOLVER_DEPENDENCY | ON, OFF | ON | Link against cusolver library in `raft::raft` | +| RAFT_ENABLE_CUSPARSE_DEPENDENCY | ON, OFF | ON | Link against cusparse library in `raft::raft` | +| RAFT_ENABLE_CUSOLVER_DEPENDENCY | ON, OFF | ON | Link against curand library in `raft::raft` | | RAFT_NVTX | ON, OFF | OFF | Enable NVTX Markers | ### Build documentation @@ -306,8 +290,6 @@ PROPERTIES CXX_STANDARD 17 INTERFACE_POSITION_INDEPENDENT_CODE ON) ``` -The [C++ example template project](https://github.com/rapidsai/raft/tree/HEAD/cpp/template) provides an end-to-end buildable example of what a `CMakeLists.txt` that uses RAFT should look like. The items below point out some of the needed details. - #### CMake Targets The `raft::raft` CMake target is made available when including RAFT into your CMake project but additional CMake targets can be made available by adding to the `COMPONENTS` option in CMake's `find_package(raft)` (refer to [CMake docs](https://cmake.org/cmake/help/latest/command/find_package.html#basic-signature) to learn more). The components should be separated by spaces. The `raft::raft` target will always be available. Note that the `distributed` component also exports additional dependencies. @@ -316,4 +298,4 @@ The `raft::raft` CMake target is made available when including RAFT into your CM |-------------|---------------------|----------------------------------------------------------|----------------------------------------| | n/a | `raft::raft` | Full RAFT header library | CUDA toolkit, RMM, NVTX, CCCL, CUTLASS | | compiled | `raft::compiled` | Pre-compiled template instantiations and runtime library | raft::raft | -| distributed | `raft::distributed` | Dependencies for `raft::comms` APIs | raft::raft, UCX, NCCL \ No newline at end of file +| distributed | `raft::distributed` | Dependencies for `raft::comms` APIs | raft::raft, UCX, NCCL diff --git a/docs/source/cpp_api.rst b/docs/source/cpp_api.rst index e60ef4e697..74f706bf46 100644 --- a/docs/source/cpp_api.rst +++ b/docs/source/cpp_api.rst @@ -8,13 +8,10 @@ C++ API :maxdepth: 4 cpp_api/core.rst - cpp_api/cluster.rst - cpp_api/distance.rst cpp_api/linalg.rst cpp_api/matrix.rst cpp_api/mdspan.rst cpp_api/mnmg.rst - cpp_api/neighbors.rst cpp_api/random.rst cpp_api/solver.rst cpp_api/sparse.rst diff --git a/docs/source/cpp_api/cluster.rst b/docs/source/cpp_api/cluster.rst deleted file mode 100644 index b0485992b3..0000000000 --- a/docs/source/cpp_api/cluster.rst +++ /dev/null @@ -1,18 +0,0 @@ -Cluster -======= - -This page provides C++ API references for the publicly-exposed elements of the `raft/cluster` headers. RAFT provides -fundamental clustering algorithms which are, themselves, considered reusable building blocks for other algorithms. - -.. role:: py(code) - :language: c++ - :class: highlight - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - cluster_kmeans.rst - cluster_kmeans_balanced.rst - cluster_slhc.rst - cluster_spectral.rst \ No newline at end of file diff --git a/docs/source/cpp_api/cluster_kmeans.rst b/docs/source/cpp_api/cluster_kmeans.rst deleted file mode 100644 index fa040ddc18..0000000000 --- a/docs/source/cpp_api/cluster_kmeans.rst +++ /dev/null @@ -1,13 +0,0 @@ -K-Means -======= - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -.. doxygennamespace:: raft::cluster::kmeans - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/cluster_kmeans_balanced.rst b/docs/source/cpp_api/cluster_kmeans_balanced.rst deleted file mode 100644 index 5d07fcc1e3..0000000000 --- a/docs/source/cpp_api/cluster_kmeans_balanced.rst +++ /dev/null @@ -1,13 +0,0 @@ -K-Means -======= - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -.. doxygennamespace:: raft::cluster::kmeans_balanced - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/cluster_slhc.rst b/docs/source/cpp_api/cluster_slhc.rst deleted file mode 100644 index fc45ae699a..0000000000 --- a/docs/source/cpp_api/cluster_slhc.rst +++ /dev/null @@ -1,13 +0,0 @@ -Hierarchical Clustering -======================= - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -.. doxygennamespace:: raft::cluster::hierarchy - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/cluster_spectral.rst b/docs/source/cpp_api/cluster_spectral.rst deleted file mode 100644 index a71f431ab8..0000000000 --- a/docs/source/cpp_api/cluster_spectral.rst +++ /dev/null @@ -1,13 +0,0 @@ -Spectral Clustering -=================== - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -.. doxygennamespace:: raft::spectral - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/distance.rst b/docs/source/cpp_api/distance.rst deleted file mode 100644 index fd81295def..0000000000 --- a/docs/source/cpp_api/distance.rst +++ /dev/null @@ -1,28 +0,0 @@ -Distance -======== - -This page provides C++ class references for the publicly-exposed elements of the `raft/distance` package. RAFT's -distances have been highly optimized and support a wide assortment of different distance measures. - -.. role:: py(code) - :language: c++ - :class: highlight - -Distance Types --------------- - -``#include `` - -namespace *raft::distance* - -.. doxygenenum:: raft::distance::DistanceType - :project: RAFT - - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - distance_pairwise.rst - distance_1nn.rst - diff --git a/docs/source/cpp_api/distance_1nn.rst b/docs/source/cpp_api/distance_1nn.rst deleted file mode 100644 index 8c1c00d6c9..0000000000 --- a/docs/source/cpp_api/distance_1nn.rst +++ /dev/null @@ -1,24 +0,0 @@ -1-Nearest Neighbors -=================== - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::distance* - -.. doxygengroup:: fused_l2_nn - :project: RAFT - :members: - :content-only: - -``#include `` -namespace *raft::distance* - -.. doxygengroup:: masked_nn - :project: RAFT - :members: - :content-only: - diff --git a/docs/source/cpp_api/distance_pairwise.rst b/docs/source/cpp_api/distance_pairwise.rst deleted file mode 100644 index 2a9c9a92f5..0000000000 --- a/docs/source/cpp_api/distance_pairwise.rst +++ /dev/null @@ -1,17 +0,0 @@ -Pairwise Distance -================= - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::distance* - -.. doxygengroup:: distance_mdspan - :project: RAFT - :members: - :content-only: - - diff --git a/docs/source/cpp_api/matrix.rst b/docs/source/cpp_api/matrix.rst index 17953bc128..c983faa9db 100644 --- a/docs/source/cpp_api/matrix.rst +++ b/docs/source/cpp_api/matrix.rst @@ -17,4 +17,3 @@ headers cover many operations on matrices that are otherwise not covered by `raf matrix_manipulation.rst matrix_ordering.rst matrix_reduction.rst - matrix_selection.rst \ No newline at end of file diff --git a/docs/source/cpp_api/matrix_selection.rst b/docs/source/cpp_api/matrix_selection.rst deleted file mode 100644 index 4842a75e0e..0000000000 --- a/docs/source/cpp_api/matrix_selection.rst +++ /dev/null @@ -1,69 +0,0 @@ -Matrix Selection -================ - -.. role:: py(code) - :language: c++ - :class: highlight - - -Copy ----- - -``#include `` - -namespace *raft::matrix* - -.. doxygengroup:: matrix_copy - :project: RAFT - :members: - :content-only: - -Diagonal --------- - -``#include `` - -namespace *raft::matrix* - -.. doxygengroup:: matrix_diagonal - :project: RAFT - :members: - :content-only: - - -Gather ------- - -``#include `` - -namespace *raft::matrix* - -.. doxygengroup:: matrix_gather - :project: RAFT - :members: - :content-only: - - -Slicing -------- - -``#include `` - -namespace *raft::matrix* - -.. doxygengroup:: matrix_slice - :project: RAFT - :members: - :content-only: - -Triangular ----------- - -``#include `` - -namespace *raft::matrix* - -.. doxygengroup:: matrix_triangular - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/neighbors.rst b/docs/source/cpp_api/neighbors.rst deleted file mode 100644 index 876f68b1bf..0000000000 --- a/docs/source/cpp_api/neighbors.rst +++ /dev/null @@ -1,19 +0,0 @@ -Neighbors -========= - -This page provides C++ class references for the publicly-exposed elements of the neighbors package. - -.. role:: py(code) - :language: c++ - :class: highlight - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - neighbors_brute_force.rst - neighbors_ivf_flat.rst - neighbors_ivf_pq.rst - neighbors_epsilon_neighborhood.rst - neighbors_ball_cover.rst - neighbors_cagra.rst \ No newline at end of file diff --git a/docs/source/cpp_api/neighbors_ball_cover.rst b/docs/source/cpp_api/neighbors_ball_cover.rst deleted file mode 100644 index 85bd6b2d8e..0000000000 --- a/docs/source/cpp_api/neighbors_ball_cover.rst +++ /dev/null @@ -1,17 +0,0 @@ -Random Ball Cover -================= - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::neighbors::ball_cover* - -.. doxygengroup:: random_ball_cover - :project: RAFT - :members: - :content-only: - - diff --git a/docs/source/cpp_api/neighbors_brute_force.rst b/docs/source/cpp_api/neighbors_brute_force.rst deleted file mode 100644 index 525addf428..0000000000 --- a/docs/source/cpp_api/neighbors_brute_force.rst +++ /dev/null @@ -1,18 +0,0 @@ -Brute-Force -=========== - -.. role:: py(code) - :language: c++ - :class: highlight - - -``#include `` - -namespace *raft::neighbors::brute_force* - -.. doxygengroup:: brute_force_knn - :project: RAFT - :members: - :content-only: - - diff --git a/docs/source/cpp_api/neighbors_cagra.rst b/docs/source/cpp_api/neighbors_cagra.rst deleted file mode 100644 index 99ecd3a985..0000000000 --- a/docs/source/cpp_api/neighbors_cagra.rst +++ /dev/null @@ -1,31 +0,0 @@ -CAGRA -===== - -CAGRA is a graph-based nearest neighbors implementation with state-of-the art query performance for both small- and large-batch sized search. - -Please note that the CAGRA implementation is currently experimental and the API is subject to change from release to release. We are currently working on promoting CAGRA to a top-level stable API within RAFT. - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::neighbors::cagra* - -.. doxygengroup:: cagra - :project: RAFT - :members: - :content-only: - - -Serializer Methods ------------------- -``#include `` - -namespace *raft::neighbors::cagra* - -.. doxygengroup:: cagra_serialize - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/neighbors_epsilon_neighborhood.rst b/docs/source/cpp_api/neighbors_epsilon_neighborhood.rst deleted file mode 100644 index f291a7605f..0000000000 --- a/docs/source/cpp_api/neighbors_epsilon_neighborhood.rst +++ /dev/null @@ -1,15 +0,0 @@ -Epsilon Neighborhood -==================== - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::neighbors::epsilon_neighborhood* - -.. doxygengroup:: epsilon_neighbors - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/neighbors_hnsw.rst b/docs/source/cpp_api/neighbors_hnsw.rst deleted file mode 100644 index 86f9544c35..0000000000 --- a/docs/source/cpp_api/neighbors_hnsw.rst +++ /dev/null @@ -1,29 +0,0 @@ -HNSW -===== - -HNSW is a graph-based nearest neighbors implementation for the CPU. -This implementation provides the ability to serialize a CAGRA graph and read it as a base-layer-only hnswlib graph. - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::neighbors::hnsw* - -.. doxygengroup:: hnsw - :project: RAFT - :members: - :content-only: - -Serializer Methods ------------------- -``#include `` - -namespace *raft::neighbors::hnsw* - -.. doxygengroup:: hnsw_serialize - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/neighbors_ivf_flat.rst b/docs/source/cpp_api/neighbors_ivf_flat.rst deleted file mode 100644 index aa03fa8a80..0000000000 --- a/docs/source/cpp_api/neighbors_ivf_flat.rst +++ /dev/null @@ -1,37 +0,0 @@ -IVF-Flat -======== - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::neighbors::ivf_flat* - -.. doxygengroup:: ivf_flat - :project: RAFT - :members: - :content-only: - -Serializer Methods ------------------- -``#include `` - -namespace *raft::neighbors::ivf_flat* - -.. doxygengroup:: ivf_flat_serialize - :project: RAFT - :members: - :content-only: - -Helper Methods --------------- -``#include `` - -namespace *raft::neighbors::ivf_flat::helpers* - -.. doxygengroup:: ivf_flat_helpers - :project: RAFT - :members: - :content-only: \ No newline at end of file diff --git a/docs/source/cpp_api/neighbors_ivf_pq.rst b/docs/source/cpp_api/neighbors_ivf_pq.rst deleted file mode 100644 index 5301105c56..0000000000 --- a/docs/source/cpp_api/neighbors_ivf_pq.rst +++ /dev/null @@ -1,48 +0,0 @@ -IVF-PQ -====== - -.. role:: py(code) - :language: c++ - :class: highlight - -``#include `` - -namespace *raft::neighbors::ivf_pq* - -.. doxygengroup:: ivf_pq - :project: RAFT - :members: - :content-only: - -Serializer Methods ------------------- -``#include `` - -namespace *raft::neighbors::ivf_pq* - -.. doxygengroup:: ivf_pq_serialize - :project: RAFT - :members: - :content-only: - -Candidate Refinement --------------------- -``#include `` - -namespace *raft::neighbors* - -.. doxygengroup:: ann_refine - :project: RAFT - :members: - :content-only: - -Helper Methods --------------- -``#include `` - -namespace *raft::neighbors::ivf_pq::helpers* - -.. doxygengroup:: ivf_pq_helpers - :project: RAFT - :members: - :content-only: \ No newline at end of file diff --git a/docs/source/cpp_api/sparse.rst b/docs/source/cpp_api/sparse.rst index 9ad7b6aeda..64197accaf 100644 --- a/docs/source/cpp_api/sparse.rst +++ b/docs/source/cpp_api/sparse.rst @@ -13,9 +13,7 @@ Core to RAFT's computational patterns for sparse data is its vocabulary of spars :caption: Contents: sparse_types.rst - sparse_distance.rst sparse_linalg.rst sparse_matrix.rst - sparse_neighbors.rst sparse_solver.rst diff --git a/docs/source/cpp_api/sparse_distance.rst b/docs/source/cpp_api/sparse_distance.rst deleted file mode 100644 index e85e43695d..0000000000 --- a/docs/source/cpp_api/sparse_distance.rst +++ /dev/null @@ -1,7 +0,0 @@ -Sparse Distance -=============== - -.. doxygennamespace:: raft::sparse::distance - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/sparse_neighbors.rst b/docs/source/cpp_api/sparse_neighbors.rst deleted file mode 100644 index 9610913da6..0000000000 --- a/docs/source/cpp_api/sparse_neighbors.rst +++ /dev/null @@ -1,7 +0,0 @@ -Sparse Neighbors -================ - -.. doxygennamespace:: raft::sparse::neighbors - :project: RAFT - :members: - :content-only: diff --git a/docs/source/cpp_api/stats.rst b/docs/source/cpp_api/stats.rst index fd23ce2149..b1eca53e0b 100644 --- a/docs/source/cpp_api/stats.rst +++ b/docs/source/cpp_api/stats.rst @@ -16,4 +16,3 @@ This page provides C++ class references for the publicly-exposed elements of the stats_regression.rst stats_classification.rst stats_clustering.rst - stats_neighborhood.rst diff --git a/docs/source/cpp_api/stats_neighborhood.rst b/docs/source/cpp_api/stats_neighborhood.rst deleted file mode 100644 index 7c7ad90a49..0000000000 --- a/docs/source/cpp_api/stats_neighborhood.rst +++ /dev/null @@ -1,30 +0,0 @@ -Neighborhood Model Scoring -========================== - -.. role:: py(code) - :language: c++ - :class: highlight - -Trustworthiness ---------------- - -``#include `` - -namespace *raft::stats* - -.. doxygengroup:: stats_trustworthiness - :project: RAFT - :members: - :content-only: - -Neighborhood Recall -------------------- - -``#include `` - -namespace *raft::stats* - -.. doxygengroup:: stats_neighborhood_recall - :project: RAFT - :members: - :content-only: diff --git a/docs/source/developer_guide.md b/docs/source/developer_guide.md index 516819b1c1..c4a099fabb 100644 --- a/docs/source/developer_guide.md +++ b/docs/source/developer_guide.md @@ -187,7 +187,7 @@ RAFT relies on `clang-format` to enforce code style across all C++ and CUDA sour 1. Do not split empty functions/records/namespaces. 2. Two-space indentation everywhere, including the line continuations. 3. Disable reflowing of comments. - The reasons behind these deviations from the Google style guide are given in comments [here](https://github.com/rapidsai/raft/blob/branch-24.10/cpp/.clang-format). + The reasons behind these deviations from the Google style guide are given in comments [here](https://github.com/rapidsai/raft/blob/branch-24.12/cpp/.clang-format). [`doxygen`](https://doxygen.nl/) is used as documentation generator and also as a documentation linter. In order to run doxygen as a linter on C++/CUDA code, run @@ -205,7 +205,7 @@ you can run `codespell -i 3 -w .` from the repository root directory. This will bring up an interactive prompt to select which spelling fixes to apply. ### #include style -[include_checker.py](https://github.com/rapidsai/raft/blob/branch-24.10/cpp/scripts/include_checker.py) is used to enforce the include style as follows: +[include_checker.py](https://github.com/rapidsai/raft/blob/branch-24.12/cpp/scripts/include_checker.py) is used to enforce the include style as follows: 1. `#include "..."` should be used for referencing local files only. It is acceptable to be used for referencing files in a sub-folder/parent-folder of the same algorithm, but should never be used to include files in other algorithms or between algorithms and the primitives or other dependencies. 2. `#include <...>` should be used for referencing everything else @@ -230,7 +230,7 @@ Call CUDA APIs via the provided helper macros `RAFT_CUDA_TRY`, `RAFT_CUBLAS_TRY` ## Logging ### Introduction -Anything and everything about logging is defined inside [logger.hpp](https://github.com/rapidsai/raft/blob/branch-24.10/cpp/include/raft/core/logger.hpp). It uses [spdlog](https://github.com/gabime/spdlog) underneath, but this information is transparent to all. +Anything and everything about logging is defined inside [logger.hpp](https://github.com/rapidsai/raft/blob/branch-24.12/cpp/include/raft/core/logger.hpp). It uses [spdlog](https://github.com/gabime/spdlog) underneath, but this information is transparent to all. ### Usage ```cpp diff --git a/docs/source/index.rst b/docs/source/index.rst index bee0e948ff..21be1bafbd 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -11,7 +11,6 @@ Useful Resources .. _raft_reference: https://docs.rapids.ai/api/raft/stable/ -- `Example Notebooks `_: Example Jupyter notebooks - `RAPIDS Community `_: Get help, contribute, and collaborate. - `GitHub repository `_: Download the RAFT source code. - `Issue tracker `_: Report issues or request features. @@ -36,16 +35,12 @@ While not exhaustive, the following general categories help summarize the accele * - Category - Examples - * - Nearest Neighbors - - pairwise distances, vector search, epsilon neighborhoods, neighborhood graph construction * - Data Formats - sparse & dense, conversions, data generation * - Dense Operations - linear algebra, matrix and vector operations, slicing, norms, factorization, least squares, svd & eigenvalue problems * - Sparse Operations - linear algebra, eigenvalue problems, slicing, norms, reductions, factorization, symmetrization, components & labeling - * - Basic Clustering - - spectral clustering, hierarchical clustering, k-means * - Solvers - combinatorial optimization, iterative solvers * - Statistics @@ -61,9 +56,6 @@ While not exhaustive, the following general categories help summarize the accele build.md cpp_api.rst pylibraft_api.rst - using_libraft.md - vector_search_tutorial.md - raft_ann_benchmarks.md raft_dask_api.rst using_raft_comms.rst developer_guide.md diff --git a/docs/source/pylibraft_api.rst b/docs/source/pylibraft_api.rst index df25b76985..ad7d2873d7 100644 --- a/docs/source/pylibraft_api.rst +++ b/docs/source/pylibraft_api.rst @@ -7,9 +7,6 @@ Python API .. toctree:: :maxdepth: 4 - pylibraft_api/cluster.rst pylibraft_api/common.rst - pylibraft_api/distance.rst - pylibraft_api/matrix.rst - pylibraft_api/neighbors.rst pylibraft_api/random.rst + pylibraft_api/sparse.rst diff --git a/docs/source/pylibraft_api/cluster.rst b/docs/source/pylibraft_api/cluster.rst deleted file mode 100644 index 085297fe34..0000000000 --- a/docs/source/pylibraft_api/cluster.rst +++ /dev/null @@ -1,20 +0,0 @@ -Cluster -======= - -This page provides pylibraft class references for the publicly-exposed elements of the `pylibraft.cluster` package. - -.. role:: py(code) - :language: python - :class: highlight - -KMeans -###### - -.. autoclass:: pylibraft.cluster.kmeans.KMeansParams - :members: - -.. autofunction:: pylibraft.cluster.kmeans.fit - -.. autofunction:: pylibraft.cluster.kmeans.cluster_cost - -.. autofunction:: pylibraft.cluster.kmeans.compute_new_centroids diff --git a/docs/source/pylibraft_api/distance.rst b/docs/source/pylibraft_api/distance.rst deleted file mode 100644 index d14ed6fc08..0000000000 --- a/docs/source/pylibraft_api/distance.rst +++ /dev/null @@ -1,15 +0,0 @@ -Distance -======== - -This page provides `pylibraft` class references for the publicly-exposed elements of the `pylibraft.distance` package. RAFT's -distances have been highly optimized and support a wide assortment of different distance measures. - - -.. role:: py(code) - :language: python - :class: highlight - -.. autofunction:: pylibraft.distance.pairwise_distance - -.. autofunction:: pylibraft.distance.fused_l2_nn_argmin - diff --git a/docs/source/pylibraft_api/matrix.rst b/docs/source/pylibraft_api/matrix.rst deleted file mode 100644 index 884a466ec1..0000000000 --- a/docs/source/pylibraft_api/matrix.rst +++ /dev/null @@ -1,11 +0,0 @@ -Matrix -====== - -This page provides `pylibraft` class references for the publicly-exposed elements of the `pylibraft.matrix` package. - - -.. role:: py(code) - :language: python - :class: highlight - -.. autofunction:: pylibraft.matrix.select_k diff --git a/docs/source/pylibraft_api/neighbors.rst b/docs/source/pylibraft_api/neighbors.rst deleted file mode 100644 index e9e890fccb..0000000000 --- a/docs/source/pylibraft_api/neighbors.rst +++ /dev/null @@ -1,99 +0,0 @@ -Neighbors -========= - -This page provides pylibraft class references for the publicly-exposed elements of the neighbors package. - -.. role:: py(code) - :language: python - :class: highlight - - -Brute Force -########### - -.. autofunction:: pylibraft.neighbors.brute_force.knn - - -CAGRA -##### - -.. autoclass:: pylibraft.neighbors.cagra.IndexParams - :members: - -.. autofunction:: pylibraft.neighbors.cagra.build - -.. autoclass:: pylibraft.neighbors.cagra.SearchParams - :members: - -.. autofunction:: pylibraft.neighbors.cagra.search - -Serializer Methods ------------------- -.. autofunction:: pylibraft.neighbors.cagra.save - -.. autofunction:: pylibraft.neighbors.cagra.load - -HNSW -#### - -.. autoclass:: pylibraft.neighbors.hnsw.SearchParams - :members: - -.. autofunction:: pylibraft.neighbors.hnsw.from_cagra - -.. autofunction:: pylibraft.neighbors.hnsw.search - -Serializer Methods ------------------- -.. autofunction:: pylibraft.neighbors.hnsw.save - -.. autofunction:: pylibraft.neighbors.hnsw.load - -IVF-Flat -######## - -.. autoclass:: pylibraft.neighbors.ivf_flat.IndexParams - :members: - -.. autofunction:: pylibraft.neighbors.ivf_flat.build - -.. autofunction:: pylibraft.neighbors.ivf_flat.extend - -.. autoclass:: pylibraft.neighbors.ivf_flat.SearchParams - :members: - -.. autofunction:: pylibraft.neighbors.ivf_flat.search - -Serializer Methods ------------------- - -.. autofunction:: pylibraft.neighbors.ivf_flat.save - -.. autofunction:: pylibraft.neighbors.ivf_flat.load - -IVF-PQ -###### - -.. autoclass:: pylibraft.neighbors.ivf_pq.IndexParams - :members: - -.. autofunction:: pylibraft.neighbors.ivf_pq.build - -.. autofunction:: pylibraft.neighbors.ivf_pq.extend - -.. autoclass:: pylibraft.neighbors.ivf_pq.SearchParams - :members: - -.. autofunction:: pylibraft.neighbors.ivf_pq.search - -Serializer Methods ------------------- - -.. autofunction:: pylibraft.neighbors.ivf_pq.save - -.. autofunction:: pylibraft.neighbors.ivf_pq.load - -Candidate Refinement --------------------- - -.. autofunction:: pylibraft.neighbors.refine diff --git a/docs/source/pylibraft_api/sparse.rst b/docs/source/pylibraft_api/sparse.rst new file mode 100644 index 0000000000..b2c3f7a2b1 --- /dev/null +++ b/docs/source/pylibraft_api/sparse.rst @@ -0,0 +1,11 @@ +Sparse +====== + +This page provides pylibraft class references for the publicly-exposed elements of the `pylibraft.sparse.linalg.eigsh` package. + + +.. role:: py(code) + :language: python + :class: highlight + +.. autofunction:: pylibraft.sparse.linalg.eigsh \ No newline at end of file diff --git a/docs/source/quick_start.md b/docs/source/quick_start.md index 3909b40f20..309e801597 100644 --- a/docs/source/quick_start.md +++ b/docs/source/quick_start.md @@ -23,7 +23,7 @@ auto vector = raft::make_device_vector(handle, n_cols); auto matrix = raft::make_device_matrix(handle, n_rows, n_cols); ``` -The `mdspan` is a lightweight non-owning view that can wrap around any pointer, maintaining shape, layout, and indexing information for accessing elements. +The `mdspan` is a lightweight non-owning view that can wrap around any pointer, maintaining shape, layout, and indexing information for accessing elements. We can construct `mdspan` instances directly from the above `mdarray` instances: @@ -95,13 +95,13 @@ auto vector = raft::make_device_vector_view(vector_ptr, raft::make_vector_stride Most of the primitives in RAFT accept a `raft::handle_t` object for the management of resources which are expensive to create, such CUDA streams, stream pools, and handles to other CUDA libraries like `cublas` and `cusolver`. The example below demonstrates creating a RAFT handle and using it with `device_matrix` and `device_vector` to allocate memory, generating random clusters, and computing -pairwise Euclidean distances: +pairwise Euclidean distances with [NVIDIA cuVS](https://github.com/rapidsai/cuvs): ```c++ #include #include #include -#include +#include raft::handle_t handle; @@ -114,20 +114,20 @@ auto output = raft::make_device_matrix(handle, n_samples, n_samples); raft::random::make_blobs(handle, input.view(), labels.view()); -auto metric = raft::distance::DistanceType::L2SqrtExpanded; -raft::distance::pairwise_distance(handle, input.view(), input.view(), output.view(), metric); +auto metric = cuvs::distance::DistanceType::L2SqrtExpanded; +cuvs::distance::pairwise_distance(handle, input.view(), input.view(), output.view(), metric); ``` ## Python Example The `pylibraft` package contains a Python API for RAFT algorithms and primitives. `pylibraft` integrates nicely into other libraries by being very lightweight with minimal dependencies and accepting any object that supports the `__cuda_array_interface__`, such as [CuPy's ndarray](https://docs.cupy.dev/en/stable/user_guide/interoperability.html#rmm). The number of RAFT algorithms exposed in this package is continuing to grow from release to release. -The example below demonstrates computing the pairwise Euclidean distances between CuPy arrays. Note that CuPy is not a required dependency for `pylibraft`. +The example below demonstrates computing the pairwise Euclidean distances between CuPy arrays with the [NVIDIA cuVS](https://github.com/rapidsai/cuvs) library. Note that CuPy is not a required dependency for `pylibraft`. ```python import cupy as cp -from pylibraft.distance import pairwise_distance +from cuvs.distance import pairwise_distance n_samples = 5000 n_features = 50 @@ -170,7 +170,7 @@ pylibraft.config.set_output_as(lambda device_ndarray: return device_ndarray.copy ```python import cupy as cp -from pylibraft.distance import pairwise_distance +from cuvs.distance import pairwise_distance n_samples = 5000 n_features = 50 diff --git a/docs/source/raft_ann_benchmarks.md b/docs/source/raft_ann_benchmarks.md deleted file mode 100644 index 7bac2047fc..0000000000 --- a/docs/source/raft_ann_benchmarks.md +++ /dev/null @@ -1,597 +0,0 @@ -# RAFT ANN Benchmarks - -This project provides a benchmark program for various ANN search implementations. It's especially suitable for comparing GPU implementations as well as comparing GPU against CPU. - -> [!IMPORTANT] -> The vector search and clustering algorithms in RAFT are being migrated to a new library dedicated to vector search called [cuVS](https://github.com/rapidsai/cuvs). As a result, `raft-ann-bench` is being migrated to `cuvs-bench` and will be removed from RAFT altogether in the 24.12 (December) release. - - -## Table of Contents - -- [Installing the benchmarks](#installing-the-benchmarks) - - [Conda](#conda) - - [Docker](#docker) -- [How to run the benchmarks](#how-to-run-the-benchmarks) - - [Step 1: prepare dataset](#step-1-prepare-dataset) - - [Step 2: build and search index](#step-2-build-and-search-index) - - [Step 3: data export](#step-3-data-export) - - [Step 4: plot results](#step-4-plot-results) -- [Running the benchmarks](#running-the-benchmarks) - - [End to end: small-scale (<1M to 10M)](#end-to-end-small-scale-benchmarks-1m-to-10m) - - [End to end: large-scale (>10M)](#end-to-end-large-scale-benchmarks-10m-vectors) - - [Running with Docker containers](#running-with-docker-containers) - - [Evaluating the results](#evaluating-the-results) -- [Creating and customizing dataset configurations](#creating-and-customizing-dataset-configurations) -- [Adding a new ANN algorithm](#adding-a-new-ann-algorithm) -- [Parameter tuning guide](https://docs.rapids.ai/api/raft/nightly/ann_benchmarks_param_tuning/) -- [Wiki-all RAG/LLM Dataset](https://docs.rapids.ai/api/raft/nightly/wiki_all_dataset/) - -## Installing the benchmarks - -There are two main ways pre-compiled benchmarks are distributed: - -- [Conda](#Conda): For users not using containers but want an easy to install and use Python package. Pip wheels are planned to be added as an alternative for users that cannot use conda and prefer to not use containers. -- [Docker](#Docker): Only needs docker and [NVIDIA docker](https://github.com/NVIDIA/nvidia-docker) to use. Provides a single docker run command for basic dataset benchmarking, as well as all the functionality of the conda solution inside the containers. - -## Conda - -If containers are not an option or not preferred, the easiest way to install the ANN benchmarks is through conda. We provide packages for GPU enabled systems, as well for systems without a GPU. We suggest using mamba as it generally leads to a faster install time: - -```bash - -mamba create --name raft_ann_benchmarks -conda activate raft_ann_benchmarks - -# to install GPU package: -mamba install -c rapidsai -c conda-forge -c nvidia raft-ann-bench= cuda-version=11.8* - -# to install CPU package for usage in CPU-only systems: -mamba install -c rapidsai -c conda-forge raft-ann-bench-cpu -``` - -The channel `rapidsai` can easily be substituted `rapidsai-nightly` if nightly benchmarks are desired. The CPU package currently allows to run the HNSW benchmarks. - -Please see the [build instructions](ann_benchmarks_build.md) to build the benchmarks from source. - -## Docker - -We provide images for GPU enabled systems, as well as systems without a GPU. The following images are available: - -- `raft-ann-bench`: Contains GPU and CPU benchmarks, can run all algorithms supported. Will download million-scale datasets as required. Best suited for users that prefer a smaller container size for GPU based systems. Requires the NVIDIA Container Toolkit to run GPU algorithms, can run CPU algorithms without it. -- `raft-ann-bench-datasets`: Contains the GPU and CPU benchmarks with million-scale datasets already included in the container. Best suited for users that want to run multiple million scale datasets already included in the image. -- `raft-ann-bench-cpu`: Contains only CPU benchmarks with minimal size. Best suited for users that want the smallest containers to reproduce benchmarks on systems without a GPU. - -Nightly images are located in [dockerhub](https://hub.docker.com/r/rapidsai/raft-ann-bench/tags), meanwhile release (stable) versions are located in [NGC](https://hub.docker.com/r/rapidsai/raft-ann-bench), starting with release 23.12. - -- The following command pulls the nightly container for python version 10, cuda version 12, and RAFT version 23.10: - -```bash -docker pull rapidsai/raft-ann-bench:24.10a-cuda12.0-py3.10 #substitute raft-ann-bench for the exact desired container. -``` - -The CUDA and python versions can be changed for the supported values: - -Supported CUDA versions: 11.2 and 12.0 -Supported Python versions: 3.9 and 3.10. - -You can see the exact versions as well in the dockerhub site: - -- [RAFT ANN Benchmark images](https://hub.docker.com/r/rapidsai/raft-ann-bench/tags) -- [RAFT ANN Benchmark with datasets preloaded images](https://hub.docker.com/r/rapidsai/raft-ann-bench-cpu/tags) -- [RAFT ANN Benchmark CPU only images](https://hub.docker.com/r/rapidsai/raft-ann-bench-datasets/tags) - -**Note:** GPU containers use the CUDA toolkit from inside the container, the only requirement is a driver installed on the host machine that supports that version. So, for example, CUDA 11.8 containers can run in systems with a CUDA 12.x capable driver. Please also note that the Nvidia-Docker runtime from the [Nvidia Container Toolkit](https://github.com/NVIDIA/nvidia-docker) is required to use GPUs inside docker containers. - -[//]: # (- The following command (only available after RAPIDS 23.10 release) pulls the container:) - -[//]: # () -[//]: # (```bash) - -[//]: # (docker pull nvcr.io/nvidia/rapidsai/raft-ann-bench:24.10-cuda11.8-py3.10 #substitute raft-ann-bench for the exact desired container.) - -[//]: # (```) - -## How to run the benchmarks - -We provide a collection of lightweight Python scripts to run the benchmarks. There are 4 general steps to running the benchmarks and visualizing the results. -1. Prepare Dataset -2. Build Index and Search Index -3. Data Export -4. Plot Results - -### Step 1: Prepare Dataset -The script `raft_ann_bench.get_dataset` will download and unpack the dataset in directory -that the user provides. As of now, only million-scale datasets are supported by this -script. For more information on [datasets and formats](ann_benchmarks_dataset.md). - -The usage of this script is: -```bash -usage: get_dataset.py [-h] [--name NAME] [--dataset-path DATASET_PATH] [--normalize] - -options: - -h, --help show this help message and exit - --dataset DATASET dataset to download (default: glove-100-angular) - --dataset-path DATASET_PATH - path to download dataset (default: ${RAPIDS_DATASET_ROOT_DIR}) - --normalize normalize cosine distance to inner product (default: False) -``` - -When option `normalize` is provided to the script, any dataset that has cosine distances -will be normalized to inner product. So, for example, the dataset `glove-100-angular` -will be written at location `datasets/glove-100-inner/`. - -### Step 2: Build and Search Index -The script `raft_ann_bench.run` will build and search indices for a given dataset and its -specified configuration. - -The usage of the script `raft_ann_bench.run` is: -```bash -usage: __main__.py [-h] [--subset-size SUBSET_SIZE] [-k COUNT] [-bs BATCH_SIZE] [--dataset-configuration DATASET_CONFIGURATION] [--configuration CONFIGURATION] [--dataset DATASET] - [--dataset-path DATASET_PATH] [--build] [--search] [--algorithms ALGORITHMS] [--groups GROUPS] [--algo-groups ALGO_GROUPS] [-f] [-m SEARCH_MODE] - -options: - -h, --help show this help message and exit - --subset-size SUBSET_SIZE - the number of subset rows of the dataset to build the index (default: None) - -k COUNT, --count COUNT - the number of nearest neighbors to search for (default: 10) - -bs BATCH_SIZE, --batch-size BATCH_SIZE - number of query vectors to use in each query trial (default: 10000) - --dataset-configuration DATASET_CONFIGURATION - path to YAML configuration file for datasets (default: None) - --configuration CONFIGURATION - path to YAML configuration file or directory for algorithms Any run groups found in the specified file/directory will automatically override groups of the same name - present in the default configurations, including `base` (default: None) - --dataset DATASET name of dataset (default: glove-100-inner) - --dataset-path DATASET_PATH - path to dataset folder, by default will look in RAPIDS_DATASET_ROOT_DIR if defined, otherwise a datasets subdirectory from the calling directory (default: - os.getcwd()/datasets/) - --build - --search - --algorithms ALGORITHMS - run only comma separated list of named algorithms. If parameters `groups` and `algo-groups are both undefined, then group `base` is run by default (default: None) - --groups GROUPS run only comma separated groups of parameters (default: base) - --algo-groups ALGO_GROUPS - add comma separated . to run. Example usage: "--algo-groups=raft_cagra.large,hnswlib.large" (default: None) - -f, --force re-run algorithms even if their results already exist (default: False) - -m SEARCH_MODE, --search-mode SEARCH_MODE - run search in 'latency' (measure individual batches) or 'throughput' (pipeline batches and measure end-to-end) mode (default: throughput) - -t SEARCH_THREADS, --search-threads SEARCH_THREADS - specify the number threads to use for throughput benchmark. Single value or a pair of min and max separated by ':'. Example --search-threads=1:4. Power of 2 values between 'min' and 'max' will be used. If only 'min' is - specified, then a single test is run with 'min' threads. By default min=1, max=. (default: None) - -r, --dry-run dry-run mode will convert the yaml config for the specified algorithms and datasets to the json format that's consumed by the lower-level c++ binaries and then print the command to run execute the benchmarks but - will not actually execute the command. (default: False) -``` - -`dataset`: name of the dataset to be searched in [datasets.yaml](#yaml-dataset-config) - -`dataset-configuration`: optional filepath to custom dataset YAML config which has an entry for arg `dataset` - -`configuration`: optional filepath to YAML configuration for an algorithm or to directory that contains YAML configurations for several algorithms. [Here's how to configure an algorithm.](#yaml-algo-config) - -`algorithms`: runs all algorithms that it can find in YAML configs found by `configuration`. By default, only `base` group will be run. - -`groups`: run only specific groups of parameters configurations for an algorithm. Groups are defined in YAML configs (see `configuration`), and by default run `base` group - -`algo-groups`: this parameter is helpful to append any specific algorithm+group combination to run the benchmark for in addition to all the arguments from `algorithms` and `groups`. It is of the format `.`, or for example, `raft_cagra.large` - -For every algorithm run by this script, it outputs an index build statistics JSON file in `/result/build/<{algo},{group}.json>` -and an index search statistics JSON file in `/result/search/<{algo},{group},k{k},bs{batch_size}.json>`. NOTE: The filenames will not have ",{group}" if `group = "base"`. - -`dataset-path` : -1. data is read from `/` -2. indices are built in `//index` -3. build/search results are stored in `//result` - -`build` and `search` : if both parameters are not supplied to the script then -it is assumed both are `True`. - -`indices` and `algorithms` : these parameters ensure that the algorithm specified for an index -is available in `algos.yaml` and not disabled, as well as having an associated executable. - -### Step 3: Data Export -The script `raft_ann_bench.data_export` will convert the intermediate JSON outputs produced by `raft_ann_bench.run` to more -easily readable CSV files, which are needed to build charts made by `raft_ann_bench.plot`. - -```bash -usage: data_export.py [-h] [--dataset DATASET] [--dataset-path DATASET_PATH] - -options: - -h, --help show this help message and exit - --dataset DATASET dataset to download (default: glove-100-inner) - --dataset-path DATASET_PATH - path to dataset folder (default: ${RAPIDS_DATASET_ROOT_DIR}) -``` -Build statistics CSV file is stored in `/result/build/<{algo},{group}.csv>` -and index search statistics CSV file in `/result/search/<{algo},{group},k{k},bs{batch_size},{suffix}.csv>`, where suffix has three values: -1. `raw`: All search results are exported -2. `throughput`: Pareto frontier of throughput results is exported -3. `latency`: Pareto frontier of latency results is exported - - -### Step 4: Plot Results -The script `raft_ann_bench.plot` will plot results for all algorithms found in index search statistics -CSV files `/result/search/*.csv`. - -The usage of this script is: -```bash -usage: [-h] [--dataset DATASET] [--dataset-path DATASET_PATH] [--output-filepath OUTPUT_FILEPATH] [--algorithms ALGORITHMS] [--groups GROUPS] [--algo-groups ALGO_GROUPS] - [-k COUNT] [-bs BATCH_SIZE] [--build] [--search] [--x-scale X_SCALE] [--y-scale {linear,log,symlog,logit}] [--x-start X_START] [--mode {throughput,latency}] - [--time-unit {s,ms,us}] [--raw] - -options: - -h, --help show this help message and exit - --dataset DATASET dataset to plot (default: glove-100-inner) - --dataset-path DATASET_PATH - path to dataset folder (default: /home/coder/raft/datasets/) - --output-filepath OUTPUT_FILEPATH - directory for PNG to be saved (default: /home/coder/raft) - --algorithms ALGORITHMS - plot only comma separated list of named algorithms. If parameters `groups` and `algo-groups are both undefined, then group `base` is plot by default - (default: None) - --groups GROUPS plot only comma separated groups of parameters (default: base) - --algo-groups ALGO_GROUPS, --algo-groups ALGO_GROUPS - add comma separated . to plot. Example usage: "--algo-groups=raft_cagra.large,hnswlib.large" (default: None) - -k COUNT, --count COUNT - the number of nearest neighbors to search for (default: 10) - -bs BATCH_SIZE, --batch-size BATCH_SIZE - number of query vectors to use in each query trial (default: 10000) - --build - --search - --x-scale X_SCALE Scale to use when drawing the X-axis. Typically linear, logit or a2 (default: linear) - --y-scale {linear,log,symlog,logit} - Scale to use when drawing the Y-axis (default: linear) - --x-start X_START Recall values to start the x-axis from (default: 0.8) - --mode {throughput,latency} - search mode whose Pareto frontier is used on the y-axis (default: throughput) - --time-unit {s,ms,us} - time unit to plot when mode is latency (default: ms) - --raw Show raw results (not just Pareto frontier) of mode arg (default: False) -``` -`mode`: plots pareto frontier of `throughput` or `latency` results exported in the previous step - -`algorithms`: plots all algorithms that it can find results for the specified `dataset`. By default, only `base` group will be plotted. - -`groups`: plot only specific groups of parameters configurations for an algorithm. Groups are defined in YAML configs (see `configuration`), and by default run `base` group - -`algo-groups`: this parameter is helpful to append any specific algorithm+group combination to plot results for in addition to all the arguments from `algorithms` and `groups`. It is of the format `.`, or for example, `raft_cagra.large` - -The figure below is the resulting plot of running our benchmarks as of August 2023 for a batch size of 10, on an NVIDIA H100 GPU and an Intel Xeon Platinum 8480CL CPU. It presents the throughput (in Queries-Per-Second) performance for every level of recall. - -![Throughput vs recall plot comparing popular ANN algorithms with RAFT's at batch size 10](../../img/raft-vector-search-batch-10.png) - -## Running the benchmarks - -### End to end: small-scale benchmarks (<1M to 10M) - -The steps below demonstrate how to download, install, and run benchmarks on a subset of 10M vectors from the Yandex Deep-1B dataset By default the datasets will be stored and used from the folder indicated by the `RAPIDS_DATASET_ROOT_DIR` environment variable if defined, otherwise a datasets sub-folder from where the script is being called: - -```bash - -# (1) prepare dataset. -python -m raft_ann_bench.get_dataset --dataset deep-image-96-angular --normalize - -# (2) build and search index -python -m raft_ann_bench.run --dataset deep-image-96-inner --algorithms raft_cagra --batch-size 10 -k 10 - -# (3) export data -python -m raft_ann_bench.data_export --dataset deep-image-96-inner - -# (4) plot results -python -m raft_ann_bench.plot --dataset deep-image-96-inner -``` - -Configuration files already exist for the following list of the million-scale datasets. Please refer to [ann-benchmarks datasets](https://github.com/erikbern/ann-benchmarks/#data-sets) for more information, including actual train and sizes. These all work out-of-the-box with the `--dataset` argument. Other million-scale datasets from `ann-benchmarks.com` will work, but will require a json configuration file to be created in `$CONDA_PREFIX/lib/python3.xx/site-packages/raft_ann_bench/run/conf`, or you can specify the `--configuration` option to use a specific file. - -| Dataset Name | Train Rows | Columns | Test Rows | Distance | -|-----|------------|----|----------------|------------| -| `deep-image-96-angular` | 10M | 96 | 10K | Angular | -| `fashion-mnist-784-euclidean` | 60K | 784 | 10K | Euclidean | -| `glove-50-angular` | 1.1M | 50 | 10K | Angular | -| `glove-100-angular` | 1.1M | 100 | 10K | Angular | -| `mnist-784-euclidean` | 60K | 784 | 10K | Euclidean | -| `nytimes-256-angular` | 290K | 256 | 10K | Angular | -| `sift-128-euclidean` | 1M | 128 | 10K | Euclidean| - -All of the datasets above contain ground test datasets with 100 neighbors. Thus `k` for these datasets must be less than or equal to 100. - -### End to end: large-scale benchmarks (>10M vectors) - -`raft_ann_bench.get_dataset` cannot be used to download the [billion-scale datasets](ann_benchmarks_dataset.md#billion-scale) -due to their size. You should instead use our billion-scale datasets guide to download and prepare them. -All other python commands mentioned below work as intended once the -billion-scale dataset has been downloaded. -To download billion-scale datasets, visit [big-ann-benchmarks](http://big-ann-benchmarks.com/neurips21.html) - -We also provide a new dataset called `wiki-all` containing 88 million 768-dimensional vectors. This dataset is meant for benchmarking a realistic retrieval-augmented generation (RAG)/LLM embedding size at scale. It also contains 1M and 10M vector subsets for smaller-scale experiments. See our [Wiki-all Dataset Guide](https://docs.rapids.ai/api/raft/nightly/wiki_all_dataset/) for more information and to download the dataset. - -The steps below demonstrate how to download, install, and run benchmarks on a subset of 100M vectors from the Yandex Deep-1B dataset. Please note that datasets of this scale are recommended for GPUs with larger amounts of memory, such as the A100 or H100. -```bash - -mkdir -p datasets/deep-1B -# (1) prepare dataset -# download manually "Ground Truth" file of "Yandex DEEP" -# suppose the file name is deep_new_groundtruth.public.10K.bin -python -m raft_ann_bench.split_groundtruth --groundtruth datasets/deep-1B/deep_new_groundtruth.public.10K.bin -# two files 'groundtruth.neighbors.ibin' and 'groundtruth.distances.fbin' should be produced - -# (2) build and search index -python -m raft_ann_bench.run --dataset deep-1B --algorithms raft_cagra --batch-size 10 -k 10 - -# (3) export data -python -m raft_ann_bench.data_export --dataset deep-1B - -# (4) plot results -python -m raft_ann_bench.plot --dataset deep-1B -``` - -The usage of `python -m raft_ann_bench.split_groundtruth` is: -```bash -usage: split_groundtruth.py [-h] --groundtruth GROUNDTRUTH - -options: - -h, --help show this help message and exit - --groundtruth GROUNDTRUTH - Path to billion-scale dataset groundtruth file (default: None) -``` - -### Running with Docker containers - -Two methods are provided for running the benchmarks with the Docker containers. - -#### End-to-end run on GPU - -When no other entrypoint is provided, an end-to-end script will run through all the steps in [Running the benchmarks](#running-the-benchmarks) above. - -For GPU-enabled systems, the `DATA_FOLDER` variable should be a local folder where you want datasets stored in `$DATA_FOLDER/datasets` and results in `$DATA_FOLDER/result` (we highly recommend `$DATA_FOLDER` to be a dedicated folder for the datasets and results of the containers): -```bash -export DATA_FOLDER=path/to/store/datasets/and/results -docker run --gpus all --rm -it -u $(id -u) \ - -v $DATA_FOLDER:/data/benchmarks \ - rapidsai/raft-ann-bench:24.10a-cuda11.8-py3.10 \ - "--dataset deep-image-96-angular" \ - "--normalize" \ - "--algorithms raft_cagra,raft_ivf_pq --batch-size 10 -k 10" \ - "" -``` - -Usage of the above command is as follows: - -| Argument | Description | -|-----------------------------------------------------------|----------------------------------------------------------------------------------------------------| -| `rapidsai/raft-ann-bench:24.10a-cuda11.8-py3.10` | Image to use. Can be either `raft-ann-bench` or `raft-ann-bench-datasets` | -| `"--dataset deep-image-96-angular"` | Dataset name | -| `"--normalize"` | Whether to normalize the dataset | -| `"--algorithms raft_cagra,hnswlib --batch-size 10 -k 10"` | Arguments passed to the `run` script, such as the algorithms to benchmark, the batch size, and `k` | -| `""` | Additional (optional) arguments that will be passed to the `plot` script. | - -***Note about user and file permissions:*** The flag `-u $(id -u)` allows the user inside the container to match the `uid` of the user outside the container, allowing the container to read and write to the mounted volume indicated by the `$DATA_FOLDER` variable. - -#### End-to-end run on CPU - -The container arguments in the above section also be used for the CPU-only container, which can be used on systems that don't have a GPU installed. - -***Note:*** the image changes to `raft-ann-bench-cpu` container and the `--gpus all` argument is no longer used: -```bash -export DATA_FOLDER=path/to/store/datasets/and/results -docker run --rm -it -u $(id -u) \ - -v $DATA_FOLDER:/data/benchmarks \ - rapidsai/raft-ann-bench-cpu:24.10a-py3.10 \ - "--dataset deep-image-96-angular" \ - "--normalize" \ - "--algorithms hnswlib --batch-size 10 -k 10" \ - "" -``` - -#### Manually run the scripts inside the container - -All of the `raft-ann-bench` images contain the Conda packages, so they can be used directly by logging directly into the container itself: - -```bash -export DATA_FOLDER=path/to/store/datasets/and/results -docker run --gpus all --rm -it -u $(id -u) \ - --entrypoint /bin/bash \ - --workdir /data/benchmarks \ - -v $DATA_FOLDER:/data/benchmarks \ - rapidsai/raft-ann-bench:24.10a-cuda11.8-py3.10 -``` - -This will drop you into a command line in the container, with the `raft-ann-bench` python package ready to use, as described in the [Running the benchmarks](#running-the-benchmarks) section above: - -``` -(base) root@00b068fbb862:/data/benchmarks# python -m raft_ann_bench.get_dataset --dataset deep-image-96-angular --normalize -``` - -Additionally, the containers can be run in detached mode without any issue. - - -### Evaluating the results - -The benchmarks capture several different measurements. The table below describes each of the measurements for index build benchmarks: - -| Name | Description | -|------------|--------------------------------------------------------| -| Benchmark | A name that uniquely identifies the benchmark instance | -| Time | Wall-time spent training the index | -| CPU | CPU time spent training the index | -| Iterations | Number of iterations (this is usually 1) | -| GPU | GPU time spent building | -| index_size | Number of vectors used to train index | - - -The table below describes each of the measurements for the index search benchmarks. The most important measurements `Latency`, `items_per_second`, `end_to_end`. - -| Name | Description | -|------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| -| Benchmark | A name that uniquely identifies the benchmark instance | -| Time | The wall-clock time of a single iteration (batch) divided by the number of threads. | -| CPU | The average CPU time (user + sys time). This does not include idle time (which can also happen while waiting for GPU sync). | -| Iterations | Total number of batches. This is going to be `total_queries` / `n_queries`. | -| GPU | GPU latency of a single batch (seconds). In throughput mode this is averaged over multiple threads. | -| Latency | Latency of a single batch (seconds), calculated from wall-clock time. In throughput mode this is averaged over multiple threads. | -| Recall | Proportion of correct neighbors to ground truth neighbors. Note this column is only present if groundtruth file is specified in dataset configuration.| -| items_per_second | Total throughput, a.k.a Queries per second (QPS). This is approximately `total_queries` / `end_to_end`. | -| k | Number of neighbors being queried in each iteration | -| end_to_end | Total time taken to run all batches for all iterations | -| n_queries | Total number of query vectors in each batch | -| total_queries | Total number of vectors queries across all iterations ( = `iterations` * `n_queries`) | - -Note the following: -- A slightly different method is used to measure `Time` and `end_to_end`. That is why `end_to_end` = `Time` * `Iterations` holds only approximately. -- The actual table displayed on the screen may differ slightly as the hyper-parameters will also be displayed for each different combination being benchmarked. -- Recall calculation: the number of queries processed per test depends on the number of iterations. Because of this, recall can show slight fluctuations if less neighbors are processed then it is available for the benchmark. - -## Creating and customizing dataset configurations - -A single configuration will often define a set of algorithms, with associated index and search parameters, that can be generalize across datasets. We use YAML to define dataset specific and algorithm specific configurations. - -A default `datasets.yaml` is provided by RAFT in `${RAFT_HOME}/python/raft-ann-bench/src/raft_ann_bench/run/conf` with configurations available for several datasets. Here's a simple example entry for the `sift-128-euclidean` dataset: - -```yaml -- name: sift-128-euclidean - base_file: sift-128-euclidean/base.fbin - query_file: sift-128-euclidean/query.fbin - groundtruth_neighbors_file: sift-128-euclidean/groundtruth.neighbors.ibin - dims: 128 - distance: euclidean -``` - -Configuration files for ANN algorithms supported by `raft-ann-bench` are provided in `${RAFT_HOME}/python/raft-ann-bench/src/raft_ann_bench/run/conf`. `raft_cagra` algorithm configuration looks like: -```yaml -name: raft_cagra -groups: - base: - build: - graph_degree: [32, 64] - intermediate_graph_degree: [64, 96] - graph_build_algo: ["NN_DESCENT"] - search: - itopk: [32, 64, 128] - - large: - build: - graph_degree: [32, 64] - search: - itopk: [32, 64, 128] -``` -The default parameters for which the benchmarks are run can be overridden by creating a custom YAML file for algorithms with a `base` group. - -There config above has 2 fields: -1. `name` - define the name of the algorithm for which the parameters are being specified. -2. `groups` - define a run group which has a particular set of parameters. Each group helps create a cross-product of all hyper-parameter fields for `build` and `search`. - -The table below contains all algorithms supported by RAFT. Each unique algorithm will have its own set of `build` and `search` settings. The [ANN Algorithm Parameter Tuning Guide](ann_benchmarks_param_tuning.md) contains detailed instructions on choosing build and search parameters for each supported algorithm. - -| Library | Algorithms | -|-----------|---------------------------------------------------------------------------------------| -| FAISS GPU | `faiss_gpu_flat`, `faiss_gpu_ivf_flat`, `faiss_gpu_ivf_pq` | -| FAISS CPU | `faiss_cpu_flat`, `faiss_cpu_ivf_flat`, `faiss_cpu_ivf_pq` | -| GGNN | `ggnn` | -| HNSWlib | `hnswlib` | -| RAFT | `raft_brute_force`, `raft_cagra`, `raft_ivf_flat`, `raft_ivf_pq`, `raft_cagra_hnswlib`| - -## Adding a new ANN algorithm - -### Implementation and Configuration -Implementation of a new algorithm should be a C++ class that inherits `class ANN` (defined in `cpp/bench/ann/src/ann.h`) and implements all the pure virtual functions. - -In addition, it should define two `struct`s for building and searching parameters. The searching parameter class should inherit `struct ANN::AnnSearchParam`. Take `class HnswLib` as an example, its definition is: -```c++ -template -class HnswLib : public ANN { -public: - struct BuildParam { - int M; - int ef_construction; - int num_threads; - }; - - using typename ANN::AnnSearchParam; - struct SearchParam : public AnnSearchParam { - int ef; - int num_threads; - }; - - // ... -}; -``` - -The benchmark program uses JSON format in a configuration file to specify indexes to build, along with the build and search parameters. To add the new algorithm to the benchmark, need be able to specify `build_param`, whose value is a JSON object, and `search_params`, whose value is an array of JSON objects, for this algorithm in configuration file. The `build_param` and `search_param` arguments will vary depending on the algorithm. Take the configuration for `HnswLib` as an example: -```json -{ - "name" : "hnswlib.M12.ef500.th32", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "/path/to/file", - "search_params" : [ - {"ef":10, "numThreads":1}, - {"ef":20, "numThreads":1}, - {"ef":40, "numThreads":1}, - ], - "search_result_file" : "/path/to/file" -}, -``` -How to interpret these JSON objects is totally left to the implementation and should be specified in `cpp/bench/ann/src/factory.cuh`: -1. First, add two functions for parsing JSON object to `struct BuildParam` and `struct SearchParam`, respectively: - ```c++ - template - void parse_build_param(const nlohmann::json& conf, - typename cuann::HnswLib::BuildParam& param) { - param.ef_construction = conf.at("efConstruction"); - param.M = conf.at("M"); - if (conf.contains("numThreads")) { - param.num_threads = conf.at("numThreads"); - } - } - - template - void parse_search_param(const nlohmann::json& conf, - typename cuann::HnswLib::SearchParam& param) { - param.ef = conf.at("ef"); - if (conf.contains("numThreads")) { - param.num_threads = conf.at("numThreads"); - } - } - ``` - -2. Next, add corresponding `if` case to functions `create_algo()` (in `cpp/bench/ann/) and `create_search_param()` by calling parsing functions. The string literal in `if` condition statement must be the same as the value of `algo` in configuration file. For example, - ```c++ - // JSON configuration file contains a line like: "algo" : "hnswlib" - if (algo == "hnswlib") { - // ... - } - ``` - - -### Adding a CMake Target -In `raft/cpp/bench/ann/CMakeLists.txt`, we provide a `CMake` function to configure a new Benchmark target with the following signature: -``` -ConfigureAnnBench( - NAME - PATH - INCLUDES - CXXFLAGS - LINKS -) -``` - -To add a target for `HNSWLIB`, we would call the function as: -``` -ConfigureAnnBench( - NAME HNSWLIB PATH bench/ann/src/hnswlib/hnswlib_benchmark.cpp INCLUDES - ${CMAKE_CURRENT_BINARY_DIR}/_deps/hnswlib-src/hnswlib CXXFLAGS "${HNSW_CXX_FLAGS}" -) -``` - -This will create an executable called `HNSWLIB_ANN_BENCH`, which can then be used to run `HNSWLIB` benchmarks. - -Add a new entry to `algos.yaml` to map the name of the algorithm to its binary executable and specify whether the algorithm requires GPU support. -```yaml -raft_ivf_pq: - executable: RAFT_IVF_PQ_ANN_BENCH - requires_gpu: true -``` - -`executable` : specifies the name of the binary that will build/search the index. It is assumed to be -available in `raft/cpp/build/`. -`requires_gpu` : denotes whether an algorithm requires GPU to run. diff --git a/docs/source/using_libraft.md b/docs/source/using_libraft.md deleted file mode 100644 index 70a17e289b..0000000000 --- a/docs/source/using_libraft.md +++ /dev/null @@ -1,64 +0,0 @@ -# Using The Pre-Compiled Binary - -At its core, RAFT is a header-only template library, which makes it very powerful in that APIs can be called with various different combinations of data types and only the templates which are actually used will be compiled into your binaries. This increased flexibility comes with a drawback that all the APIs need to be declared inline and thus calls which are made frequently in your code could be compiled again in each source file for which they are invoked. - -For most functions, compile-time overhead is minimal but some of RAFT's APIs take a substantial time to compile. As a rule of thumb, most functionality in `raft::distance`, `raft::neighbors`, and `raft::cluster` is expensive to compile and most functionality in other namespaces has little compile-time overhead. - -There are three ways to speed up compile times: - -1. Continue to use RAFT as a header-only library and create a CUDA source file - in your project to explicitly instantiate the templates which are slow to - compile. This can be tedious and will still require compiling the slow code - at least once, but it's the most flexible option if you are using types that - aren't already compiled into `libraft` - -2. If you are able to use one of the template types that are already being - compiled into `libraft`, you can use the pre-compiled template - instantiations, which are described in more detail in the following section. - -3. If you would like to use RAFT but either cannot or would prefer not to - compile any CUDA code yourself, you can simply add `libraft` to your link - libraries and use the growing set of `raft::runtime` APIs. - -### How do I verify template instantiations didn't compile into my binary? - -To verify that you are not accidentally instantiating templates that have not been pre-compiled in RAFT, set the `RAFT_EXPLICIT_INSTANTIATE_ONLY` macro. This only works if you are linking with the pre-compiled libraft (i.e., when `RAFT_COMPILED` has been defined). To check if, for instance, `raft::distance::distance` has been precompiled with specific template arguments, you can set `RAFT_EXPLICIT_INSTANTIATE_ONLY` at the top of the file you are compiling, as in the following example: - -```c++ - -#ifdef RAFT_COMPILED -#define RAFT_EXPLICIT_INSTANTIATE_ONLY -#endif - -#include -#include -#include - -int main() -{ - raft::resources handle{}; - - // Change IdxT to uint64_t and you will get an error because you are - // instantiating a template that has not been pre-compiled. - using IdxT = int; - - const float* x = nullptr; - const float* y = nullptr; - float* out = nullptr; - int m = 1024; - int n = 1024; - int k = 1024; - bool row_major = true; - raft::distance::distance( - handle, x, y, out, m, n, k, row_major, 2.0f); -} -``` - -## Runtime APIs - -RAFT contains a growing list of runtime APIs that, unlike the pre-compiled -template instantiations, allow you to link against `libraft` and invoke RAFT -directly from `cpp` files. The benefit to RAFT's runtime APIs is that they can -be used from code that is compiled with a `c++` compiler (rather than the CUDA -compiler `nvcc`). This enables the `runtime` APIs to power `pylibraft`. - diff --git a/docs/source/vector_search_tutorial.md b/docs/source/vector_search_tutorial.md deleted file mode 100644 index d1d5c57700..0000000000 --- a/docs/source/vector_search_tutorial.md +++ /dev/null @@ -1,409 +0,0 @@ -# Vector Search in C++ Tutorial - -## Table of Contents - -- [Step 1: Starting off with RAFT](#step-1-starting-off-with-raft) -- [Step 2: Generate some data](#step-2-generate-some-data) -- [Step 3: Using brute-force indexes](#step-3-using-brute-force-indexes) -- [Step 4: Using the ANN indexes](#step-4-using-the-ann-indexes) -- [Step 5: Evaluate neighborhood quality](#step-5-evaluate-neighborhood-quality) -- [Advanced Features](#advanced-features) - - [Serialization](#serialization) - - [Filtering](#filtering) - - [Stream Pools](#stream-pools) - - [Device Resources Manager](#device-resources-manager) - - [Device Memory Resources](#device-memory-resources) - - [Workspace Memory Resource](#workspace-memory-resource) - -RAFT has several important algorithms for performing vector search on the GPU and this tutorial walks through the primary vector search APIs from start to finish to provide a reference for quick setup and C++ API usage. - -This tutorial assumes RAFT has been installed and/or added to your build so that you are able to compile and run RAFT code. If not done already, please follow the [build and install instructions](build.md) and consider taking a look at the [example c++ template project](https://github.com/rapidsai/raft/tree/HEAD/cpp/template) for ready-to-go examples that you can immediately build and start playing with. Also take a look at RAFT's library of [reproducible vector search benchmarks](raft_ann_benchmarks.md) to run benchmarks that compare RAFT against other state-of-the-art nearest neighbors algorithms at scale. - -For more information about the various APIs demonstrated in this tutorial, along with comprehensive usage examples of all the APIs offered by RAFT, please refer to the [RAFT's C++ API Documentation](https://docs.rapids.ai/api/raft/nightly/cpp_api/). - -## Step 1: Starting off with RAFT - -### CUDA Development? - -If you are reading this tuturial then you probably know about CUDA and its relationship to general-purpose GPU computing (GPGPU). You probably also know about Nvidia GPUs but might not necessarily be familiar with the programming model nor GPU computing. The good news is that extensive knowledge of CUDA and GPUs are not needed in order to get started with or build applications with RAFT. RAFT hides away most of the complexities behind simple single-threaded stateless functions that are inherently asynchronous, meaning the result of a computation isn't necessarily read to be used when the function executes and control is given back to the user. The functions are, however, allowed to be chained together in a sequence of calls that don't need to wait for subsequent computations to complete in order to continue execution. In fact, the only time you need to wait for the computation to complete is when you are ready to use the result. - -A common structure you will encounter when using RAFT is a `raft::device_resources` object. This object is a container for important resources for a single GPU that might be needed during computation. If communicating with multiple GPUs, multiple `device_resources` might be needed, one for each GPU. `device_resources` contains several methods for managing its state but most commonly, you'll call the `sync_stream()` to guarantee all recently submitted computation has completed (as mentioned above.) - -A simple example of using `raft::device_resources` in RAFT: - -```c++ -#include - -raft::device_resources res; -// Call a bunch of RAFT functions in sequence... -res.sync_stream() -``` - -### Host vs Device Memory - -We differentiate between two different types of memory. `host` memory is your traditional RAM memory that is primarily accessible by applications on the CPU. `device` memory, on the other hand, is what we call the special memory on the GPU, which is not accessible from the CPU. In order to access host memory from the GPU, it needs to be explicitly copied to the GPU and in order to access device memory by the CPU, it needs to be explicitly copied there. We have several mechanisms available for allocating and managing the lifetime of device memory on the stack so that we don't need to explicitly allocate and free pointers on the heap. For example, instead of a `std::vector` for host memory, we can use `rmm::device_uvector` on the device. The following function will copy an array from host memory to device memory: - -```c++ -#include -#include -#include - -raft::device_resources res; - -std::vector my_host_vector = {0, 1, 2, 3, 4}; -rmm::device_uvector my_device_vector(my_host_vector.size(), res.get_stream()); - -raft::copy(my_device_vector.data(), my_host_vector.data(), my_host_vector.size(), res.get_stream()); -``` - -Since a stream is involved in the copy operation above, RAFT functions can be invoked immediately so long as the same `device_resources` instances is used (or, more specifically, the same main stream from the `devices_resources`.) As you might notice in the example above, `res.get_stream()` can be used to extract the main stream from a `device_resources` instance. - -### Multi-dimensional data representation - -`rmm::device_uvector` is a great mechanism for allocating and managing a chunk of device memory. While it's possible to use a single array to represent objects in higher dimensions like matrices, it lacks the means to pass that information along. For example, in addition to knowing that we have a 2d structure, we would need to know the number of rows, the number of columns, and even whether we read the columns or rows first (referred to as column- or row-major respectively). - -For this reason, RAFT relies on the `mdspan` standard, which was composed specifically for this purpose. To be even more, `mdspan` itself doesn't actually allocate or own any data on host or device because it's just a view over an existing memory on host device. The `mdspan` simply gives us a way to represent multi-dimensional data so we can pass along the needed metadata to our APIs. Even more powerful is that we can design functions that only accept a matrix of `float` in device memory that is laid out in row-major format. - -The memory-owning counterpart to the `mdspan` is the `mdarray` and the `mdarray` can allocate memory on device or host and carry along with it the metadata about its shape and layout. An `mdspan` can be produced from an `mdarray` for invoking RAFT APIs with `mdarray.view()`. They also follow similar paradigms to the STL, where we represent an immutable `mdspan` of `int` using `mdspan` instead of `const mdspan` to ensure it's the type carried along by the `mdspan` that's not allowed to change. - -Many RAFT functions require `mdspan` to represent immutable input data and there's no implicit conversion between `mdspan` and `mdspan` we use `raft::make_const_mdspan()` to alleviate the pain of constructing a new `mdspan` to invoke these functions. - -The following example demonstrates how to create `mdarray` matrices in both device and host memory, copy one to the other, and create mdspans out of them: - -```c++ -#include -#include -#include - -raft::device_resources res; - -int n_rows = 10; -int n_cols = 10; - -auto device_matrix = raft::make_device_matrix(res, n_rows, n_cols); -auto host_matrix = raft::make_host_matrix(res, n_rows, n_cols); - -// Set the diagonal to 1 -for(int i = 0; i < n_rows; i++) { - host_matrix(i, i) = 1; -} - -raft::copy(res, device_matrix.view(), host_matrix.view()); -``` - -## Step 2: Generate some data - -Let's build upon the fundamentals from the prior section and actually invoke some of RAFT's computational APIs on the device. A good starting point is data generation. - -```c++ -#include -#include - -raft::device_resources res; - -int n_rows = 10000; -int n_cols = 10000; - -auto dataset = raft::make_device_matrix(res, n_rows, n_cols); -auto labels = raft::make_device_vector(res, n_rows); - -raft::random::make_blobs(res, dataset.view(), labels.view()); -``` - -That's it. We've now generated a random 10kx10k matrix with points that cleanly separate into Gaussian clusters, along with a vector of cluster labels for each of the data points. Notice the `cuh` extension in the header file include for `make_blobs`. This signifies to us that this file contains CUDA device functions like kernel code so the CUDA compiler, `nvcc` is needed in order to compile any code that uses it. Generally, any source files that include headers with a `cuh` extension use the `.cu` extension instead of `.cpp`. The rule here is that `cpp` source files contain code which can be compiled with a C++ compiler like `g++` while `cu` files require the CUDA compiler. - -Since the `make_blobs` code generates the random dataset on the GPU device, we didn't need to do any host to device copies in this one. `make_blobs` is also asynchronous, so if we don't need to copy and use the data in host memory right away, we can continue calling RAFT functions with the `device_resources` instance and the data transformations will all be scheduled on the same stream. - -## Step 3: Using brute-force indexes - -### Build brute-force index - -Consider the `(10k, 10k)` shaped random matrix we generated in the previous step. We want to be able to find the k-nearest neighbors for all points of the matrix, or what we refer to as the all-neighbors graph, which means finding the neighbors of all data points within the same matrix. -```c++ -#include - -raft::device_resources res; - -// set number of neighbors to search for -int const k = 64; - -auto bfknn_index = raft::neighbors::brute_force::build(res, - raft::make_const_mdspan(dataset.view())); -``` - -### Query brute-force index - -```c++ - -// using matrix `dataset` from previous example -auto search = raft::make_const_mdspan(dataset.view()); - -// Indices and Distances are of dimensions (n, k) -// where n is number of rows in the search matrix -auto reference_indices = raft::make_device_matrix(res, search.extent(0), k); // stores index of neighbors -auto reference_distances = raft::make_device_matrix(res, search.extent(0), k); // stores distance to neighbors - -raft::neighbors::brute_force::search(res, - bfknn_index, - search, - reference_indices.view(), - reference_distances.view()); -``` - -We have established several things here by building a flat index. Now we know the exact 64 neighbors of all points in the matrix, and this algorithm can be generally useful in several ways: -1. Creating a baseline to compare against when building an approximate nearest neighbors index. -2. Directly using the brute-force algorithm when accuracy is more important than speed of computation. Don't worry, our implementation is still the best in-class and will provide not only significant speedups over other brute force methods, but also be quick relatively when the matrices are small! - - -## Step 4: Using the ANN indexes - -### Build a CAGRA index - -Next we'll train an ANN index. We'll use our graph-based CAGRA algorithm for this example but the other index types use a very similar pattern. - -```c++ -#include - -raft::device_resources res; - -// use default index parameters -raft::neighbors::cagra::index_params index_params; - -auto index = raft::neighbors::cagra::build(res, index_params, raft::make_const_mdspan(dataset.view())); -``` - -### Query the CAGRA index - -Now that we've trained a CAGRA index, we can query it by first allocating our output `mdarray` objects and passing the trained index model into the search function. - -```c++ -// create output arrays -auto indices = raft::make_device_matrix(res, n_rows, k); -auto distances = raft::make_device_matrix(res, n_rows, k); - -// use default search parameters -raft::neighbors::cagra::search_params search_params; - -// search K nearest neighbors -raft::neighbors::cagra::search( -res, search_params, index, search, indices.view(), distances.view()); -``` - -## Step 5: Evaluate neighborhood quality - -In step 3 we built a flat index and queried for exact neighbors while in step 4 we build an ANN index and queried for approximate neighbors. How do you quickly figure out the quality of our approximate neighbors and whether it's in an acceptable range based on your needs? Just compute the `neighborhood_recall` which gives a single value in the range [0, 1]. Closer the value to 1, higher the quality of the approximation. - -```c++ -#include - -raft::device_resources res; - -// Assuming matrices as type raft::device_matrix_view and variables as -// indices : approximate neighbor indices -// reference_indices : exact neighbor indices -// distances : approximate neighbor distances -// reference_distances : exact neighbor distances - -// We want our `neighborhood_recall` value in host memory -float const recall_scalar = 0.0; -auto recall_value = raft::make_host_scalar(recall_scalar); - -raft::stats::neighborhood_recall(res, - raft::make_const_mdspan(indices.view()), - raft::make_const_mdspan(reference_indices.view()), - recall_value.view(), - raft::make_const_mdspan(distances.view()), - raft::make_const_mdspan(reference_distances.view())); - -res.sync_stream(); -``` - -Notice we can run invoke the functions for index build and search for both algorithms, one right after the other, because we don't need to access any outputs from the algorithms in host memory. We will need to synchronize the stream on the `raft::device_resources` instance before we can read the result of the `neighborhood_recall` computation, though. - -Similar to a Numpy array, when we use a `host_scalar`, we are really using a multi-dimensional structure that contains only a single dimension, and further a single element. We can use element indexing to access the resulting element directly. -```c++ -std::cout << recall_value(0) << std::endl; -``` - -While it may seem like unnecessary additional work to wrap the result in a `host_scalar` mdspan, this API choice is made intentionally to support the possibility of also receiving the result as a `device_scalar` so that it can be used directly on the device for follow-on computations without having to incur the synchronization or transfer cost of bringing the result to host. This pattern becomes even more important when the result is being computed in a loop, such as an iterative solver, and the cost of synchronization and device-to-host (d2h) transfer becomes very expensive. - -## Advanced features - -The following sections present some advanced features that we have found can be useful for squeezing more utilization out of GPU hardware. As you've seen in this tutorial, RAFT provides several very useful tools and building blocks for developing accelerated applications beyond vector search capabilities. - -### Serialization - -Most of the indexes in `raft::neighbors` can be serialized to/from streams and files on disk. The index types that support this feature have include files with the naming convention `_serialize.cuh`. The serialization functions are similar across the different index types, with the primary difference being that some index types require a pointer to all the training data for search. Since the original training dataset can be quite large, the `serialize()` function for these index types includes an argument `include_dataset`, which allows the user to specify whether the dataset should be included in the serialized form. The index types that allow for this also include a method `update_datasets()` to allow for the dataset to be re-attached to the index after it is deserialized. - -The following example demonstrates serializing and deserializing a CAGRA index to and from a file. For index types that don't require the training data, you can remove the `include_dataset` and `update_dataset()` parts. We will assume the CAGRA index has been built using the code from [Step 4](#build-a-cagra-index) above: - -```c++ -#include -#include - -using namespace raft::neighbors; - -raft::neighbors::cagra::serialize(res, "cagra_serialized.dat", index, false); - -auto index_deser = raft::neighbors::cagra::deserialize(res, "cagra_serialized.dat"); -index_deser.update_dataset(dataset); -``` - -### Filtering - -As of RAFT 23.10, support for pre-filtering of neighbors has been added to ANN index. This search feature can enable multiple use-cases, such as filtering a vector based on it's attributes (hybrid searches), the removal of vectors already added to the index, or the control of access in searches for security purposes. -The filtering is available through the `search_with_filtering()` function of the ANN index, and is done by applying a predicate function on the GPU, which usually have the signature `(uint32_t query_ix, uint32_t sample_ix) -> bool`. - -One of the most commonly used mechanism for filtering is the bitset: the bitset is a data structure that allows to test the presence of a value in a set through a fast lookup, and is implemented as a bit array so that every element contains a `0` or a `1` (respectively `false` and `true` in boolean logic). RAFT provides a `raft::core::bitset` class that can be used to create and manipulate bitsets on the GPU, and a `raft::core::bitset_view` class that can be used to pass bitsets to filtering functions. - -The following example demonstrates how to use the filtering API (assume the CAGRA index is built using the code from [Step 4](#build-a-cagra-index) above: - -```c++ -#include -#include - -using namespace raft::neighbors; - -cagra::search_params search_params; - -// create a bitset to filter the search -auto removed_indices = raft::make_device_vector(res, n_removed_indices); -raft::core::bitset removed_indices_bitset( - res, removed_indices.view(), dataset.extent(0)); - -// ... Populate the bitset ... - -// search K nearest neighbours according to a bitset filter -auto neighbors = raft::make_device_matrix(res, n_queries, k); -auto distances = raft::make_device_matrix(res, n_queries, k); -cagra::search_with_filtering(res, search_params, index, queries, neighbors, distances, - filtering::bitset_filter(removed_indices_bitset.view())); -``` - -### Stream pools - -Within each CPU thread, CUDA uses `streams` to submit asynchronous work. You can think of a stream as a queue. Each stream can submit work to the GPU independently of other streams but work submitted within each stream is queued and executed in the order in which it was submitted. Similar to how we can use thread pools to bound the parallelism of CPU threads, we can use CUDA stream pools to bound the amount of concurrent asynchronous work that can be scheduled on a GPU. Each instance of `device_resources` has a main stream, but can also create a stream pool. For a single CPU thread, multiple different instances of `device_resources` can be created with different main streams and used to invoke a series of RAFT functions concurrently on the same or different GPU devices, so long as the target devices have available resources to perform the work. Once a device is saturated, queued work on streams will be scheduled and wait for a chance to do more work. During this time the streams are waiting, the CPU thread will still continue its own execution asynchronously unless `sync_stream_pool()` is called, causing the thread to block and wait for the thread pools to complete. - -Also, beware that before splitting GPU work onto multiple different concurrent streams, it can often be important to wait for the main stream in the `device_resources`. This can be done with `wait_stream_pool_on_stream()`. - -To summarize, if wanting to execute multiple different streams in parallel, we would often use a stream pool like this: -```c++ -#include - -#include -#include - -int n_streams = 5; - -rmm::cuda_stream stream; -std::shared_ptr stream_pool(5) -raft::device_resources res(stream.view(), stream_pool); - -// Submit some work on the main stream... - -res.wait_stream_pool_on_stream() -for(int i = 0; i < n_streams; ++i) { - rmm::cuda_stream_view stream_from_pool = res.get_next_usable_stream(); - raft::device_resources pool_res(stream_from_pool); - // Submit some work with pool_res... -} - -res.sync_stream_pool(); -``` - -### Device resources manager - -In multi-threaded applications, it is often useful to create a set of -`raft::device_resources` objects on startup to avoid the overhead of -re-initializing underlying resources every time a `raft::device_resources` object -is needed. To help simplify this common initialization logic, RAFT -provides a `raft::device_resources_manager` to handle this for downstream -applications. On startup, the application can specify certain limits on the -total resource consumption of the `raft::device_resources` objects that will be -generated: -```c++ -#include - -void initialize_application() { - // Set the total number of CUDA streams to use on each GPU across all CPU - // threads. If this method is not called, the default stream per thread - // will be used. - raft::device_resources_manager::set_streams_per_device(16); - - // Create a memory pool with given max size in bytes. Passing std::nullopt will allow - // the pool to grow to the available memory of the device. - raft::device_resources_manager::set_max_mem_pool_size(std::nullopt); - - // Set the initial size of the memory pool in bytes. - raft::device_resources_manager::set_init_mem_pool_size(16000000); - - // If neither of the above methods are called, no memory pool will be used -} -``` -While this example shows some commonly used settings, -`raft::device_resources_manager` provides support for several other -resource options and constraints, including options to initialize entire -stream pools that can be used by an individual `raft::device_resources` object. After -this initialization method is called, the following function could be called -from any CPU thread: -```c++ -void foo() { - raft::device_resources const& res = raft::device_resources_manager::get_device_resources(); - // Submit some work with res - res.sync_stream(); -} -``` - -If any `raft::device_resources_manager` setters are called _after_ the first -call to `raft::device_resources_manager::get_device_resources()`, these new -settings are ignored, and a warning will be logged. If a thread calls -`raft::device_resources_manager::get_device_resources()` multiple times, it is -guaranteed to access the same underlying `raft::device_resources` object every -time. This can be useful for chaining work in different calls on the same -thread without keeping a persistent reference to the resources object. - -### Device memory resources - -The RAPIDS software ecosystem makes heavy use of the [RAPIDS Memory Manager](https://github.com/rapidsai/rmm) (RMM) to enable zero-copy sharing of device memory across various GPU-enabled libraries such as PyTorch, Jax, Tensorflow, and FAISS. A really powerful feature of RMM is the ability to set a memory resource, such as a pooled memory resource that allocates a block of memory up front to speed up subsequent smaller allocations, and have all the libraries in the GPU ecosystem recognize and use that same memory resource for all of their memory allocations. - -As an example, the following code snippet creates a `pool_memory_resource` and sets it as the default memory resource, which means all other libraries that use RMM will now allocate their device memory from this same pool: -```c++ -#include - -rmm::mr::cuda_memory_resource cuda_mr; -// Construct a resource that uses a coalescing best-fit pool allocator -// set the initial size to half of the free device memory -auto init_size = rmm::percent_of_free_device_memory(50); -rmm::mr::pool_memory_resource pool_mr{&cuda_mr, init_size}; -rmm::mr::set_current_device_resource(&pool_mr); // Updates the current device resource pointer to `pool_mr` -``` - -The `raft::device_resources` object will now also use the `rmm::current_device_resource`. This isn't limited to C++, however. Often a user will be interacting with PyTorch, RAPIDS, or Tensorflow through Python and so they can set and use RMM's `current_device_resource` [right in Python](https://github.com/rapidsai/rmm#using-rmm-in-python-code). - -### Workspace memory resource - -As mentioned above, `raft::device_resources` will use `rmm::current_device_resource` by default for all memory allocations. However, there are times when a particular algorithm might benefit from using a different memory resource such as a `managed_memory_resource`, which creates a unified memory space between device and host memory, paging memory in and out of device as needed. Most of RAFT's algorithms allocate temporary memory as needed to perform their computations and we can control the memory resource used for these temporary allocations through the `workspace_resource` in the `raft::device_resources` instance. - -For some applications, the `managed_memory_resource`, can enable a memory space that is larger than the GPU, thus allowing a natural spilling to host memory when needed. This isn't always the best way to use managed memory, though, as it can quickly lead to thrashing and severely impact performance. Still, when it can be used, it provides a very powerful tool that can also avoid out of memory errors when enough host memory is available. - -The following creates a managed memory allocator and set it as the `workspace_resource` of the `raft::device_resources` instance: -```c++ -#include -#include - -std::shared_ptr managed_resource; -raft::device_resource res(managed_resource); -``` - -The `workspace_resource` uses an `rmm::mr::limiting_resource_adaptor`, which limits the total amount of allocation possible. This allows RAFT algorithms to work within the confines of the memory constraints imposed by the user so that things like batch sizes can be automatically set to reasonable values without exceeding the allotted memory. By default, this limit restricts the memory allocation space for temporary workspace buffers to the memory available on the device. - -The below example specifies the total number of bytes that RAFT can use for temporary workspace allocations to 3GB: -```c++ -#include -#include - -#include - -std::shared_ptr managed_resource; -raft::device_resource res(managed_resource, std::make_optional(3 * 1024^3)); -``` \ No newline at end of file diff --git a/docs/source/wiki_all_dataset.md b/docs/source/wiki_all_dataset.md deleted file mode 100644 index c001bdc409..0000000000 --- a/docs/source/wiki_all_dataset.md +++ /dev/null @@ -1,47 +0,0 @@ -# Wiki-all Dataset - -The `wiki-all` dataset was created to stress vector search algorithms at scale with both a large number of vectors and dimensions. The entire dataset contains 88M vectors with 768 dimensions and is meant for testing the types of vectors one would typically encounter in retrieval augmented generation (RAG) workloads. The full dataset is ~251GB in size, which is intentionally larger than the typical memory of GPUs. The massive scale is intended to promote the use of compression and efficient out-of-core methods for both indexing and search. - -The dataset is composed of English wiki texts from [Kaggle](https://www.kaggle.com/datasets/jjinho/wikipedia-20230701) and multi-lingual wiki texts from [Cohere Wikipedia](https://huggingface.co/datasets/Cohere/wikipedia-22-12). - -Cohere's English Texts are older (2022) and smaller than the Kaggle English Wiki texts (2023) so the English texts have been removed from Cohere completely. The final Wiki texts include English Wiki from Kaggle and the other languages from Cohere. The English texts constitute 50% of the total text size. - -To form the final dataset, the Wiki texts were chunked into 85 million 128-token pieces. For reference, Cohere chunks Wiki texts into 104-token pieces. Finally, the embeddings of each chunk were computed using the [paraphrase-multilingual-mpnet-base-v2](https://huggingface.co/sentence-transformers/paraphrase-multilingual-mpnet-base-v2) embedding model. The resulting dataset is an embedding matrix of size 88 million by 768. Also included with the dataset is a query file containing 10k query vectors and a groundtruth file to evaluate nearest neighbors algorithms. - -## Getting the dataset - -### Full dataset - -A version of the dataset is made available in the binary format that can be used directly by the [raft-ann-bench](https://docs.rapids.ai/api/raft/nightly/raft_ann_benchmarks/) tool. The full 88M dataset is ~251GB and the download link below contains tarballs that have been split into multiple parts. - -The following will download all 10 the parts and untar them to a `wiki_all_88M` directory: -```bash -curl -s https://data.rapids.ai/raft/datasets/wiki_all/wiki_all.tar.{00..9} | tar -xf - -C wiki_all_88M/ -``` - -The above has the unfortunate drawback that if the command should fail for any reason, all the parts need to be re-downloaded. The files can also be downloaded individually and then untarred to the directory. Each file is ~27GB and there are 10 of them. - -```bash -curl -s https://data.rapids.ai/raft/datasets/wiki_all/wiki_all.tar.00 -... -curl -s https://data.rapids.ai/raft/datasets/wiki_all/wiki_all.tar.09 - -cat wiki_all.tar.* | tar -xf - -C wiki_all_88M/ -``` - -### 1M and 10M subsets - -Also available are 1M and 10M subsets of the full dataset which are 2.9GB and 29GB, respectively. These subsets also include query sets of 10k vectors and corresponding groundtruth files. - -```bash -curl -s https://data.rapids.ai/raft/datasets/wiki_all_1M/wiki_all_1M.tar -curl -s https://data.rapids.ai/raft/datasets/wiki_all_10M/wiki_all_10M.tar -``` - -## Using the dataset - -After the dataset is downloaded and extracted to the `wiki_all_88M` directory (or `wiki_all_1M`/`wiki_all_10M` depending on whether the subsets are used), the files can be used in the benchmarking tool. The dataset name is `wiki_all` (or `wiki_all_1M`/`wiki_all_10M`), and the benchmarking tool can be used by specifying the appropriate name `--dataset wiki_all_88M` in the scripts. - -## License info - -The English wiki texts available on Kaggle come with the [CC BY-NCSA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/) license and the Cohere wikipedia data set comes with the [Apache 2.0](https://choosealicense.com/licenses/apache-2.0/) license. \ No newline at end of file diff --git a/notebooks/VectorSearch_QuestionRetrieval.ipynb b/notebooks/VectorSearch_QuestionRetrieval.ipynb deleted file mode 100644 index 33a2f60228..0000000000 --- a/notebooks/VectorSearch_QuestionRetrieval.ipynb +++ /dev/null @@ -1,632 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "f5499b54", - "metadata": {}, - "source": [ - "\n", - "# Similar Questions Retrieval\n", - "\n", - "This notebook is inspired by the [similar search example of Sentence-Transformers](https://www.sbert.net/examples/applications/semantic-search/README.html#similar-questions-retrieval), and adapted to support [RAFT ANN](https://github.com/rapidsai/raft) algorithm.\n", - "\n", - "The model was pre-trained on the [Natural Questions dataset](https://ai.google.com/research/NaturalQuestions). It consists of about 100k real Google search queries, together with an annotated passage from Wikipedia that provides the answer. It is an example of an asymmetric search task. As corpus, we use the smaller [Simple English Wikipedia](http://sbert.net/datasets/simplewiki-2020-11-01.jsonl.gz) so that it fits easily into memory.\n", - "\n", - "The steps to install the latest stable `pylibraft` package are available in the [documentation](https://docs.rapids.ai/api/raft/stable/build)." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "e8d55ede", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: sentence_transformers in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (2.2.2)\n", - "Requirement already satisfied: torch in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (2.0.1)\n", - "Requirement already satisfied: transformers<5.0.0,>=4.6.0 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (4.31.0)\n", - "Requirement already satisfied: tqdm in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (4.65.0)\n", - "Requirement already satisfied: torchvision in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (0.15.2)\n", - "Requirement already satisfied: numpy in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (1.24.4)\n", - "Requirement already satisfied: scikit-learn in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (1.3.0)\n", - "Requirement already satisfied: scipy in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (1.11.1)\n", - "Requirement already satisfied: nltk in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (3.8.1)\n", - "Requirement already satisfied: sentencepiece in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (0.1.99)\n", - "Requirement already satisfied: huggingface-hub>=0.4.0 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sentence_transformers) (0.16.4)\n", - "Requirement already satisfied: filelock in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (3.12.2)\n", - "Requirement already satisfied: typing-extensions in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (4.7.1)\n", - "Requirement already satisfied: sympy in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (1.12)\n", - "Requirement already satisfied: networkx in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (3.1)\n", - "Requirement already satisfied: jinja2 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (3.1.2)\n", - "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (11.7.99)\n", - "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (11.7.99)\n", - "Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (11.7.101)\n", - "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (8.5.0.96)\n", - "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (11.10.3.66)\n", - "Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (10.9.0.58)\n", - "Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (10.2.10.91)\n", - "Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (11.4.0.1)\n", - "Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (11.7.4.91)\n", - "Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (2.14.3)\n", - "Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (11.7.91)\n", - "Requirement already satisfied: triton==2.0.0 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torch) (2.0.0)\n", - "Requirement already satisfied: setuptools in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch) (68.0.0)\n", - "Requirement already satisfied: wheel in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch) (0.41.0)\n", - "Requirement already satisfied: cmake in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from triton==2.0.0->torch) (3.27.0)\n", - "Requirement already satisfied: lit in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from triton==2.0.0->torch) (16.0.6)\n", - "Requirement already satisfied: fsspec in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence_transformers) (2023.6.0)\n", - "Requirement already satisfied: requests in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence_transformers) (2.31.0)\n", - "Requirement already satisfied: pyyaml>=5.1 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence_transformers) (6.0)\n", - "Requirement already satisfied: packaging>=20.9 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence_transformers) (23.1)\n", - "Requirement already satisfied: regex!=2019.12.17 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (2023.6.3)\n", - "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (0.13.3)\n", - "Requirement already satisfied: safetensors>=0.3.1 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (0.3.1)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from jinja2->torch) (2.1.3)\n", - "Requirement already satisfied: click in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from nltk->sentence_transformers) (8.1.6)\n", - "Requirement already satisfied: joblib in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from nltk->sentence_transformers) (1.3.0)\n", - "Requirement already satisfied: threadpoolctl>=2.0.0 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from scikit-learn->sentence_transformers) (3.2.0)\n", - "Requirement already satisfied: mpmath>=0.19 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from sympy->torch) (1.3.0)\n", - "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from torchvision->sentence_transformers) (10.0.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (3.2.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (3.4)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (2.0.4)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /raid/danteg/miniconda3/envs/raft-ann/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (2023.7.22)\n" - ] - } - ], - "source": [ - "!pip install sentence_transformers torch\n", - "\n", - "# Note: if you have a Hopper based GPU, like an H100, use these to install:\n", - "# pip install torch --index-url https://download.pytorch.org/whl/cu118\n", - "# pip install sentence_transformers" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "eb1e81c3", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Jul 31 14:35:31 2023 \n", - "+-----------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 525.105.17 Driver Version: 525.105.17 CUDA Version: 12.0 |\n", - "|-------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|===============================+======================+======================|\n", - "| 0 NVIDIA H100 80G... On | 00000000:1B:00.0 Off | 0 |\n", - "| N/A 30C P0 75W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 1 NVIDIA H100 80G... On | 00000000:43:00.0 Off | 0 |\n", - "| N/A 31C P0 72W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 2 NVIDIA H100 80G... On | 00000000:52:00.0 Off | 0 |\n", - "| N/A 34C P0 70W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 3 NVIDIA H100 80G... On | 00000000:61:00.0 Off | 0 |\n", - "| N/A 33C P0 70W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 4 NVIDIA H100 80G... On | 00000000:9D:00.0 Off | 0 |\n", - "| N/A 32C P0 74W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 5 NVIDIA H100 80G... On | 00000000:C3:00.0 Off | 0 |\n", - "| N/A 30C P0 73W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 6 NVIDIA H100 80G... On | 00000000:D1:00.0 Off | 0 |\n", - "| N/A 33C P0 73W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 7 NVIDIA H100 80G... On | 00000000:DF:00.0 Off | 0 |\n", - "| N/A 35C P0 73W / 700W | 0MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-------------------------------+----------------------+----------------------+\n", - " \n", - "+-----------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=============================================================================|\n", - "| No running processes found |\n", - "+-----------------------------------------------------------------------------+\n" - ] - } - ], - "source": [ - "!nvidia-smi" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "ee4c5cc0", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/raid/danteg/miniconda3/envs/raftann/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "import json\n", - "from sentence_transformers import SentenceTransformer, CrossEncoder, util\n", - "import time\n", - "import gzip\n", - "import os\n", - "import torch\n", - "import pylibraft\n", - "from pylibraft.neighbors import ivf_flat, ivf_pq\n", - "pylibraft.config.set_output_as(lambda device_ndarray: device_ndarray.copy_to_host())\n", - "\n", - "if not torch.cuda.is_available():\n", - " print(\"Warning: No GPU found. Please add GPU to your notebook\")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "0a1a6307", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Passages: 509663\n" - ] - } - ], - "source": [ - "# We use the Bi-Encoder to encode all passages, so that we can use it with semantic search\n", - "model_name = 'nq-distilbert-base-v1'\n", - "bi_encoder = SentenceTransformer(model_name)\n", - "\n", - "# As dataset, we use Simple English Wikipedia. Compared to the full English wikipedia, it has only\n", - "# about 170k articles. We split these articles into paragraphs and encode them with the bi-encoder\n", - "\n", - "wikipedia_filepath = 'data/simplewiki-2020-11-01.jsonl.gz'\n", - "\n", - "if not os.path.exists(wikipedia_filepath):\n", - " util.http_get('http://sbert.net/datasets/simplewiki-2020-11-01.jsonl.gz', wikipedia_filepath)\n", - "\n", - "passages = []\n", - "with gzip.open(wikipedia_filepath, 'rt', encoding='utf8') as fIn:\n", - " for line in fIn:\n", - " data = json.loads(line.strip())\n", - " for paragraph in data['paragraphs']:\n", - " # We encode the passages as [title, text]\n", - " passages.append([data['title'], paragraph])\n", - "\n", - "# If you like, you can also limit the number of passages you want to use\n", - "print(\"Passages:\", len(passages))\n", - "\n", - "# To speed things up, pre-computed embeddings are downloaded.\n", - "# The provided file encoded the passages with the model 'nq-distilbert-base-v1'\n", - "if model_name == 'nq-distilbert-base-v1':\n", - " embeddings_filepath = 'simplewiki-2020-11-01-nq-distilbert-base-v1.pt'\n", - " if not os.path.exists(embeddings_filepath):\n", - " util.http_get('http://sbert.net/datasets/simplewiki-2020-11-01-nq-distilbert-base-v1.pt', embeddings_filepath)\n", - "\n", - " corpus_embeddings = torch.load(embeddings_filepath)\n", - " corpus_embeddings = corpus_embeddings.float() # Convert embedding file to float\n", - " if torch.cuda.is_available():\n", - " corpus_embeddings = corpus_embeddings.to('cuda')\n", - "else: # Here, we compute the corpus_embeddings from scratch (which can take a while depending on the GPU)\n", - " corpus_embeddings = bi_encoder.encode(passages, convert_to_tensor=True, show_progress_bar=True)" - ] - }, - { - "cell_type": "markdown", - "id": "1f4e9b9d", - "metadata": {}, - "source": [ - "# Vector Search using RAPIDS RAFT\n", - "Now that our embeddings are ready to be indexed and that the model has been loaded, we can use RAPIDS RAFT to do our vector search.\n", - "\n", - "This is done in two step: First we build the index, then we search it.\n", - "With `pylibraft` all you need is those four Python lines:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "ad90b4be", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[W] [14:35:48.810785] [raft::ivf_pq::build] the default cuda resource is used for the raft workspace allocations. This may lead to a significant slowdown for this algorithm. Consider using the default pool resource (`raft::resource::set_workspace_to_pool_resource`) or set your own resource explicitly (`raft::resource::set_workspace_resource`).\n", - "[W] [14:35:53.831753] [raft::ivf_pq::extend] the default cuda resource is used for the raft workspace allocations. This may lead to a significant slowdown for this algorithm. Consider using the default pool resource (`raft::resource::set_workspace_to_pool_resource`) or set your own resource explicitly (`raft::resource::set_workspace_resource`).\n", - "CPU times: user 2.21 s, sys: 2.49 s, total: 4.7 s\n", - "Wall time: 5.13 s\n" - ] - } - ], - "source": [ - "%%time\n", - "params = ivf_pq.IndexParams(n_lists=150, pq_dim=96)\n", - "pq_index = ivf_pq.build(params, corpus_embeddings)\n", - "search_params = ivf_pq.SearchParams()\n", - "\n", - "def search_raft_pq(query, top_k = 5):\n", - " # Encode the query using the bi-encoder and find potentially relevant passages\n", - " question_embedding = bi_encoder.encode(query, convert_to_tensor=True)\n", - "\n", - " hits = ivf_pq.search(search_params, pq_index, question_embedding[None], top_k)\n", - "\n", - " # Output of top-k hits\n", - " print(\"Input question:\", query)\n", - " for k in range(top_k):\n", - " print(\"\\t{:.3f}\\t{}\".format(hits[0][0, k], passages[hits[1][0, k]]))" - ] - }, - { - "cell_type": "markdown", - "id": "07935bca", - "metadata": {}, - "source": [ - "For IVF-PQ we want to reduce the memory footprint while keeping a good accuracy." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "724dcacb", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "IVF-PQ memory footprint: 373.3 MB\n", - "Original dataset: 1493.2 MB\n", - "Memory saved: 75.0%\n" - ] - } - ], - "source": [ - "pq_index_mem = pq_index.pq_dim * pq_index.size * pq_index.pq_bits\n", - "print(\"IVF-PQ memory footprint: {:.1f} MB\".format(pq_index_mem / 2**20))\n", - "\n", - "original_mem = corpus_embeddings.shape[0] * corpus_embeddings.shape[1] * 4\n", - "print(\"Original dataset: {:.1f} MB\".format(original_mem / 2**20))\n", - "\n", - "print(\"Memory saved: {:.1f}%\".format(100 * (1 - pq_index_mem / original_mem)))" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "c27d4715", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[W] [14:36:07.640223] [raft::ivf_pq::search] the default cuda resource is used for the raft workspace allocations. This may lead to a significant slowdown for this algorithm. Consider using the default pool resource (`raft::resource::set_workspace_to_pool_resource`) or set your own resource explicitly (`raft::resource::set_workspace_resource`).\n", - "Input question: Who was Grace Hopper?\n", - "\t190.855\t['Leona Helmsley', 'Leona Helmsley (July 4, 1920 – August 20, 2007) was an American businesswoman. She was known for having a flamboyant personality. She had a reputation for tyrannical behavior; she was nicknamed the Queen of Mean.']\n", - "\t195.364\t['Grace Hopper', 'Hopper was born in New York, USA. Hopper graduated from Vassar College in 1928 and Yale University in 1934 with a Ph.D degree in mathematics. She joined the US Navy during the World War II in 1943. She worked on computers in the Navy for 43 years. She then worked in other private industry companies after 1949. She retired from the Navy in 1986 and died on January 1, 1992.']\n", - "\t202.536\t['Anita Borg', 'Anita Borg (January 17, 1949 – April 6, 2003) was an American computer scientist. She founded the Institute for Women and Technology and the Grace Hopper Celebration of Women in Computing.']\n", - "\t203.717\t['Brett Butler', 'Brett Butler (born January 30, 1958) is an American actress and stand-up comedian. She is best known for playing Grace in the sitcom \"Grace Under Fire\". She has also done other television programs and comedy acts.']\n", - "\t203.991\t['Nellie Bly', 'Elizabeth Cochrane Seaman (born Elizabeth Jane Cochran; May 5, 1864 – January 27, 1922), better known by her pen name Nellie Bly, was an American journalist, novelist and inventor. She was a newspaper reporter, who worked at various jobs for exposing poor working conditions. Nellie Bly, also, fought for women\\'s right and was known for investigative reporting. She best known for her record-breaking trip around the world in 72 days, inspired by the adventure novel \"Around the World in Eighty Days\" by Jules Verne. In the 1880s, she went undercover as a mentally ill patient in a psychiatric hospital for ten days, with the report being made public in a book called \"\"Ten Days in a Mad-House\"\". She was added to the National Women\\'s Hall of Fame in 1998.']\n", - "CPU times: user 98.3 ms, sys: 81.2 ms, total: 180 ms\n", - "Wall time: 120 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "search_raft_pq(query=\"Who was Grace Hopper?\")" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "bc375518", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Input question: Who was Alan Turing?\n", - "\t139.827\t['Alan Turing', 'Alan Mathison Turing OBE FRS (London, 23 June 1912 – Wilmslow, Cheshire, 7 June 1954) was an English mathematician and computer scientist. He was born in Maida Vale, London.']\n", - "\t169.849\t['William Kahan', 'William Morton Kahan (born June 5, 1933) is a Canadian mathematician and computer scientist. He received the Turing Award in 1989 for \"\"his fundamental contributions to numerical analysis\".\" He was named an ACM Fellow in 1994, and added to the National Academy of Engineering in 2005.']\n", - "\t177.520\t['Rolf Noskwith', 'Rolf Noskwith (19 June 1919 – 3 January 2017) was a British businessman. During the Second World War, he worked under Alan Turing as a cryptographer at the British military base Bletchley Park in Milton Keynes, Buckinghamshire.']\n", - "\t179.202\t['Marvin Minsky', \"Marvin Lee Minsky (August 9, 1927 – January 24, 2016) was an American cognitive scientist in the field of artificial intelligence (AI). He was the co-founder of the Massachusetts Institute of Technology's AI laboratory, and author of several texts on AI and philosophy. He won the Turing Award in 1969.\"]\n", - "\t179.819\t['Edsger W. Dijkstra', 'Edsger Wybe Dijkstra (May 11, 1930 – August 6, 2002; ) was a Dutch computer scientist. He received the 1972 Turing Award for fundamental contributions to developing programming languages, and was the Schlumberger Centennial Chair of Computer Sciences at The University of Texas at Austin from 1984 until 2000.']\n", - "CPU times: user 4.89 ms, sys: 7.52 ms, total: 12.4 ms\n", - "Wall time: 12 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "search_raft_pq(query=\"Who was Alan Turing?\")" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "ab154181", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Input question: What is creating tides?\n", - "\t125.037\t['Tide', \"A tide is the periodic rising and falling of Earth's ocean surface caused mainly by the gravitational pull of the Moon acting on the oceans. Tides cause changes in the depth of marine and estuarine (river mouth) waters. Tides also make oscillating currents known as tidal streams (~'rip tides'). This means that being able to predict the tide is important for coastal navigation. The strip of seashore that is under water at high tide and exposed at low tide, called the intertidal zone, is an important ecological product of ocean tides.\"]\n", - "\t163.835\t['Tidal energy', \"Many things affect tides. The pull of the Moon is the largest effect, and most of the energy comes from the slowing of the Earth's spin.\"]\n", - "\t167.368\t['Storm surge', 'A storm surge is a sudden rise of water hitting areas close to the coast. Storm surges are usually created by a hurricane or other tropical cyclone. The surge happens because a storm has fast winds and low atmospheric pressure. Water is pushed on shore, and the water level rises. Strong storm surges can flood coastal towns and destroy homes. A storm surge is considered the deadliest part of a hurricane. They kill many people each year.']\n", - "\t177.143\t['Tidal force', 'Tidal force is caused by gravity and makes tides happen. This is because the gravitational field changes across the middle of a body (the diameter).']\n", - "\t186.108\t['Tsunami', \"A tsunami is a natural disaster which is a series of fast-moving waves in the ocean caused by powerful earthquakes, volcanic eruptions, landslides, or simply an asteroid or a meteor crash inside the ocean. A tsunami has a very long wavelength. It can be hundreds of kilometers long. Usually, a tsunami starts suddenly. The waves travel at a great speed across an ocean with little energy loss. They can remove sand from beaches, destroy trees, toss and drag vehicles, houses and even destroy whole towns. Tsunamis can even be caused when a meteorite strikes the earth's surface, though it is very rare. A tsunami normally occurs in the Pacific Ocean, especially in what is called the ring of fire, but can occur in any large body of water.\"]\n", - "CPU times: user 4.44 ms, sys: 4.65 ms, total: 9.09 ms\n", - "Wall time: 12.4 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "search_raft_pq(query = \"What is creating tides?\")" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "2d6017ed", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 208 ms, sys: 63.8 ms, total: 271 ms\n", - "Wall time: 286 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "params = ivf_flat.IndexParams(n_lists=150)\n", - "flat_index = ivf_flat.build(params, corpus_embeddings)\n", - "search_params = ivf_flat.SearchParams()\n", - "\n", - "def search_raft_flat(query, top_k = 5):\n", - " # Encode the query using the bi-encoder and find potentially relevant passages\n", - " question_embedding = bi_encoder.encode(query, convert_to_tensor=True)\n", - " \n", - " start_time = time.time()\n", - " hits = ivf_flat.search(search_params, flat_index, question_embedding[None], top_k)\n", - " end_time = time.time()\n", - "\n", - " # Output of top-k hits\n", - " print(\"Input question:\", query)\n", - " print(\"Results (after {:.3f} seconds):\".format(end_time - start_time))\n", - " for k in range(top_k):\n", - " print(\"\\t{:.3f}\\t{}\".format(hits[0][0, k], passages[hits[1][0, k]]))" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "f5cfb644", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Input question: Who was Grace Hopper?\n", - "Results (after 0.002 seconds):\n", - "\t181.650\t['Grace Hopper', 'Hopper was born in New York, USA. Hopper graduated from Vassar College in 1928 and Yale University in 1934 with a Ph.D degree in mathematics. She joined the US Navy during the World War II in 1943. She worked on computers in the Navy for 43 years. She then worked in other private industry companies after 1949. She retired from the Navy in 1986 and died on January 1, 1992.']\n", - "\t192.946\t['Leona Helmsley', 'Leona Helmsley (July 4, 1920 – August 20, 2007) was an American businesswoman. She was known for having a flamboyant personality. She had a reputation for tyrannical behavior; she was nicknamed the Queen of Mean.']\n", - "\t194.951\t['Grace Hopper', 'Grace Murray Hopper (December 9 1906 – January 1 1992) was an American computer scientist and United States Navy officer.']\n", - "\t202.192\t['Nellie Bly', 'Elizabeth Cochrane Seaman (born Elizabeth Jane Cochran; May 5, 1864 – January 27, 1922), better known by her pen name Nellie Bly, was an American journalist, novelist and inventor. She was a newspaper reporter, who worked at various jobs for exposing poor working conditions. Nellie Bly, also, fought for women\\'s right and was known for investigative reporting. She best known for her record-breaking trip around the world in 72 days, inspired by the adventure novel \"Around the World in Eighty Days\" by Jules Verne. In the 1880s, she went undercover as a mentally ill patient in a psychiatric hospital for ten days, with the report being made public in a book called \"\"Ten Days in a Mad-House\"\". She was added to the National Women\\'s Hall of Fame in 1998.']\n", - "\t205.038\t['Abbie Hoffman', 'Abbot Howard \"Abbie\" Hoffman (November 30, 1936 – April 12, 1989) was an American social and political activist.']\n", - "CPU times: user 6.48 ms, sys: 0 ns, total: 6.48 ms\n", - "Wall time: 6.22 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "search_raft_flat(query=\"Who was Grace Hopper?\")" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "b5694d00", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Input question: Who was Alan Turing?\n", - "Results (after 0.002 seconds):\n", - "\t106.131\t['Alan Turing', 'Alan Mathison Turing OBE FRS (London, 23 June 1912 – Wilmslow, Cheshire, 7 June 1954) was an English mathematician and computer scientist. He was born in Maida Vale, London.']\n", - "\t158.646\t['William Kahan', 'William Morton Kahan (born June 5, 1933) is a Canadian mathematician and computer scientist. He received the Turing Award in 1989 for \"\"his fundamental contributions to numerical analysis\".\" He was named an ACM Fellow in 1994, and added to the National Academy of Engineering in 2005.']\n", - "\t165.094\t['Alan Turing', 'A brilliant mathematician and cryptographer Alan was to become the founder of modern-day computer science and artificial intelligence; designing a machine at Bletchley Park to break secret Enigma encrypted messages used by the Nazi German war machine to protect sensitive commercial, diplomatic and military communications during World War 2. Thus, Turing made the single biggest contribution to the Allied victory in the war against Nazi Germany, possibly saving the lives of an estimated 2 million people, through his effort in shortening World War II.']\n", - "\t167.321\t['Rolf Noskwith', 'Rolf Noskwith (19 June 1919 – 3 January 2017) was a British businessman. During the Second World War, he worked under Alan Turing as a cryptographer at the British military base Bletchley Park in Milton Keynes, Buckinghamshire.']\n", - "\t176.480\t['Marvin Minsky', \"Marvin Lee Minsky (August 9, 1927 – January 24, 2016) was an American cognitive scientist in the field of artificial intelligence (AI). He was the co-founder of the Massachusetts Institute of Technology's AI laboratory, and author of several texts on AI and philosophy. He won the Turing Award in 1969.\"]\n", - "CPU times: user 4.81 ms, sys: 1.19 ms, total: 6 ms\n", - "Wall time: 6.06 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "search_raft_flat(query=\"Who was Alan Turing?\")" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "fcfc3c5b", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Input question: What is creating tides?\n", - "Results (after 0.002 seconds):\n", - "\t94.909\t['Tide', \"A tide is the periodic rising and falling of Earth's ocean surface caused mainly by the gravitational pull of the Moon acting on the oceans. Tides cause changes in the depth of marine and estuarine (river mouth) waters. Tides also make oscillating currents known as tidal streams (~'rip tides'). This means that being able to predict the tide is important for coastal navigation. The strip of seashore that is under water at high tide and exposed at low tide, called the intertidal zone, is an important ecological product of ocean tides.\"]\n", - "\t159.539\t['Tidal energy', \"Many things affect tides. The pull of the Moon is the largest effect, and most of the energy comes from the slowing of the Earth's spin.\"]\n", - "\t159.740\t['Storm surge', 'A storm surge is a sudden rise of water hitting areas close to the coast. Storm surges are usually created by a hurricane or other tropical cyclone. The surge happens because a storm has fast winds and low atmospheric pressure. Water is pushed on shore, and the water level rises. Strong storm surges can flood coastal towns and destroy homes. A storm surge is considered the deadliest part of a hurricane. They kill many people each year.']\n", - "\t178.283\t['Sea', 'Wind blowing over the surface of a body of water forms waves. The friction between air and water caused by a gentle breeze on a pond causes ripples to form. A strong blow over the ocean causes larger waves as the moving air pushes against the raised ridges of water. The waves reach their greatest height when the rate at which they travel nearly matches the speed of the wind. The waves form at right angles to the direction from which the wind blows. In open water, if the wind continues to blow, as happens in the Roaring Forties in the southern hemisphere, long, organized masses of water called swell roll across the ocean. If the wind dies down, the wave formation is reduced but waves already formed continue to travel in their original direction until they meet land. Small waves form in small areas of water with islands and other landmasses but large waves form in open stretches of sea where the wind blows steadily and strongly. When waves meet other waves coming from different directions, interference between the two can produce broken, irregular seas.']\n", - "\t181.498\t['Tidal force', 'Tidal force is caused by gravity and makes tides happen. This is because the gravitational field changes across the middle of a body (the diameter).']\n", - "CPU times: user 5.91 ms, sys: 0 ns, total: 5.91 ms\n", - "Wall time: 5.65 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "search_raft_flat(query = \"What is creating tides?\")" - ] - }, - { - "cell_type": "markdown", - "id": "a59d7b32-0832-4c3a-864e-aeb2e6e7fe1f", - "metadata": {}, - "source": [ - "## Using CAGRA: GPU graph-based Vector Search\n", - "\n", - "CAGRA is a graph-based nearest neighbors implementation with state-of-the art query performance for both small- and large-batch sized vector searches. \n", - "\n", - "CAGRA follows the same two-step APIs as IVF-FLAT and IVF-PQ in RAFT. First we build the index:" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "50df1f43-c580-4019-949a-06bdc7185536", - "metadata": {}, - "outputs": [], - "source": [ - "from pylibraft.neighbors import cagra" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "091cde52-4652-4230-af2b-75c35357f833", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 35.3 s, sys: 4.5 s, total: 39.8 s\n", - "Wall time: 2.16 s\n" - ] - } - ], - "source": [ - "%%time\n", - "params = cagra.IndexParams(intermediate_graph_degree=32, graph_degree=16, build_algo=\"nn_descent\")\n", - "cagra_index = cagra.build(params, corpus_embeddings)\n", - "search_params = cagra.SearchParams(algo=\"multi_cta\")" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "df229e21-f6b6-4d6c-ad54-2724f8738934", - "metadata": {}, - "outputs": [], - "source": [ - "def search_raft_cagra(query, top_k = 5):\n", - " # Encode the query using the bi-encoder and find potentially relevant passages\n", - " question_embedding = bi_encoder.encode(query, convert_to_tensor=True)\n", - "\n", - " start_time = time.time()\n", - " hits = cagra.search(search_params, cagra_index, question_embedding[None], top_k)\n", - " end_time = time.time()\n", - "\n", - " # Output of top-k hits\n", - " print(\"Results (after {:.3f} seconds):\".format(end_time - start_time))\n", - " print(\"Input question:\", query)\n", - " for k in range(top_k):\n", - " print(\"\\t{:.3f}\\t{}\".format(hits[0][0, k], passages[hits[1][0, k]]))" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "b5e862fd-b7e5-4423-8fbf-36918f02c8f3", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Results (after 0.005 seconds):\n", - "Input question: Who was Grace Hopper?\n", - "\t181.649\t['Grace Hopper', 'Hopper was born in New York, USA. Hopper graduated from Vassar College in 1928 and Yale University in 1934 with a Ph.D degree in mathematics. She joined the US Navy during the World War II in 1943. She worked on computers in the Navy for 43 years. She then worked in other private industry companies after 1949. She retired from the Navy in 1986 and died on January 1, 1992.']\n", - "\t192.946\t['Leona Helmsley', 'Leona Helmsley (July 4, 1920 – August 20, 2007) was an American businesswoman. She was known for having a flamboyant personality. She had a reputation for tyrannical behavior; she was nicknamed the Queen of Mean.']\n", - "\t194.951\t['Grace Hopper', 'Grace Murray Hopper (December 9 1906 – January 1 1992) was an American computer scientist and United States Navy officer.']\n", - "\t202.192\t['Nellie Bly', 'Elizabeth Cochrane Seaman (born Elizabeth Jane Cochran; May 5, 1864 – January 27, 1922), better known by her pen name Nellie Bly, was an American journalist, novelist and inventor. She was a newspaper reporter, who worked at various jobs for exposing poor working conditions. Nellie Bly, also, fought for women\\'s right and was known for investigative reporting. She best known for her record-breaking trip around the world in 72 days, inspired by the adventure novel \"Around the World in Eighty Days\" by Jules Verne. In the 1880s, she went undercover as a mentally ill patient in a psychiatric hospital for ten days, with the report being made public in a book called \"\"Ten Days in a Mad-House\"\". She was added to the National Women\\'s Hall of Fame in 1998.']\n", - "\t205.038\t['Abbie Hoffman', 'Abbot Howard \"Abbie\" Hoffman (November 30, 1936 – April 12, 1989) was an American social and political activist.']\n", - "CPU times: user 4.18 ms, sys: 3.88 ms, total: 8.07 ms\n", - "Wall time: 9.97 ms\n" - ] - } - ], - "source": [ - "%%time \n", - "search_raft_cagra(query=\"Who was Grace Hopper?\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/ivf_flat_example.ipynb b/notebooks/ivf_flat_example.ipynb deleted file mode 100644 index 08b9d78169..0000000000 --- a/notebooks/ivf_flat_example.ipynb +++ /dev/null @@ -1,674 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4f49c5c4-1170-42a7-9d6a-b90acd00c3c3", - "metadata": {}, - "source": [ - "# RAFT IVF Flat Example Notebook" - ] - }, - { - "cell_type": "markdown", - "id": "4bcfe810-f120-422c-b2bb-72cc43d0c4ca", - "metadata": {}, - "source": [ - "## Introduction\n", - "\n", - "This notebook demonstrates how to run approximate nearest neighbor search using RAFT IVF-Flat algorithm.\n", - "It builds and searches an index using a dataset from the ann-benchmarks million-scale datasets, saves/loads the index to disk, and explores important parameters for fine-tuning the search performance and accuracy of the index." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "fe73ada7-7b7f-4005-9440-85428194311b", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import cupy as cp\n", - "import numpy as np\n", - "from pylibraft.common import DeviceResources\n", - "from pylibraft.neighbors import ivf_flat\n", - "import matplotlib.pyplot as plt\n", - "import tempfile\n", - "from utils import BenchmarkTimer, calc_recall, load_dataset" - ] - }, - { - "cell_type": "markdown", - "id": "da9e8615-ea9f-4735-b70f-15ccab36c0d9", - "metadata": {}, - "source": [ - "For best performance it is recommended to use an RMM pooling allocator, to minimize the overheads of repeated CUDA allocations." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "5350e4d9-0993-406a-80af-29538b5677c2", - "metadata": {}, - "outputs": [], - "source": [ - "import rmm\n", - "from rmm.allocators.cupy import rmm_cupy_allocator\n", - "mr = rmm.mr.PoolMemoryResource(\n", - " rmm.mr.CudaMemoryResource(),\n", - " initial_pool_size=2**30\n", - ")\n", - "rmm.mr.set_current_device_resource(mr)\n", - "cp.cuda.set_allocator(rmm_cupy_allocator)" - ] - }, - { - "cell_type": "markdown", - "id": "b0d935f2-ba24-44fc-bdfe-a769b7fcd8e6", - "metadata": {}, - "source": [ - "The following GPU is used for this notebook" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "a5daa4b4-96de-4e74-bfd6-505b13595f62", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Thu Sep 21 02:30:53 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.104.05 Driver Version: 535.104.05 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 NVIDIA H100 PCIe On | 00000000:41:00.0 Off | 0 |\n", - "| N/A 35C P0 69W / 350W | 1487MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "| 0 N/A N/A 3940 C /opt/conda/envs/rapids/bin/python 1474MiB |\n", - "+---------------------------------------------------------------------------------------+\n" - ] - } - ], - "source": [ - "# Report the GPU in use\n", - "!nvidia-smi" - ] - }, - { - "cell_type": "markdown", - "id": "88a654cc-6389-4526-a3e6-826de5606a09", - "metadata": {}, - "source": [ - "## Load dataset\n", - "\n", - "The ANN benchmarks website provides the datasets in HDF5 format.\n", - "\n", - "The list of prepared datasets can be found at https://github.com/erikbern/ann-benchmarks/#data-sets" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "5f529ad6-b0bd-495c-bf7c-43f10fb6aa14", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The index and data will be saved in /tmp/raft_example\n" - ] - } - ], - "source": [ - "WORK_FOLDER = os.path.join(tempfile.gettempdir(), \"raft_example\")\n", - "f = load_dataset(\"http://ann-benchmarks.com/sift-128-euclidean.hdf5\", work_folder=WORK_FOLDER)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "3d68a7db-bcf4-449c-96c3-1e8ab146c84d", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded dataset of size (1000000, 128), 0.5 GiB; metric: 'euclidean'.\n", - "Number of test queries: 10000\n" - ] - } - ], - "source": [ - "metric = f.attrs['distance']\n", - "\n", - "dataset = cp.array(f['train'])\n", - "queries = cp.array(f['test'])\n", - "gt_neighbors = cp.array(f['neighbors'])\n", - "gt_distances = cp.array(f['distances'])\n", - "\n", - "itemsize = dataset.dtype.itemsize \n", - "\n", - "print(f\"Loaded dataset of size {dataset.shape}, {dataset.size*itemsize/(1<<30):4.1f} GiB; metric: '{metric}'.\")\n", - "print(f\"Number of test queries: {queries.shape[0]}\")" - ] - }, - { - "cell_type": "markdown", - "id": "9f463c50-d1d3-49be-bcfe-952602efa603", - "metadata": {}, - "source": [ - "## Build index\n", - "We set [IndexParams](https://docs.rapids.ai/api/raft/nightly/pylibraft_api/neighbors/#pylibraft.neighbors.ivf_flat.IndexParams) and build the index. The index parameters will be discussed in more detail in later sections of this notebook." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "737f8841-93f9-4c8e-b2e1-787d4474ef94", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 120 ms, sys: 5.33 ms, total: 125 ms\n", - "Wall time: 124 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "build_params = ivf_flat.IndexParams(\n", - " n_lists=1024,\n", - " metric=\"euclidean\",\n", - " kmeans_trainset_fraction=0.1,\n", - " kmeans_n_iters=20,\n", - " add_data_on_build=True\n", - " )\n", - "\n", - "index = ivf_flat.build(build_params, dataset)" - ] - }, - { - "cell_type": "markdown", - "id": "a16a0cf6-3b05-4afd-9bb8-54431e0d7439", - "metadata": {}, - "source": [ - "The index is built. We can print some basic information of the index" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "1aec7024-6e5d-4d2c-82e6-7b5734aec958", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Index(type=IVF-FLAT, metric=euclidean, size=1000000, dim=128, n_lists=1024, adaptive_centers=False)\n" - ] - } - ], - "source": [ - "print(index)" - ] - }, - { - "cell_type": "markdown", - "id": "df7d4958-56a3-48ea-bd64-3486fdb57fb7", - "metadata": {}, - "source": [ - "## Search neighbors" - ] - }, - { - "cell_type": "markdown", - "id": "89ba2eaa-4c85-4e1c-b07c-920394e55dce", - "metadata": {}, - "source": [ - "It is recommended to reuse [device recosources](https://docs.rapids.ai/api/raft/nightly/pylibraft_api/common/#pylibraft.common.DeviceResources) across multiple invocations of search, since constructing these can be time consuming. We will reuse the resources by passing the same handle to each RAFT API call." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "46e0421b-9335-47a2-8451-a91f56c2f086", - "metadata": {}, - "outputs": [], - "source": [ - "handle = DeviceResources()" - ] - }, - { - "cell_type": "markdown", - "id": "a6365229-18fd-468f-af30-e24b950cbd6e", - "metadata": {}, - "source": [ - "After setting [SearchParams](https://docs.rapids.ai/api/raft/nightly/pylibraft_api/neighbors/#pylibraft.neighbors.ivf_flat.SearchParams) we search for for `k=10` neighbors." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "595454e1-7240-4b43-9a73-963d5670b00c", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 171 ms, sys: 52.6 ms, total: 224 ms\n", - "Wall time: 236 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "n_queries=10000\n", - "# n_probes is the number of clusters we select in the first (coarse) search step. This is the only hyper parameter for search.\n", - "search_params = ivf_flat.SearchParams(n_probes=30)\n", - "\n", - "# Search 10 nearest neighbors.\n", - "distances, indices = ivf_flat.search(search_params, index, cp.asarray(queries[:n_queries,:]), k=10, handle=handle)\n", - " \n", - "# RAFT calls are asynchronous (when handle arg is provided), we need to sync before accessing the results.\n", - "handle.sync()\n", - "distances, neighbors = cp.asnumpy(distances), cp.asnumpy(indices)" - ] - }, - { - "cell_type": "markdown", - "id": "43d20ca7-7b9e-4046-bb52-640a2744db75", - "metadata": {}, - "source": [ - "The returned arrays have shape {n_queries x 10] and store the distance values and the indices of the searched vectors. We check how accurate the search is. The accuracy of the search is quantified as `recall`, which is a value between 0 and 1 and tells us what fraction of the returned neighbors are actual k nearest neighbors. " - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "8cd9cd20-ca00-4a35-a0a0-86636521b31a", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.97406" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "calc_recall(neighbors, gt_neighbors)" - ] - }, - { - "cell_type": "markdown", - "id": "cde5079c-9777-45a1-9545-cffbcc59988f", - "metadata": {}, - "source": [ - "## Save and load the index\n", - "You can serialize the index to file using [save](https://docs.rapids.ai/api/raft/nightly/pylibraft_api/neighbors/#pylibraft.neighbors.ivf_flat.save), and [load](https://docs.rapids.ai/api/raft/nightly/pylibraft_api/neighbors/#pylibraft.neighbors.ivf_flat.load) it later." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "bf94e45c-e7fb-4aa3-a611-ddaee7ac41ae", - "metadata": {}, - "outputs": [], - "source": [ - "index_file = os.path.join(WORK_FOLDER, \"my_ivf_flat_index.bin\")\n", - "ivf_flat.save(index_file, index)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "1622d9be-be41-4d25-be99-d348c5e54957", - "metadata": {}, - "outputs": [], - "source": [ - "index = ivf_flat.load(index_file)" - ] - }, - { - "cell_type": "markdown", - "id": "15d503e5-05e8-47ce-8501-e13fc512099c", - "metadata": {}, - "source": [ - "## Tune search parameters\n", - "Search has a single hyper parameter: `n_probes`, which describes how many neighboring cluster is searched (probed) for each query. Within a probed cluster, the distance is computed between all the vectors in the cluster and the query point, and the top-k neighbors are selected. Finally, the top-k neighbors are selected from all the neighbor candidates from the probed clusters.\n", - "\n", - "Let's see how search accuracy and latency changes when we change the `n_probes` parameter." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "ace0c31f-af75-4352-a438-123a9a03612c", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Benchmarking search with n_probes = 10\n", - "recall 0.86625\n", - "Average search time: 0.026 +/- 0.000259 s\n", - "Queries per second (QPS): 384968\n", - "\n", - "Benchmarking search with n_probes = 20\n", - "recall 0.94705\n", - "Average search time: 0.050 +/- 5.43e-05 s\n", - "Queries per second (QPS): 198880\n", - "\n", - "Benchmarking search with n_probes = 30\n", - "recall 0.97406\n", - "Average search time: 0.075 +/- 8.59e-05 s\n", - "Queries per second (QPS): 133954\n", - "\n", - "Benchmarking search with n_probes = 50\n", - "recall 0.99169\n", - "Average search time: 0.123 +/- 4.78e-05 s\n", - "Queries per second (QPS): 80997\n", - "\n", - "Benchmarking search with n_probes = 100\n", - "recall 0.99844\n", - "Average search time: 0.244 +/- 0.000249 s\n", - "Queries per second (QPS): 40934\n", - "\n", - "Benchmarking search with n_probes = 200\n", - "recall 0.99932\n", - "Average search time: 0.468 +/- 0.000367 s\n", - "Queries per second (QPS): 21382\n", - "\n", - "Benchmarking search with n_probes = 500\n", - "recall 0.99933\n", - "Average search time: 1.039 +/- 0.000209 s\n", - "Queries per second (QPS): 9625\n", - "\n", - "Benchmarking search with n_probes = 1024\n", - "recall 0.99935\n", - "Average search time: 0.701 +/- 0.00579 s\n", - "Queries per second (QPS): 14273\n" - ] - } - ], - "source": [ - "n_probes = np.asarray([10, 20, 30, 50, 100, 200, 500, 1024]);\n", - "qps = np.zeros(n_probes.shape);\n", - "recall = np.zeros(n_probes.shape);\n", - "\n", - "for i in range(len(n_probes)):\n", - " print(\"\\nBenchmarking search with n_probes =\", n_probes[i])\n", - " timer = BenchmarkTimer(reps=1, warmup=1)\n", - " for rep in timer.benchmark_runs():\n", - " distances, neighbors = ivf_flat.search(\n", - " ivf_flat.SearchParams(n_probes=n_probes[i]),\n", - " index,\n", - " cp.asarray(queries),\n", - " k=10,\n", - " handle=handle,\n", - " )\n", - " handle.sync()\n", - " \n", - " recall[i] = calc_recall(cp.asnumpy(neighbors), gt_neighbors)\n", - " print(\"recall\", recall[i])\n", - "\n", - " timings = np.asarray(timer.timings)\n", - " avg_time = timings.mean()\n", - " std_time = timings.std()\n", - " qps[i] = queries.shape[0] / avg_time\n", - " print(\"Average search time: {0:7.3f} +/- {1:7.3} s\".format(avg_time, std_time))\n", - " print(\"Queries per second (QPS): {0:8.0f}\".format(qps[i]))" - ] - }, - { - "cell_type": "markdown", - "id": "20b2498c-7231-4211-990e-600d5c26a9a1", - "metadata": {}, - "source": [ - "The plots below illustrate how the accuracy (recall) and the throughput (queries per second) depends on the `n_probes` parameter." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e1ac370f-91c8-4054-95c7-a749df5f16d2", - "metadata": {}, - "outputs": [], - "source": [ - "fig = plt.figure(figsize=(12,3))\n", - "ax = fig.add_subplot(131)\n", - "ax.plot(n_probes, recall,'o-')\n", - "#ax.set_xticks(bench_k, bench_k)\n", - "ax.set_xlabel('n_probes')\n", - "ax.grid()\n", - "ax.set_ylabel('recall (@k=10)')\n", - "\n", - "ax = fig.add_subplot(132)\n", - "ax.plot(n_probes, qps,'o-')\n", - "#ax.set_xticks(bench_k, bench_k)\n", - "ax.set_xlabel('n_probes')\n", - "ax.grid()\n", - "ax.set_ylabel('queries per second');\n", - "\n", - "ax = fig.add_subplot(133)\n", - "ax.plot(recall, qps,'o-')\n", - "#ax.set_xticks(bench_k, bench_k)\n", - "ax.set_xlabel('recall')\n", - "ax.grid()\n", - "ax.set_ylabel('queries per second');\n", - "#ax.set_yscale('log')" - ] - }, - { - "cell_type": "markdown", - "id": "81e7ad6a-bddc-45de-9cce-0fb913f91efe", - "metadata": {}, - "source": [ - "## Adjust build parameters\n", - "### n_lists\n", - "The number of clusters (or lists) is set by the n_list parameter. Let's change it to 100 clusters." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "addbfff3-7773-4290-9608-5489edf4886d", - "metadata": {}, - "outputs": [], - "source": [ - "%%time\n", - "build_params = ivf_flat.IndexParams(\n", - " n_lists=100,\n", - " metric=\"euclidean\",\n", - " kmeans_trainset_fraction=1,\n", - " kmeans_n_iters=20,\n", - " add_data_on_build=True\n", - " )\n", - "\n", - "index = ivf_flat.build(build_params, dataset, handle=handle)" - ] - }, - { - "cell_type": "markdown", - "id": "48db27f9-54c8-4dac-839b-af94ada8885f", - "metadata": {}, - "source": [ - "The ratio of n_probes / n_list will determine how large fraction of the dataset is searched for each query. The right combination depends on the use case. Here we will search 10 of the clusters for each query." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8a0149ad-de38-4195-97a5-ce5d5d877036", - "metadata": {}, - "outputs": [], - "source": [ - "%%time\n", - "n_queries=10000\n", - "\n", - "search_params = ivf_flat.SearchParams(n_probes=10)\n", - "\n", - "# Search 10 nearest neighbors.\n", - "distances, indices = ivf_flat.search(search_params, index, cp.asarray(queries[:n_queries,:]), k=10, handle=handle)\n", - " \n", - "handle.sync()\n", - "distances, neighbors = cp.asnumpy(distances), cp.asnumpy(indices)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "eedc3ec4-06af-42c5-8cdf-490a5c2bc49a", - "metadata": {}, - "outputs": [], - "source": [ - "calc_recall(neighbors, gt_neighbors)" - ] - }, - { - "cell_type": "markdown", - "id": "0c44800f-1e9e-4f7b-87fe-0f25e6590faa", - "metadata": {}, - "source": [ - "### trainset_fraction\n", - "During clustering we can sub-sample the dataset. The parameter `trainset_fraction` determines what fraction to use. Often we get good results by using only 1/10th of the dataset for clustering. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5a54d190-64d4-4cd4-a497-365cbffda871", - "metadata": {}, - "outputs": [], - "source": [ - "%%time\n", - "build_params = ivf_flat.IndexParams( \n", - " n_lists=100, \n", - " metric=\"sqeuclidean\", \n", - " kmeans_trainset_fraction=0.1, \n", - " kmeans_n_iters=20 \n", - " ) \n", - "index = ivf_flat.build(build_params, dataset, handle=handle)" - ] - }, - { - "cell_type": "markdown", - "id": "9d86a213-d6ae-4fca-9082-cb5a4d1dab36", - "metadata": {}, - "source": [ - "We see only a minimal change in the recall" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4cc992e8-a5e5-4508-b790-0e934160b660", - "metadata": {}, - "outputs": [], - "source": [ - "search_params = ivf_flat.SearchParams(n_probes=10)\n", - "\n", - "distances, indices = ivf_flat.search(search_params, index, cp.asarray(queries[:n_queries,:]), k=10, handle=handle)\n", - " \n", - "handle.sync()\n", - "distances, neighbors = cp.asnumpy(distances), cp.asnumpy(indices)\n", - "calc_recall(neighbors, gt_neighbors)" - ] - }, - { - "cell_type": "markdown", - "id": "25289ebc-7d89-4fa6-bc62-e25b6e77750c", - "metadata": {}, - "source": [ - "### Add vectors on build\n", - "Currently you cannot configure how RAFT sub-samples the input. If you want to have a fine control on how the training set is selected, then create the index in two steps:\n", - "1. Define cluster centers on a training set, but do not add any vector to the index\n", - "2. Add vectors to the index (extend)\n", - "\n", - "This workflow shall be familiar to FAISS users. Note that raft does not require adding the data in batches, internal batching is used when necessary.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7ebcf970-94ed-4825-9885-277bd984b90c", - "metadata": {}, - "outputs": [], - "source": [ - "# subsample the dataset\n", - "n_train = 10000\n", - "train_set = dataset[cp.random.choice(dataset.shape[0], n_train, replace=False),:]\n", - "\n", - "# build using training set\n", - "build_params = ivf_flat.IndexParams(\n", - " n_lists=1024,\n", - " metric=\"sqeuclidean\",\n", - " kmeans_trainset_fraction=1,\n", - " kmeans_n_iters=20,\n", - " add_data_on_build=False\n", - " )\n", - "index = ivf_flat.build(build_params, train_set)\n", - "\n", - "print(\"Index before adding vectors\", index)\n", - "\n", - "ivf_flat.extend(index, dataset, cp.arange(dataset.shape[0], dtype=cp.int64))\n", - "\n", - "print(\"Index after adding vectors\", index)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "029d48a9-baf7-4263-af43-9e500ef3cce4", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/tutorial_ivf_pq.ipynb b/notebooks/tutorial_ivf_pq.ipynb deleted file mode 100644 index 397e39bfba..0000000000 --- a/notebooks/tutorial_ivf_pq.ipynb +++ /dev/null @@ -1,1365 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# RAFT IVF-PQ tutorial\n", - "In this tutorial you will learn to build IVF-PQ index and use it to search approximate nearest neighbors (ANN).\n", - "We will start with a brief overview of the functionality, but then dive into details to gain the understanding of the model parameters.\n", - "Along the way, we will benchmark the model and give some practical recommendations on how to maximize its performance for various use cases.\n", - "\n", - "This tutorial uses the data from [ANN benchmarks website](https://ann-benchmarks.com)." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting adjustText\n", - " Downloading adjustText-0.8-py3-none-any.whl (9.1 kB)\n", - "Collecting h5py\n", - " Downloading h5py-3.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.8 MB)\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.8/4.8 MB\u001b[0m \u001b[31m46.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n", - "\u001b[?25hCollecting matplotlib\n", - " Downloading matplotlib-3.7.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.6 MB)\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.6/11.6 MB\u001b[0m \u001b[31m97.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: numpy in /opt/conda/envs/cuml_dev/lib/python3.9/site-packages (from adjustText) (1.24.4)\n", - "Collecting contourpy>=1.0.1 (from matplotlib)\n", - " Downloading contourpy-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (300 kB)\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m300.4/300.4 kB\u001b[0m \u001b[31m86.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting cycler>=0.10 (from matplotlib)\n", - " Downloading cycler-0.11.0-py3-none-any.whl (6.4 kB)\n", - "Collecting fonttools>=4.22.0 (from matplotlib)\n", - " Downloading fonttools-4.41.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.5 MB)\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.5/4.5 MB\u001b[0m \u001b[31m115.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m\n", - "\u001b[?25hCollecting kiwisolver>=1.0.1 (from matplotlib)\n", - " Downloading kiwisolver-1.4.4-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.6 MB)\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m119.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /opt/conda/envs/cuml_dev/lib/python3.9/site-packages (from matplotlib) (23.1)\n", - "Requirement already satisfied: pillow>=6.2.0 in /opt/conda/envs/cuml_dev/lib/python3.9/site-packages (from matplotlib) (10.0.0)\n", - "Collecting pyparsing<3.1,>=2.3.1 (from matplotlib)\n", - " Downloading pyparsing-3.0.9-py3-none-any.whl (98 kB)\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m98.3/98.3 kB\u001b[0m \u001b[31m43.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: python-dateutil>=2.7 in /opt/conda/envs/cuml_dev/lib/python3.9/site-packages (from matplotlib) (2.8.2)\n", - "Collecting importlib-resources>=3.2.0 (from matplotlib)\n", - " Downloading importlib_resources-6.0.0-py3-none-any.whl (31 kB)\n", - "Requirement already satisfied: zipp>=3.1.0 in /opt/conda/envs/cuml_dev/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib) (3.15.0)\n", - "Requirement already satisfied: six>=1.5 in /opt/conda/envs/cuml_dev/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n", - "Installing collected packages: pyparsing, kiwisolver, importlib-resources, h5py, fonttools, cycler, contourpy, matplotlib, adjustText\n", - "Successfully installed adjustText-0.8 contourpy-1.1.0 cycler-0.11.0 fonttools-4.41.1 h5py-3.9.0 importlib-resources-6.0.0 kiwisolver-1.4.4 matplotlib-3.7.2 pyparsing-3.0.9\n" - ] - } - ], - "source": [ - "!pip install adjustText h5py matplotlib" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import tempfile\n", - "import cupy as cp\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "import rmm\n", - "import urllib.request\n", - "import h5py\n", - "\n", - "from rmm.allocators.cupy import rmm_cupy_allocator\n", - "from pylibraft.common import DeviceResources\n", - "from pylibraft.neighbors import ivf_pq, refine\n", - "from adjustText import adjust_text\n", - "from utils import calc_recall, load_dataset\n", - "\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# A clumsy helper for inspecting properties of an object\n", - "def show_properties(obj):\n", - " return {\n", - " attr: getattr(obj, attr)\n", - " for attr in dir(obj)\n", - " if type(getattr(type(obj), attr)).__name__ == 'getset_descriptor'\n", - " }" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The index and data will be saved in /tmp/raft_ivf_pq_tutorial\n" - ] - } - ], - "source": [ - "# We'll need to load store some data in this tutorial\n", - "WORK_FOLDER = os.path.join(tempfile.gettempdir(), 'raft_ivf_pq_tutorial')\n", - "\n", - "if not os.path.exists(WORK_FOLDER):\n", - " os.makedirs(WORK_FOLDER)\n", - "print(\"The index and data will be saved in\", WORK_FOLDER)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fri Jul 28 08:21:25 2023 \n", - "+---------------------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 535.49 Driver Version: 535.49 CUDA Version: 12.2 |\n", - "|-----------------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|=========================================+======================+======================|\n", - "| 0 NVIDIA H100 PCIe On | 00000000:41:00.0 Off | 0 |\n", - "| N/A 34C P0 46W / 350W | 4MiB / 81559MiB | 0% Default |\n", - "| | | Disabled |\n", - "+-----------------------------------------+----------------------+----------------------+\n", - " \n", - "+---------------------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=======================================================================================|\n", - "| No running processes found |\n", - "+---------------------------------------------------------------------------------------+\n" - ] - } - ], - "source": [ - "# Report the GPU in use to put the measurements into perspective\n", - "!nvidia-smi" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Use the pool memory resource\n", - "RAFT uses RMM allocator widely across its algorithms, including the performance-sensitive parts like IVF-PQ search.\n", - "It's strongly advised to set up the RMM pool memory resource to minimize the overheads of repeated CUDA allocations.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "pool = rmm.mr.PoolMemoryResource(\n", - " rmm.mr.CudaMemoryResource(),\n", - " initial_pool_size=2**30\n", - ")\n", - "rmm.mr.set_current_device_resource(pool)\n", - "cp.cuda.set_allocator(rmm_cupy_allocator)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Get the data\n", - "The [ANN benchmarks website](https://ann-benchmarks.com) provides the datasets in [HDF5 format](https://www.hdfgroup.org/solutions/hdf5/).\n", - "\n", - "The list of prepared datasets can be found at https://github.com/erikbern/ann-benchmarks/#data-sets" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The index and data will be saved in /tmp/raft_example\n" - ] - } - ], - "source": [ - "DATASET_URL = \"http://ann-benchmarks.com/sift-128-euclidean.hdf5\"\n", - "f = load_dataset(DATASET_URL)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Load the dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded dataset of size (1000000, 128); metric: 'euclidean'.\n", - "Number of test queries: 10000\n" - ] - } - ], - "source": [ - "metric = f.attrs['distance']\n", - "\n", - "dataset = cp.array(f['train'])\n", - "queries = cp.array(f['test'])\n", - "gt_neighbors = cp.array(f['neighbors'])\n", - "gt_distances = cp.array(f['distances'])\n", - "\n", - "print(f\"Loaded dataset of size {dataset.shape}; metric: '{metric}'.\")\n", - "print(f\"Number of test queries: {queries.shape[0]}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Build the index\n", - "Construction of the index generally consists of two phases: training (building the clusters) and filling-in (extending the index with data).\n", - "In the first phase, a balanced hierarchical k-means algorithm clusters the training data.\n", - "In the second phase, the new data is classified and added into the appropriate clusters in the index.\n", - "Hence, a user should call `ivf_pq.build` once and then possibly `ivf_pq.extend` several times.\n", - "Though for user convenience `ivf_pq.build` by default adds the whole training set into the index." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "# RAFT's DeviceResources controls the GPU, cuda stream, memory policies etc.\n", - "# For now, we just create a default instance.\n", - "resources = DeviceResources()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'add_data_on_build': True,\n", - " 'codebook_kind': 0,\n", - " 'conservative_memory_allocation': False,\n", - " 'force_random_rotation': False,\n", - " 'kmeans_n_iters': 20,\n", - " 'kmeans_trainset_fraction': 0.5,\n", - " 'metric': 1,\n", - " 'n_lists': 1024,\n", - " 'pq_bits': 8,\n", - " 'pq_dim': 64}" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# First, we need to initialize the build/indexing parameters.\n", - "# One of the more important parameters is the product quantisation (PQ) dim.\n", - "# Effectively, this parameter says\n", - "# \"shrink the dataset to this dimensionality to reduce the index size\".\n", - "# It must be not bigger than the dataset dim,\n", - "# and it should be divisible by 32 for better GPU performance.\n", - "pq_dim = 1\n", - "while pq_dim * 2 < dataset.shape[1]:\n", - " pq_dim = pq_dim * 2\n", - "# We'll use the ANN-benchmarks-provided metric and sensible defaults for the rest of parameters.\n", - "index_params = ivf_pq.IndexParams(n_lists=1024, metric=metric, pq_dim=pq_dim)\n", - "\n", - "show_properties(index_params)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1.71 s, sys: 16.2 ms, total: 1.72 s\n", - "Wall time: 1.71 s\n" - ] - }, - { - "data": { - "text/plain": [ - "Index(type=IVF-PQ, metric=euclidean, codebook=subspace, size=1000000, dim=128, pq_dim=64, pq_bits=8, n_lists=1024, rot_dim=128)" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "## Build the index\n", - "# This function takes a row-major either numpy or cupy (GPU) array.\n", - "# Generally, it's a bit faster with GPU inputs, but the CPU version may come in handy\n", - "# if the whole dataset cannot fit into GPU memory.\n", - "index = ivf_pq.build(index_params, dataset, handle=resources)\n", - "# This function is asynchronous so we need to explicitly synchronize the GPU before we can measure the execution time\n", - "resources.sync()\n", - "index" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Index serialization\n", - "For bigger datasets, building an index can take some time. To avoid building the index from scratch every time you need it, you can save it to a file. Here is how this works:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 89.7 ms, sys: 56 ms, total: 146 ms\n", - "Wall time: 145 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "Index(type=IVF-PQ, metric=euclidean, codebook=subspace, size=1000000, dim=128, pq_dim=64, pq_bits=8, n_lists=1024, rot_dim=128)" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "index_filepath = os.path.join(WORK_FOLDER, \"ivf_pq.bin\")\n", - "ivf_pq.save(index_filepath, index) \n", - "loaded_index = ivf_pq.load(index_filepath)\n", - "resources.sync()\n", - "index" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Search\n", - "The search function returns the requested number `k` of (approximate) nearest neighbor in no particular order.\n", - "Besides the queries and `k`, the function can take a few more parameters to tweak the performance of the algorithm.\n", - "Again, these are passed via the struct with some sensible defaults." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'internal_distance_dtype': 0, 'lut_dtype': 0, 'n_probes': 20}" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "k = 10\n", - "search_params = ivf_pq.SearchParams()\n", - "show_properties(search_params)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 19.9 ms, sys: 12.3 ms, total: 32.2 ms\n", - "Wall time: 31.5 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "distances, neighbors = ivf_pq.search(search_params, index, queries, k, handle=resources)\n", - "# Sync the GPU to make sure we've got the timing right\n", - "resources.sync()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Measuring the quality of the predictions\n", - "We use [recall](https://en.wikipedia.org/wiki/Precision_and_recall) to measure the quality of the prediction." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Got recall = 0.85409 with the default parameters (k = 10).\n" - ] - } - ], - "source": [ - "recall_first_try = calc_recall(neighbors, gt_neighbors)\n", - "print(f\"Got recall = {recall_first_try} with the default parameters (k = {k}).\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Refine\n", - "Let's improve our results a little bit!\n", - "The refinement operation follows an approximate NN search.\n", - "It recomputes the exact distances for the already selected candidates and selects a subset of them thus improving the recall." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 193 ms, sys: 142 µs, total: 193 ms\n", - "Wall time: 191 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "candidates = ivf_pq.search(search_params, index, queries, k * 2, handle=resources)[1]\n", - "distances, neighbors = refine(dataset, queries, candidates, k, handle=resources)\n", - "resources.sync()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Got recall = 0.94953 with 2x refinement (k = 10).\n" - ] - } - ], - "source": [ - "recall_refine2x = calc_recall(neighbors, gt_neighbors)\n", - "print(f\"Got recall = {recall_refine2x} with 2x refinement (k = {k}).\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tweaking search parameters\n", - "Before diving deep into tweaking the model, let's quickly define the performance metrics.\n", - "As we've mentioned earlier, we use the recall to measure the quality of prediction.\n", - "The other important metric is the speed of the search.\n", - "We measure the speed in terms of queries per second (QPS).\n", - "\n", - "Most of the time, by changing the model parameters we balance the trade-off between the QPS and the recall." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Number of neighbors\n", - "Let's see how QPS depens on `k`. " - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "16.5 ms ± 13.3 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "17 ms ± 2.12 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "17.5 ms ± 2.92 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "18 ms ± 3.05 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "18.7 ms ± 4.25 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "23.4 ms ± 45.5 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "25.9 ms ± 5.49 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "40.2 ms ± 12.7 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "23.6 ms ± 26.6 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "28.7 ms ± 18.5 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA00AAAGwCAYAAAB1kI7CAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAACENklEQVR4nOzdeViVdf7/8ec5h305ICAisoi7uIsbZqalUpHVZGXlmGOmo1+tUWZabEzLFsvfpDVl2ao1ZYsz7ZaKlpqJqSju+4aKgAubIHCA8/sDPRPjdizgZnk9rotLz7k/575fvD0gb+7787lNdrvdjoiIiIiIiFyU2egAIiIiIiIiNZmaJhERERERkctQ0yQiIiIiInIZappEREREREQuQ02TiIiIiIjIZahpEhERERERuQw1TSIiIiIiIpfhYnSA+qSsrIy0tDR8fX0xmUxGxxERERERqdfsdjt5eXmEhoZiNl/6fJKapmqUlpZGeHi40TFERERERORXjhw5QlhY2CW3q2mqRr6+vkD5P4rVajUsh81mY+nSpQwaNAhXV1fDctQGqpXzVCvnqVbOU62cp1pdHdXLeaqV81Qr59WUWuXm5hIeHu74Of1S1DRVo/OX5FmtVsObJi8vL6xWq76gr0C1cp5q5TzVynmqlfNUq6ujejlPtXKeauW8mlarK02d0UIQIiIiIiIil6GmSURERERE5DLUNImIiIiIiFyGmiYREREREZHLUNMkIiIiIiJyGWqaRERERERELkNNk4iIiIiIyGWoaRIREREREbkMNU0iIiIiIiKXoaZJRERERETkMtQ0iYiIiIiIXIaaJhERERERkctQ0yQiIiIiInIZLkYHkOp16GQ+j/9nM9ZiM757T9K9WRC+Hq5GxxIRERERqbHUNNUzq/edZO3BLMDM0g82YjZBmxAr3Zs2oFvTALo1bUBjP0+jY4qIiIiI1BhqmuqZ69sE8+xt0Xy1ZhvpJd4cyTrLjuO57Diey/tJhwFo4u9ZoYlqFeyL2WwyOLmIiIiIiDEMn9N07Ngx/vjHPxIYGIinpycdOnRgw4YNju12u52pU6fSuHFjPD09GTBgAHv37q2wj9OnTzNs2DCsViv+/v6MGjWKM2fOVBizZcsWrr32Wjw8PAgPD2fmzJkXZFm4cCFt2rTBw8ODDh068N1331XY7kyWmi7U35Oh3cL4Y4syfki4ll+euIE593Vl5DVN6dDED7MJjmWf5cuUNKZ8uY0bX/6JztOXMnLeOub8uI9fDpyi0FZq9KchIiIiIlJtDD3TlJWVxTXXXEP//v35/vvvadiwIXv37qVBgwaOMTNnzuSf//wn77//PlFRUTz55JPExcWxY8cOPDw8ABg2bBjHjx8nMTERm83GyJEjGTNmDAsWLAAgNzeXQYMGMWDAAObOncvWrVt54IEH8Pf3Z8yYMQCsWbOGe++9lxkzZnDLLbewYMECbr/9djZu3Ej79u2dzlLbNLJ6EN+xMfEdGwNwpqiElNRs1h86TfLhLDamZpFbWMKPu0/w4+4TALhaTHRo4kf3pgF0axpATGQDArzdjPw0RERERESqjKFN04svvkh4eDjz5s1zPBcVFeX4u91u5+WXX2bKlCncdtttAHzwwQc0atSIL7/8knvuuYedO3eyePFi1q9fT7du3QB49dVXufnmm/nHP/5BaGgoH330EcXFxbz33nu4ubnRrl07UlJSmDVrlqNpeuWVV7jxxht55JFHAHjmmWdITEzktddeY+7cuU5lqQt83F3o0zKIPi2DACgpLWPn8TzWHzrNhsOnWX8oixN5RWxMzWZjajZvrjoAQPOG3o4mqnvTBkQEeGEy6ZI+EREREan9DG2avv76a+Li4rjrrrtYuXIlTZo04f/+7/8YPXo0AAcPHiQ9PZ0BAwY4XuPn50fPnj1JSkrinnvuISkpCX9/f0fDBDBgwADMZjO//PILf/jDH0hKSqJv3764uf33bEhcXBwvvvgiWVlZNGjQgKSkJBISEirki4uL48svv3Q6y/8qKiqiqKjI8Tg3NxcAm82GzWb7HZX7fc4f29kMbRp50aaRF8N7hmG32zmSdZbkw9kkp2ax4XA2+0/kOz4+WX8EgCAfN2Ii/ImJbEC3SH/ahvjiYjH8atCrdrW1qs9UK+epVs5TrZynWl0d1ct5qpXzVCvn1ZRaOXt8Q5umAwcO8MYbb5CQkMATTzzB+vXrefjhh3Fzc2PEiBGkp6cD0KhRowqva9SokWNbeno6wcHBFba7uLgQEBBQYcyvz2D9ep/p6ek0aNCA9PT0Kx7nSln+14wZM3j66acveH7p0qV4eXldoirVJzEx8Te/1h3o7Qq9W0B+JBzMM3Hg3EfqGTh5ppglOzJZsiMTADeznUgfO82s0MzXTlNfOx6WSvpEqsHvqVV9o1o5T7VynmrlPNXq6qhezlOtnKdaOc/oWhUUFDg1ztCmqaysjG7duvH8888D0KVLF7Zt28bcuXMZMWKEkdEqxeTJkyucvcrNzSU8PJxBgwZhtVoNy2Wz2UhMTGTgwIG4ulb+PZqKbKVsTcsl+XA2Gw5nsTE1m9zCEvbmmthbfrINswnaNvala0QDukX4ExPpTyNrzZsXVtW1qktUK+epVs5TrZynWl0d1ct5qpXzVCvn1ZRanb8S7EoMbZoaN25MdHR0hefatm3Lf/7zHwBCQkIAyMjIoHHjxo4xGRkZdO7c2TEmMzOzwj5KSko4ffq04/UhISFkZGRUGHP+8ZXG/Hr7lbL8L3d3d9zd3S943tXVtUZ8IVVVDldXV2JbeBDbovwMYFmZnb2ZZ9hw+DQbDmWx/tBpjmadZXtaHtvT8vjX2lQAwgM86RZZvsx596YBtGjoU2OWOq8p/2a1gWrlPNXKeaqV81Srq6N6OU+1cp5q5Tyja+XssQ1tmq655hp2795d4bk9e/YQGRkJlC8KERISwvLlyx2NSW5uLr/88gvjxo0DIDY2luzsbJKTk4mJiQHghx9+oKysjJ49ezrG/P3vf8dmszkKk5iYSOvWrR0r9cXGxrJ8+XImTpzoyJKYmEhsbKzTWeTizGYTrUN8aR3iy7Ce5f+26TmFFZqoncdzOXL6LEdOH+OLTccA8PN0LZ8Tda6J6tDEDw/XWnRNn4iIiIjUCYY2TZMmTaJ37948//zz3H333axbt4633nqLt956CwCTycTEiRN59tlnadmypWOZ79DQUG6//Xag/MzUjTfeyOjRo5k7dy42m40JEyZwzz33EBoaCsB9993H008/zahRo3jsscfYtm0br7zyCrNnz3Zk+ctf/sJ1113HSy+9RHx8PJ988gkbNmy4qizivBA/D27pGMotHcv/jfIKbWxKLb+cb8Oh02xKzSbnrI0fdmXyw65z86IsZjqG+RHTtAHdI8uXOm+gpc5FREREpIoZ2jR1796dL774gsmTJzN9+nSioqJ4+eWXGTZsmGPMo48+Sn5+PmPGjCE7O5s+ffqwePHiCvdF+uijj5gwYQI33HADZrOZIUOG8M9//tOx3c/Pj6VLlzJ+/HhiYmIICgpi6tSpjuXGAXr37s2CBQuYMmUKTzzxBC1btuTLL7903KPJ2Szy2/h6uNK3VUP6tmoIgK20jB1puY77Ra0/lMXJM0XlTdXhLN6kfKnzlsE+dGsaQLfI8rNR4QGeWupcRERERCqVoU0TwC233MItt9xyye0mk4np06czffr0S44JCAhw3Mj2Ujp27MhPP/102TF33XUXd9111+/KIpXD1WKmU7g/ncL9efDa8nt2HT5VUH6/qENZbDh8mv0n8tmbeYa9mWf4eF35vKhgX/dz94tqQLfIANo2rp1LnYuIiIhIzWF40yTiDJPJRNMgb5oGeXNXt3AATp0pIvncmaf1h06z7VgOmXlFLNp6nEVbjwPg5WYpX6Hv3LyozuH+eLvrbS8iIiIiztNPj1JrBfq4M6hdCIPala9sWGgrZfORbEcTlXw4i7zCElbvO8nqfScBsJhNRDe2OpqobpENCK6BS52LiIiISM2hpknqDA9XCz2bBdKzWSBQvtT5nsw81h8qX1xiw6EsjmWfZeuxHLYey2Hez4cAiAjwcjRR3Zs2oFlQzVnqXERERESMp6ZJ6iyz2USbECttQqwM71W+1Hla9lnHCn3rD2WxKz2X1NMFpJ4u4PON5Uud+3u50i2yAd2aBtAlzEpJmZGfhYiIiIgYTU2T1Cuh/p7c6u/JrZ3KlzrPLbSx8XDWuRX6TpNyJJvsAhvLdmaybGf5UucWk4X5R9fSObwBnc8tTtEsyFtno0RERETqCTVNUq9ZPVzp1zqYfq2DASguKWN7Wo6jiVp/6DSn821sPZbL1mO5/GvtYQB83V3oGO5Hp7DyJqpzuD+NNDdKREREpE5S0yTyK24uZrpENKBLRAMevLYZxcXFfPjF9wS07Mq2tDw2H81m67Ec8opK+HnfKX7ed8rx2hCrh+NMVKdwPzo08cPXw9XAz0ZEREREKoOaJpHLMJlMBHrAzR1CuL1r+VLnJaVl7Mk4Q8qRbDYfyWbz0Wz2ZOSRnlvI4u3pLN6efu610KKhj+N+U53D/Gkd4oubi+4bJSIiIlKbqGkSuUouFjPRoVaiQ63c1zMCgPyiErYdy2Hz0Ww2H8kh5Ug2x7LPOm6+++/ko0D5maz2oVbHJX2dwvyJDPTCZNL8KBEREZGaSk2TSCXwdnepsNw5wIm8IseZqPNnpXILS9iYms3G1GzHOD9P13NnovwcZ6WCfNwN+CxERERE5GLUNIlUkYa+7gyIbsSA6EYA2O12Dp0qYPORc03U0Wy2p+WSc9bGqj0nWLXnhOO1YQ08HZf0dQr3p30TK15u+nIVERERMYJ+ChOpJiaTiaggb6KCvLm9SxOgfLW+3el5pBzNJiW1vJHaf+IMR7POcjTrLIu2HAfAbIJWjXwdC010DvenZbAPLhbNjxIRERGpamqaRAzk5mKmQ5gfHcL8HDfgzS20se1oDilHzy00cSSH9NxCdqXnsSs9j0/WHwHA09VChyZ+dAo/d1lfmD9hDTw1P0pERESkkqlpEqlhrB6u9G4RRO8WQY7n0nMKK8yN2nI0hzNFJaw7dJp1h047xgV6u/13kYlwfzqF+eHv5WbEpyEiIiJSZ6hpEqkFQvw8CPELIa5dCABlZXYOnDxDypEcx2ITO4/nciq/mB92ZfLDrkzHa5sGejnORHUK96ddqBUPV4tRn4qIiIhIraOmSaQWMptNtAj2pUWwL3fGhAFQaCtl5/HcXy00kcPBk/kcOlXAoVMFfJWSBoCL2USbxr6OJc87h/vTrKEPFrMu6xMRERG5GDVNInWEh6uFLhEN6BLRwPFcdkExW47mVFj6/OSZYrYdy2XbsVw+JBUAH3eXc/Oj/Okc7kfn8AaE+HkY9amIiIiI1ChqmkTqMH8vN/q2akjfVg2B8mXP03IKzy0wkc2mI9lsPTc/KunAKZIOnHK8tpHV3XFJX+dwfzqE+WH1cDXqUxERERExjJomkXrEZDLRxN+TJv6e3NyhMQAlpWXsO3Hm3GV95WeldmfkkZFbxNIdGSzdkeF4ffOG3nQK96fLuYUm2oRYcXPRsuciIiJSt6lpEqnnXCxm2oRYaRNiZWj38ufOFpeyPS3HMTcq5UgWR06fZf+JfPafyOfzjccAcLOYiQ610jncn/aNfTh9tnyRChEREZG6RE2TiFzA081Ct6YBdGsa4Hju1Jkithw930iVX96XVWAj5dzCE+VceHnHD7RtbKV9qJV2oX60a2KlZbCvzkiJiIhIraWmSUScEujjTv82wfRvEwyUz486cvqs4ya8m1Kz2Ho0i4LiUpIPZ5F8OMvxWleLiVaNfGl/rolqF2qlbWMrXm76FiQiIiI1n35iEZHfxGQyERHoRUSgF7d2CsVms/Htou9o070ve04UsO1YDtvTctl2LIfcwhK2p+WyPS0XNpx/PTQL8qZ9Ez/ahVrLG6pQP/y8tNiEiIiI1CxqmkSk0phN0CLYh7ZNGnBb5yZA+Rmpo1lnzzVN/22kMvOKHHOkzt9DCqCJvyftm5Rf2nf+z2Bfd0wm3UdKREREjKGmSUSqlMlkIjzAi/AAL25sH+J4PjOvkO1puew410xtO5ZL6ukCjmWf5Vj2WZZs/++qfUE+buXzo0L/20xFBHipkRIREZFqoaZJRAwR7OtBcGsP+rcOdjyXc9bmaKJ2pOWyLS2HfZlnOHmmmJV7TrByzwnHWF93F6LPLzYRaqV9Ez+aN/TGxaIFJ0RERKRyqWkSkRrDz9OV2OaBxDYPdDx3triUXem5jjlR29Ny2JWeR15RCb8cPM0vB087xrq7mGnT2HrujFT5PKnWIb54uFqM+HRERESkjlDTJCI1mqebhS4RDegS0cDxnK20jH2ZZ/47T+pYLjuO53KmqITNR8pX8zvPYjbRoqHPuVX7/GgfaiU61IqvhxacEBEREeeoaRKRWsfVYqZt4/Jly++MCQPKb6p7+HSBY37U+UUnTucXszsjj90ZeY6b8gJEBnrRPtSP6HOX9rULtRLk427UpyQiIiI1mJomEakTzGYTUUHeRAV5c0vHUKB85b703EK2HyufH3V+4Ylj2Wc5fKqAw6cKWLT1uGMfIVYPx6V97c41Uk38PbXghIiISD2npklE6iyTyURjP08a+3kyILqR4/nT+cWOhSbOX+J38GQ+6bmFpOcWsnxXpmOsv5erY9W+839GBXljMauREhERqS/UNIlIvRPg7UaflkH0aRnkeC6/qISdx3P/e1PetFz2ZuSRXWDj532n+HnfKcdYLzcLbRv/d7GJ6FArrRr54uailftERETqIjVNIiKAt7sL3ZoG0K1pgOO5opJS9mac+VUjlcPO47kUFJeSfDiL5MNZjrGuFhOtGvk6lj9vF1o+58rLTd9mRUREajv9by4icgnuLhbaN/GjfRM/x3OlZXYOnjzjWGzi/J+5hSWOZdE/23AUAJMJmgV5O27Ie/4SP29XXdonIiJSm6hpEhG5ChaziRbBvrQI9uX2Lk2A8gUnjmaddazYd/7MVGZeEftP5LP/RD5fb05z7KOJvwdBZjOnA1Pp0zKYFsE+WmxCRESkBlPTJCLyO5lMJsIDvAgP8OLG9o0dz2fmFTpW7DvfSKWeLuBYdiHHMLP5213ALoJ83Ol97qa+vZsHEhHgpSZKRESkBjF01vJTTz2FyWSq8NGmTRvH9n79+l2wfezYsRX2kZqaSnx8PF5eXgQHB/PII49QUlJSYcyKFSvo2rUr7u7utGjRgvnz51+QZc6cOTRt2hQPDw969uzJunXrKmwvLCxk/PjxBAYG4uPjw5AhQ8jIyKi8YohInRPs60H/1sGM79+CN/4Yw6pH+7N52iA+fKAb8eGl9G4WgLuLmZNnivh6cxqTP9/Kdf9vBX1e/JG/fraZ/yQfJS37rNGfhoiISL1n+Jmmdu3asWzZMsdjF5eKkUaPHs306dMdj728vBx/Ly0tJT4+npCQENasWcPx48e5//77cXV15fnnnwfg4MGDxMfHM3bsWD766COWL1/Ogw8+SOPGjYmLiwPg008/JSEhgblz59KzZ09efvll4uLi2L17N8HBwQBMmjSJRYsWsXDhQvz8/JgwYQJ33HEHP//8c5XVRkTqHj9PV3pGBXAqzM7NN3ejFDMpR7JZs/8USftPknIkm2PZZ/nPxqP8Z2P53KimgV7ENg+id/NAejULpKGvbsIrIiJSnQxvmlxcXAgJCbnkdi8vr0tuX7p0KTt27GDZsmU0atSIzp0788wzz/DYY4/x1FNP4ebmxty5c4mKiuKll14CoG3btqxevZrZs2c7mqZZs2YxevRoRo4cCcDcuXNZtGgR7733Ho8//jg5OTm8++67LFiwgOuvvx6AefPm0bZtW9auXUuvXr0qsyQiUo94uFro1ay8GWJgKwqKS9hwKKu8iTpwiq1Hszl0qoBDp1L5eF0qAK0a+dC7eRCxzQPpFRWIn5erwZ+FiIhI3WZ407R3715CQ0Px8PAgNjaWGTNmEBER4dj+0Ucf8eGHHxISEsLgwYN58sknHWebkpKS6NChA40a/femlXFxcYwbN47t27fTpUsXkpKSGDBgQIVjxsXFMXHiRACKi4tJTk5m8uTJju1ms5kBAwaQlJQEQHJyMjabrcJ+2rRpQ0REBElJSZdsmoqKiigqKnI8zs3NBcBms2Gz2X5LuSrF+WMbmaG2UK2cp1o573K1cjVBbJQ/sVH+QHPyCm2sO5TFLwezSDpwml3peezJOMOejDPMX3MIkwmiG/vSKyqAXs0C6BbZAB93w7+1Vxq9r5ynWl0d1ct5qpXzVCvn1ZRaOXt8Q/9n7dmzJ/Pnz6d169YcP36cp59+mmuvvZZt27bh6+vLfffdR2RkJKGhoWzZsoXHHnuM3bt38/nnnwOQnp5eoWECHI/T09MvOyY3N5ezZ8+SlZVFaWnpRcfs2rXLsQ83Nzf8/f0vGHP+OBczY8YMnn766QueX7p0aYXLDI2SmJhodIRaQ7VynmrlvKupVWegcxScCYN9uSb25pjYm2si46yJ7Wl5bE/L492fD2PGToQPtPSz09LPTpSPHTdLlX0K1UbvK+epVldH9XKeauU81cp5RteqoKDAqXGGNk033XST4+8dO3akZ8+eREZG8tlnnzFq1CjGjBnj2N6hQwcaN27MDTfcwP79+2nevLkRka/K5MmTSUhIcDzOzc0lPDycQYMGYbVaDctls9lITExk4MCBuLrqsp7LUa2cp1o5rzJrlZFbyC8Hs1h78DRJB05zNOssh87AoTMmEo+V33S3S7g/vZoFENssgI5N/HBzMXQNoKui95XzVKuro3o5T7VynmrlvJpSq/NXgl1JjbqGw9/fn1atWrFv376Lbu/ZsycA+/bto3nz5oSEhFywyt35Fe3Oz4MKCQm5YJW7jIwMrFYrnp6eWCwWLBbLRcf8eh/FxcVkZ2dXONv06zEX4+7ujrv7hRO2XV1da8QXUk3JURuoVs5TrZxXGbUKC3QlLNCXId3KL2s+crqApAOnSNp/ijX7T5KRW8S6Q1msO5TFP3/Yj6erhW5NGzjmRLUPteJiqflNlN5XzlOtro7q5TzVynmqlfOMrpWzx65RTdOZM2fYv38/w4cPv+j2lJQUABo3Lr8PSmxsLM899xyZmZmOVe4SExOxWq1ER0c7xnz33XcV9pOYmEhsbCwAbm5uxMTEsHz5cm6//XYAysrKWL58ORMmTAAgJiYGV1dXli9fzpAhQwDYvXs3qampjv2IiNQE5+8XdXe3cOx2OwdP5jsWlVi7/xSn8ov5ae9Jftp7EgBfdxd6NgugV7NAejcPok2IL2az7hElIiLya4Y2TX/7298YPHgwkZGRpKWlMW3aNCwWC/feey/79+9nwYIF3HzzzQQGBrJlyxYmTZpE37596dixIwCDBg0iOjqa4cOHM3PmTNLT05kyZQrjx493nOEZO3Ysr732Go8++igPPPAAP/zwA5999hmLFi1y5EhISGDEiBF069aNHj168PLLL5Ofn+9YTc/Pz49Ro0aRkJBAQEAAVquVhx56iNjYWK2cJyI1lslkollDH5o19OGPvSIpK7OzJzPv3FmoU6w9cIq8whKW7cxk2c5MABp4uZ5roAKJbR5E84beutGuiIjUe4Y2TUePHuXee+/l1KlTNGzYkD59+rB27VoaNmxIYWEhy5YtczQw4eHhDBkyhClTpjheb7FY+Pbbbxk3bhyxsbF4e3szYsSICvd1ioqKYtGiRUyaNIlXXnmFsLAw3nnnHcdy4wBDhw7lxIkTTJ06lfT0dDp37szixYsrLA4xe/ZszGYzQ4YMoaioiLi4OF5//fXqKZSISCUwm020CbHSJsTKyGuiKC2zsyMtlzX7T7Jm/ynWHzpNVoGN77el8/228kVugn3diW1e3kT1bh5EeIDxi9iIiIhUN0Obpk8++eSS28LDw1m5cuUV9xEZGXnB5Xf/q1+/fmzatOmyYyZMmOC4HO9iPDw8mDNnDnPmzLliJhGR2sBiNtEhzI8OYX78+brm2ErL2HI0mzX7yi/n23A4i8y8Ir5KSeOrlDQAmvh7ljdQLQKJbRZEiJ+HwZ+FiIhI1atRc5pERMQ4rhYzMZEBxEQG8NANLSm0lbIxNYu15y7nSzmSzbHssyxMPsrC5KMANAvyJrZ5YPmNdpsFEuRz4eI3IiIitZ2aJhERuSgPVwu9mwfRu3kQCUB+UQnrD512rM637VgOB07mc+BkPh/9kgpAmxBfx5yons0C8fPU6lEiIlL7qWkSERGneLu70K91MP1al69WmnPWxrqDp1mz/yRJ+0+xKz3P8TF/zSHMJmgX6nduUYlAujcNwNtd/+2IiEjto/+9RETkN/HzdGVgdCMGRpcvmnPqTBFrD5xrog6c4sCJfLYey2HrsRzeXHUAF7OJTuH+5U1Us0C6RjbAw9Vi8GchIiJyZWqaRESkUgT6uBPfsTHxHcvvpZeeU0jSgfKzUD/vO8Wx7LMkH84i+XAWr/6wDzcXMzERDRyr83UM88fNpebfaFdEROofNU0iIlIlQvw8+EOXMP7QJQyAI6cLzt0jqnyJ88y8ovL5UQdOMSsRvNwsdGsacG5580DahfoZ/BmIiIiUU9MkIiLVIjzAi/AAL+7uHo7dbufAyXzW7D9F0rk5UVkFNlbtOcGqPScA8PVwoUfTBgQVmxhUWoar1pQQERGDqGkSEZFqZzKZaN7Qh+YNfRjeK5KyMju7M/IcTdQvB06TV1jC8l0nAAuu3+/m2T90NDq2iIjUU2qaRETEcGazibaNrbRtbGVUnyhKSsvYnpbLkm3HeX3lAT785QjXtgomrl2I0VFFRKQe0oxbERGpcVwsZjqF+zNpQAv6NS4D4G8LN3PkdIHByUREpD5S0yQiIjXarRFldA73I6+whPELNlJUUmp0JBERqWfUNImISI1mMcMrd3fEz9OVLUdzmPHdLqMjiYhIPaOmSUREarxQf09m3d0JgPlrDvH91uMGJxIRkfpETZOIiNQKN7RtxJ/7NgPg0X9v4fCpfIMTiYhIfaGmSUREao2/xbUmJrIBeUXl85sKbZrfJCIiVU9Nk4iI1BquFjOv3deFBl6ubDuWy3OLdhodSURE6gE1TSIiUqs09vNk1tDOAPxr7WG+2ZxmbCAREanz1DSJiEit0791MOP6NQdg8udbOXhS85tERKTqqGkSEZFa6a8DW9GjaQBnikoY/5HmN4mISNVR0yQiIrWSi8XMP+/tQqC3GzuO5zL92x1GRxIRkTpKTZOIiNRaIX4ezB7aGZMJFvySylcpx4yOJCIidZCaJhERqdX6tmrIhP4tAHji863sP3HG4EQiIlLXqGkSEZFa7y83tKRnVAD5xaWa3yQiIpVOTZOIiNR6LhYzr97bhSAfN3al5/HU19uNjiQiInWImiYREakTgq0evHJPF0wm+GT9Eb7YdNToSCIiUkeoaRIRkTrjmhZBPHx9SwCe+Hwb+zLzDE4kIiJ1gZomERGpUx6+oSW9mwdy1lbK/320kbPFmt8kIiK/j5omERGpUyxmEy/f05kgH3f2ZJxh6lfbjI4kIiK1nJomERGpc4J9PfjnvZ0xm2Bh8lH+naz5TSIi8tupaRIRkTqpd/MgJg5oBcCUL7eyJ0Pzm0RE5LdR0yQiInXW+P4tuLZlEIW2Mv7vo43kF5UYHUlERGohNU0iIlJnWcwmZg/tTLCvO/syz/Dkl9uw2+1GxxIRkVpGTZOIiNRpQT7u/PPeLphN8PmmYyzcoPlNIiJyddQ0iYhInderWSB/HdQagCe/2sau9FyDE4mISG2ipklEROqFcdc157pWDSkqKZ/fdEbzm0RExElqmkREpF4wm03MursTIVYPDpzI5+9fbNX8JhERcYqhTdNTTz2FyWSq8NGmTRvH9sLCQsaPH09gYCA+Pj4MGTKEjIyMCvtITU0lPj4eLy8vgoODeeSRRygpqfjbwxUrVtC1a1fc3d1p0aIF8+fPvyDLnDlzaNq0KR4eHvTs2ZN169ZV2O5MFhERqdkCfdx59b4uWMwmvkpJ45P1R4yOJCIitYDhZ5ratWvH8ePHHR+rV692bJs0aRLffPMNCxcuZOXKlaSlpXHHHXc4tpeWlhIfH09xcTFr1qzh/fffZ/78+UydOtUx5uDBg8THx9O/f39SUlKYOHEiDz74IEuWLHGM+fTTT0lISGDatGls3LiRTp06ERcXR2ZmptNZRESkdujeNIC/nZvfNO3r7exI0/wmERG5PMObJhcXF0JCQhwfQUFBAOTk5PDuu+8ya9Ysrr/+emJiYpg3bx5r1qxh7dq1ACxdupQdO3bw4Ycf0rlzZ2666SaeeeYZ5syZQ3FxMQBz584lKiqKl156ibZt2zJhwgTuvPNOZs+e7cgwa9YsRo8ezciRI4mOjmbu3Ll4eXnx3nvvOZ1FRERqjz/3bUb/1g0pLilj/IKN5BXajI4kIiI1mIvRAfbu3UtoaCgeHh7ExsYyY8YMIiIiSE5OxmazMWDAAMfYNm3aEBERQVJSEr169SIpKYkOHTrQqFEjx5i4uDjGjRvH9u3b6dKlC0lJSRX2cX7MxIkTASguLiY5OZnJkyc7tpvNZgYMGEBSUhKAU1kupqioiKKiIsfj3Nzy32babDZsNuP+gz5/bCMz1BaqlfNUK+epVs6rylq9eEc7bp2TxMGT+Tz+7y3MvrsDJpOp0o9TXfS+ujqql/NUK+epVs6rKbVy9viGNk09e/Zk/vz5tG7dmuPHj/P0009z7bXXsm3bNtLT03Fzc8Pf37/Caxo1akR6ejoA6enpFRqm89vPb7vcmNzcXM6ePUtWVhalpaUXHbNr1y7HPq6U5WJmzJjB008/fcHzS5cuxcvL65Kvqy6JiYlGR6g1VCvnqVbOU62cV1W1uicC/rndwqJt6XjlH6NPSO1fGELvq6ujejlPtXKeauU8o2tVUFDg1DhDm6abbrrJ8feOHTvSs2dPIiMj+eyzz/D09DQwWeWYPHkyCQkJjse5ubmEh4czaNAgrFarYblsNhuJiYkMHDgQV1dXw3LUBqqV81Qr56lWzquOWrmvPsSLS/bw1RFXht3Yg3ahxn1//j30vro6qpfzVCvnqVbOqym1On8l2JUYfnner/n7+9OqVSv27dvHwIEDKS4uJjs7u8IZnoyMDEJCQgAICQm5YJW78yva/XrM/65yl5GRgdVqxdPTE4vFgsViueiYX+/jSlkuxt3dHXd39wued3V1rRFfSDUlR22gWjlPtXKeauW8qqzV2H4tSE7NZtnOTP7y2Ra+eagPVo/a+++i99XVUb2cp1o5T7VyntG1cvbYhi8E8Wtnzpxh//79NG7cmJiYGFxdXVm+fLlj++7du0lNTSU2NhaA2NhYtm7dWmGVu8TERKxWK9HR0Y4xv97H+THn9+Hm5kZMTEyFMWVlZSxfvtwxxpksIiJSO5lMJv5xVyea+Hty+FQBj/9ni+7fJCIiFRjaNP3tb39j5cqVHDp0iDVr1vCHP/wBi8XCvffei5+fH6NGjSIhIYEff/yR5ORkRo4cSWxsrGPhhUGDBhEdHc3w4cPZvHkzS5YsYcqUKYwfP95xhmfs2LEcOHCARx99lF27dvH666/z2WefMWnSJEeOhIQE3n77bd5//3127tzJuHHjyM/PZ+TIkQBOZRERkdrL38uN1+7rgovZxHdb0/nX2sNGRxIRkRrE0Mvzjh49yr333supU6do2LAhffr0Ye3atTRs2BCA2bNnYzabGTJkCEVFRcTFxfH66687Xm+xWPj2228ZN24csbGxeHt7M2LECKZPn+4YExUVxaJFi5g0aRKvvPIKYWFhvPPOO8TFxTnGDB06lBMnTjB16lTS09Pp3LkzixcvrrA4xJWyiIhI7dYlogGP39SGZxft5Nlvd9IlvAEdwvyMjiUiIjWAoU3TJ598ctntHh4ezJkzhzlz5lxyTGRkJN99991l99OvXz82bdp02TETJkxgwoQJvyuLiIjUbqP6RLHu4GmW7sjg/xYk8+1D1+LnqXkJIiL1XY2a0yQiImIkk8nE/7uzE2ENPDly+iyP/nuz5jeJiIiaJhERkV/z83Jlzn1dcbWYWLI9g3k/HzI6koiIGExNk4iIyP/oFO7PEze3BWDG9ztJOZJtbCARETGUmiYREZGL+FPvptzUPgRbqZ3xH20kp8BmdCQRETGImiYREZGLMJlMvHhnRyICvDiWfZa/aX6TiEi9paZJRETkEqwerrw+rCtuFjOJOzJ4d/VBoyOJiIgB1DSJiIhcRvsmfjx5S/n8phe+38XG1CyDE4mISHVT0yQiInIFf+wVSXyHxpSU2XlowSayC4qNjiQiItVITZOIiMgVmEwmXhjSgaaB5fOb/vrZZsrKNL9JRKS+UNMkIiLiBF8PV+YM64qbi5nluzJ5+6cDRkcSEZFqoqZJRETESe1C/Zg2OBqAmUt2s+HQaYMTiYhIdVDTJCIichXu6xHBrZ1CKS2z89DHmzidr/lNIiJ1nZomERGRq2AymXj+jg40C/LmeE4hCZ+laH6TiEgdp6ZJRETkKvm4uzBnWFfcXcys2H2Cuav2Gx1JRESqkJomERGR36BtYytP39oOgJeW7mHdQc1vEhGpq9Q0iYiI/EZDu4fzhy5Nzs1v2sipM0VGRxIRkSqgpklEROQ3MplMPHt7e5o39CYjt4hJun+TiEidpKZJRETkd/B2d+H1YTF4uJpZtecEr6/YZ3QkERGpZGqaREREfqfWIb5Mv609ALMS95C0/5TBiUREpDKpaRIREakEd3cLZ0jXMMrs8PAnmziRp/lNIiJ1hZomERGRSvLM7e1oGezDibwiJn2aQqnmN4mI1AlqmkRERCqJl5sLrw/riqerhdX7TvLaD5rfJCJSF6hpEhERqUQtG/ny7O3l85teXr6HNftOGpxIRER+LzVNIiIilWxITBh3dwvDboeHP0khM6/Q6EgiIvI7qGkSERGpAk/f2p7WjXw5eaaIv3ys+U0iIrWZmiYREZEq4OlmYc6wrni5WUg6cIpXlu81OpKIiPxGappERESqSItgH57/QwcAXv1hLz/tPWFwIhER+S3UNImIiFSh27s04d4e4djtMPGTFDJyNb9JRKS2UdMkIiJSxaYNbkfbxlZO5Rfz0MebKCktMzqSiIhcBTVNIiIiVczD1cKc+7rg7WZh3cHTvLxM85tERGoTNU0iIiLVoFlDH2YM6QjAnBX7WLlH85tERGoLNU0iIiLV5NZOoQzrGYHdDpM+TeF4zlmjI4mIiBPUNImIiFSjJ2+Jpl2oldP5xTys+U0iIrWCmiYREZFqVD6/qSs+7i6sP5TFP5buMTqSiIhcgZomERGRatY0yJsXz81vmrtyPz/uyjQ4kYiIXI6aJhEREQPEd2zM/bGRAEz6LIW0bM1vEhGpqWpM0/TCCy9gMpmYOHGi47l+/fphMpkqfIwdO7bC61JTU4mPj8fLy4vg4GAeeeQRSkpKKoxZsWIFXbt2xd3dnRYtWjB//vwLjj9nzhyaNm2Kh4cHPXv2ZN26dRW2FxYWMn78eAIDA/Hx8WHIkCFkZGRU2ucvIiL1z9/j29KhiR/ZBTYmLNiITfObRERqpBrRNK1fv54333yTjh07XrBt9OjRHD9+3PExc+ZMx7bS0lLi4+MpLi5mzZo1vP/++8yfP5+pU6c6xhw8eJD4+Hj69+9PSkoKEydO5MEHH2TJkiWOMZ9++ikJCQlMmzaNjRs30qlTJ+Li4sjM/O/lEpMmTeKbb75h4cKFrFy5krS0NO64444qqoiIiNQH7i7l85t8PVzYmJrN/1uy2+hIIiJyEYY3TWfOnGHYsGG8/fbbNGjQ4ILtXl5ehISEOD6sVqtj29KlS9mxYwcffvghnTt35qabbuKZZ55hzpw5FBcXAzB37lyioqJ46aWXaNu2LRMmTODOO+9k9uzZjv3MmjWL0aNHM3LkSKKjo5k7dy5eXl689957AOTk5PDuu+8ya9Ysrr/+emJiYpg3bx5r1qxh7dq1VVwhERGpyyICvfh/d5b/0vCtVQdYtkNXMYiI1DQuRgcYP3488fHxDBgwgGefffaC7R999BEffvghISEhDB48mCeffBIvLy8AkpKS6NChA40aNXKMj4uLY9y4cWzfvp0uXbqQlJTEgAEDKuwzLi7OcRlgcXExycnJTJ482bHdbDYzYMAAkpKSAEhOTsZms1XYT5s2bYiIiCApKYlevXpd9HMrKiqiqKjI8Tg3NxcAm82GzWa7mjJVqvPHNjJDbaFaOU+1cp5q5bz6UqsbWgdxf68IPlibyl8XpvDV/8XSxN/zqvZRX2pVWVQv56lWzlOtnFdTauXs8Q1tmj755BM2btzI+vXrL7r9vvvuIzIyktDQULZs2cJjjz3G7t27+fzzzwFIT0+v0DABjsfp6emXHZObm8vZs2fJysqitLT0omN27drl2Iebmxv+/v4XjDl/nIuZMWMGTz/99AXPL1261NH4GSkxMdHoCLWGauU81cp5qpXz6kOtOtkhwttCan4JI95cxcPtSnH5DdeD1IdaVSbVy3mqlfNUK+cZXauCggKnxhnWNB05coS//OUvJCYm4uHhcdExY8aMcfy9Q4cONG7cmBtuuIH9+/fTvHnz6or6m02ePJmEhATH49zcXMLDwxk0aFCFywyrm81mIzExkYEDB+Lq6mpYjtpAtXKeauU81cp59a1WXa85y22vJ3H4TAnbLM154qbWTr+2vtXq91K9nKdaOU+1cl5NqdX5K8GuxLCmKTk5mczMTLp27ep4rrS0lFWrVvHaa69RVFSExWKp8JqePXsCsG/fPpo3b05ISMgFq9ydX9EuJCTE8ef/rnKXkZGB1WrF09MTi8WCxWK56Jhf76O4uJjs7OwKZ5t+PeZi3N3dcXd3v+B5V1fXGvGFVFNy1AaqlfNUK+epVs6rL7WKCnblH3d1Ysy/kpm35jC9mgcR1+7S/89cTH2pVWVRvZynWjlPtXKe0bVy9tiGLQRxww03sHXrVlJSUhwf3bp1Y9iwYaSkpFzQMAGkpKQA0LhxYwBiY2PZunVrhVXuEhMTsVqtREdHO8YsX768wn4SExOJjY0FwM3NjZiYmApjysrKWL58uWNMTEwMrq6uFcbs3r2b1NRUxxgREZHKMKhdCA/2iQLgkYWbOXLauUtHRESk6hh2psnX15f27dtXeM7b25vAwEDat2/P/v37WbBgATfffDOBgYFs2bKFSZMm0bdvX8fS5IMGDSI6Oprhw4czc+ZM0tPTmTJlCuPHj3ec4Rk7diyvvfYajz76KA888AA//PADn332GYsWLXIcNyEhgREjRtCtWzd69OjByy+/TH5+PiNHjgTAz8+PUaNGkZCQQEBAAFarlYceeojY2NhLLgIhIiLyWz16Yxs2HM4i5Ug2ExZsZOHY3rj9lglOIiJSKQxfPe9S3NzcWLZsmaOBCQ8PZ8iQIUyZMsUxxmKx8O233zJu3DhiY2Px9vZmxIgRTJ8+3TEmKiqKRYsWMWnSJF555RXCwsJ45513iIuLc4wZOnQoJ06cYOrUqaSnp9O5c2cWL15cYXGI2bNnYzabGTJkCEVFRcTFxfH6669XTzFERKRecXMx89p9XYj/52o2H83h+e928tSt7YyOJSJSb9WopmnFihWOv4eHh7Ny5corviYyMpLvvvvusmP69evHpk2bLjtmwoQJTJgw4ZLbPTw8mDNnDnPmzLliJhERkd8rrIEXs+7uxKj3NzB/zSF6RgVwU4fGRscSEamXdK5fRESkhrqhbSP+3LcZAI/+ewuppzS/SUTECGqaREREarC/xbUmJrIBeUUljF+wkaKSUqMjiYjUO2qaREREajBXi5lX7+1CAy9Xth7L4blFO42OJCJS71RK03T48GF27NhBWVlZZexOREREfiXU35NZQzsD8EHSYb7dkmZsIBGReuaqmqb33nuPWbNmVXhuzJgxNGvWjA4dOtC+fXuOHDlSqQFFREQE+rcOZly/5gA8/p+tHDyZb3AiEZH646qaprfeeosGDRo4Hi9evJh58+bxwQcfsH79evz9/Xn66acrPaSIiIjAXwe2onvTBpwpKmH8RxsptGl+k4hIdbiqpmnv3r1069bN8firr77itttuY9iwYXTt2pXnn3+e5cuXV3pIERERAReLmVfv7UqAtxs7jufyzLc7jI4kIlIvXFXTdPbsWaxWq+PxmjVr6Nu3r+Nxs2bNSE9Pr7x0IiIiUkGInwezh3bGZIKPfknlq5RjRkcSEanzrqppioyMJDk5GYCTJ0+yfft2rrnmGsf29PR0/Pz8KjehiIiIVHBdq4aM79cCgCc+38r+E2cMTiQiUrddVdM0YsQIxo8fzzPPPMNdd91FmzZtiImJcWxfs2YN7du3r/SQIiIiUtHEAS3pGRVAfnGp5jeJiFSxq2qaHn30UUaPHs3nn3+Oh4cHCxcurLD9559/5t57763UgCIiInIhl3P3bwrycWNXeh7PfrfL6EgiInWWy9UMNpvNTJ8+nenTp190+/82USIiIlJ1gq0evDy0C8Pf+4VPNxzDrYWJm40OJSJSB131zW0//fRThg0bxl133cXcuXOrIpOIiIg4qU/LIB66viUAnx4w8/P+UwYnEhGpe66qaXrjjTe499572bBhA3v37mX8+PE88sgjVZVNREREnDD62igAistM/Gl+MsPeWUvy4SyDU4mI1B1X1TS99tprTJs2jd27d5OSksL777/P66+/XlXZRERExAm+Hq4kP9GfaxuV4Wox8fO+Uwx5Yw0PzF/PtmM5RscTEan1rqppOnDgACNGjHA8vu+++ygpKeH48eOVHkxEREScZ/V05c5mZSRO7MPQbuFYzCZ+2JXJLa+uZtyHyezJyDM6okidt+1YLukFRqeQqnBVTVNRURHe3t7/fbHZjJubG2fPnq30YCIiInL1mvh78uKdHVmWcB23dw7FZILvt6UT9/IqJn6yiYMn842OKFIn7UjL4Q9z1zJjswvfbUs3Oo5UsqtaPQ/gySefxMvLy/G4uLiY5557rsJNbWfNmlU56UREROQ3iQry5uV7uvB//VswO3EP329L58uUNL7Zcpw7u4bx0A0tCGvgdeUdiYhTPvwl1fH3vy7cisVi4ZaOoQYmksp0VU1T37592b17d4XnevfuzYEDBxyPTSZT5SQTERGR361VI1/e+GMM247lMCtxDz/syuTTDUf4fNNR7u0Rwfj+LWhk9TA6pkitdvJMEf9OPgpAM187B/Lg4Y83cba4lLu6hRucTirDVTVNK1asqPD45MmTuLm5YbVaKzOTiIiIVLL2Tfx470/dST6cxazE3fy87xQfJB3m0/VHuD82krHXNSfQx93omCK10gdJhykuKaNjEysjw0+TZIvks+RjPPLvLZy1lXJ/bFOjI8rvdNX3acrOzmb8+PEEBQXRqFEjGjRoQEhICJMnT6agQDPfREREarKYyAZ89GAvFozuSUxkA4pKynj7p4NcO/NH/rFkNzkFNqMjitQqZ4tL+VfSIQAe7NMUswmevS2akdc0BWDqV9uZu3K/cQGlUlzVmabTp08TGxvLsWPHGDZsGG3btgVgx44dvPrqqyQmJrJ69Wq2bNnC2rVrefjhh6sktIiIiPw+vZsHETs2kBV7TvDS0t1sO5bLaz/u4/2kQ4y5thkj+0Th437VU59F6p1/Jx8hq8BGeIAnA9sGszS1fLrK1Fui8XZz4bUf9/HC97soKCph0sBWmspSS13Vd8Pp06fj5ubG/v37adSo0QXbBg0axPDhw1m6dCn//Oc/KzWoiIiIVC6TyUT/1sH0a9WQJdszmJW4mz0ZZ3gpcQ/v/XyQcf2aM7xXUzzdLEZHFamRSsvsvLP6IAAP9mmGi+W/F3GZTCb+FtcaL3cLMxfv5p8/7CO/uJQp8W3VONVCV3V53pdffsk//vGPCxomgJCQEGbOnMl//vMfEhISKtzPSURERGouk8nEje1D+P4vfXnlns5EBXmTVWDj+e92cd3/+5EPkg5RVFJqdEyRGmfp9nQOnyrA38uVu7qFXXTM//VrwVODowF4d/VBnvhiG2Vl9uqMKZXgqpqm48eP065du0tub9++PWazmWnTpv3uYCIiIlK9LGYTt3VuQuKkvsy8syNN/D3JzCti6lfbuf4fK/l0fSq20jKjY4rUCHa7nTdXla8gPbxXJF5ul76A60/XRDFzSEdMJvh4XSp/XbiZEn0t1SpX1TQFBQVx6NChS24/ePAgwcHBvzeTiIiIGMjFYububuH8+Ld+PHN7expZ3TmWfZbH/rOVgbNW8uWmY5TqN+VSz204nEXKkWzcXMxOrY53d/dwXrmnCxaziS82HWPCgk0Ul6hxqi2uqmmKi4vj73//O8XFxRdsKyoq4sknn+TGG2+stHAiIiJiHDcXM8N7RbLykf5MiW9LoLcbh04VMPHTFG58eRXfbz2uy4yk3nrr3FmmIV2b0NDXueX6b+0UyhvDuuJmMbN4ezpj/rWBQpsufa0Nrqppmj59Ort376Zly5bMnDmTr7/+mq+++ooXXniBli1bsnPnTp566qkqiioiIiJG8HC18OC1zVj1aH8eiWuN1cOFvZlnGPfRRga/tpofdmVgt6t5kvpj/4kzLNuZAcCoPs2u6rWD2oXw7p+64eFqZsXuE/xp3jrOFJVURUypRFfVNIWFhZGUlER0dDSTJ0/m9ttv5w9/+AN///vfiY6O5ueffyYiIqKqsoqIiIiBvN1dGN+/BT89dj0PX98CbzcL29NyeWD+Bu54Yw0/7zup5knqhXd+OojdDgPaNqJFsM9Vv/7alg354IGe+Li7sPbAaYa/+4vukVbDXfXNbaOiovj+++85efIka9euZe3atZw4cYLFixfTokWLqsgoIiIiNYifpysJg1rz02PX8+e+zfBwNbMpNZth7/zCvW+vZcOh00ZHFKkyJ/KK+M/GowCM6Xt1Z5l+rUdUAB892BN/L1c2pWZz79trOXWmqLJiSiW76qbpvAYNGtCjRw969OhBQEBAZWYSERGRWiDA243JN7dl1SP9+VPvprhZzKw9cJo75yYx4r11bDmabXREkUr3r6RDFJeU0Tncn+5NG/yufXUK9+eTMb0I8nFjx/Fc7n4zifScwkpKKpXpNzdNIiIiIgDBVg+eurUdPz7Sj3t7hGMxm1i55wS3vvYzYz7YwK70XKMjilSKguISPlh7GCg/y1QZN6ltE2Llsz/H0tjPg/0n8rn7zSSOnC743fuVyqWmSURERCpFE39PZtzRkeUJ13FHlyaYTLB0RwY3vfITD328if0nzhgdUeR3+XfyUbILbEQEeBHXLqTS9tusoQ+f/TmWiAAvUk8XcPebSRzQ10uNoqZJREREKlXTIG9mDe3M0ol9ie/QGLsdvtmcxsBZK3lk4Wb9Fl1qpdIyO+/8dBCAB6+NwmL+/WeZfi08wIvP/hxL84beHM8p5O431+osbQ2ipklERESqRMtGvswZ1pVFD/dhQNtgyuywMPko17+0gilfbtXcDalVlmxPJ/V0Af5ertwZE1Ylxwjx8+CzP8cS3djKyTNFDH1zLZuPZFfJseTqqGkSERGRKtUu1I93RnTni//rzbUtg7CV2vlwbSp9/9+PPPPtDk5qxTCp4ex2O2+eu5nt/b0i8XJzqbJjBfq48/HoXnQO9yfnrI1h7/zCeq1Iabga0zS98MILmEwmJk6c6HiusLCQ8ePHExgYiI+PD0OGDCEjI6PC61JTU4mPj8fLy4vg4GAeeeQRSkoq3iBsxYoVdO3aFXd3d1q0aMH8+fMvOP6cOXNo2rQpHh4e9OzZk3Xr1lXY7kwWERERubQuEQ3416iefDKmF92bNqC4pIx3Vx/k2hd/ZObiXWQXFBsdUeSi1h/KYvORbNxczAyPbVrlx/PzcuXDB3vSMyqAM0UlDH/3F37ae6LKjyuXViOapvXr1/Pmm2/SsWPHCs9PmjSJb775hoULF7Jy5UrS0tK44447HNtLS0uJj4+nuLiYNWvW8P777zN//nymTp3qGHPw4EHi4+Pp378/KSkpTJw4kQcffJAlS5Y4xnz66ackJCQwbdo0Nm7cSKdOnYiLiyMzM9PpLCIiIuKcXs0C+ezPsXzwQA86hflx1lbK6yv2c+2LP/LKsr3kFeomn1KzvHXuLNOQrmE09HWvlmP6uLswf2QPrmvVkEJbGaPmbyBxh35hbxTDm6YzZ84wbNgw3n77bRo0+O9a9zk5Obz77rvMmjWL66+/npiYGObNm8eaNWtYu3YtAEuXLmXHjh18+OGHdO7cmZtuuolnnnmGOXPmUFxc/tuquXPnEhUVxUsvvUTbtm2ZMGECd955J7Nnz3Yca9asWYwePZqRI0cSHR3N3Llz8fLy4r333nM6i4iIiDjPZDLRt1VDvhx/DW8Nj6FNiC95RSXMXraHa2f+yNyV+ykoLrnyjkSq2L7MMyzbmYHJVL4ARHXydLPw1v0x3NguhOLSMsZ+mMzXm9OqNYOUq7oLMp00fvx44uPjGTBgAM8++6zj+eTkZGw2GwMGDHA816ZNGyIiIkhKSqJXr14kJSXRoUMHGjVq5BgTFxfHuHHj2L59O126dCEpKanCPs6POX8ZYHFxMcnJyUyePNmx3Ww2M2DAAJKSkpzOcjFFRUUUFf33Ou3c3PIVUGw2Gzabcb9FO39sIzPUFqqV81Qr56lWzlOtnFeba9W/VSDXtejF99sz+OcP+zhwsoAXvt/FOz8dYGzfKO7pFoa7q6VSj1mb61Xd6nut3l61D4AbWjckwt/9snWoilqZgdl3tcfdxcRXm4/zl082ceZsMXfFNKm0YxihpryvnD2+oU3TJ598wsaNG1m/fv0F29LT03Fzc8Pf37/C840aNSI9Pd0x5tcN0/nt57ddbkxubi5nz54lKyuL0tLSi47ZtWuX01kuZsaMGTz99NMXPL906VK8vLwu+brqkpiYaHSEWkO1cp5q5TzVynmqlfNqc61MwIQWkOxnYvFRMyfPFPPsd7t5bdku4sLK6NnQjqWSr5GpzfWqbvWxVrnF8J+NFsBEtOU433133KnXVUWt+nnCiWAzazLNPPHldpJTttC3sb3Sj1PdjH5fFRQ4dwsEw5qmI0eO8Je//IXExEQ8PDyMilGlJk+eTEJCguNxbm4u4eHhDBo0CKvValgum81GYmIiAwcOxNXV1bActYFq5TzVynmqlfNUK+fVpVoNBp4oKeM/m44xZ8UBMnKL+PSAhTVZnjzUvzm3dmr8u++RU5fqVdXqc61mL9tHif0AncP9mDC0BybT5d93VV2reLudGYv3MG/NYf5zyEJUy5b8uW/1XjJYWWrK++r8lWBXYljTlJycTGZmJl27dnU8V1payqpVq3jttddYsmQJxcXFZGdnVzjDk5GRQUhI+R2YQ0JCLljl7vyKdr8e87+r3GVkZGC1WvH09MRisWCxWC465tf7uFKWi3F3d8fd/cLJgq6urjXim05NyVEbqFbOU62cp1o5T7VyXl2plasr3N+7GXd3j2TBL6m8vmIfR7LO8ujn23jzp4NMGtiKm9s3xvw7m6e6Uq/qUN9qVVBcwoL1RwD4c9/muLm5Of3aqqzV1MHt8PVw5Z8/7OMfiXspLLHz10GtrtjQ1VRGv6+cPbZhC0HccMMNbN26lZSUFMdHt27dGDZsmOPvrq6uLF++3PGa3bt3k5qaSmxsLACxsbFs3bq1wip3iYmJWK1WoqOjHWN+vY/zY87vw83NjZiYmApjysrKWL58uWNMTEzMFbOIiIhI5fNwtfBAnyhWPdqfx25sg5+nK/tP5DNhwSbiX13Nsh0Z2O21/xIlqXkWbjhKdoGNyEAvBrW79C/Jq5vJZCJhUGseu7ENAK/9uI9nvt2pr4MqZtiZJl9fX9q3b1/hOW9vbwIDAx3Pjxo1ioSEBAICArBarTz00EPExsY6Fl4YNGgQ0dHRDB8+nJkzZ5Kens6UKVMYP3684wzP2LFjee2113j00Ud54IEH+OGHH/jss89YtGiR47gJCQmMGDGCbt260aNHD15++WXy8/MZOXIkAH5+flfMIiIiIlXHy82Fcf2aM6xXBO+tPsg7Px1k5/FcHvxgA53C/fnboFb0aRFUa3/bLjVLSWkZ76wuX2b8wT5Rv/ty0Kowrl9zvN0tTP1qO+/9fJCC4hKe+0OHGpm1LjB89bzLmT17NmazmSFDhlBUVERcXByvv/66Y7vFYuHbb79l3LhxxMbG4u3tzYgRI5g+fbpjTFRUFIsWLWLSpEm88sorhIWF8c477xAXF+cYM3ToUE6cOMHUqVNJT0+nc+fOLF68uMLiEFfKIiIiIlXP6uHKxAGtGBHblLd+OsD8nw+x+Ug2w99dR4+oAP46sBU9mwUaHVNquSXbMzhy+iwNvFy5Mybc6DiXdH9sUzxdLTz2ny18sv4IZ22lvHRXJ1wqe8UUqVlN04oVKyo89vDwYM6cOcyZM+eSr4mMjOS777677H779evHpk2bLjtmwoQJTJgw4ZLbnckiIiIi1aOBtxuP3diGB66J4vUV+/hobSrrDp5m6FtrubZlEH8d1JrO4f5Gx5RayG6389aq/QAMj22Kp1vlLndf2e7qFo6Hq4VJn6bwVUoahbZS/nlvF9xdanbu2kZtqIiIiNRaDX3dmTa4HSse6cd9PSNwMZv4ae9Jbp/zMw++v4Edac6tjCVy3rqDp9l8NAd3FzP3x0YaHccpgzuFMvePMbi5mFmyPYPRHyRztrjU6Fh1ipomERERqfVC/T15/g8d+OGv/RjSNQyzCZbtzODmf/7E+AUb2ZeZZ3REqSXeWlU+l2lITBhBPheuglxTDYhuxHsjuuPpamHVnhOMmLeOM0UlRseqM9Q0iYiISJ0REejFS3d3Yumk67ilY2MAFm05zqDZq0j4LIXUU87dyFLqp32ZeSzflYnJVL4ARG3Tp2UQH4zqga+7C+sOnmbYO7+QU2AzOladoKZJRERE6pwWwT68dl9Xvv/LtQyMbkSZHT7feIzrX1rB5M+3cjyn0OiIUgO9veogAAPbNqJZQx+D0/w23ZsG8NHonvh7ubL5SDb3vL2Wk2eKjI5V66lpEhERkTqrbWMrb9/fjS/HX0PfVg0pKbPz8bpUbpj9E+tPaGlm+a/MvEK+2HQMgDF9mxmc5vfpGObPp2NiCfJxZ+fxXO5+M4l0/aLgd1HTJCIiInVe53B/PnigB5/9OZYeUQHYSu38+6BZv4EXhw/WHKa4tIyuEf50axpgdJzfrXWILwvHxhLq58GBE/nc9eYajpzW5am/lZomERERqTd6RAXwyehetA+1UlhqYtayfUZHkhogv6iEf609DNT+s0y/FhXkzWdjY4kM9OLI6bPcNTeJ/SfOGB2rVlLTJCIiIvWK2Wziyfg2APx74zG2Hs0xOJEYbeGGI+SctdE00IuB0SFGx6lUYQ28+OzPsbQM9iE9t5Chbyax87iW4r9aappERESk3uka4U9MUBl2Ozz9zXbsdrvRkcQgJaVlvLO6fAGIUdc2w2Kue3PdGlk9+GRML9qFWjl5pph73lpLypFso2PVKmqaREREpF66NaIMT1czGw5n8fXmNKPjiEEWb0/naNZZArzduLNrmNFxqkygjzsLRveia4Q/OWdt/PGdX/jlwCmjY9UaappERESkXvJ3hz+fm7/ywve7KCjWjUDrG7vd7riZ7fBekXi6WQxOVLX8PF3516iexDYL5ExRCSPmrWPlnhNGx6oV1DSJiIhIvTXqmkjCGnhyPKeQuSsPGB1HqtkvB0+z5WgO7i5m7o+NNDpOtfB2d2HeyO70b92QQlsZo9/fwJLt6UbHqvHUNImIiEi95eFq4e83twXgzZX7OZqlJZnrk/Nnme6MCSPQx93gNNXHw9XCm8O7cVP7EIpLy/i/jzbyVcoxo2PVaGqaREREpF67sX0IvZoFUFRSxozvdhkdR6rJ3ow8ftiVickED15bd5YZd5abi5lX7+3CHV2aUFpmZ+KnKXyyLtXoWDWWmiYRERGp10wmE9MGt8NsgkVbj7NWk+Prhbd/Kj/LNCi6EVFB3ganMYaLxcw/7urEsJ4R2O3w+Odbee/cSoJSkZomERERqffaNrZyX88IAJ7+ZgelZVqCvC7LzC3ky03lKyaO6dvc4DTGMptNPHt7e0ZfGwXA9G93MOdH3fT5f6lpEhEREQESBrbG6uHCzuO5fLJelynVZfPXHKK4tIyYyAbERDYwOo7hTCYTT9zclr/c0BKA/7dkNzMX79L9y35FTZOIiIgIEODtRsLAVgD8Y8lucgpsBieSqpBfVMKHaw8DMKZv/ZvLdCkmk4lJA1sx+aY2ALy+Yj9Pf7ODMp11BdQ0iYiIiDgM6xVJy2AfsgpsvLx8j9FxpAp8uv4IuYUlRAV5M6BtI6Pj1Dh/vq45z9zWDig/Izf58626XBU1TSIiIiIOrhYz0waX/8D4QdJh9mbkGZxIKlNJaRnvnlvoYFSfKCxmk8GJaqbhsU35x12dMJvg0w1HmPRpCrbSMqNjGUpNk4iIiMiv9GkZxMDoRpSW2Zn+7Q7N66hDvt+WzrHsswR4u3FnTJjRcWq0O2PCePXerriYTXy9OY3/+2gjhbZSo2MZRk2TiIiIyP+YEt8WN4uZn/aeZPnOTKPjSCWw2+2Om9neHxuJh6vF4EQ1X3zHxrw5PAY3FzOJOzIY/cEGzhbXz8ZJTZOIiIjI/4gM9GbUuSWYn120g6KS+vmDYl2y9sBpth7Lwd3FzP2xTY2OU2vc0LYR8/7UHS83Cz/tPcmI99aRV1j/FklR0yQiIiJyEeP7t6ChrzuHThUw7+dDRseR3+mtVfsBuKtbGAHebganqV2uaRHEv0b1wNfdhXWHTvPHd34hu6DY6FjVSk2TiIiIyEX4uLvw2I3lyy+/unwvmXmFBieS32pPRh4/7j6ByQQP9tEy479FTGQAH4/pRQMvVzYfzeGet9ZyIq/I6FjVRk2TiIiIyCXc0aUJncL9yS8u5f8t3m10HPmN3j43lykuOoSmQd4Gp6m92jfx49M/x9LQ151d6XkMfTOJ4zlnjY5VLdQ0iYiIiFyC2Wxi2uBoABYmH2XzkWxjA8lVy8wt5MuUYwCMuU5nmX6vVo18WfjnWJr4e3LgZD53zU0i9VSB0bGqnJomERERkcvoGtGAO7o0AeCpb7ZrCfJaZt6aQ9hK7XSLbEDXiAZGx6kTmgZ58+mfe9E00IujWWe568017Mus2/c0U9MkIiIicgWP3dQGLzcLm1Kz+Solzeg44qQzRSV8tPYwAGP66ixTZQpr4MVnf46lZbAPGblFDH1zLdvTcoyOVWXUNImIiIhcQSOrB+P7twBgxvc7yS8qMTiROOPT9UfILSyhWZA3A9o2MjpOnRNs9eDTP8fSvomVU/nF3PvWWjalZhkdq0qoaRIRERFxwqg+UUQEeJGRW8QbK/YbHUeuoKS0jPdWHwTgwWubYTabDE5UNwV4u7FgdC9iIhuQW1jCH9/5hbUHThkdq9KpaRIRERFxgoerhb/HtwXgrZ8OcOR03Z/8Xpst2nqcY9lnCfR2446uTYyOU6dZPVz54IEe9G4eSH5xKSPeW8eK3ZlGx6pUappEREREnDQouhHXtAikuKSM5xbtNDqOXILdbuftn8qXGR/RuykerhaDE9V93u4uvPen7lzfJpiikjJGf7CBxdvSjY5VadQ0iYiIiDjJZDIx9ZZ2WMwmFm9PZ82+k0ZHkotIOnCKbcdy8XA188dekUbHqTc8XC3M/WMM8R0aYyu1M37BRr7cdMzoWJVCTZOIiIjIVWgd4ssfe0YAMP3bHZSUlhmcSP7XW+duZntXTDgB3m4Gp6lf3FzMvHJPZ4Z0DaO0zM6kz1JY8Euq0bF+NzVNIiIiIldp0sBW+Hu5sis9j4/X1f4fCOuS3el5rNh9ApMJHrw2yug49ZKLxcz/u7Mjf+wVgd0OT3yxlXfOXS5ZWxnaNL3xxht07NgRq9WK1WolNjaW77//3rG9X79+mEymCh9jx46tsI/U1FTi4+Px8vIiODiYRx55hJKSisuArlixgq5du+Lu7k6LFi2YP3/+BVnmzJlD06ZN8fDwoGfPnqxbt67C9sLCQsaPH09gYCA+Pj4MGTKEjIyMyiuGiIiI1Br+Xm4kDGwFwEuJe8guKDY4kZx3fi7Tje1CiAz0NjhN/WU2m3jmtvb8+dz9sZ5dtJNXl++ttTeHNrRpCgsL44UXXiA5OZkNGzZw/fXXc9ttt7F9+3bHmNGjR3P8+HHHx8yZMx3bSktLiY+Pp7i4mDVr1vD+++8zf/58pk6d6hhz8OBB4uPj6d+/PykpKUycOJEHH3yQJUuWOMZ8+umnJCQkMG3aNDZu3EinTp2Ii4sjM/O/q35MmjSJb775hoULF7Jy5UrS0tK44447qrhCIiIiUlPd1yOC1o18yS6w8fKyvUbHESAjt5CvUsrn0OhmtsYzmUw8flMbJg347y8YZi7ZXSsbJ0ObpsGDB3PzzTfTsmVLWrVqxXPPPYePjw9r1651jPHy8iIkJMTxYbVaHduWLl3Kjh07+PDDD+ncuTM33XQTzzzzDHPmzKG4uPw3PnPnziUqKoqXXnqJtm3bMmHCBO68805mz57t2M+sWbMYPXo0I0eOJDo6mrlz5+Ll5cV7770HQE5ODu+++y6zZs3i+uuvJyYmhnnz5rFmzZoKWUVERKT+cLGYmTo4GoB/rT3M7vQ8gxPJvJ8PYSu1071pA7pENDA6jlDeOP1lQEv+fnP5cv1vrNjPU19vp6ysdjVOLkYHOK+0tJSFCxeSn59PbGys4/mPPvqIDz/8kJCQEAYPHsyTTz6Jl5cXAElJSXTo0IFGjf57h+e4uDjGjRvH9u3b6dKlC0lJSQwYMKDCseLi4pg4cSIAxcXFJCcnM3nyZMd2s9nMgAEDSEpKAiA5ORmbzVZhP23atCEiIoKkpCR69ep10c+pqKiIoqIix+Pc3FwAbDYbNpvtt5SpUpw/tpEZagvVynmqlfNUK+epVs5Tra5OZdWrR6QfA9sGk7gzk6e/3sb8P8VgMtWtm6jWlvfWmaISPvrlMACjekcakre21MoIf4oNx80C077ZyftJh8k9W0RfD+Nr5ezxDW+atm7dSmxsLIWFhfj4+PDFF18QHV3+W5v77ruPyMhIQkND2bJlC4899hi7d+/m888/ByA9Pb1CwwQ4Hqenp192TG5uLmfPniUrK4vS0tKLjtm1a5djH25ubvj7+18w5vxxLmbGjBk8/fTTFzy/dOlSR+NnpMTERKMj1BqqlfNUK+epVs5TrZynWl2dyqhXLw/40WRhzYHTvPjRYjoG1K7foDurpr+3fkwzkVdoIdjDztkDG/juoHFZanqtjOIPDGthYsE+M1+kpHMo0IzdnojFwGvfCgqcu0m14U1T69atSUlJIScnh3//+9+MGDGClStXEh0dzZgxYxzjOnToQOPGjbnhhhvYv38/zZs3NzC1cyZPnkxCQoLjcW5uLuHh4QwaNKjCZYbVzWazkZiYyMCBA3F1dTUsR22gWjlPtXKeauU81cp5qtXVqex6nfTdyxurDrI004dJQ3vjXoduplob3lu20jJenL0aKOThuHbc0i3MmBy1oFZGuxnoujmNv/57G5tOmRnRvwM3d2piWJ7zV4JdieFNk5ubGy1atAAgJiaG9evX88orr/Dmm29eMLZnz54A7Nu3j+bNmxMSEnLBKnfnV7QLCQlx/Pm/q9xlZGRgtVrx9PTEYrFgsVguOubX+yguLiY7O7vC2aZfj7kYd3d33N3dL3je1dW1Rnwh1ZQctYFq5TzVynmqlfNUK+epVlensuo14YZWfJ6SxpGss3yw7ij/169FJaSrWWrye+u77cdIyykkyMeNO7tF4Gpw01qTa1UTDOkWidXTlUWrNhDfqYmhtXL22DXuPk1lZWUV5gH9WkpKCgCNGzcGIDY2lq1bt1ZY5S4xMRGr1eq4xC82Npbly5dX2E9iYqJj3pSbmxsxMTEVxpSVlbF8+XLHmJiYGFxdXSuM2b17N6mpqRXmX4mIiEj95O3uwuM3tQHgtR/2kZFbaHCi+sNutztuZjsitikedegsX13Wr1VD+ofWnktZDW2aJk+ezKpVqzh06BBbt25l8uTJrFixgmHDhrF//36eeeYZkpOTOXToEF9//TX3338/ffv2pWPHjgAMGjSI6Ohohg8fzubNm1myZAlTpkxh/PjxjjM8Y8eO5cCBAzz66KPs2rWL119/nc8++4xJkyY5ciQkJPD222/z/vvvs3PnTsaNG0d+fj4jR44EwM/Pj1GjRpGQkMCPP/5IcnIyI0eOJDY29pKLQIiIiEj9clunJnSJ8KeguJQXF+8yOk69sWb/Kban5eLpauGPvSKNjiN1lKGX52VmZnL//fdz/Phx/Pz86NixI0uWLGHgwIEcOXKEZcuW8fLLL5Ofn094eDhDhgxhypQpjtdbLBa+/fZbxo0bR2xsLN7e3owYMYLp06c7xkRFRbFo0SImTZrEK6+8QlhYGO+88w5xcXGOMUOHDuXEiRNMnTqV9PR0OnfuzOLFiyssDjF79mzMZjNDhgyhqKiIuLg4Xn/99eoplIiIiNR4ZrOJaYPbcfucn/l84zGG94rUstfV4PxZpru7hdHA283gNFJXGdo0vfvuu5fcFh4ezsqVK6+4j8jISL777rvLjunXrx+bNm267JgJEyYwYcKES2738PBgzpw5zJkz54qZREREpH7qHO7PnTFh/Dv5KE99s4MvxvXGbK5bS5DXJLvSc1m55wRmE4zqo5vZStWpcXOaRERERGqzR+Na4+1mYfORbL7YdMzoOHXa26vK1xW/qX1jIgKNv52L1F1qmkREREQqUbDVgwnXtwTgxcW7OFNUYnCiuik9p5CvN5c3paP76iyTVC01TSIiIiKV7IE+TYkM9CIzr4g5P+4zOk6dNG/NQWyldnpEBdA53N/oOFLHqWkSERERqWTuLhamxJff/uTdnw5y+FS+wYnqlrxCGwvWpgIw5lqdZZKqp6ZJREREpAoMaBvMtS2DKC4t47lFO42OU6d8uv4IeUUlNG/ozfVtgo2OI/WAmiYRERGRKmAymZh6SzQWs4mlOzJYvfek0ZHqBFtpGe+tLl8AYvS1zbQ6oVQLNU0iIiIiVaRlI1+Gn7vh6tPfbKektMzgRLXfoi3HScspJMjHndu7NDE6jtQTappEREREqtCkAa1o4OXK3swzfPRLqtFxajW73e64me2fekfi4WoxOJHUF2qaRERERKqQn5crfx3UGoBZiXvIyi82OFHt9fO+U+w4nounq4VhPSONjiP1iJomERERkSp2b48I2oT4knPWxqzEPUbHqbXe+qn8LNPQ7uE08HYzOI3UJ2qaRERERKqYxWxi2uB2AHz0y2F2Hs81OFHts/N4Lqv2nMBsglF9ooyOI/WMmiYRERGRahDbPJCbO4RQZofp3+zAbrcbHalWefvcWaabOjQmPMDL4DRS36hpEhEREakmk29qi7uLmaQDp1iyPd3oOLXG8ZyzfJ2SBuhmtmIMNU0iIiIi1SQ8wIs/9y3/of/ZRTsptJUanKh2mP/zIUrK7PSMCqBTuL/RcaQeUtMkIiIiUo3G9mtOiNWDo1lneefcJWdyaXmFNhacW6p9TF+dZRJjqGkSERERqUZebi5MvrkNAHN+3E96TqHBiWq2T9YdIa+ohBbBPvRvHWx0HKmn1DSJiIiIVLNbO4XSLbIBZ22lvPD9TqPj1Fi20jLe+/kgAKOvjcJsNhmcSOorNU0iIiIi1cxkKl+C3GSCL1PSSD582uhINdK3W9I4nlNIkI87t3dpYnQcqcfUNImIiIgYoEOYH3fFhAHw9Dc7KCvTEuS/ZrfbeWtV+Vmmkdc0xd3FYnAiqc/UNImIiIgY5JG4Nvi4u7DlaA7/3njU6Dg1yup9J9l5PBcvNwvDekYYHUfqOTVNIiIiIgZp6OvOwze0AGDm4t3kFdoMTlRzvLWqfGXBu7uF4+/lZnAaqe/UNImIiIgY6E+9o4gK8ubkmSJe+3Gf0XFqhB1pufy09yRmE4zqE2V0HBE1TSIiIiJGcnMx8+QtbQF4b/VBDp7MNziR8c7fv+rmDo0JD/AyOI2ImiYRERERw/VvHcx1rRpiK7Xz3KIdRscxVFr2Wb7enAboZrZSc6hpEhERETGYyWTiyVuicTGbWLYzk5V7ThgdyTDz1xyipMxOr2YBdAzzNzqOCKCmSURERKRGaBHsw4jeTQGY/s12bKVlxgYyQG6hjQW/pAI6yyQ1i5omERERkRri4RtaEuDtxv4T+fwr6bDRcardJ+tSOVNUQstgH/q1CjY6joiDmiYRERGRGsLP05W/DWoNwOxlezh1psjgRNWnuKSM91YfAmD0tc0wm03GBhL5FTVNIiIiIjXI0O7hRDe2kldYwkuJe4yOU22+3ZJGem4hDX3dua1LqNFxRCpQ0yQiIiJSg1jMJqYNjgbg43WpbE/LMThR1bPb7Y6b2f6pd1PcXSwGJxKpSE2TiIiISA3Ts1kg8R0bY7fD09/swG63Gx2pSv209yS70vPwcrPwx56RRscRuYCaJhEREZEa6Imb2+LuYmbdwdN8tzXd6DhV6u1zN7Md2j0cPy9Xg9OIXEhNk4iIiEgN1MTfk7HXNQfg+e92UmgrNThR1dielsNPe09iMZt44Jooo+OIXJSaJhEREZEaaux1zQn18+BY9lneXHnA6DhV4p2fDgJwc4fGhAd4GZxG5OLUNImIiIjUUJ5uFibf3BaAN1buIy37rMGJKlda9lm+2ZwGwJhrdTNbqbnUNImIiIjUYLd0bEyPpgEU2sp44ftdRsepVPN+PkhJmZ3YZoF0CPMzOo7IJRnaNL3xxht07NgRq9WK1WolNjaW77//3rG9sLCQ8ePHExgYiI+PD0OGDCEjI6PCPlJTU4mPj8fLy4vg4GAeeeQRSkpKKoxZsWIFXbt2xd3dnRYtWjB//vwLssyZM4emTZvi4eFBz549WbduXYXtzmQRERERqWwmk4mpg6MxmeDrzWmsP3Ta6EiVIrfQxsfrjgAwpq/OMknNZmjTFBYWxgsvvEBycjIbNmzg+uuv57bbbmP79u0ATJo0iW+++YaFCxeycuVK0tLSuOOOOxyvLy0tJT4+nuLiYtasWcP777/P/PnzmTp1qmPMwYMHiY+Pp3///qSkpDBx4kQefPBBlixZ4hjz6aefkpCQwLRp09i4cSOdOnUiLi6OzMxMx5grZRERERGpKu2b+HFP93AAnvp6O6VltX8J8o9/SeVMUQktg33o17qh0XFELsvQpmnw4MHcfPPNtGzZklatWvHcc8/h4+PD2rVrycnJ4d1332XWrFlcf/31xMTEMG/ePNasWcPatWsBWLp0KTt27ODDDz+kc+fO3HTTTTzzzDPMmTOH4uJiAObOnUtUVBQvvfQSbdu2ZcKECdx5553Mnj3bkWPWrFmMHj2akSNHEh0dzdy5c/Hy8uK9994DcCqLiIiISFX666DW+Hq4sD0tl38nHzE6zu9SXFLGvJ8PATC6bzNMJpOxgUSuwMXoAOeVlpaycOFC8vPziY2NJTk5GZvNxoABAxxj2rRpQ0REBElJSfTq1YukpCQ6dOhAo0aNHGPi4uIYN24c27dvp0uXLiQlJVXYx/kxEydOBKC4uJjk5GQmT57s2G42mxkwYABJSUkATmW5mKKiIoqKihyPc3NzAbDZbNhstt9Yqd/v/LGNzFBbqFbOU62cp1o5T7Vynmp1dWpjvfzczTzUvznPf7+bmYt3M7BNEL4eVX9Po6qo1Zeb0kjPLSTY152b2wXXqn+Hy6mN7yuj1JRaOXt8w5umrVu3EhsbS2FhIT4+PnzxxRdER0eTkpKCm5sb/v7+FcY3atSI9PTyG7ylp6dXaJjObz+/7XJjcnNzOXv2LFlZWZSWll50zK5duxz7uFKWi5kxYwZPP/30Bc8vXboULy/jl9RMTEw0OkKtoVo5T7VynmrlPNXKearV1alt9Qosg2APC5n5xfz1veXc3rSs2o5dWbWy22H2FgtgokeDApYvXVwp+61Jatv7ykhG16qgoMCpcYY3Ta1btyYlJYWcnBz+/e9/M2LECFauXGl0rEoxefJkEhISHI9zc3MJDw9n0KBBWK1Ww3LZbDYSExMZOHAgrq666/blqFbOU62cp1o5T7Vynmp1dWpzvfxaneDBf23ipwwLj991Lc0aelfp8Sq7Vj/tPcnxtRvxdrPw9B/7Y/WsXfW/nNr8vqpuNaVW568EuxLDmyY3NzdatGgBQExMDOvXr+eVV15h6NChFBcXk52dXeEMT0ZGBiEhIQCEhIRcsMrd+RXtfj3mf1e5y8jIwGq14unpicViwWKxXHTMr/dxpSwX4+7ujru7+wXPu7q61ogvpJqSozZQrZynWjlPtXKeauU81erq1MZ6DWgXyvVtjvHDrkxeWLKHeSN7VMtxK6tW7645DMDQ7hEEWo2/8qYq1Mb3lVGMrpWzx65x92kqKyujqKiImJgYXF1dWb58uWPb7t27SU1NJTY2FoDY2Fi2bt1aYZW7xMRErFYr0dHRjjG/3sf5Mef34ebmRkxMTIUxZWVlLF++3DHGmSwiIiIi1WVKfFtczCZ+3H2CH3dlXvkFNcS2Yzn8vO8UFrOJB/o0NTqOiNMMPdM0efJkbrrpJiIiIsjLy2PBggWsWLGCJUuW4Ofnx6hRo0hISCAgIACr1cpDDz1EbGysY+GFQYMGER0dzfDhw5k5cybp6elMmTKF8ePHO87wjB07ltdee41HH32UBx54gB9++IHPPvuMRYsWOXIkJCQwYsQIunXrRo8ePXj55ZfJz89n5MiRAE5lEREREakuzRr6MPKaprz900Ge+XYH17QIws2lxv0u/AJv/3QAgPgOjQlrUDfPMkndZGjTlJmZyf3338/x48fx8/OjY8eOLFmyhIEDBwIwe/ZszGYzQ4YMoaioiLi4OF5//XXH6y0WC99++y3jxo0jNjYWb29vRowYwfTp0x1joqKiWLRoEZMmTeKVV14hLCyMd955h7i4OMeYoUOHcuLECaZOnUp6ejqdO3dm8eLFFRaHuFIWERERker00A0t+WLTMQ6czOeDpEM8eG3NvkHsseyzfLvlOKCb2UrtY2jT9O677152u4eHB3PmzGHOnDmXHBMZGcl333132f3069ePTZs2XXbMhAkTmDBhwu/KIiIiIlJdrB6uPBLXmsf+s5VXlu3l9i5NCPK5cC51TTFv9UFKy+z0bh5I+yZ+RscRuSo1/zyuiIiIiFzUnTHhtG9iJa+ohH8s2W10nEvKOWvj43WpgM4ySe2kpklERESklrKYTTw1uB0An244wrZjOQYnuriP16WSX1xK60a+XNeqodFxRK6amiYRERGRWqxb0wBu7RSK3Q5Pf7Mdu91udKQKikvKmPfzQQAevDYKk8lkcCKRq6emSURERKSWe/ymNni4mll/KItvzi22UFN8vTmNjNwiGlndua1zE6PjiPwmappEREREarlQf0/+r18LAGZ8t5OzxaUGJypnt9t5e1X5MuN/6h1VK5ZFF7kYvXNFRERE6oAxfZvRxN+T4zmFzF253+g4AKzcc4LdGXl4u1m4r2eE0XFEfjM1TSIiIiJ1gIerhSdubgvA3JX7OZpVYHAieOvcWaZ7ekTg5+lqcBqR305Nk4iIiEgdcXOHEHpGBVBUUsaM73cZmmXbsRzW7D+FxWzigT5RhmYR+b3UNImIiIjUESaTiamDozGbYNGW4/xy4JRhWc6fZbqlY2Oa+HsalkOkMqhpEhEREalD2oX6cU+P8vlDT32zg9Ky6l+C/GhWAYu2lq/iN/pa3cxWaj81TSIiIiJ1zF8HtsLq4cLO47l8uv5ItR//vdWHKC2zc02LQNo38av244tUNjVNIiIiInVMoI87Ewe0AuAfS3eTU2CrtmPnFNj4ZH0qAGP6Nq+244pUJTVNIiIiInXQ8NhIWgT7cDq/mFeW762243607jAFxaW0CfGlb8ugajuuSFVS0yQiIiJSB7lazDx5SzQAHyQdYl9mXpUfs6iklPk/HwLK5zKZTKYqP6ZIdVDTJCIiIlJHXdeqIQPaBlNSZmf6tzux26t2UYivU9LIzCsixOrB4E6hVXoskeqkpklERESkDvt7fDSuFhOr9pzgh12ZVXYcu93O2z+VLzM+8pqmuLnox0ypO/RuFhEREanDooK8HTeXfebbHRSXlFXJcVbsOcGejDP4uLtwb8+IKjmGiFHUNImIiIjUcRP6tyDIx51DpwqY9/PBKjnGWyvLzzLd2yMcq4drlRxDxChqmkRERETqOF8PVx67sTUAr/6wj8y8wkrd/9ajOSQdOIWL2cTIa6Iqdd8iNYGaJhEREZF6YEjXMDqG+XGmqIR/LNldqft+69xcpls6NibU37NS9y1SE6hpEhEREakHzGYT0wa3A2Bh8lG2HM2ulP0eOV3Ad1uPAzC6b7NK2adITaOmSURERKSeiIlswB+6NMFuh6e+3l4pS5C/9/NBSsvs9GkRRLtQv0pIKVLzqGkSERERqUceu7ENnq4WNqZm8/XmtN+1r5wCG5+uPwLAGJ1lkjpMTZOIiIhIPRLi58H4/s0BmPHdLgqKS37zvj785TAFxaW0CfHl2pZBlRVRpMZR0yQiIiJSzzx4bTPCGniSnlvIGyv2/6Z9FJWUMn/NIaD8LJPJZKrEhCI1i5omERERkXrGw9XClPi2ALy56gBHThdc9T6+2pTGibwiQqwe3NIxtLIjitQoappERERE6qG4diHENgukuKSM57/beVWvLSuzO5YZf6BPU9xc9COl1G16h4uIiIjUQyaTiWm3RmM2wffb0lmz/6TTr12xJ5N9mWfwcXfhnh4RVZhSpGZQ0yQiIiJST7UJsTKsZyQA07/ZQUlpmVOve2tV+Vmm+3pGYPVwrbJ8IjWFmiYRERGReixhYCv8PF3ZlZ7Hx+eWD7+cLUezWXvgNC5mE3/q3bTqA4rUAGqaREREROqxBt5uJAxsBcCspbvJLii+7PjzZ5lu7RRKqL9nlecTqQnUNImIiIjUc8N6RtCqkQ9ZBTZeXrb3kuOOnC7gu63HgfJly0XqCzVNIiIiIvWci8XM1FvaAfCvtYfZk5F30XHvrj5ImR2ubRlEdKi1OiOKGEpNk4iIiIjQp2UQg6IbUVpm55lvd2C32ytszy6w8dmG8jlPY/rqLJPUL2qaRERERASAv8e3xc1i5qe9J0nckVFh28frj1BQXErbxlb6tAgyKKGIMdQ0iYiIiAgAkYHejLo2CoBnF+2kqKQUgJIy+GBtKgBj+kZhMpkMyyhiBEObphkzZtC9e3d8fX0JDg7m9ttvZ/fu3RXG9OvXD5PJVOFj7NixFcakpqYSHx+Pl5cXwcHBPPLII5SUlFQYs2LFCrp27Yq7uzstWrRg/vz5F+SZM2cOTZs2xcPDg549e7Ju3boK2wsLCxk/fjyBgYH4+PgwZMgQMjIyLtiPiIiISG01vn8Lgn3dST1dwHurDwGw/oSJk2eKaeznwS0dQ40NKGIAQ5umlStXMn78eNauXUtiYiI2m41BgwaRn59fYdzo0aM5fvy442PmzJmObaWlpcTHx1NcXMyaNWt4//33mT9/PlOnTnWMOXjwIPHx8fTv35+UlBQmTpzIgw8+yJIlSxxjPv30UxISEpg2bRobN26kU6dOxMXFkZmZ6RgzadIkvvnmGxYuXMjKlStJS0vjjjvuqMIKiYiIiFQvH3cXHruxDQCv/bCX9NxCfjxe/iPjA9dE4WrRhUpS/7gYefDFixdXeDx//nyCg4NJTk6mb9++jue9vLwICQm56D6WLl3Kjh07WLZsGY0aNaJz584888wzPPbYYzz11FO4ubkxd+5coqKieOmllwBo27Ytq1evZvbs2cTFxQEwa9YsRo8ezciRIwGYO3cuixYt4r333uPxxx8nJyeHd999lwULFnD99dcDMG/ePNq2bcvatWvp1atXpddHRERExAh/6NKED9YeZvORbB54P5mMsyZ83F24p0e40dFEDGFo0/S/cnJyAAgICKjw/EcffcSHH35ISEgIgwcP5sknn8TLywuApKQkOnToQKNGjRzj4+LiGDduHNu3b6dLly4kJSUxYMCACvuMi4tj4sSJABQXF5OcnMzkyZMd281mMwMGDCApKQmA5ORkbDZbhf20adOGiIgIkpKSLto0FRUVUVRU5Hicm5sLgM1mw2azXXV9Ksv5YxuZobZQrZynWjlPtXKeauU81erqqF5XNuWmVtz11jr2ZpZfAXR318Z4WFSzy9H7ynk1pVbOHr/GNE1lZWVMnDiRa665hvbt2zuev++++4iMjCQ0NJQtW7bw2GOPsXv3bj7//HMA0tPTKzRMgONxenr6Zcfk5uZy9uxZsrKyKC0tveiYXbt2Ofbh5uaGv7//BWPOH+d/zZgxg6effvqC55cuXepo+oyUmJhodIRaQ7VynmrlPNXKeaqV81Srq6N6XV73hmbWnzBjNtmJLDrId98dNDpSraD3lfOMrlVBQYFT42pM0zR+/Hi2bdvG6tWrKzw/ZswYx987dOhA48aNueGGG9i/fz/Nmzev7phXZfLkySQkJDge5+bmEh4ezqBBg7BajbshnM1mIzExkYEDB+Lq6mpYjtpAtXKeauU81cp5qpXzVKuro3o5p3teEX/5dDON7Ke46xbV6kr0vnJeTanV+SvBrqRGNE0TJkzg22+/ZdWqVYSFhV12bM+ePQHYt28fzZs3JyQk5IJV7s6vaHd+HlRISMgFq9xlZGRgtVrx9PTEYrFgsVguOubX+yguLiY7O7vC2aZfj/lf7u7uuLu7X/C8q6trjfhCqik5agPVynmqlfNUK+epVs5Tra6O6nV5oQGuLHiwB999951qdRVUK+cZXStnj23o8id2u50JEybwxRdf8MMPPxAVFXXF16SkpADQuHFjAGJjY9m6dWuFVe4SExOxWq1ER0c7xixfvrzCfhITE4mNjQXAzc2NmJiYCmPKyspYvny5Y0xMTAyurq4VxuzevZvU1FTHGBERERERqXsMPdM0fvx4FixYwFdffYWvr69jbpCfnx+enp7s37+fBQsWcPPNNxMYGMiWLVuYNGkSffv2pWPHjgAMGjSI6Ohohg8fzsyZM0lPT2fKlCmMHz/ecZZn7NixvPbaazz66KM88MAD/PDDD3z22WcsWrTIkSUhIYERI0bQrVs3evTowcsvv0x+fr5jNT0/Pz9GjRpFQkICAQEBWK1WHnroIWJjY7VynoiIiIhIHWZo0/TGG28A5Tew/bV58+bxpz/9CTc3N5YtW+ZoYMLDwxkyZAhTpkxxjLVYLHz77beMGzeO2NhYvL29GTFiBNOnT3eMiYqKYtGiRUyaNIlXXnmFsLAw3nnnHcdy4wBDhw7lxIkTTJ06lfT0dDp37szixYsrLA4xe/ZszGYzQ4YMoaioiLi4OF5//fUqqo6IiIiIiNQEhjZNdrv9stvDw8NZuXLlFfcTGRnJd999d9kx/fr1Y9OmTZcdM2HCBCZMmHDJ7R4eHsyZM4c5c+ZcMZOIiIiIiNQNuqWziIiIiIjIZahpEhERERERuQw1TSIiIiIiIpehpklEREREROQy1DSJiIiIiIhchpomERERERGRy1DTJCIiIiIichlqmkRERERERC5DTZOIiIiIiMhlqGkSERERERG5DBejA9QndrsdgNzcXENz2Gw2CgoKyM3NxdXV1dAsNZ1q5TzVynmqlfNUK+epVldH9XKeauU81cp5NaVW538uP/9z+qWoaapGeXl5AISHhxucREREREREzsvLy8PPz++S2032K7VVUmnKyspIS0vD19cXk8l02bHdu3dn/fr1Tu/7asbn5uYSHh7OkSNHsFqtTh+jPqqNtbra905lqY5aVfbnVhn7+y37+K21Murf1kjV9TVYm2p7qax1pVbV9XXubL1+T57f8tqa+F6sjf8X/q/qqmtl1crI90F1HLt79+4sX768RtTKbreTl5dHaGgoZvOlZy7pTFM1MpvNhIWFOTXWYrFc1RvoascDWK3WWvvNr7rVplr9lvdCZarKWlX251YZ+/s9+7jaWhn9b2ukqv4arE21vVLW2l6r6v46v1K9fk+e3/LamvxerE3/F/6v6q7r762Vke+D6jj2r49RE2p1uTNM52khiBpq/PjxVTpe6q66/F6o7M+tMvZXnfWuy/+2RqtNtTU6a1Ufv6Z9nf+e1/+W1xr971tX1ba6Gpm3Oo5dmceorlrp8rx6KDc3Fz8/P3Jycmrtb4yqi2rlPNXKeaqV81Qr56lWV0f1cp5q5TzVynm1rVY601QPubu7M23aNNzd3Y2OUuOpVs5TrZynWjlPtXKeanV1VC/nqVbOU62cV9tqpTNNIiIiIiIil6EzTSIiIiIiIpehpklEREREROQy1DSJiIiIiIhchpomERERERGRy1DTVI+sWrWKwYMHExoaislk4ssvvzQ6Uo00Y8YMunfvjq+vL8HBwdx+++3s3r3b6Fi1wgsvvIDJZGLixIlGR6mRSktLefLJJ4mKisLT05PmzZvzzDPPoPV4nPv+tHPnTm699Vb8/Pzw9vame/fupKamVn9Yg73xxht07NjRcUPI2NhYvv/+ewBOnz7NQw89ROvWrfH09CQiIoKHH36YnJwcg1Mb59ixY/zxj38kMDAQT09POnTowIYNGy46duzYsZhMJl5++eXqDWmAy33N2Ww2HnvsMTp06IC3tzehoaHcf//9pKWlVdjHnj17uO222wgKCsJqtdKnTx9+/PHHav5Mqp4zPxf069cPk8lU4WPs2LEX7Gv+/Pl07NgRDw8PgoODa939o67kqaeeuqAObdq0cWx/66236NevH1arFZPJRHZ2doXXHzp0iFGjRlX4f3LatGkUFxdX82dyITVN9Uh+fj6dOnVizpw5Rkep0VauXMn48eNZu3YtiYmJ2Gw2Bg0aRH5+vtHRarT169fz5ptv0rFjR6Oj1Fgvvvgib7zxBq+99ho7d+7kxRdfZObMmbz66qtGRzPclb4/7d+/nz59+tCmTRtWrFjBli1bePLJJ/Hw8KjmpMYLCwvjhRdeIDk5mQ0bNnD99ddz2223sX37dtLS0khLS+Mf//gH27ZtY/78+SxevJhRo0YZHdsQWVlZXHPNNbi6uvL999+zY8cOXnrpJRo0aHDB2C+++IK1a9cSGhpqQNLqd7mvuYKCAjZu3MiTTz7Jxo0b+fzzz9m9eze33nprhXG33HILJSUl/PDDDyQnJ9OpUyduueUW0tPTq+vTqBbO/lwwevRojh8/7viYOXNmhe2zZs3i73//O48//jjbt29n2bJlxMXFVeenUi3atWtXoQ6rV692bCsoKODGG2/kiSeeuOhrd+3aRVlZGW+++Sbbt29n9uzZzJ0795Ljq5Vd6iXA/sUXXxgdo1bIzMy0A/aVK1caHaXGysvLs7ds2dKemJhov+666+x/+ctfjI5UI8XHx9sfeOCBCs/dcccd9mHDhhmUqGa62PenoUOH2v/4xz8aE6gWaNCggf2dd9656LbPPvvM7ubmZrfZbNWcyniPPfaYvU+fPlccd/ToUXuTJk3s27Zts0dGRtpnz55d9eFqEGd+Jli3bp0dsB8+fNhut9vtJ06csAP2VatWOcbk5ubaAXtiYmJVxjXcxX4uuNL/fadPn7Z7enraly1bVg0JjTNt2jR7p06drjjuxx9/tAP2rKysK46dOXOmPSoq6veH+510pknkCs5f1hIQEGBwkppr/PjxxMfHM2DAAKOj1Gi9e/dm+fLl7NmzB4DNmzezevVqbrrpJoOT1WxlZWUsWrSIVq1aERcXR3BwMD179tQlxpRf8vnJJ5+Qn59PbGzsRcfk5ORgtVpxcXGp5nTG+/rrr+nWrRt33XUXwcHBdOnShbfffrvCmLKyMoYPH84jjzxCu3btDEpa8+Xk5GAymfD39wcgMDCQ1q1b88EHH5Cfn09JSQlvvvkmwcHBxMTEGBu2il3q54KPPvqIoKAg2rdvz+TJkykoKHBsS0xMpKysjGPHjtG2bVvCwsK4++67OXLkSLVmrw579+4lNDSUZs2aMWzYsN99GXVOTk6N+Bms/n0HFbkKZWVlTJw4kWuuuYb27dsbHadG+uSTT9i4cSPr1683OkqN9/jjj5Obm0ubNm2wWCyUlpby3HPPMWzYMKOj1WiZmZmcOXOGF154gWeffZYXX3yRxYsXc8cdd/Djjz9y3XXXGR2x2m3dupXY2FgKCwvx8fHhiy++IDo6+oJxJ0+e5JlnnmHMmDEGpDTegQMHeOONN0hISOCJJ55g/fr1PPzww7i5uTFixAig/LJZFxcXHn74YYPT1lyFhYU89thj3HvvvVitVgBMJhPLli3j9ttvx9fXF7PZTHBwMIsXL77o5Y91xaV+LrjvvvuIjIwkNDSULVu28Nhjj7F7924+//xzoPy9WFZWxvPPP88rr7yCn58fU6ZMYeDAgWzZsgU3NzejPqVK1bNnT+bPn0/r1q05fvw4Tz/9NNdeey3btm3D19f3qve3b98+Xn31Vf7xj39UQdqrZPSpLjEGujzPKWPHjrVHRkbajxw5YnSUGik1NdUeHBxs37x5s+M5XZ53aR9//LE9LCzM/vHHH9u3bNli/+CDD+wBAQH2+fPnGx2tRvnf70/Hjh2zA/Z77723wrjBgwfb77nnnmpOVzMUFRXZ9+7da9+wYYP98ccftwcFBdm3b99eYUxOTo69R48e9htvvNFeXFxsUFJjubq62mNjYys899BDD9l79eplt9vt9g0bNtgbNWpkP3bsmGO7Ls+rqLi42D548GB7ly5d7Dk5OY7ny8rK7Lfeeqv9pptusq9evdqenJxsHzdunL1Jkyb2tLS0akpe/Zz9uWD58uV2wL5v3z673W63P/fcc3bAvmTJEseYzMxMu9lsti9evLhKMxspKyvLbrVaL7h82JnL844ePWpv3ry5fdSoUVWc0jm6PE/kEiZMmMC3337Ljz/+SFhYmNFxaqTk5GQyMzPp2rUrLi4uuLi4sHLlSv75z3/i4uJCaWmp0RFrlEceeYTHH3+ce+65hw4dOjB8+HAmTZrEjBkzjI5WowUFBeHi4nLBmZS2bdvWy9XzANzc3GjRogUxMTHMmDGDTp068corrzi25+XlceONN+Lr68sXX3yBq6urgWmN07hx48u+b3766ScyMzOJiIhwfA87fPgwf/3rX2natKkBiWsWm83G3XffzeHDh0lMTHScZQL44Ycf+Pbbb/nkk0+45ppr6Nq1K6+//jqenp68//77BqauOlfzc0HPnj2B8jMlUP5eBCq8Hxs2bEhQUFCd/j7m7+9Pq1atHHVwVlpaGv3796d379689dZbVZTu6ujyPJH/Ybfbeeihh/jiiy9YsWIFUVFRRkeqsW644Qa2bt1a4bmRI0fSpk0bHnvsMSwWi0HJaqaCggLM5oq/q7JYLJSVlRmUqHZwc3Oje/fuFyzxu2fPHiIjIw1KVbOUlZVRVFQEQG5uLnFxcbi7u/P111/XyxUGz7vmmmsu+74ZPnz4BXMx4+LiGD58OCNHjqy2nDXR+YZp7969/PjjjwQGBlbYfn6+zv9+TzObzXXue9pv+bkgJSUF+G+zdM011wCwe/duR8N1+vRpTp48Wae/j505c4b9+/czfPhwp19z7Ngx+vfvT0xMDPPmzbvgPWYUNU31yJkzZyp0+gcPHiQlJYWAgAAiIiIMTFazjB8/ngULFvDVV1/h6+vrWDrVz88PT09Pg9PVLL6+vhfM9fL29iYwMFBzwC5i8ODBPPfcc0RERNCuXTs2bdrErFmzeOCBB4yOZrgrfX965JFHGDp0KH379qV///4sXryYb775hhUrVhgX2iCTJ0/mpptuIiIigry8PBYsWMCKFStYsmQJubm5DBo0iIKCAj788ENyc3PJzc0Fyn+rXd9+kTFp0iR69+7N888/z9133/3/27t71ijWMAzAjyRuYhE/JgQShCxoSFIIYiFoE1YECz8QQQhWgVQiSBS0EEH/gAEbmxVN5VYWStTCwq1sFCy0loWAErSzs5DHQs7C+cho5GxG8bpgu2W432Fmd+6d992JFy9eRLPZ7P5yPTw8/K8ysHnz5hgdHY2pqakqIm+YsnNubGwsTp8+Ha9evYpHjx7Fly9fut+FRVFErVaLgwcPxo4dO2Jubi6uXbsWW7Zsidu3b0en04ljx45VNaye+N51wdu3b6PVasXRo0djeHg4Xr9+HRcvXoyZmZnuYzgmJyfj5MmTsbCwEM1mM7Zu3RpXrlyJ6enpOHToUJXD+19dunQpTpw4EfV6Pd6/fx/Xr1+Pvr6+OHPmTERErK6uxurqavfYe/PmTQwNDcX4+HgURRHv3r2LRqMR9Xo9bty4ER8/fuxue3R0tJIxdVU9P5CN89f80X++5ubmqo72S/mvfRQRubS0VHW034I1TWv79OlTLiws5Pj4eA4ODuauXbvy6tWr+fnz56qjVe5HPp/u3LmTExMTOTg4mHv37s0HDx5UF7hC8/PzWa/Xs1ar5cjISB4+fDifPn2amWvvx4jITqdTbfCKLC8v5549e3JgYCCnp6ez2WyWvv9PWdNUds51Op01j6N2u93dxsuXL/PIkSNZFEUODQ3lgQMH8smTJ9UNqke+d12wsrKSMzMzWRRFDgwM5MTERF6+fPlva8Ayv60znJ+fz+3bt2dRFHnq1KlcWVmpYES9Mzs7m2NjY1mr1XLnzp05OzvbXdeV+e0vycv25dLS0pr7u2qbMj2KHgAAYC2/xiRBAACAX5TSBAAAUEJpAgAAKKE0AQAAlFCaAAAASihNAAAAJZQmAACAEkoTAABACaUJAH5Qo9GICxcuVB0DgA2mNAEAAJRQmgAAAEooTQDwkx4/fhzbtm2Le/fuVR0FgB7qrzoAAPyOWq1WnD17NlqtVhw/frzqOAD0kDtNALBOt27dinPnzsXy8rLCBPAHcKcJANbh/v378eHDh3j+/Hns37+/6jgAbAB3mgBgHfbt2xcjIyNx9+7dyMyq4wCwAZQmAFiH3bt3R7vdjocPH8b58+erjgPABjA9DwDWaXJyMtrtdjQajejv74+bN29WHQmAHlKaAOAnTE1NxbNnz6LRaERfX18sLi5WHQmAHtmUJmQDAACsyZomAACAEkoTAABACaUJAACghNIEAABQQmkCAAAooTQBAACUUJoAAABKKE0AAAAllCYAAIASShMAAEAJpQkAAKDEVyQwkXs6kWTSAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "bench_k = np.exp2(np.arange(10)).astype(np.int32)\n", - "bench_avg = np.zeros_like(bench_k, dtype=np.float32)\n", - "bench_std = np.zeros_like(bench_k, dtype=np.float32)\n", - "for i, k in enumerate(bench_k):\n", - " r = %timeit -o ivf_pq.search(search_params, index, queries, k, handle=resources); resources.sync()\n", - " bench_avg[i] = (queries.shape[0] * r.loops / np.array(r.all_runs)).mean()\n", - " bench_std[i] = (queries.shape[0] * r.loops / np.array(r.all_runs)).std()\n", - "\n", - "fig, ax = plt.subplots(1, 1, figsize=plt.figaspect(1/2))\n", - "ax.errorbar(bench_k, bench_avg, bench_std)\n", - "ax.set_xscale('log')\n", - "ax.set_xticks(bench_k, bench_k)\n", - "ax.set_xlabel('k')\n", - "ax.grid()\n", - "ax.set_ylabel('QPS');" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Number of probes\n", - "IVF-PQ search runs in two phases; first it looks for nearest clusters,\n", - "then it searches for the neighbors in every selected cluster.\n", - "\n", - "We can set how many clusters we want to inspect.\n", - "For this, `ivf_pq.SearchParams` has a parameter `n_probes`.\n", - "This is the core parameter to control the QPS/recall trade-off." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3.67 ms ± 3.91 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "4.78 ms ± 1.74 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "6.65 ms ± 3.72 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "10.2 ms ± 4.86 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "17.2 ms ± 14.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "60.2 ms ± 16.6 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "115 ms ± 41.2 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "222 ms ± 184 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "430 ms ± 143 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "829 ms ± 162 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "1.6 s ± 354 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "bench_probes = np.exp2(np.arange(11)).astype(np.int32)\n", - "bench_qps = np.zeros_like(bench_probes, dtype=np.float32)\n", - "bench_recall = np.zeros_like(bench_probes, dtype=np.float32)\n", - "k = 100\n", - "for i, n_probes in enumerate(bench_probes):\n", - " sp = ivf_pq.SearchParams(n_probes=n_probes)\n", - " r = %timeit -o ivf_pq.search(sp, index, queries, k, handle=resources); resources.sync()\n", - " bench_qps[i] = (queries.shape[0] * r.loops / np.array(r.all_runs)).mean()\n", - " bench_recall[i] = calc_recall(ivf_pq.search(sp, index, queries, k, handle=resources)[1], gt_neighbors)\n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It's clear that the search time scales almost linearly with the number of probes.\n", - "This is due to the algorithm spending most of the time in the second phase scanning through individual clusters.\n", - "Thanks to the balanced nature of the clustering k-means algorithm, the sizes of the clusters are roughly similar;\n", - "hence the linear relation `n_probes` ~ query time.\n", - "\n", - "Let's draw some plots to illustrate how the number of probes affects QPS and recall." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABR8AAAFzCAYAAAC3uH7uAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAACbY0lEQVR4nOzdeVxVdf7H8de9l8u+KCCgiCK44r7gntmiZpPtZatmZctg08Q0lTOTTjWT09Q4/WaiLMtsnWyblslMs0zNfU9xR0VREFBZFS7c+/sDJUlUhMs9917ez8eDR91z7znnfbjgFz58zvdrcjgcDkRERERERERERESczGx0ABEREREREREREfFOKj6KiIiIiIiIiIhIo1DxUURERERERERERBqFio8iIiIiIiIiIiLSKFR8FBERERERERERkUah4qOIiIiIiIiIiIg0ChUfRUREREREREREpFGo+CgiIiIiIiIiIiKNwsfoAK5mt9s5ePAgISEhmEwmo+OIiMg5OBwOioqKaNWqFWaz/l5WG41rIiKeQ+Pa+WlcExHxDBcypjW54uPBgweJi4szOoaIiFyA/fv307p1a6NjuCWNayIinkfj2tlpXBMR8Sx1GdOaXPExJCQEqPrkhIaG1usYNpuN+fPnM3LkSKxWqzPjKYMyeHQOZVAGZ2coLCwkLi6u+t9uOZPGNWVQBu/N4C45lMF5GTSunZ8zxrXauMPXkLN54zWBrsuTeOM1ga6rri5kTGtyxcdTrfuhoaEN+iUtMDCQ0NBQQ3/4UQZlcLccyqAMjZVBt12dncY1ZVAG783gLjmUwfkZNK6dnTPGtdq4w9eQs3njNYGuy5N44zWBrutC1WVM00QjIiIiIiIiIiIi0ihUfBQREREREREREZFGoeKjiIiIiIiIiIiINAoVH0VERERERERERKRRqPgoIiIiIiIiIiIijULFRxEREREREREREWkUKj6KiIh4oLS0NJKSkkhOTjY6ioiISINpXBMR8V4qPoqIiHiglJQU0tPTWb16tdFRREREGkzjmoiI91LxUURERERERERERBqFj9EBRETqq6LSTkl5JSVlFZSUVVBcVkFBaRlbjpoI2J6LxWLB7gCHw4GDk/91gAOwn/b/P293YLeftq0u+zgcVec4bVtFZSVbs0xkLd2DxWyp17U5Gvi5qaysZHuWiQNL9mCx1C9DQ53KEHuggH7tIg3JIOe3OauAz9cfILLU6CQiIiIN9+aPezCbTCS2CCYxKoiYUH9MJpPRsUREmjTDi49paWk8//zzZGdn07NnT/7973/Tv3//Wl9rs9mYNm0ab731FllZWXTq1InnnnuOK664wsWpRaQ+7HYHJeUVlJRVUnyyYHiqaFha/sttVUXF4vKa20rLf97nhM1+ljNZeG3bepdeW20ZvsjcaXgG3CBD0r6jKj66sTmr9/POin2AD1/mruCGvq25umcrIoL9jI4mIiJywWb8sJucwrLqx4G+FhJaBNEuIpDKoyZMm7PpEBNGu8gg/K3G/IFWRKSpMbT4OGfOHFJTU5kxYwYDBgzgxRdfZNSoUWzfvp2oqKgzXv+nP/2Jd999l5kzZ9K5c2e++eYbrrvuOpYtW0bv3r0NuAIRATh47Dhr9h1l7d58ftph5r/56yi12Sn5RVGxtLyyUc5vtZgI8vMhyNeHIF8Lx0uKaNYsDLPZjAkwmcBsMlX/vwlT1X9P/r/ZfPo2U+37/HK76cx9zCf/32G3k3Uwi9atW2M2GTO7hd1hJ+vAAWLdIEOHqGBDzi91c2nnKA4eK+X77YfZfLCQzQfT+etXWxneKYob+sRyaZco/Hz0y5mIiLg/u93Bdb1bszu3mN25xWTml1JaXsnmrEI2ZxUCFubO2QRU/UzXunlAVYdki2ASWgRV/39ksK+6JUVEnMjQ4uP06dOZOHEiEyZMAGDGjBl89dVXzJo1iyeeeOKM17/zzjv88Y9/5MorrwTgwQcf5Ntvv+Uf//gH7777rkuzizRVlXYHWw8Vsnbf0ZMFxyMcLDhx2ivMkJ93zmNYzCaCfC0E+/kQ5OdDoJ8PwX4Wgnx9qrcFndpW/f+/2OZ76nWWGoURm83G3LlzufLKgVit1kb6LJxbVYb9XHllN4MzZLpFhmEd1PXozi7pHMXQxOZ8+PlcTkR347MNh/gpq4Bvt+bw7dYcwgKsjOnZkuv7tKZ3XDP9MiYiIm7LbDbxxOjO1Y9tlXYyj5Sy+3AxO7ILWbxhO+X+zcnILaHwRAX7jxxn/5HjLNqeW+M4of4+JEYFkxBZdev2qaJk24hArBYtmyAicqEMKz6Wl5ezdu1aJk+eXL3NbDZz+eWXs3z58lr3KSsrw9/fv8a2gIAAli5detbzlJWVUVb2c9t9YWEhUPVLsc1mq1f2U/vVd39nUAZlcFWO4rIKNuwvYF3mUdZmHmPj/gJKftHBaDGb6BITQs/WIZTkZNKnexKhgX4EnSwoBp0sGAb7Vv3Xz8fsvAKGw47ttNuv3eH9UAbnZTD6+6opCbbCzQPbcM9FiezIKeLTdVn8d/0BcgrLeHdFJu+uyCQhMojr+8Rybe9YWjcPNDqyiIjIOVkt5urC4SUdI4gr3sqVVw7Ax8eH/JJydh8uZnduCbtzi8nIrfr//UdLKTxRwfrMY6zPPFbjeD5mE23CA0loUbMomdgiiGaBvsZcpIiIBzCs+JiXl0dlZSXR0dE1tkdHR7Nt27Za9xk1ahTTp09n2LBhJCYmsnDhQj799FMqK89+K+e0adN46qmnztg+f/58AgMb9ovTggULGrS/MyiDMvxSQ3McKYM9RSb2FJrIKDJxsBQc1CwU+lkctAt2kBDqoF0ItA124Gc5AhyBVkD+FsiHUqo+cms5T2Nzh/dDGRqeobRUq6AYoWN0CE+M7szvR3Vi2e48Pl2XxbzN2WTklfDC/B28MH8HgxIiuL5PLKO7tyTYz/AppEVEROrMZDIRGexHZLAfAxIiajx3wlbJ3vwSdh8uOVmQ/LlAWVpeSUZeCRl5JXy7teYxI4J8qxe5Of027tbNA7GYddeAiDRtHvXbwv/93/8xceJEOnfujMlkIjExkQkTJjBr1qyz7jN58mRSU1OrHxcWFhIXF8fIkSMJDQ2tVw6bzcaCBQsYMWKEobczKoMyNDRHRaWd7TnFrMs8xtrMY6zdd5Ts0yboPiW2mT992jSjb5tm9GnTnI7RwbX+EOUOnwtl8K4Mp7rVxRgWs4mLOrTgog4teObaCr7+6RCfrstieUZ+9ceUz7dwRbcYru8Ty+DESP2CJSIiHs3faqFzTCidY2r+ruhwOMguPEHGyULk6V2ThwpOkF9STn7JEVbtPVJjP18fM+0igkiMCqpxG3dCi2D98U5EmgzD/rWLjIzEYrGQk5NTY3tOTg4xMTG17tOiRQs+++wzTpw4QX5+Pq1ateKJJ54gISHhrOfx8/PDz+/MFTutVmuDfyF3xjEaShmU4UJyFJdVsD7zKGv2HmXtvqOszzxa6y3USS1D6du2Of3im9OvbTgxYf61Hq8+GVxFGbwjg9HZ5WfBfj7c1C+Om/rFceBoKZ+tz+LTdVlk5JXw3/VZ/Hd9FjGh/lzbO5Yb+sTSITrE6MgiIiJOYzKZaBkWQMuwAIa0rzmfdUlZBXvyzixKZuSVUF5hZ3tOEdtzis44Zkyo/2kL3QSRGFV1G3dMqD9m/TFPRLyIYcVHX19f+vbty8KFC7n22msBsNvtLFy4kEmTJp1zX39/f2JjY7HZbHzyySfcfPPNLkgs4nmyjh1nzd4jVYvD7D3KtuxC7I6arwnx86F32+b0O/nRM64ZQforrIicQ+vmgUy6tAMpl7Rnw/5jfLLuAF9uPER24Qlm/LCbGT/spntsGDf0iWVMz1ZEBJ/5R0ARERFvEeTnQ7fYMLrFhtXYXml3cPDYcXadLEpm5JVUFyfzisvILjxBduEJlu3Or7FfgNVSY/XtU92S7SKD8LdaEBHxNIZWGFJTUxk/fjz9+vWjf//+vPjii5SUlFSvfj1u3DhiY2OZNm0aACtXriQrK4tevXqRlZXFn//8Z+x2O4899piRlyHiFioq7ewvhrdXZLJ+fwFr9x3lUI1VqKu0bh5Av7bN6RsfTr+2zekYHaLbJEWkXkwmE73bNKd3m+Y8eVUS3287zCfrsvh+22F+yirgp6wC/vLVVoZ3iuLGvrFc0jmqxur0IiIi3sxiNhEXHkhceCCXdIqq8VxBqY3decW/uI27mH35pRy3VbLlYCFbDtacfsZkgthmAT+vvh3uT26BidyiMlo293Hego4iIk5maPFx7Nix5ObmMmXKFLKzs+nVqxfz5s2rXoQmMzMTs9lc/foTJ07wpz/9iYyMDIKDg7nyyit55513aNasmUFXIGI8u93B28v3Mn3BDgpP+MBPPy/YZDGb6Nrq5C3UbcPpF9+c6NALu4VaRKQu/HwsXNGtJVd0a0l+cRlfbjzIp+uz2HSggG+35vDt1hzCAqyM6dmS6/u0pndcM/2SJCIiTVZYoJU+bZrTp03zGtttlXb2HymtvnX7VFFyd24JBcdtHDh6nANHj/PDjlNLOlp4Kf0HQvx9qouSp7om20cF0SY8CF8f85kBRERcyPB7KydNmnTW26wXLVpU4/HFF19Menq6C1KJeIbM/FJ+//FGVu6pmtg6wOIgOSGS5PgI+sY3p1dcMwJ9Df82F5EmJiLYj7uGtOOuIe3YkVPEp+uy+Gx9FtmFJ3h3RSbvrsgkITKI6/vEcl2f1sQ2CzA6soiIiFuwWswknFyQZgTR1dsdDgdHSsprFCV3HS5i875cjpSbKDpRwYb9x9iw/1iN41nMJtqGB55xG3dCZDDNg3xdfHUi0lSpKiHigex2B++s2Mffvt7GcVslAVYLj43qQLO8zVz1q75apENE3EbH6BCeGN2Z34/qxLLdeXy6Lot5m7PJyCvhhfk7eGH+DgYlRHBNzxjMlec/noiISFNkMpmICPYjItiP/u3CAbDZbMydO5fLRowiq9BWtchNbnGNAmVJeSUZeSVk5JXw7dbDNY4ZHuRbtdDNaUXJDlEhtG4eoLsTRMSpVHwU8TC/7HYc0C6c52/sSctQK3PnbjY4nYhI7SxmExd1aMFFHVrwzLUVzNuczSdrD7A8I7/6I9DHwr7AXUwYmkCkFqkRERGpEz+rhU4x/nSKCamx3eFwkFNYdrIgWbMoebDgBEdKyjlSUs7qvUdr7Bfs50OnmBA6xYTQJSaETjGhdIoJISxADQ4iUj8qPop4CLvdwbsrq7odS8uruh2fGN2ZOwe2xWw2YbPZjI4oIlInwX4+3Ni3NTf2bc2Bo6V8vuEgH6zKZP/R46QtyuD1pXu5sW9rJl6UQHxkkNFxRUREPJLJZCImzJ+YMH8Gt4+s8VxpecXPi938Yn7J4rIK1u47ytp9NYuSrcL86dyyqhDZOSaEzjGhJLQIwmrRnJIicm4qPop4gMz8Uh77ZCMrMn7udvz7jT1oG6FfykXEs7VuHkjKJe25Z3AbnntvHmtLmrMpq5D3Vmby/qpMRneL4b5hifSKa2Z0VBEREa8R6OtDt9gwusWG1dhuq7SzJ6+ErYcK2Z5dxLbsIrZnF5F17DgHC05wsOAE3237+fZtq8VEYotguvyiKBkd6qdbt0WkmoqPIm7sfN2OItJ0paWlkZaWRmWld0yUaDGb6BXhYPIdA1h3oIhXf9jN99tzmftTNnN/ymZAu3DuvziB4R2j9O+fiIgX8rZxzVNZLWY6RofQMbrmLdwFx23syCli26FCtp1WlCwuq6h+fLpmgVY6RYfUKEp2jA4hyE8lCJGmSN/5Im5q/5FSHvt4E8sz8gHo3y6c59XtKCInpaSkkJKSQmFhIWFhYeffwUOYTCYGJkQwMCGC7dlFvLY4gy82ZrFyzxFW7jlCx+hg7huWyNU9W+Hro9u8RES8hbeOa94iLMBKcnw4yfHh1dscDgcHjh4/2SH5c1FyT14Jx0pt1WP36dpGBNIpOoTOLUNPdkmG0DYiCIv+sCji1VR8FHEzdruD91buY9pp3Y6PX9GJcYPi1e0jIk1Kp5gQ/nFzTx4d1ZE3f9zL+ysz2ZFTzKMfbeSFb7Zz99B4bu3fhhB/TYAvIiLiaiaTibjwQOLCA7k8Kbp6+wlbJbsOF5/sjvy5KJlbVMa+/FL25ZcyPz2n+vX+1qpuy1NFyfaRARRrOnsRr6Lio4gbOaPbMT6c529St6OING0twwL4w5VdmHRpe95fmcmspXvILjzBs3O38e+Fu7htYBvuHtKO6FB/o6OKiIg0ef5WS63zSeYXl7E9u4itJ4uS27OL2J5TxAmbnU0HCth0oOC0V/vwz22Lqm7bPq1Tsn1UMP5Wi2svSEQaTMVHETdgtzt4b1Um0+ZupbS8En+rmcev6Mx4dTuKiFQL9bfywMWJTBgSz+cbDjJzcQY7Dxfz6g8ZzFq6h2t7xXLfsAQ6/GKeKhERETFeRLAfg9v71Vh5u9LuIPNI6WlzSRay7VARmUdKyCsuZ8nOPJbszKt+vcVsIj4isKoYeVpRMrZZgH5vEnFjKj6KGGz/kVIe/2QTy3b/3O349xt7EB+pbkcRkdr4+Vi4uV8cN/ZpzffbD/Pq4gxW7TnCR2sP8NHaA1zWOYr7L04kOb65VtoUERFxYxaziXaRQbSLDGJ095YA2Gw2/vvlXNr1GszuvOM/FyWzizhWamN3bgm7c0v4ikPVxwn286FjdPBpc0lWLXQTFqCpWUTcgYqPIgZxOBy8t7Kq27HkZLfjY6M6c9dgdTuKiNSF2Wzisi7RXNYlmvWZR3ltcQbztmSzcNthFm47TK+4ZjxwcQIjkmI0kb2IiIgH8bNAr7hmJCe0qN7mcDg4XFTG1kOFJxe5qfrYdbhq1e11mcdYl3msxnFahflXrbZ9WlEyoUUQVosWrRNxJRUfRQxw4GhVt+OPu6q6HZPjm/P8jT3V7SgiUk+92zTnlTv6sievhJlLMvh47QE27D/GA++uIz4ikInDErihT2vNEyUiIuKhTCYT0aH+RIf6M7xTVPV2W6WdPXklNYqS27OLyDp2nIMFJzhYcILvt+dWv95qMZHYIriqGNmyqkOyS0wo0aF+umNCpJGo+CjiQg6Hg/dXZfLsVz93O/5+VGcmqNtRRMQp2kUG8ex13Xnk8o68vXwvby/fx978Uv74381Mn7+DuwbHc+egtjQL9DU6qoiIiDiB1VK1WnbHX8z5XHDcxo6cotPmk6wqShaXVVQ/ZsPB6teHBVhPdkf+XJTsFB1CkJ/KJiINpe8iERc5cLSUJz75iaW7qiZM7te2Oc/f1JN26nYUEXG6FiF+/G5kJx64OJEP1+zn9SV7yDp2nH8s2MHLi3YzNjmOe4a2Iy480OioIiIi0gjCAqwkx4eTHB9evc3hcHDg6PGTHZI/FyX35JVQcNzGyj1HWLnnSI3jtAkPPKMoGR8RpCldRC6Aio8ijexs3Y53DY7XgCUi0siC/HyYMKQddw5sy1c/HeK1xRlsOVjI7GV7eWfFPn7VvSX3DUugW2yY0VFFRESkkZlMJuLCA4kLD+TypOjq7Sdslew6XHxGUTK3qIzMI6VkHillfnpO9ev9fKq6LTvHhFTdtn2yKBkZ7GfEZYm4PRUfRRqRuh1FRNyDj8XMNb1iubpnK37clc+ri3ezZGceX2w8yBcbDzK0fST3DGmLw2F0UhEREXE1f6uFbrFhZ/wxMr+47LTFbarmlNyeU8QJm52fsgr4Kaugxusjg/2quyQ7nVzgpkN0sOacliZPxUeRRuBwOPjPqv08O3crxWUV+PmY+f2oTkwY0k7djiIiBjKZTAztEMnQDpFsOVjAzMUZfLnpEEt35bF0Vx7xwRba9iqkd3yE0VFFRETEYBHBfgxu78fg9pHV2yrtDjKPlJ42l2RVUXLfkVLyistYuqusuvkEwMdsoldcMwa0a46pwERZhR2r1YirETGOio8iTpZ17DhPfLKJJTurBpy+bZvz/I09SGgRbHAyERE5XddWYbx4S28eHdWJWUv38sHqTPYWV3L9qyu4rX8bfj+qkxamERERkRosZhPtIoNoFxnE6O4tq7eXllewI6e4RlFyW3YRx0ptrNl3lDX7jgIWZv71O5LjwxncPoLBiZF0axWKj8Vs3AWJuICKjyJOUtXtmMlfv1K3o4iIJ2ndPJApY5K4e3AcD7+5iLV5Zt5bmcncnw7x+BWdublfHGb9Oy4iIiLnEOjrQ6+4ZvSKa1a9zeFwsP/IcZZn5LFkRy4/bDtEkc1efccFbCfE34cB7SIYnBjBkPaRdIwOxmTSzx3iXVR8FHGCI2Vw99vrWLorH4A+bZrx/E09SVS3o4iIx4gO9WdcBzuPXN2fZ77azvacIp749Cf+s3o/z1zTlR6tmxkdUURERDyIyWSiTUQgbSLacH2vlnz11QE6Jg9j1d5jLNudz4qMfApPVPDt1hy+3Vq1oE1ksC+DEiOripGJkcSFB6gYKR5PxUeRBlq6K5+/bbRQVpmPn4+ZR0d24u6h6nYUEfFUA9qF87/fDOXt5fv454IdbNx/jGvSfuSW5DY8NqoTzYN0K7aIiIhcOJMJOkQFkxTbnLuGtKPS7mDLwQJ+3JXPst15rN57hLzicr7ceJAvNx4EILZZQHVX5KDECKJD/Q2+CpELp+KjSAMcKjhO6kebKKs00SsujH/c3EvdjiIiXsBqMXPP0HaM6dGSaV9v47/rs/jPqky+3nyI34/qxC3JbfRHJhEREWkQi9lEj9bN6NG6GQ8OT6SsopINmVVdkct257E+8xhZx47z0doDfLT2AADto4IZnFh1m/bAhAjNTy0eQcVHkXqqqLTz8AcbOFpqo3WQg3fvTiY4wM/oWCIi4kRRof78c2wvbu3fhimfb2ZbdhF//O9m5qzez9PXdKsxr5OIiIhIQ/j5WBiQEMGAhAgeGdGRkrIKVu89wvLd+Szbnc/mgwXsOlzMrsPFvL18HyYTdG0VypDEqq7I/u3CCfRVmUfcj74qRerp39/tYtWeIwT5WhjfoQw/H61QJiLirfq3C+d/D/18K/amAwVc9/KPjO0Xx2NXdCZct2KLiIiIkwX5+TC8UxTDO0UBcKy0nBUZR1i2O49lu/PZdbiYzVmFbM4q5NXFGVgtJnrFNWNQYiRDEiPo1aYZfj4Wg69CRMVHkXpZvjuff3+3E4Cnr07CJ2u9wYlERKSx+VjM3D20HVf1bMnfvt7Gp+uy+GD1fr7enM3vR3Xi1v66FVtEREQaT7NAX67oFsMV3WIAOFx4ovoW7R935ZN17Dir9x5l9d6j/GvhTvytZpLjwxl8cgGbbrFh+llFDKHio8gFOlJSzm/nrMfugJv6tubqni2Zq+KjiEiTERXiz/Sbe3Fb/zY8+fkWth4q5E+fbeaD1Zk8fU03+rRpbnREERERaQKiQv25tncs1/aOxeFwsP/I8apC5O58lu/OI6+4nCU781iyMw+AEH8fBiZEMCQxgsHtI+kQFayVtMUlVHwUuQAOh4NHP9pITmEZCS2CeOqaroDD6FgiImKAfvHhfDlpCO+tzOSF+dvZnFXI9S8v4+Z+rXn8is5EBGseYBEREXENk8lEm4hA2kS04Zb+bXA4HOzIKa7uilyZkU/RiQoWpOewID0HgMhgv+rFa4a0jyQuPNDgqxBvpeKjyAV4Y+kevtt2GF8fMy/d2odAXx9sNpvRsURExCA+FjPjB8dzZfeWPDdvGx+vPcCHaw4wb3M2j47qxO0D2ur2JhEREXE5k8lEp5gQOsWEMGFIOyoq7Ww5WMiPu/NYvjuf1XuPkFdcxhcbD/LFxoMAtG4ewJDESAa3j2BQQgRRof4GX4V4CxUfRepo04FjPDdvGwBP/qoLSa1CDU4kIiLuokWIHy/c1JNb+8fx5GdbSD9UyJTPt/DBqv08c21X+rYNNzqiiIiINGE+FjM945rRM64Zvx7enrKKStZnHquaM3JXHhv2H+PA0ePMWbOfOWv2A9AhKriqM7J9JAPbRRAWaDX4KsRTqfgoUgdFJ2w89J/12CodXNE1hjsGtjU6koiIuKG+bcP58qGhvL9yH89/s530Q4Xc8MpybuzbmidGdyZSt2KLiIiIG/DzsTAwIYKBCRGkjuhISVkFq/YeYfnJBWy2HCxk5+Fidh4u5q3l+zCboFtsGIMSIxiSGEm/+OZYdXOH1JGKjyLn4XA4+ON/N7Mvv5TYZgE8d0MPTcorIk61Z88e7r77bnJycrBYLKxYsYKgoCCjY0k9Wcwm7hz0863YH645wMdrD/DNlmx+N6Ijdwxsi4/FbHRMEZFGo3FNxPME+flwSacoLukUBcDRknJW7snnx11VxcjduSVsOlDApgMFvPpDBlaLiZ6tw4isNNFi71H6tYvE10c/30jtVHwUOY+P1hzgi40HsZhN/OvWXmo1FxGnu+uuu/jLX/7CRRddxJEjR/DzU3ecN4gI9uPvN/bklv5tmPL5ZjZnFfLnL9OZs+YAz1zTlX7xuhVbRLyTxjURz9c8yJcrurXkim4tAcgpPMGy3Xks25XPst35ZB07zpp9xwAL895YTYDVQnK78KrFaxIjSWoVqnmvpZqKjyLnsOtwEVO+2AxA6oiOmrNLRJxuy5YtWK1WLrroIgDCw/XvjLfp06Y5n6cM5f1VmbzwzXa2HirkxhnLub5PLJNHd6FFiH4pFxHvoXFNxDtFh/pzXe/WXNe7NQ6Hg8wjpSzZcZhPlm5m3wk/jpTYWLwjl8U7cgEIC7AyMCGcwYmRDGkfQWKLYN1B2ISpJ1bkLE7YKpn0/npO2OwMbR/JgxcnGh1JRNzQ4sWLGTNmDK1atcJkMvHZZ5+d8Zq0tDTi4+Px9/dnwIABrFq1qvq5nTt3EhwczJgxY+jTpw/PPvusC9OLq1jMJu4c2JbvHx3OLclxmEzw6bosLn1hEbOW7qGi0m50RBERQOOaiJyfyWSibUQQY/u15q6OdlY8Ppx5v72IKVclcXmXKEL8fCg4buObLTlM/WILl09fTP9nF/LwB+v5cPV+9h8pNfoSxMUMLz6ea+CqzYsvvkinTp0ICAggLi6ORx55hBMnTrgorTQlf/kqnW3ZRUQG+zJ9bE/MahkXkVqUlJTQs2dP0tLSan1+zpw5pKamMnXqVNatW0fPnj0ZNWoUhw8fBqCiooIlS5bw8ssvs3z5chYsWMCCBQtceQniQuFBvvzthh7899dD6NE6jKKyCp7+XzpX/XspKzPyjY4nIqJxTUQumMlkonNMKHcPbcfr45NZP2UEn6UM4fejOjG0fSR+PmZyi8r4fMNBHvtkExf9/XuG/f17nvhkE19sPEhuUZnRlyCNzNDbrk8NXDNmzGDAgAG8+OKLjBo1iu3btxMVFXXG699//32eeOIJZs2axeDBg9mxYwd33XUXJpOJ6dOnG3AF4q2+/ukQ767IBGD6zb2ICvE3OJGIuKvRo0czevTosz4/ffp0Jk6cyIQJEwCYMWMGX331FbNmzeKJJ54gNjaWfv36ERcXB8CVV17Jhg0bGDFiRK3HKysro6zs5x/QCgsLAbDZbNhstnpdw6n96ru/MzS1DF1jgvhwYn8+WpvFPxbsZFt2EWNfW8FV3aPp79t0Pg/K4Bk5lMF5GYz+eqoLbxjXauMOX0PO5o3XBLouT3Kua+oaE0TXmCDuG9qWMlslGw4UsDzjCMszjrDpQAGZR0rJPFLKB6v3A9AhKohBCREMSghnUEI4QX7Glau88b0C51/XhRzH0OLj+QauX1q2bBlDhgzhtttuAyA+Pp5bb72VlStXujS3eLf9R0p57JNNADxwcSLDOrYwOJGIeKry8nLWrl3L5MmTq7eZzWYuv/xyli9fDkBycjKHDx/m6NGjhIWFsXjxYu6///6zHnPatGk89dRTZ2yfP38+gYGBDcrrDp0pTS1DKPBYV/hfppnlh03876ccFlos7Cn6ll4RDpflqE1Tey/cOQO4Rw5laHiG0lLPvtXQ08a12rjD15CzeeM1ga7Lk9T1mjoCHWPhRAzsLjSxs8DEzkITWSWw83AJOw+X8PaKTPwtDgZHO7g4xk4zA6fG9sb3Cpx3XRcyphlWfKzLwPVLgwcP5t1332XVqlX079+fjIwM5s6dy5133nnW86hDRBku6HiVdh76zzqKTlTQKy6M31zS7rzHdofPg7vkUAZlcHYGo7+vGiovL4/Kykqio6NrbI+Ojmbbtm0A+Pj48OyzzzJs2DAcDgcjR47kqquuOusxJ0+eTGpqavXjwsJC4uLiGDlyJKGhofXKabPZWLBgASNGjMBqtdbrGA3V1DPcBGw6UMCUL9LZcqiIN3dYuCW5NX+4ohMBvhaXZmnq74U7ZXCXHMrgvAynfhfxVJ4yrtXGHb6GnM0brwl0XZ7EWdd0tLSclXuOsjwjn8U78zlw9DjfHTSxONvCld1iuHtIW7q2ct6/B+fjje8VOP+6LmRMM6z4WJeB65duu+028vLyGDp0KA6Hg4qKCh544AH+8Ic/nPU86hBRhgvx5T4zGw6aCbA4uDoynwXfzHN5hoZyhxzKoAzOyuDpHSJ1db5b3E7n5+eHn9+ZfwK2Wq0N/iHCGcdoqKacoW+7SD68bwC/mTmfhQfNfLD6AGv3HePft/Wmc4zrfuA+pSm/F+6WwV1yKEPDMxid3VXcZVyrjTt8DTmbN14T6Lo8SUOvKSrMypheQYzp1Rq73cH32w8zc0kGKzKO8MWmQ3yx6RCDEyOYeFECF3ds4bL1GLzxvQLnXdeFHMPQ264v1KJFi3j22Wd5+eWXGTBgALt27eLhhx/mmWee4cknn6x1H3WIKENdLdmVx7fL1wHw3I09Gd0txuUZGsIdciiDMjg7g6d3iERGRmKxWMjJyamxPScnh5iYuv0bI02Lr4+Zq9vauXNkMr//ZDM7Dxdz9Us/8qdfdeHOgW0xmbT4mYgYR+OaiDQ2s9nEZV2iuaxLND8dKGDmkgy++ukQy3bns2x3Ph2igrn3onZc0ysWf6tr7w6R+jOs+FifgevJJ5/kzjvv5N577wWge/fulJSUcN999/HHP/4Rs/nMxbvVIaIMdXG46ASPfbIZgNsHtOHq3nEuz+As7pBDGZTBWRmMzt5Qvr6+9O3bl4ULF3LttdcCYLfbWbhwIZMmTTI2nLi1IYkRzHv4Ih79aCPfb89lyudbWLwjj+dv7EHzIF+j44lIE6VxTURcqXvrMP51a28eH92Z2T/u4T+r9rPzcDGPf/ITz3+znfGD4rljYFv9bOQBzqzWucjpA9cppwauQYMG1bpPaWnpGQVGi6Wq0u1wGDspu3guu91B6pyN5BWX0zkmhCevSjI6koh4kOLiYjZs2MCGDRsA2LNnDxs2bCAzMxOA1NRUZs6cyVtvvcXWrVt58MEHKSkpqV5srb7S0tJISkoiOTm5oZcgbioi2I9ZdyUz5aokfC1mvt2aw+j/W8Ly3flGRxMRL6ZxTUTcTWyzAP74qySWTb6UP17ZhVZh/uQVl/OPBTsY9LeF/Omzn9iTV2J0TDkHQ2+7Tk1NZfz48fTr14/+/fvz4osv1hi4xo0bR2xsLNOmTQNgzJgxTJ8+nd69e1ffdv3kk08yZsyY6iKkyIWasXg3S3flEWC18NJtvdW6LSIXZM2aNVxyySXVj09N9TF+/Hhmz57N2LFjyc3NZcqUKWRnZ9OrVy/mzZt3xpzHFyolJYWUlBQKCwsJCwtr0LHEfZlMJu4e2o7+7cL5zQfrycgt4bbXV5AyvD0PX94Bq8WwvyOLiJfSuCYi7irU38rEYQncNSSeuT8dYuaSDDZnFfLuikzeW5nJiC7RTByWQL+2zTVVjZsxtPh4voErMzOzRqfjn/70J0wmE3/605/IysqiRYsWjBkzhr/+9a9GXYJ4uLX7jvKP+TsAeOrqrrSPCjE4kYh4muHDh5+3+37SpEm6HU0apFtsGP97aChPfZHOnDX7een7XSzbncf/3dKbuPCGLaAnInI6jWsi4u6sFjPX9Irl6p6tWJFxhJlLMvhu22Hmp+cwPz2HnnHNuO+iBEZ1jcZHf6h1C4YvOHOugWvRokU1Hvv4+DB16lSmTp3qgmTi7QpKbfzmP+uptDu4umcrburX2uhIIiIiZxXo68NzN/ZgaIdI/vDpT6zLPMaV/7eEZ6/vzpierYyOJyIiIuJSJpOJQYkRDEqMYNfhIt5YuodP1mWxcf8xUt5fR+vmAdw9pB03J8cR7Gd4+atJUwlYmiSHw8Hjn2wi69hx2oQH8tfruqktW0REPMKYnq2Y+/BF9GnTjKKyCh76z3oe+3gjpeUVRkcTERERMUT7qBCmXd+DZU9cym8u60B4kC8Hjh7n6f+lM3jaQv729TayC04YHbPJUvFRmqR3V2Yyb0s2VouJl27rTYi/Z6+oKyJNjybmb9riwgP58P5BPHRpe0wm+HDNAa7691I2ZxUYHU1EpF40romIM0QG+5E6oiPLnriUv17XjXaRQRSeqGDGD7u56O/fkTpnA+kHC42O2eSo+ChNztZDhTzzv3QAHr+iMz1aNzM2kIhIPaSkpJCens7q1auNjiIG8bGY+d3ITrx/70BiQv3JyC3h+peX8cbSPeedr01ExN1oXBMRZ/K3Wrh9QFsWpl7MzHH96N8uHFulg0/XZ3Hlv5Zwx+srWbT9sH5mchEVH6VJKS2vYNL76yivsHNp5yjuGdrO6EgiIiINMigxgq8fvogRSdGUV9p55n/p3D17NXnFZUZHExERETGU2WxiRFI0H94/iM9ThnBVj5ZYzCaW7srjrjdXc8WLS/hwzX7KKiqNjurVVHyUJuXPX2xhd24J0aF+PH9jD83zKCIiXqF5kC+v3dmXZ67piq+Pme+35zL6/5awZGeu0dFERERE3ELPuGa8dFsfFj06nLuHtCPI18L2nCIe+3gTQ5/7nrTvd3GstNzomF5JxUdpMj7fkMWHaw5gMsGLY3sTEexndCQRERGnMZlM3Dkoni8mDaFjdDC5RWXc+cYqpn29lfIKu9HxRERERNxCXHggU8YksWzyZUwe3ZmYUH9yi8p4/pvtDJr2HU/9byt5WpvGqVR8lCZhb14Jf/j0JwAeurQDgxIjDE4kIiLSODrHhPJ5ylBuH9AGgFd/yOCmGcvYl19icDIRERER9xEWYOX+ixNZ/Ngl/HNsT5JahnLcVsm7K/fzl/UWUv6zgbX7jhgd0yuo+Cher6yikkn/WUdJeSX924Xzm0vbGx1JRKTBtCqonEuAr4W/XtedGXf0JSzAysYDBVz5f0v47/oDRkcTEamVxjURMYqvj5nrerfmq98M5b17B3Bxh0gcmJiffpgbXlnOdS//yNc/HaLSrsVp6kvFR/F6f5+3nc1ZhTQLtPJ/t/TCx6IvexHxfFoVVOriim4xfP3wRfRvF05JeSWPzNlI6pwNFJdVGB1NRKQGjWsiYjSTycSQ9pG8Pq4PT/Ss4Ka+sfhazKzPPMaD763jkhcWMfvHPZTo56gLpiqMeLWFW3N4Y+keAF64sSctwwIMTiQiIuJarZoF8J+JA0kd0RGzCT5dn8Wv/rWEjfuPGR1NRERExC21DIRnr+3K0icu4aFL29Ms0ErmkVL+/GU6g//2HX+ft43DhZoYsq5UfBSvdajgOI9+tBGACUPiuTwp2uBEIiIixrCYTfzmsg58eP8gYpsFsC+/lBteWcarP+zGrluIRERERGoVFeLP70Z2YtkTl/LMNV2Jjwik4LiNlxftZshz3/HoRxvZll1odEy3p+KjeKVKu4OHP9jA0VIb3WJDeWJ0Z6MjiYiIGK5ffDhzf3MRv+rekgq7g2lfb2P8m6s4XKS/3IuIiIicTaCvD3cOimfh74bz6p196de2ObZKBx+vPcAVLy7hzjdWsnhHLg6H/qhbGxUfxSv9+7udrNpzhCBfC/++tQ9+PhajI4mIiLiFsEArL93Wm79d3x1/q5klO/MY/eISvt9+2OhoIiIiIm7NYjYxqmsMHz84mP/+ejC/6t4SswmW7Mxj3KxVjP6/JXy89gDlFXajo7oVFR/F66zIyOdfC3cC8NfrutMuMsjgRCIiIu7FZDJxS/82/O+hoXSOCSG/pJwJb67mmf+lU6YflkVERETOq3eb5qTd3ocffn8Jdw2OJ9DXwrbsIh79aCNDn/uOlxftoqDUZnRMt6Dio3iVIyXlPPzBeuwOuLFva67tHWt0JBGRRpGWlkZSUhLJyclGRxEP1j4qhM9ShnDX4HgA3li6h5tfW0nOcWNziUjTo3FNRDxVXHggf766K8ufuIzHruhEdKgfh4vK+Pu87Qz620L+/MUWMvNLjY5pKBUfxWs4HA4e/WgjOYVlJLQI4qmruxodSUSk0aSkpJCens7q1auNjiIezt9q4c9Xd+X1cf1oHmgl/VAR03+ykHVMFUgRcR2NayLi6cICrfx6eHuWPHYp/7ipJ51jQigtr2T2sr2MfPEH0g823YVpVHwUrzHrx718t+0wvj5mXrq1D0F+PkZHEhER8RiXJ0Uz77fD6NYqlBOVJv7+zQ6jI4mIiIh4HF8fMzf0bc3XD1/EO/f0p3tsGCdsdl5etMvoaIZR8VG8wuasQv729VYAnvxVF5JahRqcSERExPNEh/oz7bqumHAwd3MOKzPyjY4kIiIi4pFMJhMXdWjBczf0AGDuT4fYf6Rp3n6t4qN4vBMV8PCHG7FVOhjVNZo7BrY1OpKIiIjH6hwTwuBoBwBP/y+dSrvD4EQiIiIiniupVSgXdYjE7qiaX7spUvFRPJrD4eDDPWYyjxwntlkAf7+hJyaTyehYIiIiHu3KODuh/j5sOVjIR2v2Gx1HRERExKPdNywBgDmr93OstNzgNK6n4qN4tP9uOMjaPDMWs4l/3dqLsECr0ZFEREQ8XrAVJl2SCMAL87dTeMJmcCIRERERzzW0fSRdWoZy3FbJeyszjY7jcio+iscqLa/g79/sBODhSxPp2zbc4EQiIiLe444BcSS2CCKvuJyXvmu6E6SLiIiINJTJZOK+Ye0AePPHvZywVRqcyLVUfBSP9dayfeSXlBPh5+DeofFGxxEREfEqVouZP12VBMCbP+5hT16JwYlEREREPNdVPVrRMsyfvOIyPt+QZXQcl1LxUTxS0Qkbry7eDcAVcXasFn0pi0jTkpaWRlJSEsnJyUZHES92Sacohndqga3SwV+/Sjc6joh4MY1rIuLtrBYzdw+p6n58bXEG9ia0qJ8qNuKR3vxxL8dKbSREBtIvsul8w4qInJKSkkJ6ejqrV682Oop4uT/9Kgkfs4lvtx5myc5co+OIiJfSuCYiTcEt/eMI8fNhd24J328/bHQcl1HxUTxOQamNmUsyAPjNpe0xa3FrERGRRtM+Kphxg+IBePrLdCoq7cYGEhEREfFQIf5WbhvQBoBXF2cYnMZ1VHwUjzNzSQZFJyroHBPC6K7RRscRERHxeg9f1oHwIF92Hi5ukis0ioiIiDjLhCHt8DGbWLXnCBv2HzM6jkuo+CgeJb+4jFk/7gHgkREdMavtUUREpNGFBVpJHdERgOkLdnC0pNzgRCIiIiKeKSbMn6t7tQJgZhPpflTxUTzKq4szKC2vpHtsGCOT1PUoIiLiKrf2b0PnmBAKjtt48dsdRscRERER8Vj3DUsA4OvNh8jMLzU4TeNT8VE8xuHCE7y1bC8AqSM6YjKp61FERMRVLGYTU8YkAfDuykx25BQZnEhERETEM3WOCWVYxxbYHfDGUu/vflTxUTzGy4t2U1Zhp0+bZgzv1MLoOCIiIk3O4MRIRnWNptLu4Jn/peNwOIyOJCIiIuKR7j/Z/fjhmgNeP6WNio/iEQ4eO877Jye4/93ITup6FBERMcgfr0zC12Jmyc48Fm49bHQcEREREY80ODGCpJahHLdV8u6KfUbHaVQqPopH+Pd3uyivtDMwIZzBiRFGxxEREWmy2kQEcs9F7QD4y1fplFVUGpxIRERExPOYTCbuv7iq+/Gt5Xs5YfPen6lUfBS3l5lfykdr9gPqehQROSUtLY2kpCSSk5ONjiJNUMol7WkR4sfe/NLq+ZhFRBpC45qINEVXdm9JqzB/8orL+e/6LKPjNBoVH8Xt/d/CnVTYHQzr2ILk+HCj44iIuIWUlBTS09NZvXq10VGkCQr28+GxUZ0A+PfCXeQWlRmcSEQ8ncY1EWmKrBYzdw+tuqNk5pIM7HbvnE/bLYqPaWlpxMfH4+/vz4ABA1i1atVZXzt8+HBMJtMZH7/61a9cmFhcZXduMf9dfwCoWuFaRERE3MMNfVrTo3UYRWUV/GP+dqPjiIiIiHikW/q3IcTfh4zcEr7dmmN0nEZhePFxzpw5pKamMnXqVNatW0fPnj0ZNWoUhw/XPoH5p59+yqFDh6o/Nm/ejMVi4aabbnJxcnGFF7/did0Bl3eJpldcM6PjiIiIyElms4mpY5IAmLNmP5uzCgxOJCIiIuJ5gv18uH1AW6Cq+9EbGV58nD59OhMnTmTChAkkJSUxY8YMAgMDmTVrVq2vDw8PJyYmpvpjwYIFBAYGqvjohbZnF/G/TQcBdT2KiIi4o75tw7m6ZyscDnj6y3QcDu+8VUhERESkMU0YEo/VYmL13qOsyzxqdByn8zHy5OXl5axdu5bJkydXbzObzVx++eUsX768Tsd44403uOWWWwgKCqr1+bKyMsrKfp6HqLCwEACbzYbNZqtX7lP71Xd/Z2gKGf4xfxsOB1zRNZoOLQJqPU9T+Dx4Ug5lUAZnZzD6+0pEzu+J0Z2Zn57Nqr1HmPtTNr/q0dLoSCIiIiIeJTrUn2t6xfLx2gPMXJzBK3f0NTqSUxlafMzLy6OyspLo6Oga26Ojo9m2bdt591+1ahWbN2/mjTfeOOtrpk2bxlNPPXXG9vnz5xMYGHjhoU+zYMGCBu3vDN6aYX8xzE/3wYSD3j5ZzJ177lWfvPXzUB/ukEMZlMFZGUpLS52YREQaQ6tmATxwcSIvfruTZ+du5bIuUfhbLUbHEhEREfEo9w1L4OO1B5i3JZu9eSXER9beZOeJDC0+NtQbb7xB9+7d6d+//1lfM3nyZFJTU6sfFxYWEhcXx8iRIwkNDa3XeW02GwsWLGDEiBFYrdZ6HaOhvD3DxHfWAXmM6dGKu2/sbkiGunKHDO6SQxmUwdkZTnWri4h7u39YIh+u3k/WsePMXJzBQ5d1MDqSiIiIiEfpGB3C8E4tWLQ9lzeW7uGZa7sZHclpDC0+RkZGYrFYyMmpuZpPTk4OMTEx59y3pKSEDz74gKeffvqcr/Pz88PPz++M7VartcG/kDvjGA3ljRnW7jvKoh15WMwmHhnZqU7H9sbPgyfnUAZlcFYGo7OLSN0E+Fp44sou/OY/63l50W5u6hdHTJi/0bFEREREPMp9wxJYtD2Xj9bu55ERHQkP8jU6klMYuuCMr68vffv2ZeHChdXb7HY7CxcuZNCgQefc96OPPqKsrIw77rijsWOKi/1zwQ4AbugTSzsvajMWERHxZmN6tKRf2+Yct1Xy3LzzT58jIiIiIjUNSoigW2woJ2x23lm+z+g4TmP4atepqanMnDmTt956i61bt/Lggw9SUlLChAkTABg3blyNBWlOeeONN7j22muJiIhwdWRpRCsy8lm6Kw+rxcRDl+qWLREREU9hMpmYOqYrJhP8d32WV67UKCIiItKYTCYT9w1LBODt5Xs5Yas0OJFzGF58HDt2LC+88AJTpkyhV69ebNiwgXnz5lUvQpOZmcmhQ4dq7LN9+3aWLl3KPffcY0RkaSQOh4Pp86u6HscmxxEX3rAFgURERMS1urcO46a+rQF46st07HaHwYlEREREPMuV3WKIbRZAfkk5H689YHQcpzC8+AgwadIk9u3bR1lZGStXrmTAgAHVzy1atIjZs2fXeH2nTp1wOByMGDHCxUmlMS3ZmceqvUfw9TEz6RJ1PYqIiHiiR0d1ItjPh437j/HZhiyj44iIiIh4FB+LmXuGtgPgjaV7qPSCP+a6RfFRxOFw8I+Tcz3eMaCtJqkXETmPtLQ0kpKSSE5ONjqKSA1RIf6kXNIegL99vY2SsgqDE4mIJ9C4JiLys7HJcYT6+7Anr4QF6Tnn38HNqfgobuG7bYfZuP8YAVYLDw5PNDqOiIjbS0lJIT09ndWrVxsdReQMdw+Np21EIIeLynhl0W6j44iIB9C4JiLysyA/H+4Y2BaAmUsyDE7TcCo+iuHsdgf/ODnX4/jB8bQI8TM4kYiIiDSEn4+FP1zZBYDXlmSw/0ipwYlEREREPMtdg+PxtZhZu+8oa/cdMTpOg6j4KIb7Zks26YcKCfbz4f5hCUbHEREREScYmRTNkPYRlFfYmfb1VqPjiIiIiHiUqFB/ru3dCoDXFnt296OKj2KoSruD6Sfnerx7aDuaB/kanEhEREScwWQy8eRVSZhNMPenbFZk5BsdSURERMSjTLyoqkFrfnoOGbnFBqepPxUfxVD/23SQnYeLCfX3qV7NSURERLxD55hQbh9QNV/RU1+me8VqjSIiIiKu0iE6hEs7R+FwVK187alUfBTDVFTaefHbnQDcNyyBsACrwYlERETE2R4Z0ZFQfx+2Hipkzur9RscRERER8Sj3nZye7uO1B8grLjM4Tf2o+CiG+XR9FnvySggP8uWuIep6FBER8UbhQb789vKOAPxj/nYKjtsMTiQiIiLiOQa0C6dn6zDKKuy8vXyf0XHqRcVHMUR5hZ1/Lazqenzg4gSC/XwMTiQiIiKN5c5BbUlsEUR+STn/Pjn+i4iIiMj5mUwmJp7sfnxn+V6Ol1canOjCqfgohvhwzX4OHD1OixA/7hwYb3QcERERaURWi5knr0oCYPayvR49YbqIiIiIq13RNYa48ACOltr4eK3nTWOj4qO43AlbJS99twuAlOGJBPhaDE4kIiIijW14pygu7RxFhd3BX7/aanQcEREREY/hYzFzz8np6l5fusfjFvFT8VFc7v2VmWQXnqBVmD+3DmhjdBwRERFxkT/+qgs+ZhMLtx3mhx25RscRERER8Rg3J8cRFmBlX34p87dkGx3ngqj4KC5VWl7By4t2AzDp0g74+ajrUUREpKlIbBHMXYPjAXjmf+nYKu3GBhIRERHxEIG+Ptw5sC0Ary7OwOHwnO5HFR/Fpd5evo+84jLahAdyU7/WRscRERERF3vosg6EB/my63Ax767wzBUbRURERIwwfnA8vhYzG/YfY82+o0bHqTMVH8Vlik7YePWHqq7H31zWAatFX34iIiJNTViAld+N7AjAPxfs4EhJucGJRERERDxDixA/ru8TC8BrizMMTlN3qv6Iy7z5416OltpIiAzi2l6tjI4jIiIiBrkluQ2dY0IoPFHBPxfsMDqOiIiIiMe496IEAL7dmsPu3GKD09SNio/iEgWlNmYuqarK/3ZER3zU9SgiItJkWcwmpo7pCsB7K/exLbvQ4EQiIiIinqF9VDCXd4nG4YDXl3hG96MqQOISry/NoOhEBZ2iQ7iqe0uj44iIiIjBBiVGMLpbDHZH1eIznjRpuoiIiIiR7htW1f34yboscovKDE5zfio+SqM7UlLOrKV7AHhkREfMZpPBiURERMQd/OHKLvj6mPlxVz4L0nOMjiMiIiLiEZLjm9MrrhnlFXbeXr7X6DjnpeKjNLpXf9hNSXkl3WJDGdU12ug4IiJeIS0tjaSkJJKTk42OIlJvceGBTLyoHQB/nbuVsopKgxOJiFE0romI1J3JZKrufnxnxT5KyysMTnRuKj5KozpcdIK3TlbhfzeiEyaTuh5FRJwhJSWF9PR0Vq9ebXQUkQb59fD2RIX4sS+/lDd/3Gt0HBExiMY1EZELM6prDG3CAzlWauOjNQeMjnNOKj5Ko3r5+92csNnp3aYZwzu1MDqOiIiIuJkgPx8eu6IzAC99t4vDRScMTiQiIiLi/ixmE/eevIPk9aUZVNrdd/5sFR+l0Rw8dpz3V2YC6noUERGRs7u+dyw9W4dRXFbBC99sNzqOiIiIiEe4qW8czQOt7D9ynHmbs42Oc1YqPkqjeen7XZRX2hnQLpwh7SOMjiMiIiJuymw2MWVMVwA+WnuAzVmFBicSERERcX8BvhbuHNgWgNcW78bhcM/uR5+6vvD666+v80E//fTTeoUR77H/SCkfrt4PwO9GqutRREREzq1v2+Zc26sVn204yF/mbuPOVkYnEhEREXF/4wbH8+riDDYeKGDVniMMSHC/5q86dz6GhYXV+UPk/xbupMLu4KIOkfRvF250HBEREfEAj4/uTIDVwtrMY6zP1x8uRURERM4nMtiPG/q2BmDmkgyD09Suzp2Pb775ZmPmEC+SkVvMp+uqVlr63chOBqcRERERT9EyLIAHhycyfcEOPt9n5pGyCppZrUbHEhEREXFr9w5tx39WZfLt1sPsOlxE+6gQoyPVoDkfxele/HYndgdc3iWKXnHNjI4jIlIn+/btIz09HbvdbnQUkSbtvmEJtArz51i5iXGz15BfXGZ0JBGPozFNRKRpSWgRzOVdogGYc3IKPHdS5+Jj79696dOnT50+pOnanl3El5sOAvDIiI4GpxEROdOsWbOYPn16jW333XcfCQkJdO/enW7durF/v/sN2CJNhb/Vwr9u6Umgj4NNBwq5ccZy9h8pNTqWiFvSmCYiIqdc1zsWgG+3HjY4yZnqfNv1tdde24gxxFv8c8EOHA4Y3S2Grq00/6eIuJ/XXnuN+++/v/rxvHnzePPNN3n77bfp0qULkyZN4qmnnuL11183MKVI09azdRi/7VbJ7D3B7Mkr4fpXljF7QrJ+thD5BY1pIiJyyrCOLfC1mNmTV8Lu3GISWwQbHalanYuPU6dObcwc4gU2ZxUwb0s2JpO6HkXEfe3cuZN+/fpVP/7888+55ppruP322wF49tlnmTBhglHxROSk6AD48L7+3PvOerZlFzH21RW8dmdfBrePNDqaiNvQmCYiIqcE+/kwMDGCxTty+TY9h8SL3af4qDkfxWmmL9gBwNU9W9Ex2r0mNxUROeX48eOEhoZWP162bBnDhg2rfpyQkEB2drYR0UTkF6JD/Zlz/yAGtAunuKyC8W+u4suNB42OJeI2NKaJiMjpRnSJAuDbrTkGJ6mpXsXHyspKXnjhBfr3709MTAzh4eE1PqTpWZd5lO+2HcZiNvHwZR2MjiMiclZt27Zl7dq1AOTl5bFlyxaGDBlS/Xx2djZhYbq1U8RdhAVYeevu/lzZPQZbpYOH/rOeWUv3GB1LxC1oTBMRkdNdenLRmbX7jrrVon31Kj4+9dRTTJ8+nbFjx1JQUEBqairXX389ZrOZP//5z06OKJ7gnye7Hq/vHUuCG80rICLyS+PHjyclJYVnnnmGm266ic6dO9O3b9/q55ctW0a3bt0MTCgiv+RvtfDvW/swflBbAJ7+XzrTvt6K3e4wOJmIsTSmiYjI6WKbBZDUMhS7A77fnmt0nGr1Kj6+9957zJw5k9/97nf4+Phw66238vrrrzNlyhRWrFhxQcdKS0sjPj4ef39/BgwYwKpVq875+mPHjpGSkkLLli3x8/OjY8eOzJ07tz6XIU6yau8RluzMw2ox8Rt1PYqIm3vssceYOHEin376Kf7+/nz00Uc1nv/xxx+59dZbDUonImdjMZv489Vd+f2oTgC8+kMGj360EVul3eBkIsbRmCYiIr90eVJV9+NCN7r1us4LzpwuOzub7t27AxAcHExBQQEAV111FU8++WSdjzNnzhxSU1OZMWMGAwYM4MUXX2TUqFFs376dqKioM15fXl7OiBEjiIqK4uOPPyY2NpZ9+/bRrFmz+lyGOIHDAf/8dhcAN/eLIy480OBEIiLnZjabefrpp3n66adrff6Xv7iJiPswmUykXNKeqBA/nvj0Jz5dn0VeSTmv3N6HIL96/Vgr4tE0pomIyC+N6BLNvxbu5IcduZywVeJvtRgdqX6dj61bt+bQoUMAJCYmMn/+fABWr16Nn59fnY8zffp0Jk6cyIQJE0hKSmLGjBkEBgYya9asWl8/a9Ysjhw5wmeffcaQIUOIj4/n4osvpmfPnvW5DHGC7QUm1uw7hq+PmUmXtjc6johIncyZM4fbb7+dm266iRkzZhgdR0Qu0E394nh9XD8CrBYW78jl1pkryHOjeY1EXEljmoiInK5bbCjRoX6UlleyIiPf6DhAPTsfr7vuOhYuXMiAAQN46KGHuOOOO3jjjTfIzMzkkUceqdMxysvLWbt2LZMnT67eZjabufzyy1m+fHmt+3zxxRcMGjSIlJQUPv/8c1q0aMFtt93G448/jsVSeyW3rKyMsrKffxgtLCwEwGazYbPZ6nrJNZzar777O4M7ZCgvL2fu/qr69a3JrYkM9HF5Hnf4PLhDBnfJoQzK4OwMjZH/lVdeISUlhQ4dOhAQEMCnn37K7t27ef75551+LhFpPJd0juL9iQO4e/ZqNh0o4MZXlvH23QNoE6G7MKTp0JgmIiK/ZDKZuKxLNO+vzOTbrTkM73TmncWuVq/i49/+9rfq/x87dixt27Zl2bJldOjQgTFjxtTpGHl5eVRWVhIdHV1je3R0NNu2bat1n4yMDL777jtuv/125s6dy65du/j1r3+NzWZj6tSpte4zbdo0nnrqqTO2z58/n8DAhv1wumDBggbt7wxGZth81MS+YgtWs4P25RnMnZthWJam/l6czh1yKIMyOCtDaWmpE5NUeemll5g6dWr1uPHuu+9y//336xc1EQ/Uu01zPnlwMONmrWJvfinXv/Ijsyf0p1usVveVpkFjmoiI1GbEyeLjwq2HeeYaByaTydA8TpkcZ+DAgQwcONAZhzonu91OVFQUr732GhaLhb59+5KVlcXzzz9/1uLj5MmTSU1NrX5cWFhIXFwcI0eOJDQ0tF45bDYbCxYsYMSIEVit1nodo6GMzuBwOHjtlRVAEeMGtuWW0Z1dngGM/zy4SwZ3yaEMyuDsDKe61Z0pIyOD8ePHVz++7bbbuOeeezh06BAtW7Z0+vlEpHEltAjm0wcHc9ebq0k/VMjYV5cz486+XNShhdHRRBqdxjQREanNoMQIAqwWDhWcYMvBQsP/MFuv4uO0adOIjo7m7rvvrrF91qxZ5Obm8vjjj5/3GJGRkVgsFnJyaq6+k5OTQ0xMTK37tGzZEqvVWuMW6y5dupCdnU15eTm+vr5n7OPn51frPJRWq7XBv5A74xgNZVSGH3flseVQEb5mB/cNS2iynwd3y+AuOZRBGZyVoTGyl5WVERQUVP3YbDbj6+vL8ePHnX4uEXGNqFB/5tw/kPvfWcuy3flMeHM1L9zUk2t7xxodTaRRaUwTEZHa+FstDOsYyTdbcvh2a45nFh9fffVV3n///TO2d+3alVtuuaVOxUdfX1/69u3LwoULufbaa4GqzsaFCxcyadKkWvcZMmQI77//Pna7HbO5aq7BHTt20LJly1oLj9J4ZvywG4ABUQ7Cg/S5FxHP8uSTT9aYeqO8vJy//vWvhIX9PChPnz7diGgiUk8h/lbenJDM7z7cyP82HeK3czaQV1zGvRclGB1NpFFpTBMRkdpc3iW6uvj428s7GpqlXsXH7OzsWtv4W7RoUb0Kdl2kpqYyfvx4+vXrR//+/XnxxRcpKSlhwoQJAIwbN47Y2FimTZsGwIMPPshLL73Eww8/zEMPPcTOnTt59tln+c1vflOfy5B6Sj9YyJKdeZhNcElLu9FxREQuyLBhw9i+fXuNbYMHDyYj4+d5a42eE0VE6sfPx8K/bulNixA/3vxxL3/5ais5hSeYPLoLZrO+r8X7aEwTEZGzuaRzFCYTbM4q5FDBcSIDnTLzYr3U68xxcXH8+OOPtGvXrsb2H3/8kVatWtX5OGPHjiU3N5cpU6aQnZ1Nr169mDdvXvUiNJmZmdUdjqfO+8033/DII4/Qo0cPYmNjefjhh+vUaSnOM3NJ1Q8zo7vGEOF/wOA0IiIXZtGiRTUe5+Xl4evrW+95gEXEvZjNJqZclURMqD/Tvt7GzCV7OFxUxvM39sTXx3z+A4h4EI1pIiJyNpHBfvRp05y1+47y7dbD3NK37vU6Z6vXT2ATJ07kt7/9LW+++Sb79u1j3759zJo1i0ceeYSJEyde0LEmTZrEvn37KCsrY+XKlQwYMKD6uUWLFjF79uwarx80aBArVqzgxIkT7N69mz/84Q815oCUxpV17DhfbDwIwL1D440NIyJST8eOHSMlJYXIyEiio6Np3rw5MTExTJ48uVFW2BYR1zKZTNx/cSLTb+6Jj9nE5xsOcvfs1RSXVRgdTcTpNKaJiMjZXN6lqrlv4dac87yycdWr8/H3v/89+fn5/PrXv6a8vBwAf39/Hn/8cSZPnuzUgOJeZi3dQ6XdweDECLrFhpK50ehEIiIX5siRIwwaNIisrCxuv/12unTpAkB6ejr//ve/WbBgAUuXLmXTpk2sWLHCJVN7xMfHExoaitlspnnz5nz//feNfk6RpuD6Pq2JCPbjwXfXsnRXHre8tpw37+pPi5AzFyMU8UTuOKaBxjUREXcxIimK5+ZtY9mufEoM/CNsvYqPJpOJ5557jieffJKtW7cSEBBAhw4dal1VWrxHQamND1ZlAnDfME3eLiKe6emnn8bX15fdu3dXT/Nx+nMjR47kzjvvZP78+fzrX/9yWa5ly5YRHBzssvOJNBUXd2zBB/cNZMKbq9mcVcgNryzjrbv70y4y6Pw7i7g5dx3TQOOaiIg7SGwRTHxEIHvzS1m6K9+wHA2a+CY7O5sjR46QmJiIn58fDofDWbnEDb27ch8l5ZV0jgnh4o4tjI4jIlIvn332GS+88MIZv6QBxMTE8Pe//51PPvmkelE0EfF8PVo345MHB9MmPJDMI6Xc+MoyNh04ZnQskQbTmCYiIudiMpm47NSt19tzDctRr+Jjfn4+l112GR07duTKK6+sXuH6nnvu4Xe/+51TA4p7KKuoZPayvUBV16NWzRMRT3Xo0CG6du161ue7deuG2Wxm6tSpdTre4sWLGTNmDK1atcJkMvHZZ5+d8Zq0tDTi4+Px9/dnwIABrFq1qsbzJpOJiy++mOTkZN57770Luh4RqZv4yCA+fnAQXVuFkl9Szi2vreCHHcb9EC7iDM4e00DjmoiItzk17+Oi7bnYDeoZrFfx8ZFHHsFqtZKZmUlgYGD19rFjxzJv3jynhRP38dn6LHKLymgZ5s+YnsatkCQi0lCRkZHs3bv3rM/v2bOHqKioOh+vpKSEnj17kpaWVuvzc+bMITU1lalTp7Ju3Tp69uzJqFGjOHz4cPVrli5dytq1a/niiy949tln2bRpU53PLyJ1FxXiz5z7BzG0fSSl5ZXcM3s1n647YHQskXpz9pgGGtdERLxNv/jmhAVYOVpqY2+RMRnqNefj/Pnz+eabb2jdunWN7R06dGDfvn1OCSbuw2538OriDADuHtIOq6VBd+uLiBhq1KhR/PGPf2TBggX4+vrWeK6srIwnn3ySK664os7HGz16NKNHjz7r89OnT2fixIlMmDABgBkzZvDVV18xa9YsnnjiCQBiY2MBaNmyJVdeeSXr1q2jR48etR6vrKyMsrKy6seFhYUA2Gw2bDZbnXOf7tR+9d3fGZRBGVyVwc8Mr97eiyf+u5kvN2WT+uFGsgtKuXdIfI07O9zh8+AuOZTBeRmcnd/ZYxp4x7hWG3f4GnI2b7wm0HV5Em+8JvDO67q4QyRfbDrE5qNmp13XhRynXsXHkpKSGh2Ppxw5ckSLznihhdsOk5FbQoifD7f0jzM6johIgzz99NP069ePDh06kJKSQufOnXE4HGzdupWXX36ZsrIy3n77baecq7y8nLVr1zJ58uTqbWazmcsvv5zly5cDVWOq3W4nJCSE4uJivvvuO26++eazHnPatGk89dRTZ2yfP39+rWPzhViwYEGD9ncGZVAGV2W4NBCKW5r5/pCZv3+zk5WbtnNtWzvmX8ws4w6fB3CPHMrQ8AylpaVOTOLaMQ08b1yrjTt8DTmbN14T6Lo8iTdeE3jXdYUfNwEWNh81Oe26LmRMq1fx8aKLLuLtt9/mmWeeAarm9LDb7fz973/nkksuqc8hxY29tng3ALcPbEuIv9XgNCIiDdO6dWuWL1/Or3/9ayZPnly9WJrJZGLEiBG89NJLtGnTxinnysvLo7Ky8oyFAKKjo9m2bRsAOTk5XHfddQBUVlYyceJEkpOTz3rMyZMnk5qaWv24sLCQuLg4Ro4cSWhoaL1y2mw2FixYwIgRI7Bajfl3XhmUwYgMVwFv/LiXv83bwQ+HzARHtOK5G7rh52N2i88DNK33oylkONXV5yyuHNPAc8a12rjD15CzeeM1ga7Lk3jjNYF3XtdFJ2y8O20ROcehQ58hdIgJa/AxL2RMq1fx8fnnn+fSSy9lzZo1lJeX89hjj7FlyxaOHDnCjz/+WJ9Diptau+8oq/cexWoxMWFIvNFxREScol27dnz99dccPXqUnTt3AtC+fXvCw8NdniUhIYGNGzfW+fV+fn613mVgtVob/MORM47RUMqgDK7O8MDwDrRsFsijH23kq83ZHD1u49U7++J/8rzu8HlwlxzK0PAMjZHdncY0cK9xrTbu8DXkbN54TaDr8iTeeE3gXdcVbrXSv11zlu0+wpLdR0mKi2zwMS/kc3PBk/fZbDZ+85vf8OWXXzJ06FCuueYaSkpKuP7661m/fj2JiYkXekhxY6e6Hq/tFUt0qL/BaUREnKt58+b079+f/v37N8ovaZGRkVgsFnJycmpsz8nJISYmxunnE5H6uaZXLG/e1Z8gXwvLducz9tUVHC4qO/+OIm6kscc00LgmIuLJLutctQDZwm25Lj/3BRcfrVYrmzZtonnz5vzxj3/kww8/ZO7cufzlL3+hZcuWjZFRDJKRW8z89KofLO4blmBwGhERz+Pr60vfvn1ZuHBh9Ta73c7ChQsZNGiQgclE5JeGdohkzv2DiAz2Jf1QIWNfW8nh40anEnEvGtdERDzXpZ1aALA28xhHS8pdeu56LVt8xx138MYbbzg7i7iZmUv24HBUVcc7RIcYHUdExC0VFxezYcMGNmzYAMCePXvYsGEDmZmZAKSmpjJz5kzeeusttm7dyoMPPkhJSUn1KqH1lZaWRlJS0jnn0RKRC9MtNoxPHhxMfEQgB46d4MXNFjYeKDA6lohLaVwTEfFOrZsH0CrQQaXdwffbD7v03PWa87GiooJZs2bx7bff0rdvX4KCgmo8P336dKeEE+PkFpXxyboDgLoeRUTOZc2aNTUWWzs1af748eOZPXs2Y8eOJTc3lylTppCdnU2vXr2YN2/eGZP1X6iUlBRSUlIoLCwkLKzhE0aLSJW2EUF8/OBgJry5ip+yChn35hpm3NGXYR1bGB1NxCU0romIeK9OYQ4OlprYdKCA6/u0dtl561V83Lx5M3369AFgx44dNZ4zmUwNTyWGe3v5Xsor7PSKa0b/dsZMVi0i4gmGDx9evbro2UyaNIlJkya5KJGINFRksB/vTOjH2H9/y/YCuOet1bxwU0+u6RVrdDSRRqdxTUTEe4VYq/59Lzxuc+l561V8/P77752dQ9xISVkFby/fB8D9wxJUUBYREZEmJ8jPh/s62/mupBVfbc7mt3M2cLSknLuGtDM6moiIiEi9BJysAhaecG3xsV5zPop3+3DNfgqO24iPCGRkV61aJyIiIk2Tjxmm39Sd8YPa4nDAn79M5x/zt5+3K0xERETEHQWeLD4WuLjzUcVHqaGi0s4bS/cAcO9FCVjM6noUEXFHmphfxDXMZhN/vrorvxvREYB/f7eLP/x3M5V2FSBFnEnjmohI4wtQ8VHcwdzN2Rw4epyIIF9u7Ou6yUdFROTCpKSkkJ6ezurVq42OIuL1TCYTD13WgWev647ZBP9ZlUnKe+s4Yas0OpqI19C4JiLS+AItVX88VfFRDONwOHj1h90AjBsUj7/VYnAiEREREfdx24A2vHx7H3wtZuZtyeauN1e5fM4kERERkfqqnvPxeIVLz6vio1RbtjufLQcLCbBaGDeordFxRERERNzOFd1aMvvuZIL9fFiRcYRbXl1BblGZ0bFEREREzuvUnI/HbZWUV9hddl4VH6Xaq4szALi5X2uaB/kanEZERETEPQ1OjOSD+wYSGexL+qFCbpyxjMz8UqNjiYiIiJyTvwVMJ5f2cOWt1yo+CgDpBwtZvCMXs6lqoRkRERERObtusWF8/MBg4sID2JdfyvWvLGPLwQKjY4mIiIicldkEIX5V7Y8qPorLzVxS1fV4ZfeWxIUHGpxGRETOR6uCihgvPjKITx4YTJeWoeQVl3HLqytYkZFvdCwRj6RxTUTENUIDrICKj+JiWceO8+XGgwDcPyzR4DQiIlIXWhVUxD1Ehfoz5/6B9G8XTlFZBeNmrWLe5myjY4l4HI1rIiKuEepf1fnoykXzVHwUZi3dQ4XdwaCECLq3DjM6joiIiIhHCfW38vbd/RmZFE15hZ1fv7eWD1ZlGh1LRERE5AxhJzsfC9X5KK5ScNxW/cPxfRdrrkcRERGR+vC3Wnj59j6M7ReH3QFPfPoTad/vwuFwGB1NREREpNqpzkfddi0u897KfZSUV9IpOoThHVsYHUdERETEY/lYzPzthu6kXFI1jc3z32znqS/TsdtVgBQRERH3cKrzsaBUxUdxgbKKSt78cS8A9w1LwHRqvXURERERqReTycTvR3VmylVJAMxetpffztlAeYXd4GQiIiIiEKI5H8WVPlufRW5RGTGh/ozp2croOCIiIiJe4+6h7XhxbC98zCa+2HiQe99eQ0lZhdGxREREpIkL02rX4ip2u4PXFmcAcM/Qdvj66EtBRMSTpKWlkZSURHJystFRROQsru0dy+vj+xFgtbB4Ry63vb6SIyXlRscScUsa10REXCNUxUdxlYXbDrM7t4QQPx9u6R9ndBwREblAKSkppKens3r1aqOjiMg5DO8UxfsTB9As0MrG/ce4acYyso4dNzqWiNvRuCYi4hphWnBGXOW1xbsBuG1gG0L8rQanEREREfFevds05+MHBtEyzJ/duSXc+MoyduYUGR1LREREmqCfb7t23XQwKj42QWv3HWX13qNYLSbuHtLO6DgiIiIiXq99VAifPDiY9lHBHCo4wY0zlrN231GjY4mIiEgTU73gjDofpTGd6nq8tlcs0aH+BqcRERERaRpaNQvgo/sH0SuuGQXHbdz++gq+337Y6FgiIiLShJzqfGxyxce0tDTi4+Px9/dnwIABrFq16qyvnT17NiaTqcaHv78KaHWVkVvM/PQcAO4blmBwGhEREZGmpXmQL+9PHMDFHVtwwmZn4ltr+O/6A0bHEhERkSbi1IIzRWUVVNodLjmn4cXHOXPmkJqaytSpU1m3bh09e/Zk1KhRHD589r8Ch4aGcujQoeqPffv2uTCxZ3t96R4cDriscxQdokOMjiMiIiLS5AT6+vD6+H5c26sVFXYHj8zZyOtLMoyOJSIiIk1A6MnbrsF13Y+GFx+nT5/OxIkTmTBhAklJScyYMYPAwEBmzZp11n1MJhMxMTHVH9HR0S5M7Llyi8r4eG3VX9bV9SgiIiJiHKvFzPSbe1XPv/2Xr7by3LxtOByu6UAQERGRpslqMRPoawGg8IRrio8+539J4ykvL2ft2rVMnjy5epvZbObyyy9n+fLlZ92vuLiYtm3bYrfb6dOnD88++yxdu3at9bVlZWWUlZVVPy4sLATAZrNhs9Xvk3xqv/ru7wz1yfDm0gzKK+z0aB1K79YhDc7vqZ8Hb8zgLjmUQRmcncHo7ysRkcZkNpt48qouRIb48vd523ll0W7yi8t49rru+FgM7xEQERERLxUWYKW0vJICF3U+Glp8zMvLo7Ky8ozOxejoaLZt21brPp06dWLWrFn06NGDgoICXnjhBQYPHsyWLVto3br1Ga+fNm0aTz311Bnb58+fT2BgYIPyL1iwoEH7O0NdM5RVwux1FsBE38CjfP311y7P0JiU4WfukEMZlMFZGUpLS52YxLukpaWRlpZGZWWl0VFEpAFMJhO/Ht6e8EBf/vDfn/hwzQGOlNh46bbe+FstRscTcRmNayIirhMWYOVQwYmmUXysj0GDBjFo0KDqx4MHD6ZLly68+uqrPPPMM2e8fvLkyaSmplY/LiwsJC4ujpEjRxIaGlqvDDabjQULFjBixAisVmu9jtFQF5rh7RWZlFZso214II/fPgSL2eTyDI1BGdwrhzIog7MznOpWlzOlpKSQkpJCYWEhYWFhRscRkQa6pX8bmgf58tB/1vPt1hzGvbGKmeP7EehxP62L1I/GNRER1zm16EyTKD5GRkZisVjIycmpsT0nJ4eYmJg6HcNqtdK7d2927dpV6/N+fn74+fnVul9DfyF3xjEaqi4ZKirtvLmsalGee4cl4O/n6/IMjU0Z3CuHMiiDszIYnV1ExJVGdY3hnbv7c+9ba1i19whjX13OG+P6GB1LREREvEyYi4uPhk4m4+vrS9++fVm4cGH1NrvdzsKFC2t0N55LZWUlP/30Ey1btmysmB5v7uZsDhw9TkSQLzf1PfPWdBERERFxDwMSIphz/yBahPixLbuIsa+t5PBxo1OJiIiINwn1ryo+Fh6vcMn5DJ/JOjU1lZkzZ/LWW2+xdetWHnzwQUpKSpgwYQIA48aNq7EgzdNPP838+fPJyMhg3bp13HHHHezbt497773XqEtwaw6Hg9cW7wZg3KB4zR0kIiIi4uaSWoXyyQODaRsRyIFjJ/i/LRa2HNQ0FCIiIuIcoQFVN0IXNYXVrgHGjh1Lbm4uU6ZMITs7m169ejFv3rzqRWgyMzMxm3+ukR49epSJEyeSnZ1N8+bN6du3L8uWLSMpKcmoS3Bry3bnszmrEH+rmTsHtTU6joiIiIjUQZuIQD5+YDDj3ljJ1uwi7pi1hjfG92NAQoTR0URERMTD+ZxcB6TS4XDN+VxylvOYNGkSkyZNqvW5RYsW1Xj8z3/+k3/+858uSOUdXl2cAcDN/eIID3LuXI8iIiIi0nhahPjx3j39uOlf37G7qIJxs1bx0m19GJEUbXQ0ERER8WBmU1Xx0UW1R+Nvu5bGs/VQIYt35GI2wb1DE4yOIyIiIiIXKMTfygNdKrmscwvKKuw88O5aPll7wOhYIiIi4smqao/Y7a6pPqr46MVeO9n1OLp7S9pEBBqcRkRERETqw9cCL93Skxv6tKbS7uB3H23k9SUZRscSERERD1Xd+eiq87noPOJiB48d58uNBwG4f5i6HkVEREQ8mY/FzPM39uDeoe0A+MtXW3n+m204XHW/lIiIiHgN86nORxf9HKHio5eatXQPFXYHgxIi6NG6mdFxRETEydLS0khKSiI5OdnoKCLiImaziT/+qgu/H9UJgLTvd/OH/26m0kW3TIk0Jo1rIiKuY0JzPkoDFRy38Z9VmQDcd7G6HkVEvFFKSgrp6emsXr3a6Cgi4kImk4mUS9rz7HXdMZngP6syeeg/6yirqDQ6mkiDaFwTEXEddT5Kg723ch8l5ZV0ig5heMcWRscRERERESe7bUAb0m7rg6/FzNyfsrln9hpKyiqMjiUiIiIewKTVrqUhyioqefPHvQDcNyyh+gtKRERERLzLld1bMuuuZAJ9LSzdlcdtr6/kaEm50bFERETEzZ1acEadj1Ivn63PIreojJhQf8b0bGV0HBERERFpREM7RPL+xIE0C7Sycf8xbnp1OYcKjhsdS0RERNyYqfq2a9ecT8VHL2K3O3htcQYAdw+Nx9dHb6+IiIiIt+sV14yPHxhEyzB/dh0u5sZXlrM7t9joWCIiIuKmzNU3yarzUS7Qd9sOszu3hBA/H27t38boOCIiIiLiIu2jQvj4wcEkRAaRdew4N81Yzk8HCoyOJSIiIm7o1BR9drtrzqfioxc51fV428A2hPhbDU4jIiIiIq4U2yyAjx4YRPfYMI6UlHPrzBUs251ndCwRERFxMyatdi31sS7zKKv2HsFqMXH3kHZGxxERERERA0QE+/H+xAEMSoiguKyCu2atZt7mbKNjiYiIiBs5teCMi6Z8VPHRW7z2Q1XX47W9YokO9Tc4jYiIiIgYJcTfypsTkhnVNZrySju/fm8tH67eb3QsERERcRNmdT7KhdqTV8I36VV/0b5vWILBaURERETEaP5WC2m39WFsvzjsDnjsk028+sNuo2OJiIiIGzBxsvNRq11LXc1ckoHDAZd2jqJDdIjRcURERETEDfhYzPzthu7cf3HVH6enfb2NaV9vxeGq3zRERETELWnOR7kg+cVlfLz2AKCuRxERERGpyWQyMXl0FyaP7gzAqz9k8MQnP1FR6aLlLUVERMTtVM/5qM5HqYt3Vu6nvMJOz7hmDGgXbnQcERFxkbS0NJKSkkhOTjY6ioh4gPsvTuTvN/TAbII5a/aT8v46TtgqjY4lUk3jmoiI62jOR6mzskp4b2XV5OH3D0vAdKpvVkREvF5KSgrp6emsXr3a6Cgi4iFuTo7j5dv74msx882WHCa8uZqiEzajY4kAGtdERFzJpM5HqauVh00cO26jbUQgo7rGGB1HRERERNzcFd1imH13MsF+PizPyOe2mSvJLy4zOpaIiIi40KnORwfqfJRzqKi08/2hqrfv3osSsJjV9SgiIiIi5zc4MZL/TBxIeJAvP2UVcNOry8k6dtzoWCIiIuIipzof7S6aAlrFRw81b0sOR8pMNA+0clPf1kbHEREREREP0r11GB89MIhWYf5k5JZw4yvL2HW4yOhYIiIi4gJa7VrOy+Fw8PqPewG4c0Ab/K0WYwOJiIiIiMdJbBHMxw8Opn1UMIcKTnDTjOVs2H/M6FgiIiLSyKpXu3bV+Vx0HnGiRTty2XKwCKvZwe0D4oyOIyIiIiIeqlWzAD68fxA9W4dxtNTGbTNXsHRnntGxREREpBFVz/mozkepjcPh4MVvdwJwUbSD8CBfgxOJiIiIiCcLD/LlvYkDGdo+ktLySu6evZq5Px0yOpaIiIg0kuo5H7XatdRm0Y5cNu4/hr/VzKWxLpoZVERERES8WrCfD2/c1Y8ru8dQXmkn5f11/GdVptGxREREpBGcWrJYcz7KGU7very9fxwhVoMDiYiIiIjX8POx8O9b+3Br/zY4HDD50594edEul92SJSIiIq5RPeejOh/ll07vepw4NN7oOCIiIiLiZSxmE89e142USxIB+Pu87Tz3zQ6X/XIiIiIijc98shqozkep4fSux3GD4okI9jM4kYiIiIh4I5PJxO9HdeZPv+oCwBs/7uP93WZslZryR0RExBuYUOej1OL0rsf7hiUYHUdEREREvNy9FyXwj5t6YjGbWJVrJuU/GzheXml0LBEREWkg06nVrlHno5z0y67HSHU9ioiIiIgL3NC3NWm39sRqcvD99jzufGMlBaU2o2OJiIiIB1Hx0QOo61FEREREjHJZ5ygeTKok1N+HNfuOcvOry8kuOGF0LBEREfEQKj66udO7Hu8c2FZdjyIiIiLicomh8P49yUSH+rE9p4gbXlnG7txio2OJiIiIB1Dx0c3V7HpMNDqOiIiIiDRRnWJC+PiBwbSLDCLr2HFumrGcjfuPGR1LRERE3JyKj27sl12PLULU9SgiIiIixokLD+SjBwbRPTaMIyXl3DpzBUt25hodS0RERNyYio9uTF2PIiIiIuJuIoP9+M99AxnaPpLS8krunr2aLzceNDqWiIiIuCm3KD6mpaURHx+Pv78/AwYMYNWqVXXa74MPPsBkMnHttdc2bkADqOtRRETOJS0tjaSkJJKTk42OIiJNULCfD2/c1Y9f9WiJrdLBbz5Yz1vL9hodSzyYxjUREe9lePFxzpw5pKamMnXqVNatW0fPnj0ZNWoUhw8fPud+e/fu5dFHH+Wiiy5yUVLXUtejiIicS0pKCunp6axevdroKCLSRPn5WPjXLb0ZN6gtDgdM/WIL0xfswOFwGB1NPJDGNRER72V48XH69OlMnDiRCRMmkJSUxIwZMwgMDGTWrFln3aeyspLbb7+dp556ioSEBBemdQ11PYqIiIiIJ7CYTTx1dVceubwjAP9auJM/fraZSrsKkCIiIlLFx8iTl5eXs3btWiZPnly9zWw2c/nll7N8+fKz7vf0008TFRXFPffcw5IlS855jrKyMsrKyqofFxYWAmCz2bDZbPXKfWq/+u5/Pj+c1vV49+A2tZ6nsTPUhTK4TwZ3yaEMyuDsDEZ/X4mIyPmZTCYevrwDEcG+PPn5Zt5fmcnRknL+ObYX/laL0fFERETEYIYWH/Py8qisrCQ6OrrG9ujoaLZt21brPkuXLuWNN95gw4YNdTrHtGnTeOqpp87YPn/+fAIDAy848+kWLFjQoP1r43DAPzdbABODIitYtXihyzNcKGVwnwzgHjmUQRmclaG0tNSJSUREpDHdMbAt4UG+/PaDDXy9OZtjpat5bVxfQvytRkcTERERAxlafLxQRUVF3HnnncycOZPIyMg67TN58mRSU1OrHxcWFhIXF8fIkSMJDQ2tVw6bzcaCBQsYMWIEVqtzf5havDOPfSvW4W818+y4i4kMrv2W68bMUFfK4D4Z3CWHMiiDszOc6lYXERHPcGX3ljQLsHLfO2tZnpHPLa+tYPaE/ppGSEREpAkztPgYGRmJxWIhJyenxvacnBxiYmLOeP3u3bvZu3cvY8aMqd5mt9sB8PHxYfv27SQm1lycxc/PDz+/M3/YsVqtDf6F3BnHOJ3D4eDf32cAVXM9tmwe7PIM9aEM7pPBXXIogzI4K4PR2UVE5MINbh/JB/cNZPysVWw5WMiNM5bxzt0DaBPRsLuORERExDMZuuCMr68vffv2ZeHCn28tttvtLFy4kEGDBp3x+s6dO/PTTz+xYcOG6o+rr76aSy65hA0bNhAXF+fK+E73w45cNmiFaxERERHxcN1iw/j4wcG0bh7AvvxSbpixjPSD6mYXERFpigy/7To1NZXx48fTr18/+vfvz4svvkhJSQkTJkwAYNy4ccTGxjJt2jT8/f3p1q1bjf2bNWsGcMZ2T6MVrkVERETEm7SLDOLTBwczbtYqtmUXMfbV5bw+vh8DEiKMjiYiIiJUrTviCoZ2PgKMHTuWF154gSlTptCrVy82bNjAvHnzqhehyczM5NChQwanbHzqehQRERERbxMV6s+c+wfRPz6corIK7py1im+2ZBsdS0REpEkzmUwuPZ/hnY8AkyZNYtKkSbU+t2jRonPuO3v2bOcHcjF1PYqIiIiItwoLsPL2Pf156D/rWZCew4PvrmXa9d0Zm9zG6GgiIiLiAoZ3Poq6HkVERETEu/lbLbxyex9u7tcauwMe/+Qn0r7fhcNV93uJiIiIYVR8NNjpXY93DFDXo4iIiIh4Jx+Lmedu6MGDw6v+2P78N9t5+n/p2O0qQIqIiHgzFR8NVqPr8eIEo+OIiIiIiDQak8nE41d05smrkgB488e9PPLhBsor7AYnExERkcai4qOBftn1GBXib3AiEREREZHGd8/Qdrw4thc+ZhOfbzjIvW+vobS8wuhYIiIi0ghUfDSQuh5FREREpKm6tncsM8f3I8BqYfGOXG6buZKjJeVGxxIREREnU/HRIOp6FBEREZGm7pJOUbw3cQDNAq1s2H+MG2csI+vYcaNjiYiIiBOp+GgQdT2KiIiIiECfNs356P5BtAzzZ3duCTe+soydOUVGxxIREREnUfHRAOp6FBERERH5WYfoED55cDCJLYI4VHCCm15dztp9R42OJSIiIk6g4qMB1PUoIiIiIlJTq2YBfPzAYHrFNeNYqY07Xl/J99sPGx1LREREGkjFRxdT16OIiIiISO2aB/ny/sQBDOvYguO2Sia+tYbP1mcZHUtEREQaQMVHF1u8M09djyIiIiIiZxHo68Pr4/pxTa9WVNgd/HbOBt5cts/oWCIiIlJPKj66UFXX4w5AXY8iIiIiImfj62Pmnzf3YsKQeACe/Xo7X+4z43A4jA0mIiIiF0zFRxdavDOP9ZnqehQREREROR+z2cSUq5L4/ahOAHx70MwfPkunotJucDIRERHv4Kq/6an46CLqehQRERERuTAmk4mUS9rz7LVJmHDw8bosHnh3HSdslUZHExERkTpS8dFF1PUoIiIiIlI/N/Vtzd2d7Pj6mPl2aw7j3lhFwXGb0bFERESkDlR8dAF1PYqIyPmUlpbStm1bHn30UaOjiIi4pR7hDmaN60OInw+r9h5h7KvLOVx4wuhYUguNaSIicjoVH11AXY8iInI+f/3rXxk4cKDRMURE3NqAduHMuX8QkcF+bMsu4vpXlrEnr8ToWPILGtNEROR0Kj42MnU9iojI+ezcuZNt27YxevRoo6OIiLi9pFahfPrgYNpGBHLg6HFufGUZm7MKjI4lJ2lMExGRX1LxsZGd6nr081HXo4iIN1q8eDFjxoyhVatWmEwmPvvsszNek5aWRnx8PP7+/gwYMIBVq1bVeP7RRx9l2rRpLkosIuL52kQE8vEDg+naKpT8knJueW0Fy3blGR3L42lMExGRxqDiYyOq0fU4UF2PIiLeqKSkhJ49e5KWllbr83PmzCE1NZWpU6eybt06evbsyahRozh8+DAAn3/+OR07dqRjx46ujC0i4vFahPjxwX0DGZQQQXFZBXe9uZq5Px0yOpZH05gmIiKNwcfoAN7s9K7H+9X1KCLilUaPHn3OW8umT5/OxIkTmTBhAgAzZszgq6++YtasWTzxxBOsWLGCDz74gI8++oji4mJsNhuhoaFMmTKl1uOVlZVRVlZW/biwsBAAm82GzVa/lV9P7Vff/Z1BGZRBGdw3hztn8LfAzDt6kfrxT8xPP0zK++v481VduK1/nMsy1OcY7srVYxo0zrhWG3f4OnY2b7wm0HV5Em+8Jmga11VZUQGAw2Fv8O8QdaHiYyNR16OIiJSXl7N27VomT55cvc1sNnP55ZezfPlyAKZNm1Z9e9rs2bPZvHnzOX9JmzZtGk899dQZ2+fPn09gYGCD8i5YsKBB+zuDMiiDMtTOHXK4c4bRoVAcbWZZjpmpX25lxfrNjGrtwGRyXYa6KC0tdWIS12qMMe3UPo01rtXGHb6Onc0brwl0XZ7EG68JvPu6NuSZAAv5+fnMnTu3Xse5kDFNxcdGoq5HERHJy8ujsrKS6OjoGtujo6PZtm1bvY45efJkUlNTqx8XFhYSFxfHyJEjCQ0NrdcxbTYbCxYsYMSIEVit1nodo6GUQRmUwX1zeEqGXzkc/N93u0lblMHXByxExMbxp9GdsZidU4F0xufhVFefJ2qMMQ0aZ1yrjTt8HTubN14T6Lo8iTdeEzSN67JvzeOtnT8RERHBlVcm1+t4FzKmqfjYCNT1KCIi9XHXXXed9zV+fn74+fmdsd1qtTb4hyNnHKOhlEEZlMF9c3hCht9f0YWo0AD+/OUW3l25n6PHK5h+c0/8fCwuy3C+fZuKuoxp0LjjWm3c4evY2bzxmkDX5Um88ZrAu6/L4lNVDjSZzC4Z07TgTCNQ16OIiABERkZisVjIycmpsT0nJ4eYmBiDUomIeLfxg+P5v1t6Y7WY+GrTIe6ZvYbisgqjY3k8jWkiIlJfKj46mboeRUTkFF9fX/r27cvChQurt9ntdhYuXMigQYMMTCYi4t2u7tmKWXclE+hrYemuPG6buYL84rLz7yhnpTFNRETqS8VHJ1PXo4hI01JcXMyGDRvYsGEDAHv27GHDhg1kZmYCkJqaysyZM3nrrbfYunUrDz74ICUlJdUrhdZXWloaSUlJJCfXb44WERFvd1GHFvxn4kDCg3zZdKCAm2YsZ/8Rz13wxRWMGtNA45qIiDfTnI9O5HA4+D91PYqINClr1qzhkksuqX58atL88ePHM3v2bMaOHUtubi5TpkwhOzubXr16MW/evDMm7L9QKSkppKSkUFhYSFhYWIOOJSLirXrGNeOjBwYx7o1VZOSVcOOMZbx1d386xzhvIRNvYtSYBhrXRES8mYqPTrRkZx7r1PUoItKkDB8+HIfDcc7XTJo0iUmTJrkokYiInC6xRTCfPDiYcbNWsiOnmJtnLGfWXcn0iw83Oprb0ZgmIiKNQbddO4nmehQRERERcU8xYf58eP8g+rZtTuGJCm5/fSULt+acf0cRERFpMBUfnURdjyIiIiIi7qtZoC/v3jOASztHUVZh57531vLx2gNGxxIREfF6Kj46gboeRUTE1TQxv4jIhQvwtfDqnX25oU9rKu0OHv1oI6/+sNvoWILGNRERb6bioxOo61FERFwtJSWF9PR0Vq9ebXQUERGPYrWYeeGmHtw3rOrn9mlfb+PZuVux288916E0Lo1rIiLeS8XHBlLXo4iIiIiIZzGZTPzhyi5MHt0ZgNcWZ/D7jzdhq7QbnExERMT7uEXxMS0tjfj4ePz9/RkwYACrVq0662s//fRT+vXrR7NmzQgKCqJXr1688847Lkxbk7oeRUREREQ80/0XJ/LCTT2xmE18su4A97+zluPllUbHEhER8SqGFx/nzJlDamoqU6dOZd26dfTs2ZNRo0Zx+PDhWl8fHh7OH//4R5YvX86mTZuYMGECEyZM4JtvvnFx8ppdj7cPUNejiIiIiIinubFva167sy9+Pma+23aYO95YSUGpzehYIiIiXsPw4uP06dOZOHEiEyZMICkpiRkzZhAYGMisWbNqff3w4cO57rrr6NKlC4mJiTz88MP06NGDpUuXujg5LN2dX931+IC6HkVEREREPNJlXaJ5794BhPr7sHbfUW56dRnZBSeMjiUiIuIVfIw8eXl5OWvXrmXy5MnV28xmM5dffjnLly8/7/4Oh4PvvvuO7du389xzz9X6mrKyMsrKyqofFxYWAmCz2bDZ6vcXTZvNhsMB/1q4C4Bbk1vTPMBS7+PVN8Pp/zWCMrhPBnfJoQzK4OwMRn9fubO0tDTS0tKorNTtgSIiztAvPpyPHhjMuFkr2ZFTzA2vLOPte/qT2CLY6GhNgsY1ERHvZWjxMS8vj8rKSqKjo2tsj46OZtu2bWfdr6CggNjYWMrKyrBYLLz88suMGDGi1tdOmzaNp5566ozt8+fPJzAwsN7ZtxeY2HCgEKvJQUJ5BnPnZtT7WA2xYMECQ86rDO6ZAdwjhzIog7MylJaWOjGJd0lJSSElJYXCwkLCwsKMjiMi4hU6xYTw8QODGT9rFRl5Jdw0Yzlv3pVMz7hmRkfzehrXRES8l6HFx/oKCQlhw4YNFBcXs3DhQlJTU0lISGD48OFnvHby5MmkpqZWPy4sLCQuLo6RI0cSGhpar/OXl5fzz39+D8DtA9ty65Wd63WchrDZbCxYsIARI0ZgtVpdfn5lcK8M7pJDGZTB2RlOdauLiIi4Slx4IB89MIgJs1ez6UABt85cwat39mVgfDOjo4mIiHgkQ4uPkZGRWCwWcnJyamzPyckhJibmrPuZzWbat28PQK9evdi6dSvTpk2rtfjo5+eHn5/fGdutVmu9fxlesiuPvcUm/HzM/PqSDoYWnBpyHcrgfRncJYcyKIOzMhidXUREmqaIYD/enziQB95Zy9Jdedw9ezXP39Adk9HBREREPJChC874+vrSt29fFi5cWL3NbrezcOFCBg0aVOfj2O32GvM6NiaHw8G/v9sNVM31GBWqFa5FRERERLxNsJ8Pb9zVj6t6tMRW6eCRjzax+JDKjyIiIhfK8NuuU1NTGT9+PP369aN///68+OKLlJSUMGHCBADGjRtHbGws06ZNA6rmcOzXrx+JiYmUlZUxd+5c3nnnHV555RWX5F2yM4/1+wuwmhxMvKidS84pIiIiIiKu5+dj4V+39CYiyJe3lu/jk70WfrUrj0u7tDQ6moiIiMcwvPg4duxYcnNzmTJlCtnZ2fTq1Yt58+ZVL0KTmZmJ2fxzg2ZJSQm//vWvOXDgAAEBAXTu3Jl3332XsWPHuiRvp5gQxg1sw8HMvUSFnHk7t4iIiIiIeA+z2cSfr+5KswAflv+0k6GJEUZHEhERaZC45gHc0Kc17aOCXXI+w4uPAJMmTWLSpEm1Prdo0aIaj//yl7/wl7/8xQWpahcd6s+Tv+ps2OrWIiIiAGlpaaSlpVFZWWl0FBERr2cymZh0SSLtSrdjMunW68agcU1ExHV6t2lO7zbNXXY+Q+d8FBERkfpJSUkhPT2d1atXGx1FRKTJUN2x8WhcExHxXio+ioiIiIiIiIiISKNQ8VFEREREREREREQahYqPIiIiIiIiIiIi0ihUfBQREREREREREZFGoeKjiIiIiIiIiIiINAoVH0VERERERERERKRRqPgoIiIiIiIiIiIijULFRxEREQ+UlpZGUlISycnJRkcRERFpMI1rIiLeS8VHERERD5SSkkJ6ejqrV682OoqIiEiDaVwTEfFeKj6KiIiIiIiIiIhIo/AxOoCrORwOAAoLC+t9DJvNRmlpKYWFhVitVmdFUwZl8PgcyqAMzs5w6t/qU/92y5k0rimDMnhvBnfJoQzOy6Bx7fycMa7Vxh2+hpzNG68JdF2exBuvCXRddXUhY1qTKz4WFRUBEBcXZ3ASERGpq6KiIsLCwoyO4ZY0romIeB6Na2encU1ExLPUZUwzOZrYn93sdjsHDx4kJCQEk8lU47nk5ORa5xj55fbCwkLi4uLYv38/oaGhjZ65Ns7OcLZrd2aGupzjXK+p7blzZajPNdVHY3weLsSp411IjgvJcCHvW20Z6vp9VZ9stWms788LyVWXDPW5zgvZp2/fvuzataveGS70uQv9/qwrh8NBUVERrVq1wmzWTCG1udBxrbHeq4bytHGtoWPa2Z5v6uNafca0C81Q1/flbBk0rp2Zob7X6MxxrT7fbxe6XeOaa5xrXGsIdxjrnM0brwl0XZ7EG68JdF11dSFjWpPrfDSbzbRu3brW5ywWS61vwNm2h4aGGv6F6KwMZ7tGZ2aoyznO9ZpzPVdbhoZcU3048/NwIX55vLrkuJAM9XnfTs9wod9Xzvr8OPv7sz65zpWhPse70PetIRku9LkL/f68EOoMObcLHdca871yBk8Z1xo6pp3v+aY6rtVnTLvQDBf6vvwyg8Y1512jM8e1+n6/1ed907jWuM41rjmDO4x1zuaN1wS6Lk/ijdcEuq66qOuYpj+3nSYlJeWCtnsTV1xjXc5xrtdcaEZ3fd+cnas+x7uQfRrrffO07zdPe98mTpzYoONd6HPu+r41dU35vWrs62zov411PUZDXu8qzsxV32M5c1yr7/NNeVxzxft2vnGtqbxvIiIinqjJ3XbtDIWFhYSFhVFQUGDo7WnKoAzulkMZlMHdMkjduMN7pQzKoAzum0MZ3CeD1J83vn/eeE2g6/Ik3nhNoOtqDOp8rAc/Pz+mTp2Kn5+fMiiDW2RwlxzKoAzulkHqxh3eK2VQBmVw3xzK4D4ZpP688f3zxmsCXZcn8cZrAl1XY1Dno4iIiIiIiIiIiDQKdT6KiIiIiIiIiIhIo1DxUURERERERERERBqFio8iIiIiIiIiIiLSKFR8FBERERERERERkUah4uMFWLx4MWPGjKFVq1aYTCY+++wzl2eYNm0aycnJhISEEBUVxbXXXsv27dtdnuOUv/3tb5hMJn7729+69LyVlZU8+eSTtGvXjoCAABITE3nmmWdozPWT6vL+b926lauvvpqwsDCCgoJITk4mMzPTaRleeeUVevToQWhoKKGhoQwaNIivv/4agCNHjvDQQw/RqVMnAgICaNOmDb/5zW8oKChw2vlPycrK4o477iAiIoKAgAC6d+/OmjVran3tAw88gMlk4sUXX6z3+c71ubfZbDz++ON0796doKAgWrVqxbhx4zh48GCNY+zYsYNrrrmGyMhIQkNDGTp0KN9//32dM9Tle2/48OGYTKYaHw888MAZx5o9ezY9evTA39+fqKgoUlJS6pThz3/+8xnH79y5c/Xzr732GsOHDyc0NBSTycSxY8dq7L93717uueeeGt83U6dOpby8/KznPN/XvcPhYMqUKbRs2ZKAgAAuv/xydu7cWe9z7tq1i5CQEJo1a1anz4k0jNHjmruNaaBx7Zc0rp1J45rGNY1r7istLY34+Hj8/f0ZMGAAq1atOutrP/30U/r160ezZs0ICgqiV69evPPOOy5MWzcXck2n++CDDzCZTFx77bWNG7CeLuS6Zs+efca/Ff7+/i5MW3cX+n4dO3aMlJQUWrZsiZ+fHx07dmTu/7d350FRnVkbwJ8GZImCER0QRBlZ4i46EhmWqCMKGqMYJ66EIWppGbFco5AoHy7RwS0uJC5B1MyEiImRiZM4RMQlcVekRVBREOOauERBNCLQ5/vDoistIN1NL2ieX1VX2Xd7z7lLH/vlvbd37jRRtNrRJafq6oZCocCAAQNMGLF2dD1WK1euVP+fpGXLlpg2bRoePXpkomi1p0teZWVlmD9/Pjw9PWFrawsfHx+kpaUZJS52PurgwYMH8PHxwSeffGK2GPbv34+oqCgcOXIE6enpKCsrQ0hICB48eGDyWI4fP47169ejc+fOJm978eLFWLt2LT7++GOcPXsWixcvxpIlS5CQkGC0Nms7/gUFBQgKCkLbtm2xb98+ZGdnIzY21qCF0c3NDfHx8cjMzMSJEyfQu3dvhIWFITc3F9evX8f169exbNky5OTkYPPmzUhLS8PYsWMN1j4A3L17F4GBgWjQoAH+97//4cyZM1i+fDmaNGlSZdnU1FQcOXIErq6udWrzWfv+4cOHOHnyJGJjY3Hy5Els374deXl5GDRokMZyb7zxBsrLy7Fnzx5kZmbCx8cHb7zxBn7++WetYtD22hs3bhxu3Lihfi1ZskRj/kcffYTZs2cjJiYGubm52L17N0JDQ7XeFx06dNDY/oEDBzT2Rb9+/fDBBx9Uu+65c+egUqmwfv165ObmYsWKFVi3bl2NywO1n/dLlizB6tWrsW7dOhw9ehQNGzZEaGiouhDr0mZZWRlGjhyJ1157Tev9QXVj7rpWn2oawLr2NNY11jWAdY117fmxdetWTJ8+HXFxcTh58iR8fHwQGhqKmzdvVru8o6MjZs+ejcOHDyM7OxujR4/G6NGj8f3335s48prpmlOlS5cu4b333qu3554+eTk4OGh8Vvz0008mjFg7uub1+PFj9O3bF5cuXcK2bduQl5eHxMREtGjRwsSR10zXnLZv365xnHJycmBpaYmhQ4eaOPJn0zWvL774AjExMYiLi8PZs2eRlJSErVu3PrPemIOuec2ZMwfr169HQkICzpw5gwkTJuDNN99EVlaW4YMT0gsASU1NNXcYcvPmTQEg+/fvN2m79+/fF29vb0lPT5eePXvKlClTTNr+gAEDZMyYMRrThgwZIuHh4SZpv7rjP3z4cHn77bdN0v7vNWnSRDZs2FDtvC+//FKsra2lrKzMYO1FR0dLUFBQrctdvXpVWrRoITk5OeLu7i4rVqwwSPvaXHvHjh0TAPLTTz+JiMitW7cEgPzwww/qZYqLiwWApKen6xVHdddebdfCr7/+KnZ2drJ792692oyLixMfH59al9u7d68AkLt379a67JIlS6R169Zatf/0vlepVNK8eXNZunSpetq9e/fExsZGtmzZonObs2bNkrfffls2bdokjRs31iomMpz6UNfMVdNEWNdY11jXWNdY15533bt3l6ioKPX7iooKcXV1lX/+859ab6Nr164yZ84cY4SnF31yKi8vl4CAANmwYYNERkZKWFiYCSLVja55PS/XkK55rV27Vjw8POTx48emClFndb2uVqxYIfb29lJSUmKsEPWia15RUVHSu3dvjWnTp0+XwMBAo8apK13zcnFxkY8//lhjmrH+/8mRj8+5yluPHB0dTdpuVFQUBgwYgD59+pi03UoBAQHIyMjA+fPnAQCnTp3CgQMH0L9/f7PEo1Kp8N133+GVV15BaGgonJyc4OfnZ9RbGCsqKpCSkoIHDx7A39+/2mWKiorg4OAAKysrg7W7Y8cO+Pr6YujQoXByckLXrl2RmJiosYxKpUJERARmzpyJDh06GKxtbRUVFUGhUKhvcWratCnatGmDf/3rX3jw4AHKy8uxfv16ODk5oVu3bnq3AVS99pKTk9GsWTN07NgR77//Ph4+fKiel56eDpVKhWvXrqFdu3Zwc3PDsGHDcOXKFa3bvXDhAlxdXeHh4YHw8PA63/5YVFSk9+dHYWEhfv75Z43PgcaNG8PPzw+HDx/Wqc09e/bgq6++MuvIcjI/c9U0gHXtaaxrrGuVWNdY154Hjx8/RmZmpsaxs7CwQJ8+fZ557CqJCDIyMpCXl4cePXoYM1St6ZvT/Pnz4eTkZPBR4oaib14lJSVwd3dHy5Yt1SPk6xN98tqxYwf8/f0RFRUFZ2dndOzYEYsWLUJFRYWpwn6mul5XAJCUlIQRI0agYcOGxgpTZ/rkFRAQgMzMTPUtzBcvXsTOnTvx+uuvmyRmbeiTV2lpaZU7Wuzs7DTuQjAUw/3PjUxOpVJh6tSpCAwMRMeOHU3WbkpKCk6ePInjx4+brM2nxcTEoLi4GG3btoWlpSUqKiqwcOFChIeHmyWemzdvoqSkBPHx8fjwww+xePFipKWlYciQIdi7dy969uxpsLZOnz4Nf39/PHr0CI0aNUJqairat29fZbnbt29jwYIFGD9+vMHaBp580K5duxbTp0/HBx98gOPHj2Py5MmwtrZGZGQkgCe3D1pZWWHy5MkGbVsbjx49QnR0NEaOHAkHBwcAgEKhwO7duzF48GDY29vDwsICTk5OSEtLq/a2utrUdO2NGjUK7u7ucHV1RXZ2NqKjo5GXl4ft27cDeLLvVCoVFi1ahFWrVqFx48aYM2cO+vbti+zsbFhbWz+zXT8/P2zevBlt2rTBjRs3MG/ePLz22mvIycmBvb29znnk5+cjISEBy5Yt03ldAOpb+5ydnTWmOzs713jbX3Vt3rlzB++88w4+//xz9TGjPx5z1TSAda06rGusawDrWiXWtfrv9u3bqKioqPbYnTt3rsb1ioqK0KJFC5SWlsLS0hJr1qxB3759jR2uVvTJ6cCBA0hKSoJSqTRBhPrRJ682bdpg48aN6Ny5M4qKirBs2TIEBAQgNzcXbm5upgi7VvrkdfHiRezZswfh4eHYuXMn8vPzMXHiRJSVlSEuLs4UYT+TvtdVpWPHjiEnJwdJSUnGClEv+uQ1atQo3L59G0FBQRARlJeXY8KECfXqtmt98goNDcVHH32EHj16wNPTExkZGdi+fbtROsDZ+fgci4qKQk5OjlF6pWty5coVTJkyBenp6WZ9yO+XX36J5ORkfPHFF+jQoQOUSiWmTp0KV1dX9RcFU1KpVACAsLAwTJs2DQDQpUsXHDp0COvWrTPol7Q2bdpAqVSiqKgI27ZtQ2RkJPbv36/xRa24uBgDBgxA+/btMXfuXIO1DTzJ1dfXF4sWLQIAdO3aFTk5OVi3bh0iIyORmZmJVatW4eTJk1AoFAZtuzZlZWUYNmwYRARr165VTxcRREVFwcnJCT/++CPs7OywYcMGDBw4EMePH4eLi4tO7dR07f3+C3GnTp3g4uKC4OBgFBQUwNPTEyqVCmVlZVi9ejVCQkIAAFu2bEHz5s2xd+/eWp+R9fsRUJ07d4afnx/c3d3x5Zdf6vzX7WvXrqFfv34YOnQoxo0bp9O6+qqpzXHjxmHUqFH1ZqQBmYc5ahrAulYT1jXWNYB1Td82WdeeH/b29lAqlSgpKUFGRgamT58ODw8P9OrVy9yh6ez+/fuIiIhAYmIimjVrZu5wDMrf319jRHxAQADatWuH9evXY8GCBWaMrG5UKhWcnJzw6aefwtLSEt26dcO1a9ewdOnSetH5WFdJSUno1KkTunfvbu5Q6mzfvn1YtGgR1qxZAz8/P+Tn52PKlClYsGABYmNjzR2e3latWoVx48ahbdu2UCgU8PT0xOjRo7Fx40bDN2bwG7n/IGDmZ2NFRUWJm5ubXLx40aTtpqamCgCxtLRUvwCIQqEQS0tLKS8vN0kcbm5uVZ5NsGDBAmnTpo1J2n/6+JeWloqVlZUsWLBAY7lZs2ZJQECAUWMJDg6W8ePHq98XFxeLv7+/BAcHy2+//Wbw9lq1aiVjx47VmLZmzRpxdXUVkSfP9ag8H35/jlhYWIi7u3ud26/p2nv8+LEMHjxYOnfuLLdv39aYt3v3brGwsJCioiKN6V5eXjo9B0hEt2uvpKREAEhaWpqIiGzcuFEAyJUrVzSWc3Jykk8//VSnOCr5+vpKTEyMxrTano117do18fb2loiICKmoqNC6raf3fUFBgQCQrKwsjeV69OghkydP1rrNxo0ba5wvFhYW6s+ZpKQkreOjujFnXTNXTRNhXavEusa6xrrGuvY8Ky0tFUtLyyrX0j/+8Q8ZNGiQ1tsZO3ashISEGDg6/eiaU1ZWVpV6plAo1J9f+fn5Jor82Qx1rN566y0ZMWKEgaPTnz559ejRQ4KDgzWm7dy5UwBIaWmpsULVWl2OVUlJiTg4OMjKlSuNGKF+9MkrKChI3nvvPY1p//73v8XOzk6numNMdTlev/32m1y9elVUKpXMmjVL2rdvb/D4+MzH54yIYNKkSUhNTcWePXvQunVrk7YfHByM06dPQ6lUql++vr4IDw+HUqmEpaWlSeJ4+PAhLCw0T19LS0v1SA1Ts7a2xquvvoq8vDyN6efPn4e7u7tR21apVCgtLQXwZGRISEgIrK2tsWPHDqOM4gkMDHxmnhEREcjOztY4R1xdXTFz5kyj/Xpg5ciQCxcuYPfu3WjatKnG/MrnUz19zlhYWGh9zuhz7VXe8lI5AiUwMBAANPbfr7/+itu3b+t1npSUlKCgoECnES7Xrl1Dr1690K1bN2zatKnKPtFF69at0bx5c2RkZKinFRcX4+jRoxp/na6tzcOHD2ucL/Pnz1ePRHjzzTf1jo/qP3PXNIB1rSasa6xr1WFd065N1jXTs7a2Rrdu3TSOnUqlQkZGRo3PkK3O7z9/zE3XnNq2bVulng0aNAh/+9vfoFQq0bJlS1OGXyNDHKuKigqcPn1a51HexqRPXoGBgcjPz9f43D5//jxcXFxqfWyFKdTlWH311VcoLS3F22+/bewwdaZPXjX9Pw14Uk/rg7ocL1tbW7Ro0QLl5eX4+uuvERYWZvgADd6d+QK7f/++ZGVlqf+q9NFHH0lWVpb6lwdN4d1335XGjRvLvn375MaNG+rXw4cPTRbD08zxq6CRkZHSokUL+fbbb6WwsFC2b98uzZo1k1mzZhmtzdqO//bt26VBgwby6aefyoULFyQhIUEsLS3lxx9/NFgMMTExsn//fiksLJTs7GyJiYkRhUIhu3btkqKiIvHz85NOnTpJfn6+xvlhyJE7x44dEysrK1m4cKFcuHBBkpOT5aWXXpLPP/+8xnXq+qugz9r3jx8/lkGDBombm5solUqNvCv/Ynjr1i1p2rSpDBkyRJRKpeTl5cl7770nDRo0EKVSqVUMtV17+fn5Mn/+fDlx4oQUFhbKN998Ix4eHtKjRw+N7YSFhUmHDh3k4MGDcvr0aXnjjTekffv2Wv3K3YwZM2Tfvn1SWFgoBw8elD59+kizZs3k5s2bIiJy48YNycrKksTERPWvoGZlZcmdO3dE5MkvtXp5eUlwcLBcvXpVIw999r2ISHx8vLz88svyzTffSHZ2toSFhUnr1q3Vo5P0afN5+UXDF4G561p9rGkirGusa6xrIqxrrGvPn5SUFLGxsZHNmzfLmTNnZPz48fLyyy/Lzz//LCIiERERGqNqFy1aJLt27ZKCggI5c+aMLFu2TKysrCQxMdFcKVSha05Pq6+/dq1rXvPmzZPvv/9eCgoKJDMzU0aMGCG2traSm5trrhSqpWtely9fFnt7e5k0aZLk5eXJt99+K05OTvLhhx+aK4Uq9D0Hg4KCZPjw4aYOV2u65hUXFyf29vayZcsWuXjxouzatUs8PT1l2LBh5kqhWrrmdeTIEfn666+loKBAfvjhB+ndu7e0bt26xjsN6oKdjzqovOXj6VdkZKTJYqiufQCyadMmk8XwNHN8SSsuLpYpU6ZIq1atxNbWVjw8PGT27NlGHZ6uzfFPSkoSLy8vsbW1FR8fH/nPf/5j0BjGjBkj7u7uYm1tLX/6058kODhYdu3a9cz4AEhhYaFB4/jvf/8rHTt2FBsbG2nbtm2tt1bV9Uvas/Z9YWFhjXnv3btXvY3jx49LSEiIODo6ir29vfz1r3+VnTt3ah1Dbdfe5cuXpUePHuLo6Cg2Njbi5eUlM2fOrHJLXFFRkYwZM0ZefvllcXR0lDfffFMuX76sVQzDhw8XFxcXsba2lhYtWsjw4cM1bqGJi4t7ZoybNm2qMY+a1Hbeq1QqiY2NFWdnZ7GxsZHg4GDJy8tTr69Pm/ySZjrmrmv1saaJsK6xrrGuibCusa49nxISEqRVq1ZibW0t3bt3lyNHjqjn9ezZU+Pzbfbs2erPtyZNmoi/v7+kpKSYIepn0yWnp9XXzkcR3fKaOnWqellnZ2d5/fXX5eTJk2aIuna6Hq9Dhw6Jn5+f2NjYiIeHhyxcuNBkj3zRlq45nTt3TgCo63l9pUteZWVlMnfuXPH09BRbW1tp2bKlTJw40SiddHWlS1779u2Tdu3aiY2NjTRt2lQiIiLk2rVrRolLIVJPxogSERERERERERHRC4XPfCQiIiIiIiIiIiKjYOcjERERERERERERGQU7H4mIiIiIiIiIiMgo2PlIRERERERERERERsHORyIiIiIiIiIiIjIKdj4SERERERERERGRUbDzkYiIiIiIiIiIiIyCnY9EfwCXLl2CQqGAUqk0dyhERER1xrpGREQvsrlz56JLly7q9++88w4GDx5stniI6oqdj0RERERERERERGQU7Hwkes6VlZWZOwQiIiKDYV0jIqL67PHjx+YOgei5w85HIgPr1asXJk+ejFmzZsHR0RHNmzfH3LlztVpXoVBg7dq16N+/P+zs7ODh4YFt27ap51feZrZ161b07NkTtra2SE5Ohkqlwvz58+Hm5gYbGxt06dIFaWlpVbZ/7tw5BAQEwNbWFh07dsT+/fs15ufk5KB///5o1KgRnJ2dERERgdu3b6vnb9u2DZ06dYKdnR2aNm2KPn364MGDB/rtKCIiei6wrhER0R9Zr169MGnSJEydOhXNmjVDaGhorfVFpVJhyZIl8PLygo2NDVq1aoWFCxeq50dHR+OVV17BSy+9BA8PD8TGxvKPb/RCY+cjkRF89tlnaNiwIY4ePYolS5Zg/vz5SE9P12rd2NhY/P3vf8epU6cQHh6OESNG4OzZsxrLxMTEYMqUKTh79ixCQ0OxatUqLF++HMuWLUN2djZCQ0MxaNAgXLhwQWO9mTNnYsaMGcjKyoK/vz8GDhyIO3fuAADu3buH3r17o2vXrjhx4gTS0tLwyy+/YNiwYQCAGzduYOTIkRgzZgzOnj2Lffv2YciQIRARA+wxIiKqz1jXiIjoj+yzzz6DtbU1Dh48iPj4+GfWFwB4//33ER8fj9jYWJw5cwZffPEFnJ2d1fPt7e2xefNmnDlzBqtWrUJiYiJWrFhhjtSITEOIyKB69uwpQUFBGtNeffVViY6OrnVdADJhwgSNaX5+fvLuu++KiEhhYaEAkJUrV2os4+rqKgsXLqzS5sSJEzXWi4+PV88vKysTNzc3Wbx4sYiILFiwQEJCQjS2ceXKFQEgeXl5kpmZKQDk0qVLteZBREQvDtY1IiL6I+vZs6d07dpV/b62+lJcXCw2NjaSmJiodRtLly6Vbt26qd/HxcWJj4+P+n1kZKSEhYXpnQORuVmZqc+T6IXWuXNnjfcuLi64efOmVuv6+/tXef/0r3n6+vqq/11cXIzr168jMDBQY5nAwECcOnWqxm1bWVnB19dXPfrk1KlT2Lt3Lxo1alQlpoKCAoSEhCA4OBidOnVCaGgoQkJC8NZbb6FJkyZa5UVERM8v1jUiIvoj69atm/rftdWXe/fuobS0FMHBwTVub+vWrVi9ejUKCgpQUlKC8vJyODg4GCV2ovqAnY9ERtCgQQON9wqFAiqVymDbb9iwocG2VamkpAQDBw7E4sWLq8xzcXGBpaUl0tPTcejQIezatQsJCQmYPXs2jh49itatWxs8HiIiqj9Y14iI6I/s93Wqtvpy8eLFZ27r8OHDCA8Px7x58xAaGorGjRsjJSUFy5cvN3jcRPUFn/lIVM8cOXKkyvt27drVuLyDgwNcXV1x8OBBjekHDx5E+/bta9x2eXk5MjMz1dv+y1/+gtzcXPz5z3+Gl5eXxquy2CoUCgQGBmLevHnIysqCtbU1UlNT65QvERG92FjXiIjoRVJbffH29oadnR0yMjKqXf/QoUNwd3fH7Nmz4evrC29vb/z0008mzoLItDjykaie+eqrr+Dr64ugoCAkJyfj2LFjSEpKeuY6M2fORFxcHDw9PdGlSxds2rQJSqUSycnJGst98skn8Pb2Rrt27bBixQrcvXsXY8aMAQBERUUhMTERI0eOVP+iaX5+PlJSUrBhwwacOHECGRkZCAkJgZOTE44ePYpbt2498wskERER6xoREb1Iaqsvtra2iI6OxqxZs2BtbY3AwEDcunULubm5GDt2LLy9vXH58mWkpKTg1VdfxXfffcc/fNELj52PRPXMvHnzkJKSgokTJ8LFxQVbtmypMtLjaZMnT0ZRURFmzJiBmzdvon379tixYwe8vb01louPj0d8fDyUSiW8vLywY8cONGvWDADUo0yio6MREhKC0tJSuLu7o1+/frCwsICDgwN++OEHrFy5EsXFxXB3d8fy5cvRv39/o+0LIiJ6/rGuERHRi6S2+gIAsbGxsLKywv/93//h+vXrcHFxwYQJEwAAgwYNwrRp0zBp0iSUlpZiwIABiI2Nxdy5c82YFZFxKUREzB0EET2hUCiQmpqKwYMHmzsUIiKiOmNdIyIiIiI+85GIiIiIiIiIiIiMgp2PRCaSnJyMRo0aVfvq0KGDucMjIiLSCesaEREREWmDt10Tmcj9+/fxyy+/VDuvQYMGcHd3N3FERERE+mNdIyIiIiJtsPORiIiIiIiIiIiIjIK3XRMREREREREREZFRsPORiIiIiIiIiIiIjIKdj0RERERERERERGQU7HwkIiIiIiIiIiIio2DnIxERERERERERERkFOx+JiIiIiIiIiIjIKNj5SEREREREREREREbBzkciIiIiIiIiIiIyiv8HLDeAqlBTzTQAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots(1, 3, figsize=plt.figaspect(1/4))\n", - "\n", - "ax[0].plot(bench_probes, bench_recall)\n", - "ax[0].set_xscale('log')\n", - "ax[0].set_xticks(bench_probes, bench_probes)\n", - "ax[0].set_xlabel('n_probes')\n", - "ax[0].set_ylabel('recall')\n", - "ax[0].grid()\n", - "\n", - "ax[1].plot(bench_probes, bench_qps)\n", - "ax[1].set_xscale('log')\n", - "ax[1].set_xticks(bench_probes, bench_probes)\n", - "ax[1].set_xlabel('n_probes')\n", - "ax[1].set_ylabel('QPS')\n", - "ax[1].set_yscale('log')\n", - "ax[1].grid()\n", - "\n", - "ax[2].plot(bench_recall, bench_qps)\n", - "ax[2].set_xlabel('recall')\n", - "ax[2].set_ylabel('QPS')\n", - "ax[2].set_yscale('log')\n", - "ax[2].grid();" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Internal search types\n", - "Besides `n_probes`, `ivf_pq.SearchParams` contains a couple more parameters, which affect the internal workings of the algorithm.\n", - "\n", - "`internal_distance_dtype` controls the representation of the distance/similarity during the search.\n", - "By default, it's `np.float32`, but you can change it to `np.float16` when appropriate to save the memory bandwidth.\n", - "This can be a good idea when the dataset type is low precision anyway (e.g. `np.uint8`),\n", - "yet it may help with 32-bit float datasets too.\n", - "\n", - "`lut_dtype` is the Look-Up Table Data Type.\n", - "The specifics of the PQ algorithm is that it stores the data in the Product Quantizer (PQ) encoded format,\n", - "which needs to be decoded during the second-phase (in-cluster) search.\n", - "Thus, the algorithm constructs a lookup table for each cluster.\n", - "This is a costly operation, and the table itself can be rather large.\n", - "By default, the individual elements in the table are stored as 32-bit floats,\n", - "but you can change this to `np.float16` or `np.uint8` to reduce the table size.\n", - "\n", - "The exact size of the table is as follows:\n", - "\n", - "$ \\mathtt{lut\\_size} = \\mathtt{pq\\_dim} \\cdot \\mathtt{sizeof(lut\\_dtype) \\cdot 2^{\\mathtt{pq\\_bits}}} $\n", - "\n", - "Ideally, the lookup table should fit in the shared memory of a GPU's multiprocessor,\n", - "but it's not the case for wider datasets.\n", - "The logic of deciding whether this table should stay in the shared or the global memory of the GPU is somewhat complicated.\n", - "Yet, you can see the outcome when you gradually change `pq_dim` and observe a sudden drop in QPS after a certain threshold.\n", - "The shared-memory kernel version is typically 2-5x faster than the global-memory version.\n", - "\n", - "However `pq_dim` strongly affects the recall and requires the index to be re-build on change.\n", - "This is where `lut_dtype` comes in handy: you can halve or quarter the lookup table size by changing it.\n", - "Though it does affect the recall too.\n", - "\n", - "Also note, it does not make sense to set the `lut_dtype` to a more precise type than `internal_distance_dtype`,\n", - "as the former is converted to the latter internally.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "209 ms ± 151 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "178 ms ± 485 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "182 ms ± 297 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "176 ms ± 220 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "181 ms ± 439 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "bench_qps_s1 = np.zeros((5,), dtype=np.float32)\n", - "bench_recall_s1 = np.zeros((5,), dtype=np.float32)\n", - "k = 10\n", - "n_probes = 256\n", - "search_params_32_32 = ivf_pq.SearchParams(n_probes=n_probes, internal_distance_dtype=np.float32, lut_dtype=np.float32)\n", - "search_params_32_16 = ivf_pq.SearchParams(n_probes=n_probes, internal_distance_dtype=np.float32, lut_dtype=np.float16)\n", - "search_params_32_08 = ivf_pq.SearchParams(n_probes=n_probes, internal_distance_dtype=np.float32, lut_dtype=np.uint8)\n", - "search_params_16_16 = ivf_pq.SearchParams(n_probes=n_probes, internal_distance_dtype=np.float16, lut_dtype=np.float16)\n", - "search_params_16_08 = ivf_pq.SearchParams(n_probes=n_probes, internal_distance_dtype=np.float16, lut_dtype=np.uint8)\n", - "search_ps = [search_params_32_32, search_params_32_16, search_params_32_08, search_params_16_16, search_params_16_08]\n", - "bench_names = ['32/32', '32/16', '32/8', '16/16', '16/8']\n", - "\n", - "for i, sp in enumerate(search_ps):\n", - " r = %timeit -o ivf_pq.search(sp, index, queries, k, handle=resources); resources.sync()\n", - " bench_qps_s1[i] = (queries.shape[0] * r.loops / np.array(r.all_runs)).mean()\n", - " bench_recall_s1[i] = calc_recall(ivf_pq.search(sp, index, queries, k, handle=resources)[1], gt_neighbors)" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0UAAAHgCAYAAABqycbBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAACcU0lEQVR4nOzdd1wUx/8/8NfRexGpioBdsGNUrNgANbZoQGzYjYo11mgENPbejRVj+dh7QbE31IgaY29YoiIqCiJSb35/+Lv9ct7RFEW51/PxIHFn52Znd/Z2730zOycTQggQERERERFpKK38rgAREREREVF+YlBEREREREQajUERERERERFpNAZFRERERESk0RgUERERERGRRmNQREREREREGo1BERERERERaTQGRUREREREpNEYFBERERERkUZjUEQqEhIS0LNnT9jZ2UEmk2Hw4MEAgOfPn6Ndu3awsrKCTCbDnDlz8rWeee3OnTvw8vKCubk5ZDIZduzYkd9VyjPBwcGQyWR4+fJlfleFqEDz9PSEp6entPzgwQPIZDKEhobmW52y4uzsjK5du+bb9qdPn47ixYtDW1sblStXBgCkpaVhxIgRcHR0hJaWFlq3bp1lGXK5HOXLl8fEiRNzte2uXbvC2dlZKS2z+x99OsX9J6OcnnehoaGQyWR48ODBl6lcHvD09ET58uU/+fXHjh2DTCbDsWPHlNLXrFmDsmXLQldXFxYWFrkqMywsDCYmJnjx4sUn10sTMSjSEIoLS2Z/Z8+elfJOmjQJoaGh6Nu3L9asWYPOnTsDAIYMGYIDBw5g9OjRWLNmDXx8fPK8npMmTcq3YCQgIAD//vsvJk6ciDVr1qBatWr5Ug/69uzbtw/BwcH5XY189e7dO0yYMAEVK1aEkZERzM3NUbduXaxZswZCCJX8Ga8vWlpacHBwgJeXl8qNPyUlBXPnzkWVKlVgZmYGCwsLuLm5oXfv3rh586ZKuXK5HNbW1pg2bdqX2tV8cf36dQQHB3/TH/7y2sGDBzFixAjUrl0bq1atwqRJkwAAK1euxPTp09GuXTusXr0aQ4YMybKc//3vf3j8+DECAwM/u07q7n9nzpxBcHAw3rx5k+NyNm7ciE6dOqFUqVKQyWRKgXJGf//9NwIDA+Hm5gZjY2MUK1YMvr6+uH37ttr8mzZtQs2aNWFhYQErKyvUr18fe/fu/YQ9pW/ZzZs30bVrV5QoUQLLli3D0qVLAXwI5NV9hitbtqzS6318fFCyZElMnjw5P6r/3dLJ7wrQ1zV+/Hi4uLiopJcsWVL695EjR1CzZk0EBQUp5Tly5AhatWqFYcOGfbH6TZo0Ce3atcv2m8G89v79e0RERGDMmDF5cmOlgmXfvn1YuHChxgZGz58/R6NGjXDjxg20b98egYGBSEpKwtatW9GlSxeEhYVhzZo10NJS/p6tSZMm6NKlC4QQiIqKwqJFi9CwYUPs3bsXTZs2BQC0bdsW+/fvh7+/P3r16oXU1FTcvHkTe/bsQa1atVRu9ufPn8fLly/RvHnzr7b/X8P169cREhICT09Pld6LgurIkSPQ0tLCihUroKenp5RepEgRzJ49O0flTJ8+He3bt4e5uXmutr9s2TLI5XKVOn18/5sxYwZCQkLQtWvXHH9jv3jxYkRGRuKHH37Aq1evMs03depUnD59Gj///DMqVqyI6OhoLFiwAFWrVsXZs2eVeiDmz5+PgQMHonnz5pgyZQqSkpIQGhqKH3/8EVu3bsVPP/2Uq/3PT7du3VK5XtD/OXbsGORyOebOnav0+QwA9PX1sXz5cqU0ded+nz59MGzYMISEhMDU1PSL1regYFCkYZo2bZptD0hMTAxcXV3Vpue2C/d7oehi/p72Ly0tDXK5XOnDxPesoO1PdoQQSEpKgqGhYX5XJVsBAQG4ceMGtm/fjpYtW0rpAwcOxPDhwzFjxgxUrlwZw4cPV3pd6dKl0alTJ2m5TZs2qFixIubMmYOmTZvi77//xp49ezBx4kT89ttvSq9dsGCB2m/m9+3bBycnJ7i5uamt6/d0XD9VQdnHmJgYGBoaqrznc3OvuXTpEv755x/MnDkz19vX1dVVWyd197/cWrNmDYoUKQItLa0sh1YNHToU69evVzoGfn5+qFChAqZMmYK1a9dK6fPnz8cPP/yA3bt3S8PRunfvjiJFimD16tXfVVCkr6+f31X4psXExABQ/5lER0dH6bqambZt22LAgAHYvHkzunfvntdVLJAYppNEMa41KioKe/fulbplFUPvhBBYuHChlK7w5s0bDB48GI6OjtDX10fJkiUxdepUlW/gFN96VKhQAQYGBrC2toaPjw8uXLgA4MNwm3fv3mH16tXSNhRjjt++fYvBgwfD2dkZ+vr6sLGxQZMmTXDx4sVs9+vSpUto2rQpzMzMYGJigkaNGikNFwwODoaTkxMAYPjw4ZDJZNl+Uzt//ny4ubnByMgIlpaWqFatGtavX6+U58mTJ+jevTtsbW2hr68PNzc3rFy5UilPSkoKxo0bB3d3d5ibm8PY2Bh169bF0aNHlfIpnkuYMWMG5syZgxIlSkBfXx/Xr18H8KGr3dfXF9bW1jA0NESZMmUwZswYlXq/efNG+rbT3Nwc3bp1Q2JiYrbHUDFmOjIyErVq1YKhoSFcXFywZMmSPN+fTylj4cKFKF68OIyMjODl5YXHjx9DCIEJEyagaNGiMDQ0RKtWrRAbG6uyb/v370fdunVhbGwMU1NTNG/eHNeuXZPWd+3aFQsXLgSgPCRMQS6XY86cOXBzc4OBgQFsbW3Rp08fvH79Wmk7zs7O+PHHH3HgwAFUq1YNhoaG+PPPPwEA4eHhqFOnDiwsLGBiYoIyZcqoBAnqpKWlYcKECdLxc3Z2xm+//Ybk5GS12z516hSqV68OAwMDFC9eHH/99Ve22zh79iwOHDiArl27KgVECpMnT0apUqUwZcoUvH//PsuyKlSogMKFCyMqKgoAcO/ePQBA7dq1VfJqa2vDyspKJX3v3r1KvURZHde8ujYBwKpVq9CwYUPY2NhAX18frq6uWLx4cZb7m1OhoaH4+eefAQANGjSQzjHFUMOs9jGn9RJC4I8//kDRokVhZGSEBg0aKJ3nGeX0uGUmJ+elTCbDqlWr8O7dO5V7zdGjR3Ht2jWV46DOjh07oKenh3r16iml5+SekfGZoszuf127dpWCfRcXFyk9u2GOiuehslOrVi2VoLBUqVJwc3PDjRs3lNLj4+NhY2OjdP1R3NdyGiDn5L6U2XM8mT37cu7cOTRr1gyWlpYwNjZGxYoVMXfu3Czroe6ZomvXrqFhw4YwNDRE0aJF8ccff2R6zmV33QaAK1euoGvXrihevDgMDAxgZ2eH7t27q/TcKZ55unv37ifdHxWuX7+OBg0awMjICEWKFFE7xPe///5D69atYWxsDBsbGwwZMkTt9VrRU2ltbQ2ZTKYySiE9PR3x8fFZ1sfGxgYVK1bEzp07c7wPmo49RRomLi5O5WF7mUwGKysrlCtXDmvWrMGQIUNQtGhR/PrrrwCAKlWqSGOrFcNhFBITE1G/fn08efIEffr0QbFixXDmzBmMHj0az549U5qMoUePHggNDUXTpk3Rs2dPpKWl4eTJkzh79iyqVauGNWvWoGfPnqhevTp69+4NAChRogQA4JdffsGWLVsQGBgIV1dXvHr1CqdOncKNGzdQtWrVTPf32rVrqFu3LszMzDBixAjo6urizz//hKenJ44fP44aNWrgp59+goWFBYYMGQJ/f380a9YMJiYmmZa5bNkyDBw4EO3atcOgQYOQlJSEK1eu4Ny5c+jQoQOAD8ONatasCZlMhsDAQFhbW2P//v3o0aMH4uPjpYd34+PjsXz5cmno0Nu3b7FixQp4e3vj/Pnz0oPHCqtWrUJSUhJ69+4NfX19FCpUCFeuXEHdunWhq6uL3r17w9nZGffu3cPu3btVHjz29fWFi4sLJk+ejIsXL2L58uWwsbHB1KlTM91fhdevX6NZs2bw9fWFv78/Nm3ahL59+0JPT0/6Fiov9ie3Zaxbtw4pKSkYMGAAYmNjMW3aNPj6+qJhw4Y4duwYRo4cibt372L+/PkYNmyY0geANWvWICAgAN7e3pg6dSoSExOxePFi1KlTB5cuXYKzszP69OmDp0+fIjw8HGvWrFE5Ln369EFoaCi6deuGgQMHIioqCgsWLMClS5dw+vRppW+jb926BX9/f/Tp0we9evVCmTJlcO3aNfz444+oWLEixo8fD319fdy9exenT5/Otk169uyJ1atXo127dvj1119x7tw5TJ48WerVyeju3bto164devTogYCAAKxcuRJdu3aFu7t7pr0uALB7924AUHrfZ6Sjo4MOHTogJCQEZ86cQaNGjTIt6/Xr13j9+rU0HETxZcS6detQu3Zt6OhkfUuKjo7GpUuXMH78eKV0dcc1L69NwIfhUG5ubmjZsiV0dHSwe/du9OvXD3K5HP3798+y3tmpV68eBg4ciHnz5uG3335DuXLlAED6f2b7mJt6jRs3Dn/88QeaNWuGZs2a4eLFi/Dy8kJKSopSXXJz3DKTk/NyzZo1WLp0Kc6fPy8NBVLcayZOnIiEhATpeYiMx+FjZ86cQfny5VV6fXJ7z8js/lehQgWkpKTgf//7H2bPno3ChQsD+PBh9UsRQuD58+cq70tPT09s2bIF8+fPR4sWLZCUlIT58+cjLi4OgwYNyrbcnN6XciM8PBw//vgj7O3tMWjQINjZ2eHGjRvYs2dPjuqkEB0djQYNGiAtLQ2jRo2CsbExli5dqjbYy8l1W1G3+/fvo1u3brCzs8O1a9ewdOlSXLt2DWfPnlWZ/OFz748+Pj746aef4Ovriy1btmDkyJGoUKGCNFT4/fv3aNSoER49eoSBAwfCwcEBa9aswZEjR5TKmjNnDv766y9s374dixcvhomJCSpWrCitT0xMhJmZGRITE2FpaQl/f39MnTpV7ecWd3f3AjVp1BcnSCOsWrVKAFD7p6+vr5TXyclJNG/eXKUMAKJ///5KaRMmTBDGxsbi9u3bSumjRo0S2tra4tGjR0IIIY4cOSIAiIEDB6qUK5fLpX8bGxuLgIAAlTzm5uYq286J1q1bCz09PXHv3j0p7enTp8LU1FTUq1dPSouKihIAxPTp07Mts1WrVsLNzS3LPD169BD29vbi5cuXSunt27cX5ubmIjExUQghRFpamkhOTlbK8/r1a2Frayu6d++uUj8zMzMRExOjlL9evXrC1NRUPHz4UCk943ENCgoSAJTKFEKINm3aCCsrq2z2WIj69esLAGLmzJlSWnJysqhcubKwsbERKSkpebY/uS3D2tpavHnzRkofPXq0ACAqVaokUlNTpXR/f3+hp6cnkpKShBBCvH37VlhYWIhevXopbSs6OlqYm5srpffv31+ou1yePHlSABDr1q1TSg8LC1NJd3JyEgBEWFiYUt7Zs2cLAOLFixcq5Wfl8uXLAoDo2bOnUvqwYcMEAHHkyBGVbZ84cUJKi4mJEfr6+uLXX3/NcjutW7cWAMTr168zzbNt2zYBQMybN09KAyB69OghXrx4IWJiYsS5c+dEo0aNlM4juVwunVu2trbC399fLFy4UOVcVlixYoUwNDSU3j8Z9+3j45rX16aM21Tw9vYWxYsXV0qrX7++qF+/vrSsOE9XrVqldp8UNm/eLACIo0ePqqzLbB9zWq+YmBihp6cnmjdvrrRPv/32mwCgdM3N6XHLTG7Oy4CAAGFsbKxSRv369bO9xioULVpUtG3bViU9J/eMgIAA4eTkpJSm7v43ffp0AUBERUXlqE4fc3NzUzonsrNmzRoBQKxYsUIp/fnz59J7SPFXuHBhcebMmRyVm9P7kuLzwsf7e/ToUaVzNC0tTbi4uAgnJyeV64O6+09GTk5OSufd4MGDBQBx7tw5KS0mJkaYm5sr1SU31211743//e9/KtfCvLo//vXXX1JacnKysLOzUzo358yZIwCITZs2SWnv3r0TJUuWVHnvK+r08X1h1KhRYuTIkWLjxo3if//7nwgICBAARO3atZXudwqTJk0SAMTz58+z3Q8SgsPnNMzChQsRHh6u9Ld///5PLm/z5s2oW7cuLC0t8fLlS+mvcePGSE9Px4kTJwAAW7duhUwmU5m8AYDKtzXqWFhY4Ny5c3j69GmO65aeno6DBw+idevWKF68uJRub2+PDh064NSpU9l2P2dWl//++w9///232vVCCGzduhUtWrSAEELpuHh7eyMuLk4awqGtrS0NnZDL5YiNjUVaWhqqVaumdmhg27Ztlb6hfPHiBU6cOIHu3bujWLFiSnnVHddffvlFablu3bp49epVjo6Djo4O+vTpIy3r6emhT58+iImJQWRkZJ7sz6eU8fPPPys9ZFqjRg0AQKdOnZR6HmrUqIGUlBQ8efIEwIdvEd+8eQN/f3+lNtLW1kaNGjVUhuups3nzZpibm6NJkyZKZbi7u8PExESlDBcXF3h7eyulKcaM79y5M8fDk4APz9YAH55JyEjxDffHM1K5urqibt260rK1tTXKlCmD+/fvZ7mdt2/fAkCWD+oq1inyKqxYsQLW1tawsbFBjRo1cPr0aQwdOlT6Rlomk+HAgQP4448/YGlpif/973/o378/nJyc4Ofnp/JM0b59+9CgQQOVb4/VHde8vjZl3Kaix71+/fq4f/8+4uLiMj02eUXdPua0XocOHZJ6UzPuk7qegZwet8zk9rz8XK9evYKlpaVK+qfcM74FN2/eRP/+/eHh4YGAgACldUZGRihTpgwCAgKwefNmrFy5Evb29vjpp59w9+7dLMvNzX0ppy5duoSoqCgMHjxY5dmXnNzXM9q3bx9q1qyJ6tWrS2nW1tbo2LGjUr7cXLczvjeSkpLw8uVL1KxZEwDU7uvn3B9NTEyUnvPR09ND9erVla6v+/btg729Pdq1ayelGRkZSSNjcmLy5MmYMmUKfH190b59e4SGhmLixIk4ffo0tmzZopJf8d7gz3HkDIfPaZjq1avn6VTTd+7cwZUrVzIdSqB4WPDevXtwcHBAoUKFPmk706ZNQ0BAABwdHeHu7o5mzZqhS5cuSsHOx168eIHExERpmElG5cqVg1wux+PHj7McOqTOyJEjcejQIVSvXh0lS5aEl5cXOnToID0X8eLFC7x58wZLly6VptH8mOK4AMDq1asxc+ZM3Lx5E6mpqVK6ulkCP05TXHBz+hsJHwdOigvm69evYWZmluVrHRwcYGxsrJRWunRpAB+e71HcbD5nfxRyU8bH+6QIkBwdHdWmK571uXPnDgCgYcOGauuQ3fFQlBEXFwcbGxu16zO2M6C+/n5+fli+fDl69uyJUaNGoVGjRvjpp5/Qrl27LJ9JePjwIbS0tFRmJrKzs4OFhQUePnyolP7xcQI+tP/Hzz59LGPAk9nD74pg6OPj0KpVKwQGBkImk8HU1FSadjgjfX19jBkzBmPGjMGzZ89w/PhxzJ07F5s2bYKurq70oHlqairCw8PVTjGr7rjm9bXp9OnTCAoKQkREhMpzBnFxcbme+Sy3Mnuv5KReinOhVKlSSuutra1VAoqcHrcXL14gPT1dSjcxMYGJiUmuz8u8INRMCf8p94zcio2NVRp+aGho+FnnQXR0NJo3bw5zc3Ns2bIF2traSut//vlnaYikQqtWrVCqVCmMGTMGGzduRHp6uspv0xQqVAhv3rzJ1X0pJxTPBH7Ob/QoPHz4UPpCK6OP79+5uW7HxsYiJCQEGzZsUNk3dV9kfM79sWjRoiqBoKWlJa5cuSItP3z4ECVLllTJp+4zSm4MGTIEv//+Ow4dOoT27dsrrVO8N3IbpGoqBkX0WeRyOZo0aYIRI0aoXa/40Py5fH19UbduXWzfvh0HDx7E9OnTMXXqVGzbtk0ar/u1lCtXDrdu3cKePXsQFhaGrVu3YtGiRRg3bhxCQkKkb/s7deqk8k2fgmJ88Nq1a9G1a1e0bt0aw4cPh42NDbS1tTF58mTphpPR58429fFNVkHdh4pPkRf7k9syMtun7PZV0U5r1qyBnZ2dSr7snm9RlGFjY4N169apXf/xB0t1+2toaIgTJ07g6NGj2Lt3L8LCwrBx40Y0bNgQBw8ezHQ/FHJ6s/vUtnd1dcWOHTtw5coVlYfZFRQ3/o8/cBYtWhSNGzfOUf2AD7247du3R9u2beHm5oZNmzYhNDQUOjo6Us9us2bNVF6n7rjm5bXp3r17aNSoEcqWLYtZs2bB0dERenp62LdvH2bPnp2rHr5PpW4fv0S9cnrcfvjhB6UAJygoSOlh8K/1IczKykptYP817hk//fQTjh8/Li0HBAR88o/0xsXFoWnTpnjz5g1OnjwJBwcHpfX3799HWFiYSkBTqFAh1KlTR3oG8fHjxyoB9NGjR6Wp7XNyX8qs7TIGwfklN9dtX19fnDlzBsOHD0flypVhYmICuVwOHx8fte+Nz7k/ful7a1YMDQ1hZWWldiIhxXtD8SwcZY1BEX2WEiVKICEhIdsPPiVKlMCBAwcQGxub5TeyWd1I7e3t0a9fP/Tr1w8xMTGoWrUqJk6cmOkNztraGkZGRrh165bKups3b0JLS0ulNyGnjI2N4efnBz8/P6SkpOCnn37CxIkTMXr0aFhbW8PU1BTp6enZHpctW7agePHi2LZtm9K+qxvKo47iQ+jVq1c/aT9y4+nTp3j37p3SN/2KHxhUPNj6ufuTV2XkhGISDxsbm2zbKbPzskSJEjh06BBq1679WQGrlpYWGjVqhEaNGmHWrFmYNGkSxowZg6NHj2ZaNycnJ8jlcty5c0fpQfTnz5/jzZs30iQGn6tFixaYNGkS/vrrL7VBUXp6OtavXw9bW9tMg6bc0tXVRcWKFXHnzh28fPkSdnZ22Lt3L1xdXXP8Gz55eW3avXs3kpOTsWvXLqVvk3MyxDKnPiWIyGm9FOfCnTt3lALXFy9eqAQUOT1u69atU5ptUFHu1zovFcqWLSvNZvix3N4zMpNZ28ycOVPp+H0cyORUUlISWrRogdu3b+PQoUNqpwR//vw5APWBSWpqKtLS0gB86JELDw9XWl+pUiWYmZnl+L6k6CH5ePjqx718imvo1atXc/XlhzpOTk5SL1BGH9+/c3rdfv36NQ4fPoyQkBCMGzdOSle3ja/FyckJV69ehRBC6ZxS9xklN96+fYuXL1+q7d2NiopC4cKFv+jEIAUJnymiz+Lr64uIiAgcOHBAZd2bN2+kC3Xbtm0hhEBISIhKvozfpBgbG6tciNPT01W6um1sbODg4KAylWVG2tra8PLyws6dO5WmFn3+/DnWr1+POnXq5GiI1Mc+ns5TT08Prq6uEEIgNTUV2traaNu2LbZu3ao2WMk4tEHx7VLGY3Du3DlERETkqC7W1taoV68eVq5ciUePHimty+tvqNLS0qRpgIEP02//+eefsLa2hru7O4DP35+8KiMnvL29YWZmhkmTJikN0VPI2E6KQPDjc9PX1xfp6emYMGGCyuvT0tLU/s7Ox9R9u6eYYS+r81vRY/LxbGCzZs0CgDz7cdOaNWvCy8sLq1atwp49e1TWjxkzBrdv38aIESNy1LuW0Z07d1TOW+DDcY6IiIClpaV0M9+3b1+u9ikvr03qzsm4uDisWrUqx/XJTmbnWFZyWq/GjRtDV1cX8+fPV8qrbia5nB632rVro3HjxtKfIij6WuelgoeHB65evar0XvnUe0ZmMmsbd3d3pWPwKb9vlJ6eDj8/P0RERGDz5s3w8PBQm69kyZLQ0tLCxo0bldrwv//+w8mTJ1GlShUAgIGBgVKdGjduDEtLy1zdlxSBR8bnx9LT01V6qapWrQoXFxfMmTNH5djk9v7TrFkznD17FufPn1eq08e98Dm9bqt7bwDqz/ncePToEW7evPlJr23WrBmePn2q9OxPYmJipsMZP5aUlKTy3CYATJgwAUII+Pj4qKyLjIzM9JwiVewp0jD79+9X+4auVavWJ421Hj58OHbt2oUff/xRmt733bt3+Pfff7FlyxY8ePAAhQsXRoMGDdC5c2fMmzcPd+7ckbqvT548iQYNGiAwMBDAh5vMoUOHMGvWLDg4OMDFxQVlypRB0aJF0a5dO1SqVAkmJiY4dOgQ/v7772x/sO+PP/6QfgOmX79+0NHRwZ9//onk5GS1vyGQE15eXrCzs0Pt2rVha2uLGzduYMGCBWjevLn0/MWUKVNw9OhR1KhRA7169YKrqytiY2Nx8eJFHDp0SPog/OOPP2Lbtm1o06YNmjdvjqioKCxZsgSurq5ISEjIUX3mzZuHOnXqoGrVqujduzdcXFzw4MED7N27F5cvX/6kfVTHwcEBU6dOxYMHD1C6dGls3LgRly9fxtKlS6XpcPNif/KijJwwMzPD4sWL0blzZ1StWhXt27eHtbU1Hj16hL1796J27dpYsGABAEhB38CBA+Ht7Q1tbW20b98e9evXR58+fTB58mRcvnwZXl5e0NXVxZ07d7B582bMnTtX6aFadcaPH48TJ06gefPmcHJyQkxMDBYtWoSiRYuiTp06mb6uUqVKCAgIwNKlS/HmzRvUr18f58+fx+rVq9G6dWs0aNAgz47VX3/9hYYNG6JVq1bo0KED6tati+TkZGzbtg3Hjh1Dp06dMGTIkFyX+88//6BDhw5o2rQp6tati0KFCuHJkydYvXo1nj59ijlz5kBbWxtRUVG4ceNGrn4XKC+vTV5eXtDT00OLFi3Qp08fJCQkYNmyZbCxscGzZ89yvd/qVK5cGdra2pg6dSri4uKgr68v/f5QZnJaL2trawwbNgyTJ0/Gjz/+iGbNmuHSpUvYv3+/yrCanB63zHzN8xL48EzNhAkTcPz4cXh5eQH48M35p94z1FG8/8eMGYP27dtDV1cXLVq0UHk+LqMTJ05IQcWLFy/w7t07/PHHHwA+TMGu6FX99ddfsWvXLrRo0QKxsbFKP9YKQHp439raGt27d8fy5cul5w7fvn2LRYsW4f379xg9enS2+5HT+5Kbmxtq1qyJ0aNHSz2oGzZskAJiBS0tLSxevBgtWrRA5cqV0a1bN9jb2+PmzZu4du2a2sA6MyNGjMCaNWvg4+ODQYMGSVNyOzk5KT2Xk9PrtpmZGerVq4dp06YhNTUVRYoUwcGDBzPtVcypLl264Pjx45/0pWOvXr2wYMECdOnSBZGRkbC3t8eaNWtgZGSUo9dHR0ejSpUq8Pf3l4ZDHjhwAPv27YOPjw9atWqllD8mJgZXrlz57J8M0ChfZY47yndZTcmNj6aLzc2U3EJ8mCJz9OjRomTJkkJPT08ULlxY1KpVS8yYMUOaqlmID9N3Tp8+XZQtW1bo6ekJa2tr0bRpUxEZGSnluXnzpqhXr54wNDSUpopNTk4Ww4cPF5UqVRKmpqbC2NhYVKpUSSxatChH+37x4kXh7e0tTExMhJGRkWjQoIHKFKa5mZL7zz//FPXq1RNWVlZCX19flChRQgwfPlzExcUp5Xv+/Lno37+/cHR0FLq6usLOzk40atRILF26VMojl8vFpEmThJOTk9DX1xdVqlQRe/bsUZkqNrv6Xb16VbRp00ZYWFgIAwMDUaZMGfH7779L6zOb3jOzqVc/ppgi98KFC8LDw0MYGBgIJycnsWDBAqV8ebE/n1uGYtrYzZs3q93Xv//+WyW/t7e3MDc3FwYGBqJEiRKia9eu4sKFC1KetLQ0MWDAAGFtbS1kMpnK9LJLly4V7u7uwtDQUJiamooKFSqIESNGiKdPn0p5MntfHT58WLRq1Uo4ODgIPT094eDgIPz9/VWmRFYnNTVVhISECBcXF6GrqyscHR3F6NGjpWnHs9v2x9NHZ+Xt27ciJCREuLm5CQMDA+nakfE8yyiz60VGz58/F1OmTBH169cX9vb2QkdHR1haWoqGDRuKLVu2SPkWLFggzM3N1U45m9m+KeqcV9emXbt2iYoVKwoDAwPh7Owspk6dKlauXKny/vnUKbmFEGLZsmWiePHiQltbW2mK3qz2Maf1Sk9PFyEhIcLe3l4YGhoKT09PcfXqVZWpkXNz3DKT0/MyL6bkFkKIihUrih49ekjLOb1n5HRKbiE+TFVepEgRoaWllaNrpuKaq+4vKChIaV+zujdnlJqaKubPny8qV64sTExMhImJiWjQoIHSNOfZycl9SQgh7t27Jxo3biz09fWFra2t+O2330R4eLjaaeNPnTolmjRpIh3rihUrivnz56sci4zUnXdXrlwR9evXFwYGBqJIkSJiwoQJYsWKFZlOD57ddfu///6T7ovm5ubi559/Fk+fPlVpg9zcHxXtlVFm56u68+vhw4eiZcuWwsjISBQuXFgMGjRI+gmH7Kbkfv36tejUqZMoWbKkMDIyEvr6+sLNzU1MmjRJ7fty8eLFwsjISMTHx6usI/VkQnyFp8CI6Lvm6emJly9ffpVnl+j78OTJE9SqVQtpaWmIiIhQO7tdXlH8oPKmTZu+2Dbo+7VmzRr0798fjx49ynSGRCJNU6VKFXh6emL27Nn5XZXvBp8pIiKiXCtSpAjCwsKQlJSEpk2bZju19+fw9PT8pOF5pBk6duyIYsWKYeHChfldFaJvQlhYGO7cuZOjYZX0f9hTRETZYk8RERERFWTsKSIiIiIiIo3GniIiIiIiItJo7CkiIiIiIiKNxqCIiIiIiIg0GoMiIsoXwcHBkMlkePnyZX5XhT5B165dYWJikt/VoG/AgwcPIJPJEBoaKqUp3t9ERN8LBkVEpPEmTpyIli1bwtbWFjKZDMHBwZnmffLkCXx9fWFhYQEzMzO0atUK9+/f/3qVpS9q27Zt8PPzQ/HixWFkZIQyZcrg119/xZs3b1TyOjs7QyaTqfz98ssvass+dOgQGjZsCHNzc5iamsLd3R0bN278wntEuXXv3j106NABNjY2MDQ0RKlSpTBmzJhM86empsLV1RUymQwzZsz4ijUlorykk98VICLKb2PHjoWdnR2qVKmCAwcOZJovISEBDRo0QFxcHH777Tfo6upi9uzZqF+/Pi5fvgwrK6uvWGv6Enr37g0HBwd06tQJxYoVw7///osFCxZg3759uHjxIgwNDZXyV65cGb/++qtSWunSpVXKXbVqFXr06IEmTZpg0qRJ0NbWxq1bt/D48eMvuj/5ZezYsRg1alR+VyPXLl++DE9PTxQpUgS//vorrKys8OjRoyzbaf78+Xj06NFXrCURfQkMiohI40VFRcHZ2RkvX76EtbV1pvkWLVqEO3fu4Pz58/jhhx8AAE2bNkX58uUxc+ZMTJo06WtVOc8lJSVBT08PWlqaPYBgy5Yt8PT0VEpzd3dHQEAA1q1bh549eyqtK1KkCDp16pRlmQ8ePED//v0xYMAAzJ07N6+r/E3S0dGBjs739RFDLpejc+fOKFu2LI4ePaoSAKsTExOD8ePHY+TIkRg3btxXqCURfSmaffcjom/Kw4cPUbJkSZQvXx7Pnz//att1dnbOUb4tW7bghx9+kAIiAChbtiwaNWqETZs2fdK2Fc/mPHnyBK1bt4aJiQmsra0xbNgwpKen56osT09PlC9fHpGRkahVqxYMDQ3h4uKCJUuWKOU7duwYZDIZNmzYgLFjx6JIkSIwMjJCfHw8AGDz5s1wd3eHoaEhChcujE6dOuHJkydqt3n//n14e3vD2NgYDg4OGD9+PD7+pQe5XI45c+bAzc0NBgYGsLW1RZ8+ffD69WulfBcuXIC3tzcKFy4s1b179+65Ogaf6+OACADatGkDALhx44ba16SkpODdu3eZlrlkyRKkp6dj/PjxAD70OH7ur2Eontm5efMmfH19YWZmBisrKwwaNAhJSUlKeZOTkzFkyBBYW1vD1NQULVu2xH///ZftUFF13rx5g65du8Lc3BwWFhYICAhQO7RQ3TNFMpkMgYGB2Lx5M1xdXWFoaAgPDw/8+++/AIA///wTJUuWhIGBATw9PfHgwYNc1e1zHTx4EFevXkVQUBAMDQ2RmJiY7Xtw1KhRKFOmTLaBMRF9+76vr3GIqMC6d+8eGjZsiEKFCiE8PByFCxfONG9qairi4uJyVG6hQoXypPdDLpfjypUraj+kV69eHQcPHsTbt29hamqa67LT09Ph7e2NGjVqYMaMGTh06BBmzpyJEiVKoG/fvrkq6/Xr12jWrBl8fX3h7++PTZs2oW/fvtDT01Op+4QJE6Cnp4dhw4YhOTkZenp6CA0NRbdu3fDDDz9g8uTJeP78OebOnYvTp0/j0qVLsLCwUKq3j48PatasiWnTpiEsLAxBQUFIS0uTAgAA6NOnj1TuwIEDERUVhQULFuDSpUs4ffo0dHV1ERMTAy8vL1hbW2PUqFGwsLDAgwcPsG3btmz3OSEhQSUQUEdXVxfm5uY5P5j/X3R0NACoPSePHDkCIyMjpKenw8nJCUOGDMGgQYOU8hw6dAhly5bFvn37MHz4cDx58gSWlpbo378/QkJCPuv89PX1hbOzMyZPnoyzZ89i3rx5eP36Nf766y8pT8+ePbF27Vp06NABtWrVwpEjR9C8efNcb0sIgVatWuHUqVP45ZdfUK5cOWzfvh0BAQE5LuPkyZPYtWsX+vfvDwCYPHkyfvzxR4wYMQKLFi1Cv3798Pr1a0ybNg3du3fHkSNHsiwvL68Fhw4dAgDo6+ujWrVqiIyMhJ6eHtq0aYNFixahUKFCSvnPnz+P1atX49SpU5xUgqggEERE+SAoKEgAEC9evBA3btwQDg4O4ocffhCxsbHZvvbo0aMCQI7+oqKiclynFy9eCAAiKCgo03Xjx49XWbdw4UIBQNy8eTPH21IICAhQW26VKlWEu7t7rsqqX7++ACBmzpwppSUnJ4vKlSsLGxsbkZKSIoT4v+NXvHhxkZiYKOVNSUkRNjY2onz58uL9+/dS+p49ewQAMW7cOJV6DxgwQEqTy+WiefPmQk9PT7x48UIIIcTJkycFALFu3TqluoaFhSmlb9++XQAQf//9d672OWNdsvurX79+rssWQogePXoIbW1tcfv2baX0Fi1aiKlTp4odO3aIFStWiLp16woAYsSIEUr5zMzMhKWlpdDX1xe///672LJli+jQoYMAIEaNGvVJdVK8f1q2bKmU3q9fPwFA/PPPP0IIIS5fviwAiH79+inlU2xf3bmemR07dggAYtq0aVJaWlqatN+rVq1SqV9GAIS+vr7Se/LPP/8UAISdnZ2Ij4+X0kePHp2j929eXgtatmwpAAgrKyvRsWNHsWXLFvH7778LHR0dUatWLSGXy6W8crlcVK9eXfj7+wshhIiKihIAxPTp07PcBhF9u9hTRET56urVq/Dz80PJkiWxf/9+mJmZZfuaSpUqITw8PEfl29nZfW4VAQDv378H8OFb5I8ZGBgo5fkUH89YVrduXaxZsybX5ejo6KBPnz7Ssp6eHvr06YO+ffsiMjISNWvWlNYFBAQoPTdx4cIFxMTEIDg4WNonAGjevDnKli2LvXv3IiQkRGl7gYGB0r8Vw6P27t2LQ4cOoX379ti8eTPMzc3RpEkTpenX3d3dYWJigqNHj6JDhw5SD9SePXtQqVIl6Orq5nifR4wYkaPhS5aWljkuU2H9+vVYsWIFRowYgVKlSimt27Vrl9Jyt27d0LRpU8yaNQsDBgxA0aJFAXzoyZLL5ZgyZQpGjhwJAGjbti1iY2Mxd+5c/Pbbb5/UwwhA6nFRGDBgABYtWoR9+/ahYsWK2LdvHwBg4MCBSvkGDx6M9evX52pb+/btg46OjlLvpba2NgYMGICTJ0/mqIxGjRopDVetUaMGgA/HI+MxUKTfv38/y+GteXktSEhIAAD88MMPWLt2rVQvIyMjjB49GocPH0bjxo0BAKGhofj333+xZcuWHG2biL59DIqIKF+1aNECtra2OHDgQI5/98bS0lL6cPK1KIKH5ORklXWKoVs5eTBbHQMDA5UJHiwtLVWeuckJBwcHGBsbK6UpZkN78OCBUlDk4uKilO/hw4cAgDJlyqiUW7ZsWZw6dUopTUtLC8WLF890WwBw584dxMXFwcbGRm19Y2JiAAD169dH27ZtERISgtmzZ8PT0xOtW7dGhw4d1AaiGbm6usLV1TXLPJ/i5MmT6NGjB7y9vTFx4sRs88tkMgwZMgQHDhzAsWPHpEDN0NAQ7969g7+/v1J+f39/hIWF4dKlS6hXr94n1fHjQK1EiRLQ0tKSjv/Dhw+hpaWFEiVKKOVT18bZefjwIezt7VXep7kpq1ixYkrLiuGMjo6OatOzew/k5bVA8f79uJ06dOiA0aNH48yZM2jcuDHi4+MxevRoDB8+XKXeRPT9YlBERPmqbdu2WL16NdatW6fUw5GVlJQUxMbG5iivtbU1tLW1P6eKAD48j6Cvr49nz56prFOkOTg4fFLZeVG/T/GpQVxuyOVy2NjYYN26dWrXK4JBmUyGLVu24OzZs9i9ezcOHDiA7t27Y+bMmTh79myWAXNcXFyOeun09PRUngvJzD///IOWLVuifPny2LJlS45nUlN8SM54fjo4OODOnTuwtbVVyqsIFD8l+M3Mt/5sS2bnembpIpsJKfLyWqB4/2bXTjNmzEBKSgr8/Pyk4PO///6T8jx48AAODg7Q09PLUb2I6NvAoIiI8tX06dOho6ODfv36wdTUFB06dMj2NWfOnEGDBg1yVL5iuu3PpaWlhQoVKuDChQsq686dO4fixYt/8hCovPT06VO8e/dOqbfo9u3bALKfZc/JyQkAcOvWLTRs2FBp3a1bt6T1CnK5HPfv31f6XZ6Pt1WiRAkcOnQItWvXzlEQVrNmTdSsWRMTJ07E+vXr0bFjR2zYsEFlKuyMBg0ahNWrV2dbdv369XHs2LFs8927dw8+Pj6wsbHBvn37ctyDCUD6Id+MPX/u7u64c+cOnjx5otSz9vTpU5W8uXXnzh2lHr+7d+9CLpdLx9/JyQlyuRz37t1T6tG5detWrrfl5OSEw4cPIyEhQemYfEpZeSUvrwXu7u5YtmyZykyLH7fTo0eP8Pr1a7i5uamUMWnSJEyaNAmXLl1C5cqVc7YTRPRNYFBERPlKJpNh6dKlePv2LQICAmBiYoKWLVtm+Zr8eKYIANq1a4dRo0bhwoULqFatGoAPHwiPHDmCYcOG5dl2PkdaWhr+/PNPDB06FMCHb9L//PNPWFtbw93dPcvXVqtWDTY2NliyZAm6d+8uDVvbv38/bty4ofZ3WBYsWIB58+YB+PCt/oIFC6Crq4tGjRoB+DA72qJFizBhwgSV33FKS0tDQkICLCws8Pr1a1hYWCj1dCg+VKobsphRXj5TFB0dDS8vL2hpaeHAgQOZBiyxsbEwNzdX6nlITU3FlClToKenp/RB3c/PDxs2bMCKFSukYXhyuRyrVq1CoUKFsm2XrCxcuBBeXl7S8vz58wF8+P0sxf9/++03zJs3DwsXLpTyzZkzJ9fbatasGZYuXYrFixdj+PDhAD7MQKjYZn7Iy2tBq1atMGjQIKxatQpdu3aVZqpbvnw5AKBJkyYAPjyf1bp1a6XXxsTEoE+fPujatStatWqlMjSViL59DIqIKN9paWlh7dq1aN26NXx9fbFv3z6VnoqM8vqZojVr1uDhw4dITEwEAJw4cQJ//PEHAKBz585SD0m/fv2wbNkyNG/eHMOGDYOuri5mzZoFW1tb/Prrr0plenp64vjx45/9ezS55eDggKlTp+LBgwcoXbo0Nm7ciMuXL2Pp0qXZTl6gq6uLqVOnolu3bqhfvz78/f2lKbmdnZ0xZMgQpfwGBgYICwtDQEAAatSogf3792Pv3r347bffpGCifv366NOnDyZPnozLly/Dy8sLurq6uHPnDjZv3oy5c+eiXbt2WL16NRYtWoQ2bdqgRIkSePv2LZYtWwYzMzM0a9Ysy3rn5TNFPj4+uH//PkaMGIFTp04pPUdla2srfTDetWsX/vjjD7Rr1w4uLi6IjY3F+vXrcfXqVUyaNEnpA3irVq3QqFEjTJ48GS9fvkSlSpWwY8cOnDp1Cn/++afSM1Ndu3bF6tWrc9zDGRUVhZYtW8LHxwcRERHS1NuVKlUC8CGw9Pf3x6JFixAXF4datWrh8OHDuHv3bq6PTYsWLVC7dm2MGjUKDx48gKurK7Zt25bjKbG/hLy8FtjZ2WHMmDEYN24cfHx80Lp1a/zzzz9YtmwZ/P39pd8nq1q1KqpWrar0WsUwOjc3N5WAiYi+E/k8+x0RaaiMU3IrJCYmivr16wsTExNx9uzZr1YXxVTW6v6OHj2qlPfx48eiXbt2wszMTJiYmIgff/xR3LlzR6VMd3d3YWdnl+22AwIChLGxsUq6uimNc7Ifbm5u4sKFC8LDw0MYGBgIJycnsWDBAqV8immMN2/erLacjRs3iipVqgh9fX1RqFAh0bFjR/Hff/+prfe9e/eEl5eXMDIyEra2tiIoKEikp6erlLl06VLh7u4uDA0NhampqahQoYIYMWKEePr0qRBCiIsXLwp/f39RrFgxoa+vL2xsbMSPP/4oLly4kKtj8LkyOw/w0ZTeFy5cEC1atBBFihQRenp6wsTERNSpU0ds2rRJbblv374VgwYNEnZ2dkJPT09UqFBBrF27ViVf27ZthaGhoXj9+nWW9VScH9evXxft2rUTpqamwtLSUgQGBipNpy6EEO/fvxcDBw4UVlZWwtjYWLRo0UI8fvw411NyCyHEq1evROfOnYWZmZkwNzcXnTt3FpcuXcrxlNz9+/dXSstsKuvsztEvRS6Xi/nz54vSpUsLXV1d4ejoKMaOHStNZ58ZTslN9P2TCfGVv8YkIirg3r59i0KFCmHOnDkqUyZ/SZ6ennj58iWuXr361bZJecvW1hZdunTB9OnTs8wXHByMkJAQvHjxIssfOs6KTCZDUFAQgoODP+n1REQFyef/zDsRESk5ceIEihQpgl69euV3Veg7cu3aNbx//176LSMiIvp6+EwREVEea968OZo3b55n5cXGxiIlJSXT9dra2p81gxl9G9zc3BAfH//Vt5ueno4XL15kmcfExCRXs/AREX1vGBQREX3jfvrpJxw/fjzT9U5OTtKD3kS59fjx42xnS+MwOyIq6PhMERHRNy4yMjLLH/g0NDRE7dq1v2KNqCBJSkpSmmVPneLFiyv9xhIRUUHDoIiIiIiIiDQaJ1ogIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcbfKcojcrkcT58+hampKWQyWX5Xh4iIiIhIowkh8PbtWzg4OEBLK+u+IAZFeeTp06dwdHTM72oQEREREVEGjx8/RtGiRbPMw6Aoj5iamgL4cNDNzMzyuTYfpKam4uDBg/Dy8oKurm5+V4c+Edvx+8c2LBjYjgUD27FgYDsWDF+6HePj4+Ho6Ch9Ts8Kg6I8ohgyZ2Zm9k0FRUZGRjAzM+MF4zvGdvz+sQ0LBrZjwcB2LBjYjgXD12rHnDzawokWiIiIiIhIozEoos+WlJSErl27okKFCtDR0UHr1q3V5ktOTsaYMWPg5OQEfX19ODs7Y+XKlSr5QkJC0KlTJwDA0qVL4enpCTMzM8hkMrx580Zt2Xv37kWNGjVgaGgIS0vLTOtARERERPQxDp+jz5aeng5DQ0MMHDgQW7duzTSfr68vnj9/jhUrVqBkyZJ49uwZ5HK5Sr6dO3di1KhRAIDExET4+PjAx8cHo0ePVlvu1q1b0atXL0yaNAkNGzZEWloarl69mjc7R0REREQFHoMi+mzGxsZYvHgxAOD06dNqe3PCwsJw/Phx3L9/H4UKFQIAODs7q+R7/Pgxrl27Bh8fHwDA4MGDAQDHjh1Tu+20tDQMGjQI06dPR48ePaR0V1fXT98hIiIiItIoHD5HX8WuXbtQrVo1TJs2DUWKFEHp0qUxbNgwvH//XiWfYrhcTly8eBFPnjyBlpYWqlSpAnt7ezRt2pQ9RURERESUY+wpoq/i/v37OHXqFAwMDLB9+3a8fPkS/fr1w6tXr7Bq1Sop386dO9GqVatclQsAwcHBmDVrFpydnTFz5kx4enri9u3bUq8UEREREVFm2FNEX4VcLodMJsO6detQvXp1NGvWDLNmzcLq1aul3qL4+HgcP34cLVu2zFW5ADBmzBi0bdsW7u7uWLVqFWQyGTZv3vxF9oWIiIiIChYGRfRV2Nvbo0iRIjA3N5fSypUrByEE/vvvPwDA/v374erqCkdHx1yVCyg/Q6Svr4/ixYvj0aNHeVR7IiIiIirIGBTRV1G7dm08ffoUCQkJUtrt27ehpaWFokWLAsj90DkAcHd3h76+Pm7duiWlpaam4sGDB3BycsqbyhMRERFRgcagiLKVLheIuPcKOy8/QcS9V0iXC5U8169fx+XLlxEbG4u4uDhcvnwZly9fltZ36NABVlZW6NatG65fv44TJ05g+PDh6N69OwwNDZGWlob9+/erDJ2Ljo7G5cuXcffuXQDAv//+K20HAMzMzPDLL78gKCgIBw8exK1bt9C3b18AwM8///yFjggRERERFSScaIGyFHb1GUJ2X8ezuCQpzd7cAEEtXOFT3l5Ka9asGR4+fCgtV6lSBQAgxIcAysTEBOHh4RgwYACqVasGKysr+Pr64o8//gAAHD9+HCYmJqhatarS9pcsWYKQkBBpuV69egCAVatWoWvXrgCA6dOnQ0dHB507d8b79+9Ro0YNHDlyBJaWlnl4JIiIiIiooGJQRJkKu/oMfddexMf9QtFxSei79iIWd6oqBUYPHjzItryyZcsiPDxc7bqdO3eiRYsWKunBwcEIDg7OslxdXV3MmDEDM2bMyLYOREREREQfY1BEaqXLBUJ2X1cJiABAAJABCNl9HU1c7aCtJfvs7ZUvXx4eHh6fXQ4RERERUW4xKCK1zkfFKg2Z+5gA8CwuCeejYuFRwuqzt9e7d+/PLoOIiIiI6FNwogVSK+Zt5gHRp+QjIiIiIvpWMSgitWxMDfI0HxERERHRt4pBEalV3aUQ7M0NkNnTQjJ8mIWuukuhr1ktIiIiIqI8x6CI1NLWkiGohSsAqARGiuWgFq55MskCEREREVF+YlBEmfIpb4/FnarCzlx5iJyduYHSdNxERERERN8zzj5HWfIpb48mrnY4HxWLmLdJsDH9MGSOPUREREREVFAwKKJsaWvJ8mTabSIiIiKibxGHzxERERERkUZjUERERERERBqNQREREREREWk0BkVERERERKTRGBQREREREZFGY1BERERERPQNu3XrFho0aABbW1sYGBigePHiGDt2LFJTU6U8y5YtQ926dWFpaQlLS0s0btwY58+fV1tegwYNsHz5cgDAwIED4e7uDn19fVSuXFltfiEEZsyYgdKlS0NfXx9FihTBxIkT83w/8xOn5CYiIiIi+obp6uqiS5cuqFq1KiwsLPDPP/+gV69ekMvlmDRpEgDg2LFj8Pf3R61atWBgYICpU6fCy8sL165dQ5EiRaSyYmNjcfr0aWzYsEFK6969O86dO4crV66o3f6gQYNw8OBBzJgxAxUqVEBsbCxiY2O/7E5/ZQyKiIiIiIi+YcWLF0fx4sWlZScnJxw7dgwnT56U0tatW6f0muXLl2Pr1q04fPgwunTpIqXv3bsXVatWha2tLQBg3rx5AIAXL16oDYpu3LiBxYsX4+rVqyhTpgwAwMXFJe927hvB4XNERERERN+Ru3fvIiwsDPXr1880T2JiIlJTU1GoUCGl9F27dqFVq1Y53tbu3btRvHhx7NmzBy4uLnB2dkbPnj0LXE8RgyIiIiIiou+AYmhcqVKlULduXYwfPz7TvCNHjoSDgwMaN24spSUnJyMsLAwtW7bM8Tbv37+Phw8fYvPmzfjrr78QGhqKyMhItGvX7rP25VvDoIiIiIiI6DuwceNGXLx4EevXr8fevXsxY8YMtfmmTJmCDRs2YPv27TAwMJDSjxw5AhsbG7i5ueV4m3K5HMnJyfjrr79Qt25deHp6YsWKFTh69Chu3br12fv0reAzRURERERE3wFHR0cAgKurK9LT09G7d2/8+uuv0NbWlvLMmDEDU6ZMwaFDh1CxYkWl1+/atStXvUQAYG9vDx0dHZQuXVpKK1euHADg0aNH0nNG3zv2FBERERERfWfkcjlSU1Mhl8ultGnTpmHChAkICwtDtWrVlPILIbB79+5cPU8EALVr10ZaWhru3bsnpd2+fRvAhwkfCgr2FBERERERfcPWrVsHXV1dVKhQAfr6+rhw4QJGjx4NPz8/6OrqAgCmTp2KcePGYf369XB2dkZ0dDQAwMTEBCYmJoiMjERiYiLq1KmjVPbdu3eRkJCA6OhovH//HpcvXwbwoTdKT08PjRs3RtWqVdG9e3fMmTMHcrkc/fv3R5MmTZR6jxTS5QLno2IR8zYJNqYGqO5SCNpasi97gPIAgyIiIiIiom+Yjo4Opk6ditu3b0MIAScnJwQGBmLIkCFSnsWLFyMlJUVlAoSgoCAEBwdj586daNasGXR0lD/+9+zZE8ePH5eWq1SpAgCIioqCs7MztLS0sHv3bgwYMAD16tWDsbExmjZtipkzZ6rUM+zqM4Tsvo5ncUlSmr25AYJauMKnvH2eHIsvhUEREREREdE3zM/PD35+flnmefDgQZbrd+7cibFjx6qkHzt2LNvtOzg4YOvWrVnmCbv6DH3XXoT4KD06Lgl9117E4k5Vv+nAiM8UEREREREVYCkpKWjbti2aNm36RcpPlwuE7L6uEhABkNJCdl9Hulxdjm9DvgZFwcHBkMlkSn9ly5ZVyhMREYGGDRvC2NgYZmZmqFevHt6/fy+tj42NRceOHWFmZgYLCwv06NEDCQkJSmVcuXIFdevWhYGBARwdHTFt2jSVumzevBlly5aFgYEBKlSogH379n2ZnSYiIiIi+or09PQQFBQEU1PTL1L++ahYpSFzHxMAnsUl4XzUt/uDr/neU+Tm5oZnz55Jf6dOnZLWRUREwMfHB15eXjh//jz+/vtvBAYGQkvr/6rdsWNHXLt2DeHh4dizZw9OnDiB3r17S+vj4+Ph5eUFJycnREZGYvr06QgODsbSpUulPGfOnIG/vz969OiBS5cuoXXr1mjdujWuXr36dQ4CEREREdF3KuZt5gHRp+TLD/n+TJGOjg7s7OzUrhsyZAgGDhyIUaNGSWkZ50K/ceMGwsLC8Pfff0vTDs6fPx/NmjXDjBkz4ODggHXr1iElJQUrV66Enp4e3NzccPnyZcyaNUsKnubOnQsfHx8MHz4cADBhwgSEh4djwYIFWLJkidq6JScnIzk5WVqOj48HAKSmpiI1NfUzjkjeUdTjW6kPfRq24/ePbVgwsB0LBrZjwcB2/LZYGeUspLAy0lFqsy/djrkpN9+Dojt37sDBwQEGBgbw8PDA5MmTUaxYMcTExODcuXPo2LEjatWqhXv37qFs2bKYOHGiNJVgREQELCwslOZhb9y4MbS0tHDu3Dm0adMGERERqFevHvT09KQ83t7emDp1Kl6/fg1LS0tERERg6NChSvXy9vbGjh07Mq335MmTERISopJ+8OBBGBkZfeZRyVvh4eH5XQXKA2zH7x/bsGBgOxYMbMeCge34bZALwEJPG29SAEDd9NsCFnrAi+tnse+G6tov1Y6JiYk5zpuvQVGNGjUQGhqKMmXK4NmzZwgJCUHdunVx9epV3L9/H8CH545mzJiBypUr46+//kKjRo1w9epVlCpVCtHR0bCxsVEqU0dHB4UKFZLmZo+OjoaLi4tSHltbW2mdpaUloqOjpbSMeRRlqDN69GilQCo+Ph6Ojo7w8vKCmZnZpx+UPJSamorw8HA0adJEmsOevj9sx+8f27BgYDsWDGzHgoHt+O3RdX6OARv+AQClCRdk//+/f/xUCd5uyp+3v3Q7KkZy5US+BkUZZ8CoWLEiatSoAScnJ2zatAnlypUDAPTp0wfdunUD8GHe9MOHD2PlypWYPHlyvtRZQV9fH/r6+irpurq639yb81usE+Ue2/H7xzYsGNiOBQPbsWBgO347fqxcFDo62iq/U2SXg98p+lLtmJsy8334XEYWFhYoXbo07t69i4YNGwL48Gu6GZUrVw6PHj0CANjZ2SEmJkZpfVpaGmJjY6XnlOzs7PD8+XOlPIrl7PJk9qwTEREREREp8ylvjyaudjgfFYuYt0mwMTVAdZdC0NZSN6Tu25Lvs89llJCQgHv37sHe3h7Ozs5wcHDArVu3lPLcvn0bTk5OAAAPDw+8efMGkZGR0vojR45ALpejRo0aUp4TJ04oPWgVHh6OMmXKwNLSUspz+PBhpe2Eh4fDw8Pji+wnEREREVFBpK0lg0cJK7SqXAQeJay+i4AIyOegaNiwYTh+/DgePHiAM2fOoE2bNtDW1oa/vz9kMhmGDx+OefPmYcuWLbh79y5+//133Lx5Ez169ADwodfIx8cHvXr1wvnz53H69GkEBgaiffv2cHBwAAB06NABenp66NGjB65du4aNGzdi7ty5Ss8DDRo0CGFhYZg5cyZu3ryJ4OBgXLhwAYGBgflyXIiIiIiI6OvJ1+Fz//33H/z9/fHq1StYW1ujTp06OHv2LKytrQEAgwcPRlJSEoYMGYLY2FhUqlQJ4eHhKFGihFTGunXrEBgYiEaNGkFLSwtt27bFvHnzpPXm5uY4ePAg+vfvD3d3dxQuXBjjxo1T+i2jWrVqYf369Rg7dix+++03lCpVCjt27ED58uW/3sEgIiIiIqJ8ka9B0YYNG7LNM2rUKKXfKfpYoUKFsH79+izLqFixIk6ePJllnp9//hk///xztvUhIiIiIqKC5Zt6poiIiIiIiOhrY1BEREREREQajUERERERERFpNAZFGiQpKQldu3ZFhQoVoKOjg9atW6vNl5ycjDFjxsDJyQn6+vpwdnbGypUrVfKFhISgU6dOAIDo6Gh07twZdnZ2MDY2RtWqVbF169YvuTtERERERHnim/rxVvqy0tPTYWhoiIEDB2YZsPj6+uL58+dYsWIFSpYsiWfPnkEul6vk27lzpzQJRpcuXfDmzRvs2rULhQsXxvr16+Hr64sLFy6gSpUqX2yfiIiIiIg+F4MiDWJsbIzFixcDAE6fPo03b96o5AkLC8Px48dx//59FCpUCADg7Oysku/x48e4du0afHx8AABnzpzB4sWLUb16dQDA2LFjMXv2bERGRjIoIiIiIqJvGofPkZJdu3ahWrVqmDZtGooUKYLSpUtj2LBheP/+vUo+T09PmJmZAfjwW08bN25EbGws5HI5NmzYgKSkJHh6eubDXhARERER5Rx7ikjJ/fv3cerUKRgYGGD79u14+fIl+vXrh1evXmHVqlVSvp07d6JVq1bS8qZNm+Dn5wcrKyvo6OjAyMgI27dvR8mSJfNjN4iIiIiIcoxBESmRy+WQyWRYt24dzM3NAQCzZs1Cu3btsGjRIhgaGiI+Ph7Hjx/HihUrpNf9/vvvePPmDQ4dOoTChQtjx44d8PX1xcmTJ1GhQoX82h0iIiIiomwxKCIl9vb2KFKkiBQQAUC5cuUghMB///2HUqVKYf/+/XB1dYWjoyMA4N69e1iwYAGuXr0KNzc3AEClSpVw8uRJLFy4EEuWLMmXfSEiIiIiygk+U0RKateujadPnyIhIUFKu337NrS0tFC0aFEAqkPnEhMTAQBaWsqnk7a2ttpZ64iIiIiIviUMigqQdLlAxL1X2Hn5CSLuvUK6XKjkuX79Oi5fvozY2FjExcXh8uXLuHz5srS+Q4cOsLKyQrdu3XD9+nWcOHECw4cPR/fu3WFoaIi0tDTs378fLVu2lF5TtmxZlCxZEn369MH58+dx7949zJw5E+Hh4Zn+FhIRERER0beCw+cKiLCrzxCy+zqexSVJaXZm+mhmJ0OzDPmaNWuGhw8fSsuK6bKF+BBAmZiYIDw8HAMGDEC1atVgZWUFX19f/PHHHwCA48ePw8TEBFWrVpXK0NXVxb59+zBq1Ci0aNECCQkJKFmyJFavXo1mzTJunYiIiIjo28OgqAAIu/oMfddexMf9Qs/jk7EyXgtVrz3Hj5U/DH178OBBtuWVLVsW4eHhatft3LkTLVq0UEkvVapUlj8IS0RERET0reLwue9culwgZPd1lYAIgJQ2cf9NtUPpPkX58uXRt2/fPCmLiIiIiL4tt27dQoMGDWBrawsDAwMUL14cY8eORWpqqpRn2bJlqFu3LiwtLWFpaYnGjRvj/Pnzastr0KABli9fDgD4+++/0ahRI1hYWMDS0hLNmzdHVFTUV9mv7DAo+s6dj4pVGjKnSoZncck4HxWbJ9vr3bs3p9gmIiIiKqB0dXXRpUsXHDx4ELdu3cKcOXOwbNkyBAUFSXmOHTsGf39/HD16FBEREXB0dISXlxeePHmiVFZsbCxOnz4tPV7h4+ODYsWK4dy5czh16hRMTEwQEhKiFHDlFw6f+87FvM0qIMp9PiIiIiLSXMWLF0fx4sWlZScnJxw7dgwnT56U0tatW6f0muXLl2Pr1q04fPgwunTpIqXv3bsXVatWha2tLS5cuIDY2FiMHz9e+lmXsWPHYvv27Xj48CHKlSv3hfcsa+wp+s7ZmBrkaT4iIiIiIoW7d+8iLCwM9evXzzRPYmIiUlNTUahQIaX0Xbt2ST/jUqZMGVhZWWHFihVISUnB+/fvERoaiqJFi8LZ2flL7kKOMCj6zlV3KQR7cwPIMs0hYG+uj+ouhTLNQURERESUUa1atWBgYIBSpUqhbt26GD9+fKZ5R44cCQcHBzRu3FhKS05ORlhYmPQzLqampjh27BjWrl0LQ0NDmJiY4MCBAxg3bhx0dPJ/8BqDou+ctpYMQS1cAUAlMFIsj2laFtpamYdNREREREQZbdy4ERcvXsT69euxd+9ezJgxQ22+KVOmYMOGDdi+fTsMDP5vZNKRI0dgY2MDNzc3AMD79+/Ro0cP1K5dG2fPnsXp06fh5uaGP/74A+/fv/8q+5SV/A/L6LP5lLfH4k5VVX+nyFwfTW0T4e1mm4+1IyIiIqLvjeK5H1dXV6Snp6N379749ddfoa2tLeWZMWMGpkyZgkOHDqFixYpKr9+1a5fUSwQA69evx4MHDxAREQEtrQ/9MmvWrIGVlRV27dqFTp06fYW9yhyDogLCp7w9mrja4XxULGLeJsHG1ABVipriQNj+/K4aEREREX3H5HI5UlNTIZfLpaBo2rRpmDhxIg4cOIBq1aop5RdCYPfu3Vi7dq2UlpiYCC0tLchk/zd6SbEsl8u/zo5kgUFRAaKtJYNHCStp+VuY3pCIiIiIvi3pcqH0RXp1l0LSoxbr1q2Drq4uKlSoAH19fVy4cAGjR4+Gn58fdHV1AQBTp07FuHHjsH79ejg7OyM6OhoAYGJiAhMTE0RGRiIxMRF16tSRttmkSRMMHz4c/fv3x4ABAyCXyzFp0iRoaWnB09Pzqx+DjzEoIiIiIiLSEGFXn6k8cmFvboCgFq7wKW8PHR0dTJ06Fbdv34YQAk5OTggMDMSQIUOk/IsXL0ZKSgratWunVHZQUBCCg4Oxc+dONGvWTGkChbJly2L37t0ICQmBh4cHtLS0ULlyZQQFBcHe3v7L73g2GBQREREREWmAsKvP0HftRYiP0qPjktB37UUs7lQVfn5+8PPzy7KcBw8eZLl+586dGDt2rEp6kyZN0KRJE2k5NTUV+/bty2n1vyjOPkdEREREVMClywVCdl9XCYgASGkhu68jXa4uR86lpKSgbdu2aNq06WeV87UxKCIiIiIiKuDOR8UqDZn7mADwLC4J56NiP2s7enp6CAoKgqmp6WeV87UxKCIiIiIiKuBi3mYeEH1KvoKGQRERERERUQFnY2qQfaZc5CtoGBQRERERERVw1V0Kwd7cALJM1svwYRa66i6Fvma1vhkMioiIiIiICjhtLRmCWrgCgEpgpFgOauEq/V6RpmFQRERERESkAXzK22Nxp6qwM1ceImdnboDFnarCp3z+/15QfuHvFBERERERaQif8vZo4mqH81GxiHmbBBvTD0PmNLWHSIFBERERERGRBtHWksGjhFV+V+ObwuFzRERERESk0RgUERERERGRRmNQREREREREGo1BERERERERaTQGRUREREREpNEYFBERERERkUZjUERERERERBqNQREREREREWk0BkVERERERKTRGBQREREREZFGY1BEREREREQajUERERERERFpNAZFRERERESk0RgUERERERGRRmNQREREREREGo1BERERERERaTQGRUREREREpNEYFBERERERkUZjUERERERERBqNQREREREREWk0BkVERERERKTRGBQREREREZFGY1BEREREREQajUERERERERFpNAZFRERERESk0RgUERERERGRRmNQREREREREGo1BERERERERaTQGRUREREREpNEYFBERERERkUbL16AoODgYMplM6a9s2bIq+YQQaNq0KWQyGXbs2KG07tGjR2jevDmMjIxgY2OD4cOHIy0tTSnPsWPHULVqVejr66NkyZIIDQ1V2cbChQvh7OwMAwMD1KhRA+fPn8/LXSUiIiIiom9UvvcUubm54dmzZ9LfqVOnVPLMmTMHMplMJT09PR3NmzdHSkoKzpw5g9WrVyM0NBTjxo2T8kRFRaF58+Zo0KABLl++jMGDB6Nnz544cOCAlGfjxo0YOnQogoKCcPHiRVSqVAne3t6IiYn5MjtNRERERETfDJ18r4CODuzs7DJdf/nyZcycORMXLlyAvb290rqDBw/i+vXrOHToEGxtbVG5cmVMmDABI0eORHBwMPT09LBkyRK4uLhg5syZAIBy5crh1KlTmD17Nry9vQEAs2bNQq9evdCtWzcAwJIlS7B3716sXLkSo0aNUluv5ORkJCcnS8vx8fEAgNTUVKSmpn76AclDinp8K/WhT8N2/P6xDQsGtmPBwHYsGNiOBcOXbsfclJvvQdGdO3fg4OAAAwMDeHh4YPLkyShWrBgAIDExER06dMDChQvVBk4RERGoUKECbG1tpTRvb2/07dsX165dQ5UqVRAREYHGjRsrvc7b2xuDBw8GAKSkpCAyMhKjR4+W1mtpaaFx48aIiIjItN6TJ09GSEiISvrBgwdhZGSUq2PwpYWHh+d3FSgPsB2/f2zDgoHtWDCwHQsGtmPB8KXaMTExMcd58zUoqlGjBkJDQ1GmTBk8e/YMISEhqFu3Lq5evQpTU1MMGTIEtWrVQqtWrdS+Pjo6WikgAiAtR0dHZ5knPj4e79+/x+vXr5Genq42z82bNzOt++jRozF06FBpOT4+Ho6OjvDy8oKZmVnOD8IXlJqaivDwcDRp0gS6urr5XR36RGzH7x/bsGBgOxYMbMeCge1YMHzpdlSM5MqJfA2KmjZtKv27YsWKqFGjBpycnLBp0yZYW1vjyJEjuHTpUj7WMHP6+vrQ19dXSdfV1f3m3pzfYp0o99iO3z+2YcHAdiwY2I4FA9uxYPhS7ZibMvN9ooWMLCwsULp0ady9exdHjhzBvXv3YGFhAR0dHejofIjf2rZtC09PTwCAnZ0dnj9/rlSGYlkx3C6zPGZmZjA0NEThwoWhra2tNk9WzzoREREREVHB8E0FRQkJCbh37x7s7e0xatQoXLlyBZcvX5b+AGD27NlYtWoVAMDDwwP//vuv0ixx4eHhMDMzg6urq5Tn8OHDStsJDw+Hh4cHAEBPTw/u7u5KeeRyOQ4fPizlISIiIiKigitfh88NGzYMLVq0gJOTE54+fYqgoCBoa2vD398f1tbWantqihUrBhcXFwCAl5cXXF1d0blzZ0ybNg3R0dEYO3Ys+vfvLw1t++WXX7BgwQKMGDEC3bt3x5EjR7Bp0ybs3btXKnPo0KEICAhAtWrVUL16dcyZMwfv3r2TZqMjIiIiIqKCK1+Dov/++w/+/v549eoVrK2tUadOHZw9exbW1tY5er22tjb27NmDvn37wsPDA8bGxggICMD48eOlPC4uLti7dy+GDBmCuXPnomjRoli+fLk0HTcA+Pn54cWLFxg3bhyio6NRuXJlhIWFqUy+QEREREREBU++BkUbNmzIVX4hhEqak5MT9u3bl+XrPD09s52wITAwEIGBgbmqDxERERERff++qWeKiIiIiIiIvjYGRUREREREpNEYFBERERERkUZjUERERERERBqNQREREREREWk0BkVERERERKTRGBQREREREZFGY1BEREREREQajUERERERERFpNAZFRERERESk0RgUERERERGRRmNQREREREREGo1BERERERERaTQGRUREREREpNEYFBERERERkUZjUERERERERBqNQREREREREWk0BkVERERERKTRGBQREREREZFGY1BEREREREQajUERERERERFpNAZFRERERESk0RgUERERERGRRmNQREREREREGo1BERERERERaTQGRUREREREpNEYFBERERERkUZjUERERERERBqNQREREREREWk0BkVERERERKTRGBQREREREZFGY1BEREREREQajUERERERERFpNAZFRERERESk0RgUERERERGRRmNQREREREREGo1BERERERERaTQGRUREREREpNEYFBERERERkUZjUERERERERBqNQREREREREWk0BkVERERERKTRGBQREREREZFGY1BEREREREQajUERERERERFpNAZFRERERESk0RgUERERERGRRsuToOjhw4e4fv065HJ5XhRHRERERET01eQqKFq5ciVmzZqllNa7d28UL14cFSpUQPny5fH48eM8rSAREREREdGXlKugaOnSpbC0tJSWw8LCsGrVKvz111/4+++/YWFhgZCQkDyvJBERERER0Zeik5vMd+7cQbVq1aTlnTt3olWrVujYsSMAYNKkSejWrVve1pCIiIiIiOgLylVP0fv372FmZiYtnzlzBvXq1ZOWixcvjujo6LyrHRERERER0ReWq6DIyckJkZGRAICXL1/i2rVrqF27trQ+Ojoa5ubmeVtDIiIiIiKiLyhXw+cCAgLQv39/XLt2DUeOHEHZsmXh7u4urT9z5gzKly+f55UkIiIiIiL6UnIVFI0YMQKJiYnYtm0b7OzssHnzZqX1p0+fhr+/f55WkIiIiIiI6EvKVVCkpaWF8ePHY/z48WrXfxwkERERERERfetyFRQBwMaNG7Fr1y6kpKSgUaNG+OWXX75EvYiIiIiIiL6KXAVFixcvRv/+/VGqVCkYGhpi27ZtuHfvHqZPn/6l6kdERERERPRF5Wr2uQULFiAoKAi3bt3C5cuXsXr1aixatOhL1Y2IiIiIiOiLy1VQdP/+fQQEBEjLHTp0QFpaGp49e5bnFSMiIiIiIvoachUUJScnw9jY+P9erKUFPT09vH//Ps8rRkRERERE9DXkeqKF33//HUZGRtJySkoKJk6cqPSjrbNmzcqb2hEREREREX1huQqK6tWrh1u3biml1apVC/fv35eWZTJZ3tSMiIiIiIjoK8hVUHTs2DGl5ZcvX0JPTw9mZmZ5WSciIiIiIqKvJlfPFAHAmzdv0L9/fxQuXBi2trawtLSEnZ0dRo8ejcTExFyVFRwcDJlMpvRXtmxZAEBsbCwGDBiAMmXKwNDQEMWKFcPAgQMRFxenVMajR4/QvHlzGBkZwcbGBsOHD0daWppSnmPHjqFq1arQ19dHyZIlERoaqlKXhQsXwtnZGQYGBqhRowbOnz+fuwNDRERERETfpVz1FMXGxsLDwwNPnjxBx44dUa5cOQDA9evXMX/+fISHh+PUqVO4cuUKzp49i4EDB2ZbppubGw4dOvR/FdL5UKWnT5/i6dOnmDFjBlxdXfHw4UP88ssvePr0KbZs2QIASE9PR/PmzWFnZ4czZ87g2bNn6NKlC3R1dTFp0iQAQFRUFJo3b45ffvkF69atw+HDh9GzZ0/Y29vD29sbwIcfpB06dCiWLFmCGjVqYM6cOfD29satW7dgY2OTm0NERERERETfmVwFRePHj4eenh7u3bsHW1tblXVeXl7o3LkzDh48iHnz5uWsAjo6sLOzU0kvX748tm7dKi2XKFECEydORKdOnZCWlgYdHR0cPHgQ169fx6FDh2Bra4vKlStjwoQJGDlyJIKDg6Gnp4clS5bAxcUFM2fOBACUK1cOp06dwuzZs6WgaNasWejVqxe6desGAFiyZAn27t2LlStXYtSoUbk5RERERERE9J3JVVC0Y8cO/PnnnyoBEQDY2dlh2rRpaNasGYKCgpR+zygrd+7cgYODAwwMDODh4YHJkyejWLFiavPGxcXBzMxM6k2KiIhAhQoVlOrj7e2Nvn374tq1a6hSpQoiIiLQuHFjpXK8vb0xePBgAB9mz4uMjMTo0aOl9VpaWmjcuDEiIiIyrXdycjKSk5Ol5fj4eABAamoqUlNTc7TvX5qiHt9KfejTsB2/f2zDgoHtWDCwHQsGtmPB8KXbMTfl5iooevbsGdzc3DJdX758eWhpaSEoKChH5dWoUQOhoaEoU6YMnj17hpCQENStWxdXr16FqampUt6XL19iwoQJ6N27t5QWHR2tEqAplqOjo7PMEx8fj/fv3+P169dIT09Xm+fmzZuZ1n3y5MkICQlRST948KDSlOXfgvDw8PyuAuUBtuP3j21YMLAdCwa2Y8HAdiwYvlQ75ma+g1wFRYULF8aDBw9QtGhRteujoqJy9QxO06ZNpX9XrFgRNWrUgJOTEzZt2oQePXpI6+Lj49G8eXO4uroiODg4N1X+YkaPHo2hQ4dKy/Hx8XB0dISXl9c3MxtfamoqwsPD0aRJE+jq6uZ3degTsR2/f2zDgoHtWDCwHQsGtmPB8KXbUTGSKydyFRR5e3tjzJgxCA8Ph56entK65ORk/P777/Dx8clNkUosLCxQunRp3L17V0p7+/YtfHx8YGpqiu3btysdMDs7O5VZ4p4/fy6tU/xfkZYxj5mZGQwNDaGtrQ1tbW21edQ966Sgr68PfX19lXRdXd1v7s35LdaJco/t+P1jGxYMbMeCge1YMLAdC4Yv1Y65KTNXU3KPHz8et27dQqlSpTBt2jTs2rULO3fuxJQpU1CqVCncuHHjs3pyEhIScO/ePdjb2wP4EN15eXlBT08Pu3btgoGBgVJ+Dw8P/Pvvv4iJiZHSwsPDYWZmBldXVynP4cOHlV4XHh4ODw8PAICenh7c3d2V8sjlchw+fFjKQ0REREREBVeueoqKFi2KiIgI9OvXD6NHj4YQAgAgk8nQpEkTLFiwINNJEtQZNmwYWrRoAScnJzx9+hRBQUHQ1taGv7+/FBAlJiZi7dq1iI+Pl7rArK2toa2tDS8vL7i6uqJz586YNm0aoqOjMXbsWPTv31/qxfnll1+wYMECjBgxAt27d8eRI0ewadMm7N27V6rH0KFDERAQgGrVqqF69eqYM2cO3r17J81GR0REREREBVeugiIAcHFxwf79+/H69WvcuXMHAFCyZEkUKlQo1xv/77//4O/vj1evXsHa2hp16tTB2bNnYW1tjWPHjuHcuXNS+RlFRUXB2dkZ2tra2LNnD/r27QsPDw8YGxsjICAA48ePV6rv3r17MWTIEMydOxdFixbF8uXLpem4AcDPzw8vXrzAuHHjEB0djcqVKyMsLEztLHtERERERFSw5DooUrC0tET16tU/a+MbNmzIdJ2np6fUE5UVJycn7Nu3L8s8np6euHTpUpZ5AgMDERgYmO32iIiIiIioYMnVM0VEREREREQFDYMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijZavQVFwcDBkMpnSX9myZaX1SUlJ6N+/P6ysrGBiYoK2bdvi+fPnSmU8evQIzZs3h5GREWxsbDB8+HCkpaUp5Tl27BiqVq0KfX19lCxZEqGhoSp1WbhwIZydnWFgYIAaNWrg/PnzX2SfiYiIiIjo25LvPUVubm549uyZ9Hfq1Clp3ZAhQ7B7925s3rwZx48fx9OnT/HTTz9J69PT09G8eXOkpKTgzJkzWL16NUJDQzFu3DgpT1RUFJo3b44GDRrg8uXLGDx4MHr27IkDBw5IeTZu3IihQ4ciKCgIFy9eRKVKleDt7Y2YmJivcxCIiIiIiCjf5HtQpKOjAzs7O+mvcOHCAIC4uDisWLECs2bNQsOGDeHu7o5Vq1bhzJkzOHv2LADg4MGDuH79OtauXYvKlSujadOmmDBhAhYuXIiUlBQAwJIlS+Di4oKZM2eiXLlyCAwMRLt27TB79mypDrNmzUKvXr3QrVs3uLq6YsmSJTAyMsLKlSu//gEhIiIiIqKvSie/K3Dnzh04ODjAwMAAHh4emDx5MooVK4bIyEikpqaicePGUt6yZcuiWLFiiIiIQM2aNREREYEKFSrA1tZWyuPt7Y2+ffvi2rVrqFKlCiIiIpTKUOQZPHgwACAlJQWRkZEYPXq0tF5LSwuNGzdGREREpvVOTk5GcnKytBwfHw8ASE1NRWpq6mcdk7yiqMe3Uh/6NGzH7x/bsGBgOxYMbMeCge1YMHzpdsxNufkaFNWoUQOhoaEoU6YMnj17hpCQENStWxdXr15FdHQ09PT0YGFhofQaW1tbREdHAwCio6OVAiLFesW6rPLEx8fj/fv3eP36NdLT09XmuXnzZqZ1nzx5MkJCQlTSDx48CCMjo5wdgK8kPDw8v6tAeYDt+P1jGxYMbMeCge1YMLAdC4Yv1Y6JiYk5zpuvQVHTpk2lf1esWBE1atSAk5MTNm3aBENDw3ysWfZGjx6NoUOHSsvx8fFwdHSEl5cXzMzM8rFm/yc1NRXh4eFo0qQJdHV187s69InYjt8/tmHBwHYsGNiOBQPbsWD40u2oGMmVE/k+fC4jCwsLlC5dGnfv3kWTJk2QkpKCN2/eKPUWPX/+HHZ2dgAAOzs7lVniFLPTZczz8Yx1z58/h5mZGQwNDaGtrQ1tbW21eRRlqKOvrw99fX2VdF1d3W/uzfkt1olyj+34/WMbFgxsx4KB7VgwsB0Lhi/VjrkpM98nWsgoISEB9+7dg729Pdzd3aGrq4vDhw9L62/duoVHjx7Bw8MDAODh4YF///1XaZa48PBwmJmZwdXVVcqTsQxFHkUZenp6cHd3V8ojl8tx+PBhKQ8RERERERVc+RoUDRs2DMePH8eDBw9w5swZtGnTBtra2vD394e5uTl69OiBoUOH4ujRo4iMjES3bt3g4eGBmjVrAgC8vLzg6uqKzp07459//sGBAwcwduxY9O/fX+rF+eWXX3D//n2MGDECN2/exKJFi7Bp0yYMGTJEqsfQoUOxbNkyrF69Gjdu3EDfvn3x7t07dOvWLV+OCxERERERfT35Onzuv//+g7+/P169egVra2vUqVMHZ8+ehbW1NQBg9uzZ0NLSQtu2bZGcnAxvb28sWrRIer22tjb27NmDvn37wsPDA8bGxggICMD48eOlPC4uLti7dy+GDBmCuXPnomjRoli+fDm8vb2lPH5+fnjx4gXGjRuH6OhoVK5cGWFhYSqTLxARERERUcGTr0HRhg0bslxvYGCAhQsXYuHChZnmcXJywr59+7Isx9PTE5cuXcoyT2BgIAIDA7PMQ0REREREBc839UwRERERERHR18agiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNxqCIiIiIiIg0GoMiIiIiIiLSaAyKiIiIiIhIozEoIiIiIiIijcagiIiIiIiINBqDIiIiIiIi0mgMioiIiIiISKMxKCIiIiIiIo3GoIiIiIiIiDQagyIiIiIiItJoDIqIiIiIiEijMSgiIiIiIiKNppPfFdA0cXFxSExM/CrbSktLQ2JiIqKjo6Gjw6b+XrEdv39sw4KB7fh5jIyMYG5unt/VICJSi1f1ryguLg4LFy5EamrqV93u7du3v+r26MtgO37/2IYFA9vx0+jq6qJ///4MjIjom8Sg6CtKTExEamoq2rRpA2tr6/yuDhER0Vfx4sULbN++HYmJiQyKiOibxKAoH1hbW8Pe3j6/q0FEREREROBEC0REREREpOEYFBERERERkUZjUERERERE9I27desWGjRoAFtbWxgYGKB48eIYO3as0gRey5YtQ926dWFpaQlLS0s0btwY58+fV1tegwYNsHz5crx69Qo+Pj5wcHCAvr4+HB0dERgYiPj4eCnvtm3b0KRJE1hbW8PMzAweHh44cODAF9/nr+mbCYqmTJkCmUyGwYMHS2nR0dHo3Lkz7OzsYGxsjKpVq2Lr1q1Kr4uNjUXHjh1hZmYGCwsL9OjRAwkJCUp5rly5grp168LAwACOjo6YNm2ayvY3b96MsmXLwsDAABUqVMC+ffu+yH5+SaGhobCwsPjscmQyGXbs2PHZ5Sh07doVrVu3zrPyvgUPHjyATCbD5cuX87Tc4OBgVK5cOcs8n3s8v1Tdc2vp0qVwdHSElpYW5syZk6PX5PW5mR9y0sZZ+bj9PT09la6bmuJb3u/8fI8VhPcIEamnq6uLLl264ODBg7h16xbmzJmDZcuWISgoSMpz7Ngx+Pv74+jRo4iIiICjoyO8vLzw5MkTpbJiY2Nx+vRptGjRAlpaWmjVqhV27dqF27dvIzQ0FIcOHcIvv/wi5T9x4gSaNGmCffv2ITIyEg0aNECLFi1w6dKlr7b/X9o3MdHC33//jT///BMVK1ZUSu/SpQvevHmDXbt2oXDhwli/fj18fX1x4cIFVKlSBQDQsWNHPHv2DOHh4UhNTUW3bt3Qu3dvrF+/HgAQHx8PLy8vNG7cGEuWLMG///6L7t27w8LCAr179wYAnDlzBv7+/pg8eTJ+/PFHrF+/Hq1bt8bFixdRvnz5r3Ycunbtijdv3vCGlkdkMhm2b9/+2QHZt9Yuc+fOhRBCWvb09ETlypVzHFg4Ojri2bNnKFy48BeqYfbi4+MRGBiIWbNmoW3btjmejerZs2ewtLTM8XZCQ0MxePBgvHnz5hNr+u3btm0bdHV1c5Q3t+fK98TZ2RmDBw/O10CpW7duKFKkCHr27Jnr1wYHB2PHjh15Gkg9ePAALi4uuHTp0mcF4kT0bShevDiKFy8uLTs5OeHYsWM4efKklLZu3Tql1yxfvhxbt27F4cOH0aVLFyl97969qFq1KmxtbQEAffv2VSq3X79+mD59upT28X1j0qRJ2LlzJ3bv3i19Jv/e5XtPUUJCAjp27Ihly5apfNg5c+YMBgwYgOrVq0tdhBYWFoiMjAQA3LhxA2FhYVi+fDlq1KiBOnXqYP78+diwYQOePn0K4MPJkZKSgpUrV8LNzQ3t27fHwIEDMWvWLGk7c+fOhY+PD4YPH45y5cphwoQJqFq1KhYsWPD1DgRRDpmbm39Wj6C2tjbs7Ozy9ccnHz16hNTUVDRv3hz29vYwMjLK0evs7Oygr6//hWunKj09HXK5/KtvNycKFSoEU1PT/K6GxktPT8eePXvQsmXL/K4KEWmIu3fvIiwsDPXr1880j+LnYAoVKqSUvmvXLrRq1Urta54+fYpt27ZlWa5cLsfbt29Vyv2e5XtQ1L9/fzRv3hyNGzdWWVerVi1s3LgRsbGxkMvl2LBhA5KSkuDp6QkAiIiIgIWFBapVqya9pnHjxtDS0sK5c+ekPPXq1YOenp6Ux9vbG7du3cLr16+lPB9v39vbGxEREZnWOzk5GfHx8Up/AJCamprpX1pa2qcdpP9v1qxZqFChAoyNjeHo6Ih+/fqpDBUEgB07dqBUqVIwMDCAt7c3Hj9+rLR+586dqFq1qjQeNSQkJNO6paSkIDAwEPb29jAwMICTkxMmT56caR3T09MxdOhQWFhYwMrKCiNGjFDq1QA+vJEmT54MFxcXGBoaolKlStiyZYu0/tixY5DJZDh8+DCqVasGIyMj1KpVC7du3VIqZ/HixShRogT09PRQpkwZrFmzRlrn7OwMAGjTpg1kMpm0nNv9Dw4OxurVq7Fz507IZDLIZDIcO3ZMWn///n00aNAARkZGqFSpkso5c+rUKdStWxeGhoZwdHTEwIED8e7du0yPn8Kff/4JR0dHGBkZwdfXF3FxcdK6jMOnunbtiuPHj2Pu3LlS/R48eIDXr1+jY8eOsLa2hqGhIUqVKoVVq1YBUB3a07VrV+m1Gf8U+5mcnIxhw4ahSJEiMDY2Ro0aNZSOgTqPHj1Cq1atYGJiAjMzM/j6+uL58+cAPvTeVKhQAcCHb70Udc6JjEODFPuxbds2tW1w7NgxdOvWDXFxcdI+BQcH52ifFENRd+3aBVdXV+jr6+PRo0dwdnbGpEmT0L17d5iamqJYsWJYunSpUh1HjhyJ0qVLw8jICMWLF8fvv//+yT/YnJP308fDyBYtWiS9/21tbdGuXTsAmZ8r6enp6NGjh/R+LFOmDObOnau0DcU5N2PGDNjb28PKygr9+/dX2q/k5GSMHDkSjo6O0NfXR8mSJbFixQpp/dWrV9G0aVOYmJjA1tYWnTt3xsuXL3N0HN69e4cuXbrAxMQE9vb2mDlzpsoxePjwIYYMGSLt27t372BmZqZ0bQE+XB+NjY3x9u1b6RzasGEDatWqBQMDA5QvXx7Hjx9Xek1O6n7mzBno6urihx9+UKm/uqHNO3bsgEwmk9aHhITgn3/+keofGhqa7XG5c+cO6tWrBwMDA7i6uiI8PFxpvYuLCwCgSpUqkMlk8PT0xIkTJ6Crq4vo6GilvIMHD0bdunWV6puX9xKFtLS0LO+TX+sPyPp+zb/v409T29HDwwMGBgYoVaoUateuLd1n1P0NHz4cDg4OqF+/vpSWkJCAsLAwNG3aVCmvn58fjIyMUKRIEZiYmGDx4sWZljt16lQkJCSgTZs233w75lS+Dp/bsGEDLl68iL///lvt+k2bNsHPzw9WVlbQ0dGBkZERtm/fjpIlSwL48MyRjY2N0mt0dHRQqFAh6YIfHR0t3RgUFF2F0dHRsLS0RHR0tJSWMc/HN42MJk+ejJCQEJX0gwcPZvqtd2JiYqbl5YSWlhbmzZsHFxcX3L9/H/369cOIESOwaNEipW1MnDgRf/31F/T09NCvXz+0b98ep0+fBgCcPHkSXbp0wbx581C3bl3cu3dPGkaYcUyqwrx587Br1y5s2rQJxYoVw+PHj1VujBnNnDkToaGhWLlyJcqVK4eZM2di+/btaNiwoZRn8uTJWLt2LZYsWYJSpUrhxIkT6NSpE6ytrZW+lRgzZgxmzpwJa2tr/PLLL+jevbu0H9u3b8egQYMwZ84cNG7cGHv27EG3bt1QtGhRNGjQAH///TdsbGywatUq+Pj4QFtb+5P2f9iwYbhx4wbi4+OloKJQoUJST+SYMWMwY8YMlCpVCmPGjIG/vz/u3r0LHR0d3Lt3Dz4+Pvjjjz+wcuVKvHjxAoGBgQgMDJTKUufu3bvYtGkTdu/ejfj4ePTo0QP9+vVT6RIHPvRy3r59G+XLl8f48eMBfPgdrEGDBuH69evYv38/ChcujLt37+L9+/dqtzd37lxMmTJFWp4yZQr+97//oWzZsgCAwMBAXL9+HRs2bICDgwO2b98OHx8f/PvvvyhVqpRKeXK5XAqIjh8/jrS0NPTv3x9+fn44duwY/Pz84OjoKD386ejoCGtra3Tt2hUPHjzINuD6WGZtUKtWLcyZMwfjxo2TAmoTE5Mc71NiYiKmTp2K5cuXw8rKSrrWzJw5ExMmTMBvv/2GLVu2oG/fvqhfvz7KlCkDADA1NUVoaCgcHBzw77//olevXjA1NcWIESNytV+KbWX3fsrowoULGDhwINasWYNatWohNjZWGlaR2bkil8tRtGhRbN68GVZWVjhz5gx69+4Ne3t7+Pr6SmUfPXoU9vb2OHr0KO7evQs/Pz9UrlwZvXr1AvBhuHNERATmzZuHSpUqISoqSgoc3rx5g4YNG6Jnz56YPXs23r9/j5EjR8LX1xdHjhzJ9jgMHz4cx48fx86dO2FjY4PffvsNFy9elIaEbdu2DZUqVULv3r2l+hgbG6N9+/ZYtWqVFBgCkJZNTU3x6tUrqfw5c+bA1dUVs2bNQosWLRAVFQUrK6sc133Xrl1o0aKFFOjkhp+fH65evYqwsDAcOnQIALIdUiqXy/HTTz/B1tYW586dQ1xcnMrQwfPnz6N69eo4dOgQ3NzcoKenh0KFCqF48eJYs2YNhg8fDuDDh5F169YpPW+b1/cShVOnTuW4Z/hL+ziIpO+TJrZjjx49kJSUhKioKKxevRopKSn46aefVPJt3boV27dvxx9//KF0vYqMjISxsTEePnyIhw8fSulNmzZFvXr18PTpU6xZswa+vr5KzxUpHD9+HIsWLcJvv/2GCxcu5Mk+fal2zNVnb5FPHj16JGxsbMQ///wjpdWvX18MGjRIWg4MDBTVq1cXhw4dEpcvXxbBwcHC3NxcXLlyRQghxMSJE0Xp0qVVyra2thaLFi0SQgjRpEkT0bt3b6X1165dEwDE9evXhRBC6OrqivXr1yvlWbhwobCxscm0/klJSSIuLk76e/z4sQAgXr58KVJSUtT+PXr0SAQHB4unT5+qLTMgIEC0atUq84P2kc2bNwsrKytpedWqVQKAOHv2rJR248YNAUCcO3dOCCFEo0aNxKRJk5TKWbNmjbC3t5eWAYjt27cLIYQYMGCAaNiwoZDL5Tmqk729vZg2bZq0nJqaKooWLSrtV1JSkjAyMhJnzpxRel2PHj2Ev7+/EEKIo0ePCgDi0KFD0vq9e/cKAOL9+/dCCCFq1aolevXqpVTGzz//LJo1a6Z2PxRysv8fU9cuUVFRAoBYvny5lKY4r27cuCHt08fn3smTJ4WWlpa0Hx8LCgoS2tra4r///pPS9u/fL7S0tMSzZ8/U1ufj940QQrRo0UJ069ZN7TYUdb906ZLKuq1btwoDAwNx6tQpIYQQDx8+FNra2uLJkydK+Ro1aiRGjx6ttvyDBw8KbW1t8ejRIylNcWzOnz8vhBDi0qVLAoCIioqS8owaNUp07txZbZkKGds0J22watUqYW5urlRGTvZJ8V66fPmyUh4nJyfRqVMnaVkulwsbGxuxePHiTOs8ffp04e7uLi0HBQWJSpUqZbmfCtm9n4RQbv+tW7cKMzMzER8fr7Y8deeKOv379xdt27aVlgMCAoSTk5NIS0uT0n7++Wfh5+cnhBDi1q1bAoAIDw9XW96ECROEl5eXUprimnnr1q0s6/L27Vuhp6cnNm3aJKW9evVKGBoaKu2Lk5OTmD17ttJrz507J7S1taVr7vPnz4WOjo44duyYEOL/zqEpU6ZIr1Ec46lTp+aq7qVKlRJ79uxRKlfxHlN3Hm7fvl1kvAXn5rwQQogDBw4IHR0dpfN4//79at8jH7/Xp06dKsqVKyctb926VZiYmIiEhASpvnlxL8no6dOnIjg4WDx69CjTe+TX+nv37p3YsWOHePfuXb7XhX9sx8/9W7VqlTA0NBTv379XSp8yZYowNzcXERERKq/p3bu3GDhwYJblKj6LPXz4UCl9zZo1wtDQUOzYseO7aMeXL18KACIuLi7b62q+9RRFRkYiJiYGVatWldLS09Nx4sQJLFiwALdu3cKCBQtw9epVuLm5AQAqVaqEkydPYuHChViyZAns7OwQExOjVG5aWhpiY2NhZ2cH4MMzCIphOwqK5ezyKNaro6+vr/bZBl1d3Uwfev7cZzgOHTqEyZMn4+bNm4iPj0daWhqSkpKQmJgoffOmo6OjNHyjbNmysLCwwI0bN1C9enX8888/OH36NCZOnCjlSU9PVylHoWvXrmjSpAnKlCkDHx8f/Pjjj/Dy8lJbv7i4ODx79gw1atRQ2udq1apJQ37u3r2LxMRENGnSROm1KSkpKg/qZZx4w97eHgAQExODYsWK4caNG9K3kgq1a9dWGfbzsdzuf3Yyq2PZsmXxzz//4MqVK0o9PEIIyOVyREVFoVy5cmrLLFasGIoUKSIte3h4QC6X49atW1mekxn17dsXbdu2xcWLF+Hl5YXWrVujVq1aWb7m0qVL6Ny5MxYsWIDatWsDAP7991+kp6ejdOnSSnmTk5NhZWWltpwbN27A0dERjo6OUpqrq6t0HqobXgQgy2GZWcmqDdTJ6T7p6empTP7y8fZkMpnKdWjjxo2YN28e7t27h4SEBKSlpcHMzCzX+5WT99PHmjRpAicnJxQvXhw+Pj7w8fFBmzZtsj2vFy5ciJUrV+LRo0d4//49UlJSVB7Md3Nzk3pcgQ/H+t9//wUAXL58Gdra2pmOP//nn39w9OhRqacuo3v37qm0xcfrU1JSlI5DoUKFpJ65rFSvXh1ubm5YvXo1Ro0ahbVr18LJyQn16tVTyufh4SH9W3GMb9y4keO637hxA0+fPkWjRo2yrVNeUbzPHBwcpLSM+5GVrl27YuzYsTh79ixq1qyJ0NBQ+Pr6wtjYWMqT1/eSjOXmdGKQLy2r+zV9PzS9HbW0tJCamgptbW3pOEybNg2TJk3CgQMHULNmTaX8Qgjs3bsXa9euzfK4aWl9eMJGLpdL+f73v/+hV69e2LBhQ6bPI32qL9WOuSkz34KiRo0aSTdUhW7duqFs2bIYOXKk1N2laBQFbW1t6YFnDw8PvHnzBpGRkXB3dwcAHDlyBHK5XLqBenh4YMyYMUhNTZUOTHh4OMqUKSNN7ODh4YHDhw8rDT0IDw/P8Q3ma3jw4AF+/PFH9O3bFxMnTkShQoVw6tQp9OjRAykpKTn+MJ+QkICQkBC13awGBgYqaVWrVkVUVBT279+PQ4cOwdfXF40bN1YZp59Timeg9u7dq/TBH4BKkJnxRFYMSfnch91zu//ZyaqOCQkJ6NOnDwYOHKjyumLFiuV6W7nRtGlTPHz4EPv27UN4eDgaNWqE/v37Y8aMGWrzR0dHo2XLlujZsyd69OghpSckJEBbWxuRkZFKH4gBqP2QmB9ye57kdJ8MDQ3VDoX6+AIrk8mk7UVERKBjx44ICQmBt7c3zM3NsWHDBpVnYL4UU1NTXLx4EceOHcPBgwcxbtw4BAcH4++//850co4NGzZg2LBhmDlzJjw8PGBqaorp06dLz2UqZLXfhoaGWdYrISEBLVq0wNSpU1XWKQLZL6Vnz55YuHAhRo0ahVWrVqFbt265GuKWk7rv2rULTZo0yfQaoqWlpRLI5mace16zsbFBixYtsGrVKri4uGD//v25Hraa19dSIsreunXroKuriwoVKkBfXx8XLlzA6NGj4efnJ12jp06dinHjxmH9+vVwdnaWHgUxMTGBiYkJIiMjkZiYiDp16kjl7tu3D8+fP8cPP/wAExMTXLt2DcOHD0ft2rWlZ7LXr1+PgIAAzJ07FzVq1JDKNTQ0VBnumy4XOB8Vi5i3SbAxNUB1l0LQ1sr90OKvLd+CIlNTU5Xpro2NjWFlZYXy5csjNTUVJUuWRJ8+fTBjxgxYWVlhx44dCA8Px549ewAA5cqVg4+PD3r16oUlS5YgNTUVgYGBaN++vfTtWYcOHRASEoIePXpg5MiRuHr1KubOnYvZs2dL2x00aBDq16+PmTNnonnz5tiwYQMuXLig8gB1foqMjIRcLsfMmTOlQHHTpk0q+dLS0nDhwgVUr14dwIcf+nrz5o3UK1G1alXcunVLei4rJ8zMzODn5wc/Pz+0a9cOPj4+iI2NVZlxxNzcHPb29jh37pz0TWxaWhoiIyOlHsGMD61nNatJdsqVK4fTp08jICBASjt9+jRcXV2lZV1dXaSnpyu97lP2X09PT6WcnKhatSquX7+eq20BHyYpePr0qXQOnz17FlpaWpl+M55Z/aytrREQEICAgADUrVsXw4cPVxsUJSUloVWrVihbtqzSrIzAhwe009PTERMTIz2EnZ1y5cpJz54peouuX7+ON2/eKLXP16Du2HzKPuXUmTNn4OTkhDFjxkhpGcdr50ZO3k/q6OjooHHjxmjcuDGCgoJgYWGBI0eO4KefflJ7PE6fPo1atWqhX79+Utq9e/dyVdcKFSpALpfj+PHjaifNUfzGnLOzc657zEuUKAFdXV2cO3dO+jLh9evXuH37ttI1JLP3QadOnTBixAjMmzcP169fV7pmKJw9e1blGAcGBua47jt37lTpuc7I2toab9++xbt376TemI+n3s7tdUbxPnv27JkUnJ09e1alTABqy+3Zsyf8/f1RtGhRlChRQuodVvgS9xIi+jw6OjqYOnUqbt++DSEEnJycEBgYiCFDhkh5Fi9ejJSUFKVnKYEPz/oFBwdj586daNasmdL1zNDQEMuWLcOQIUOQnJwMR0dH/PTTTxg1apSUZ+nSpdIzwv3795fSAwIClCaGCbv6DCG7r+NZXJKUZv//2rv3qCjr/A/gb+4zwDAIpsJyGRXwfkETXeSEmySbFywrzcWE1TQNVsDNeyi7XiJX07SshcxbGK0nUVMPWESteUNl9cS6aYKpaxqaxuWwIjCf3x/+mBy5wzwgzPt1zhzxme985/udzwzPfHie7+fRqrBsXG/8vq+yfwRrrkfiOkU1sbGxwcGDB7Fw4UKMGzcOJSUl8PHxwbZt2zB69GhDu5SUFERHR2PkyJGwtLTEc889hw0bNhju12q1OHToEKKiojB48GB07NgRS5cuNdqBBQYGYufOnXj99dexePFi+Pr6Ys+ePS16jaIqhYWF1XaWrq6u8PHxQXl5OTZu3Ihx48bhyJEjeP/996s93sbGBn/605+wYcMGWFtbIzo6GsOGDTPs2JYuXYqxY8fCy8sLzz//PCwtLXH27Fnk5uZixYoV1fp766234ObmBn9/f1haWmLXrl3o0qVLrX91jomJQWJiInx9fQ1fsh+8RoxGo8Frr72GuLg46PV6BAUFobCwEEeOHIGTk1ONX1hqMm/ePEycOBH+/v4ICQnBZ599ht27dxsWKQP3K9BlZmZi+PDhsLOzQ4cOHRo9/6p+MjIycP78ebi6ujb4mjoLFizAsGHDEB0djZdffhkODg44d+4cPv/88zrLvatUKkRERGDNmjUoKirCnDlzMHHixFpPndPpdDhx4gR++OEHODo6wsXFBQkJCRg8eDD69OmDsrIy7N+/v9bT9V555RVcvXoVmZmZuHnzpmG7i4sL/Pz8EB4ejqlTp2Lt2rXw9/fHzZs3kZmZif79+2PMmDHV+gsJCUG/fv0QHh6O9evXo6KiAq+++iqCg4ONKkU+bNGiRbh27Rq2b99ea5vG0ul0KCkpQWZmJgYMGAB7e/smzamhfH19ceXKFaSmpmLIkCE4cOAA0tLSmtxffZ+nh+3fvx/5+fl44okn0KFDBxw8eBB6vd6QUNf0XvH19cX27duRkZGBrl27YseOHTh58mS1AjV10el0iIiIwLRp0wyFFi5fvoyCggJMnDgRUVFRSE5OxuTJkzF//ny4uLjg4sWLSE1NxQcffFDtiN2DHB0dMX36dMybN89Q8GLJkiXVziLQ6XT45z//iRdffBF2dnaG63B16NABEyZMwLx58zBq1Ch4eHhUe453330Xvr6+6NWrF9atW4c7d+5g2rRpAFDv2H/++WecOnUK+/btq3UOQ4cOhb29PRYvXow5c+bgxIkT1arL6XQ6XLp0CWfOnIGHhwc0Gk2d5edDQkLg5+eHiIgI/O1vf0NRUZFRMg7cPyKkVquRnp4ODw8PqFQqw++v0NBQODk5YcWKFYbCGw8y9b6EiJqv6g/UdamvmuvevXvx+uuvG2373e9+h6NHj9b5uIYcTU7PvY7ZH+Xg4RO8bxTexeyPcvDelEGPdmJU76ojapDCwsJ6F3JVLTStq9ACgGq36dOni4jIW2+9JW5ubqJWqyU0NFS2b98uAOTOnTsi8uti3k8//VS6desmdnZ2EhISIpcvXzZ6nvT0dAkMDBS1Wi1OTk4SEBAgSUlJhvvxwELdpKQkGThwoDg4OIiTk5OMHDlScnJyap1jeXm5xMTEiJOTkzg7O8vcuXNl6tSpRgvD9Xq9rF+/Xnr06CE2Njby2GOPSWhoqHz99dci8muhhap5idS8MH/Tpk3SrVs3sbGxET8/P9m+fbvRWPbt2yc+Pj5ibW0t3t7eDZ7/wwoKCuSpp54SR0dHASBZWVk1LmC+c+eO4f4q2dnZhsc6ODhI//79ZeXKlbU+V9Vi602bNom7u7uoVCp5/vnn5fbt24Y2DxdaOH/+vAwbNkzUarXhNVq+fLn06tVL1Gq1uLi4yPjx4yU/P19Eqi++9vb2rvF9VzWPe/fuydKlS0Wn04mNjY24ubnJs88+ayh4UpPLly9LWFiYODg4iEajkRdeeEFu3LhhuL+meEZEREhwcHCtfYrUXGihvhjMmjVLXF1dBYAsW7asQXOqaWF81Wv18GL+AQMGGPoVEZk3b564urqKo6OjTJo0SdatW2fUV2MW1Dfk8/Rg8YTDhw9LcHCwdOjQQdRqtfTv318++eQTQ9ua3it3796VyMhI0Wq14uzsLLNnz5aFCxcajbGmYiMxMTFG8frf//4ncXFx4ubmJra2tuLj4yMffvih4f4LFy7Is88+K87OzqJWq6Vnz54SGxvboCIuxcXFMmXKFLG3t5fOnTvL6tWrqxWNOHbsmPTv31/s7Ozk4V1bZmamADAq1iDy63to586dEhAQILa2ttK7d2/58ssvjdrVNfYPPvhAhg8fXmO/D74309LSxMfHR9RqtYwdO1aSkpKMxnn37l157rnnxNnZWQDIli1b6n1dzp8/L0FBQWJrayt+fn6Snp5ercBMcnKyeHp6iqWlZbXPV3x8vFEhiiqm2pc8qL79X0u6d++eYZE4tV2MY+OVlZVJQkJCrcV4mqOiUi/DVn0h3gv213jTLdgvw1Z9IRWVxr/zlY5jQ76fV7EQqWXFLjVKUVERtFotCgsLa11Uff36dSQlJRnK3RIRkfJ27NiBuLg4/Pjjj0bXrPvhhx/QtWtX/Otf/6pWWKKhwsLCEBQU1KSS661t+vTpuHnzZrWjXFu3bkVsbGydRyUb61Ha/5WXl+PgwYMYPXq0WS/Qb+sYx0fLsbyfMTn5eL3tPp4xDL/t/mtRI6Xj2JDv51Ue2dPniIiImqO0tBTXr19HYmIiXnnlFaOEyFSCgoIwefJkk/erpMLCQnz77bfYuXNnnaf9ERE1VEHx3fobNaJda7CsvwkRESmhqhpQTbeqi662d1euXKnzdbhy5UqT+169ejV69uyJLl26YNGiRSYc9a/mz59vVH7eVFJSUmp9TaouU9FU48ePx6hRozBr1qxql0cgImqKTpqGVZ1saLvWwCNFRESt5OGiKg96uGR9e+Xu7l7n6/DgdXgaKyEhAQkJCbXer9Ppar3mU2sLCwszujbTg5p7ikl9C6YjIyMRGRnZrOcgIvMS0NUFbloVbhTerVZoAQAsAHTR3i/P/ahiUkRE1EpYzvh+iVm+DtVpNBpoNJrWHgYRUYNYWVpg2bjemP1RDiwAo8So6gpFy8b1fqSvV8TT54iIiIiIqFl+39cN700ZhC5a41PkumhVj345bvBIERERERERmcDv+7rhqd5dkH3pNgqK76KT5v4pc4/yEaIqTIpawYMXyCQiImrvuN8jMh9WlhZGZbfbCiZFLcje3h42NjbNusI9ERFRW2RjYwN7e/vWHgYRUY2YFLUgrVaLqKgolJaWtsjzVVRU4JtvvkFQUBCsrRnqtopxbPsYw/aBcWwee3t7aLXa1h4GEVGN+Fu9hWm12hbbKZSXl8Pe3h5dunTh1Z7bMMax7WMM2wfGkYio/WL1OSIiIiIiMmtMioiIiIiIyKwxKSIiIiIiIrPGpIiIiIiIiMwakyIiIiIiIjJrTIqIiIiIiMissSS3iYgIAKCoqKiVR/Kr8vJylJaWoqioiOVj2zDGse1jDNsHxrF9YBzbB8axfVA6jlXfy6u+p9eFSZGJFBcXAwA8PT1beSRERERERFSluLi43uuEWkhDUieql16vx48//giNRgMLC4vWHg6A+9mxp6cnrl69Cicnp9YeDjUR49j2MYbtA+PYPjCO7QPj2D4oHUcRQXFxMdzd3WFpWfeqIR4pMhFLS0t4eHi09jBq5OTkxF8Y7QDj2PYxhu0D49g+MI7tA+PYPigZx/qOEFVhoQUiIiIiIjJrTIqIiIiIiMisMSlqx+zs7LBs2TLY2dm19lCoGRjHto8xbB8Yx/aBcWwfGMf24VGKIwstEBERERGRWeORIiIiIiIiMmtMioiIiIiIyKwxKSIiIiIiIrPGpIiIiIiIiMwak6I25N1334VOp4NKpcLQoUORnZ1da9sRI0bAwsKi2m3MmDFG7f7zn/8gLCwMWq0WDg4OGDJkCK5cuaL0VMyaqeNYUlKC6OhoeHh4QK1Wo3fv3nj//fdbYipmrTFxBID169ejR48eUKvV8PT0RFxcHO7evdusPqn5TB3HN954A0OGDIFGo0GnTp3wzDPP4Pz580pPw+wp8XmskpiYCAsLC8TGxiowcqqiRAyvXbuGKVOmwNXVFWq1Gv369cOpU6eUnIbZM3UcKysrER8fj65du0KtVqN79+5Yvnw5FKkTJ9QmpKamiq2trXz44Yfy73//W2bMmCHOzs7y008/1dj+559/luvXrxtuubm5YmVlJVu2bDG0uXjxori4uMi8efMkJydHLl68KHv37q21T2o+JeI4Y8YM6d69u2RlZcmlS5fk73//u1hZWcnevXtbaFbmp7FxTElJETs7O0lJSZFLly5JRkaGuLm5SVxcXJP7pOZTIo6hoaGyZcsWyc3NlTNnzsjo0aPFy8tLSkpKWmpaZkeJOFbJzs4WnU4n/fv3l5iYGIVnYr6UiOHt27fF29tbIiMj5cSJE5Kfny8ZGRly8eLFlpqW2VEijitXrhRXV1fZv3+/XLp0SXbt2iWOjo7y9ttvm3z8TIraiICAAImKijL8v7KyUtzd3eWNN95o0OPXrVsnGo3GaMc8adIkmTJlisnHSrVTIo59+vSRv/71r0btBg0aJEuWLDHNoKmaxsYxKipKnnzySaNtc+fOleHDhze5T2o+JeL4sIKCAgEgX3/9tWkGTdUoFcfi4mLx9fWVzz//XIKDg5kUKUiJGC5YsECCgoKUGTDVSIk4jhkzRqZNm2bUZsKECRIeHm7Ckd/H0+fagHv37uH06dMICQkxbLO0tERISAiOHTvWoD42b96MF198EQ4ODgAAvV6PAwcOwM/PD6GhoejUqROGDh2KPXv2KDEFgjJxBIDAwEDs27cP165dg4ggKysLFy5cwKhRo0w+B2paHAMDA3H69GnDaQT5+fk4ePAgRo8e3eQ+qXmUiGNNCgsLAQAuLi4mHD1VUTKOUVFRGDNmjFHfZHpKxXDfvn14/PHH8cILL6BTp07w9/dHcnKyspMxY0rFMTAwEJmZmbhw4QIA4OzZs/jmm2/w9NNPm3wO1ibvkUzu1q1bqKysROfOnY22d+7cGd999129j8/OzkZubi42b95s2FZQUICSkhIkJiZixYoVePPNN5Geno4JEyYgKysLwcHBJp+HuVMijgCwceNGzJw5Ex4eHrC2toalpSWSk5PxxBNPmHT8dF9T4viHP/wBt27dQlBQEEQEFRUVmDVrFhYvXtzkPql5lIjjw/R6PWJjYzF8+HD07dvX5HMg5eKYmpqKnJwcnDx5UtHxk3IxzM/Px3vvvYe5c+di8eLFOHnyJObMmQNbW1tEREQoOidzpFQcFy5ciKKiIvTs2RNWVlaorKzEypUrER4ebvI58EiRGdi8eTP69euHgIAAwza9Xg8AGD9+POLi4jBw4EAsXLgQY8eO5SL9R1RNcQTuJ0XHjx/Hvn37cPr0aaxduxZRUVH44osvWmmk9LCvvvoKq1atwqZNm5CTk4Pdu3fjwIEDWL58eWsPjRqhsXGMiopCbm4uUlNTW3ikVJf64nj16lXExMQgJSUFKpWqlUdLNWnIZ1Gv12PQoEFYtWoV/P39MXPmTMyYMYPfcR4hDYnjP/7xD6SkpGDnzp3IycnBtm3bsGbNGmzbts30AzL5CXlkcmVlZWJlZSVpaWlG26dOnSphYWF1PrakpEScnJxk/fr11fq0traW5cuXG22fP3++BAYGmmTcZEyJOJaWloqNjY3s37/faPv06dMlNDTUJOMmY02JY1BQkLz22mtG23bs2CFqtVoqKyub9d6gplEijg+KiooSDw8Pyc/PN+m4yZgScUxLSxMAYmVlZbgBEAsLC7GyspKKigqlpmOWlPosenl5yfTp043abNq0Sdzd3U03eDJQKo4eHh7yzjvvGLVZvny59OjRw3SD/388UtQG2NraYvDgwcjMzDRs0+v1yMzMxG9/+9s6H7tr1y6UlZVhypQp1focMmRItVKxFy5cgLe3t+kGTwZKxLG8vBzl5eWwtDT+KFtZWRmOBpJpNSWOpaWlNcYIAESkWe8Nahol4lj1b3R0NNLS0vDll1+ia9euCs2AAGXiOHLkSHz77bc4c+aM4fb4448jPDwcZ86cMbQl01Dqszh8+HB+x2lBSsWxtjaKfMcxeZpFikhNTRU7OzvZunWrnDt3TmbOnCnOzs5y48YNERF56aWXZOHChdUeFxQUJJMmTaqxz927d4uNjY0kJSXJ999/Lxs3bhQrKys5fPiwonMxZ0rEMTg4WPr06SNZWVmSn58vW7ZsEZVKJZs2bVJ0LuassXFctmyZaDQa+fjjjyU/P18OHTok3bt3l4kTJza4TzI9JeI4e/Zs0Wq18tVXXxmV0y8tLW3x+ZkLJeL4MFafU5YSMczOzhZra2tZuXKlfP/995KSkiL29vby0Ucftfj8zIUScYyIiJDf/OY3hpLcu3fvlo4dO8r8+fNNPn4mRW3Ixo0bxcvLS2xtbSUgIECOHz9uuC84OFgiIiKM2n/33XcCQA4dOlRrn5s3bxYfHx9RqVQyYMAA2bNnj1LDp/9n6jhev35dIiMjxd3dXVQqlfTo0UPWrl0rer1eyWmYvcbEsby8XBISEqR79+6iUqnE09NTXn31Vblz506D+yRlmDqOAGq8PXhtMTI9JT6PD2JSpDwlYvjZZ59J3759xc7OTnr27ClJSUktNBvzZeo4FhUVSUxMjHh5eYlKpZJu3brJkiVLpKyszORjtxBR4pKwREREREREbQPXFBERERERkVljUkRERERERGaNSREREREREZk1JkVERERERGTWmBQREREREZFZY1JERERERERmjUkRERERERGZNSZFRERERERk1pgUERERNUNCQgIGDhxo+H9kZCSeeeaZVhsPERE1HpMiIiIiIiIya0yKiIio3bp3715rD4GIiNoAJkVERNRujBgxAtHR0YiNjUXHjh0RGhqK3NxcPP3003B0dETnzp3x0ksv4datW4bH6PV6rF69Gj4+PrCzs4OXlxdWrlxpuH/BggXw8/ODvb09unXrhvj4eJSXl7fG9IiISCFMioiIqF3Ztm0bbG1tceTIESQmJuLJJ5+Ev78/Tp06hfT0dPz000+YOHGiof2iRYuQmJiI+Ph4nDt3Djt37kTnzp0N92s0GmzduhXnzp3D22+/jeTkZKxbt641pkZERAqxEBFp7UEQERGZwogRI1BUVIScnBwAwIoVK3D48GFkZGQY2vz3v/+Fp6cnzp8/Dzc3Nzz22GN455138PLLLzfoOdasWYPU1FScOnUKwP1CC3v27MGZM2cA3C+08Msvv2DPnj0mnRsRESnHurUHQEREZEqDBw82/Hz27FlkZWXB0dGxWru8vDz88ssvKCsrw8iRI2vt75NPPsGGDRuQl5eHkpISVFRUwMnJSZGxExFR62BSRERE7YqDg4Ph55KSEowbNw5vvvlmtXZubm7Iz8+vs69jx44hPDwcf/nLXxAaGgqtVovU1FSsXbvW5OMmIqLWw6SIiIjarUGDBuHTTz+FTqeDtXX1XZ6vry/UajUyMzNrPH3u6NGj8Pb2xpIlSwzbLl++rOiYiYio5bHQAhERtVtRUVG4ffs2Jk+ejJMnTyIvLw8ZGRn44x//iMrKSqhUKixYsADz58/H9u3bkZeXh+PHj2Pz5s0A7idNV65cQWpqKvLy8rBhwwakpaW18qyIiMjUmBQREVG75e7ujiNHjqCyshKjRo1Cv379EBsbC2dnZ1ha3t8FxsfH489//jOWLl2KXr16YdKkSSgoKAAAhIWFIS4uDtHR0Rg4cCCOHj2K+Pj41pwSEREpgNXniIiIiIjIrPFIERERERERmTUmRUREREREZNaYFBERERERkVljUkRERERERGaNSREREREREZk1JkVERERERGTWmBQREREREZFZY1JERERERERmjUkRERERERGZNSZFRERERERk1pgUERERERGRWfs/FNNVXBYe5NQAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots(1, 1, figsize=plt.figaspect(1/2))\n", - "fig.suptitle(\n", - " f'Effects of search parameters on QPS/recall trade-off ({DATASET_FILENAME})\\n' + \\\n", - " f'k = {k}, n_probes = {n_probes}, pq_dim = {pq_dim}')\n", - "ax.plot(bench_recall_s1, bench_qps_s1, 'o')\n", - "ax.set_xlabel('recall')\n", - "ax.set_ylabel('QPS')\n", - "ax.grid()\n", - "annotations = []\n", - "for i, label in enumerate(bench_names):\n", - " annotations.append(ax.text(\n", - " bench_recall_s1[i], bench_qps_s1[i],\n", - " f\" {label} \",\n", - " ha='center', va='center'))\n", - "clutter = [\n", - " ax.text(\n", - " 0.02, 0.08,\n", - " 'Labels denote the bitsize of: internal_distance_dtype/lut_dtype',\n", - " verticalalignment='top',\n", - " bbox={'facecolor': 'white', 'edgecolor': 'grey'},\n", - " transform = ax.transAxes)\n", - "]\n", - "adjust_text(annotations, objects=clutter);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This figure represents the trade-offs one does by choosing different combintations of the internal search types (the bit sizes of the data types are shown as point labels).\n", - "Depending on the GPU and the selected dataset, you may see different pictures.\n", - "With SIFT-128 (`pq_dim = 64`), reducing the `internal_distance_dtype` comes at a huge cost to recall,\n", - "whereas `lut_dtype` doesn't cost too much while significantly improving QPS.\n", - "\n", - "Also, often you may see `16/16` version being faster than `16/8`.\n", - "This indicates that ALU is the bottleneck in this configuration, and a few extra ALU operations for converting between fp8 and fp16 do more harm than the saved L1 bandwidth does good for the performance.\n", - "\n", - "\n", - "Let's try the same experiment, but with refinement.\n", - "We'll try ratio 2 and 4 and see how it affects recall and QPS." - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "210 ms ± 129 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "181 ms ± 331 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "184 ms ± 536 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "179 ms ± 331 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "182 ms ± 329 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "410 ms ± 203 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "344 ms ± 304 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "338 ms ± 632 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "320 ms ± 269 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "323 ms ± 194 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "425 ms ± 743 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "389 ms ± 688 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "381 ms ± 519 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "325 ms ± 552 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "340 ms ± 876 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "def search_refine(ps, ratio):\n", - " k_search = k * ratio\n", - " candidates = ivf_pq.search(ps, index, queries, k_search, handle=resources)[1]\n", - " return candidates if ratio == 1 else refine(dataset, queries, candidates, k, handle=resources)[1]\n", - "\n", - "ratios = [1, 2, 4]\n", - "bench_qps_sr = np.zeros((len(ratios), len(search_ps)), dtype=np.float32)\n", - "bench_recall_sr = np.zeros((len(ratios), len(search_ps)), dtype=np.float32)\n", - "\n", - "for j, ratio in enumerate(ratios): \n", - " for i, ps in enumerate(search_ps):\n", - " r = %timeit -o search_refine(ps, ratio); resources.sync()\n", - " bench_qps_sr[j, i] = (queries.shape[0] * r.loops / np.array(r.all_runs)).mean()\n", - " bench_recall_sr[j, i] = calc_recall(search_refine(ps, ratio), gt_neighbors)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0UAAAHgCAYAAABqycbBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAADexklEQVR4nOzdd1gUx/8H8PdxdI6qVEFAsYEigopoVFQElaBGjYrGgC02EtFYE2NN7I3YO0bNN5aYxFhQULFi7L0jlihFY0FU2t3+/uB3G887yimK8d6v5+HRnZ2bnV2G3fvszM5KBEEQQEREREREpKP0yroCREREREREZYlBERERERER6TQGRUREREREpNMYFBERERERkU5jUERERERERDqNQREREREREek0BkVERERERKTTGBQREREREZFOY1BEREREREQ6jUERqcnKykKfPn3g4OAAiUSC6OhoAEB6ejo6deqEcuXKQSKRYO7cuWVaz9J27do1BAcHw9LSEhKJBL///ntZV6nUjB8/HhKJBA8ePCjrqhB90AIDAxEYGCgu37x5ExKJBLGxsWVWp6K4ubkhMjKyzLY/Y8YMVKpUCVKpFD4+PgCA/Px8jBgxAi4uLtDT00P79u2LLEOhUKBmzZr44YcftNp2ZGQk3NzcVNIKu/7R61Nef15W0nYXGxsLiUSCmzdvvp3KlYLAwEDUrFnztT+fmJgIiUSCxMRElfQ1a9agevXqMDAwgJWVlVZlxsXFQSaT4f79+69dL13EoEhHKE8shf0cOXJEzDt58mTExsZiwIABWLNmDXr06AEAGDJkCHbu3InRo0djzZo1aNWqVanXc/LkyWUWjERERODcuXP44YcfsGbNGtStW7dM6kHvn+3bt2P8+PFlXY0y9ezZM0yaNAne3t4wNTWFpaUlGjdujDVr1kAQBLX8L59f9PT04OTkhODgYLULf25uLmJiYlCnTh1YWFjAysoKXl5e+OKLL3D58mW1chUKBWxtbTF9+vS3tatl4uLFixg/fvx7/eWvtO3atQsjRoxAo0aNsGrVKkyePBkAsHLlSsyYMQOdOnXC6tWrMWTIkCLL+d///oc7d+4gKirqjeuk6fp3+PBhjB8/Ho8fPy5xOevXr8dnn32GKlWqQCKRqATKLzt27BiioqLg5eUFMzMzVKxYEZ07d8bVq1c15t+wYQMaNGgAKysrlCtXDk2bNsW2bdteY0/pfXb58mVERkaicuXKWLZsGZYuXQqgIJDX9B2uevXqKp9v1aoVPDw8MGXKlLKo/n+WfllXgN6tiRMnwt3dXS3dw8ND/P+ePXvQoEEDjBs3TiXPnj170K5dOwwbNuyt1W/y5Mno1KlTsXcGS9uLFy+QlJSEb7/9tlQurPRh2b59OxYsWKCzgVF6ejpatGiBS5cuoWvXroiKikJ2djZ+/fVXfP7554iLi8OaNWugp6d6n61ly5b4/PPPIQgCUlJSsHDhQjRv3hzbtm1D69atAQAdO3bEjh07EB4ejr59+yIvLw+XL1/G1q1b0bBhQ7WL/dGjR/HgwQOEhoa+s/1/Fy5evIgJEyYgMDBQrffiQ7Vnzx7o6elhxYoVMDQ0VEmvUKEC5syZU6JyZsyYga5du8LS0lKr7S9btgwKhUKtTq9e/2bOnIkJEyYgMjKyxHfsFy1ahBMnTqBevXr4559/Cs03bdo0HDp0CJ9++im8vb2RlpaG+fPnw9fXF0eOHFHpgZg3bx6++uorhIaGYurUqcjOzkZsbCw+/vhj/Prrr+jQoYNW+1+Wrly5ona+oH8lJiZCoVAgJiZG5fsZABgZGWH58uUqaZrafr9+/TBs2DBMmDAB5ubmb7W+HwoGRTqmdevWxfaAZGRkwNPTU2O6tl24/xXKLub/0v7l5+dDoVCofJn4L/vQ9qc4giAgOzsbJiYmZV2VYkVERODSpUv47bff0LZtWzH9q6++wvDhwzFz5kz4+Phg+PDhKp+rWrUqPvvsM3H5k08+gbe3N+bOnYvWrVvj2LFj2Lp1K3744Qd88803Kp+dP3++xjvz27dvh6urK7y8vDTW9b90XF/Xh7KPGRkZMDExUfub1+Zac+rUKZw5cwazZs3SevsGBgYa66Tp+qetNWvWoEKFCtDT0ytyaNXQoUPx888/qxyDLl26oFatWpg6dSrWrl0rps+bNw/16tXDn3/+KQ5H69WrFypUqIDVq1f/p4IiIyOjsq7Cey0jIwOA5u8k+vr6KufVwnTs2BFffvklNm7ciF69epV2FT9IDNNJpBzXmpKSgm3btondssqhd4IgYMGCBWK60uPHjxEdHQ0XFxcYGRnBw8MD06ZNU7sDp7zrUatWLRgbG8PW1hatWrXC8ePHARQMt3n27BlWr14tbkM55vjp06eIjo6Gm5sbjIyMYGdnh5YtW+LkyZPF7tepU6fQunVrWFhYQCaToUWLFirDBcePHw9XV1cAwPDhwyGRSIq9Uztv3jx4eXnB1NQU1tbWqFu3Ln7++WeVPHfv3kWvXr1gb28PIyMjeHl5YeXKlSp5cnNzMXbsWPj5+cHS0hJmZmZo3Lgx9u7dq5JP+VzCzJkzMXfuXFSuXBlGRka4ePEigIKu9s6dO8PW1hYmJiaoVq0avv32W7V6P378WLzbaWlpiZ49e+L58+fFHkPlmOkTJ06gYcOGMDExgbu7OxYvXlzq+/M6ZSxYsACVKlWCqakpgoODcefOHQiCgEmTJsHZ2RkmJiZo164dHj58qLZvO3bsQOPGjWFmZgZzc3OEhobiwoUL4vrIyEgsWLAAgOqQMCWFQoG5c+fCy8sLxsbGsLe3R79+/fDo0SOV7bi5ueHjjz/Gzp07UbduXZiYmGDJkiUAgPj4eHz00UewsrKCTCZDtWrV1IIETfLz8zFp0iTx+Lm5ueGbb75BTk6Oxm0fPHgQ9evXh7GxMSpVqoSffvqp2G0cOXIEO3fuRGRkpEpApDRlyhRUqVIFU6dOxYsXL4osq1atWihfvjxSUlIAAMnJyQCARo0aqeWVSqUoV66cWvq2bdtUeomKOq6ldW4CgFWrVqF58+aws7ODkZERPD09sWjRoiL3t6RiY2Px6aefAgCaNWsmtjHlUMOi9rGk9RIEAd9//z2cnZ1hamqKZs2aqbTzl5X0uBWmJO1SIpFg1apVePbsmdq1Zu/evbhw4YLacdDk999/h6GhIZo0aaKSXpJrxsvPFBV2/YuMjBSDfXd3dzG9uGGOyuehitOwYUO1oLBKlSrw8vLCpUuXVNIzMzNhZ2encv5RXtdKGiCX5LpU2HM8hT378tdff6FNmzawtraGmZkZvL29ERMTU2Q9ND1TdOHCBTRv3hwmJiZwdnbG999/X2ibK+68DQBnz55FZGQkKlWqBGNjYzg4OKBXr15qPXfKZ56uX7/+WtdHpYsXL6JZs2YwNTVFhQoVNA7x/fvvv9G+fXuYmZnBzs4OQ4YM0Xi+VvZU2traQiKRqI1SkMvlyMzMLLI+dnZ28Pb2xh9//FHifdB17CnSMU+ePFF72F4ikaBcuXKoUaMG1qxZgyFDhsDZ2Rlff/01AKBOnTri2GrlcBil58+fo2nTprh79y769euHihUr4vDhwxg9ejRSU1NVJmPo3bs3YmNj0bp1a/Tp0wf5+fk4cOAAjhw5grp162LNmjXo06cP6tevjy+++AIAULlyZQBA//79sWnTJkRFRcHT0xP//PMPDh48iEuXLsHX17fQ/b1w4QIaN24MCwsLjBgxAgYGBliyZAkCAwOxb98++Pv7o0OHDrCyssKQIUMQHh6ONm3aQCaTFVrmsmXL8NVXX6FTp04YPHgwsrOzcfbsWfz111/o1q0bgILhRg0aNIBEIkFUVBRsbW2xY8cO9O7dG5mZmeLDu5mZmVi+fLk4dOjp06dYsWIFQkJCcPToUfHBY6VVq1YhOzsbX3zxBYyMjGBjY4OzZ8+icePGMDAwwBdffAE3NzckJyfjzz//VHvwuHPnznB3d8eUKVNw8uRJLF++HHZ2dpg2bVqh+6v06NEjtGnTBp07d0Z4eDg2bNiAAQMGwNDQULwLVRr7o20Z69atQ25uLr788ks8fPgQ06dPR+fOndG8eXMkJiZi5MiRuH79OubNm4dhw4apfAFYs2YNIiIiEBISgmnTpuH58+dYtGgRPvroI5w6dQpubm7o168f7t27h/j4eKxZs0btuPTr1w+xsbHo2bMnvvrqK6SkpGD+/Pk4deoUDh06pHI3+sqVKwgPD0e/fv3Qt29fVKtWDRcuXMDHH38Mb29vTJw4EUZGRrh+/ToOHTpU7O+kT58+WL16NTp16oSvv/4af/31F6ZMmSL26rzs+vXr6NSpE3r37o2IiAisXLkSkZGR8PPzK7TXBQD+/PNPAFD5u3+Zvr4+unXrhgkTJuDw4cNo0aJFoWU9evQIjx49EoeDKG9GrFu3Do0aNYK+ftGXpLS0NJw6dQoTJ05USdd0XEvz3AQUDIfy8vJC27Ztoa+vjz///BMDBw6EQqHAoEGDiqx3cZo0aYKvvvoKP/74I7755hvUqFEDAMR/C9tHbeo1duxYfP/992jTpg3atGmDkydPIjg4GLm5uSp10ea4FaYk7XLNmjVYunQpjh49Kg4FUl5rfvjhB2RlZYnPQ7x8HF51+PBh1KxZU63XR9trRmHXv1q1aiE3Nxf/+9//MGfOHJQvXx5AwZfVt0UQBKSnp6v9XQYGBmLTpk2YN28ewsLCkJ2djXnz5uHJkycYPHhwseWW9Lqkjfj4eHz88cdwdHTE4MGD4eDggEuXLmHr1q0lqpNSWloamjVrhvz8fIwaNQpmZmZYunSpxmCvJOdtZd1u3LiBnj17wsHBARcuXMDSpUtx4cIFHDlyRG3yhze9PrZq1QodOnRA586dsWnTJowcORK1atUShwq/ePECLVq0wO3bt/HVV1/ByckJa9aswZ49e1TKmjt3Ln766Sf89ttvWLRoEWQyGby9vcX1z58/h4WFBZ4/fw5ra2uEh4dj2rRpGr+3+Pn5fVCTRr11AumEVatWCQA0/hgZGankdXV1FUJDQ9XKACAMGjRIJW3SpEmCmZmZcPXqVZX0UaNGCVKpVLh9+7YgCIKwZ88eAYDw1VdfqZWrUCjE/5uZmQkRERFqeSwtLdW2XRLt27cXDA0NheTkZDHt3r17grm5udCkSRMxLSUlRQAgzJgxo9gy27VrJ3h5eRWZp3fv3oKjo6Pw4MEDlfSuXbsKlpaWwvPnzwVBEIT8/HwhJydHJc+jR48Ee3t7oVevXmr1s7CwEDIyMlTyN2nSRDA3Nxdu3bqlkv7ycR03bpwAQKVMQRCETz75RChXrlwxeywITZs2FQAIs2bNEtNycnIEHx8fwc7OTsjNzS21/dG2DFtbW+Hx48di+ujRowUAQu3atYW8vDwxPTw8XDA0NBSys7MFQRCEp0+fClZWVkLfvn1VtpWWliZYWlqqpA8aNEjQdLo8cOCAAEBYt26dSnpcXJxauqurqwBAiIuLU8k7Z84cAYBw//59tfKLcvr0aQGA0KdPH5X0YcOGCQCEPXv2qG17//79YlpGRoZgZGQkfP3110Vup3379gIA4dGjR4Xm2bx5swBA+PHHH8U0AELv3r2F+/fvCxkZGcJff/0ltGjRQqUdKRQKsW3Z29sL4eHhwoIFC9TastKKFSsEExMT8e/n5X179biW9rnp5W0qhYSECJUqVVJJa9q0qdC0aVNxWdlOV61apXGflDZu3CgAEPbu3au2rrB9LGm9MjIyBENDQyE0NFRln7755hsBgMo5t6THrTDatMuIiAjBzMxMrYymTZsWe45VcnZ2Fjp27KiWXpJrRkREhODq6qqSpun6N2PGDAGAkJKSUqI6vcrLy0ulTRRnzZo1AgBhxYoVKunp6eni35Dyp3z58sLhw4dLVG5Jr0vK7wuv7u/evXtV2mh+fr7g7u4uuLq6qp0fNF1/Xubq6qrS7qKjowUAwl9//SWmZWRkCJaWlip10ea8relv43//+5/aubC0ro8//fSTmJaTkyM4ODiotM25c+cKAIQNGzaIac+ePRM8PDzU/vaVdXr1ujBq1Chh5MiRwvr164X//e9/QkREhABAaNSokcr1Tmny5MkCACE9Pb3Y/SBB4PA5HbNgwQLEx8er/OzYseO1y9u4cSMaN24Ma2trPHjwQPwJCgqCXC7H/v37AQC//vorJBKJ2uQNANTu1mhiZWWFv/76C/fu3Stx3eRyOXbt2oX27dujUqVKYrqjoyO6deuGgwcPFtv9XFhd/v77bxw7dkzjekEQ8OuvvyIsLAyCIKgcl5CQEDx58kQcwiGVSsWhEwqFAg8fPkR+fj7q1q2rcWhgx44dVe5Q3r9/H/v370evXr1QsWJFlbyajmv//v1Vlhs3box//vmnRMdBX18f/fr1E5cNDQ3Rr18/ZGRk4MSJE6WyP69TxqeffqrykKm/vz8A4LPPPlPpefD390dubi7u3r0LoOAu4uPHjxEeHq7yO5JKpfD391cbrqfJxo0bYWlpiZYtW6qU4efnB5lMplaGu7s7QkJCVNKUY8b/+OOPEg9PAgqerQEKnkl4mfIO96szUnl6eqJx48bisq2tLapVq4YbN24UuZ2nT58CQJEP6irXKfMqrVixAra2trCzs4O/vz8OHTqEoUOHinekJRIJdu7cie+//x7W1tb43//+h0GDBsHV1RVdunRRe6Zo+/btaNasmdrdY03HtbTPTS9vU9nj3rRpU9y4cQNPnjwp9NiUFk37WNJ6JSQkiL2pL++Tpp6Bkh63wmjbLt/UP//8A2tra7X017lmvA8uX76MQYMGISAgABERESrrTE1NUa1aNURERGDjxo1YuXIlHB0d0aFDB1y/fr3IcrW5LpXUqVOnkJKSgujoaLVnX0pyXX/Z9u3b0aBBA9SvX19Ms7W1Rffu3VXyaXPefvlvIzs7Gw8ePECDBg0AQOO+vsn1USaTqTznY2hoiPr166ucX7dv3w5HR0d06tRJTDM1NRVHxpTElClTMHXqVHTu3Bldu3ZFbGwsfvjhBxw6dAibNm1Sy6/82+DrOEqGw+d0TP369Ut1qulr167h7NmzhQ4lUD4smJycDCcnJ9jY2LzWdqZPn46IiAi4uLjAz88Pbdq0weeff64S7Lzq/v37eP78uTjM5GU1atSAQqHAnTt3ihw6pMnIkSORkJCA+vXrw8PDA8HBwejWrZv4XMT9+/fx+PFjLF26VJxG81XK4wIAq1evxqxZs3D58mXk5eWJ6ZpmCXw1TXnCLek7El4NnJQnzEePHsHCwqLIzzo5OcHMzEwlrWrVqgAKnu9RXmzeZH+UtCnj1X1SBkguLi4a05XP+ly7dg0A0Lx5c411KO54KMt48uQJ7OzsNK5/+fcMaK5/ly5dsHz5cvTp0wejRo1CixYt0KFDB3Tq1KnIZxJu3boFPT09tZmJHBwcYGVlhVu3bqmkv3qcgILf/6vPPr3q5YCnsIfflcHQq8ehXbt2iIqKgkQigbm5uTjt8MuMjIzw7bff4ttvv0Vqair27duHmJgYbNiwAQYGBuKD5nl5eYiPj9c4xaym41ra56ZDhw5h3LhxSEpKUnvO4MmTJ1rPfKatwv5WSlIvZVuoUqWKynpbW1u1gKKkx+3+/fuQy+Viukwmg0wm07pdlgZBw5Twr3PN0NbDhw9Vhh+amJi8UTtIS0tDaGgoLC0tsWnTJkilUpX1n376qThEUqldu3aoUqUKvv32W6xfvx5yuVzt3TQ2NjZ4/PixVtelklA+E/gm7+hRunXrlnhD62WvXr+1OW8/fPgQEyZMwC+//KK2b5puZLzJ9dHZ2VktELS2tsbZs2fF5Vu3bsHDw0Mtn6bvKNoYMmQIvvvuOyQkJKBr164q65R/G9oGqbqKQRG9EYVCgZYtW2LEiBEa1yu/NL+pzp07o3Hjxvjtt9+wa9cuzJgxA9OmTcPmzZvF8brvSo0aNXDlyhVs3boVcXFx+PXXX7Fw4UKMHTsWEyZMEO/2f/bZZ2p3+pSU44PXrl2LyMhItG/fHsOHD4ednR2kUimmTJkiXnBe9qazTb16kVXS9KXidZTG/mhbRmH7VNy+Kn9Pa9asgYODg1q+4p5vUZZhZ2eHdevWaVz/6hdLTftrYmKC/fv3Y+/evdi2bRvi4uKwfv16NG/eHLt27Sp0P5RKerF73d+9p6cnfv/9d5w9e1btYXYl5YX/1S+czs7OCAoKKlH9gIJe3K5du6Jjx47w8vLChg0bEBsbC319fbFnt02bNmqf03RcS/PclJycjBYtWqB69eqYPXs2XFxcYGhoiO3bt2POnDla9fC9Lk37+DbqVdLjVq9ePZUAZ9y4cSoPg7+rL2HlypXTGNi/i2tGhw4dsG/fPnE5IiLitV/S++TJE7Ru3RqPHz/GgQMH4OTkpLL+xo0biIuLUwtobGxs8NFHH4nPIN65c0ctgN67d684tX1JrkuF/e5eDoLLijbn7c6dO+Pw4cMYPnw4fHx8IJPJoFAo0KpVK41/G29yfXzb19aimJiYoFy5chonElL+bSifhaOiMSiiN1K5cmVkZWUV+8WncuXK2LlzJx4+fFjkHdmiLqSOjo4YOHAgBg4ciIyMDPj6+uKHH34o9AJna2sLU1NTXLlyRW3d5cuXoaenp9abUFJmZmbo0qULunTpgtzcXHTo0AE//PADRo8eDVtbW5ibm0Mulxd7XDZt2oRKlSph8+bNKvuuaSiPJsovoefPn3+t/dDGvXv38OzZM5U7/coXDCofbH3T/SmtMkpCOYmHnZ1dsb+nwtpl5cqVkZCQgEaNGr1RwKqnp4cWLVqgRYsWmD17NiZPnoxvv/0We/fuLbRurq6uUCgUuHbtmsqD6Onp6Xj8+LE4icGbCgsLw+TJk/HTTz9pDIrkcjl+/vln2NvbFxo0acvAwADe3t64du0aHjx4AAcHB2zbtg2enp4lfodPaZ6b/vzzT+Tk5GDLli0qd5NLMsSypF4niChpvZRt4dq1ayqB6/3799UCipIet3Xr1qnMNqgs9121S6Xq1auLsxm+SttrRmEK+93MmjVL5fi9GsiUVHZ2NsLCwnD16lUkJCRonBI8PT0dgObAJC8vD/n5+QAKeuTi4+NV1teuXRsWFhYlvi4pe0heHb76ai+f8hx6/vx5rW5+aOLq6ir2Ar3s1et3Sc/bjx49wu7duzFhwgSMHTtWTNe0jXfF1dUV58+fhyAIKm1K03cUbTx9+hQPHjzQ2LubkpKC8uXLv9WJQT4kfKaI3kjnzp2RlJSEnTt3qq17/PixeKLu2LEjBEHAhAkT1PK9fCfFzMxM7UQsl8vVurrt7Ozg5OSkNpXly6RSKYKDg/HHH3+oTC2anp6On3/+GR999FGJhki96tXpPA0NDeHp6QlBEJCXlwepVIqOHTvi119/1RisvDy0QXl36eVj8NdffyEpKalEdbG1tUWTJk2wcuVK3L59W2Vdad+hys/PF6cBBgqm316yZAlsbW3h5+cH4M33p7TKKImQkBBYWFhg8uTJKkP0lF7+PSkDwVfbZufOnSGXyzFp0iS1z+fn52t8z86rNN3dU86wV1T7VvaYvDob2OzZswGg1F5u2qBBAwQHB2PVqlXYunWr2vpvv/0WV69exYgRI0rUu/aya9euqbVboOA4JyUlwdraWryYb9++Xat9Ks1zk6Y2+eTJE6xatarE9SlOYW2sKCWtV1BQEAwMDDBv3jyVvJpmkivpcWvUqBGCgoLEH2VQ9K7apVJAQADOnz+v8rfyuteMwhT2u/Hz81M5Bq/zfiO5XI4uXbogKSkJGzduREBAgMZ8Hh4e0NPTw/r161V+h3///TcOHDiAOnXqAACMjY1V6hQUFARra2utrkvKwOPl58fkcrlaL5Wvry/c3d0xd+5ctWOj7fWnTZs2OHLkCI4ePapSp1d74Ut63tb0twFobvPauH37Ni5fvvxan23Tpg3u3bun8uzP8+fPCx3O+Krs7Gy15zYBYNKkSRAEAa1atVJbd+LEiULbFKljT5GO2bFjh8Y/6IYNG77WWOvhw4djy5Yt+Pjjj8XpfZ89e4Zz585h06ZNuHnzJsqXL49mzZqhR48e+PHHH3Ht2jWx+/rAgQNo1qwZoqKiABRcZBISEjB79mw4OTnB3d0d1apVg7OzMzp16oTatWtDJpMhISEBx44dK/aFfd9//734DpiBAwdCX18fS5YsQU5OjsZ3CJREcHAwHBwc0KhRI9jb2+PSpUuYP38+QkNDxecvpk6dir1798Lf3x99+/aFp6cnHj58iJMnTyIhIUH8Ivzxxx9j8+bN+OSTTxAaGoqUlBQsXrwYnp6eyMrKKlF9fvzxR3z00Ufw9fXFF198AXd3d9y8eRPbtm3D6dOnX2sfNXFycsK0adNw8+ZNVK1aFevXr8fp06exdOlScTrc0tif0iijJCwsLLBo0SL06NEDvr6+6Nq1K2xtbXH79m1s27YNjRo1wvz58wFADPq++uorhISEQCqVomvXrmjatCn69euHKVOm4PTp0wgODoaBgQGuXbuGjRs3IiYmRuWhWk0mTpyI/fv3IzQ0FK6ursjIyMDChQvh7OyMjz76qNDP1a5dGxEREVi6dCkeP36Mpk2b4ujRo1i9ejXat2+PZs2aldqx+umnn9C8eXO0a9cO3bp1Q+PGjZGTk4PNmzcjMTERn332GYYMGaJ1uWfOnEG3bt3QunVrNG7cGDY2Nrh79y5Wr16Ne/fuYe7cuZBKpUhJScGlS5e0ei9QaZ6bgoODYWhoiLCwMPTr1w9ZWVlYtmwZ7OzskJqaqvV+a+Lj4wOpVIpp06bhyZMnMDIyEt8/VJiS1svW1hbDhg3DlClT8PHHH6NNmzY4deoUduzYoTaspqTHrTDvsl0CBc/UTJo0Cfv27UNwcDCAgjvnr3vN0ET59//tt9+ia9euMDAwQFhYmNrzcS/bv3+/GFTcv38fz549w/fffw+gYAp2Za/q119/jS1btiAsLAwPHz5UeVkrAPHhfVtbW/Tq1QvLly8Xnzt8+vQpFi5ciBcvXmD06NHF7kdJr0teXl5o0KABRo8eLfag/vLLL2JArKSnp4dFixYhLCwMPj4+6NmzJxwdHXH58mVcuHBBY2BdmBEjRmDNmjVo1aoVBg8eLE7J7erqqvJcTknP2xYWFmjSpAmmT5+OvLw8VKhQAbt27Sq0V7GkPv/8c+zbt++1bjr27dsX8+fPx+eff44TJ07A0dERa9asgampaYk+n5aWhjp16iA8PFwcDrlz505s374drVq1Qrt27VTyZ2Rk4OzZs2/8ygCd8k7muKMyV9SU3HhlulhtpuQWhIIpMkePHi14eHgIhoaGQvny5YWGDRsKM2fOFKdqFoSC6TtnzJghVK9eXTA0NBRsbW2F1q1bCydOnBDzXL58WWjSpIlgYmIiThWbk5MjDB8+XKhdu7Zgbm4umJmZCbVr1xYWLlxYon0/efKkEBISIshkMsHU1FRo1qyZ2hSm2kzJvWTJEqFJkyZCuXLlBCMjI6Fy5crC8OHDhSdPnqjkS09PFwYNGiS4uLgIBgYGgoODg9CiRQth6dKlYh6FQiFMnjxZcHV1FYyMjIQ6deoIW7duVZsqtrj6nT9/Xvjkk08EKysrwdjYWKhWrZrw3XffiesLm96zsKlXX6WcIvf48eNCQECAYGxsLLi6ugrz589XyVca+/OmZSinjd24caPGfT127Jha/pCQEMHS0lIwNjYWKleuLERGRgrHjx8X8+Tn5wtffvmlYGtrK0gkErXpZZcuXSr4+fkJJiYmgrm5uVCrVi1hxIgRwr1798Q8hf1d7d69W2jXrp3g5OQkGBoaCk5OTkJ4eLjalMia5OXlCRMmTBDc3d0FAwMDwcXFRRg9erQ47Xhx2351+uiiPH36VJgwYYLg5eUlGBsbi+eOl9vZywo7X7wsPT1dmDp1qtC0aVPB0dFR0NfXF6ytrYXmzZsLmzZtEvPNnz9fsLS01DjlbGH7pqxzaZ2btmzZInh7ewvGxsaCm5ubMG3aNGHlypVqfz+vOyW3IAjCsmXLhEqVKglSqVRlit6i9rGk9ZLL5cKECRMER0dHwcTERAgMDBTOnz+vNjWyNsetMCVtl6UxJbcgCIK3t7fQu3dvcbmk14ySTsktCAVTlVeoUEHQ09Mr0TlTec7V9DNu3DiVfS3q2vyyvLw8Yd68eYKPj48gk8kEmUwmNGvWTGWa8+KU5LokCIKQnJwsBAUFCUZGRoK9vb3wzTffCPHx8RqnjT948KDQsmVL8Vh7e3sL8+bNUzsWL9PU7s6ePSs0bdpUMDY2FipUqCBMmjRJWLFiRaHTgxd33v7777/F66KlpaXw6aefCvfu3VP7HWhzfVT+vl5WWHvV1L5u3boltG3bVjA1NRXKly8vDB48WHyFQ3FTcj969Ej47LPPBA8PD8HU1FQwMjISvLy8hMmTJ2v8u1y0aJFgamoqZGZmqq0jzSSC8A6eAiOi/7TAwEA8ePDgnTy7RP8Nd+/eRcOGDZGfn4+kpCSNs9uVFuULlTds2PDWtkH/XWvWrMGgQYNw+/btQmdIJNI1derUQWBgIObMmVPWVfnP4DNFRESktQoVKiAuLg7Z2dlo3bp1sVN7v4nAwMDXGp5HuqF79+6oWLEiFixYUNZVIXovxMXF4dq1ayUaVkn/Yk8RERWLPUVERET0IWNPERERERER6TT2FBERERERkU5jTxEREREREek0BkVERERERKTTGBQRUZkYP348JBIJHjx4UNZVodcQGRkJmUxW1tWg98DNmzchkUgQGxsrpin/vomI/isYFBGRzvvhhx/Qtm1b2NvbQyKRYPz48YXmvXv3Ljp37gwrKytYWFigXbt2uHHjxrurLL1VmzdvRpcuXVCpUiWYmpqiWrVq+Prrr/H48WO1vG5ubpBIJGo//fv311h2QkICmjdvDktLS5ibm8PPzw/r169/y3tE2kpOTka3bt1gZ2cHExMTVKlSBd9++22h+fPy8uDp6QmJRIKZM2e+w5oSUWnSL+sKEBGVtTFjxsDBwQF16tTBzp07C82XlZWFZs2a4cmTJ/jmm29gYGCAOXPmoGnTpjh9+jTKlSv3DmtNb8MXX3wBJycnfPbZZ6hYsSLOnTuH+fPnY/v27Th58iRMTExU8vv4+ODrr79WSatatapauatWrULv3r3RsmVLTJ48GVKpFFeuXMGdO3fe6v6UlTFjxmDUqFFlXQ2tnT59GoGBgahQoQK+/vprlCtXDrdv3y7y9zRv3jzcvn37HdaSiN4GBkVEpPNSUlLg5uaGBw8ewNbWttB8CxcuxLVr13D06FHUq1cPANC6dWvUrFkTs2bNwuTJk99VlUtddnY2DA0Noaen2wMINm3ahMDAQJU0Pz8/REREYN26dejTp4/KugoVKuCzzz4rssybN29i0KBB+PLLLxETE1PaVX4v6evrQ1//v/UVQ6FQoEePHqhevTr27t2rFgBrkpGRgYkTJ2LkyJEYO3bsO6glEb0tun31I6L3yq1bt+Dh4YGaNWsiPT39nW3Xzc2tRPk2bdqEevXqiQERAFSvXh0tWrTAhg0bXmvbymdz7t69i/bt20Mmk8HW1hbDhg2DXC7XqqzAwEDUrFkTJ06cQMOGDWFiYgJ3d3csXrxYJV9iYiIkEgl++eUXjBkzBhUqVICpqSkyMzMBABs3boSfnx9MTExQvnx5fPbZZ7h7967Gbd64cQMhISEwMzODk5MTJk6ciFff9KBQKDB37lx4eXnB2NgY9vb26NevHx49eqSS7/jx4wgJCUH58uXFuvfq1UurY/CmXg2IAOCTTz4BAFy6dEnjZ3Jzc/Hs2bNCy1y8eDHkcjkmTpwIoKDH8U3fhqF8Zufy5cvo3LkzLCwsUK5cOQwePBjZ2dkqeXNycjBkyBDY2trC3Nwcbdu2xd9//13sUFFNHj9+jMjISFhaWsLKygoREREahxZqeqZIIpEgKioKGzduhKenJ0xMTBAQEIBz584BAJYsWQIPDw8YGxsjMDAQN2/e1Kpub2rXrl04f/48xo0bBxMTEzx//rzYv8FRo0ahWrVqxQbGRPT++2/dxiGiD1ZycjKaN28OGxsbxMfHo3z58oXmzcvLw5MnT0pUro2NTan0figUCpw9e1bjl/T69etj165dePr0KczNzbUuWy6XIyQkBP7+/pg5cyYSEhIwa9YsVK5cGQMGDNCqrEePHqFNmzbo3LkzwsPDsWHDBgwYMACGhoZqdZ80aRIMDQ0xbNgw5OTkwNDQELGxsejZsyfq1auHKVOmID09HTExMTh06BBOnToFKysrlXq3atUKDRo0wPTp0xEXF4dx48YhPz9fDAAAoF+/fmK5X331FVJSUjB//nycOnUKhw4dgoGBATIyMhAcHAxbW1uMGjUKVlZWuHnzJjZv3lzsPmdlZakFApoYGBjA0tKy5Afz/6WlpQGAxja5Z88emJqaQi6Xw9XVFUOGDMHgwYNV8iQkJKB69erYvn07hg8fjrt378La2hqDBg3ChAkT3qh9du7cGW5ubpgyZQqOHDmCH3/8EY8ePcJPP/0k5unTpw/Wrl2Lbt26oWHDhtizZw9CQ0O13pYgCGjXrh0OHjyI/v37o0aNGvjtt98QERFR4jIOHDiALVu2YNCgQQCAKVOm4OOPP8aIESOwcOFCDBw4EI8ePcL06dPRq1cv7Nmzp8jySvNckJCQAAAwMjJC3bp1ceLECRgaGuKTTz7BwoULYWNjo5L/6NGjWL16NQ4ePMhJJYg+BAIRURkYN26cAEC4f/++cOnSJcHJyUmoV6+e8PDhw2I/u3fvXgFAiX5SUlJKXKf79+8LAIRx48YVum7ixIlq6xYsWCAAEC5fvlzibSlFRERoLLdOnTqCn5+fVmU1bdpUACDMmjVLTMvJyRF8fHwEOzs7ITc3VxCEf49fpUqVhOfPn4t5c3NzBTs7O6FmzZrCixcvxPStW7cKAISxY8eq1fvLL78U0xQKhRAaGioYGhoK9+/fFwRBEA4cOCAAENatW6dS17i4OJX03377TQAgHDt2TKt9frkuxf00bdpU67IFQRB69+4tSKVS4erVqyrpYWFhwrRp04Tff/9dWLFihdC4cWMBgDBixAiVfBYWFoK1tbVgZGQkfPfdd8KmTZuEbt26CQCEUaNGvVadlH8/bdu2VUkfOHCgAEA4c+aMIAiCcPr0aQGAMHDgQJV8yu1rauuF+f333wUAwvTp08W0/Px8cb9XrVqlVr+XARCMjIxU/iaXLFkiABAcHByEzMxMMX306NEl+vstzXNB27ZtBQBCuXLlhO7duwubNm0SvvvuO0FfX19o2LChoFAoxLwKhUKoX7++EB4eLgiCIKSkpAgAhBkzZhS5DSJ6f7GniIjK1Pnz59GlSxd4eHhgx44dsLCwKPYztWvXRnx8fInKd3BweNMqAgBevHgBoOAu8quMjY1V8ryOV2csa9y4MdasWaN1Ofr6+ujXr5+4bGhoiH79+mHAgAE4ceIEGjRoIK6LiIhQeW7i+PHjyMjIwPjx48V9AoDQ0FBUr14d27Ztw4QJE1S2FxUVJf5fOTxq27ZtSEhIQNeuXbFx40ZYWlqiZcuWKtOv+/n5QSaTYe/evejWrZvYA7V161bUrl0bBgYGJd7nESNGlGj4krW1dYnLVPr555+xYsUKjBgxAlWqVFFZt2XLFpXlnj17onXr1pg9eza+/PJLODs7AyjoyVIoFJg6dSpGjhwJAOjYsSMePnyImJgYfPPNN6/VwwhA7HFR+vLLL7Fw4UJs374d3t7e2L59OwDgq6++UskXHR2Nn3/+Wattbd++Hfr6+iq9l1KpFF9++SUOHDhQojJatGihMlzV398fQMHxePkYKNNv3LhR5PDW0jwXZGVlAQDq1auHtWvXivUyNTXF6NGjsXv3bgQFBQEAYmNjce7cOWzatKlE2yai9x+DIiIqU2FhYbC3t8fOnTtL/N4ba2tr8cvJu6IMHnJyctTWKYduleTBbE2MjY3VJniwtrZWe+amJJycnGBmZqaSppwN7ebNmypBkbu7u0q+W7duAQCqVaumVm716tVx8OBBlTQ9PT1UqlSp0G0BwLVr1/DkyRPY2dlprG9GRgYAoGnTpujYsSMmTJiAOXPmIDAwEO3bt0e3bt00BqIv8/T0hKenZ5F5XseBAwfQu3dvhISE4Icffig2v0QiwZAhQ7Bz504kJiaKgZqJiQmePXuG8PBwlfzh4eGIi4vDqVOn0KRJk9eq46uBWuXKlaGnpyce/1u3bkFPTw+VK1dWyafpd1ycW7duwdHRUe3vVJuyKlasqLKsHM7o4uKiMb24v4HSPBco/35f/T1169YNo0ePxuHDhxEUFITMzEyMHj0aw4cPV6s3Ef13MSgiojLVsWNHrF69GuvWrVPp4ShKbm4uHj58WKK8tra2kEqlb1JFAAXPIxgZGSE1NVVtnTLNycnptcoujfq9jtcN4rShUChgZ2eHdevWaVyvDAYlEgk2bdqEI0eO4M8//8TOnTvRq1cvzJo1C0eOHCkyYH7y5EmJeukMDQ3VngspzJkzZ9C2bVvUrFkTmzZtKvFMasovyS+3TycnJ1y7dg329vYqeZWB4usEv4V5359tKaytF5YuFDMhRWmeC5R/v8X9nmbOnInc3Fx06dJFDD7//vtvMc/Nmzfh5OQEQ0PDEtWLiN4PDIqIqEzNmDED+vr6GDhwIMzNzdGtW7diP3P48GE0a9asROUrp9t+U3p6eqhVqxaOHz+utu6vv/5CpUqVXnsIVGm6d+8enj17ptJbdPXqVQDFz7Ln6uoKALhy5QqaN2+usu7KlSvieiWFQoEbN26ovJfn1W1VrlwZCQkJaNSoUYmCsAYNGqBBgwb44Ycf8PPPP6N79+745Zdf1KbCftngwYOxevXqYstu2rQpEhMTi82XnJyMVq1awc7ODtu3by9xDyYA8UW+L/f8+fn54dq1a7h7965Kz9q9e/fU8mrr2rVrKj1+169fh0KhEI+/q6srFAoFkpOTVXp0rly5ovW2XF1dsXv3bmRlZakck9cpq7SU5rnAz88Py5YtU5tp8dXf0+3bt/Ho0SN4eXmplTF58mRMnjwZp06dgo+PT8l2gojeCwyKiKhMSSQSLF26FE+fPkVERARkMhnatm1b5GfK4pkiAOjUqRNGjRqF48ePo27dugAKvhDu2bMHw4YNK7XtvIn8/HwsWbIEQ4cOBVBwJ33JkiWwtbWFn59fkZ+tW7cu7OzssHjxYvTq1UsctrZjxw5cunRJ43tY5s+fjx9//BFAwV39+fPnw8DAAC1atABQMDvawoULMWnSJLX3OOXn5yMrKwtWVlZ49OgRrKysVHo6lF8qNQ1ZfFlpPlOUlpaG4OBg6OnpYefOnYUGLA8fPoSlpaVKz0NeXh6mTp0KQ0NDlS/qXbp0wS+//IIVK1aIw/AUCgVWrVoFGxubYn8vRVmwYAGCg4PF5Xnz5gEoeH+W8t9vvvkGP/74IxYsWCDmmzt3rtbbatOmDZYuXYpFixZh+PDhAApmIFRusyyU5rmgXbt2GDx4MFatWoXIyEhxprrly5cDAFq2bAmg4Pms9u3bq3w2IyMD/fr1Q2RkJNq1a6c2NJWI3n8MioiozOnp6WHt2rVo3749OnfujO3bt6v1VLystJ8pWrNmDW7duoXnz58DAPbv34/vv/8eANCjRw+xh2TgwIFYtmwZQkNDMWzYMBgYGGD27Nmwt7fH119/rVJmYGAg9u3b98bvo9GWk5MTpk2bhps3b6Jq1apYv349Tp8+jaVLlxY7eYGBgQGmTZuGnj17omnTpggPDxen5HZzc8OQIUNU8hsbGyMuLg4RERHw9/fHjh07sG3bNnzzzTdiMNG0aVP069cPU6ZMwenTpxEcHAwDAwNcu3YNGzduRExMDDp16oTVq1dj4cKF+OSTT1C5cmU8ffoUy5Ytg4WFBdq0aVNkvUvzmaJWrVrhxo0bGDFiBA4ePKjyHJW9vb34xXjLli34/vvv0alTJ7i7u+Phw4f4+eefcf78eUyePFnlC3i7du3QokULTJkyBQ8ePEDt2rXx+++/4+DBg1iyZInKM1ORkZFYvXp1iXs4U1JS0LZtW7Rq1QpJSUni1Nu1a9cGUBBYhoeHY+HChXjy5AkaNmyI3bt34/r161ofm7CwMDRq1AijRo3CzZs34enpic2bN5d4Suy3oTTPBQ4ODvj2228xduxYtGrVCu3bt8eZM2ewbNkyhIeHi+8n8/X1ha+vr8pnlcPovLy81AImIvqPKOPZ74hIR708JbfS8+fPhaZNmwoymUw4cuTIO6uLciprTT979+5VyXvnzh2hU6dOgoWFhSCTyYSPP/5YuHbtmlqZfn5+goODQ7HbjoiIEMzMzNTSNU1pXJL98PLyEo4fPy4EBAQIxsbGgqurqzB//nyVfMppjDdu3KixnPXr1wt16tQRjIyMBBsbG6F79+7C33//rbHeycnJQnBwsGBqairY29sL48aNE+RyuVqZS5cuFfz8/AQTExPB3NxcqFWrljBixAjh3r17giAIwsmTJ4Xw8HChYsWKgpGRkWBnZyd8/PHHwvHjx7U6Bm+qsHaAV6b0Pn78uBAWFiZUqFBBMDQ0FGQymfDRRx8JGzZs0Fju06dPhcGDBwsODg6CoaGhUKtWLWHt2rVq+Tp27CiYmJgIjx49KrKeyvZx8eJFoVOnToK5ublgbW0tREVFqUynLgiC8OLFC+Grr74SypUrJ5iZmQlhYWHCnTt3tJ6SWxAE4Z9//hF69OghWFhYCJaWlkKPHj2EU6dOlXhK7kGDBqmkFTaVdXFt9G1RKBTCvHnzhKpVqwoGBgaCi4uLMGbMGHE6+8JwSm6i/z6JILzj25hERB+4p0+fwsbGBnPnzlWbMvltCgwMxIMHD3D+/Pl3tk0qXfb29vj8888xY8aMIvONHz8eEyZMwP3794t80XFRJBIJxo0bh/Hjx7/W54mIPiRv/pp3IiJSsX//flSoUAF9+/Yt66rQf8iFCxfw4sUL8V1GRET07vCZIiKiUhYaGorQ0NBSK+/hw4fIzc0tdL1UKn2jGczo/eDl5YXMzMx3vl25XI779+8XmUcmk2k1Cx8R0X8NgyIiovdchw4dsG/fvkLXu7q6ig96E2nrzp07xc6WxmF2RPSh4zNFRETvuRMnThT5gk8TExM0atToHdaIPiTZ2dkqs+xpUqlSJZV3LBERfWgYFBERERERkU7jRAtERERERKTTGBQREREREZFOY1BEREREREQ6jUERERERERHpNAZFRERERESk0/ieolKiUChw7949mJubQyKRlHV1iIiIiIh0miAIePr0KZycnKCnV3RfEIOiUnLv3j24uLiUdTWIiIiIiOgld+7cgbOzc5F5GBSVEnNzcwAFB93CwkJlXV5eHnbt2oXg4GAYGBiURfXoP4ZthrTFNkPaYpshbbHNkLbKus1kZmbCxcVF/J5eFAZFpUQ5ZM7CwkJjUGRqagoLCwueRKhE2GZIW2wzpC22GdIW2wxp631pMyV5tIUTLRARERERkU5jTxEVKjtPjm9/O4/zd5/g+v0sNK9uh2Wf11XLl5Mvx4+7r+H3U/dw/2kObM2NMLhFFXSup/qM1dyEq7j54Bnmdq2Dn/+6jT9O38WFe5nIysnHmXHBsDRRv4Ow53I6YnZfx+XUTBjp68G/UjmNdSAiIiIiel0MiqhQCkGAsYEeIhu5Ycf5tELzDVp3Cg+ycjCtozdcy5ki42kOBEFQyxd/MR0DAisDAF7kydG0mi2aVrPF9LgrGsvdcS4Vozafw/CQamhYuRzkCgFX0p+Wzs4REREREf0/BkVUKFNDffzwSS0AwPGbj5CZnaeWJ/FKBv5K+QcHRjSDlakhAMDFxlQt373HL3AtPQtNq9oCAHp/5A4ASEr+R+O28+UKTPjzIr5pUx1d6lUU06vYF/+gHBEREb0f5HI58vLUvz+QbsjLy4O+vj6ys7Mhl8tLvXwDAwNIpdJSKYtBEb2RhEvp8Ha2xOJ9N/Dbqb9haqiPoBp2+Dq4GowNpCr5/CvZwNy4ZA/Znb+XibTMbEgkErSJOYD7WTnwdLTAN21qoJoDAyMiIqL3mSAISEtLw+PHj8u6KlSGBEGAg4MD7ty589be42llZQUHB4c3Lp9BEb2R2w9f4NjNRzDSl2JJj7p49CwXY34/j0fP8zDz09pivviL6Wjpaa9Fuc8BADEJ1zAmtAacrU2x7MANdF2ahL3DAsVeKSIiInr/KAMiOzs7mJqa8sX2OkqhUCArKwsymazYl6dqSxAEPH/+HBkZGQAAR0fHNyqPQRG9EUEQIAEwt6sPLP6/F+i7j2tgwLqT+L59TRgbSPE0Ow9/3XiIaR29tSoXAAY180DrWgWNfMan3giYsgfbzqWiu79rqe8LERERvTm5XC4GROXKlSvr6lAZUigUyM3NhbGxcakHRQBgYmICAMjIyICdnd0bDaXjlNz0RmzNjeBgaSwGRADgYSeDIACpT7IBAIlX7sPDTgYnKxOtygWAKvYyMc1IXwoXG1Pce/yilGpPREREpU35DJGpqfozxkSlTdnO3vTZNQZF9EbqutogPTMbz3LyxbQb959BTwI4WhoD0H7oHADUqmAJQ3093LifJablyRW4++g5KljxJEtERPS+45A5ehdKq51x+JyOkisEHE15iIyn2bAzN0Z9dxtI9dQb1bX0p8iVK/DkRS6ycvJx4d4TAICXkyUAoJ2PE+btuYbhm85gSFBVPHyWiyk7LqNzXRcYG0iRL1cg8UoGvmjSQKXcjKfZuP80B7f+eQYAuJL2FGZGUlSwMoGVqSHMjQ3Q3b8i5sRfg6OlCSpYm2DpvhsAgNBabzZmlIiIiIjoZQyKdFDc+VRM+POiOLwNKOjVGRfmiVY1VQOOyFXHcPel4WqhPx4EANycGgoAMDPSx5re/hi/5QLC5h+EtakhQms5YlhINQDAXykPYWakj5oVLFXKXXfkNmJ2XxOXOy9JAgDM6OSNT+sWvPT1mzY1oK8nwdANp5Gdp4CPixV+7tsAlqYlm8GOiKg0JN/Pwre/ncP1jCxkZufD3sII7WpXwOCgKjCQFgy4+N/R29h88m9cSSt4l1otZ0sMD6kOHxcrtfK6Lk1Ce58K6Fq/IsZvuYDjtx7ialoWKtvJsGNwY7X8giBg2YEb+N/RO7j76AWszQzQo4EroppXeav7TUSkSxgU6Zi486kYsPYkXn21atqTbAxYexKLPvNVCYwOjWpebJkedjKs7eOvcV38xXS0qGGnlj6kZVUMaVm1yHINpHr4NtQT34Z6FlsHIqK3xUBPDx18nVHTyRIWJvq4lPoUozefhUIQMKJVdQDAkRv/oG1tJ/i2tYaRvhSL9yWjx4q/ED+kKRz+fygxADx+nosTtx5hXrivmNa5rgtO336MS2maX0494c+L2H/tPr5pUwPVHczx+HkeHr/Ifbs7TfSeKOnIlrImCAL69euHTZs24dGjRzh16hSio6Ph4+ODuXPnlnX1Siw2NhbR0dE6OZU6gyIdIlcImPDnRbWACAAEABIUXHxbejqU2gmnqr05fF2tSqUsIqKyULGcKSqW+/dZRmdrUxy5UQHHbj4U02K61lH5zLSO3og7n4ZD1x+go5+zmL7ncga8nCzFyWTGt/UCAPyTdVVjUHQ94ynWHrmFnUOaoLJtwcQzLjalt29E7zNtRraUtbi4OMTGxiIxMRGVKlVC+fLlsXnzZhgYvL+jW9zc3BAdHY3o6GgxrUuXLmjTpk3ZVQrAhQsXMHbsWJw4cQK3bt3CnDlzVOr4tnCiBR1yNOWhyonlVQIKZow7mvKw0Dza6uZfEdUdLEqtPCKisnbzwTPsu3of/u6FTzX8Ik+OPLkCVq8M9024pN3EMwmXMlDRxhR7LmXgo2l70GjqHozcdBaPn7OniD5sypEtr35vUY5siTuf+k7qkZtbsr+15ORkODo6omHDhnBwcIC+vj5sbGxgbv5uXzgvCALy8/OLz1gIExMT2Nmpj/B5l54/f45KlSph6tSpcHBweGfbZVCkQzKeFh4QvU4+IiJd0mHhIVQdswOBMxNRz80GQ4sYAjx1xyXYWxijkUd5MS0nX459V+5r/SLrvx+/wLZzqZjd2QczP62Nc3efYMDak2+0L0Tvs+JGtgAFI1vkCk053kxgYCCioqIQHR2N8uXLIyQkBABw/vx5tG7dGjKZDPb29ujRowcePHgAAIiMjMSXX36J27dvQyKRwM3NTSzr5R4ONzc3TJ48Gb169YK5uTkqVqyIpUuXqmz/zp076Ny5M6ysrGBjY4N27drh5s2bhdY3MTEREokEO3bsgJ+fH4yMjHDw4EEkJyejXbt2sLe3h0wmQ7169ZCQkKCyn7du3cKQIUMgkUjEGdxiY2NhZWWlso1FixahcuXKMDQ0RLVq1bBmzZrXPLolU69ePcyYMQNdu3aFkZHRW93WyxgU6RA7c+PiM2mRj4hIl8zv5ottX36EmK4+2Hs5A0sP3NCYb2Hidfx5JhVLevjB2ODfFwkeTv4H5WRGqGpf8jvHgiAgN1+B2Z1ro767DQIql8P0Tt5IuvEPkl96ZQHRh6QsRra8bPXq1TA0NMShQ4ewePFiPH78GM2bN0edOnVw/PhxxMXFIT09HZ07dwYAxMTEYOLEiXB2dkZqaiqOHTtWaNmzZs1C3bp1cerUKQwcOBADBgzAlStXABS8ZyckJATm5uY4cOAADh06BJlMhlatWhXbYzVq1ChMnToVly5dgre3N7KystCmTRvs3r0bp06dQqtWrRAWFobbt28DADZv3gxnZ2dMnDgRqampSE3V3PP222+/YfDgwfj6669x/vx59OvXDz179sTevXsLrcu6desgk8kgk8lgYWEBZ2dnWFhYiGkymQwHDhwocn/KAp8p0iH13W3gaGmMtCfZGu++SAA4WBY8xEhERKqUL6CuYm8OhSBg9OZz6Nu4ksozmEv3J2NRYjLW9fFHDUfVocMJF9MRVEO7d7bZmhtDX0+CSrb/vsjaw67g//cevxCfMyL6kJT1yJYqVapg+vTp4vL333+POnXqYPLkyWLaypUr4eLigqtXr6Jq1aowNzeHVCotdrhXmzZtMHDgQADAyJEjMWfOHOzduxfVqlXD+vXroVAosHz5crHnZtWqVbCyskJiYiKCg4MLLXfixIlo2bKluGxjY4PatWuLy5MmTcJvv/2GLVu2ICoqCjY2NpBKpTA3Ny+yzjNnzkRkZKRY56FDh+LIkSOYOXMmmjVrpvEzbdu2hb9/wQRcCoUCWVlZkMlk0NP7ty+mQoUKRR6nssCgSIdI9SQYF+aJAWtPQgKoBEbKS/q4MM/3clYXIqL3iUIB5MsFKAQB0v8/gy7el4wFe65jde/68Ha2UskvCAJ2X8rAnC4+Wm2nrqs18hUCbv3zDK7lzAAUvCAbACr8f5BG9KEp65Etfn5+KstnzpzB3r17IZOp34RITk5G1apFz6b7Mm9vb/H/EokEDg4OyMjIELdz/fp1teeQsrOzkZycXGS5devWVVnOysrC+PHjsW3bNqSmpiI/Px8vXrwQe4pK6tKlS/jiiy9U0ho1aoSYmJhCP2Nubi7ug0KhQGZmJiwsLFSCovcRgyId06qmIxZ95qs2m4vDezqbCxHR21bclL+/n7oLfakE1R3MYSiV4uzdx5i+8zI+9nYU31O0KDEZc+KvIqarD5ytTcQ72GaG+jAz0se5u0/wIk+Oem7WKtu++eAZnuXm435WDnLy5OILsqvYmcNQXw8feZRHzQoWGL7pLMZ+7AlBAL774zwaVymv0ntE9CEp65EtZmZmKstZWVkICwvDtGnT1PI6Omr3venV2egkEgkUCoW4HT8/P6xbt07tc7a2tlrVediwYYiPj8fMmTPh4eEBExMTdOrUqcQTR7yJdevWoV+/fkXm2bFjBxo3Vn8vW1liUKSDWtV0REtPh//EvP9ERG9TSab8lepJsHhfMlLuP4OAgh6azwPc0Psjd/Eza4/cQq5cgQHrVCdAGNyiCoa0rIr4i+loVs0W+lLVO6Ujfz2Lv156LkL5guwDI5rBxcYUenoSrIioh3F/XECXJUkwMdRHYDVbjAmtUdqHgui98b6NbPH19cWvv/4KNzc36Ou/va/Ovr6+WL9+Pezs7GBh8WYz9x46dAiRkZH45JNPABQEXK9O2GBoaAi5XF5kOTVq1MChQ4cQERGhUranZ+HvkOTwOSpz2rzgTKonQUDlwqeTJSL60JX0ZdZhtZ0QVtupyLKKe9F1/MV0RDX3UEtf3y+g2HraWxhjcQ+/YvMRfUjep5EtgwYNwrJlyxAeHo4RI0bAxsYG169fxy+//ILly5dDKpUWX0gJdO/eHTNmzEC7du3EiRtu3bqFzZs3Y8SIEXB2di6+kP9XpUoVbN68GWFhYZBIJPjuu+/EHiklNzc37N+/X5zlrXz58mrlDB8+HJ07d0adOnUQFBSEP//8E5s3b1aZye5Vbzp8Ljc3FxcvXhT/f/fuXZw+fRoymQweHurn0dJSpoP7xo8fL04DqPypXr26uD4wMFBtff/+/VXKuH37NkJDQ2Fqago7OzsMHz5cbX72xMRE+Pr6wsjICB4eHoiNjVWry4IFC+Dm5gZjY2P4+/vj6NGjb2Wf35a486n4aNoehC87gsG/nEb4siP4aNqedzaPPxHRf8m7nPI3N1+BVjUdEFitbN/9QfRf06qmIw6ObI7/9W2AmK4++F/fBjg4svk7H+rv5OSEQ4cOQS6XIzg4GLVq1UJ0dDSsrKxK9TkZU1NT7N+/HxUrVkSHDh1Qo0YN9O7dG9nZ2Vr3HM2ePRvW1tZo2LAhwsLCEBISAl9fX5U8EydOxM2bN1G5cuVCh+e1b98eMTExmDlzJry8vLBkyRKsWrUKgYGBr7ubxbp37x7q1KmDOnXqIDU1FTNnzkSdOnXQp0+ft7ZNAJAIglD6k7yX0Pjx47Fp0yaVaFNfX1+MVAMDA1G1alVMnDhRXG9qaio2DLlcDh8fHzg4OGDGjBlITU3F559/jr59+4ozhKSkpKBmzZro378/+vTpg927dyM6Ohrbtm0T555fv349Pv/8cyxevBj+/v6YO3cuNm7ciCtXrpT4BVaZmZmwtLTEkydP1BpuXl4etm/fjjZt2ryVNxsXdrdT2UekvNtJ/x1vu83Qh4dtRjtJyf8gfNmRYvP9r2+DD7ZXnW2GtFXSNpOdnY2UlBS4u7vD2Jiv+dBl72KihaLaW1Hfz19V5sPn9PX1i5wK0NTUtND1u3btwsWLF5GQkAB7e3v4+Phg0qRJGDlyJMaPHw9DQ0MsXrwY7u7umDVrFoCCsZEHDx7EnDlzxKBo9uzZ6Nu3L3r27AkAWLx4MbZt24aVK1di1KhRGredk5ODnJwccTkzMxNAwQkjLy9PJa9y+dX00iBXCBi/5UKhdzslACb8eQGBVcrxmaH/kLfZZujDxDajndTHz0qcLy/vzcb2v6/YZkhbJW0zeXl5EAQBCoVCbcgW6RZl34uyPbwNCoUCgiAgLy9PbSijNue3Mg+Krl27BicnJxgbGyMgIABTpkxBxYoVxfXr1q3D2rVr4eDggLCwMHz33XcwNTUFACQlJaFWrVqwt//3vQ8hISEYMGAALly4gDp16iApKQlBQUEq2wwJCRHfMJybm4sTJ05g9OjR4no9PT0EBQUhKSmp0HpPmTIFEyZMUEvftWuXWL9XxcfHF39AtHTtiQRpmYWPZS14wVkO5q+PQxXLMusUpNf0NtoMfdjYZkrmxhMJgOKfA7hx4TS2/33q7VeoDLHNkLaKazPKG95ZWVnvZLYzev89ffr0rZWdm5uLFy9eYP/+/WqP0Dx//rzE5ZRpUOTv74/Y2FhUq1YNqampmDBhAho3bozz58/D3Nwc3bp1g6urK5ycnHD27FmMHDkSV65cwebNmwEAaWlpKgERAHE5LS2tyDyZmZl48eIFHj16BLlcrjHP5cuXC6376NGjMXToUHE5MzMTLi4uCA4O1jh8Lj4+Hi1btiz1IQp/nk0FLp4rNl8lLx+08eYQuv+Kt9lm6MPENqMduULApln7kZ6ZU8SUv0aI6tLkg+1lZ5shbZW0zWRnZ+POnTuQyWQcPqfjBEHA06dPYW5uLr6QtrRlZ2fDxMQETZo00Th8rqTKNChq3bq1+H9vb2/4+/vD1dUVGzZsQO/evVVeFlWrVi04OjqiRYsWSE5ORuXKlcuiyiIjIyMYGRmppRsYGBR6oihq3etytDIrPtP/5+NF77/nbbQZ+rCxzRQobjZOAwDj23oVM+WvF4yNDN9hrcsG2wxpq7g2I5fLIZFIoKen996/sJPeLuWQOWV7eBv09PQgkUg0tkttzm1lPnzuZVZWVqhatSquX7+ucb1yzvPr16+jcuXKcHBwUJslLj09HQDE55AcHBzEtJfzWFhYwMTEBFKpFFKpVGOeop51el+U9QvOiIjeNyV59xDwfk35S0REZeu9Ct+zsrKQnJxc6NuBT58+DeDftwcHBATg3LlzyMjIEPPEx8fDwsJCfKlUQEAAdu/erVJOfHw8AgIK3g1haGgIPz8/lTwKhQK7d+8W87zPlC84A/69u6lUFi84IyIqS8rZOF8OcoB/3z306msK3pcpf4mIqGyVaU/RsGHDEBYWBldXV9y7dw/jxo2DVCpFeHg4kpOT8fPPP6NNmzYoV64czp49iyFDhqBJkybw9vYGAAQHB8PT0xM9evTA9OnTkZaWhjFjxmDQoEHi0Lb+/ftj/vz5GDFiBHr16oU9e/Zgw4YN2LZtm1iPoUOHIiIiAnXr1kX9+vUxd+5cPHv2TJyN7n1X0rud2XlyfPvbeZy/+wTX72eheXU7LPu8rlp5Ofly/Lj7Gn4/dQ/3n+bA1twIg1tUQed6Lir55iZcxc0HzzC3ax1kPM3GlO2XceDaAzzLyUclWzNENfNA61r8YkFE70Zx7x4qmI3zIlp6OqjcKOLLrImIqEyDor///hvh4eH4559/YGtri48++ghHjhyBra0tsrOzkZCQIAYoLi4u6NixI8aMGSN+XiqVYuvWrRgwYAACAgJgZmaGiIgIlfcaubu7Y9u2bRgyZAhiYmLg7OyM5cuXi9NxA0CXLl1w//59jB07FmlpafDx8UFcXJza5Avvs1Y1HdHS06HIMfQKQYCxgR4iG7lhx/m0QssatO4UHmTlYFpHb7iWM0XG0xxoep1V/MV0DAgseLbr6w1nkPkiD8sj6sLG1BB/nL6LQT+fxJaoj1CzgmXp7zAR0SuOpjxU6yF6WcFsnNk4mvKQQRAREako06Dol19+KXSdi4sL9u3bV2wZrq6u2L59e5F5AgMDcepU0VOqRkVFISoqqtjtvc+Ku9tpaqiPHz6pBQA4fvMRMrPV525PvJKBv1L+wYERzWBlWvCAsYuN+hTj9x6/wLX0LDStWvAG5BO3HuH79jXh42IFAPiyRRWsOJSC83efMCgionci42nhAdHr5CMiKilBENCvXz9s2rQJjx49wqlTpxAdHQ0fHx/MnTu3rKtXYrGxsYiOjsbjx4/Luirv3Hv1TBGVvYRL6fB2tsTifTfgPzkBzWYm4odtF5GdJ1fL51/JBubGBbN6+LlaY+vZVDx+nguFQsCWM/eQk6dAg0q8G0tE74adecmm/i1pPiJ6DyjkQMoB4Nymgn8V8uI/Uwbi4uIQGxuLrVu3IjU1FTVr1sTmzZsxadKksq5aodzc3NQCti5duuDq1atlU6H/t2zZMjRu3BjW1tawtrZGUFCQ2sRqb8N7Nfsclb3bD1/g2M1HMNKXYkmPunj0LBdjfj+PR8/zMPPT2mK++IvpaOn57/DC+d18EfXzSfhMjIe+ngQmBlIs6eEHt/IlmzKciOhNcTZOog/MxS1A3Egg896/aRZOQKtpgGfbd1KF3NxcGBoWPzW/cqKwhg0bimk2Nu/+XCMIAuRyOfT1X+8rvomJCUxMTEq5VtpJTExEeHg4GjZsCGNjY0ybNg3BwcG4cOECKlSo8Na2y54iUiEIAiQA5nb1gY+LFZpVt8N3H9fAryf/FnuLnmbn4a8bDxFU49+gaPauK8jMzse6Pv7YEvURejd2x6CfT+JyWslfmkVE9CY4GyfRB+TiFmDD56oBEQBkphakX9zyVjYbGBiIqKgoREdHo3z58uIz6OfPn0fr1q0hk8lgb2+PHj164MGDBwCAyMhIfPnll7h9+zYkEgnc3NzEsqKjo8Wy3dzcMHnyZPTq1Qvm5uaoWLEili5dqrL9O3fuoHPnzrCysoKNjQ3atWuHmzdvFlrfxMRESCQS7NixA35+fjAyMsLBgweRnJyMdu3awd7eHjKZDPXq1UNCQoLKft66dQtDhgyBRCIRX6waGxsLKysrlW0sWrQIlStXhqGhIapVq4Y1a9a85tEtmXXr1mHgwIHw8fFB9erVsXz5cnFm6LeJQRGpsDU3goOlMSyM/33ZlYedDIIA8QHmxCv34WEng5NVwZ2EW/88w+qkW5jRyRuNPMrD08kC0UFV4e1siZ+SbpXJfhCRblLOxulgqTpEzsHSGIs+8xVn40y+n4WuS5NQ9/t4VB2zA42n78HMnVeQJ1eIn/nf0dv4dPFheI/fCe/xO9F9+RGcvvNY43a7Lk3CL0dvAwDO3HmMbsuOoNb/f67Hir9w8R5vEBGVmEJe0ENU6FySAOJGvbWhdKtXr4ahoSEOHTqExYsX4/Hjx2jevDnq1KmD48ePIy4uDunp6ejcuTMAICYmBhMnToSzszNSU1Nx7NixQsueNWsW6tati1OnTmHgwIEYMGAArly5AgDIy8tDSEgIzM3NceDAARw6dAgymQytWrVCbm5ukXUeNWoUpk6dikuXLsHb2xtZWVlo06YNdu/ejVOnTqFVq1YICwvD7dsF56nNmzfD2dkZEydORGpqKlJTUzWW+9tvv2Hw4MH4+uuvcf78efTr1w89e/bE3r17C63LunXrIJPJIJPJYGFhAWdnZ1hYWIhpMpkMBw4cKHJ/Xvb8+XPk5eW99Z43Dp8jFXVdbbD9XCqe5eTDzKigedy4/wx6koKXHwLqQ+de/H8P0qs3X/UkEo2z1hERvU0lmY3TQE8PHXydUdPJEhYm+riU+hSjN5+FQhAwolV1AMCRG/+gbW0n+La1hpG+FIv3JaPHir8QP6SpStD1+HkuTtx6hHnhvniWk4+IVUcRVMMek9rXhFwhYE78VXy+8iiSRjeHgZT3IomKdeuweg+RCgHIvFuQz71xqW++SpUqmD59urj8/fffo06dOpg8ebKYtnLlSri4uODq1auoWrUqzM3NIZVK4eDgUGTZbdq0wcCBAwEAI0eOxJw5c7B3715Uq1YN69evh0KhwPLly8Wem1WrVsHKygqJiYkIDg4utNyJEyeiZcuW4rKNjQ1q1/73sYdJkybht99+w5YtWxAVFQUbGxtIpVKYm5sXWeeZM2ciMjJSrPPQoUNx5MgRzJw5E82aNdP4mbZt28Lf3x9Awbs/s7KyIJPJoKf37/lPm2FwI0eOhJOTE4KCgkr8mdfBoEjHXEt/ily5Ak9e5CIrJx8X7j0BAHg5FcwQ187HCfP2XMPwTWcwJKgqHj7LxZQdl9G5rguMDaTIlyuQeCUDXzRpIJZZ2VYGt3Km+GbzeXwTWgPWpgbYdSEdB68/wMqIemWyn0Sk24qbjbNiOVNULPfvzJrO1qY4cqMCjt18KKbFdK2j8plpHb0Rdz4Nh64/QEc/ZzF9z+UMeDlZwtbcCGf/fozHz/MwtGVVsTd9cFAVtJp7AHcfveBzlkQlkZVeuvm05Ofnp7J85swZ7N27FzKZTC1vcnIyqlatWuKyle/aBACJRAIHBwdkZGSI27l+/TrMzc1VPpOdnY3k5OQiy61bV/W9k1lZWRg/fjy2bduG1NRU5Ofn48WLF2JPUUldunQJX3zxhUpao0aNEBMTU+hnzM3NxX1QKBTIzMyEhYWFSlBUUlOnTsUvv/yCxMREGBu/3UlyGBTpmMhVx3D38QtxOfTHgwCAm1NDAQBmRvpY09sf47dcQNj8g7A2NURoLUcMC6kGAPgr5SHMjPRVptk2kOphVc/6mLbjMvqsPoZnOXK4ljPFrE9ro1l1u3e4d0REr+fmg2fYd/U+WnkVfsf0RZ4ceXIFrEwNVNITLv3be17JVgZrUwOsP3YHg5p5QCEIWH/sDjzsZHC2LtuHl4n+M2QlfE9kSfNpycxM9eZFVlYWwsLCMG3aNLW8jo7avaTewED1/CGRSKBQKMTt+Pn5Yd26dWqfs7W11arOw4YNQ3x8PGbOnAkPDw+YmJigU6dOxQ7DKw3r1q1Dv379isyzY8cONG5cdC/fzJkzMXXqVCQkJKgEk28LgyIdc2hU82LzeNjJsLaPv8Z18RfT0aKGeqDjXt4Mi3v4afgEEdH7q8PCQzh/LxO5+QqE16+IoS0Lv+M7dccl2FsYo5FHeTEtJ1+OfVfuIzqo4HMyI3388kUAvlhzHPP2XAMAuJU3w0+96kOfQ+eISsa1YcEsc5mp0PxckaRgvWtDDetKn6+vL3799Ve4ubm99qxuJd3O+vXrYWdnBwsLizcq69ChQ4iMjMQnn3wCoCDgenXCBkNDQ8jlRT+XVaNGDRw6dAgREREqZXt6ehb6mdIYPjd9+nT88MMP2Llzp1ov2NvCMzRppaq9OT5r4FrW1SAiKhXzu/li25cfIaarD/ZezsDSAzc05luYeB1/nknFkh5+MDaQiumHk/9BOZkRqtoXDBXJzpNjxK9n4edqjd8GNsKmAQ1Rzd4cvWKPqb3vjYgKoSctmHYbQKFzSbaaWpDvHRg0aBAePnyI8PBwHDt2DMnJydi5cyd69uxZbFChje7du6N8+fJo164dDhw4gJSUFCQmJuKrr77C33//rVVZVapUwebNm3H69GmcOXMG3bp1E3uklNzc3LB//37cvXtXnEnvVcOHD0dsbCwWLVqEa9euYfbs2di8eTOGDRtW6LbNzc3h4eEh/lSqVEllWdlzVZhp06bhu+++w8qVK+Hm5oa0tDSkpaUhKytLq2OgLQZFpJVu/hVR3eHN7l4QEb0vnKxMUMXeHO18KmBk62qYm3AVcoXqneml+5OxKDEZa3rXRw1H1fNfwsV0ldcT/HH6Lu4+eo6ZnWqjtosVfCtaI6ZrHdx5+AK7Lr6d5x+IPkiebYHOPwEWrwxPs3AqSH9H7ykCACcnJxw6dAhyuRzBwcGoVasWoqOjYWVl9VrPyRTG1NQU+/fvR8WKFdGhQwfUqFEDvXv3RnZ2ttY9R7Nnz4a1tTUaNmyIsLAwhISEwNfXVyXPxIkTcfPmTVSuXLnQ4Xnt27dHTEwMZs6cCS8vLyxZsgSrVq1CYGDg6+5msRYtWoTc3Fx06tQJjo6O4s/MmTPf2jYBDp8jIiICACgUQL5cgEIQIP3/u9GL9yVjwZ7rWN27PrydrVTyC4KA3ZcyMKeLj5j2Ilf+/+/8+DefngSQSMDZOIm05dkWqB5aMMtcVnrBM0SuDd9qD1FiYqLGdGXPS2Gio6NV3kmkqSxN7xs6ffq0yrKDgwNWr15dgpoWCAwM1HhucXNzw549e1TSBg0apLLcoEEDnDlzRiUtMjISkZGRKmkDBgzAgAEDSlynN1XUe5neJgZFRESkc34/dRf6UgmqO5jDUCrF2buPMX3nZXzs7ShOm70oMRlz4q8ipqsPnK1NkPG04F1tZob6MDPSx7m7T/AiT456btZiuR9VscXkHZfx3R/nEdnQDQqhoBypngQBlQqfDY+ICqEnfSvTbhO9ikERERHpHKmeBIv3JSPl/jMIACpYmeDzADf0/shdzLP2yC3kyhUYsO6kymcHt6iCIS2rIv5iOppVs1WZQMHDToYVEXURk3ANnyw8DD2JBF5OFljdqz7sLN7udLJERPT6GBQREZHOCavthLDaTkXmKW62zviL6Yhq7qGW3riKLRpXKXr6XCIier9wogUiIiIt5eYr0KqmAwKr8V1sREQfAvYUERERaclQX098NxEREf33saeIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIincagiIiIiIjoDQiCgC+++AI2NjaQSCQ4ffo0AgMDER0dXdZV00psbCysrKzKuhplgkEREREREb2X5Ao5jqUdw/Yb23Es7RjkCnlZV0mjuLg4xMbGYuvWrUhNTUXNmjWxefNmTJo0qayrVig3NzfMnTtXJa1Lly64evVq2VRIg19++QUSiQTt27d/69vilNxERERE9N5JuJWAqUenIv15uphmb2qPUfVHIcg16J3UITc3F4aGhsXmS05OhqOjIxo2bCim2djYvM2qaSQIAuRyOfT1X+8rvomJCUxMTEq5Vq/n5s2bGDZsGBo3bvxOtseeIiIiIiJ6ryTcSsDQxKEqAREAZDzPwNDEoUi4lfBWthsYGIioqChER0ejfPnyCAkJAQCcP38erVu3hkwmg729PXr06IEHDx4AACIjI/Hll1/i9u3bkEgkcHNzE8t6eficm5sbJk+ejF69esHc3BwVK1bE0qVLVbZ/584ddO7cGVZWVrCxsUG7du1w8+bNQuubmJgIiUSCHTt2wM/PD0ZGRjh48CCSk5PRrl072NvbQyaToV69ekhI+PeYBQYG4tatWxgyZAgkEgkkEgkAzcPnFi1ahMqVK8PQ0BDVqlXDmjVrXvPolpxcLkf37t0xYcIEVKpU6a1vD2BQRERERETvEblCjqlHp0KAoLZOmTbt6LS3NpRu9erVMDQ0xKFDh7B48WI8fvwYzZs3R506dXD8+HHExcUhPT0dnTt3BgDExMRg4sSJcHZ2RmpqKo4dO1Zo2bNmzULdunVx6tQpDBw4EAMGDMCVK1cAAHl5eQgJCYG5uTkOHDiAQ4cOQSaToVWrVsjNzS2yzqNGjcLUqVNx6dIleHt7IysrC23atMHu3btx6tQptGrVCmFhYbh9+zYAYPPmzXB2dsbEiRORmpqK1NRUjeX+9ttvGDx4ML7++mucP38e/fr1Q8+ePbF3795C67Ju3TrIZDLIZDJYWFjA2dkZFhYWYppMJsOBAweK3J+JEyfCzs4OvXv3LjJfaeLwOSIiIiJ6b5zMOKnWQ/QyAQLSnqfhZMZJ1HOoV+rbr1KlCqZPny4uf//996hTpw4mT54spq1cuRIuLi64evUqqlatCnNzc0ilUjg4OBRZdps2bTBw4EAAwMiRIzFnzhzs3bsX1apVw/r166FQKLB8+XKx52bVqlWwsrJCYmIigoODCy134sSJaNmypbhsY2OD2rVri8uTJk3Cb7/9hi1btiAqKgo2NjaQSqUwNzcvss4zZ85EZGSkWOehQ4fiyJEjmDlzJpo1a6bxM23btoW/vz8AQKFQICsrCzKZDHp6//bFVKhQodBtHjx4ECtWrMDp06cLzfM2MCgiIiIiovfG/ef3SzWftvz8/FSWz5w5g71790Imk6nlTU5ORtWqVUtctre3t/h/iUQCBwcHZGRkiNu5fv06zM3NVT6TnZ2N5OTkIsutW7euynJWVhbGjx+Pbdu2ITU1Ffn5+Xjx4oXYU1RSly5dwhdffKGS1qhRI8TExBT6GXNzc3EfFAoFMjMzYWFhoRIUFebp06fo0aMHli1bhvLly2tV1zfFoIiIiIiI3hu2pralmk9bZmZmKstZWVkICwvDtGnT1PI6OjpqVbaBgYHKskQigUKhELfj5+eHdevWqX3O1rbofX21zsOGDUN8fDxmzpwJDw8PmJiYoFOnTsUOwysN69atQ79+/YrMs2PHDo0TKCQnJ+PmzZsICwsT05THR19fH1euXEHlypVLt8L/j0EREREREb03fO18YW9qj4znGRqfK5JAAntTe/ja+b6b+vj64tdff4Wbm9trz+pW0u2sX78ednZ2sLCweKOyDh06hMjISHzyyScACgKuVydsMDQ0hFxe9HNZNWrUwKFDhxAREaFStqenZ6GfeZPhc9WrV8e5c+dU0saMGYOnT58iJiYGLi4uRdb3TXCiBSIiIiJ6b0j1pBhVfxSAggDoZcrlkfVHQqonfSf1GTRoEB4+fIjw8HAcO3YMycnJ2LlzJ3r27FlsUKGN7t27o3z58mjXrh0OHDiAlJQUJCYm4quvvsLff/+tVVlVqlTB5s2bcfr0aZw5cwbdunUTe1yU3NzcsH//fty9e1ecSe9Vw4cPR2xsLBYtWoRr165h9uzZ2Lx5M4YNG1bots3NzeHh4SH+VKpUSWVZ2XOlibGxMWrWrKnyY2VlBXNzc9SsWbNE06O/LgZFRERERPReCXINwuzA2bAztVNJtze1x+zA2e/sPUUA4OTkhEOHDkEulyM4OBi1atVCdHQ0rKysSvScTEmZmppi//79qFixIjp06IAaNWqgd+/eyM7O1rrnaPbs2bC2tkbDhg0RFhaGkJAQ+Pqq9qxNnDgRN2/eROXKlQsdnte+fXvExMRg5syZ8PLywpIlS7Bq1SoEBga+7m6+tySCIKj3S5LWMjMzYWlpiSdPnqg13Ly8PGzfvh1t2rRRG0tKpAnbDGmLbYa0xTZD2ippm8nOzkZKSgrc3d1hbGz8RtuUK+Q4mXES95/fh62pLXztfN9ZDxG9OW0nWngdRbW3or6fv4rPFBERERHRe0mqJ30r024TvYrD54iIiIiISKcxKCIiIiIiIp3GoIiIiIiIiHQagyIiIiIiKnWcy4vehdJqZwyKiIiIiKjUKGeme/78eRnXhHSBsp296SyanH2OiIiIiEqNVCqFlZUVMjIyABS8f0cikRTzKfoQKRQK5ObmIjs7u9Sn5BYEAc+fP0dGRgasrKwglb7ZVO0MioiIiIioVDk4OACAGBiRbhIEAS9evICJiclbC4ytrKzE9vYmyjQoGj9+PCZMmKCSVq1aNVy+fBlAwcuYvv76a/zyyy/IyclBSEgIFi5cCHt7ezH/7du3MWDAAOzduxcymQwRERGYMmUK9PX/3bXExEQMHToUFy5cgIuLC8aMGYPIyEiV7S5YsAAzZsxAWloaateujXnz5qF+/fpvb+eJiIiIPlASiQSOjo6ws7NDXl5eWVeHykheXh7279+PJk2avJWXRBsYGLxxD5FSmfcUeXl5ISEhQVx+OZgZMmQItm3bho0bN8LS0hJRUVHo0KEDDh06BACQy+UIDQ2Fg4MDDh8+jNTUVHz++ecwMDDA5MmTAQApKSkIDQ1F//79sW7dOuzevRt9+vSBo6MjQkJCAADr16/H0KFDsXjxYvj7+2Pu3LkICQnBlStXYGdn9w6PBhEREdGHQyqVltqXVvrvkUqlyM/Ph7Gx8VsJikpTmU+0oK+vDwcHB/GnfPnyAIAnT55gxYoVmD17Npo3bw4/Pz+sWrUKhw8fxpEjRwAAu3btwsWLF7F27Vr4+PigdevWmDRpEhYsWIDc3FwAwOLFi+Hu7o5Zs2ahRo0aiIqKQqdOnTBnzhyxDrNnz0bfvn3Rs2dPeHp6YvHixTA1NcXKlSvf/QEhIiIiIqJ3qsx7iq5duwYnJycYGxsjICAAU6ZMQcWKFXHixAnk5eUhKChIzFu9enVUrFgRSUlJaNCgAZKSklCrVi2V4XQhISEYMGAALly4gDp16iApKUmlDGWe6OhoAEBubi5OnDiB0aNHi+v19PQQFBSEpKSkQuudk5ODnJwccTkzMxNAQTfhq93EymV2H1NJsc2QtthmSFtsM6QtthnSVlm3GW22W6ZBkb+/P2JjY1GtWjWkpqZiwoQJaNy4Mc6fP4+0tDQYGhrCyspK5TP29vZIS0sDAKSlpakERMr1ynVF5cnMzMSLFy/w6NEjyOVyjXmUzzZpMmXKFLXnoYCC3itTU1ONn4mPjy+0PCJN2GZIW2wzpC22GdIW2wxpq6zajDbTwpdpUNS6dWvx/97e3vD394erqys2bNgAExOTMqxZ8UaPHo2hQ4eKy5mZmXBxcUFwcDAsLCxU8ubl5SE+Ph4tW7Z878dT0vuBbYa0xTZD2mKbIW2xzZC2yrrNKEdylUSZD597mZWVFapWrYrr16+jZcuWyM3NxePHj1V6i9LT08Vp9xwcHHD06FGVMtLT08V1yn+VaS/nsbCwgImJifgAoKY8RU3vZ2RkBCMjI7V0AwODQn/pRa0j0oRthrTFNkPaYpshbbHNkLbKqs1os80yn2jhZVlZWUhOToajoyP8/PxgYGCA3bt3i+uvXLmC27dvIyAgAAAQEBCAc+fOqcyBHx8fDwsLC3h6eop5Xi5DmUdZhqGhIfz8/FTyKBQK7N69W8xDREREREQfrjINioYNG4Z9+/bh5s2bOHz4MD755BNIpVKEh4fD0tISvXv3xtChQ7F3716cOHECPXv2REBAABo0aAAACA4OhqenJ3r06IEzZ85g586dGDNmDAYNGiT24vTv3x83btzAiBEjcPnyZSxcuBAbNmzAkCFDxHoMHToUy5Ytw+rVq3Hp0iUMGDAAz549Q8+ePcvkuBARERER0btTpsPn/v77b4SHh+Off/6Bra0tPvroIxw5cgS2trYAgDlz5kBPTw8dO3ZUeXmrklQqxdatWzFgwAAEBATAzMwMERERmDhxopjH3d0d27Ztw5AhQxATEwNnZ2csX75cfEcRAHTp0gX379/H2LFjkZaWBh8fH8TFxalNvkBERERERB+eMg2KfvnllyLXGxsbY8GCBViwYEGheVxdXbF9+/YiywkMDMSpU6eKzBMVFYWoqKgi8xARERER0YfnvXqmiIiIiIiI6F1jUERERERERDqNQREREREREek0BkVERERERKTT3quXtxIREb2Pku9n4dvfzuF6RhYys/Nhb2GEdrUrYHBQFRhIC+4v/u/obWw++TeupD0FANRytsTwkOrwcbFSK6/r0iS096mAEC8HDF5/GpdTM/H4eR7KyQzR0tMew0Oqwdy44KWDcedTsfbIbVxMzURuvgJV7GWIDqqKplVt39n+ExF96BgUERERFcNATw8dfJ1R08kSFib6uJT6FKM3n4VCEDCiVXUAwJEb/6BtbSf4trWGkb4Ui/clo8eKvxA/pCkcLI3Fsh4/z8WJW48wL9wXehIJWnraY1hwVdiYGeLWP8/x3R/n8fh5Hn4MrwMA+CvlIT6qUh7DQ6rBwsQAG4/fQZ/Vx/DbwEaoWcGyTI4HEdGHhkERERFRMSqWM0XFcqbisrO1KY7cqIBjNx+KaTFd66h8ZlpHb8SdT8Oh6w/Q0c9ZTN9zOQNeTpawNS94yXiPBq4q5fZo4Iql+2+IaePCvFTKHdGqOuIvpmP3pQwGRUREpYTPFBEREWnp5oNn2Hf1PvzdyxWa50WeHHlyBaxMDVTSEy6lo6Wn5peDp2dmI+58GvzdbQotV6EQ8CwnX61cIiJ6fewpIiIiKqEOCw/h/L2CZ3vC61fE0JZVC807dccl2FsYo5FHeTEtJ1+OfVfuIzpI9XNf/u8U4i+mITtPgaAadpja0bvQcpceuIFnuXKEeju++Q4REREA9hQRERGV2Pxuvtj25UeI6eqDvZczsPTADY35FiZex59nUrGkhx+MDaRi+uHkf1BOZoSq9uYq+b/7uAa2ftkYyz6vi1v/PMf32y5qLPeP03cRk3ANC7r5orzMqPR2jIhIx7GniIiIqIScrEwAAFXszaEQBIzefA59G1eCVE8i5lm6PxmLEpOxro8/ajhaqHw+4WI6gmqoD52zMzeGnTngYSeDlakBPl2chK+aV4Gdxb8TNGw5cw8jfz2Lhd198VGV8mplEBHR62NPERER0WtQKIB8uQCFIIhpi/clY97u61jdqz68na1U8guCgN2XMgp9nujfcgvKy8lXiGl/nL6L4RvP4MeuddC8etGfJyIi7bGniIiIdJpcIeBoykNkPM2Gnbkx6rvbqPT8AMDvp+5CXypBdQdzGEqlOHv3MabvvIyPvR3F9xQtSkzGnPiriOnqA2drE2Q8zQYAmBnqw8xIH+fuPsGLPDnquVmL5e69nIH7WTmo7WwFU0MprmU8xeTtl1HX1RouNgWz3f1x+i6+3nAG48I84VPRSizX2EAKC2NOtkBEVBoYFBERkc6KO5+KCX9eROqTbDHN0dIY48I80armvxMZSPUkWLwvGSn3n0EAUMHKBJ8HuKH3R+5inrVHbiFXrsCAdSdVtjG4RRUMaVkV8RfT0ayaLfSl/w7SMDLQwy9Hb2PS1ovIzVfAycoEIV4OGBBYWczz81+3ka8Q8N0fF/DdHxfE9I6+zpjVuXZpHg4iIp3FoIiIiHRS3PlUDFh7EsIr6WlPsjFg7Uks+sxXDIzCajshrLZTkeUdGtW8yPXxF9MR1dxDJa1h5fLYPLDo54PW9wsocj0REb05PlNEREQ6R64QMOHPi2oBEQAxbcKfFyFXaMqhvdx8BVrVdEBgNbtSKY+IiEoXgyIiItI5R1MeqgyZe5UAIPVJNo6mPCyV7Rnq6yE6qCpkRhygQUT0PmJQREREOkc5WUFp5SMiov82BkVERKRz7MyNi8+kRT4iIvpvY1BEREQ6p767DRwtjSEpZL0EBbPQ1Xe3eZfVIiKiMsKgiIiIdI5UT4JxYZ4AoBYYKZfHhXmqva+IiIg+TAyKiIhIJ7Wq6YhFn/nCwVJ1iJyDpbHKdNxERPTh4zQ4RESks1rVdERLTwccTXmIjKfZsDMvGDLHHiIiIt3CoIiIiHSaVE+CgMrlyroaRERUhjh8joiIiIiIdBqDIiIiIiIi0mkMioiIiIiISKcxKCIiIiIiIp3GoIiIiIiIiHQagyIiIiIiItJpDIqIiIiIiEinMSgiIiIiIiKdxqCIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIincagiIiIiIiIdBqDIiIiIiIi0mkMioiIiIiISKcxKCIiIiIiIp3GoIiIiIiIiHTaexMUTZ06FRKJBNHR0WJaYGAgJBKJyk///v1VPnf79m2EhobC1NQUdnZ2GD58OPLz81XyJCYmwtfXF0ZGRvDw8EBsbKza9hcsWAA3NzcYGxvD398fR48efRu7SURERERE75n3Iig6duwYlixZAm9vb7V1ffv2RWpqqvgzffp0cZ1cLkdoaChyc3Nx+PBhrF69GrGxsRg7dqyYJyUlBaGhoWjWrBlOnz6N6Oho9OnTBzt37hTzrF+/HkOHDsW4ceNw8uRJ1K5dGyEhIcjIyHi7O05ERERERGWuzIOirKwsdO/eHcuWLYO1tbXaelNTUzg4OIg/FhYW4rpdu3bh4sWLWLt2LXx8fNC6dWtMmjQJCxYsQG5uLgBg8eLFcHd3x6xZs1CjRg1ERUWhU6dOmDNnjljO7Nmz0bdvX/Ts2ROenp5YvHgxTE1NsXLlyrd/AIiIiIiIqEzpl3UFBg0ahNDQUAQFBeH7779XW79u3TqsXbsWDg4OCAsLw3fffQdTU1MAQFJSEmrVqgV7e3sxf0hICAYMGIALFy6gTp06SEpKQlBQkEqZISEh4jC93NxcnDhxAqNHjxbX6+npISgoCElJSYXWOycnBzk5OeJyZmYmACAvLw95eXkqeZXLr6YTFYZthrTFNkPaYpshbbHNkLbKus1os90yDYp++eUXnDx5EseOHdO4vlu3bnB1dYWTkxPOnj2LkSNH4sqVK9i8eTMAIC0tTSUgAiAup6WlFZknMzMTL168wKNHjyCXyzXmuXz5cqF1nzJlCiZMmKCWvmvXLjFoe1V8fHyh5RFpwjZD2mKbIW2xzZC22GZIW2XVZp4/f17ivGUWFN25cweDBw9GfHw8jI2NNeb54osvxP/XqlULjo6OaNGiBZKTk1G5cuV3VVWNRo8ejaFDh4rLmZmZcHFxQXBwsMoQP6AgSo2Pj0fLli1hYGDwrqtK/0FsM6QtthnSFtsMaYtthrRV1m1GOZKrJMosKDpx4gQyMjLg6+srpsnlcuzfvx/z589HTk4OpFKpymf8/f0BANevX0flypXh4OCgNktceno6AMDBwUH8V5n2ch4LCwuYmJhAKpVCKpVqzKMsQxMjIyMYGRmppRsYGBT6Sy9qHZEmbDOkLbYZ0hbbDGmLbYa0VVZtRpttltlECy1atMC5c+dw+vRp8adu3bro3r07Tp8+rRYQAcDp06cBAI6OjgCAgIAAnDt3TmWWuPj4eFhYWMDT01PMs3v3bpVy4uPjERAQAAAwNDSEn5+fSh6FQoHdu3eLeYiIiIiI6MNVZj1F5ubmqFmzpkqamZkZypUrh5o1ayI5ORk///wz2rRpg3LlyuHs2bMYMmQImjRpIk7dHRwcDE9PT/To0QPTp09HWloaxowZg0GDBom9OP3798f8+fMxYsQI9OrVC3v27MGGDRuwbds2cbtDhw5FREQE6tati/r162Pu3Ll49uwZevbs+e4OCBERERERlYkyn32uMIaGhkhISBADFBcXF3Ts2BFjxowR80ilUmzduhUDBgxAQEAAzMzMEBERgYkTJ4p53N3dsW3bNgwZMgQxMTFwdnbG8uXLERISIubp0qUL7t+/j7FjxyItLQ0+Pj6Ii4tTm3yBiIiIiIg+PO9VUJSYmCj+38XFBfv27Sv2M66urti+fXuReQIDA3Hq1Kki80RFRSEqKqpE9SQiIiIiog9Hmb+8lYiIiIiIqCwxKCIiIiIiIp3GoIiIiIiIiHQagyIiIiIiItJpDIqIiIiIiEinMSgiIiIiIiKdxqCIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIincagiIiIiIiIdBqDIiIiIiIi0mkMioiIiIiISKcxKCIiIiIiIp3GoIiIiIiIiHQagyIiIiIiItJpDIqIiIiIiEinMSgiIiIiIiKdxqCIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIincagiIiIiIiIdBqDIiIiIiIi0mkMioiIiIiISKcxKCIiIiIiIp3GoIiIiIiIiHQagyIiIiIiItJpDIqIiIiIiEinMSgiIiIiIiKdxqCIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIincagiIiIiIiIdBqDIiIiIiIi0mmlEhTdunULFy9ehEKhKI3iiIiIiIiI3hmtgqKVK1di9uzZKmlffPEFKlWqhFq1aqFmzZq4c+dOqVaQiIiIiIjobdIqKFq6dCmsra3F5bi4OKxatQo//fQTjh07BisrK0yYMKHUK0lERERERPS26GuT+dq1a6hbt664/Mcff6Bdu3bo3r07AGDy5Mno2bNn6daQiIiIiIjoLdKqp+jFixewsLAQlw8fPowmTZqIy5UqVUJaWlrp1Y6IiIiIiOgt0yoocnV1xYkTJwAADx48wIULF9CoUSNxfVpaGiwtLV+rIlOnToVEIkF0dLSYlp2djUGDBqFcuXKQyWTo2LEj0tPTVT53+/ZthIaGwtTUFHZ2dhg+fDjy8/NV8iQmJsLX1xdGRkbw8PBAbGys2vYXLFgANzc3GBsbw9/fH0ePHn2t/SAiIiIiov8WrYKiiIgIDBo0CJMmTcKnn36K6tWrw8/PT1x/+PBh1KxZU+tKHDt2DEuWLIG3t7dK+pAhQ/Dnn39i48aN2LdvH+7du4cOHTqI6+VyOUJDQ5Gbm4vDhw9j9erViI2NxdixY8U8KSkpCA0NRbNmzXD69GlER0ejT58+2Llzp5hn/fr1GDp0KMaNG4eTJ0+idu3aCAkJQUZGhtb7QkRERERE/y1aBUUjRoxA3759sXnzZhgbG2Pjxo0q6w8dOoTw8HCtKpCVlYXu3btj2bJlKpM4PHnyBCtWrMDs2bPRvHlz+Pn5YdWqVTh8+DCOHDkCANi1axcuXryItWvXwsfHB61bt8akSZOwYMEC5ObmAgAWL14Md3d3zJo1CzVq1EBUVBQ6deqEOXPmiNuaPXs2+vbti549e8LT0xOLFy+GqakpVq5cqdW+EBERERHRf49WEy3o6elh4sSJmDhxosb1rwZJJTFo0CCEhoYiKCgI33//vZh+4sQJ5OXlISgoSEyrXr06KlasiKSkJDRo0ABJSUmoVasW7O3txTwhISEYMGAALly4gDp16iApKUmlDGUe5TC93NxcnDhxAqNHj1bZz6CgICQlJRVa75ycHOTk5IjLmZmZAIC8vDzk5eWp5FUuv5pOVBi2GdIW2wxpi22GtMU2Q9oq6zajzXa1CoqAgqFmW7ZsQW5uLlq0aIH+/ftrW4Tol19+wcmTJ3Hs2DG1dWlpaTA0NISVlZVKur29vTiZQ1pamkpApFyvXFdUnszMTLx48QKPHj2CXC7XmOfy5cuF1n3KlCkapx/ftWsXTE1NNX4mPj6+0PKINGGbIW2xzZC22GZIW2wzpK2yajPPnz8vcV6tgqJFixZh0KBBqFKlCkxMTLB582YkJydjxowZWlfyzp07GDx4MOLj42FsbKz158va6NGjMXToUHE5MzMTLi4uCA4OVpmhDyiIUuPj49GyZUsYGBi866rSfxDbDGmLbYa0xTZD2mKbIW2VdZtRjuQqCa2Covnz52PcuHEYN24cAGDt2rXo16/fawVFJ06cQEZGBnx9fcU0uVyO/fv3Y/78+di5cydyc3Px+PFjld6i9PR0ODg4AAAcHBzUZolTzk73cp5XZ6xLT0+HhYUFTExMIJVKIZVKNeZRlqGJkZERjIyM1NINDAwK/aUXtY5IE7YZ0hbbDGmLbYa0xTZD2iqrNqPNNrWaaOHGjRuIiIgQl7t164b8/HykpqZqUwwAoEWLFjh37hxOnz4t/tStWxfdu3cX/29gYIDdu3eLn7ly5Qpu376NgIAAAEBAQADOnTunMktcfHw8LCws4OnpKeZ5uQxlHmUZhoaG8PPzU8mjUCiwe/duMQ8REREREX24tOopysnJgZmZmbisp6cHQ0NDvHjxQusNm5ubq03fbWZmhnLlyonpvXv3xtChQ2FjYwMLCwt8+eWXCAgIQIMGDQAAwcHB8PT0RI8ePTB9+nSkpaVhzJgxGDRokNiL079/f8yfPx8jRoxAr169sGfPHmzYsAHbtm0Ttzt06FBERESgbt26qF+/PubOnYtnz56hZ8+eWu8XERERERH9t2g90cJ3332nMpFAbm4ufvjhB5WXts6ePbtUKjdnzhzo6emhY8eOyMnJQUhICBYuXCiul0ql2Lp1KwYMGICAgACYmZkhIiJCZXY8d3d3bNu2DUOGDEFMTAycnZ2xfPlyhISEiHm6dOmC+/fvY+zYsUhLS4OPjw/i4uLUJl8gIiIiIqIPj1ZBUZMmTXDlyhWVtIYNG+LGjRviskQiee3KJCYmqiwbGxtjwYIFWLBgQaGfcXV1xfbt24ssNzAwEKdOnSoyT1RUFKKiokpcVyIiIiIi+jBoFRS9GrQ8ePAAhoaGarOtERERERER/VdoNdECADx+/BiDBg1C+fLlYW9vD2trazg4OGD06NFazQVORERERET0PtCqp+jhw4cICAjA3bt30b17d9SoUQMAcPHiRcybNw/x8fE4ePAgzp49iyNHjuCrr756K5UmIiIiIiIqLVoFRRMnToShoSGSk5PVJiGYOHEigoOD0aNHD+zatQs//vhjqVaUiIiIiIjobdAqKPr999+xZMkSjbOyOTg4YPr06WjTpg3GjRun8j4jIiIiIiKi95VWzxSlpqbCy8ur0PU1a9aEnp4exo0b98YVIyIiIiIiehe0CorKly+PmzdvFro+JSUFdnZ2b1onIiIiIiKid0aroCgkJATffvstcnNz1dbl5OTgu+++Q6tWrUqtckRERERERG+b1hMt1K1bF1WqVMGgQYNQvXp1CIKAS5cuYeHChcjJycFPP/30tupKRERERERU6rQKipydnZGUlISBAwdi9OjREAQBACCRSNCyZUvMnz8fFStWfCsVJSIiIiIiehu0CooAwN3dHTt27MCjR49w7do1AICHhwdsbGxKvXJERERERERvm9ZBkZK1tTXq169fmnUhIiIiIiJ657SaaIGIiIiIiOhDw6CIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIincagiIiIiIiIdBqDIiIiIiIi0mkMioiIiIiISKcxKCIiIiIiIp3GoIiIiIiIiHQagyIiIiIiItJpDIqIiIiIiEinMSgiIiIiIiKdxqCIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIincagiIiIiIiIdBqDIiIiIiIi0mkMioiIiIiISKcxKCIiIiIiIp3GoIiIiIiIiHQagyIiIiIiItJpDIqIiIiIiEinMSgiIiIiIiKdxqCIiIiIiIh0GoMiIiIiIiLSaQyKiIiIiIhIpzEoIiIiIiIinVamQdGiRYvg7e0NCwsLWFhYICAgADt27BDXBwYGQiKRqPz0799fpYzbt28jNDQUpqamsLOzw/Dhw5Gfn6+SJzExEb6+vjAyMoKHhwdiY2PV6rJgwQK4ubnB2NgY/v7+OHr06FvZZyIiIiIier+UaVDk7OyMqVOn4sSJEzh+/DiaN2+Odu3a4cKFC2Kevn37IjU1VfyZPn26uE4ulyM0NBS5ubk4fPgwVq9ejdjYWIwdO1bMk5KSgtDQUDRr1gynT59GdHQ0+vTpg507d4p51q9fj6FDh2LcuHE4efIkateujZCQEGRkZLybA0FERERERGVGvyw3HhYWprL8ww8/YNGiRThy5Ai8vLwAAKampnBwcND4+V27duHixYtISEiAvb09fHx8MGnSJIwcORLjx4+HoaEhFi9eDHd3d8yaNQsAUKNGDRw8eBBz5sxBSEgIAGD27Nno27cvevbsCQBYvHgxtm3bhpUrV2LUqFEat52Tk4OcnBxxOTMzEwCQl5eHvLw8lbzK5VfTiQrDNkPaYpshbbHNkLbYZkhbZd1mtNlumQZFL5PL5di4cSOePXuGgIAAMX3dunVYu3YtHBwcEBYWhu+++w6mpqYAgKSkJNSqVQv29vZi/pCQEAwYMAAXLlxAnTp1kJSUhKCgIJVthYSEIDo6GgCQm5uLEydOYPTo0eJ6PT09BAUFISkpqdD6TpkyBRMmTFBL37Vrl1i/V8XHxxd/IIhewjZD2mKbIW2xzZC22GZIW2XVZp4/f17ivGUeFJ07dw4BAQHIzs6GTCbDb7/9Bk9PTwBAt27d4OrqCicnJ5w9exYjR47ElStXsHnzZgBAWlqaSkAEQFxOS0srMk9mZiZevHiBR48eQS6Xa8xz+fLlQus9evRoDB06VFzOzMyEi4sLgoODYWFhoZI3Ly8P8fHxaNmyJQwMDLQ5PKSj2GZIW2wzpC22GdIW2wxpq6zbjHIkV0mUeVBUrVo1nD59Gk+ePMGmTZsQERGBffv2wdPTE1988YWYr1atWnB0dESLFi2QnJyMypUrl2GtASMjIxgZGamlGxgYFPpLL2odkSZsM6QtthnSFtsMaYtthrRVVm1Gm22W+ZTchoaG8PDwgJ+fH6ZMmYLatWsjJiZGY15/f38AwPXr1wEADg4OSE9PV8mjXFY+h1RYHgsLC5iYmKB8+fKQSqUa8xT2LBMREREREX04yjwoepVCoVCZwOBlp0+fBgA4OjoCAAICAnDu3DmVWeLi4+NhYWEhDsELCAjA7t27VcqJj48Xn1syNDSEn5+fSh6FQoHdu3erPNtEREREREQfpjIdPjd69Gi0bt0aFStWxNOnT/Hzzz8jMTERO3fuRHJyMn7++We0adMG5cqVw9mzZzFkyBA0adIE3t7eAIDg4GB4enqiR48emD59OtLS0jBmzBgMGjRIHNrWv39/zJ8/HyNGjECvXr2wZ88ebNiwAdu2bRPrMXToUERERKBu3bqoX78+5s6di2fPnomz0RERERER0YerTIOijIwMfP7550hNTYWlpSW8vb2xc+dOtGzZEnfu3EFCQoIYoLi4uKBjx44YM2aM+HmpVIqtW7diwIABCAgIgJmZGSIiIjBx4kQxj7u7O7Zt24YhQ4YgJiYGzs7OWL58uTgdNwB06dIF9+/fx9ixY5GWlgYfHx/ExcWpTb5AREREREQfnjINilasWFHoOhcXF+zbt6/YMlxdXbF9+/Yi8wQGBuLUqVNF5omKikJUVFSx2yMiIiIiog/Le/dMERERERER0bvEoIiIiIiIiHQagyIiIiIiItJpDIqIiIiIiEinMSgi+r/27jwuqup94Phn2HdwYVXEfcFU1HJfMFFM0zJL2wxLrVxKpXL5lrlVLuVWuWS5ZOXPzK1yFxJN3NdUlBQRXEBcQURhmDm/PyYGxwEEBFF53q8XL51zzz1zznDmzjzcc58rhBBCCCFKNQmKhBBCCCGEEKWaBEVCCCGEEEKIUk2CIiGEEEIIIUSpJkGREEIIIYQQolSToEgIIYQQQghRqklQJIQQQgghhCjVJCgSQgghhBBClGoSFAkhhBBCCCFKNQmKhBBCCCGEEKWaBEVCCCGEEEKIUk2CIiGEEEIIIUSpJkGREEIIIYQQolSToEgIIYQQQghRqlmVdAeEEEKUAO1tWDMMEg7BpWio2QleWWJeLzMdtk6Gf5ZB6kVw8oK2w6FRb9N6EZPgSgz0+B72LYQjyyHhMGTcgBFxYO9m3va/Gw1tXzwGVrbg1yrnPgghhBDFTIIiIYQojZQOrO2g6TsQ9Ufu9X7rA6lJ0O0bKFvVEBgpvXm9E2uh1TDD/7W3oHp7w0/4uJzbjfod/ngf2n8KVdqCPhOSou57WEIIIURhSFAkhBClkY0jPDvd8P/43XA72bzOyTA4EwlDDoFDWUNZGT/zesnn4NIJqB5keNx8oOHf2L9zfm5dJqwfCR0nQKM3sss9ahdqKEIIIcT9kqBICCEKqziXoN24CJtHQ8wWyEiFctWhzYfg/9wDGRoA0evAJwAiZ8I/v4K1A9R6Bp7+BKzt76i3Hiq3AjuX/LWbcBhuXACNBcxtZTgT5VUPOkwAT/9iGYoQQgiRFwmKhBCisIpzCdqqdwxnb15ZajhLc2S5oZ23I8C7QTEMJgfXzkD8LrCyg16/QNoVWPsB3LoGz8827XftLgVoN9bwb8QkCP4c3CrBjm9hURd4b3/2WSkhhBDiAZHsc0IIUVhZS9Aa9wEnz5zrZC1Be+03qNbOsPzMtwlUamZa7+4laGf3GIKtio2hbBVo+xHYucKFQ8U5IlNKDxqN4cxVxcZQs6MhiDm0xHDdEMDtFIiLNJxByne7yvBv6w8MZ758GhqCLI0GolYX+TCEEEKIe5GgSAghitOdS9Cm1oavG8HGj7ODCmO9u5ag+TaBoysh7Sro9YYzRZnphjoPirMXOHsbgrEs7rUABSkXDI9PbTaUuVYsQLv/BZDud1xDZGULZSobgkMhhBDiAZPlc0IIUZwKuwTtpUWw/E2YUgUsrAzX8/T6GcpVe3B9920Kx1ZDeirYOhnKrpwyXAvk4vNfv9dBrQIsnQPwDgBLW7hyEvyaG8p0WrgeD66+RdV7IYQQIt/kTJEQQhSnwi5B2/K54ZqiN343XEfUfBD89qbhnj75odcZsr8dWW74V68zr5N0AhL+MQRo6SmG/yf8k7293kuG63t+H2ioeyYSNo2Ghq8bEi3oMg1niu5eOnfjoqGdq6f/e54ow+O0q4bHdi7w5FuwZSKcCofLJw0JKwDqPp+/8QkhhBBFSM4UCSFEcbrXErRy1cyXoF09DXvmwcBd4FHHUOZVD+J2wJ7voeuMvJ8z6g/YMCJ7iRsYzux0mgz+3bLLfnkJkuOzH3/X2vDv2P/Sc9s6Qe/VsP4jmBdoCJDqdjdknwOI2w42ToblgXfatwC2Tsp+vPC/oOm52dDwNcP/O04AC0tDQgntbUPAGPIn2JfJe2xCCCFEMZCgSAghilNhlqBlnUHS3HUy38Iy56x1d9CcWAMr3gSU6YaUBFj2BvRcnB0YDTty7/671zScrcrJiXWGNOR3azfK8JMXS2vDGbPgz+/dByGEEKKYyfI5IYS4H8WxBK18TUPq7j+Hwrn9hjNHO74x3LOo9rPmfdDr0MRtp8LVHViu+wCzgAiyyzaMzHkpXWF41IGn+hZNW0IIIUQJkjNFQghxP4pjCZqlNby2HMLGwP/1goybhiCp+1zDNUl3+m+pnFXKBZ68Z2cVpJw3LMOr0rqwI8725Jv334YQQgjxEJCgSAgh7kdxLUErV82QbS4vUX8YlsTleGYoD6kXC1ZfCCGEeMxJUCSEECXJo47hnkQFpdcZkikUMCDSAQcyk7l0eh3uDu408miEpYVlwZ9fCCGEeIxIUCSEECWpsEvQ4naYZpfLhzAHByaVL8fFf2YYyzwdPBnZZCRBfkGF64cQQgjxGJBEC0II8Sgq4BK4MAcHQj3KcdFSY1KelJZEaEQoYXFhRdk7IYQQ4pEiZ4qEEOJR5OSZ76o6YFL5ciiNxmybQqFBw+Q9k2nn206W0gkhhCiYrBtwXzphuBm5s5ch82rgSGMVzcHFcPQ3w828AbwDoP0Ywz3q7rboWcP+jUPg/H4IGwsXDoMGqNAYOow33LuviMmZIiGEeBT5tfjvPkfmgY6BBhzKwwvfc+C5aWZniO6kUCSmJXIg6UCxdFUIIcRjzMIKGrwMvVfBe/ug0yQ48CNs+SK7SlwkPNEDQtZA3zDDzcp/6m6+DDztKsTvMtymIj0Vfu4Brr7QPxze2mjI1vrTC6DTFv0wirxFIYQQxc/CEjpN/u/B3QHPf4+fnQ71e3LJ1TtfTV5Ku1Rk3RNCCFFKlK1iuPeeVz1wqwS1O0O9nhC/01hF9/x30KQ/eNc3ZGTt9o3hZuSnt5q2dXITeDcAJw+4/K/hHoDt/gflaxgSEwWOhJtJcD2eoiZBkRBCPKr8u0HPxeByV9Dj4mMo9+8GgLuDe76ay289IYQQIldXYuBUGPi1zL2ONg30WrAvY1oevc4QVIEhELIvCwd+gswM0N4y/L98LXDzK/JuyzVFQgjxKPPvBrW7kHl6G4f+3khA62CsqrYxnEn6TyOPRng6eJKUloTKIYW3Bg2eDp408mj0IHsuhBDicfJDB0g4DLp0aNwH2n0MOl3OdTePMVx7VDUwuywzHU6FQ+Aow2NbZ+izFpa+CtumGMrKVoPeK8Gy6EMYCYqEEOJRZ2GJ8mvF+WMpNPBrZRIQAVhaWDKyyUhCI0LRoDEJjDT/LbUb0WSEMclCui6d8TvHE3UlitjkWNpUbMPXT39t9rQZugzmHp7LmtNruHzrMu727rzb4F261+huUm/OoTnE3YhjUutJ/Pbvb6w7vY7jV49zU3uTyFcicbFxMWt727ltzD08l3+v/YuNpQ1Pej6ZYx+EEEI8JF5aaLgO6OJR2DQaynwNTQeZ1/t7GhxdYQh4rO2yy2O3gWN5wzI5MJwZ+mMwVGoGL84HvR52fA2/9IS3t4C1fZF2v0SXz82ZM4f69evj4uKCi4sLzZs3Z/369cbtt2/fZtCgQZQrVw4nJyd69OjBxYumaWjj4+Pp0qULDg4OeHh48NFHH5GZmWlSJyIigkaNGmFra0v16tVZtGiRWV9mzZpF5cqVsbOzo2nTpuzZs6dYxiyEECUhyC+IaYHT8HDwMCn3dPBkWuA0k/sU6fQ67CzteK3OazTzbpZrmx9s/YDdCbsZ12Icf3b/k8ltJlPZtbJZvS1ntxDoGwjA7czbtKzQkn71+uXa7ua4zYz6exTPV3+e5V2X89MzP9G5aueCDVgIIcSD5VoRPGpDvRchaCxETDLcaPxOkV/D9hmGpAxeT5hui14Hte441h/5zXDt0HOzDVnnfJ+CHvPhehycWFvk3S/RM0UVK1Zk0qRJ1KhRA6UUP/74I8899xwHDx6kbt26DBs2jLVr1/Lbb7/h6urK4MGDeeGFF4iMjARAp9PRpUsXvLy82LFjBwkJCbzxxhtYW1vzxReGjBexsbF06dKFd999l19++YXw8HD69euHt7c3wcHBAPz666+EhoYyd+5cmjZtyowZMwgODiY6OhoPD49c+y+EEI+SIL8g2vm240DSAS6lXcLdwZ1GHo3M0nA7WDswuvloAA4mHeRGxg2ztraf387+xP2s77EeV1tXACo4VTCrl3gzkVPXT9HKpxUAvf17A7A3cW+OfczUZzJpzyQ+ePIDXqjxgrG8mlu1QoxYCCFEiVB6wzVDSp9dtn0G/D0VXl8JFe5arq0URG+AF+Zll2lvgcYC7rydhMYC0BjqF7ESDYq6du1q8vjzzz9nzpw57Nq1i4oVKzJ//nyWLFnC008/DcDChQupU6cOu3btolmzZmzatImoqCjCwsLw9PQkICCACRMmMGLECMaOHYuNjQ1z586lSpUqTJ06FYA6deqwfft2pk+fbgyKpk2bRv/+/XnzTcOd5efOncvatWtZsGABI0eOJCfp6emkp6cbH6ekpACg1WrRak3TBGY9vrtciNzInBEFVZA5E1AuAMoZ/q/X6dHr9LnWVXqFUsqs3b/i/qJO2Tr88M8PrI1di72VPW0rtGVA/QHYWWUvhwg7E0Zjj8bYamxN2sg6o5+pzUSryS4/evkoSWlJ6HV6XvzjRa7cukLNMjUZ2nAo1d2q33NsIv/kOCMKSuZMKaXXoTm703DTcCdPlG9zk2XamqO/gYU1ysMfLG3QJBzCMmwsyv95tP99vKjt01Hbv0T3/HcoJ2+4ds6wwcYRbJzQXDiIpTaNTJ8nIWt+VWqN1a3R6P8chv6p/qD0WO6YicbCksyKzbLr5aEgc/WhuaZIp9Px22+/cfPmTZo3b87+/fvRarUEBWUv6ahduzaVKlVi586dNGvWjJ07d1KvXj08PbNvYhgcHMyAAQM4duwYDRs2ZOfOnSZtZNUZOnQoABkZGezfv59Ro0YZt1tYWBAUFMTOnTvJzcSJExk3bpxZ+aZNm3BwcMhxn82bN+frtRAii8wZUVBFPWfO3TzHbXWbdevWmZQfSj1EbGYsyVeS6WHXg5v6m/wZ/SdHTx+lh0MPY73lqcupY13HbP/T2tOA4Zhpb5G9LvyfjH8AmLFnBp3tO+Nm5Ubk5Uj6rO/DUOehOFjkfHwVhSfHGVFQMmdKD+/re6l37hfstVeNZbesy3Kk4mskuD0FgM+1I9S4uA6n9ERAkWZTnnNlWhNjGYz+v7mSufM7bHQZWK1406T9E17PE+39ArUvLMfB3p8DGzaZbHevPIRa0atwOfwrCg1X7P047jeUa3/n7756aWlp+R5riQdFR44coXnz5ty+fRsnJydWrVqFv78/hw4dwsbGBjc3N5P6np6eJCYmApCYmGgSEGVtz9qWV52UlBRu3brFtWvX0Ol0OdY5ceJErv0eNWoUoaGhxscpKSn4+vrSsWNHXFxMLxrWarVs3ryZDh06YG1tnY9XRZR2MmdEQeU1Z3R6HQcvHeTyrcuUty9PQ/eGZkvmcrN7525uaG/QuY3pNT1r/lpD/KV45j0/D2cbZwDqn63P8L+HM+v5WdhZ2ZGqTWXcinF80/EbvBy9TPbfd3EfC8IX0LFjR+P+AJozGpbtWMbgpwbTo7ohuArRhdBpdSeoBZ1ryLVFRUWOM6KgZM6ULpoTa7Bc8S3clbXUTnuNp2K/RddjIar2s0BnYLyxlj1Q47+frDnD0CNoc5gz1f77sfp+ErrAUDr7332M7wwMN/QHcAOaF2AMWSu58qPEg6JatWpx6NAhkpOTWb58OSEhIWzduvXeO5YwW1tbbG1tzcqtra1zPVDktU2InMicEQV195wJiwtj0p5JXEzLTlLj6eDJyCYjTZIr5EZjoUGj0ZjNQw9HDzxuelDWsayxrGbZmigUV7VX8bP3Y/e53VRzq4avm69Zu1ZWho8fK2srk7a9nLyMbWWVW1tb4+vsy6Xbl+T9UAzkOCMKSuZMKaDXweb/cXdABKBBARqsNn8MdbuZZTzNSZ5zJjMD/J/DqnYnKOJ5VZB5WuI3b7WxsaF69eo0btyYiRMn0qBBA2bOnImXlxcZGRlcv37dpP7Fixfx8jJ8aHp5eZllo8t6fK86Li4u2NvbU758eSwtLXOsk9WGEEI8isLiwgiNCDUJiACS0pIIjQglLC6s0G0HeARwKe0SadrspQlnUs5gobHA08Fw5v2vs3/Rzrddgdr1L+ePjYUNZ1LOGMu0ei3nU8/j7eSd+45CCCGKTtwOSLmQRwUFKecN9e6XlQ0EjjTcl6gElXhQdDe9Xk96ejqNGzfG2tqa8PBw47bo6Gji4+Np3txw4qx58+YcOXKEpKQkY53Nmzfj4uKCv7+/sc6dbWTVyWrDxsaGxo0bm9TR6/WEh4cb6wghxKNGp9cxac+kHG/WmlU2ec9kdHenS/1PzPUYTlw9QUp6CqnaVE5cPcGJq9lLirtU6YKrrSufRH5CzPUY9iXuY9r+aXSv3h07Kzsy9ZlsP7/dmIo7y+Vblzlx9QTxKfEAnLx2khNXT5CcngyAk40TPWv1ZNahWew4v4PY5Fg+2/UZAB39Ot736yKEECIfUi/eu05B6j0CSnT53KhRo3jmmWeoVKkSN27cYMmSJURERLBx40ZcXV3p27cvoaGhlC1bFhcXF9577z2aN29Os2aG+2Z07NgRf39/evfuzZQpU0hMTOSTTz5h0KBBxqVt7777Lt9++y3Dhw/nrbfe4q+//mLZsmWsXZud3zw0NJSQkBCefPJJmjRpwowZM7h586YxG50QQjxqDiQdMDtDdCeFIjEtkQNJB3jK6ymz7QPDBnLhZvZfCV/68yUAjoQcAQxpu+d1nMfE3RN5ec3LuNq6Elw5mPcavgcYrhlysHLAv5y/SbvLopcx5/Ac4+M+G/oAMKHlBJ6v/jwAoU+GYqmxZNT2UaTr0qlXvh7zO843pv4WQghRzJw8710H4EpM8fbjASrRoCgpKYk33niDhIQEXF1dqV+/Phs3bqRDhw4ATJ8+HQsLC3r06EF6ejrBwcHMnj3buL+lpSVr1qxhwIABNG/eHEdHR0JCQhg/fryxTpUqVVi7di3Dhg1j5syZVKxYkR9++MGYjhugV69eXLp0iU8//ZTExEQCAgLYsGGDWfIFIYR4VFxKu3Rf9Ta+uPGe+1Z1rcr3Hb/PcduW+C1mZ4kABgYMZGDAwDzbtbaw5sOnPuTDpz68Zx+EEEIUA78W4OIDKQnkdF2RUcQX4FEH/Ls9sK4VlxINiubPn5/ndjs7O2bNmsWsWbNyrePn52eW6vVugYGBHDx4MM86gwcPZvDgwXnWEUKIR4W7g3uR1iuo6mWq08C9QbG0LYQQophZWEKnybDsjXtU1MCGkVC7S74SLjzMHrprioQQQty/Rh6N8HTwRIMmx+0aNHg5eNHIo1GO2+/XSzVfomaZmsXSthBCiAfAvxsEjrpHpSJMuFDCJCgSQojHkKWFJSObjAQwC4yyHo9oMiLf9ysSQghRCpWrlr96j0HChRK/T5EQQjyu0nXpjN85nqgrUcQmx9KmYhu+fvprs3oZugzmHp7LmtNruHzrMu727rzb4F261+huUm/OoTnE3YhjUutJXL51man7prLzwk7SMtPwc/ajYUZDOpN947sgvyCmBU7L8T5FI5qMyNd9ioQQQpRi+U24kN96DzEJioQQopjo9DrsLO14rc5red4T6IOtH3D11lXGtRhHJZdKXEq7lGMq7S1nt/BWvbcA+N/f/+NGxg2+efob3OzcWHNqDXP+mUPXq12p51nPuE+QXxDtfNtxIOkAl9Iu4e7gTiOPRnKGSAghStLlk7BmGFw6AbdTwNkL6r1kuF+P5X83HN2/CA4vhaQow2PvAGg/Bio2Nm9v0bOG/RuHwLrhcHYXJB2H8rVgwHbz+krBjm8Mz5F8FhzKwVN9oc1HpvXumXBBY9ju16LQL8XDQoIiIYQoJg7WDoxuPhqAg0kHuZFxw6zO9vPb2Z+4n/U91htTTldwqmBWL/FmIqeun6KVTysADl06xOhmo6nnbgiA+j3Rj4VHFnL86nGToAgMS+lySrsthBCihFhYQYOXwbsB2LlC4lH4831QeggaY6hzZjs80QN8p4CVHUTOgJ+6w6BdhkAkS9pViN8FLy7ILmvYG87tg4vHcn7+9SMg5i/o+Bl4+sOta4Yfs37emXBBg2lg9N/S7E6THvkkCyBBkRBClKiIsxH4l/dnwdEFrIlZg721PYEVAxnccDB2VnbGelvObuEpr6dwsnECIMA9gA1nNtCmYhucbZzZeGYjmSqTxp45/AVRCCHEw6VsFcNPFrdKhiAofmd2WY8fTPfp9g1E/QGnt0LAK9nlJzcZgisnD8PjzlMM/968nHNQdCka9s2HgbugfA1DWZnKuffVvxv0XAwbRkBK9v3rcPExBESPQTpukKBICCFK1Lkb5zh48SC2lrbMaDeDa+nX+HzX51xPv85nrT4z1tsSv4V2ldoZH38V+BUfbf2IVktbYaWxws7KjlcdX6WSc6WSGIYQQoj7cSUGToVBna6519GmgV4L9mVMy6PXQe3OOe+Tk+j1hiDo3w3w8wuGkz9V20KH8eBQNud9/LsZ0m7H7TAkVXDyNCyZewzOEGWRoEgIIUqQXunRaDRMaj0JZxtnADKeyiA0IpRPmn2CnZUdqRmp7Lu4j/Ets29M/e3Bb7mRcYPvO35PGdsybD6zmR+P/Ejn653xd/cvqeEIIYQoiB86QMJh0KVD4z7Q7uPc624eY7j2qGpgdllmOpwKz0fq7DtcOwPXz8Kx1dD9O9DrYOMowxK5Pmty38/CEqq0zv/zPGIkJbcQQpQgdwd3PBw8jAERQFXXqiiUMWPc9vPbqeZWDS9HLwDOppzl/078H+NbjKeZdzNqla3FO/XewcfKh2X/LiuRcQghhCiElxbCO9ugx3z4dxPsMM9QCsDf0+DoCuj1C1hnL60mdhs4lgePOvl/TqU3BGHdvzOc7anSGrp9C2f+NiSAKKUkKBJCiBIU4BHApbRLpGnTjGVnUs5gobHA08GQ4vSvs3/Rzjd76dwt3S0ALDSmh3ALLNAr/QPotRBCiCLhWhE8akO9FyFoLERMMpy5uVPk17B9BvReBV5PmG6LXge1CrB0DgxnmyysoHz17DL3WoZ/k88WdASPDQmKhBCiGMVcj+HE1ROkpKeQqk3lxNUTnLh6wri9S5UuuNq68knkJ8Rcj2Ff4j6m7Z9G9+rdsbOyI1Ofyfbz2wn0DTTuU8W1CpWcKzFu5ziOXDrC2ZSz/HT8J2IyY2hXsV0OvRBCCPHQU3rDNUN3/nFr+wzY9iW8vgIqNLqrvoLoDQUPinybgj4Trp7OLrtyyvCva+m9LlWuKRJCiGI0MGwgF25mZ+t56c+XADgScgQwpO2e13EeE3dP5OU1L+Nq60pw5WDea/geAPsu7sPBygH/ctnXCVlbWDM7aDYz9s9g8F+DuZV5i4pOFXnB4QVaVWj1AEcnhBCiUP5ZZjhb41kXLG3gwkEIHwd1X8i+T9H26bDlC0MWOrdKcOO/m3DbOIKtk2EfbRpUam7a9pUYyLhpSIiQeQsS/jGUu9cGKxuo2s6Qre73wdBpoiEIW/uhofzOs0eljARFQghRjDa+uPGedaq6VuX7jt/nuG1L/BaTs0RZ/Fz8mN5uuvGxVqtl3bp1he6nEEKIB8jC0nDfoSsxhjM+br7QpD80G5RdZ+8C0GX8d4+gO7QdCe1GGZbO1egIlnd9nf/jfYi744at3/2XHGHIP1DGDyws4JVfYf1HsLAzWDtAjQ6GexaVYhIUCSHEQ6x6meo0cG9Q0t0QQghRlJ7oYfjJy7AjeW8/sQ7afGhe/ubaez+/izf0+vne9UoRCYqEEOIh9lLNl0q6C0IIIR42mRmGewfV6FDSPXlsSFAkhBBCCCHEo8TKBgJHlnQvHiuSfU4IIYQQQghRqklQJIQQQgghhCjVZPmcEEIIIYQQosjEJscyYdcEYq7HkHI7hTm/z6Fz1c4MCBiAtYUh5fjyf5fzZ8yfnLx+EgD/cv4MaTiEeu71zNp7a+NbdKnShR41e3D08lFm7J9B1JUo0EC98vUIbRxKrbK17qvPcqZICCGEEEIIUWSsLKzoWrUrs9vNZojLED5s/CErTq5g9qHZxjp7E/fyTJVnWBC8gJ87/4yXgxfvbH6HizcvmrSVnJ7MwaSDtPVtS5o2jXfD3sXL0YtfuvzC4k6LcbR25J3N76DVa++vz/e1txBCCCGEEELcwdfZF19nX7RaLacsTtG2YlsOXD7AgYsHjHUmt5lsss+4FuMIiw9jd+JuulXrZizfdm4b/mX9KW9fnmOXj5GcnszghoPxcvQC4N0G79Ljjx4kpCZQyaVSofssZ4qEEEIIIYQQxSb+RjyR5yNp7Nk41zq3dbfJ1GfiauNqUr7l7BbaVWoHQGXXyrjZurHy5Eq0Oi23M2+z6uQqqrpWxcfJ5776KGeKhBBCCCGEEEWuz6Y+RF2PIvPPTF6s+SKDGw7Ote70/dNxt3enmU8zY1mGLoPI85EMbDAQAEdrRxYEL2DIliF89893AFRyrsR3Hb7DyuL+who5UySEEEIIIYQocpNaTmKg80C+aPEF285tY9GxRTnW++HID6yPXc+MdjOwtbQ1lu9O2E1Zu7JUL1MdgNuZtxmzYwwNPRryS+dfWPzMYmqUqcGg8EHczrx9X32VoEgIIYQQQghR5LwcvfCw9KBT5U4MbTSUOYfmoNPrTOosOrqIBUcWMK/DPLMMchFnIwj0DTQ+Xhe7jvOp55nQcgJPlH+CBu4NmNx6MudTz7Pl7Jb76qsERUIIIYQQQohipVBk6jPRozeWLTi6gO/++Y45HeZQt3xd0/pKEXEugqcrPW0su5V5CwuNBRo0xjKNxvB/vdJzP+SaIiGEEEIIIUSR0Ol1zDo0i5vam9Ryq8U13TU2xW1i5oGZBFcJNt6naP6R+cw6NIvJbSZTwakCl29dBsDBygEHaweirkRxO/M2DT0aGttu7tOcafum8fnuz3m19qvolZ75R+djpbGiiVeT++q3BEVCCCGEEEKI+xYWF8akPZO4mGZ6ryHPg568UucVevv3NpYti16GVq8lNCLUpO6ABgMYGDCQv87+ReuKrU0SKFR1rco37b9h7uG5vL7udTQaDXXK1mFOhzm4O7jfV98lKBJCCCGEEELcl7C4MEIjQlEos21JaUlUdqlskkRh44sb82xvy9ktvF3/bbPyFj4taOHT4v47fBe5pkgIIYQQQghRaDq9jkl7JuUYEGWZvGeyWZKF3Gh1WjpU6kDrCq2Lqov3JEGREEIIIYQQotAOJB0wWzJ3J4UiMS2RA0kH8tWetaU1AwIG4GjtWFRdvCcJioQQQgghhBCFdintUpHWKwkSFAkhhBBCCCEKLb9JDu43GUJxkqBICCGEEEIIUWiNPBrh6eCZZx0vBy8aeTR6QD0qOAmKhBBCCCGEEIVmaWFJ5yqd86zzTJVnsLSwfEA9KjgJioQQQgghhBCFptPrWBe7Ls8662PX5zv7XEmQoEgIIYQQQghRaPfKPgcUKPtcSZCgSAghhBBCCFFokn1OCCGEEEIIUao9DtnnrEq6A0IIIYQQQoiHT2xyLBN2TSDmegypGam4O7jTuUpnBgQMwNrCGoDl/y7nj5g/0KBBoXJsR4MGTwdPZh+azbNVn6VHzR5M3D2Rg0kHOXX9FFVdq7K823Kz/ZRS/HjsR5afXM6F1AuUsS1Dr9q9eLv+20U+1hINiiZOnMjKlSs5ceIE9vb2tGjRgsmTJ1OrVi1jncDAQLZu3Wqy3zvvvMPcuXONj+Pj4xkwYABbtmzBycmJkJAQJk6ciJVV9vAiIiIIDQ3l2LFj+Pr68sknn9CnTx+TdmfNmsWXX35JYmIiDRo04JtvvqFJkyZFOubk5GTS0tKKtE3x+MnMzCQtLY3ExESTeSxEbmTOiIJ60HPGwcEBV1fXYn8eIUTRsbKwomvVrviX88fZxpnoq9GM3TkWhWJIoyEA7E3cS+cqnWnv256v9n+Va1uDGw5m7M6xfNn2S2NZ9xrdOXLpCP9e+zfHfSbtmcSOCzv4oPEH1ChTg+SMZJLTk4t2kP8p0U/OrVu3MmjQIJ566ikyMzP53//+R8eOHYmKisLR0dFYr3///owfP9742MHBwfh/nU5Hly5d8PLyYseOHSQkJPDGG29gbW3NF198AUBsbCxdunTh3Xff5ZdffiE8PJx+/frh7e1NcHAwAL/++iuhoaHMnTuXpk2bMmPGDIKDg4mOjsbDw6NIxpucnMy8efPQarVF0p54/P37b84HCSFyI3NGFNSDmjPW1tYMGjRIAiMhHiG+zr74OvsaH/s4+bD34l4OXMxOmDC5zWTj/ys4V2Di7okk3UoylrlqXPmk1SdolRb/sv6Uty8PwKimowC4dvtajkHR6eunWRa9jJXPraSKaxUAKlKxaAd4hxINijZs2GDyeNGiRXh4eLB//37atGljLHdwcMDLyyvHNjZt2kRUVBRhYWF4enoSEBDAhAkTGDFiBGPHjsXGxoa5c+dSpUoVpk6dCkCdOnXYvn0706dPNwZF06ZNo3///rz55psAzJ07l7Vr17JgwQJGjhxZJOO9desWWq2W7t274+7+8K6pFEIIIYrSpUuXWLVqFWlpaRIUCfEIi0+JJ/J8JO0rtc9xe5BfEE28mhC4LJDXar9GC+8WJO5LpL1ve0ZEjqBdpXb5fq6IcxFUdK7ItnPbGBA2AKUUzXyaEdo4FFfboj+OPFRrLJKTDafDypYta1L+yy+/8PPPP+Pl5UXXrl0ZPXq08WzRzp07qVevHp6e2XfRDQ4OZsCAARw7doyGDRuyc+dOgoKCTNoMDg5m6NChAGRkZLB//35GjRpl3G5hYUFQUBA7d+7Msa/p6emkp6cbH6ekpACg1WrNzgRlPc7MzATA3d0db2/v/L0oQgghxGMiMzNTVks8wrJ+d/I7LH36bOrDiasnyNBn8EL1F3jniXdynQcz9s/A08GTd+q9g4XegiRNEjdv3yTyfCRv133bbD+dTodSyqw8PjmeC6kX2BC7gXHNxqFXeqYemMrQLUOZ135evvpdkLn60ARFer2eoUOH0rJlS5544glj+auvvoqfnx8+Pj78888/jBgxgujoaFauXAlAYmKiSUAEGB8nJibmWSclJYVbt25x7do1dDpdjnVOnDiRY38nTpzIuHHjzMo3bdpksrzvTrt27crrJRBCCCEea9u3b8/1M1I8OjZv3lzSXRAPWEd9RwIdA0nQJbAxZiM3z92ktV1rs3pbb29le/p2+jr1JXxjuLH8+43fY6u3JXpHNNFEm+xz8tZJUrQprFtnevPXuLQ4MvQZBGUEkbjP8J2+fWZ7Zl+bzY9//oi75b1XXRXkOv6HJigaNGgQR48eZfv27Sblb7+dnV2iXr16eHt70759e2JiYqhWrdqD7qbRqFGjCA0NNT5OSUnB19eXjh074uLiYlJXq9WyefNmmjVrJuv9hRBClFqtWrXKdTm8ePhlfZ/p0KED1tbWJd0d8aDpdWjO7qTB2XQ+O7eez7qNw9LKxrh58fHF7Dy6k+87fo9/OX8ge86keqbyjPUzdG7c2azZ+H/iOX/uPJ07m26L+yeOQ8cO8UbXN4xltzNvM3vZbGo9WYtm3s3u2eWslVz58VAERYMHD2bNmjVs27aNihXzvoCqadOmAJw6dYpq1arh5eXFnj17TOpcvGi4o27WgdfLy8tYdmcdFxcX7O3tsbS0xNLSMsc6uR28bW1tsbW1NSu3trbO9UAhGaGEEEKUZlZWVvJl+jGQ13cd8ZiK+gM2jICUC2icHMksXxbLOU9i3Wky+HdjwdEF/HD0B+Z2mEsD9wYmuyql2J6wnUltJuU4bywtLdFoNGbbnvR6ku+Pfk/irUR8XQzJHk7fOA2Ar6tvvuZgQeZpid68VSnF4MGDWbVqFX/99RdVqlS55z6HDh0CMF6T07x5c44cOUJSUnaWi82bN+Pi4oK/v7+xTnh4uEk7mzdvpnnz5gDY2NjQuHFjkzp6vZ7w8HBjnUfBokWLcHNzu+92NBoNq1evvu92svTp04fnn3++yNp7GJw5cwaNRmOcj0Vl7NixBAQE5Fnnfl/P4up7Qc2bNw9fX18sLCyYMWNGvvYp6rlZEvLzO87L3b//wMBA4/WRpcnDPO6SfI89Du8RIcTDY83pNWzY/jmnV73F2bSLbHB0YGYZV4JvpmGdkgDL3mB++Id8e/BbxrccTwWnCly+dZnLty6TpjUsXbugu8Bt3W0aejQ0aTs+JZ4TV09w+dZl0nXpnLh6ghNXT6DVGa4DaubTjDpl6zB6x2iOXznOsSvHGL9zPM29m1PZtXKRj7VET10MGjSIJUuW8Pvvv+Ps7Gy8BsjV1RV7e3tiYmJYsmQJnTt3ply5cvzzzz8MGzaMNm3aUL9+fQA6duyIv78/vXv3ZsqUKSQmJvLJJ58waNAg45mcd999l2+//Zbhw4fz1ltv8ddff7Fs2TLWrl1r7EtoaCghISE8+eSTNGnShBkzZnDz5k1jNroHoU+fPly/fl0+0IqIRqNh1apV9x2QPWy/l5kzZ6JU9s3RAgMDCQgIyHdg4evrS0JCAuXLly+mHt5bSkoKgwcPZtq0afTo0SPf2agSEhIoU6ZMvp9n0aJFDB06lOvXrxeypw+/lStX5vsvYQWdK4+SypUrM3To0BINlN58800qVKhAv379Crzv2LFjWb16dZEGUmfOnKFKlSocPHjwvgJxIUTpZYUFC6L/jzgfTxTgk5nJKymp9DYuS9OwLG4DWksNoRGhJvsOaDCA/nX7c1x7nJY+LbGyMA07xuwYw76L+4yPX/rzJQA29NhABacKWGgs+Lb9t0zcPZE+G/pgb2VPqwqt+Oipj4pprCVozpw5gOGD+k4LFy6kT58+2NjYEBYWZgxQfH196dGjB5988omxrqWlJWvWrGHAgAE0b94cR0dHQkJCTO5rVKVKFdauXcuwYcOYOXMmFStW5IcffjCm4wbo1asXly5d4tNPPyUxMZGAgAA2bNhglnxBiJJ2v+lsLS0tS3xNf3x8PFqtli5duhQoE2NJ9Vun06HRaLCwKNGT6zm6O1unKBk6nY41a9aY/LFNCCEedZ00znQ6ezaPGoqN8WchZA1UMU+8oNVqOa49zrAKw8y2Ley08J7P7+HgwfR20wvS5UIr8eVzOf306dMHMPxFe+vWrVy5coXbt29z8uRJpkyZYpbIwM/Pj3Xr1pGWlsalS5f46quvzK7fCQwM5ODBg6SnpxMTE2N8jjsNHjyYuLg40tPT2b17t/H6pYfFtGnTqFevHo6Ojvj6+jJw4EBSU1PN6q1evZoaNWpgZ2dHcHAwZ++azL///juNGjXCzs6OqlWrMm7cOGO68LtlZGQwePBgvL29sbOzw8/Pj4kTJ+baR51OR2hoKG5ubpQrV47hw4ebnNUAw9LEiRMnUqVKFezt7WnQoAHLly83bo+IiECj0RAeHs6TTz6Jg4MDLVq0IDraNFvJnDlzqFatGjY2NtSqVYuffvrJuK1y5coAdO/eHY1GY3xc0PGPHTuWH3/8kd9//x2NRoNGoyEiIsK4/fTp07Rr1w4HBwcaNGhglsJ9+/bttG7dGnt7e3x9fXn//fe5efNmrq9flu+++w5fX18cHBzo2bOnMV09mC6f6tOnD1u3bmXmzJnG/p05c4Zr167x2muv4e7ujr29PTVq1GDhQsPB5+6lPX369DHue+dP1jjT09P58MMPqVChAo6OjjRt2tTkNchJfHw8zz33HE5OTri4uNCzZ0/jNXuLFi2iXr16AFStWtXY5/y4c2lQ1jhWrlyZ4+8gIiKCN998k+TkZOOYxo4dm68xZS1F/eOPP/D398fW1pb4+HgqV67MF198wVtvvYWzszOVKlVi3jzTtKAjRoygZs2aODg4ULVqVUaPHl3o9LX5eT/dvYxs9uzZxve/p6cnL774IpD7XNHpdPTt29f4fqxVqxYzZ840eY6sOffVV1/h7e1NuXLlGDRokMm40tPTGTFiBL6+vtja2lK9enXmz59v3H706FGeeeYZnJyc8PT0pHfv3ly+fDlfr8PNmzd54403cHJywtvb23jPuTtfg7i4OIYNG2Yc282bN3FxcTE5toDh+Ojo6MiNGzeMc2jp0qW0aNECOzs7nnjiCbZu3WqyT376vmPHDqytrXnqqafM+p/T0ubVq1ej0WiM28eNG8fhw4eN/V+0aNE9X5eTJ0/Spk0b7Ozs8Pf3N8sIlrUkvWHDhmg0GgIDA9m2bRvW1tbGlRlZhg4dSuvWrU36W5SfJUKIR1TqxXvXyaOeVqelrk1dWvq0LMJOFRMlikRycrICVHJystm2jIwMtXr1ahUfH6/Gjh2rLly4kGMbISEh6rnnnsv1OaZPn67++usvFRsbq8LDw1WtWrXUgAEDjNsXLlyorK2t1ZNPPql27Nih9u3bp5o0aaJatGhhrLNt2zbl4uKiFi1apGJiYtSmTZtU5cqV1dixY411ALVq1SqllFJffvml8vX1Vdu2bVNnzpxRf//9t1qyZEmufZw8ebIqU6aMWrFihYqKilJ9+/ZVzs7OJuP67LPPVO3atdWGDRtUTEyMWrhwobK1tVURERFKKaW2bNmiANW0aVMVERGhjh07plq3bm0yjpUrVypra2s1a9YsFR0draZOnaosLS3VX3/9pZRSKikpSQFq4cKFKiEhQSUlJeV7/He6ceOG6tmzp+rUqZNKSEhQCQkJKj09XcXGxipA1a5dW61Zs0ZFR0erF198Ufn5+SmtVquUUurUqVPK0dFRTZ8+Xf37778qMjJSNWzYUPXp0yfX12/MmDHK0dFRPf300+rgwYNq69atqnr16urVV1811rlznly/fl01b95c9e/f39i/zMxMNWjQIBUQEKD27t2rYmNj1ebNm9Uff/yhlFLGvh88eNDYRta+CQkJasiQIcrDw0MlJCQopZTq16+fatGihdq2bZs6deqU+vLLL5Wtra36999/cxyDTqdTAQEBqlWrVmrfvn1q165dqnHjxqpt27ZKKaXS0tJUWFiYAtSePXuMfQ4JCTHWyc2dc/Nev4P09HQ1Y8YM5eLiYhzbjRs38jWmrPdSixYtVGRkpDpx4oS6efOm8vPzU2XLllWzZs1SJ0+eVBMnTlQWFhbqxIkTxj5OmDBBRUZGqtjYWPXHH38oT09PNXnyZJPfcYMGDfIcZ5b8vJ/atm2rhgwZopRSau/evcrS0lItWbJEnTlzRh04cEDNnDlTKZX7XMnIyFCffvqp2rt3rzp9+rT6+eeflYODg/r111+NzxESEqJcXFzUu+++q44fP67+/PNP5eDgoObNm2es07NnT+Xr66tWrlypYmJiVFhYmFq6dKlSSqlr164pd3d3NWrUKHX8+HF14MAB1aFDB9WuXbt8vQ4DBgxQlSpVUmFhYeqff/5Rzz77rHJ2djaO+8qVK6pixYpq/PjxxrEppVT//v1V586dTdrq1q2beuONN5RS2XOoYsWKavny5SoqKkr169dPOTs7q8uXLxeo7x9++KF6++23TdrNeo8tXLhQubq6mtRftWqVyvoITktLUx988IGqW7eusf9paWl5viY6nU498cQTqn379urQoUNq69atqmHDhibvkT179ihAhYWFqYSEBHXlyhWllFI1a9ZUU6ZMMbaVkZGhypcvrxYsWGDsb1F8ltzpwoULeX7+iUdD1veZjIyMku6KeFBOb1NqjMu9f05vy3H3kp4zeX0/v5sERUXkQQRFd/vtt99UuXLljI8XLlyoALVr1y5j2fHjxxWgdu/erZRSqn379uqLL74waeenn35S3t7exsd3fqi+99576umnn1Z6vT5fffL29jb5sNVqtapixYrGcd2+fVs5ODioHTt2mOzXt29f9corryilsoOisLAw4/a1a9cqQN26dUsppVSLFi1U//79Tdp46aWXTL4A3TmOLPkZ/91y+r1kfen54YcfjGXHjh1TgDp+/LhxTFlfkrL8/fffysLCwjiOu40ZM0ZZWlqqc+fOGcvWr1+vLCwsjF/07u7PnV+Ks3Tt2lW9+eabOT7H3V/Y7rRixQplZ2entm/frpRSKi4uTllaWqrz58+b1Gvfvr0aNWpUju1v2rRJWVpaqvj4eGNZ1muzZ88epZRSBw8eVICKjY011hk5cqTq3bt3jm1mySkoyut3kNOX0fyMKeu9dOjQIZM6fn5+6vXXXzc+1uv1ysPDQ82ZMyfXPn/55ZeqcePGxscFCYru9X5SyvT3v2LFCuXi4qJSUlJybC+nuZKTQYMGqR49ehgfh4SEKD8/P5WZmWkse+mll1SvXr2UUkpFR0crQG3evDnH9iZMmKA6duxoUnb27FkFqOjo6Dz7cuPGDWVjY6OWLVtmLLty5Yqyt7c3GYufn5+aPn26yb67d+9WlpaWxmPuxYsXlZWVlfEPMFlzaNKkScZ9sl7jrEA2v32vUaOGWrNmjUm7+Q2KlCrYvFBKqY0bNyorKyuTebx+/foc3yN3v9cnT56s6tSpY3y8YsUK5eTkpFJTU439LYrPkjtJUPR4KOkvuKIE6DKVmlpbqTGuuQRErkpNrWOol4OSnjMFCYoevgXyIldhYWG0b9+eChUq4OzsTO/evbly5YrJjamsrKxMlm/Url0bNzc3jh8/DsDhw4cZP348Tk5Oxp/+/fuTkJCQ4w2u+vTpw6FDh6hVqxbvv/8+mzZtyrV/ycnJJCQkmCw7tLKy4sknnzQ+PnXqFGlpaXTo0MGkD4sXLyYmJsakvaxkGpCdbTAry+Dx48dp2dL0VGzLli2N48xNQcd/L3n18fDhwyxatMjkuYKDg9Hr9cTGxubaZqVKlahQoYLxcfPmzdHr9WbLB/MyYMAAli5dSkBAAMOHD2fHjh333OfgwYP07t2bb7/91vjaHjlyBJ1OR82aNU3GsXXrVrPfV5bjx4/j6+uLr6+vsczf399kHuZk4sSJLF68ON9jzJLX7yAn+R2TjY2NSds5PZ9Go8HLy8vk+X799VdatmyJl5cXTk5OfPLJJ8THxxd4XPl5P92tQ4cO+Pn5UbVqVXr37s0vv/ySr3k9a9YsGjdujLu7O05OTsybN8+sz3Xr1sXS0tL42Nvb2zjuQ4cOYWlpSdu2bXNs//Dhw2zZssXk9a5duzZArvMoS0xMDBkZGSavQ9myZalVq9Y9x9WkSRPq1q3Ljz/+CMDPP/+Mn58fbdq0Mal3Z5bRrNf4zmPmvfp+/PhxLly4QPv27e/Zp6KS9T7z8fHJcRx56dOnD6dOnTLeUHzRokX07NkTR0dHY52i/iwRQjyiLCyh0+T/Hmju2vjf406TDPUecXLjnEfEmTNnePbZZxkwYACff/45ZcuWZfv27fTt25eMjIx83yE8NTWVcePG8cILL5hts7OzMytr1KgRsbGxrF+/nrCwMHr27ElQUJDZOv38yroGau3atSZf/AGz+z7dmVEra+29Xq8v1PPe+fwFGf+95NXH1NRU3nnnHd5//32z/SpVqlTg5yqIZ555hri4ONatW8fmzZtp3749gwYN4quvvsqxfmJiIt26daNfv3707dvXWJ6amoqlpSX79+83+UIM4OTkVKxjyK+CzpP8jsne3t7YXm7Pl/WcWc+3c+dOXnvtNcaNG0dwcDCurq4sXbrU7BqY4uLs7MyBAweIiIhg06ZNfPrpp4wdO5a9e/fmmq5/6dKlfPjhh0ydOpXmzZvj7OzMl19+ye7du03q5TVue3v7PPuVmppK165dmTx5stm2giTaKIx+/foxa9YsRo4cycKFC3nzzTdz/L3mJj99/+OPP+jQoUOuxxALCwuza8EKe51ZUfDw8KBr164sXLiQKlWqsH79+nteJ3i3oj6WCiEeYv7doOdi432KjFx8DAGRfzeT6jq9jgNJB7iUdokyNmXQq/v77vagSFD0iNi/fz96vZ6pU6caM2AtW7bMrF5mZib79u2jSZMmAERHR3P9+nXq1KkDGIKc6Ohoqlevnu/ndnFxoVevXvTq1YsXX3yRTp06cfXqVbOsV66urnh7e7N7927jX2IzMzPZv38/jRo1AjC5aD23vyrnR506dYiMjCQkJMRYFhkZabw3FRi+xOl0OpP9CjN+Gxsbs3byo1GjRkRFRRXoucCQpODChQvGvwDv2rULCwuLXP8ynlv/3N3dCQkJISQkhNatW/PRRx/lGBTdvn2b5557jtq1azNt2jSTbQ0bNkSn05GUlGS8CPte6tSpw9mzZzl79qzxbFFUVBTXr183+f08CDm9NoUZU37t2LEDPz8/Pv74Y2NZXFxcodrKz/spJ1ZWVgQFBREUFMSYMWNwc3Pjr7/+4oUXXsjx9YiMjKRFixYMHDjQWHavszd3q1evHnq9nq1btxIUFGS2vVGjRqxYsYLKlSsX+CbW1apVw9ramt27dxv/mHDt2jX+/fdfk2NIbu+D119/neHDh/P1118TFRVlcszIsmvXLrPXePDgwfnu+++//87bb7+d6xjc3d25ceMGN2/eNJ6NuTv1dkGPM1nvs4SEBGNwlnXm5842gRzb7devH6+88goVK1akWrVqZmfei+OzRAjxCPPvBrW7QNwOQ1IFJ0/wa2F2higsLoxJeyZxMS078YKLxgX7s/Z0qtrpQfe6QCQoesgkJyebfViWK1eO6tWro9Vq+eabb+jatSuRkZHMnTvXbH9ra2vee+89vv76a6ysrBg8eDDNmjUzfrB9+umnPPvss1SqVIkXX3wRCwsLDh8+zNGjR/nss8/M2ps2bRre3t40bNgQCwsLfvvtN7y8vHL9q/OQIUOYNGkSNWrUMH7JvvMeMc7Oznz44YcMGzYMvV5Pq1atSE5OJjIyEhcXlxy/sOTko48+omfPnjRs2JCgoCD+/PNPVq5cSVhYmLFO5cqVCQ8Pp2XLltja2lKmTJkCjz+rnY0bNxIdHU25cuXynRJ7xIgRNGvWjMGDB9OvXz8cHR2Jiopi8+bNfPvtt7nuZ2dnR0hICF999RUpKSm8//779OzZM9d01JUrV2b37t2cOXMGJycnypYty9ixY2ncuDF169YlPT2dNWvWGL/M3O2dd97h7NmzhIeHc+nSJWN52bJlqVmzJq+99hpvvPEGU6dOpWHDhly6dInw8HDq169Ply5dzNoLCgqiXr16vPbaa8yYMYPMzEwGDhxI27Zt81z6NWrUKM6fP1+oJXS5qVy5MqmpqYSHh9OgQQMcHBwKNab8qlGjBvHx8SxdupSnnnqKtWvXsmrVqkK3d6/3093WrFnD6dOnadOmDWXKlGHdunXo9XpjQJ3TXKlRowaLFy9m48aNVKlShZ9++om9e/fm62baWSpXrkxISAhvvfUWX3/9NQ0aNCAuLo6kpCR69uzJoEGD+P7773nllVcYPnw4ZcuW5dSpUyxdupQffvjB7IzdnZycnOjbty8fffQR5cqVw8PDg48//tgsPXrlypXZtm0bL7/8Mra2tsb7cJUpU4YXXniBjz76iI4dO1KxYkWz55g1axY1atSgTp06TJ8+nWvXrvHWW28B3LPvV65cYd++ffzxxx+5jqFp06Y4ODjwv//9j/fff5/du3ebZZerXLkysbGxHDp0iIoVK+Ls7Gx29vxOQUFB1KxZk5CQEL788ktSUlJMgnEwnBGyt7dnw4YNVKxYETs7O+PxKzg4GBcXFz777DOTW1hkKerPEiFKncsnYc0wuHQCbqeAsxfUewkCR4Llf2fe9y+Cw0shKcrw2DsA2o+Bio3N21v0rGH/Ol1hRT+4eAxuXQVHd6jVGdp/Cnb/ZWeO+gP2zYfEI5CZAR61Dc9b3fyPVgViYZlj2u0sYXFhhEaEojA9M56iUhj+93CsLK0I8rvPPhSn4r/EqXQoqkQLgNlP3759lVJKTZs2TXl7eyt7e3sVHBysFi9erAB17do1pVT2xbwrVqxQVatWVba2tiooKEjFxcWZPM+GDRtUixYtlL29vXJxcVFNmjQxySLFHRfqzps3TwUEBChHR0fl4uKi2rdvrw4cOJDr66DVatWQIUOUi4uLcnNzU6GhoeqNN94wuTBcr9erGTNmqFq1ailra2vl7u6ugoOD1datW5VS2YkWssalVM4X5s+ePVtVrVpVWVtbq5o1a6rFixeb9OWPP/5Q1atXV1ZWVsrPzy/f479bUlKS6tChg3JyclKA2rJlS44XMF+7ds24PcuePXuM+zo6Oqr69eurzz//PNfnyrrYevbs2crHx0fZ2dmpF198UV29etVY5+5EC9HR0apZs2bK3t7e+BpNmDBB1alTR9nb26uyZcuq5557Tp0+fVopZX7xtZ+fX47zLmscWdnJKleurKytrZW3t7fq3r27+ueff3IdR1xcnOrWrZtydHRUzs7O6qWXXlKJiYnG7Tn9Pgubfe5ev4N3331XlStXTgFqzJgx+RpTThfGZ71Wd1/M36BBA2O7Sin10UcfqXLlyiknJyfVq1cvNX36dJO2CnJBfX7eT3cmT/j7779V27ZtVZkyZZS9vb2qX7++SRa5nObK7du3VZ8+fZSrq6tyc3NTAwYMUCNHjjTpY07JRoYMGWLy+7p165YaNmyY8vb2VjY2Nqp69erGbGZKKfXvv/+q7t27Kzc3N2Vvb69q166thg4dmq8kLjdu3FCvv/66cnBwUJ6enmrKlClmSSN27typ6tevr2xtbdXdH23h4eEKMEnWoFT2HFqyZIlq0qSJsrGxUf7+/sYslvnp+w8//KBatmyZY7t3zs1Vq1ap6tWrK3t7e/Xss8+qefPmmfTz9u3bqkePHsrNzc2YOfNeoqOjVatWrZSNjY2qWbOm2rBhg1mCme+//175+voqCwsLs/fX6NGjTRJRZCmqz5I7SaKFx0NJXzT/SLlyWqkDPymV8I9S1+KUOr5WqSnVlNp8R4bG5X2V2j1PqQuHlUqKVmrVAKW+8FUq2TQRkLp5Ralx5ZS6cVGptKtK7fleqXP7De3GbFHq68ZK/fZWdv11I5T6e7pS5/YpdfmU4TnHlVPqgmnyoKKUqctU7Ze1V08seiLHn3qL6qmgZUEqM5eEDMWlIIkWNErdtdBZFEpKSgqurq4kJyeb3UdJq9Wybt06GjVqxIIFC3j77beLfR29EEIIg59++olhw4Zx4cIF45IyMFyrWaVKFQ4ePEhAQECh2u7WrRutWrVi+PDhRdTbB6dv375cunTJ7CzXokWLGDp0aJ5nJQsqISGBefPmyeffIy7r+0znzp3NrjMU+bDhf3DhALy1Iefteh1M8oPOX0LAK9nlh5fCnu+hf3jO++2aCzu+htCo3J97VlOo+wIEjih8//OwN3Evb2186571FgQv4Ckv8/u5FZe8vp/fTZbPCSGEeCylpaWRkJDApEmTeOedd0wCoqLSqlUrXnnllXtXfIgkJydz5MgRlixZkueyPyFEEboSA6fCDMvfcqNNA70W7MuYlkevg9qdc94nJQGO/wl+edwcVa+H9FTzdovQpbRL965UgHolQVJyCyFECbkznfHdP3///XdJd++BiI+Pz/N1KEw68yxTpkyhdu3aeHl5MWrUqCLsdbbhw4ebpJ8vKr/88kuur0ndunXvq+3nnnuOjh078u6779KhQ4ci6rEQIkc/dIAJHvBNI/BrDu0+zr3u5jGGa4+qBmaXZabDqXDDdUN3Wv4WfOYF02qDrTN0+yb3dnd8DRmpULf7fQ0lL+4O7kVaryTImSIhhCghdydVudPdKesfVz4+Pnm+Dnfeh6egxo4dy9ixY3PdXrlyZbNU2Q+Lbt26mdyb6U73u2zpXum3+/TpQ58+fe7rOYQQ/3lpoeEszcWjsGk0lPkaWg01r/f3NDi6AvqsBes70trHbgPH8uBxV7Kk4InQdiRcOQXh42Dj/+BZ0wyyAPzzG2ydDC8vAafiC0gaeTTC08GTpLQks0QLABo0eDp40sgj9+ypJU2CIiGEKCGSztiQQlxeB3POzs44OzuXdDeEEPfL9b+Mlx61DdcM/TkEWrxnmso68mvYPgPeWA1eT5juH73O/CwRgLOn4ce9pmFZ3MJO0Ha44UxTliPL4Y/3oOePUK1dUY/MhKWFJSObjCQ0IhQNmhwDoxFNRmD5EN/kVZbPCSGEEEIIUdyU3nDN0J03M90+A7Z9Ca+vgAp3nUVRCqI35BwU3d0uGJbaZTmyHH4fBC/Oh5rBRdL9ewnyC2Ja4DQ8HDxMyl01rkxpPeXhTseNnCkSQgghhBCiaP2zDCyswLMuWNrAhYOGZW51X8i+T9H26bDlC+jxA7hVghv/3fDUxhFsnQz7aNOgUvPsdv/dBDeTwKeRod6lE4Zleb7NoIzff8/9G6x+FzpNggpPZrdrbQd2+bvXYm5ik2OZsGsCMddjSM1Ixd3Bnc5VOjMgYADWFtYE+QVx9fZVlp5YyrnUc2jQUE6Vw9PeM8f23tr4Fl2qdKF9pfaM/Hsk/177l+vp1ylrV5Z2vu0Y0mgITjZOgOE+SL9G/0r01Wgy9BlUc6vGwAYDaVkhjyQTBSBBUQm48waZQgghxONOPvdEqWNhCZEzDFnnlAI3X2jSH5oNyq6zdwHoMmDZG6b7th0J7UYZls7V6AiWd3xdt7aD/T8a0nvr0sGlgiGjXath2XX2LwJ9Jqz70PCTpcGr0H3OfQ3LysKKrlW74l/OH2cbZ6KvRjN251gUiiGNhhie/uJ+etbqSYBHABZ6Cz7b/BkDtwxk9XOr8XTMDo6S05M5mHSQKW2moNFoaOfbjvcavkcZuzLE34jn812fk7wrmSltphjbbe7TnCGNhuBs48zqU6sZ/NdglnReQp1yOd+gvkBju+8WRL7Z29tjbW19X3e4F0IIIR5F1tbWODg4lHQ3hHgwnuhh+MnLsCN5bz+xDtp8aFpWpQ3025z3fm+uvXf/CsnX2Rdf5+yMmz5OPuy9uJcDFw8Yyya3mWz8v1arpbt9dybfnMzuxN10q9bNuG3buW34l/WnvH15AHrV7mXS7su1X2bh0YXGshFNTO+xNKTRELbEbyHiXIQERY8aV1dXBg0aRFpaWkl3RTzkMjMz2b59O61atcLKSt6m4t5kzoiCetBzxsHBAVfX+1u6I0SpkZkB/t2gxsOdNj8+JZ7I85G0r9Q+1zpatGSqTFxtTN//W85uoV2lnBNAJKUlERYXxpNeT+barl7puZl506zdwpJPzgfM1dVVPhTEPWm1WhwcHPDy8pK7hot8kTkjCkrmjBAPMSsbCBxZ0r3I1evrXuf4leNk6DN4seaLDG44ONe6G29txN3enWY+zYxlGboMIs9HMrDBQJO6w7cOZ8vZLdzW3SawYiDjWozLtd1FxxaRpk0juHLRJJKQ7HNCCCGEEEKIfPuq7Vcs67qMya0ns+3cNhYdW5RjvYXHFnJEe4SvWn+FraWtsXx3wm7K2pWlehnTWzIMbzKcX7v+ytftvubsjbN8uffLHNtde3otcw/P5au2X1HOvlyRjEnOFAkhhBBCCCHyzcvRcD+kam7V0Ckd43eOJ8Q/xOQ+RIuOLmJh1EL6OPahZpmaJvtHnI0g0DfQrN3y9uUpb1+eqq5VcbV1JWRDCO/Ufwd3h+wbz66PXc/YHWOZGjiV5j7NzdooLDlTJIQQQgghhCgUhSJTn4me7PsvLTi6gO/++Y5v231LBasKpvWVIuJcBE9XejrPdvX/3X8pQ59hLFt3eh2jI0czuc1k2lRsU4SjkDNFQgghhBBCiHxYc3oNVhZW1HSribWlNceuHGPm/pkEVwnG2sJwbeL8I/OZdWgWk9tMxsfRh+P641y+dRlXXHGwdiDqShS3M2/T0KOhsd1t57Zx5dYVnij/BA7WDsRcj2Hqvqk09GhIBSdDULX29Fo+2f4JI5qMoL57fS7fugyAraUtzjbO9z02CYqEEEIIIYQQ92SlsWLBkQXEpcShUPg4+vBKnVfo7d/bWGdZ9DK0ei2hEaHGssmrJjOgwQAGBgzkr7N/0bpia6wsssMQO0s7VpxcwZd7vyRDn4GXoxftK7Wnb72+xjrL/11Opsrk892f8/nuz43l3ap14/NW2Y8LPbb7bkEIIYQQQgjx2OtUpROdqnTKs87GFzca/6/Valm3bh2dO3c2ZrnccnYLb9d/22SfJt5N+Nn75zzbXdhpYZ7b75cERUVEKQVASkqK2TatVktaWhopKSmS9lTki8wZUVAyZ0RByZwRBSVzRhTU3XNGq9fSxrMNDZwb5PiduahlPUfW9/S8aFR+aol7OnfuHL6+vveuKIQQQgghhHhgzp49S8WKFfOsI0FREdHr9Vy4cAFnZ2c0Go3JtpSUFHx9fTl79iwuLi4l1EPxKJE5IwpK5owoKJkzoqBkzoiCKuk5o5Tixo0b+Pj4YGGRd9JtWT5XRCwsLO4Zgbq4uMhBRBSIzBlRUDJnREHJnBEFJXNGFFRJzhlXV9d81ZP7FAkhhBBCCCFKNQmKhBBCCCGEEKWaBEUPgK2tLWPGjMHW1rakuyIeETJnREHJnBEFJXNGFJTMGVFQj9KckUQLQgghhBBCiFJNzhQJIYQQQgghSjUJioQQQgghhBClmgRFQgghhBBCiFJNgiIhhBBCCCFEqSZBUSHNmjWLypUrY2dnR9OmTdmzZ0+udQMDA9FoNGY/Xbp0Mdbp06eP2fZOnTo9iKGIB6QgcwZgxowZ1KpVC3t7e3x9fRk2bBi3b9++rzbFo6Wo58zYsWPNjjO1a9cu7mGIB6ggc0ar1TJ+/HiqVauGnZ0dDRo0YMOGDffVpnj0FPWckePM42vbtm107doVHx8fNBoNq1evvuc+ERERNGrUCFtbW6pXr86iRYvM6jw0xxglCmzp0qXKxsZGLViwQB07dkz1799fubm5qYsXL+ZY/8qVKyohIcH4c/ToUWVpaakWLlxorBMSEqI6depkUu/q1asPaESiuBV0zvzyyy/K1tZW/fLLLyo2NlZt3LhReXt7q2HDhhW6TfFoKY45M2bMGFW3bl2T48ylS5ce1JBEMSvonBk+fLjy8fFRa9euVTExMWr27NnKzs5OHThwoNBtikdLccwZOc48vtatW6c+/vhjtXLlSgWoVatW5Vn/9OnTysHBQYWGhqqoqCj1zTffKEtLS7VhwwZjnYfpGCNBUSE0adJEDRo0yPhYp9MpHx8fNXHixHztP336dOXs7KxSU1ONZSEhIeq5554r6q6Kh0RB58ygQYPU008/bVIWGhqqWrZsWeg2xaOlOObMmDFjVIMGDYqlv6LkFXTOeHt7q2+//dak7IUXXlCvvfZaodsUj5bimDNynCkd8hMUDR8+XNWtW9ekrFevXio4ONj4+GE6xsjyuQLKyMhg//79BAUFGcssLCwICgpi586d+Wpj/vz5vPzyyzg6OpqUR0RE4OHhQa1atRgwYABXrlwp0r6LklGYOdOiRQv2799vPIV8+vRp1q1bR+fOnQvdpnh0FMecyXLy5El8fHyoWrUqr732GvHx8cU3EPHAFGbOpKenY2dnZ1Jmb2/P9u3bC92meHQUx5zJIscZAbBz506T+QUQHBxsnF8P2zFGgqICunz5MjqdDk9PT5NyT09PEhMT77n/nj17OHr0KP369TMp79SpE4sXLyY8PJzJkyezdetWnnnmGXQ6XZH2Xzx4hZkzr776KuPHj6dVq1ZYW1tTrVo1AgMD+d///lfoNsWjozjmDEDTpk1ZtGgRGzZsYM6cOcTGxtK6dWtu3LhRrOMRxa8wcyY4OJhp06Zx8uRJ9Ho9mzdvZuXKlSQkJBS6TfHoKI45A3KcEdkSExNznF8pKSncunXroTvGSFD0gM2fP5969erRpEkTk/KXX36Zbt26Ua9ePZ5//nnWrFnD3r17iYiIKJmOihIVERHBF198wezZszlw4AArV65k7dq1TJgwoaS7Jh5S+ZkzzzzzDC+99BL169cnODiYdevWcf36dZYtW1aCPRclZebMmdSoUYPatWtjY2PD4MGDefPNN7GwkK8GImf5mTNynBGPKjnyFVD58uWxtLTk4sWLJuUXL17Ey8srz31v3rzJ0qVL6du37z2fp2rVqpQvX55Tp07dV39FySvMnBk9ejS9e/emX79+1KtXj+7du/PFF18wceJE9Hr9fc1D8fArjjmTEzc3N2rWrCnHmcdAYeaMu7s7q1ev5ubNm8TFxXHixAmcnJyoWrVqodsUj47imDM5keNM6eXl5ZXj/HJxccHe3v6hO8ZIUFRANjY2NG7cmPDwcGOZXq8nPDyc5s2b57nvb7/9Rnp6Oq+//vo9n+fcuXNcuXIFb2/v++6zKFmFmTNpaWlmf621tLQEQCl1X/NQPPyKY87kJDU1lZiYGDnOPAbu55hgZ2dHhQoVyMzMZMWKFTz33HP33aZ4+BXHnMmJHGdKr+bNm5vML4DNmzcb59dDd4x54KkdHgNLly5Vtra2atGiRSoqKkq9/fbbys3NTSUmJiqllOrdu7caOXKk2X6tWrVSvXr1Miu/ceOG+vDDD9XOnTtVbGysCgsLU40aNVI1atRQt2/fLvbxiOJX0DkzZswY5ezsrP7v//5PnT59Wm3atElVq1ZN9ezZM99tikdbccyZDz74QEVERKjY2FgVGRmpgoKCVPny5VVSUtIDH58oegWdM7t27VIrVqxQMTExatu2berpp59WVapUUdeuXct3m+LRVhxzRo4zj68bN26ogwcPqoMHDypATZs2TR08eFDFxcUppZQaOXKk6t27t7F+Vkrujz76SB0/flzNmjUrx5TcD8sxRoKiQvrmm29UpUqVlI2NjWrSpInatWuXcVvbtm1VSEiISf0TJ04oQG3atMmsrbS0NNWxY0fl7u6urK2tlZ+fn+rfv7986DxmCjJntFqtGjt2rKpWrZqys7NTvr6+auDAgSYfPPdqUzz6inrO9OrVS3l7eysbGxtVoUIF1atXL3Xq1KkHOCJR3AoyZyIiIlSdOnWUra2tKleunOrdu7c6f/58gdoUj76injNynHl8bdmyRQFmP1lzJCQkRLVt29Zsn4CAAGVjY6OqVq1qco/OLA/LMUajVC7rKoQQQgghhBCiFJBrioQQQgghhBClmgRFQgghhBBCiFJNgiIhhBBCCCFEqSZBkRBCCCGEEKJUk6BICCGEEEIIUapJUCSEEEIIIYQo1SQoEkIIIYQQQpRqEhQJIYQQQgghSjUJioQQQoj7MHbsWAICAoyP+/Tpw/PPP19i/RFCCFFwEhQJIYQQQgghSjUJioQQQjy2MjIySroLQgghHgESFAkhhHhsBAYGMnjwYIYOHUr58uUJDg7m6NGjPPPMMzg5OeHp6Unv3r25fPmycR+9Xs+UKVOoXr06tra2VKpUic8//9y4fcSIEdSsWRMHBweqVq3K6NGj0Wq1JTE8IYQQxUSCIiGEEI+VH3/8ERsbGyIjI5k0aRJPP/00DRs2ZN++fWzYsIGLFy/Ss2dPY/1Ro0YxadIkRo8eTVRUFEuWLMHT09O43dnZmUWLFhEVFcXMmTP5/vvvmT59ekkMTQghRDHRKKVUSXdCCCGEKAqBgYGkpKRw4MABAD777DP+/vtvNm7caKxz7tw5fH19iY6OxtvbG3d3d7799lv69euXr+f46quvWLp0Kfv27QMMiRZWr17NoUOHAEOihevXr7N69eoiHZsQQojiY1XSHRBCCCGKUuPGjY3/P3z4MFu2bMHJycmsXkxMDNevXyc9PZ327dvn2t6vv/7K119/TUxMDKmpqWRmZuLi4lIsfRdCCFEyJCgSQgjxWHF0dDT+PzU1la5duzJ58mSzet7e3pw+fTrPtnbu3Mlrr73GuHHjCA4OxtXVlaVLlzJ16tQi77cQQoiSI0GREEKIx1ajRo1YsWIFlStXxsrK/COvRo0a2NvbEx4enuPyuR07duDn58fHH39sLIuLiyvWPgshhHjwJNGCEEKIx9agQYO4evUqr7zyCnv37iUmJoaNGzfy5ptvotPpsLOzY8SIEQwfPpzFixcTExPDrl27mD9/PmAImuLj41m6dCkxMTF8/fXXrFq1qoRHJYQQoqhJUCSEEOKx5ePjQ2RkJDqdjo4dO1KvXj2GDh2Km5sbFhaGj8DRo0fzwQcf8Omnn1KnTh169epFUlISAN26dWPYsGEMHjyYgIAAduzYwejRo0tySEIIIYqBZJ8TQgghhBBClGpypkgIIYQQQghRqklQJIQQQgghhCjVJCgSQgghhBBClGoSFAkhhBBCCCFKNQmKhBBCCCGEEKWaBEVCCCGEEEKIUk2CIiGEEEIIIUSpJkGREEIIIYQQolSToEgIIYQQQghRqklQJIQQQgghhCjVJCgSQgghhBBClGr/DxwjjyFRq/BaAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots(1, 1, figsize=plt.figaspect(1/2))\n", - "fig.suptitle(\n", - " f'Effects of search parameters on QPS/recall trade-off ({DATASET_FILENAME})\\n' + \\\n", - " f'k = {k}, n_probes = {n_probes}, pq_dim = {pq_dim}')\n", - "labels = []\n", - "for j, ratio in enumerate(ratios):\n", - " ax.plot(bench_recall_sr[j, :], bench_qps_sr[j, :], 'o')\n", - " labels.append(f\"refine ratio = {ratio}\")\n", - "ax.legend(labels)\n", - "ax.set_xlabel('recall')\n", - "ax.set_ylabel('QPS')\n", - "ax.grid()\n", - "colors = plt.rcParams[\"axes.prop_cycle\"].by_key()[\"color\"]\n", - "annotations = []\n", - "for j, ratio in enumerate(ratios):\n", - " for i, label in enumerate(bench_names):\n", - " annotations.append(ax.text(\n", - " bench_recall_sr[j, i], bench_qps_sr[j, i],\n", - " f\" {label} \",\n", - " color=colors[j],\n", - " ha='center', va='center'))\n", - "clutter = [\n", - " ax.text(\n", - " 0.02, 0.08,\n", - " 'Labels denote the bitsize of: internal_distance_dtype/lut_dtype',\n", - " verticalalignment='top',\n", - " bbox={'facecolor': 'white', 'edgecolor': 'grey'},\n", - " transform = ax.transAxes)\n", - "]\n", - "adjust_text(annotations, objects=clutter);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Depending on the dataset, you may see very different pictures here. For SIFT-128, we pick three interesting candidates candidates featuring compromizes between the QPS and the recall:\n", - " - `internal_distance_dtype = 16, lut_dtype = 16`\n", - " - `internal_distance_dtype = 32, lut_dtype = 8`\n", - " - `internal_distance_dtype = 32, lut_dtype = 8, refine_ratio = 2`\n", - "\n", - "This is all for the search parameters, but we will come back to the look-up table question in the next section." - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "def search_refine(internal_distance_dtype, lut_dtype, ratio, n_probes):\n", - " k_search = k * ratio\n", - " ps = ivf_pq.SearchParams(\n", - " n_probes=n_probes,\n", - " internal_distance_dtype=internal_distance_dtype,\n", - " lut_dtype=lut_dtype)\n", - " candidates = ivf_pq.search(ps, index, queries, k_search, handle=resources)[1]\n", - " return candidates if ratio == 1 else refine(dataset, queries, candidates, k, handle=resources)[1]\n", - "\n", - "search_configs = [\n", - " lambda n_probes: search_refine(np.float16, np.float16, 1, n_probes),\n", - " lambda n_probes: search_refine(np.float32, np.uint8, 1, n_probes),\n", - " lambda n_probes: search_refine(np.float32, np.uint8, 2, n_probes)\n", - "]\n", - "search_config_names = [\n", - " '16/16', '32/8', '32/8/r2'\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tweaking indexing parameters\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Deciding on the indexing parameters is a bit more involved than on the search parameters. This is obviously because `ivf_pq.IndexParams` has more members than `ivf_pq.SearchParams`, but also because the try-test loop takes longer time when it includes training.\n", - "Since RAFT's IVF-PQ algorithm uses balanced-hierarchical k-means clustering and efficient logic for encoding, we find significantly improved index build times.\n", - "\n", - "First of all, let's pick the parameters we __don't need__ to tweak:\n", - "\n", - " - `metric` - the distance metric often depens on the problem and thus fixed (currently RAFT supports variations of eucliean and inner product distances).\n", - " - `conservative_memory_allocation` only affects how data is allocated - does not affect the search performance.\n", - " - `add_data_on_build` is a convenience flag. When activated, it automatically adds the training data to the index during `ivf_pq.build`. Otherwise, no data is added during `ivf_pq.build` and vectors need to be explicitly added to the index using `ivf_pq.extend`.\n", - " - `force_random_rotation` may slightly affect performance when the data dimensionality is a power of two (see the module docs), but normally you don't need to change the defaults. \n", - "\n", - "The rest of the parameters can be divided in two categories: influencing the coarse search (`kmeans_n_iters`, `kmeans_trainset_fraction` , `n_lists`) and the fine search / product quantization (`codebook_kind`, `pq_dim`, `pq_bits`)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Indexing parameters affecting the coarse search\n", - "\n", - "#### n_lists\n", - "\n", - "`n_lists` is the first parameter to look at. It has a profound impact on overall performance during both training and search.\n", - "`n_lists` defines the number of clusters into which the index data is partitioned; you should keep this in mind when selecting the `n_probes` search parameter.\n", - "\n", - "The ratio `n_probes/n_lists` tells how large fraction of the dataset is compared to each query. If `n_lists == n_probes`, that is like a brute force search: we compare all dataset vectors to all query vectors. One would expect the recall is equal to `1` in such a case, but that does not take into account the PQ compression, which is lossy; in reality the recall is always lower unless you refine the search results.\n", - "\n", - "As `n_probes` approaches `n_lists`, IVF-PQ becomes slower than brute force because of all the extra work the algorithm does: dimension padding / transform, two-step search, extra PQ compute, etc. In practice searching around 0.1-1% of lists is enough for many datasets. But this depends on how well the input can be clustered. (e.g. for uniform random numbers as inputs, IVF methods don't work well).\n", - "\n", - "`n_lists = sqrt(n_samples)` is a good starting point for the balance of coarse/fine search time. To make sure the GPU resources are utilized efficiently, keep in mind:\n", - " - The average cluster size (i.e. `n_smaples / n_lists`) should be in the range of at least ~2k records to keep individual SMs busy\n", - " - Total amount of search work (`n_queries * n_probes`) should be a good multiple of number of SMs\n" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4.36 ms ± 2.38 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "4.36 ms ± 1.9 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "4.37 ms ± 2.47 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "7.74 ms ± 19.7 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "33.8 ms ± 733 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "44.1 ms ± 714 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "1.83 ms ± 1.66 µs per loop (mean ± std. dev. of 7 runs, 1,000 loops each)\n", - "3.1 ms ± 14.1 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "6.43 ms ± 16.8 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "11.9 ms ± 33 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "45.2 ms ± 622 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "87.3 ms ± 153 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "2.55 ms ± 452 ns per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "5.1 ms ± 11.1 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "9.32 ms ± 15.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "16.1 ms ± 34.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "74 ms ± 254 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "145 ms ± 295 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "3.92 ms ± 5.94 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "8.12 ms ± 6.62 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "14.7 ms ± 23.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "27.8 ms ± 131 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "132 ms ± 289 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "259 ms ± 3.04 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "7.49 ms ± 4.68 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "17.2 ms ± 48.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "32.4 ms ± 111 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "63 ms ± 149 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "303 ms ± 2.32 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "603 ms ± 1.78 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "n_list_variants = [100, 500, 1000, 2000, 5000]\n", - "pl_ratio_variants = [500, 200, 100, 50, 10, 5]\n", - "selected_search_variant = 1\n", - "search_fun = search_configs[selected_search_variant]\n", - "search_label = search_config_names[selected_search_variant]\n", - "\n", - "bench_qps_nl = np.zeros((len(n_list_variants), len(pl_ratio_variants)), dtype=np.float32)\n", - "bench_recall_nl = np.zeros_like(bench_qps_nl, dtype=np.float32)\n", - "\n", - "for i, n_lists in enumerate(n_list_variants):\n", - " index_params = ivf_pq.IndexParams(n_lists=n_lists, metric=metric, pq_dim=pq_dim)\n", - " index = ivf_pq.build(index_params, dataset, handle=resources)\n", - " for j, pl_ratio in enumerate(pl_ratio_variants):\n", - " n_probes = max(1, n_lists // pl_ratio)\n", - " r = %timeit -o search_fun(n_probes); resources.sync()\n", - " bench_qps_nl[i, j] = (queries.shape[0] * r.loops / np.array(r.all_runs)).mean()\n", - " bench_recall_nl[i, j] = calc_recall(search_fun(n_probes), gt_neighbors)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzkAAAHgCAYAAACGvKPXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAADZlklEQVR4nOzdd3hUVfrA8e+0zGTSe2dSKEkAKUmkKE3pTSxgWwHRVde6+rOsu66KbVdR1wKuomvDDhZUQJqgFFFChyQQQhrpvZeZzP39McmQIQEChBbez/PcB3LunXvPvXOnvHPOeY9KURQFIYQQQgghhOgi1Oe6AkIIIYQQQgjRmSTIEUIIIYQQQnQpEuQIIYQQQgghuhQJcoQQQgghhBBdigQ5QgghhBBCiC5FghwhhBBCCCFElyJBjhBCCCGEEKJLkSBHCCGEEEII0aVIkCOEEEIIIYToUiTIEReU6upqbr/9dgIDA1GpVPz1r38FoKCggOuuuw4fHx9UKhWvvfbaOa1nZ0tNTWXs2LF4eHigUqn47rvvzvgx169fj0qlYv369fay2bNnEx4efsaPLc4vH374ISqVioyMDHvZyJEjGTly5Dmr0/E8/fTTqFSqc3b8rVu3MnToUFxcXFCpVOzcuROAn376if79+2MwGFCpVJSXlx93Py+99BLR0dFYrdYOH7u91y3AokWLiI6ORqfT4enpeXInJNrIyMhApVLx4Ycf2stO5r5TqVQ8/fTTZ6ZynaDlNZ+YmHjK+wgPD2f27NkOZafzWWY2mwkLC+Ott9465TqJi4sEOeKca3kzPdayZcsW+7YvvPACH374IX/5y19YtGgRt9xyCwAPPvggK1eu5PHHH2fRokWMHz++0+v5wgsvnJXgoj2zZs1iz549PP/88yxatIj4+PhzUo9TkZSUxNNPP+3wBfl88uOPPzJ+/Hh8fHwwGAz07NmTRx55hNLS0jbbzp492+HedHd3p1+/frzyyis0NDQ4bLtx40YmTJhASEgIBoOBbt26MWXKFD777LN26/F///d/xMbGnpFzPJfO5evmXDCbzUyfPp3S0lL+85//sGjRIkwmEyUlJcyYMQNnZ2cWLFjAokWLcHFxOeZ+KisrefHFF3nsscdQq0/vozolJYXZs2cTFRXFu+++y8KFC6mtreXpp59uEwwdz/79+3nwwQcZOnSoPVBr73VdUlLCvHnzGD58OH5+fnh6ejJ48GC+/PLLdvebmprKDTfcQGhoKEajkejoaJ555hlqa2tP8YzF+aq9z7KWwPxEn/86nY6HHnqI559/nvr6+nN4FuJCoT3XFRCixTPPPENERESb8u7du9v///PPPzN48GCeeuoph21+/vlnrrrqKh5++OEzVr8XXniB6667jmnTpp2xY7Snrq6O3377jX/84x/ce++9Z/XYR3v33XdP6ldlsAU5c+fOZeTIkeddK9DDDz/MK6+8Qr9+/Xjsscfw9vZm+/btvPnmm3z55ZesXbuWHj16ODxGr9fz3nvvAVBeXs7XX3/Nww8/zNatW/niiy8AWLx4Mddffz39+/fngQcewMvLi/T0dH799VfeffddbrrppjZ1WbZsGVOmTDnzJ32WnavXzbmSlpZGZmYm7777Lrfffru9/KeffqKqqopnn32W0aNHn3A/77//PhaLhRtvvPGkjj98+HDq6upwcnKyl61fvx6r1crrr79ufz8tLi5m7ty5AB1ukfvtt9944403iI2NJSYmxt5C1d52//jHP5g4cSJPPPEEWq2Wr7/+mhtuuMH+ftAiOzubSy+9FA8PD+699168vb357bffeOqpp9i2bRtLly49qfM/l5544gn+9re/netqnLeO9Vl28OBBAO6//34SEhIcHtP68x/g1ltv5W9/+xufffYZc+bMOfOVFhc0CXLEeWPChAknbKEoLCxs99fuwsLCLtsFo6ioCOC8OD+dTneuq9BpPv/8c1555RWuv/56Pv30UzQajX3d7NmzGTVqFNOnTycxMRGt9shbpVar5U9/+pP977vvvptBgwbx5Zdf8uqrrxIcHMzTTz9NbGwsW7ZscfiyCbZ79WiHDh1i//79vP3228esb01NzXF/+e8KusI5tjy/R79ej1V+LB988AFTp07FYDCc1PHVanWbx5zssY9l6tSplJeX4+bmxssvv3zMIKd3796kpqZiMpnsZXfffTejR4/mxRdf5NFHH7U/z4sWLaK8vJyNGzfSu3dvAO644w6sVisff/wxZWVleHl5nVa9zxatVuvwXiEcneizbNiwYVx33XXH3Yenpydjx47lww8/lCBHnJB0VxMXhJbm7PT0dJYtW2Zvym7p6qYoCgsWLLCXtygvL+evf/0rYWFh6PV6unfvzosvvtimNaLlV86+fftiMBjw8/Nj/Pjx9v7IKpWKmpoaPvroI/sxWvoaV1VV8de//pXw8HD0ej3+/v6MGTOG7du3n/C8duzYwYQJE3B3d8fV1ZUrr7zSoXn+6aeftn9ReOSRR1CpVMdtDWm5Tl999RXPP/88oaGhGAwGrrzySvuvZaejvTE5X3zxBXFxcbi5ueHu7k7fvn15/fXXAVtXxOnTpwMwatQo+7U7UReZn3/+mWHDhuHi4oKnpydXXXUVycnJDtu09H8/ePAgs2fPxtPTEw8PD2699dYOdXOZO3cuXl5eLFy40CHAAbj00kt57LHH2LVrF998881x96NWq+2/hLd03UlLSyMhIaFNgAPg7+/fpmzZsmV4eHhw+eWXO5xbUlISN910E15eXvZ1AJ988glxcXE4Ozvj7e3NDTfcQHZ2dpv9/v7770ycOBEvLy9cXFy45JJL7M8NwO7du5k9ezaRkZEYDAYCAwOZM2cOJSUlxz3njjre6+Z453gy9dq4cSMJCQkYDAaioqJ45513jlmfjl63YznRfTl79mxGjBgBwPTp01GpVPaxS7NmzQIgISHB4Tq0Jz09nd27d7fb4nO81xu0HZMTHh5ub/n28/OzH9vPzw+wvQ5anpsTjRHx9vbGzc3thNcpIiLCIcAB270wbdo0GhoaOHTokL28srISgICAAIftg4KCUKvV7b6Gjma1Wnnttdfo3bs3BoOBgIAA7rzzTsrKytrUob1zbG/sSHl5OQ8++KD9fT00NJSZM2dSXFx8zHq0NyanoaGBBx98ED8/P9zc3Jg6dSqHDx9u9/E5OTnMmTOHgIAA9Ho9vXv35v3333fYprGxkSeffJK4uDg8PDxwcXFh2LBhrFu3zmG7ljFDL7/8MgsXLiQqKgq9Xk9CQgJbt2495jkcraGhgYceegg/Pz9cXFy4+uqr7cFKC0VReO655+zdDUeNGsW+ffvaXJuOfJZVVVVhsViOW6cxY8awcePGdrsUC9Ga/OQgzhsVFRVtPkBUKhU+Pj7ExMSwaNEiHnzwQUJDQ/m///s/AAYMGGAfmzNmzBhmzpxpf2xtbS0jRowgJyeHO++8k27durF582Yef/xx8vLyHJIT3HbbbXz44YdMmDCB22+/HYvFwoYNG9iyZQvx8fEsWrSI22+/nUsvvZQ77rgDgKioKADuuusulixZwr333ktsbCwlJSVs3LiR5ORkBg4ceMzz3bdvH8OGDcPd3Z1HH30UnU7HO++8w8iRI/nll18YNGgQ11xzDZ6enjz44IPceOONTJw4EVdX1xNey3//+9+o1WoefvhhKioqeOmll7j55pv5/fffO/x8dMTq1au58cYbufLKK3nxxRcBSE5OZtOmTTzwwAMMHz6c+++/nzfeeIO///3vxMTEANj/bc+aNWuYMGECkZGRPP3009TV1fHmm29y2WWXsX379jYfjDNmzCAiIoJ//etfbN++nffeew9/f397fdqTmprK/v37mT17Nu7u7u1uM3PmTJ566il++OEHZsyYcdzrkJaWBoCPjw8AJpOJtWvXcvjwYUJDQ4/7WIDly5czZsyYNr8CT58+nR49evDCCy+gKAoAzz//PP/85z+ZMWMGt99+O0VFRbz55psMHz6cHTt22H8lXb16NZMnTyYoKIgHHniAwMBAkpOT+fHHH3nggQfs2xw6dIhbb72VwMBA9u3bx8KFC9m3bx9btmw57cH7x3vdHO8cO1qvPXv2MHbsWPz8/Hj66aexWCw89dRTbb4wn8x1O5aO3Jd33nknISEhvPDCC/auNy116dWrFwsXLrR3yz36OrS2efNmgDbvHyd6vbXntdde4+OPP+bbb7/lv//9L66urvTt25fBgwfzl7/8hauvvpprrrkGgEsuueS41+B05efnA+Dr62svGzlyJC+++CK33XYbc+fOxcfHh82bN/Pf//6X+++/v0Mte3feeScffvght956K/fffz/p6enMnz+fHTt2sGnTppNuga6urmbYsGEkJyczZ84cBg4cSHFxMd9//z2HDx92qP+J3H777XzyySfcdNNNDB06lJ9//plJkya12a6goIDBgwejUqm499578fPzY8WKFdx2221UVlbak+xUVlby3nvvceONN/LnP/+Zqqoq/ve//zFu3Dj++OMP+vfv77Dfzz77jKqqKu68805UKhUvvfQS11xzDYcOHerQdbnvvvvw8vLiqaeeIiMjg9dee417773XYXzVk08+yXPPPcfEiROZOHEi27dvZ+zYsTQ2Ntq36chn2a233kp1dTUajYZhw4Yxb968dnt3xMXFoSgKmzdvZvLkySc8B3ERU4Q4xz744AMFaHfR6/UO25pMJmXSpElt9gEo99xzj0PZs88+q7i4uCgHDhxwKP/b3/6maDQaJSsrS1EURfn5558VQLn//vvb7Ndqtdr/7+LiosyaNavNNh4eHm2O3RHTpk1TnJyclLS0NHtZbm6u4ubmpgwfPtxelp6ergDKvHnzTrjPdevWKYASExOjNDQ02Mtff/11BVD27NnT4fq17GvdunX2slmzZikmk8n+9wMPPKC4u7srFovlmPtZvHhxm/0cT//+/RV/f3+lpKTEXrZr1y5FrVYrM2fOtJc99dRTCqDMmTPH4fFXX3214uPjc9xjfPfddwqg/Oc//znudu7u7srAgQPtf8+aNUtxcXFRioqKlKKiIuXgwYPKCy+8oKhUKuWSSy6xb/e///1PARQnJydl1KhRyj//+U9lw4YNSlNTU5tj1NTUKAaDQfnggw/anNuNN97osG1GRoai0WiU559/3qF8z549ilartZdbLBYlIiJCMZlMSllZmcO2re/p2traNvX5/PPPFUD59ddf7WUtr9H09HR72YgRI5QRI0a0vWhHOdbr5ljneDL1mjZtmmIwGJTMzEx7WVJSkqLRaJTWH28dvW7H09H7suV1s3jxYofHt1zDrVu3nvBYTzzxhAIoVVVVDuUdeb2197ptudZFRUX2sqKiIgVQnnrqqRPWpz3z5s1rc08cT0lJieLv768MGzaszbpnn31WcXZ2dnjv/8c//tGh/W7YsEEBlE8//dSh/KeffmpTfqzzNZlMDvfok08+qQDKN99802bbltdPy/tye6/bFjt37lQA5e6773bYx0033dSmLrfddpsSFBSkFBcXO2x7ww03KB4eHvbXhMVicXhvVxRFKSsrUwICAhzeC1vq5+Pjo5SWltrLly5dqgDKDz/80ObcWmu5X0ePHu3wnvHggw8qGo1GKS8vVxRFUQoLCxUnJydl0qRJDtv9/e9/VwCH63qsz7JNmzYp1157rfK///1PWbp0qfKvf/1L8fHxUQwGg7J9+/Y2dcvNzVUA5cUXXzzuOQgh3dXEeWPBggWsXr3aYVmxYsUp72/x4sUMGzYMLy8viouL7cvo0aNpamri119/BeDrr79GpVK1SWYAdOiXbE9PT37//Xdyc3M7XLempiZWrVrFtGnTiIyMtJcHBQVx0003sXHjRns3jlNx6623OnTzGDZsGIBDN5HO4OnpSU1NDatXr+6U/eXl5bFz505mz56Nt7e3vfySSy5hzJgxLF++vM1j7rrrLoe/hw0bRklJyXGvX1VVFcAJu964ubnZt21RU1ODn58ffn5+dO/enb///e8MGTKEb7/91r7NnDlz+Omnnxg5ciQbN27k2WefZdiwYfTo0cP+K32Ln3/+mYaGBiZMmHDCc/vmm2+wWq3MmDHD4Z4ODAykR48e9i4rO3bsID09nb/+9a9tWiha39POzs72/9fX11NcXMzgwYMBOtTdsjMcfY4drVdTUxMrV65k2rRpdOvWzb59TEwM48aNc9hfR6/bsZzKfXk6SkpK0Gq1bX7p7uzX29litVq5+eabKS8v580332yzPjw8nOHDh7Nw4UK+/vpr5syZwwsvvMD8+fNPuO/Fixfj4eHBmDFjHJ7buLg4XF1dT/jctufrr7+mX79+XH311W3WnUzrZst9cf/99zuUt7TKtFAUha+//popU6agKIrDeYwbN46Kigr7fa/RaOzv7VarldLSUiwWC/Hx8e2+Zq+//nqHMU0n+1lwxx13OJzzsGHDaGpqIjMzE7C1cDY2NnLfffc5bHf0OR7P0KFDWbJkCXPmzGHq1Kn87W9/s7fYPv744222bzmf43UdFAKku5o4j1x66aWdmho5NTWV3bt32/ueH61lMG5aWhrBwcEOX15OxksvvcSsWbMICwsjLi6OiRMnMnPmTIfg5WhFRUXU1tbSq1evNutiYmKwWq1kZ2fbB+KerNZf+uDIh8LRfdRP1913381XX31lT5U8duxYZsyYccopvFs+OI91XVauXNlmcPrxzvVYXdFagpujA5ijVVVVtekeZzAY+OGHHwBbprWIiIh2u6SNGzeOcePGUVtby7Zt2/jyyy95++23mTx5MikpKfaxOcuWLSM+Pr7dLlZHZxtMTU1FUZQ2Gd9atHQ/aek+16dPn+OeX2lpKXPnzuWLL75okxChoqLiuI/tLO1lVOxIvYqKiqirq2v3WvTq1csh8Ojodauurqa6utpertFo8PPzO6X78kzo7Ndbe+rq6to894GBgae1z/vuu4+ffvqJjz/+mH79+jms++KLL7jjjjs4cOCA/XV0zTXXYLVaeeyxx7jxxhvx8fGhtLTUofuTs7MzHh4epKamUlFR0e5YN2g/0ceJpKWlce211570446WmZmJWq1u0zXx6PuoqKiI8vJyFi5cyMKFC9vdV+vz+Oijj3jllVdISUnBbDbby9t7LZ3uZ8GJHt/y2jj6teXn53daCSO6d+/OVVddxTfffENTU5PDuEmluVvruZwLS1wYJMgRXZbVamXMmDE8+uij7a7v2bNnpxxnxowZDBs2jG+//ZZVq1Yxb948XnzxRb755pt2f50/G44eSN+i5cOhs/j7+7Nz505WrlzJihUrWLFiBR988AEzZ87ko48+6tRjHcupnGtLhr7du3cfc5vMzEwqKyvbBKsajaZDKYBbGI1Ghg0bxrBhw/D19WXu3LmsWLHCPhB9+fLl3Hrrre0+tnWLBtjuaZVKxYoVK9o9746M12ptxowZbN68mUceeYT+/fvj6uqK1Wpl/PjxJ50q/FQdfY5nol4dvW4vv/yyQ3pjk8l0TuZ38vHxwWKxUFVV5dDaeDZeb19++WWb+/F03jfmzp3LW2+9xb///W/7vGatvfXWWwwYMKDNDwVTp07lww8/ZMeOHYwePZprrrmGX375xb5+1qxZfPjhh1itVvz9/fn000/bPf6xfuRqramp6STPqnO13NN/+tOf7O8LR2sZL/XJJ58we/Zspk2bxiOPPIK/vz8ajYZ//etf9h83Wjvdz4Kz9VnSnrCwMBobG6mpqXH4waolwDqZsVHi4iRBjuiyoqKiqK6uPuEX0qioKFauXElpaelxW3OO96tRUFAQd999N3fffTeFhYUMHDiQ559//phBjp+fH0ajkf3797dZl5KSglqtJiws7Lj1Pl84OTkxZcoUpkyZgtVq5e677+add97hn//8J927dz+pX9tasu8c67r4+vp2yq/lPXr0oFevXnz33Xe8/vrr7XZb+/jjjwHs2eE6Q0tLZV5eHgB79+4lKyur3YHI7YmKikJRFCIiIo4bpLf8crx3795j3v9lZWWsXbuWuXPn8uSTT9rLU1NTO1SXjjrZX1s7Wi8/Pz+cnZ3bre/R909Hr9vMmTMdsti1BGBn675sER0dDdiyrB2dDOBEr7eOOtbzMm7cuE7rDrdgwQKefvpp/vrXv/LYY4+1u01BQUG7v/i3tFC0ZNp65ZVXHFofgoODAdtzu2bNGi677LJ2A+bWvLy8KC8vdyhrbGy0vx5bREVFsXfv3uOfXAeYTCasVitpaWkOrTdH30ctmdeamppO+Hm1ZMkSIiMj+eabbxyew/a6W58NLa+N1NRUhx+EioqKTrvnwKFDhzAYDG1+vElPTweOn8BGCJAU0qILmzFjBr/99hsrV65ss668vNz+4XnttdeiKIrDL7gtWv9a5eLi0uYDsqmpqU3XDn9/f4KDg2loaDhm3TQaDWPHjmXp0qUOvxQXFBTw2Wefcfnllx+zq9X55OiUvmq12v6lrOX8W778HX3t2hMUFET//v356KOPHLbfu3cvq1atYuLEiZ1TcWxfCsrKyrjrrrva/JK7bds2XnzxRQYMGHBKrXFr165tt7ylC1XLF57ly5cTEBDQ4W6a11xzDRqNhrlz57b5JVVRFPvzMXDgQCIiInjttdfaXPeWx7X8Qnv0flpnHewM7b1ujqej9dJoNIwbN47vvvuOrKwse3lycnKb13xHr1tkZCSjR4+2L5dddhlwdu9LgCFDhgDYU9i36MjrraOMRiPQ9nUZFBTkcA1OptWytS+//JL777+fm2++mVdfffWY2/Xs2ZMdO3Zw4MABh/LPP//c4fzi4uIc6tTSGjtjxgyampp49tln2+zbYrE4nF9UVJR9LGaLhQsXtnn9X3vttezatcthnF2Lk2nBaHnveOONNxzK27uXr732Wr7++ut2g6vWKZvbe338/vvv/Pbbbx2u19EqKipISUk5pS6qo0ePRqfT8eabbzrU6WTeR45OSQ2wa9cuvv/+e8aOHYta7fhVddu2bahUKvvrRIhjkZYccd5YsWIFKSkpbcqHDh163PEtx/LII4/w/fffM3nyZGbPnk1cXBw1NTXs2bOHJUuWkJGRga+vL6NGjeKWW27hjTfeIDU11d4lZsOGDYwaNco+M3NcXBxr1qyxT/gYERFBr169CA0N5brrrqNfv364urqyZs0atm7dyiuvvHLc+j333HOsXr2ayy+/nLvvvhutVss777xDQ0MDL7300kmf77lw++23U1payhVXXEFoaCiZmZm8+eab9O/f3/4rW//+/dFoNLz44otUVFSg1+u54oorjtmHft68eUyYMIEhQ4Zw22232VP1enh4nHAej5Nx4403kpiYyKuvvkpSUhI333wzXl5ebN++nffffx8/Pz+WLFlySpP7XXXVVURERDBlyhSioqKoqalhzZo1/PDDDyQkJDBlyhTANh5nwoQJHW7tiIqK4rnnnuPxxx8nIyODadOm4ebmRnp6Ot9++y133HEHDz/8MGq1mv/+979MmTKF/v37c+uttxIUFERKSgr79u1j5cqVuLu7M3z4cF566SXMZjMhISGsWrXK/itpZ2nvdTNo0KBjbn8y9Zo7dy4//fQTw4YN4+6778ZisfDmm2/Su3dvh66IHb1ux3O27kuwBVt9+vRhzZo1DhMeduT11lHOzs7Exsby5Zdf0rNnT7y9venTp89xx3FVVFTYEwds2rQJgPnz5+Pp6Ymnp6f9vfKPP/5g5syZ+Pj4cOWVV7bpStb6Pf2RRx5hxYoVDBs2jHvvvRcfHx9+/PFHVqxYwe23325vsTmWESNGcOedd/Kvf/2LnTt3MnbsWHQ6HampqSxevJjXX3/dPsHk7bffzl133cW1117LmDFj2LVrFytXrmzT7emRRx5hyZIlTJ8+nTlz5hAXF0dpaSnff/89b7/9dptxRcfSv39/brzxRt566y0qKioYOnQoa9eubXfOsn//+9+sW7eOQYMG8ec//5nY2FhKS0vZvn07a9assc8JM3nyZL755huuvvpqJk2aRHp6Om+//TaxsbEO48lOxrfffsutt97KBx98cNz5m9rj5+fHww8/zL/+9S8mT57MxIkT2bFjBytWrOhwd7Lrr78eZ2dnhg4dir+/P0lJSSxcuBCj0ci///3vNtuvXr2ayy67zJ6yX4hjOnuJ3IRo3/FSSHNUis6TSSGtKIpSVVWlPP7440r37t0VJycnxdfXVxk6dKjy8ssvK42NjfbtLBaLMm/ePCU6OlpxcnJS/Pz8lAkTJijbtm2zb5OSkqIMHz7cnup01qxZSkNDg/LII48o/fr1U9zc3BQXFxelX79+yltvvdWhc9++fbsybtw4xdXVVTEajcqoUaOUzZs3O2xzKimkj05f2166047u63gppJcsWaKMHTtW8ff3V5ycnJRu3bopd955p5KXl+ewr3fffVeJjIy0p/Y9UTrpNWvWKJdddpni7OysuLu7K1OmTFGSkpIctmkvLa6itJ/u+Hi+//57ZfTo0Yqnp6f9nuvdu7dSUVHRZtuWFNIn8vnnnys33HCDEhUVpTg7OysGg0GJjY1V/vGPfyiVlZWKoihKeXm5otVqla+++qrN4491bi2+/vpr5fLLL1dcXFwUFxcXJTo6WrnnnnuU/fv3O2y3ceNGZcyYMfZ785JLLlHefPNN+/rDhw8rV199teLp6al4eHgo06dPt6dnbZ3e9nRSSLf3ujnROXa0XoqiKL/88osSFxenODk5KZGRkcrbb7/dJpXvyV63Y+nIfdkZKaQVRVFeffVVxdXV1SGddkdebx1NIa0oirJ582b7tWvv2h6t5X2kvaX1+8LJvKcriqL8/vvvyoQJE5TAwEBFp9MpPXv2VJ5//nnFbDZ36FopiqIsXLhQiYuLU5ydnRU3Nzelb9++yqOPPqrk5ubat2lqalIee+wxxdfXVzEajcq4ceOUgwcPtkkhrSi2lNf33nuvEhISojg5OSmhoaHKrFmz7CmeO5JCWlEUpa6uTrn//vsVHx8fxcXFRZkyZYqSnZ3d7vUuKChQ7rnnHiUsLEzR6XRKYGCgcuWVVyoLFy60b2O1WpUXXnhBMZlMil6vVwYMGKD8+OOPbd6bj/e5cazXd+tzOdb92t791dTUpMydO1cJCgpSnJ2dlZEjRyp79+5tc12PVafXX39dufTSSxVvb29Fq9UqQUFByp/+9CclNTW1Td3Ly8sVJycn5b333muzToijqRTlLIweE0KIC8Ttt9/O//73P959911uv/32M3acr776iptvvpni4mI8PDzO2HHEhamiooLIyEheeuklbrvttnNdHSHOC6+99hovvfQSaWlpJxyDJYSMyRFCiFbeeecdJk+ezF/+8pdOn/+kNU9PT9544w0JcES7PDw8ePTRR5k3b95Zy3QnxPnMbDbz6quv8sQTT0iAIzpEWnKEuMi0Nw/G0by9vR0mExVCCCGEuJBI4gEhLjLtzYNxtHXr1jFy5MizUyEhhBBCiE4mLTlCXGTy8vLYt2/fcbeJi4s7rdmqhRBCCCHOJQlyhBBCCCGEEF2KJB4QQgghhBBCdCkS5AghOs3TTz+NSqWiuLj4XFdFNFu/fj0qlYr169fby2bPnk14ePg5q5M48zIyMlCpVLz88svnuipCCHFOSJAjhOiSnn/+eaZOnUpAQAAqleq4s9Ln5OQwY8YMPD09cXd356qrruLQoUNnr7Kiw7Zv387UqVPx9vbGaDTSp08f3njjjWNuX15ejr+/PyqViiVLlpzFmooWdXV13HbbbfTp0wcPDw9cXV3p168fr7/+Omaz2WHbtWvXMmfOHHr27InRaCQyMpLbb7+dvLy8Y+7/zTffxMPDw76vvLw87rjjDiIiInB2diYqKoqHHnqIkpKSM3qeQojzi2RXE0J0SU888QSBgYEMGDCAlStXHnO76upqRo0aRUVFBX//+9/R6XT85z//YcSIEezcuRMfH5+zWOuz4913370g515ZtWoVU6ZMYcCAAfzzn//E1dWVtLQ0Dh8+fMzHPPnkk9TW1p7FWoqj1dXVsW/fPiZOnEh4eDhqtZrNmzfz4IMP8vvvv/PZZ5/Zt33ssccoLS1l+vTp9OjRg0OHDjF//nx+/PFHdu7cSWBgYJv9L1u2jLFjx6LT6aiurmbIkCHU1NRw9913ExYWxq5du5g/fz7r1q1j27ZtqNXy+64QFwMJcoQQXVJ6ejrh4eEUFxfj5+d3zO3eeustUlNT+eOPP0hISABgwoQJ9OnTh1deeYUXXnjhbFX5rNHpdOe6CietsrKSmTNnMmnSJJYsWdKhL6p79+7lv//9L08++SRPPvnkWajlmVdbW4vRaDzX1Tgp3t7ebNmyxaHsrrvuwsPDg/nz5/Pqq6/ag5dXX32Vyy+/3OH5HT9+PCNGjGD+/Pk899xzDvupra3ll19+4b///S8A33//PZmZmfz4449MmjTJoQ7PPPMMu3btYsCAAWfqVIUQ5xH5OUMIcUZlZmbSvXt3+vTpQ0FBwVk7bkfHnCxZsoSEhAR7gAMQHR3NlVdeyVdffXVKx549ezaurq4cOnSIcePG4eLiQnBwMM888wxHJ7QsLy9n9uzZeHh44OnpyaxZs9i5cycqlYoPP/zwpI57+PBhpk2bhouLC/7+/jz44IM0NDS0W7/W16f1+I0FCxYQGRmJ0Whk7NixZGdnoygKzz77LKGhoTg7O3PVVVdRWlp6KpfmlH322WcUFBTw/PPPo1arqampOWFr1AMPPMDVV1/NsGHDTvv4iYmJjBs3Dl9fX5ydnYmIiGDOnDkO21itVl577TV69+6NwWAgICCAO++8k7KyMoftli5dyqRJkwgODkav1xMVFcWzzz5LU1OTw3YjR46kT58+bNu2jeHDh2M0Gvn73/8OQH19PU8//TQ9e/bEYDAQFBTENddcQ1paWpu6L1y4kKioKPR6PQkJCWzduvW0r0dnaLkHy8vL7WXDhw9vE8AOHz4cb29vkpOT2+xj7dq1NDQ0MGHCBMAWDAMEBAQ4bBcUFASAs7NzZ1VfCHGek5YcIcQZk5aWxhVXXIG3tzerV6/G19f3mNuazWYqKio6tF9vb+9O6XJitVrZvXt3my+rAJdeeimrVq2iqqoKNze3k953U1MT48ePZ/Dgwbz00kv89NNPPPXUU1gsFp555hkAFEXhqquuYuPGjdx1113ExMTw7bffMmvWrJM+Xl1dHVdeeSVZWVncf//9BAcHs2jRIn7++ecO7+PTTz+lsbGR++67j9LSUl566SVmzJjBFVdcwfr163nsscc4ePAgb775Jg8//DDvv//+cffX0NBAVVVVh459vHsDYM2aNbi7u5OTk8O0adM4cOAALi4u3HLLLfznP//BYDA4bL948WI2b95McnIyGRkZHarDsRQWFjJ27Fj8/Pz429/+hqenJxkZGXzzzTcO29155518+OGH3Hrrrdx///2kp6czf/58duzYwaZNm+wtaB9++CGurq489NBDuLq68vPPP/Pkk09SWVnJvHnzHPZZUlLChAkTuOGGG/jTn/5EQEAATU1NTJ48mbVr13LDDTfwwAMPUFVVxerVq9m7dy9RUVH2x3/22WdUVVVx5513olKpeOmll7jmmms4dOjQcVv0rFZrhwNZDw+PDrUONjY2UllZSV1dHYmJibz88suYTCa6d+9+3MdVV1dTXV3d7j2yfPly4uLi7EFNS5D0wAMP8MorrxAaGsru3bt5/vnnmTZtGtHR0R06JyFEF6AIIUQneeqppxRAKSoqUpKTk5Xg4GAlISFBKS0tPeFj161bpwAdWtLT0ztcp6KiIgVQnnrqqWOue+aZZ9qsW7BggQIoKSkpHT5Wi1mzZimAct9999nLrFarMmnSJMXJyUkpKipSFEVRvvvuOwVQXnrpJft2FotFGTZsmAIoH3zwQYeP+dprrymA8tVXX9nLampqlO7duyuAsm7dOof6mUwm+9/p6ekKoPj5+Snl5eX28scff1wBlH79+ilms9lefuONNypOTk5KfX39cev0wQcfdPg5PZFLLrlEMRqNitFoVO677z7l66+/Vu677z4FUG644QaHbWtra5Vu3bopjz/+uKIoR+6txYsXn/A47fn2228VQNm6desxt9mwYYMCKJ9++qlD+U8//dSmvLa2ts3j77zzTsVoNDpc0xEjRiiA8vbbbzts+/777yuA8uqrr7bZj9VqVRTlyHPq4+Pj8PpbunSpAig//PDDcc+55fEdWVrfW8fz+eefOzwuPj5e2b179wkf9+yzzyqAsnbt2jbrunXr1ua1/d577ymenp4Ox5o1a5bDPSyE6PqkJUcI0en27t3L9ddfT/fu3VmxYgXu7u4nfEy/fv1YvXp1h/bf3uDjU1FXVweAXq9vs66lZaBlm1Nx77332v+vUqm49957WbZsGWvWrOGGG25g+fLlaLVa/vKXv9i302g03HfffWzYsOGkjrV8+XKCgoK47rrr7GVGo5E77riDRx99tEP7mD59Oh4eHva/Bw0aBMCf/vQntFqtQ/nnn39OTk4OkZGRx9zfuHHjOvycnkh1dTW1tbXcdddd9mxq11xzDY2Njbzzzjs888wz9OjRA4B///vfmM1me9eu0+Xp6QnAjz/+SL9+/dpttVi8eDEeHh6MGTPGIYV6XFwcrq6urFu3jptuuglw7DJVVVVFQ0MDw4YN45133iElJYV+/frZ1+v1em699VaHY3399df4+vpy3333tamHSqVy+Pv666/Hy8vL/ndL170TZQ8MDAzs8HPXur7HM2rUKFavXk15eTlr165l165d1NTUHPcxv/76K3PnzrW3KLa2d+9esrKyHMbeAISEhHDppZcyceJETCYTGzZs4I033sDX11dSagtxEZEgRwjR6aZMmUJAQAArV67E1dW1Q4/x8vJi9OjRZ7hmjlq+bLY3bqW+vt5hm5OlVqvbBAA9e/YEsHefyszMJCgoqM016tWr10kfr2Xs09Ffck9mX926dXP4uyXgCQsLa7f86LEmRwsKCrKPhThdLc/DjTfe6FB+00038c477/Dbb7/Ro0cPMjIymDdvHgsWLOjwvXciI0aM4Nprr2Xu3Ln85z//YeTIkUybNo2bbrrJHiCnpqZSUVGBv79/u/soLCy0/3/fvn088cQT/Pzzz/YxJC2O7rIZEhKCk5OTQ1laWhq9evVyCDyP5ejntCXgOdFzZzAYOv31GBAQYO9Wdt111/HCCy8wZswYUlNT2/3hIiUlhauvvpo+ffrw3nvvtVm/bNkyAgICiI+Pt5dt2rSJyZMns2XLFnv5tGnTcHd3Z+7cucyZM4fY2NhOPS8hxPlJghwhRKe79tpr+eijj/j000+58847O/SYxsbGDo8B8PPzQ6PRnE4VAdvYHr1e3+4cHC1lwcHBp32cC8WxrumxypWjkigcra6ursPjrE7UOhccHMy+ffvaDChvCSpavrQ/+eSThISEMHLkSHswmZ+fD0BRUREZGRl069btpMZ0tcyxs2XLFn744QdWrlzJnDlzeOWVV9iyZQuurq5YrVb8/f359NNP291HS4a/8vJyRowYgbu7O8888wxRUVEYDAa2b9/OY4891iaZwukOlD/V566pqYmioqIOHcPb27tNINYR1113Hf/4xz9YunRpm/eJ7Oxsxo4di4eHB8uXL293XNzy5csZP368Q2D/zjvvtAl8AKZOncrTTz/N5s2bJcgR4iIhQY4QotPNmzcPrVbL3XffjZubm72bzvFs3ryZUaNGdWj/LemhT5daraZv374kJia2Wff7778TGRl5SkkHwDZw+9ChQ/bWG4ADBw4AR7JKmUwm1q5dS3V1tUOrw/79+0/6eCaTib1796IoisOXvlPZV2f58ssv23S1OpYTfemOi4tj9erV5OTkOLRO5ebmAkeCiKysLA4ePNhuN7q7774bsAVELV3QTsbgwYMZPHgwzz//PJ999hk333wzX3zxBbfffjtRUVGsWbOGyy677LiByfr16ykpKeGbb75h+PDh9vL09PQO1yMqKorff/8ds9l8xtKBZ2dnExER0aFt161bx8iRI0/6GC1dQY8OhEtKShg7diwNDQ2sXbu23dbA8vJyNm/e7NAlFKCgoKBNljrAPlGoxWI56XoKIS5MEuQIITqdSqVi4cKFVFVVMWvWLFxdXZk6depxH3MuxuSA7dfkv/3tbyQmJtp//d2/fz8///wzDz/88Gnte/78+fbxI4qiMH/+fHQ6HVdeeSUAEydOZOHChfz3v//lkUceAWy/oL/55psnfayJEyeyatUqlixZwvTp0wHbHCILFy48rXM4HZ05JmfGjBn8+9//5n//+5/D2Iz33nsPrVZr/5L93HPPOYyJAdvYjX/+8588+uijDBkyBBcXl5M6dktQ1Dp47N+/P3Ckq+OMGTN46623ePbZZ9vMrWSxWKiursbT09PestI6qGtsbOStt97qcH2uvfZali1bxvz583nwwQcd1h0d5J6qzhyTU1xcjI+PT5t6tXRBa93qUlNTw8SJE8nJyWHdunX2cVZHW7VqFQBjx451KO/ZsyerVq1i/fr1DoHX559/DiBz5AhxEZEgRwhxRqjVaj755BOmTZvGjBkzWL58eZuBw6119picRYsWkZmZaZ/t/tdff7VPJHjLLbdgMpkA26/77777LpMmTeLhhx9Gp9Px6quvEhAQwP/93/857HPkyJH88ssvJ2x1ANuYhp9++olZs2YxaNAgVqxYwbJly/j73/9ub3WYMmUKl112GX/729/IyMggNjaWb775psNdvFr785//zPz585k5cybbtm0jKCiIRYsWndOJIztzTM6AAQOYM2cO77//PhaLhREjRrB+/XoWL17M448/bu9WePnll7d5bEurTUJCAtOmTXNYp1Kp7Ps6lo8++oi33nqLq6++mqioKKqqqnj33Xdxd3dn4sSJgG3czp133sm//vUvdu7cydixY9HpdKSmprJ48WJef/11rrvuOoYOHYqXlxezZs3i/vvvR6VSsWjRog7dUy1mzpzJxx9/zEMPPcQff/zBsGHDqKmpYc2aNdx9991cddVVHd7XsXTmmJxPPvmEt99+m2nTphEZGUlVVRUrV65k9erVTJkyxeF94eabb+aPP/5gzpw5JCcnO8yN4+rqan/+li1bxuWXX+6QKANsyT4++OADpkyZwn333YfJZOKXX37h888/Z8yYMfZkGkKIi8A5y+smhOhyWqeQblFbW6uMGDFCcXV1VbZs2XLW6tKSfre95eiUt9nZ2cp1112nuLu7K66ursrkyZOV1NTUNvuMi4tTAgMDT3jsWbNmKS4uLkpaWpoyduxYxWg0KgEBAcpTTz2lNDU1OWxbUlKi3HLLLYq7u7vi4eGh3HLLLcqOHTtOOoW0oihKZmamMnXqVMVoNCq+vr7KAw88YE9h3JEU0vPmzXPY37FSL7ekhj5eSuUzobGxUXn66acVk8mk6HQ6pXv37sp//vOfEz7uWOdRVVXVbgrqo23fvl258cYblW7duil6vV7x9/dXJk+erCQmJrbZduHChUpcXJzi7OysuLm5KX379lUeffRRJTc3177Npk2blMGDByvOzs5KcHCw8uijjyorV65s8zyNGDFC6d27d7t1qq2tVf7xj38oERERik6nUwIDA5XrrrtOSUtLUxTl2M+poijHTKl+pmzdulWZPn26/fq5uLgoAwcOVF599dU2aZ1NJtMxX7ct96zValX8/f0dUq+3lpKSolx33XVKWFiYotPpFJPJpDz88MNKTU3NmT5VIcR5RKUoJ/HzkRBCXKSqqqrw9vbmtdde45577jnutrNnz2bJkiVUV1ef0rEyMjKIiIjggw8+YPbs2ae0D3Fiy5cvZ/LkyezatYu+ffue6+qIDvrjjz8YNGgQ+/btkyQCQohjOv0pw4UQ4iLw66+/EhISwp///OdzXRXRSdatW8cNN9wgAc4F6IUXXpAARwhxXDImRwghOmDSpEltJh08GzqSWtvDw+O0Uw1fjObNm3euqyBOwaWXXsqll156rqshhDjPSZAjhBDnsY6k1pZubUIIIYQjGZMjhBDnsbKyMrZt23bcbXr37t1pWcyEEEKIrkCCHCGEEEIIIUSXIokHhBBCCCGEEF2KBDlCCCGEEEKILkWCHCGEEEIIIUSXIkGOEEIIIYQQokuRIEcIIYQQQgjRpVz08+RYrVZyc3Nxc3NDpVKd6+oIIYQQQghxUVMUhaqqKoKDg1GrT61N5qIPcnJzcwkLCzvX1RBCCCGEEEK0kp2dTWho6Ck99qIPctzc3ADbRXR3dz/utmazmVWrVjF27Fh0Ot3ZqJ7oguQ+EqdL7iHRGeQ+Ep1B7iPRGY6+jyorKwkLC7N/Tz8VF32Q09JFzd3dvUNBjtFoxN3dXV7I4pTJfSROl9xDojPIfSQ6g9xHojMc6z46naEkF23igQULFhAbG0tCQsK5rooQQgghhBCiE120Qc4999xDUlISW7duPddVEUIIIYQQQnSiizbIEUIIIYQQQnRNEuQIIYQQQgghuhQJcoQQQgghhBBdigQ5QgghhBBCiC5FghwhhBBCCCFElyJBjhBCCCGEEKJLkSBHCCGEEEII0aVctEGOTAYqhBBCCCFE13TRBjkyGagQQgghhBBd00Ub5AghhBBCCCG6Ju25roAAKg6DooB7MKg157o2QgghhBBCXNAkyDkf/PIibP8Y1DrwCAUvE3iajvzb8n8XP1CpznVthRBCCCGEOK9JkHM+sDSCWgtWM5Sl25b26Izg2c0xALIHQt3A2fOsVlsIIYQQQojzkQQ554Nr3oFpb0FlLpRnQlkmlGe1+n+mbZ25FopSbEt7DB5HBUDhR/72CAMn41k9LSGEEEIIIc4FCXLOF2oNeIbZlvDL2663NNjG7pRl2IKe8qwjAVBZJtQWQ30F5O+2Le1x8W/bAtTyr0coaHRn9BSFEEIIIYQ4GyTIuVBo9eATZVva01DdtvWndYtQQyXUFNqWw+2kzVapwT2k/QDIywSugaCWZHxCCCGEEOL8J0FOV6F3hYBY23I0RYG6snYCoJYgKAss9VCRbVsyN7bdh8bJ1uWt3ZagcDB6S1IEIYQQQghxXrhog5wFCxawYMECmpqaznVVzjyVyhaEGL0heEDb9VarrYXHIQDKOPJ3RQ40NUJpmm1pj5PrkQQI7QVCerczeopCCCGEEEK0uGiDnHvuuYd77rmHyspKPDw8znV1zi21GtwCbUu3QW3XN1mgMqf9lqCyTKjOh8ZqKNxnW9rj7O2YCa6lBaglKYLOcEZPUQghhBBCXDwu2iBHnASN1haMeJkgop315uaubke3ALX8W1cGdaW2JXdH+8dwC2rV+nNUmmz3EFsdhBBCCCGE6AD55ihOn84Avj1sS3vqK4+RFKH5X3MNVOXZluwtbR+v1toCnWNNkuoaIOOBhBBCCCGEnQQ54swzuENgH9tyNEWB2pJWiRCOCoAqsm3jgVrWtUdraNv603pskLOXBEFCCCGEEBcRCXLEuaVSgYuvbQmNa7vearW18LQ3N1B5pm2skKUeig/Ylvbo3dtPjd0SCDm5nNlzFEIIIYQQZ5UEOeL8plaDR4htMQ1tu97SCJWHHVNitw6EagptcwQV7LEt7TH6HmeS1DDQOp3ZcxRCCCGEEJ1KghxxYdM6gXekbWlPY23b8UCt/19fAbXFtiVnWzs7UIF78LEnSXULArXmjJ6iEEIIIYQ4ORLkiK7NyQj+0balPXXlx54ktSwTLHW2LnGVOZC1ue3j1TrwCG0nAAq3/eviK+OBhBBCCCHOMglyxMXN2dO2BPVru05RoKaoVQCU0SoQyrIlRbCaoSzdtrRHZ2yTCEHlFop7bZYt65zO5wyenBBCCCHExUmCHCGORaUCV3/bEpbQdr21CSpzj50auyoPzLVQlGxbmmmBUQD7/wkGz6PmBgp3nC9I53x2zlUIIYQQoguRIEeIU6XWgGeYbQm/vO16SwNUHD6qBSgTa2kG5qKD6C1VUF8OeeWQt6v9Y7gGHHuSVI9Q0OjO4AkKIYQQQlyYJMgR4kzR6sEnyra00mQ289Py5UwcPRxddd6xW4Iaq6C6wLYc/qPt/lVqcA9tf24gLxO4Btqy0wkhhBBCXGQu2iBnwYIFLFiwgKampnNdFXGxcnKFgFjbcjRFgbqyYwdA5VnQ1AAVWbaFDW33odE3tzSZ+MXFyMv1GZiMAZg8Igj37UO4bywmj3D8nP1QSXIEIYQQQnQhF22Qc88993DPPfdQWVmJh4fHua6OEI5UKjB625bgAW3XW622Fh6HuYEyjgRCFTm2IKjkIJQcJM3DjQxvLzIqKqDiAGSttO/KiBqT1pVwZ3/C3U2YvGMIDxqAyScWVyfXs3fOQgghhBCd5KINcoS4oKnV4B5kW7oNbru+yeIwSeq0kgPElh4gsyaHjIYyMmgkU6clR6ulVmUl2VJJclUlVB2EnLXQPG+qr6LCpHEh3OBLuGsYJu+ehAf0JzQwDp3B7eyesxBCCCFEB0mQI0RXpNHaMrV5hQPgDQxuXgAw10F5No2laRwu3E1G2QEyqg6TWV9ChrWGDI2KUo2GYpVCsbWabbXVUJsBhRsgBTSKQogVwlUGTHpvwl1DCPfqjsnvEvz9+6LyCLXVQQghhBDiHJBvIUJcjHTO4NcTJ7+eRPaaQOTR6+srqSxKIit/B+mlKWRWZpJZV0SGpZpMVRN1ahVZGsiiAcx5UJYHZYlw6AucrVbCzRZMKidMTh6EG4MI94zE5NsHN99etsQIrv4ySaoQQgghzhgJcoQQbRnccQ8bTJ+wwfQ5apVitVJYup+M3EQyS/aRUZFBRm0+meZKchQzdWo1yXonbDMDVUBtBdSmQO5yfCxNmCxmwi0K4Vo3TEZ/wt3DCfOJRucdeSQznLPX2T9nIYQQQnQZEuQIIU6KSq0mwDeGAN8YBh21ztxk5nBlNhkF28ks2ktGeRoZNblkNpZRrJgp0Woo0WrYDoAFrLlQnou6bBMhFgsms4Vws5lwRYfJ4Eu4exj+nt1Re4c7zhfk5HLWz1sIIYQQFw4JcoQQnUan0RHhFUmEVyREX+ewrrqxmszKTDLKDpJZvJeM0gNkVB8ms6GUWixk63Rk63RsxLn5ETXQmIJzfhLdsm3Bj8lsIdxiJlzjisk1BHfPCMe5gTxN4BEGWqezf/JCCCGEOG9IkCOEOCtcnVzp7dub3r69ocdV9nJFUSiqKyKzMpP0inQyy9LILN1PZlUW2fXF1KnV7Nc7sV9/dOBSgndVIeGlGzGZLZjMZsLNFsItTYQZ/HFqafVpHQB5mcAtCNSas3vyQgghhDirJMgRQpxTKpUKf6M//kZ/EgITHNaZrWZyqnJsLUCVGWRUZpBZlkZGZTpFDeWUajSUajRsNzjuU60oBFsOYco/QLi9FcgWBAUoatTNk6Ti2a1VABRu+9fFV5IiCCGEEBc4CXKEEOctnVpHuEc44R7hjGCEw7oac40t+KnIcAyCKjKosdRyWKfjsE7HpqP2abBa6WauxVS9h/Cy7YQfaGkFMuNhVUDn4hj8OARCJjDI5MFCCCHE+U6CHCHEBclF50KsTyyxPrEO5YqiUFJfYuv6VplpD4QyKjM4XHWYerWFA3onDrTp/gZeTU3NyQ8KMBXkEH74F8LNFsIsZvRK80YGzyNBj3cE+PYC/2jbv3rXM3/iQgghhDghCXKEEF2KSqXC19kXX2ffNt3fLFYLOdU5DoFPSytQYW0hZRoNZRoNOw16x30qEGy1YmposM0BVJ1GePl+wlPNBFqaULds6BEGftHg16v53+b/G9zPzskLIYQQApAgRwhxEdGqtZjcTZjcTQwPHe6wrtZca2/5Sa9sbgWqsAVA1eZqcjRqcozObD5qn3pFRVhTExEN9ZjMlZjyNxOebWsB8rRabRu5hxwV+DQHP86eZ+W8hRBCiIuNBDlCCAEYdUZifGKI8YlxKG/p/nb0+J/MykyyqrJosFo4qFVzUGtss08Pq0J4YyMmcx3hxYmE5/2GyWyhm8WCQVFsmd7swU8v8Iux/Wv0PlunLYQQQnRJEuQIIcRxtO7+FhcQ57DOYrWQV513pOWnVTe4gtoCKtQqdhn07GrT/U0hyNKEyWLGVLHLFgDttiVACLI0oXHxt43zcej6FgMuPmfz1IUQQogLlgQ5QghxirRqLWHuYYS5h7VZV2uuJasqqznj25HWn4yKDKrMVeTqtOTqtPzm7Pg4J6tCN4sZU00S4eW7MCXZUmCHmy14GrxR+UWj9ulBRFETqgw3COoDLn6S9loIIYRo5aINchYsWMCCBQtoamo611URQnRBRp2RaO9oor2jHcoVRaG0vvRIy09lhr0bXFZVFo2YOejkxEGnttnf3JuaCG9MJTwrCZPZQt63iwk3W+imdcO5datPSyuQa4AEP0IIIS5KF22Qc88993DPPfdQWVmJh4fMeyGEODtUKhU+zj74OPswMGCgw7omaxO5NblHEiC0SoOdV5NHpUbDbo2G3Ud1fwMItGQQnpOKKeM7ws3NE6CqDQR790TjF+PY9c09WIIfIYQQXdpFG+QIIcT5RqPWEOYWRphbGJeHXO6wrs5SR1ZlFpmVmaSVpbE5eTNN7k1kVmVS2VhJvlZLvlbLlqO6v+mUPLoVZGE6vAyT2UKE2YxJ5YTJIxJv32hU/rFHAiCPUAl+hBBCdAkS5AghxAXAWetML+9e9PLuhTnETHBWMBPHTUSr1VLeUN6m5Sej4hBZVdk0Ws2kOTmR1qb7WxFupQWEF6zBZGlu+VG0hLuF0c0nBqN/7JFsbx5hoFa3Wy8hhBDifCRBjhBCXMBUKhVeBi+8DF709+/vsK7J2kR+bb7jxKflh8gsP0RefTFVGjV7NHr20Lr7WxlUbiag9FfC99gyvoVbVZiMgYR79yTY/xK0/rG24MczXIIfIYQQ5yUJcoQQoovSqDWEuIYQ4hrCZSGXOayrt9STVZV1ZPxP+SEyS/eTWX2YckstBVotBVotvzsbmh9RDdXb0VZtIyzFFvxENIHJ4EO4RwQmvz74BA5A5R8DXuGg1pz18xVCCCFaSJAjhBAXIYPWQE+vnvT06tlmXXl9uWPLT0kSGRUZZNUX0UAT6U460p10rAegDuqSICsJ14wvbN3eLFZMOk8i3EIxeUdjCorDGNQfvCJAIx87Qgghzjz5tBFCiC5OaWqiqaICrbd3h7b3NHjS39C/Tfc3q2Ilvybflva6/BCZRXvJLEsloyaXXEs11Wo1e/V69uoBGqAhDfLSIG8Z/hYL4ZYmTBoXTMZAIjy7YwroT3DIIHR+vUCj6/TzFkIIcfGSIEcIIbq4hoMHSb9qGtqAAAwxMRhiY9DHxGCIiUUXEoyqgxnV1Co1wa7BBLsGMzR4qOMxmhrIrswmo+IQGQW7bK0/VVlkNpRSplgo1Gop1Gr5AwuYD0PRYShaj3aPQqjFQrjKYOv65m7C5NuH8OBB+AbHodIZjlEbIYQQ4tgkyBFCiC6uMT0DAEtBAdUFBVSvX29fp/bwwBAdbQ9+DDExOEVEoNKe3MeDXqOnu1d3unt1h/CxDusqGirIKE8ns2A7GYV7yKxII6O2kKymGupVKjJ0OjJoAkshlBZC6VY48AEuVismRYNJ50GEawgm716YAuMID7sMF6PP6V4WIYQQXZgEOUII0cW5jx+HS2IiDftTqE9Kpj7ZtjQcPIi1ooLa33+n9vff7dur9Hr0PXs6BD76Xr1QG06tVcVD70G/gP70C+jvUG5VrBTWFJCeu5XM/O1klu0nvTqHTHMluVioUatJQiHJWg6V5VC5DzK+gS3gZwWTxoVwYwDhHpGY/C8hPHQIIV5R6NTS9U0IIS52EuQIIcRFQOPqgjEuDmNcnL1MaWyk4eBBW9DTHPw0pKRgra2lfs8e6vfsObIDtRqnyAgMMbEOwY/Gw+OU66RWqQl0DSKw51SG9JzqsK7R0kB2/nYycraQWZxERmUmmQ0lZFjrKdWoKVJDkVJDYs0hqDkEuWtgJ2gUCFU5Ea73xuRuwuQbQ0RQAiafaPyc/TrcNU8IIcSFTYIcIYS4SKmcnDDExmKIjYVrbWWK1UpjZiYNza09LcFPU2kpjQfTaDyYRuUPP9j3oQsORt8c8BhiYjHExqANCDjtYMJJqycqdAhRoUMcVygKFaUHyTq8iYyC3WSUpZJZW0BmUzWZGhV1ajWZNJLZkA9F+VD0OyR/CIARNSatG+EuwZi8exIeMIBwn2hM7iZcnVxPq75CCCHOLxLkCCGEsFOp1egjItBHROA+cSIAiqJgKSykPinJ1trTHPyYc3Iw5+Zizs2les1a+z40Xl5tEhw4hZtQdcbEoSoVHj496OvTg76tyxUFa3UBhTm/k5m3jYySFDKqD5PZWE6G2kqOVkutykqypYLkigqoSIb0pfaH+6r1mJz9CHePJNy/jy31tYeJMNcwdJL5TQghLjgS5AghzisNliZ6PfETAA+N6YnJx0iErwsmHxc8nOXL5rmgUqnQBQSgCwjAbdQoe3lTZSX1ySnUJyfZA5+GQ4doKiujZvNmajZvPrIPoxFDr14OwY++Rw/UTk6dVUnUboEERl9FYPRVDGq9rqYYc8FesnP/IKNoL5mVGWTWFZGuspCp1VGi1VBsbaC45jDbag5D3q/2h2pQEeLkgck1jHDfGMK9e2FyNxHuHo6/0V+6vwkhxHlKghwhxHllXUqh/f+vrj7gsM7bxckW9PjYgp5wXyPhPi6E+0oAdC5o3N1xGXQpLoMutZdZ6+tpSE1t7ubW3PKz/wBKbS11O3ZQt2PHkR1otei7d2/u6tYc/ERHo3Ht5K5jLr7oIkcSGTmSyNbltaVQtJ+q/F1kFuxo7vqWT4a1nkydjgydljq1mqzGcrJKy9lQusdht85qHSZjIOFePTB59bAHP+Ee4bg5uXXuOQghhDgpEuQIIc4rl0b4cPOgbiTlVdLdz5WMkhoySmopqmqgtKaR0ppGdmSVt3mcl1FHuK+LLehpHQD5uOBhlADobFEbDDj37Ytz3yOdyRSLhcaMDIcxPvXJyVgrKmhISaEhJYWKb7+1b68zdWuT4EDr69v5lTV6g2kIbqYh9AH6tJTXlUPRfpTCZArtc/4cJqOpmkydjkydlsNaLXVWMynV2aRUZ0P2zw679ta52QIeb1vwY3I3EeEeQahbKE6aTmq9EkIIcUwS5AghziveLk48f3XfNuXVDRYyS2rIKK61BT7FNWSW1JJeUkNRVQNltWbKssrbDYA8jbrmgMd4JBDytf3taZQvnGeaqrnFRt+9Ox5TpgDN43xyc9sEPpb8fMyZWZgzs6j66Sf7PrR+fm0SHOhCQ89MdzFnT+g2CFW3QQQAAcClAPWVUHwACpMxFyZxuGgfmRXpZDSWkdEc/GTodBRrNZSaqygt2cP2EsfWHzUqgl0CCfeMItw93Nb64xFu7/6mVnXCuCUhhBAS5AghLgyuei29gz3oHdw2ZXFNg4XMkubgpzkAyiipJaO4hsKqBsprzeysLWdndnmbx3oadZh8XIjwMdr+9XWxjwOSAOjMUalU6EJC0IWE4DZ6tL3cUlrqkNygPjmZxowMLEVFWH4pouaXI+Nl1K6uGKKjm4MfW+Cjj4xEpTtDLXcGdwiNh9B4dEBE80JDlS34KdoPhclUFyaRWXaAzPpi20SnOq09AKpVqzlck8fhmjw25mx02L2zxkC35lYfk7uJCI8I+/899KeeqlsIIS5GEuQIIS54LnotscHuxAa7t1nXEgBlltSQXlJDZrGt9SezpIaCSlsAVF5bzq52AiAPZ529xccWADX/6+OCp1Eng87PAK23N66XXYbrZZfZy6w1NdTvP3BkjE9SMg2pqVirq6lNTKQ2MdG+rcrJCX2PHq0yu8Vg6NULtdF45iqtd4OQONsCuAK9gd6NNUeCn6IUlIJkiktSyKjNJ0OnsXV909qCn8M6LXVN9ewv28/+sv1tDuFt8MLU0vLjHm5vBerm3k26vwkhRDskyBFCdGnHC4BqG5tbgJpbfjJLakhv7gaXX1lPRZ2ZXdntB0DuBq0961tLINTSFc5LAqBOpXZxwThwAMaBA+xlitlMQ1paq65uSTQkp9gCon37qN+3r9UO1DiFhzuM8dHHxKD18jqzFXdygeABtgVQAX6An7mOhOJUKEppXvZjLkomtzKbDK3aoetbpk5LoVZLaX0ZpfVl7Cjc4XAItUpNkEuQPeFB6/E/AS4B0v1NCHHRkiBHCHHRMjppiQlyJyao/QAoq7TWoetbRvOYoPzKeirrLew6XMGuwxVtHutu0LZKgmALflq6wkkA1DlUOh2G6GgM0dHA1YBtIlNzdnabcT5NxcU0HjpE46FDVC5bZt+HNijIIbObISYGbVDQmX9+dM4QdIltaSkCTOZ6TCUHGdEc+FCUDEX7qSk9RKZWZc/41joIqlFDTnUOOdU5bMrd5HAYvUZPN/duDi0/LeN/pPubEKKrkyBHCCHaYXTSEh3oTnRg2wCorrGJzNIjSRBaJ0TIq7AFQLsPV7C7nQDIrXULkI/RIROct4uTBECnQaVW42Qy4WQy4T5+vL3cUlTUJvAxZ2VhycujOi+P6p+PZEbTeHqij4l2yO7mFB6OSqM58yegM0BgH9vSioulkdiSg8Tagx9bC5CSf5ASldUW9DR3e7ON/9GRrdPS0NRAalkqqWWpbQ7lqffE5GZCXasmf18+UV5R9u5veo3+zJ+rEEKcYRdtkLNgwQIWLFhAU1PTua6KEOIC4+ykOWYAVG9uOpIEoVUrUGZJDbkV9VSdIAAK93FxmAC1ZRyQjwRAp0zr54ernx+uw4fby5qqqmhISXEIfhrS0mgqL6f2ty3U/rbFvq3K2RlDz54O2d30PXug1p+lYEDrBAGxtqUVVZMZ39JD+BalEF94pOsb+alYmhrJ1WodWn4ydTrSdToKtRrKG8opbygHYMeuI13gVKgIdg3G5G6il1cv4gPjGeA/QOb9EUJccFSKoijnuhLnUmVlJR4eHlRUVODu3vYLS2tms5nly5czceJEdGcqe4/o8uQ+unjVm5vIKq1tHvdTQ3pxbXMrkC0AOh43vRZTc4tPmJeBysMHmTJqMFEBHvi6SgDUGawNDTSkHmwe39Mc/Ozfj1JX13ZjrRZ9ZOSRSUybu71p3M6DYKDJAmXpR8b8FDYHP8UHoKmBWpWKLJ1jy0+GzokMJx3V7dxGapWaaO9o4gPiiQ+IZ2DAQOnuJuzkM010hqPvo5P5fn4sF21LjhBCnG0GnYaeAW70DGj7RbglALKP/SmptXeDy62oo6rBwt6cSvbmVDY/QsOnaVsBW3ptkz3xgbHVPEAuEgCdBLVej3Of3jj36W0vU5qaaMzMbG7tORL8NJWX03DgAA0HDlCxdKl9e11YWJsEBzp//7N7Ihot+PawLTFTjpRbm6AsA2NRCtFFKfQsSKYybSse1QWoLCUoQKnalvgg3UnLbr2eRIOBbJ2WpJIkkkqS+DjpY1So6OXVk/jABOID4okLiMPT4Hl2z1EIIU5AghwhhDgPnCgAyi6ttXd9O1RUxbYDWdSojORW1FPdYGFfbiX7civbPNYeADWP/Wk9F5Cfq14CoBNQaTToIyPRR0biMXkS0DyRaX7+UeN8krDk5mHOzsacnU3VqlX2fWh8fdskONCFhaFSn+XMZ2oN+ETZluhJNJnN/LJ8ORPHj0NXk4eqaD8+Rcn4FO0nriiF64r2Q3EpBRoNiQY9Ww0Gthn0ZDjpSCnbT0rZfj5J/gSAHs7+xPsNIN40irigwfg4+5zdcxNCiKNIkCOEEOc5g05DjwA3ejQHQLZm/QwmThxOE2oOl9Xau761pMBOL64ht6LuuAGQi5OmOQV2cxDUKh22n5sEQMeiUqnQBQWhCwrC7Yor7OWWsjLbOJ9WCQ4a09NpKi6mZsMGajZssG+rdnGxJTiIPhL86KOiUDmdgzlv1BrwjrAtvY4kbMBqhYpsAor2M6kohUnN3d+K8g6wTdNkD3wOOelIrSskNWsln2etBCBKpSfeGEq8X3/iw6/ENzjBllhBCCHOEglyhBDiAmbQaeju70Z3/7YtQA2WJrJL61p1gWsVAJXXUdPYRFJeJUl5bQMgY3MA1HoC1JaECBIAtU/r5YV2yBBchgyxl1nr6mjYv98xwcGBA1hraqhL3EZd4jb7tiqdDqce3e3JDQyxzROZurici9MBtRq8TLal51h7sZ+iML4yh/HNyQ5KCnazrTSZxPoCtjqpOOjkRJrSQFpNGl/WpEHG14SbzcRb9cQbQ4j3u4SAoIHg1wt8eoDTGZyoVQhx0ZIgRwghuii9VkN3f1e6+7u2WdcSALVu/WkJhHLK6qhtbCI5r5Lk4wRA4T5Gx0DI1wV/CYAcqJ2dce7fH+f+/e1litlMw6F0xwQHKSlYq6poSEqmISmZCr6xbaxS4WQytUpuYAt+tN7e5+aEmuuER6ht6TEaH2AsMFZRoCqPstxEtmdvILFkL1vr8zmA2ZbkACtLmrIhP5uw7O9IqG8gvr6BeCc/gnxjbEGPXzT4R4NvT9tkqkIIcYokyBFCiIvQiQKgw2V19hTYrQOhw2W1xw2AnHWaVmOAjkyGGu7jQoC7BEDQPJFpr54YevWEadMA2zgfc04O9UlJttae5lYfS2EhjRkZNGZkwPIV9n1oAwKOyuwWiy4k+NxeX5UK3IPxcp/KldFTubK5uKK+gu2Za0nMXk9iyV5S6ovI1unI1un4xs0VsBJSu4u4/b+TsKuB+Pp6QixNqDy72YIeh6Un6M+DDHZCiPOeBDlCCCEc6LUaovxcifJrGwA1WqwcLmuZB6jWngkuo7iGw2W11JmbSMmvIiW/qs1jDTq1feyPydfY3AXuSAuQWn3xBkAqlQqn0FCcQkNxH3uka5iluJj65BR7coOGpGQaMzOxFBRQXVBA9fr19m3VHh4YoqMdEhw4RUSg0p7bj3oPgwejel3DqF7XAFDVWMWOwh0k5m8lMXcLSWUHyNFpydG58r2b7Z4LtFiIr68hIW8T8ek/E2axYL87PMKOtPq0Dn4MktZaCHGEBDlCCCE6zEmrJtLPlcjjBEAt434yS2pIb24JOlxWR73ZesIAyNSq5adlDFCAm+GiDYC0vr64Drsc12GX28uaqmto2O+Y4KDh4EGsFRXU/v47tb//bt9Wpdej79XLIbubvmdP1IZzlwTAzcmN4aHDGR5qm5y1xlzTHPQkkliQyL7ifeRr4UdXLT+62rqs+aMhrr6RhOpy4mvzCD+YjergmqN2HGzr6uYX3RwExdiCH2evs32KQojzgAQ5QgghOkXrAGjUUevMTdZWXeBqHAKh7A4EQCbvI0FP64xwge4XXwCkcXXBGBeHMS7OXmZtbKTx4EHHBAcpKVhra6nfvZv63btb7UCDPjICp17ReCoKtb6+uPbpg8bj3LSEuOhcuDzkci4PsQVyteZadhbtJDE/kW0F29hdvJtCq4UVBg0rDLbU1L5aF+I07sQ3NJJQlk9keR6qqlyoyoW0nx0P4BpoC3r8YxxbgIzncFyTEOKMkyBHCCHEGafTqInwtXVNO5q5yUpOWR3pJTVkNo8DsnWHOxIA7S+oYn9B2wBIr1UfNQaoOSGCrwtBF1EApHZywhAbiyE2Fq61lSlWK42ZmbbkBq2Cn6bSUhpSD9KQehB/IHfZMgB0ISGtxvjEYIiNRevvf9bH+Rh1RoYGD2Vo8FAA6ix17C7aTWJBIon5iewu2k2xpYaVlhpWAnjp8A68hDj3KOK0HiQ0NNG9LAd18QGoPAzV+bYl/RfHA7n4twp6WoKgaHDxPavnK4Q4MyTIEUIIcU7pNGpbgOLrAr0c17UEQC1BT0sAlFlSS3ZpLQ0WKwcKqjlQUN1mv05aNSZvY5sECOEXSQCkUqvRR0Sgj4jAfeJEoHki08JC6pOSqN27j8z16/EqL8OSk4s5JwdzTg5Vq490A9N4ezuM8dHHxOBkMp3ViUydtc4MChrEoKBBADQ0NdiDnm3529hVtIvShnJWF21jdfNjPPQexPUfTbxPH+KdvOlZV4OmaD+0LBVZUFNoWzI2OB7Q6NPc1e2oAMjFz5ZcQQhxQZAgRwghxHnreAGQpclKTnmdPfFBSyCUWVJLVmktjRYrqYXVpBYeOwA6OgW2ycdIsIdzlw2AVCoVuoAAdAEBGC6/nC3dwhgwcSLq2lrHBAfJyTQcSqeptJSaTZuo2bTpyD6MRgwt43yaW370PXqgPksTmeo1ehICE0gITIB+YG4ys7dkL1vzt5KYn8jOop1UNFTwc/bP/Jxt67rm5uRGnH8c8QOmEB8wl14uIWhL05qDnhRonvOH8kyoLYHMjbalNWevo5Id9IKwQTLPjxDnKQlyhBBCXJC0GjWm5gxtI3r6OayzNFnJLa+3dYFrPRdQcQ3ZZScOgLp5G22tPz4umHyPTIYa7OmMpgsGQBoPD1wGD8Jl8CB7mbW+noYDBxwTHOzfj1JbS92OHdTt2HFkBzod+qgoxwQH0TFoXM/8XDc6jY4B/gMY4D+AOy65A7PVTFJJki3oKUhkR8EOqhqrWH94PesPrwfAVefKAP8BxAfGE9//OmJ8YtCpddBYA8WptoCnKOVIEFSaDnVlkPWbbbEf3Ag9xkLvabZ/ZW4fIc4bEuQIIYTocrQaNd18jHTzMQLtB0Atk59mFDfPBVRSQ3ZzC9DBwmoOthcAadSEeTu3SoDgYg+GuloApDYYcL7kEpwvucReplgsNGZkOIzxqU9OxlpRQUNKCg0pKVR8+619e52pm20C01Zd3rS+Z3bMi06to59fP/r59eP2vrdjsVpIKU2xBz3bC7ZTba5mQ84GNuTYuqo5a50Z6D/QFvQExNO7z7XoNLojOzXXNQc/+48EQLk7bWN+kr6zLToj9BgDsdOg5zgJeIQ4xyTIEUIIcVFpHQANPyoAarIq5JYfNQaouStcdmkdjU1W0opqSCuqabPflgCo9USoLd3gukoApNJq0Xfvjr57dzymTAGax/nk5rYJfCz5+ZgzszBnZlH100/2fWj9/NA3BzyGmFgMsTHoQkPPWIIDrVpLH98+9PHtw619bqXJ2sT+sv0OQU9lYyWbcjexKdfWLc9Z60w/v37EB8QTHxhPX9++OAVdAkFHAj4UBXJ32AKcfd/ZurolLbUtWmfoOVYCHiHOIQlyhBBCiGYatYowbyNh3kaG9ThOANQc/LR0hTtRAKTT2PbbMhlqSwpsWwuQAa3m7A3k72wqlQpdSAi6kBDcRo+2l1tKS21d3FoFP40ZGViKirD8UkTNL7/at1W7udkmMrVnd4tFHxV5RiYy1ag1xPrEEusTy6zes7AqVlLLUkksSGRr/la2FWyjvKGcLXlb2JK3BbCNA2od9Fzidwl6jR5CBtqW0XMhb6ct2En6DsoyHAOeHmOau7SNA33bOaaEEJ1PghwhhBCiAxwDIMd1LQFQZqv01y2Z4LJKamlssnKoqIZDxwqAvGzZ31rPBRRxgQdAWm9vXC+7DNfLLrOXWWtqqN9/gPrkJFsAlJRMQ2oq1qoqardupXbrVvu2Kicn9D17OmZ369ULtbNzp9ZTrVLTy7sXvbx7cXPMzVgVK2nlaQ5BT2l9KX/k/8Ef+X/ALluXuL6+fUkITCA+MJ5+fv1wDh4AwQNg9NOQtwv2fXsk4En+3rZonaHH6OYWnvES8AhxBkmQI4QQQpym1gHQ5T0cx5w0WRXyKuocJkBNbx4HlNk8BuhQcQ2HitsGQFp1SwuQYwa4CF8XQjydL7gASO3ignHgAIwDB9jLlMZGGg4datXVLYmG5BRbQLR3L/V797bagRqniAiHBAeGmBg0np6dV0eVmh5ePejh1YMbo29EURTSK9LtQU9iQSLFdcVsL9zO9sLtvLP7HVuXOJ8+tqAnIJ7+/v0xjpl7JOBp6dJWlg7JP9gWrQG6j4beV9u6tOndOu0chBAS5AghhBBnlEatItTLSKiXkcu6OwZAVqtCXmW9fdxP60Aoo8QWAKUX27rEQZHDY1sCIPtkqK3mAgr1unACIJWTk62rWnQ0cDVgm8jUnJ3dZpxPU3ExjWlpNKalUfnjj/Z9aIOD2iY4CAzslHE+KpWKSM9IIj0jmdFrBoqikFmZaZuctDnwKawtZGfRTnYW7eTdPe+iVWmJ9YklLjCOhIAEBgx/GNcrn4L83Ue6tJUegpQfbYsEPEJ0OglyhBBCiHNErVYR4ulMiKdzuwFQvj0Aat0NzhYMNZwgAAr1cj4yAaqP0Z4KO8TLGd15HgCp1GqcTCacTCbcx4+3l5sLC21jfFoFP+bsbCy5eVTn5lG9dq19W42nJ/qYaIfgxyk8HJVGc3p1U6kI9wgn3COc63peh6IoHK467BD05NXksbt4N7uLd/PB3g9Qq9TEeMcQHxBPQq+RDBj2V9xLM4+08JSmHQl4NPojWdp6jZeAR4hTJEGOEEIIcR5Sq1UEezoT7OnM0O6O6+wBUOsU2C1zAZXU0GCxNgdGtbQEQEPqteRprOQ6KQR5O2PydkapUlP0WyZR/u6E+9pagM7nAEjn74/O3x/XESPsZU1VVW0SHDSkpdFUXk7tb1uo/W2LfVuVszOGnj0dsrvpe/ZArdefcp1UKhVh7mGEuYdxdQ9bS1ROdQ6J+UeCnpzqHPaV7GNfyT4+SvoIFSqivaOJC4gj4aqXiVM545G6uv2Ap/toW9KCnuPB4H7K9RTiYiNBjhBCCHGBcQiAohzXWa0KBVX1DkFPzuEqYrbZ5v1prFHIrLJwKLecNJ2aDfn77Y/VNLcA2RIfOI4DCvM2npcBkMbNDZdLL8Xl0kvtZdaGBhpSDzaP72kOfvbvR6mro27XLup27TqyA60WfWTkkUlMm8f7aNxOvQUlxDWEkO4hXNX9KgDya/LtSQwSCxLJrMwkuTSZ5NJkPkn+BBUqenj1IH7QdOL1/sQVHsJ7/09QchD2L7MtEvAIcVIkyBFCCCG6ELVaRZCHM0EeRwKgiqI6tjtnkrmnGCoa6WHR0MOigTqodVFz2KCws6mBzKYmMktqySyp5dej9qtp7lrXkvhgaJQvV0T746Q9/wIftV6Pc5/eOPfpbS9TmppozMxsbu05Evw0lZfTcOAADQcOULF0qX17XViYY2a3mBh0/v6nVJ9Al0CmRE1hSpRtbqHC2kJ7S09iQSLpFekcKDvAgbIDfNb8mO7doojrPYL42mriM7bhW3x0wHNlc5e2CRLwCNEOCXKEEEKILs7Dz5lRf4pGURSKs6vJ2FNMxp5iCjMqMdZY6VkDPXFC76rDNdyVBj89ec6QUVFnHwNUZ24iq7SWrNJaNqQW8/Fvmfi4OHH1gBBmJITRM+D8Hjui0mjQR0aij4zEY/IkoHki0/z8oxIcJGHJzcOcnY05O5uqVavs+9D4+rbJ7KYLC0OlPrlAz9/oz8TIiUyMnAhAcV2xLeDJT2RbwTYOlh/kYHkaB8vT+BLADSICBxOPM/FFmcQXZ+K/fznsXw4aJ4i60tbC02sCGDw66YoJcWGTIEcIIYS4SKhUKvy6ueHXzY3+Y0P54dsV9AoeSHZSGVlJpTRUm2nYWwaAh0bFhB6emPqEY+rrQ6Ozxp74ICW/ih9351FU1cB7G9N5b2M6/cM8mREfxuR+QbgbdOf4TDtGpVKhCwpCFxSE2xVX2MstZWU0pKQ4ZHZrTE+nqbiYmg0bqNmwwb6t2sWlTYIDfVQUKl3Hr4Gvsy/jw8czPtyWZKG0vtTWta25tedA2QHSa3JJBxYbgW4hmLRuxNfWEldeQELaKgIPrGgOeK440sLj7Nk5F0qIC5AEOUIIIcRFSqNX6DkogN6Xh9JksZJ3sJyMPSVk7i2hvKCWwyllHE4pY9OSg3j4OxPe15ehfX24bkAo/5gYw/r9RXyVmM3PKYXszC5nZ3Y5z/y4j4l9gpgeH8bgSO9OSeN8tmm9vNAOGYLLkCH2MmtdHQ379zu0+jQcOIC1poa6xG3UJW6zb6vS6dD36IFx8GDcJ03EEBt7UtfB2+DNGNMYxpjGAFBeX862wm32lp6U0hQyLVVkOsHX/rasfCFWFQk1VcTnbiD+0GpCvlc7dmmTgEdcZCTIEUIIIQQarZrQaG9Co725fHoPygtqydxbQsaeYnIPlFNRWMeutdnsWpuNzqChW6w3pj6+vDbtEmpUCt/tyOHLxGwOFlbzzY4cvtmRg8nHyPS4UK6NCyXIw/lcn+JpUTs749y/P879+9vLFLOZhkPpjgkOUlKwVlVRn5REfVISpe+/j1N4OO6TJuE+aRL6yIiTPranwZMru13Jld2uBKCysZIdBTvsk5MmlyaTo7aS4+bKd26uAARZLCSU/kH8mg3EL/8rod2Go+pzNfSaKAGPuChIkCOEEEKINjwDjHgGGOl3ZRiNdRayk0vJ2FNM5t4S6qrMpG0vIm17EaggINyd/n18uOq6AWQrFpZsO8wPu/LILKnl5VUHeHX1AYb18GNGfBijY/3Ra09vrprzhUqnw9CrJ4ZePWHaNMA2zsd8+DD1e/ZQuWo11evW0ZiRQfGCBRQvWIA+NgaPSZNwnzgRXVDQKR3X3cmdEWEjGBFmS6Vd3VjNjsIdbC3Yyrb8bewr2UeeFr53c+X75qDHv2EvCZu2Ef/zY8T7D8QUOx1VzCRw9uqUayHE+UaCHCGEEEIcl5OzlqiB/kQN9EexKhRmVtkDnqKsKgrSKylIr+SPH9Jx8XBibF9fZk3tzx5zA4t35fBHeim/HCjilwNFeBl1TBsQwoz4MGKCul5WMJVKhVNYGE5hYbhPnEhTdQ3VP6+lYtkyajZtpiEpmcKkZArnvYxzXBzukybiPn48Wm/vUz6mq5Mrw0KHMSx0GAC15lp2Fu5ka8FWEvMT2Vu8h0ItLHPVsswVaDqE347nidvyNAlu4cR3n0JEv1tQGU+9DkKcbyTIEUIIIUSHqdQqAiLcCYhwZ9DUSGrKG+zd2rKTS6mpaCRpYy5JG3PRaNXc0suTv1wezbbGOhan5FNQ2cAHmzL4YFMGfUM8mBEfytT+IXg4XxjJCk6WxtUFj6lT8Zg6FUtZGVUrV1G5bBm1iYnUbdtG3bZtFDz/Ai5DhuA+aRJuY0ajcXU9rWMadUaGhgxlaMhQAOosdewq2kVifiJbs39lT9l+irRaftJq+claAAfewzv5HeI0HiQEDyG+z01EBQ5ErTr/0oML0VES5AghhBDilLl46om9PJjYy4OxmJvIPWBLXpCxp5iqknqy9pXCvlKcgQeC3FB392drQx0/5JSwJ6eCPTkVPLcsmfF9ApkRH8aQSB/U6gsvWUFHaL288LrherxuuB5zfj6Vy1dQuWwZ9fv2UbNxIzUbN5L/1FO4jhiB++TJuI4YjtpgOO3jOmudGRw0mMFBg2HAvdRb6tlTvIfEQz+xNesXdtcXUKrRsJpqVueuhtzVeKImzj2K+MjxJISNoIdXDwl6xAVFghwhhBBCdAqtTkO33j506+3DsOt7UJZXa+/WlpdWQVleDeTVEAk84uyKJUDP1oY6NtfUsHRnLkt35hLi6cz0+FCuiwsl1Mt4rk/pjNEFBuIz51Z85txKQ3o6lcuXU7lsOY2HDlG1ejVVq1ejdnHBbfRo3CdPwmXw4JNKS308Bq2BhMAEEgIT+MvQf9LY1Mieg8tJTFnC1pK97FKZKVfD2spU1u5MhZ1v4q4xMDAgjvjgIcQHxhPtFY1G3TXGVomuSYIcIYQQQnQ6lUqFd7AL3sEuDBxnor7GTHZSc/KCfSU01Fggw0IcEKdyptFTy3ZzA8kl9by2OpXX16ZyeXdfpseHMTY2AIOu636h1kdE4HfPPfjefTcNKSlULltGxfLlWHLzqFi6lIqlS9F4eeE2fhwekybhPHDgSU9AejxOGifiek0jrtc07gTMhUns2/EBiVnrSLSUs92gp5J61uduYn3uJgDcdK4MCBjIAN8BNFgasFgt6OiaXQ7FhalLBDnp6enMmTOHgoICNBoNW7ZswcXF5VxXSwghhBDNDC46eiQE0CMhAKtVoeBQRfOcPMWU5NTgVGZhMBoGo6HeSUUyZtKSSnnwQDEuzlp7soI+IR7n+lTOGJVKZZtQNCYGv4ceom7nTip/XEblTz/RVFpK+edfUP75F2iDgnCfMOGU5uDpCJ1/LP3HzaM/cHvxQcz7viY55VsSaw+TaDCw3aCnylzNr4d/5dfDvwKwaMkiLg26lBujb2Rw0OALcn4k0bV0iSBn9uzZPPfccwwbNozS0lL0ev25rpIQQgghjkGtVhHU3ZOg7p4MuTqKypI6svaWkLGnhMP7yzA0WhmAlgGNWiwqhYwaKzvXH+abTZl0C3FnRnwoV/UPwcvF6VyfyhmjUqsxDhyIceBAAv7+ODVbfqdy2TKqVq/GkpdH6fvvd8ocPCfk2x3diMe4ZMRjXFJ8kDlJ32LZ9x37yw6QaNCTaNCzzWCgihrWZa9jXfY6enr1ZGbsTCZETMBJ03WfI3F+u+CDnH379qHT6Rg2zJY20fs0UjAKIYQQ4uxz93Gmz4hQ+owIxdzYRE5KGRl7isnYU0JNeQPdLRq6WzRQBwU1Daw5dID3l+6nbz8/rk/oxmXdfdF00WQFACqtFtfLL8P18suwPv0U1b/+SuWy5WdkDp7j8u0Owx9BO/wRepek0Xvft8zc9x3WrD3sd9Lxnasr37m5cKDsAE9seoLXtr3KjTE3M6PnDDwNnp1fHyGO45ynyfj111+ZMmUKwcHBqFQqvvvuuzbbLFiwgPDwcAwGA4MGDeKPP/6wr0tNTcXV1ZUpU6YwcOBAXnjhhbNYeyGEEEJ0Jp2ThvBLfBl5czSz/jWU659IYNDUSAIj3W0TjzapGdKg4/pKJ0wby/n8zR3c/OTPvLosmayS2nNd/TNOrdfjPmYMoa/9hx6bNhH84r9xGT4MNBrbHDzzXubgqCvIuPlPlH72GZbS0jNTEZ8oGP4wltvXsS7mJXoNfYy/O4WxOjuHv5aW4W+xUFxfyps73mTMV6N4bsM/yKjIODN1EaId57wlp6amhn79+jFnzhyuueaaNuu//PJLHnroId5++20GDRrEa6+9xrhx49i/fz/+/v5YLBY2bNjAzp078ff3Z/z48SQkJDBmzJh2j9fQ0EBDQ4P978rKSgDMZjNms/m4dW1Zf6LthDgeuY/E6ZJ7SHSGC+U+8ggw0G9MCP3GhFBX1Uh2UhlZ+0rJTCrFpcFKH7MWiqHph1zmrzhMU4CehCHBTBwSgrPTOf+ac2bpnTBOnIhx4kSaysqoXr2aquUrqG+ef6dlDh7j4MG4TpyA6xVXoD7NOXiOZjabqTEE0nDpLegu+yvG0kPMTv6eW5K/Y2XpIT72cCdFD18e+p6v0r5nhFsEN/e7h4HdRsm4HWF39PtRZ7wvqRRFUU57L51EpVLx7bffMm3aNHvZoEGDSEhIYP78+QBYrVbCwsK47777+Nvf/sZvv/3G008/zcqVKwGYN28eAI888ki7x3j66aeZO3dum/LPPvsMo7HrpqoUQgghuhLFCg1lGmoLtVTma9HWO3ZOKVdbqfVoIiDETFhwExdTtmNteQVuu3fhtnMXhpwce7lVq6UmOpqq/v2oiY5G6aSU1MdibCgguOx3Cmu38rWhll+MzvZ1Pc1qRqujCXObQJOT1xmth7jw1NbWctNNN1FRUYG7u/sp7eO8DnIaGxsxGo0sWbLEIfCZNWsW5eXlLF26FIvFQkJCAj///DMeHh5cddVV3HnnnUyePLndY7TXkhMWFkZxcfEJL6LZbGb16tWMGTMG3Rl+YxBdl9xH4nTJPSQ6Q1e7jyqK6tiTWMDexEKUwno0HGklMKvBKdiZgYODiBngh9H94hkM35iRQfWKn6hasQJzerq9XOXiguuVV+A6YQLGQYNOeQ6eDt9HZelk7f6ETzKX8YO6nobmFNgBFgs3aXy5uscMXHtfCy5+p1QPcWE7+j6qrKzE19f3tIKc87odt7i4mKamJgICAhzKAwICSElJAUCr1fLCCy8wfPhwFEVh7NixxwxwAPR6fbvZ13Q6XYff5E9mWyGORe4jcbrkHhKdoavcR77BOkZNdWfU1B7U15r5+Zcsdv6eh7agARerCuVwHduWHGLbkkMYApzpEx9AxCW++IW5oerCSQt0PXrg0qMH/vfd22YOnqrvf6Dq+x86ZQ6eE95H/j2JGv0MT/EM9+ft4qvEV/m8ZAcFWi3/oZy3D7zNNdtf4Wa3XoT1ng4xU8HV/zTOXFyIWu6jznhPOq+DnI6aMGECEyZMONfVEEIIIcR5wGDUMXFCFBMnRFFe3ch369PZ9XseLqUWgprU1BfUkbgsg8RlGejddERd4oupry+h0V44GbrEV6M22p+D50cqf1p5VufgAfAK6sedUz5idlMDy/d8zMfJizjYWMan7m58ruRw5dZ/MXPtP+gfmACxV9kCHreAE+9YiFbO61eyr68vGo2GgoICh/KCggICAwPPUa2EEEIIcaHwdHVi9uReMLkXSbmVLNmUwb7EAgJrIdyshiozSZvySNqUh1qjIqSXF+F9fTD18cXDz/nEB7gAOc7B83dqfttim4NnzRrHOXgiInCfOPGMzcGj1+i5uv+fmdbvdn7L/Y2Pd73DpqLtrHYxstrFyCX1B5i5/p9cufwRtOGXS8AjTsp5HeQ4OTkRFxfH2rVr7WNyrFYra9eu5d577z23lRNCCCHEBSU22J0np19Cw9VNrE0uZPEfWWQklxLRqCHKosazSU12UinZSaVs+DIVr0Aj4X19MfX1ITDKA43mnM+80elUWi2uwy7HddjlWBuepvqXX2xz8KxfT2N6un0OHkNsrG3S0YkTOn0OHpVKxdCQoQwNGUpqWSqLkhbx46Ef2G2Ahw1+hJgt3Fy6k6tXbMR1+SNgugx6T5OARxzXOQ9yqqurOXjwoP3v9PR0du7cibe3N926deOhhx5i1qxZxMfHc+mll/Laa69RU1PDrbfeelrHXbBgAQsWLKCpqel0T0EIIYQQFxC9VsPEvkFM7BtEXkUdX287zFdbs6kurifKrCHSoibUoqEsv5ay/Cx2rM5Cb9QSFutNeF9fuvX2xtm16yUvUOv1uI8di/vYsTRVV1O9di0Vy5ZRs2kz9UlJ1CclUThvHs7xcXhMmoTbuHHg5tapdejh1YNnLnuG+wfez5f7v+TLlC/JoYyXfLx4y9ubaysruPnwbwRlboTlj4BpKMROg9ip4Ca9fMQR5zy72vr16xk1alSb8lmzZvHhhx8CMH/+fObNm0d+fj79+/fnjTfeYNCgQZ1y/MrKSjw8PDqUvcFsNrN8+XImTpzYJQZpinND7iNxuuQeEp1B7iNHVqvCHxmlfJWYzfI9eSgNVsItGrpbNPS0atFajnxdUqkgIMKD8Ets3dp8Qly69JwvlrIyqlaupPLHZdQmJh5ZodFgHDyYtJBgLnvwQfRenZ8Kut5Szw+HfuDjfR+TUZlhOywqxjY5MTM/kz6Njc1bqqDbkCMtPO6d29okzqyj349O5vv5sZzzIOdckyBHnG1yH4nTJfeQ6AxyHx1bZb2ZH3fl8VViNjuzy1EpENSkpo/KiT4qHZpKi8P2rl56e7e20F5eaJ267qQ85vx8KpevoPLHH6lPSrKXq/R6XEeMwH3SJFxHjkDdTibb02FVrGzM2cjH+z7m9/zf7eUDDYHMrK5jZPYejlx1CXguNGciyDnn3dWEEEIIIc4n7gYdNw3qxk2DunGgoIrFidl8sz2HVTX1rKIeN3cVo9xc6avW05RfR3VZA3t/zWHvrzlodWpCo70w9fXF1McHN2/DuT6dTqULDMRnzq34zLmVhvR0yn/4gfzFS3AqKqJq1SqqVq1C7eKC2+jRuE+ehMuQIai0p/91U61SMzx0OMNDh5NSmsLH+z5mRfoKttfns10L3foM5k8ukVyVcwDj4UTI2mxbVjwG3QZD76th4CzQda3nQxybBDlCCCGEEMfQM8CNf0yK5ZFx0fycUsjixGzW7S/k+5oqvqcKN08NVwf70FftRG1GNdVlDWTsKSFjTwkAPiGuhPf1IfwSX/zD3VF3oTl59BEReP/lL2zp1o0rIyOpWbmSyuUrsOTlUbF0KRVLl6Lx8sJ9wnjcJ03CecCAU5qD52jR3tG8MOwFHhj4AF/s/4Kv9n9FVk0uL9TkMt/FneljH+amJj3++1fD4T8g6zfbcugXuP4T6IQ6iPOfBDlCCCGEECfgpFUzvk8g4/sEUlBZzzfbc1icmM2h4ho+ziwEINLHyHWDguiNjuIDFRSkV1CSU01JTjXbfsrE4KrD1NsHU18fusV6ozd2kW6CKhX6mBhcL7kE///7P+p27KBy2TL7HDxln31O2Wef2+bgmTgBj0mT0MfEnPY4pgCXAB4Y+AB/7vtnlqYtZVHSIrKrsvlf6ld8pNYyodcEZo55guicPbD2Gdi/DDa+CsMf7qQTF+czCXKEEEIIIU5CgLuBv4yM4q4RkSRmlvHV1myW7cnjUEktL5UcQqNWMbKnH9eM6km4WU1OUilZSaXUV5vZ/3s++3/PR6VWEdzdA1MfX8Iv8cEzwNglkheo1GqMcXEY4+Ic5+BZvdo2B8//3qf0f81z8EyahPukiegjTm8OHqPOyI3RNzKj5wzWH17Px/s+Znvhdn449AM/HPqBQYGDmDnsL1y+/jXUPz8Hwf2h++jOOWFx3rpogxxJIS2EEEKI06FSqUgI9yYh3JunpvZm+e48vkzMZltmGWtTClmbUoiPixPXDAzhuon9cK22krmnhIw9xZTl15JzoJycA+Vs/uYg7r4Ge/KCkB5eaHQXfpcqhzl45raag2fdOtscPPPnUzx/fqfNwaNRa7iy25Vc2e1K9hbv5eN9H7MqcxW/5//O70BEVDS3FGYz5evbMdzxC3iZOu9kxXlHsqtJdjVxlsl9JE6X3EOiM8h9dOYcLKxm8bZsvt6WQ3F1g728f5gnM+LDmNIvCGuVhcy9xWTsKSHnQBnWVimqtXoN3WK8MfX1wdTHBxePzs1U1plO5T5qqq6mas0aKpctp2bzZmj1g7N9Dp7x49F2QkrqvOo8Pk3+lK9Tv6baXA2AV1MT11uNXD/jO3zdQ0/7GOL0SXY1IYQQQojzXHd/Vx6fEMPDY3vxy/4ivkrM5ueUQnZml7Mzu5xnftzHxL5BtoBnZCjmhiYOp5SRuaeYjL0l1FY0cmhnEYd2FgHgb3LD1MeWvMAvzA3VBZ68QOPqiue0aXhOm4altJSqlSupWLaMusRt9iX/uedxGToUj8mTcL1yNBpXl1M6VpBrEA8nPMxd/e7i24Pf8sneD8mtK+RtTQP/+3Yik7tP45bYW+jh1aOTz1KcaxLkCCGEEEKcATqNmtGxAYyODaCoqoFvdxzmy63ZpBXV8M32HL7ZnoPJx8j0uFCujQtlVH8/FKtC8eFqMvYUk7G7mMLMKvuydVkGRncnW8DT15fQGC+cDBf2VzmttzdeN96I1403Ys7Ls83Bs2wZ9UlJ1GzYQM2GDbY5eEaOxH3SRFxHnNocPK5OrtwSews3Rt/I2q1v8vGu/7Jbr+fbg9/y7cFvuSz4MmbGzmRI8JAuMTZKSJAjhBBCCHHG+bnpuWN4FH8eFsmO7HIWJ2bzw648MktqeXnVAV5dfYDhPf2YER/GlTH+JHSLIGFSBDUVDWTtKyFzTwlZSaXUVjaSvDmP5M15qDUqQnp62pMXePgZz/VpnhZdUBA+t83B57Y5NBxKp3L5ciqXLaMxPZ2qlSupWrkStaurbQ6eSZNwGTL4pOfg0aq1jBv0IOMsGnb++iwfe3iw1sXIptxNbMrdRHfP7syMncmkyEk4aZzO0JmKs0GCHCGEEEKIs0SlUjGwmxcDu3nxz8mxLN+Tz1eJ2fyRXsr6/UWs31+El1HHtAEhzIgPIybInZihwcQMDabJYiU3tdyevKCiqI7s5DKyk8vYuDgVr0CjvZUnsLsHGs2Fm7xAHxmB37334HvP3TQkJ1OxbNmROXi++46K775D4+2N+/hxpzYHz9D76J+zjf5J35HtEcRnCdfzdeYKDpYf5MnNT/L69tdtGdt6zcDLcPpjg8TZJ0GOEEIIIcQ5YHTScl1cKNfFhZJeXMOSbdks2XaYgsoGPtiUwQebMrgk1IPp8WFM7ReMh7OOsBhvwmK8uXxGD8oLam3d2vYUk5daQVl+LWX5texck42Ts5Zusd6E9/WhWx8fnF0vzFYJlUqFITYWQ2ysbQ6e7dupWLaMqqPn4AkOwn3CSczBo1LBVfOhKIWwohQeS93KX274ia/TlvJp8qcU1BYwf+d83t3zLlOjpnJL7C1EeJxeqmtxdkl2NcmuJs4yuY/E6ZJ7SHQGuY/OT5YmKxtSi/kqMZs1yQWYm2xf0/TNk5FeHx/G4Egf1EclH2ios5CdVErmnmIy95VQV2U+slIFgRHu9m5tPiGunTbu5FzdR4rZTM2WLVT+uIyqNWuw1tTY1zlFRuI+cWLH5uApToWFo6CxCgbfDeP/hdlqZlXGKj7a9xHJpcn2TUeEjmBm7EwSAhNk3E4nk+xqnUjmyRFCCCHE+UarUTMq2p9R0f6UVDfw3c5cvtqazf6CKpbuzGXpzlxCvZyZHhfGdfGhhHg6A6B31tI9zp/ucf5YrQqFGZVk7rV1ayvOrib/UCX5hyr5/ftDuHrp7d3aQqK90DlpzvFZnzyVTofrsGG4DhuGtb6e6l9+pXLZMqrXr6fx0CHHOXimTsHrpptQO7XTmuXbA65+G768Gba8BSFx6Ppex6TISUyMmMi2gm18lPQRv2T/wi+HbUuMdwy3xN7C+PDx6DTyA8H5SlpypCVHnGVyH4nTJfeQ6AxyH104FEVh9+EKvkrM5vuduVQ1WABbj6vLu/syIz6MMbEBGHTtByvVZfXNAU8Jh5NLsZit9nUanZrQXl6E9/XB1NcXN2/DSdXtfLuPjjUHj6FPH0Je+w9OoceYF2ftM7DhFdAZ4fY1ENDbYXVGRQafJH/C0oNLqW+qB8Df2Z8bY25kes/peOg9zuh5dXXSkiOEEEIIcZFRqVT0C/OkX5gnT0yKZeW+fL7cms1vh0rYkFrMhtRiPJx1TOsfzPT4MPqEOH7hdvUy0HtYCL2HhWBpbCLnQLltTp49JVSV2gKgzL0l8PkBfEJcMPX1JbyPDwGRHm26xZ3vjp6Dp3LFCorfeJP6vXtJv+Zagv/9b9yuGNX2gaP+Abk7IO1n+OJmuGM9OHvaV4d7hPPE4Ce4t/+9LD6wmM9SPqOwrpDXt7/Owt0LmdZ9GrfE3EKYe9hZO1dxfBLkCCGEEEJcIJydNEwbEMK0ASFkldTakxXkVtTz0W+ZfPRbJrFB7lyfEMZV/YPxNDp20dI6aTD18cHUx4dhNyiU5tXYs7Xlp1VQklNDSU4N23/KxOCio1tvb8L7+hIW643B5dy31JwMrbc33jffjNuoURx+8EHqd+3m8N1343P7bfj99a+O6afVGrj2f/DOCChLh2/vhBs+h6MytnkaPPnzJX9mVu9ZrEhfwcdJH3Og7ACfp3zOFylfcEW3K5gZO5MB/gNk3M45JkGOEEIIIcQFqJuPkYfG9uKB0T3ZdLCYLxOzWb2vgKS8Sp76fh/PL0tmbO8AZsSHcVl3XzRHtcqoVCp8gl3xCXZl4DgT9TVmsvbZurVl7SuhvsbMgT8KOPBHASq1iqAoD0x9fQjv44tXkPGC+RKvCw4mfNEiCl5+mbKPF1Hy3v+o3bmTkFdeRRfgf2RDozdcvwjeHwcHfoJf58HIx9rdp5PGiau6X8XUqKlsydvCx0kfszFnI2uz1rI2ay19ffsyM3Ymo02j0arl6/a5IFddCCGEEOICplGrGN7Tj+E9/SiraWTpzhy+TDxMcl4lP+7O48fdeQR7GLguLpTp8WGEebc/aajBRUfPSwPpeWkg1iYr+Ycqydxr69ZWmltDbmo5uanl/PZNGu6+Bkx9fAmN9US5AHI4qZycCPz73zEOjCPvH/+gLnEb6ddcQ8jL83AZMuTIhsH9YdKrsPRuWP8vCBkIPcYce78qFUOChzAkeAhp5WksSlrED2k/sKd4D4/8+ghBLkHcHHMz1/S4BjcntzN/osJOEg9I4gFxlsl9JE6X3EOiM8h91PXtzalgcWI23+3MpaLuSErpoVE+zIgPY3yfwGMmKzhaZXGdPVtbzv5ymixHkheoNArdYn2J7OeHqY8PLp76Tj+XztSYkcHhvz5IQ0oKqFT43ncvvnfd5TiZ6I8PQuL7YPCwjc/xjuzw/kvqSvhq/1d8sf8LSutLAXDRufDmFW+SEJjQyWfTNUjiASGEEEII0SF9QjzoE+LB4xNjWJVUwOLEbDYeLGZzWgmb00pwW6rlqv7BzIgPo2+Ix3G7n7n7OtN3ZCh9R4ZibmjicEopGc1jeWorGsncU0LmnhIA/Lq52bu1+ZvcUJ1nyQucwsMJ/+JzCp5/nvLFSyh+403qtm0neN5LaL29bRuN/zfk74HDW+HLmXDbKnBqvwXsaD7OPvyl/1+Y03cOP6b9yEdJH5Fekc7r21/nk4mfnMEzE61JkCOEEEII0YUZdBqm9gtmar9gDpfVsmTbYRYnHianvI5PtmTxyZYsogPdmB4fxtUDQvB2aWc+mVZ0eg0R/fyI6OdHY2MjS79YSTevWLKTyijIqKQoq4qirCoSl2Xg7O5km5Onjw9hMd44OZ8fXz3VBgNBzz6L88A48ufOpWbTJtKvvoaQ//wH48ABoNXDjI/hneFQsAd+/Ctc/Y4tb3cH6TV6ru15LSPCRjBm8Rh2Fe3iQNkBenr1PHMnJuzUJ96ka1qwYAGxsbEkJEizoRBCCCEuDqFeRv46uicbHh3Fp7cP4qr+wThp1aTkV/Hsj0kMemENd3+6jXX7C2mynnhEg0qlwsnDysDx3bjusXhuffFyrpwVQ9RAP5wMGuoqG0nZnMdPC/fyv4c3sPS1Hexam019jfmE+z4bPK+eRviXX+IUEYGloIDMmTMp+eBDFEUB92CY/iGoNLD7S/jj3VM6hq+zL6O62dJWL96/uBNrL47nog1y7rnnHpKSkti6dev/t3fncVWW+f/HX+cAh3PYBAQFkhQVU1AUcQkMw1zILKUptbLSprRxcBx10sZSXMq20RZHGnMqtW+aZovNL7cMc1rEPSslzV1TcQNFZTks5/cHeSZySWQ5cHg/H4/zyPu+r/u+P7de6flwXdfndnQoIiIiItXKaDTQpXkAr90XzaanevBMv0ja3FCPwmIby3/I5JG5m+jywhr+sWonB05duObreviYaBkbzO3D2vDH6fH0G9WOtj1C8W3oQUmxjZ93ZvP1kt28m5LOD2t/pqS45PcvWsXMN7WgyZIl+NxxBxQVceLFFzkyciTFOTnQ5Bbo9Uxpw1Xj4WD6dd3j3hb3AvDpvk/JK8qrrNDlKupskiMiIiIiUM/DjYdim/D//nILy0fG80iXJvh5uJGZk0/qF3tJmL6WAW+k8+GWn8m1Fl3zdV1cjTRq6c8t94YzaMrNDJpyM7f0D8cv2JOCC0V8uegnFj27iUMZp6vw6a4xVi9PQmZMp2HKRAxubpxb/Tn777mX/IwMuPnPEPkHKCmCJYPhXGa5r39z8M008mrE+cLzrNy/sgqeQH5LSY6IiIiIABAR4sOkuyJZ/1R3Xh/UnltbBGIwwMb9WfxtyXd0mpbG+I++Z+uhbMpboNe3oQdtu4dy34SOdL2vBWZPN7KPXeD/zfyOZanfkZ157SNGVcFgMOD/wAM0XrgQtxtuoPDwYQ7cdz/Z7y/B1vef0CACzh+H9wdDkbVc1zYajNzT4h4APvjpg6oIX35DSY6IiIiIlOHu6sIdbYKZ/8dOfPPkbTzRqwU3+ntwvqCI9zYe5g+vr6PXK1/y1jcHyCnf932MLkbaJDRi0NSbaXtbKEajgQM/nGbR1I18vWS3w9frWNq0JuyjD/Hq1g2b1UrmpEkcnTiVkr5vgrsPHF4Pn00o93WTmifhanDl+1PfsytrVxVELr+mJEdERERErijE18KI28JZ+0QC7w29mT9E34DZzcjuE+d5YeVPTNrqwp8XbuPzjOMUlWONjdnTjVsGhHNfSicat6lPSYmN79IOsyBlPdv/69j1Oi716tEodRYNxj4BLi7k/Of/sf9PT1LQYWppg41vwHeLy3XNMgUIflIBgqqmJEdEREREfpfRaCC2WX1eHtiOjU/34Lm729C2UT1KbAZW/3iCx97ZTOwLa3hhxU72njx/zdf1C/LkzuS23PWXtvgFe5J/oZD/vvcTi6dt4vCPWVX4RFdnMBqp/+ijNJ4/D9fAQKx79rJ/3D85a+5f2uD//bX0XTrl0L9F6bnL9i0jtzC3skOWX1GSIyIiIiLl4mN244HON/LB4535e9si/hjXmPqeJk6eK2D2f/fSfcZ/ufdf63h/02EuFFxbsYIbI+vb1+u4e7qSdfQC/3ltG8te/54zxx2XEHh06EDY0o/xiL0ZW14eR+d9w7GfIikpyIPFD0Je9jVfq3NwZ0K9Q0sLEBxQAYKqpCRHRERERK5bsAeM730T6eO7M/vBGLq3bIDRAJsPZjPuw+/pOO1zxn3wHZsPZP1usYKL63UenBpL1G2NStfrfH+K96Zu4OsPdlOQ65j1Oq7163Pjm28S8OfhYDBwZms2B78IwXr4MHw4FEqubWqd0WC0l5NWAYKqpSRHRERERCrM5Grk9tZBvDWkI+njuzPu9psIC/Ak11rM+5t/5t7Z6XSf8V/+tXYvJ3Lyr3ots6cb8QNalK7XaV2fkmIb331+mHdT1rP9yyOUXMOLSiubwcWFwJEjCZ0zBxc/P/JPwf5VgZxb+xX894Vrvk6/Zv1wNbryw6kf2Jm1swojrtuU5IiIiIhIpWroY+bPCc1Z87dbWfKnWPrHNMLD5MK+Uxd4ceVOYl9Yw2PzN7FqRyaFVykw4BfkyZ0j2nLnX9riF+RB/vlC/rtwF+9P28jPOx2zXscr/hbCPv4IS3Q0JYVGfv7an+Ov/gvbjmXXdH59S32639gdgCW7VICgqtTZJCc1NZWIiAg6duzo6FBEREREnJLBYKBjE3/+0b8tG5/uwYv3tCGmsR/FJTY+//EEj//fFmKfT2Pasgx2Hz93xes0jqzPwImdiB8YjruHK6ePXOCTV7ex/F/fc+ZE9a/XcQsKovE78/EfMgSArF1eHBw+msIfN1zT+fYCBPtVgKCq1NkkJzk5mYyMDDZt2uToUEREREScnpe7KwM73siHw+P4fMytPH5rUwK83Dl13sq/v9pPz1e+5O7Xv+G9jYc4l3/p2hsXFyNR3UJ58JlY2nRrhMFoYP93p3hvyga++XAPBXnXVuCgshjc3Gj49ye54dWXMZoM5J1wYf/9j3D+v2m/e27HoI7c6H0jFwovsGL/imqItu6ps0mOiIiIiDhG8wZejO/divTxt/HvhzvQM6IhLkYD3x46w/iPfqDjtM8Z8/42Nuw7fUmxArOnG10HtuC+iZ24MdKfkmIb21YfYkFKOju+qv71Oj639yZs0Tu417dRnG/j8J9GcPKf/8RWXHzFc35dgEDvzKkaSnJERERExCHcXIz0jGjIvx/uQPr423jqjpY0C/Qkv7CEj7YeYeCc9XSbvpbUL/aQebZssQL/YE/u+ks77hxRul4n71whaxfs4v1pm/h517WXda4MpogONJn3Br7N88AGp1Jf5/DQoRSdPn3Fc/o174eb0Y0dp3eQcTqjGqOtG5TkiIiIiIjDNfA2M6xrMz4fcysfDo/jvo6heJpcOHA6l3+s2kXcC2kMmbuR5T8cw1r0v2IFjVuXrte5ZcDF9Trn+eSVb1kx+wfOnqy+9S7G8FsJnvB3Qm7OxuBi48K6dPbf/Qdyt2y5bHt/sz89buwBqJx0VVCSIyIiIiI1hsFgIKaxHy/cE8WmCT2Y3r8tncL8KbHB2l0n+fOCrdz8fBpT/18GOzNzgNL1Om1vC+XBqbG0SShdr7Nv20kWTtnAuo/2YK2u9TqdH6feXXcS1uskJl8bRSdOcPDhwZx+663LviPo4pS1ZfuWcaHwQvXEWEcoyRERERGRGsnD5Mq9MY14//FYvngigT8nNKOBtztZF6y8/c1+bn/1K/rO+pp31x/kbF4hZi83ut7XgoETOhIa4U9JkY1vPzvEuynpZHx9tOrX6xgMcNdruLdoSVj3THxaWaC4mBP/mM7PySMoPnu2TPOOQR1p4tOE3KJclu9fXrWx1TFKckRERESkxgsL8GTc7S1Z9/fbmDukI71bB+HmYuD7n88yYel2Ok37nFGLvmXdnlP4BXly11/a0ic5Ct+Gpet1vnh3J0ue38SRn6p4vY7JEwb+H0ZvH0Ki9hJ0b2sMbm6cX7OG/ffcS972HfamBoPBPpqjKWuVS0mOiIiIiNQari5GurVswL8ejGH9+O5M6NOKFg29KCgqYem2ozzw5gZunf4F/1yzB7dQT+6b2Ilb+peu1zl1+DxLX/6WFW/8wNmTeVUXpH9T+MObGAwG/Fw/o/HUwbg1akThzz9z8P77yX7vPfv0tb7N+uJmdCPjdAY7Tu/4nQvLtVKSIyIiIiK1Un0vdx6Lb8qqUV35JLkLgzrfiLe7K4ez8nh59U/c8uIahszfxKEAF+6d2InWt96AwQD7vj3JwinrSf+4CtfrtOgFCX8HwLLjJcL+NQWvHt2xFRaSOWUqR58YS8mFC/iZ/ejRuLQAwZJdKiddWZTkiIiIiEitZjAYaBvqy7S727Dx6R68MrAtsU3rY7PBV7tP8Zf3vuXWmV/yhWcRMY9HENrKj5IiG1tXHeLdSevJ+KaK1ut0HQfhiVCUj8uyx2n04hQaPPkkuLqSs2wZ+/sPoGD3bvq36A/A8v3LOW89X/lx1EFKckRERETEaVhMLtwd3Yj3ht3Ml2O78ZfbmhNcz8zZvELmrTvAwEVbeINzmG9riHegmbwcK1/8X+l6naO7K3m9jtEIf3gD/MLgzCEMHz1G/cEP0fid+bg2bIh13z72DxhI+PqjNPFpQl5RngoQVJI6m+SkpqYSERFBx44dHR2KiIiIiFSBG+t78LdeN/H1k7cx/4+d6BMVjMnFyI5j53hm6wGeKz7L8aZmjO5GTh0+z8czvmXlnB/IOVWJ63UsfjDwXXC1wN418MVzeLRvT9jHH+HZpQu2vDyO/f3vPJHmiVuhjQ9++uCy5aalfOpskpOcnExGRgabNm1ydCgiIiIiUoVcjAZubRFI6gPt2fBUdybfFUGrYB/yS0p4Jyubme4X+MkbbMDerSdZOHkD6Uv3Ys2vpPU6Qa2h7z9Lf/3VdNi5DFd/f0LnvEHAX0aAwUCDz79j2v+VkLVHBQgqQ51NckRERESk7vHzNDGkSxjLR97Cp3+5hYdjG+Pm4conLnnM887noGsxxUUlbF15kHdT1vPjuqPYKmO9TlR/6Dy89Ncf/wlO7cHg4kJgcjKhb/4bF39/mhy38eK8YtLfe7Xi96vjlOSIiIiISJ1jMBhofUM9pvZrzcanezDz/mha3lSfJV5WPvIsINtYQl6OlTXv7OSdqesr5/06vZ6BG+OgIAcWD4KC0iIDXl26EPbxRxS3aYFHAdyS+g2Hn52KzWqt+D3rKCU5IiIiIlKnmd1c6Ns2hHcf68yX47pxV+9mrAo18oW5kAJsnM/MY+nL3zL7uQ0cOpxz/TdycYP+88ArCE7uhP+MgF/W37g1bEjEgiX8t6sfAOfffY+DDw+m8NixSnjCukdJjoiIiIjIL0L9PRjVowX/fbIbo0fE8HOcLz+YiynBRvGhC3w8bROTp31D2g/HKL6eaWzeDWHAO2B0gx0fQ/os+yGjyYTnqOG8eK+RPIuRvG3b2H/3H7iQnl6JT1g3VEqSc/DgQTIyMigpKamMy4mIiIiIOJTRaKBL8wBefjiGF59LwPuuRpz2NOCKgcDDBWx+PYOHJqxh+sqdHDx9oXwXv7Ez3P586a9XT4L9X9kP9W3Wlx9uMvPEEAO2m5pSfOYMR0aPwVZcXIlP5/zKleS8/fbbvPzyy2X2DRs2jKZNm9KmTRtat27N4cOHKzVAERERERFHqufhxpA+NzFxegKtBzaj2MMFL5uBLlkGzv6/n3ng+S8Z+EY6H275mVzrNVZk6/gYtL0fbMWwZAicPVJ6L/d69GrSi5O+Bt4f3Q6jlxfFZ85Q8NNPVfeATqhcSc6cOXPw8/Ozb69cuZK5c+fyzjvvsGnTJnx9fZkyZUqlBykiIiIi4mgGg4FbuzUm+cV4OiU1xWAyElRs5IHz7gRuP8eUxd/TaVoa4z/6gW8PZV/9fTcGA9z5CgS1gdxT8P5DUFQAQP8W/QFYfmQ1bm3bAJC7ZWuVP58zKVeSs3v3bjp06GDf/uSTT+jXrx+DBg2iffv2PPfcc6SlpVV6kCIiIiIiNYWLm5GOtzdhyLNxRMSHgAFaFbry6Dkzbc/ABxsOcffr6+j1ypf8+8t9nDpfcPkLuVlKXxRq9oUjW2DFkwBEN4imWb1m5BXlcaCxGYC8rVuq5+GcRLmSnLy8PHx8fOzb69ato2vXrvbtpk2bkpmZWXnRiYiIiIjUUB4+JroNasnApztyQwtfXG0QV+DGiDwP2hW7svv4eaYt/5Gbn0vj8f/bTNqPxykq/s0adr8mcM9bgAG2zIWt/4fBYKD/Tb+M5njvA0pHcq46MiRllCvJady4MVu2lGaRp06dYseOHXTp0sV+PDMzk3r16lVuhCIiIiIiNVhAI2/6jY6m9+Nt8Akw42q10fOcG39386Wrvw9FJTZW7TjOo/M3E/vCGl5YsZN9J8//7wLhPaDb06W/XvY3OLKVO5veibuLO194/4zNxYWi48cpOnrUMQ9YC5UryRk8eDDJyck888wz9O/fn5YtWxITE2M/vm7dOlq3bl3pQYqIiIiI1GQGg4Gm0YE8MOlmYu9uhpvZheJTBXTeV8j0RjcwNOZG/D1NnDxXwOz/7uW2Gf+l/+x1vL/5MBcKiiD+b9CiNxQXwPsPU6+oiMQmiVjdDJxuXDqIkLtV63KuVbmSnHHjxjF06FA++ugjzGYzS5YsKXP8m2++4f7776/UAEVEREREagsXNyPtExvz4NRYIroEgwGOb8+i/penmdWmKa8PbMdtLRtgNMCmA9mM++B7Ok77nHEf/cC3HV7A5t8Mzh6GD/9I/+Z/AGBT4DkAcrdoXc61ci1PY6PRyNSpU5k6deplj/826RERERERqYs8fEx0e6gVrRMa8fX7uzm6+wzfrjyIZz0TTw1owfN/aMOHW39myeaf2X/qAu9v/pn3N//Mbf5/5Q3jONz2raXtD+1o7tuc7Tf8RG8gb+u3jn6sWqPcLwNdvHgxgwYNon///syePbsqYhIRERERcQqBod4kjYnm9sdb4xNg5sJZK6ve3I71SC5/TmjOmr/dypI/xXJvTCMsbi6syQpgdP5QAAzfvEpscWN2NTIAULB7N8VnzzrycWqNciU5//rXv7j//vvZvHkzu3fvJjk5mbFjx1ZVbCIiIiIitZ7BYKBZdAPun9SZm24OAht8PjeD3BwrBoOBjk38md6/LZsm9ODFe9pwLPQO/l10BwAP/bCIcxY3jvoBNht527Y59Flqi3IlObNmzWLSpEns2rWLbdu2MX/+fF5//fWqiq1KpaamEhERQceOHR0dioiIiIjUAa5uLiQ8cBP+IZ7k5lhJm5+BreR/ZaG93F0Z2PFGPhweR7cRr3PQO5pgWx7dzlvZFVo6mrN/bbqjwq9VypXk7Nu3j8GDB9u3H3jgAYqKijh27FilB1bVkpOTycjIYNOmTY4ORURERETqCFeTC70ei8TVzcihHVls+/zwZds1D/Kj8ePvY/MO4ZHzJ9j5y5S1I199U53h1lrlSnIKCgrw9PT838lGIyaTiby8vEoPTERERETEGdUP8eKWAeEArF+6l+MHci7f0KsBhgHv0LbQRl7DIgCCju3j+30nqivUWqtc1dUAJk6ciIeHh33barUybdq0Mi8BffnllysnOhERERERJxRxSwiHf8xm79YTfPbmdgY+3QmT5TJfzUM7Yuj9Irf9N4WzHj7Uyy1h4XufE/X0A9UfdC1SriSna9eu7Nq1q8y+uLg49u3bZ982GAyVE5mIiIiIiJMyGAx0e/AmThzIIedUPmsX7qLnHyMu/126wx/p8/NGPrnhSzrsBuMPy/jx2J20Cvap/sBriXIlOWvXri2zferUKUwmEz4++g0WERERESkPdw83ej0WyUfTt7J703FCW/nRKi7k0oYGAz53vopxVWfYXUjHs98yOy2D1x68ufqDriXK/Z6cM2fOkJycTEBAAA0bNsTPz4+goCDGjx9Pbm5uVcQoIiIiIuKUgprWo3PfMAC+XPQTWccuXL6hm4WoviMAaJBpo91Pz7LnxLnqCrPWKVeSk5WVRefOnZk/fz733HMPM2bMYMaMGfTt25d//vOfdO3alfz8fDZu3MjMmTOrKmYREREREafRvldjGrX0o8hawmdv7qCosPiy7SK7D6bQ1YB3HnjxPVs/erV6A61FypXkTJ06FZPJxN69e3njjTcYNWoUo0aNYs6cOezZswer1cpDDz1Ez549yxQiEBERERGRyzMYDfR4JAKLtxunj5xn3Yd7L9vO6O5OwU03ArDzjAd9j73KsR1fV2eotUa5kpylS5cyffp0GjZseMmxoKAgXnrpJT788EPGjBlT5n06IiIiIiJyZZ713Ok+JAKAH9b+zL5tJy/bLjjuNgDqHzOyw+yCZekjcP7ybeuyciU5x44dIzIy8orHW7dujdFoZNKkSRUOTERERESkLmkcWZ92PUtHata88yPnsvIvaePbsbTYQMufbcz1DsS38AQFiwZDcVG1xlrTlSvJCQgI4MCBA1c8vn//fho0aFDRmERERERE6qSb+zWlQWNvCnKLWP32DkqKS8oct7Rrh81gIDgbvje4ctRgxv3nbyBtioMirpnKleQkJiby9NNPY7VaLzlWUFDAxIkTuf322ystOBERERGRusTF1UivxyJxM7twbM9ZNi0/UPa4jw/mFi0ACD9SxJ88upceWDcTdnxczdHWXOUuPLBr1y7Cw8N56aWX+M9//sMnn3zCCy+8QHh4OD/++COTJ0+uolBFRERERJxfvUAPEgbdBMCW5Qc4siu7zHGPmPYAtDxs40jACf5V1Kf0wNJkOLGzWmOtqcqV5DRq1Ij09HQiIiIYP348SUlJ3H333Tz99NNERETwzTffcOONN1ZVrCIiIiIidUKLjkG0jAvGZoPVb+8g7/z/ZlJZokuTnIgjBqzG47zs1pn1tkgovACLB0H+WUeFXWOU+2WgYWFhrFixglOnTrF+/XrWr1/PyZMnWblyJc2bN6+KGEVERERE6pyuA1vg29CDC2etrHlnJzabDfjfSE6T4zbcrTbq3/Adfy74CzmmhnB6Dyz9M5SUXO3STq/cSc5Ffn5+dOrUiU6dOuHv71+ZMYmIiIiI1Hlu7i4kDo3ExdXIge9P8f0XP5fuDwnBNTgYY7GN5sds5Ju2ke3iwvDCUdhcTLDzU/jmFQdH71jXneSIiIiIiEjVCmjkTdw9pbOl1n20h5OHzgHgER0NQPypAIpthfTsdJjpox7BcMf00hPTnoEzhx0Sc02gJEdEREREpAZrk3ADYW0DKCmyserN7Vjzi7D8MmWt4wlvAI6VrCXIxwwxg8G/KWCDM4ccGLVjKckREREREanBDAYDtz3cCi8/d86eyOOrRT/hERMDgM/uTDyNFg7kHGDz8c2lJ5g8S/9blOegiB1PSY6IiIiISA1n9nSj5x8jMRhg5/pMDmT7YPTywnbhAveb4gBYsmtJaWNXc+l/C/MdFK3jKckREREREakFQsJ96XhnGABfLtpNcbtbAOh5phEAnx/6nKz8rP8lOUVKckREREREpIaL6d2EkHBfCguK2ebdkxKDK/V2HSOifgSFJYX8Z89/wM1S2lhJjoiIiIiI1HRGo4Gef4zA3dOV7Dwze5v2JXfLFvqH3wvAB7s/wObiXtq4UGty6pzU1FQiIiLo2LGjo0MREREREblmXn5mug+OAOBwaHcyCwPoZW6Ph6sHB3MOsslgLW2okZy6Jzk5mYyMDDZt2uToUEREREREyiUsKoCobqVrcX5s+RAXNv5In6Z9AFhSfKq0kZIcERERERGpTeL+0Bxf0wUKTd589ZWVe5uXTln7vPAUp41GVVcTEREREZHaxcXNyK0JZlyKCzhp9SV3kwet67emCBufeHtqJEdERERERGqfoK7RtPhpMQAbP91HP6/7APjQ24sSFR4QEREREZHaxjUggBvNmTTM3IitBPJX1cev2IdDbm5szM90dHgOoyRHRERERKQW84iJ4abdi/ByK+BCtpW7D/4RbLAk/2dHh+YwSnJERERERGoxj5j2uBYX0O7sKowuBkzHw4g43oU1xdmcyjvl6PAcQkmOiIiIiEgtZoluD4D7d2uJ7dsEgFsO3I3PhWA+2fOJAyNzHCU5IiIiIiK1mCmsCS5+ftgKCmjR4AyNGxditLnRY/cQPv7xE0psJY4OsdopyRERERERqcUMBgOWmNLRnLyt39L9dhsWYxb+eUE02d6J9cfWOzjC6qckR0RERESklvP4Zcpa7tatWHzc6VXvVWyU0OpEHCs+/8bB0VU/JTkiIiIiIrWch30kZys2VzON3H8gLDANAP8NERw5WbfKSSvJERERERGp5cwRERjc3SnOzsZ6LBuA3vX+Q75HDqZiC8s2pDk4wuqlJEdEREREpJYzmExYoqIAyMvYB4Cx+AL1giwAbN77fZ0qQKAkR0RERETECVja/7IuZ8dPpTuKCgi7oVHpL3Ng/dG6U4BASY6IiIiIiBO4uC4n9/uM0h2Fefj6ewLgZfVjyU9LHBVatVOSIyIiIiLiBCzt2oHBQOHhIxTlGwEbXr4uAHgV+PLF4S84mXvSoTFWFyU5IiIiIiJOwMXHB/fwcAByT5oA8PYuPVa/KIhiWzFL9yx1UHTVS0mOiIiIiIiTsL8U9FRpkuP1S5JjLvAGG3y4+8M6UYBASY6IiIiIiJPwaB8DQO4pMwCeHkVgAIoNBBiCOHL+COuOrnNghNVDSY6IiIiIiJO4WHwgP8uFkiIDLrYCPHxKR3V6B/YFYMku5y9AoCRHRERERMRJuIWE4BoUBDYDeafdoCgPb//SUZ3OXl0A2JC5wZEhVgslOSIiIiIiTsTj4vtyTpqgqAAvP3cAjBdK/1tYXOiw2KqLkhwRERERESdSpvhAYR5ev4zk5J0pAlDhARERERERqV0ujuTknTJhK8jF269sklNsK3ZYbNVFSY6IiIiIiBNxb9ECo8lASZGRgn0H7NPV8s6UTlOzYcNmszkyxCqnJEdERERExIkYXFywNPIAIHfHXvt0tdwz/1uL4+yjOUpyREREREScjEdjHwDyftz/v5GcnEKMJaVf/519XY6SHBERERERJ2MJ8wMgd9dhLF5uGF0NYAOPwnqARnJERERERKSWsTQJBIONouwLFGcew8u3dDTHq6A0+dGaHBERERERqVWMnp6Y/UrX4ORu/db+QtCLSY5GckREREREpHZxNeMRaAUgd+sWvH4pI+1l9QW0JkdERERERGqbHlOw/PlNAPK2bLUXH6grIzmujg6gMjRp0gQfHx+MRiN+fn588cUXjg5JRERERMRxXFzxiIkBoGD3bjxLK0rjZS1Ncpx9JMcpkhyAdevW4eXl5egwRERERERqBNeAAEyNG2M9eBC304cB8L44klPi3CM5mq4mIiIiIuKkLO3bA+By8EcAPLUmp3p8+eWX3HXXXYSEhGAwGFi6dOklbVJTU2nSpAlms5nOnTuzcePGMscNBgO33norHTt2ZMGCBdUUuYiIiIhIzeYRU5rkGLZvAsBc5Ilrscnp1+Q4PMm5cOECbdu2JTU19bLHFy9ezJgxY5g0aRJbt26lbdu2JCYmcuLECXubr7/+mi1btvCf//yH5557ju+//766whcRERERqbEs7UvX5RT9sBWT2QUorbDm7O/JcfianN69e9O7d+8rHn/55ZcZOnQojzzyCACzZ89m2bJlvP322/z9738H4IYbbgAgODiYO+64g61btxIVFXXZ6xUUFFBQUGDfzsnJAaCwsJDCwsKrxnrx+O+1E7ka9SOpKPUhqQzqR1IZ1I9qPkOjGzD6+VGSnY2HxYY1v7TCWkFhQY35c/ttP6qMuBye5FyN1Wply5YtjB8/3r7PaDTSo0cP0tPTgdKRoJKSEry9vTl//jxr1qxhwIABV7zm888/z5QpUy7Z/9lnn+Hh4XFNca1evbqcTyJyKfUjqSj1IakM6kdSGdSParaQ4GC8srMpOpsJNMCrwI8v/vsFDVwaODq0Mi72o9zc3Apfq0YnOadOnaK4uJiGDRuW2d+wYUN27twJwPHjx7n77rsBKC4uZujQoXTs2PGK1xw/fjxjxoyxb+fk5BAaGkqvXr3w8fG5ajyFhYWsXr2anj174ubmdr2PJXWc+pFUlPqQVAb1I6kM6ke1Q/aJE5zOyMC/5DznaYCX1Zdb4m+huW9zR4cGXNqPLs60qoganeRci6ZNm/Ldd99dc3t3d3fc3d0v2e/m5nbN/3OWp63IlagfSUWpD0llUD+SyqB+VLN5d+zIacDt2D4IbopXgR9GF2ON+zO72I8qI64aneQEBATg4uLC8ePHy+w/fvw4QUFB1RZHSUkJVquVwsJCXF1dyc/Pp7jYuStSSOVyc3PDxcXF0WGIiIhIHWSOiMDg7o7pzBEILn0hqLNXV6vRSY7JZCImJoa0tDSSkpKA0oQjLS2NESNGVEsMVquV/fv3U1JSgs1mIygoiMOHD2MwGKrl/uI8fH19qzU5FxEREQEwmExY2rTBvCcbAK8CX6d/T47Dk5zz58+zZ88e+/b+/fvZtm0b/v7+3HjjjYwZM4bBgwfToUMHOnXqxKuvvsqFCxfs1daqks1m49ixY7i4uBAaGmqP18vLC6PR4dW3pZaw2Wzk5ubay54HBAQ4OCIRERGpaywxMbhv/xAAT6sfRSVFDo6oajk8ydm8eTPdunWzb18sCjB48GDmzZvHwIEDOXnyJCkpKWRmZtKuXTtWrlx5STGC8kpNTSU1NfWq086KiorIzc0lJCQEDw8P+7Q1s9msJEfKxWKxAHDixAn8/PwcHI2IiIjUNR4x7THPeQsAtxIT1lxNV6tSCQkJv/syohEjRlT69LTk5GSSk5PJycmhXr16l21zMQEymUyVem+pmy6WKC8qcu6fnIiIiEjNY2nXDiPFuFlzKDT5kH/GuZMcDUdcA62/kcpwsR85+xuGRUREpOZx8fHBPTwcc37puhwlOSIiIiIiUutZYtpjLvglyTnr3DNLlOSIiIiIiNQBHu3b416QBUDBGeeurqYkRy5hMBhYunQpAAcOHMBgMLBt2zaHxiQiIiIiFePRvj3m/DMAFGRpJEfqsNDQUI4dO0br1q1/t21NSYimTZtGXFwcHh4e+Pr6XrbNoUOH6NOnDx4eHjRo0ICxY8deUhBg7dq1tG/fHnd3d5o3b868efOqPngRERGRKuIaEkKJy1kAik7kOziaqlVnk5zU1FQiIiLo2LGjo0Op0VxcXAgKCsLV1eGF+K6Z1Wqlf//+DB8+/LLHi4uL6dOnD1arlXXr1jF//nzmzZtHSkqKvc3+/fvp06cP3bp1Y9u2bYwaNYrHHnuMVatWVddjiIiIiFQqg8FAdmABAIU5Dg6mitXZJCc5OZmMjAw2bdp0zefYbDbyrMXkWouq/VOeilwJCQmMHDmScePG4e/vT1BQEJMnT76O36VLR2eys7MZNGgQgYGBWCwWwsPDmTt3LgBhYWEAREdHYzAYSEhIAEpHRDp16oSnpye+vr506dKFgwcPXlc812LKlCmMHj2aNm3aXPb4Z599RkZGBu+++y7t2rWjd+/ePPPMM6SmpmK1WgGYPXs2YWFhzJgxg1atWjFixAjuvfdeXnnllSqLW0RERKSqnWxUWlXNWuROSYnzVnytPT+erwHyCouJfXm9Q+6dMTURD9O1/3HNnz+fMWPGsGHDBtLT0xkyZAhdunShZ8+eFYpj4sSJZGRksGLFCgICAtizZw95eXkAbNy4kU6dOvH5558TGRmJyWSiqKiIpKQkhg4dynvvvYfVamXjxo1XLcsdGRl51SQoPj6eFStWXPczpKen06ZNmzIvlE1MTGT48OHs2LGD6Oho0tPT6dGjR5nzEhMTGTVq1HXfV0RERMTRjjRzocmmYmxGF3LPFuDlZ3Z0SFVCSY6TioqKYtKkSQCEh4cza9Ys0tLSKpzkHDp0iOjoaDp06ABAkyZN7McCAwMBqF+/PkFBQQBkZWVx9uxZ7rzzTpo1awZAq1atrnqP5cuXU1hYeMXjFoulIo9AZmZmmQQHsG9nZmZetU1OTg55eXkVjkFERETEEfJ93HC3niHfXJ9zWUpyBLC4uZA+5ma8fbwxGqt3pp/FzaVc7aOiospsBwcHc+LEiQrHMXz4cO655x62bt1Kr169SEpKIi4u7ort/f39GTJkCImJifTs2ZMePXowYMAAgoODr3hO48aNKxyniIiIiFzKaDDinp9Nvrk+57PzgXqODqlK1Nk1OdfDYDBgMbngYXKt9s/Vpnddjpub2yWxl5RUvB567969OXjwIKNHj+bo0aN0796dJ5544qrnzJ07l/T0dOLi4li8eDEtWrRg/forT/uLjIzEy8vrip/evXtX6BmCgoI4fvx4mX0Xty+OQF2pjY+Pj0ZxREREpNYyGl3sLwQ9n1Xg4GiqjkZypNwCAwMZPHgwgwcPJj4+nrFjxzJ9+nRMJhNQWr3st6Kjo4mOjmb8+PHExsaycOFCbr755stev6qnq8XGxjJt2jROnDhBgwYNAFi9ejU+Pj5ERETY2yxfvrzMeatXryY2NrZC9xYRERFxJIPBiDm/9IWg57Kdt4x0nU1yUlNTSU1NvewXcrmylJQUYmJiiIyMpKCggE8//dS+xqZBgwZYLBZWrlxJo0aNMJvNZGVlMWfOHPr27UtISAi7du1i9+7dPPzww1e8R0Wnqx06dIisrCwOHTpEcXGxvTJc8+bN8fLyolevXkRERPDQQw/x0ksvkZmZyYQJE0hOTsbd3R2AP/3pT8yaNYtx48bxxz/+kTVr1vD++++zbNmyCsUmIiIi4kguBhfc7SM5zpvk1NnpatdTQlrAZDIxfvx4oqKi6Nq1Ky4uLixatAgAV1dXZs6cyRtvvEFISAj9+vXDw8ODnTt3cs8999CiRQuGDRtGcnIyjz/+eJXFmJKSQnR0NJMmTeL8+fP2UaTNmzcDpe/++fTTT3FxcSE2NpYHH3yQhx9+mKlTp9qvERYWxrJly1i9ejVt27ZlxowZvPnmmyQmJlZZ3CIiIiJVzWAw/m+6Wramq0ktsnbt2kv2LV269JrP//U7eZo0aVJme8KECUyYMOGK5z722GM89thjZfZ9/PHH13zvyjBv3jzmzZt31TaNGze+ZDrabyUkJPDtt99WYmQiIiIijjUsahin858E+KXwgHOqsyM5IiIiIiJ1TWPfJvaRnLxzhRRZnXPphpKcOmbBggVXrFoWGRnp6PBEREREpCoZDLgW5eJSXDpVzVmnrGm6Wh3Tt29fOnfufNljvy07LSIiIiJOxmDAALjnZ5PrGcT57Hx8G3o4OqpKpySnjvH29sbb29vRYYiIiIiIA5kLssj1DOKck74rR9PVRERERETqGHsZaSctPqAkR0RERESkjjAYDACY8537XTl1NslJTU0lIiKCjh07OjoUEREREZHq8UuS4+7k78qps0mOXgYqIiIiInXVxTLS5zSSIyIiIiIitdpvp6tlF5R58buzUJIjlzAYDCxduhSAAwcOYDAY2LZtm0NjEhEREZFK8JvpaoUFxVjzihwZUZVQkiNXFRoayrFjx2jduvXvtq0pCVGTJk0wGAxlPi+88EKZNt9//z3x8fGYzWZCQ0N56aWXLrnOkiVLaNmyJWazmTZt2rB8+fLqegQRERGRqvFLkuNSUojZs/RtMs5YRlpJjlyVi4sLQUFBuLrWrlcqTZ06lWPHjtk/f/nLX+zHcnJy6NWrF40bN2bLli384x//YPLkycyZM8feZt26ddx///08+uijfPvttyQlJZGUlMT27dsd8TgiIiIilcRg/5WXnzvgnGWkleSUh80GhblgvVD9n3LMlUxISGDkyJGMGzcOf39/goKCmDx58nU98m9HZ7Kzsxk0aBCBgYFYLBbCw8OZO3cuAGFhYQBER0djMBhISEgAYO3atXTq1AlPT098fX3p0qULBw8evK54rpW3tzdBQUH2j6enp/3YggULsFqtvP3220RGRnLfffcxcuRIXn75ZXub1157jdtvv52xY8fSqlUrnnnmGdq3b8+sWbOqNG4RERGR6uLl+0uS44TFB2rXj+cdrTAX39RWjrn3U0fB5Pn77X4xf/58xowZw4YNG0hPT2fIkCF06dKFnj17ViiMiRMnkpGRwYoVKwgICGDPnj3k5eUBsHHjRjp16sTnn39OZGQkJpOJoqIikpKSGDp0KO+99x5Wq5WNGzfaa7RfTmRk5FWToPj4eFasWHHVOF944QWeeeYZbrzxRh544AFGjx5tH41KT0+na9eumEwme/vExERefPFFsrOz8fPzIz09nTFjxpS5ZmJion2tkoiIiEht9OuvYF6+pd+FzjlhGWklOU4qKiqKSZMmARAeHs6sWbNIS0urcJJz6NAhoqOj6dChA1C6/uWiwMBAAOrXr09QUBAAWVlZnD17ljvvvJNmzZoB0KrV1RPF5cuXU1hYeMXjFovlquePHDmS9u3b4+/vz7p16xg/fjzHjh2zj9RkZmbaR50uatiwof2Yn58fmZmZ9n2/bpOZmXnVe4uIiIjUaL/Kci4mORrJqevcPDiT/CM+3t4YjdU808/No1zNo6KiymwHBwdz4sSJCocxfPhw7rnnHrZu3UqvXr1ISkoiLi7uiu39/f0ZMmQIiYmJ9OzZkx49ejBgwACCg4OveE7jxo0rFOOvR2CioqIwmUw8/vjjPP/887i7u1fo2iIiIiK12q+THL9fkhwnHMnRmpzyMBhKkw2TZ/V/rjK963Lc3Nx+E7qBkpKSCv8W9O7dm4MHDzJ69GiOHj1K9+7deeKJJ656zty5c0lPTycuLo7FixfTokUL1q9ff8X2kZGReHl5XfHTu3fvcsXcuXNnioqKOHDgAABBQUEcP368TJuL2xdHoK7U5uJxERERkdrOq97FJEcjOU4jNTWV1NRUiouLHR1KrRMYGMjgwYMZPHgw8fHxjB07lunTp9vXuFzu9zQ6Opro6GjGjx9PbGwsCxcu5Oabb77s9Ss6Xe23tm3bhtFopEGDBgDExsby9NNPU1hYaE8GV69ezU033YSfn5+9TVpaGqNGjbJfZ/Xq1cTGxpbr3iIiIiI1yuWmq2UXYCuxYTCW74fqNVmdTXKSk5NJTk4mJyeHevXqOTqcWiMlJYWYmBgiIyMpKCjg008/ta+xadCgARaLhZUrV9KoUSPMZjNZWVnMmTOHvn37EhISwq5du9i9ezcPP/zwFe9Rkelq6enpbNiwgW7duuHt7U16ejqjR4/mwQcftCcwDzzwAFOmTOHRRx/lySefZPv27bz22mu88sor9uv89a9/5dZbb2XGjBn06dOHRYsWsXnz5jJlpkVERERqnV8lOR7ebhgMUFJsI/ecFc96zjOtX9PVpFxMJhPjx48nKiqKrl274uLiwqJFiwBwdXVl5syZvPHGG4SEhNCvXz88PDzYuXMn99xzDy1atGDYsGEkJyfz+OOPV0l87u7uLFq0iFtvvZXIyEimTZvG6NGjyyQn9erV47PPPmP//v3ExMTwt7/9jZSUFIYNG2ZvExcXx8KFC5kzZw5t27blgw8+YOnSpdf0UlQRERGRGutXSY7RCJ6/lJE+52TFB+rsSI4zW7t27SX7ylP62Pard/I0adKkzPaECROYMGHCFc997LHHeOyxx8rs+/jjj6/53hXVvn37q673uSgqKoqvvvrqqm369+9P//79Kys0ERERkRrHy8/M+ewCzmcVQNjvt68tNJIjIiIiIlJHlFl1Y7Ph5f/LC0GdrPiAkpw6ZsGCBVesWhYZGeno8ERERESkKv26Yq/NhrefGaB0JMeJaLpaHdO3b186d+582WO/LTstIiIiIk7mN0mOs47kKMmpY7y9vfH29nZ0GCIiIiLiCL9596LXLyM5zlZ4QNPVRERERETqIJvNhrf/L9PVsp1rupqSHBERERGRuuK3Izm/TFfLzbFSXFjiiIiqhJIcEREREZG64jdrcsyebri4laYE5884z2iOkhwRERERkTrC8JuRHIPBgJef8xUfUJIjIiIiIlIX/fLCd/u6HCcqPlBnk5zU1FQiIiLo2LGjo0OpcQwGA0uXLgXgwIEDGAwGtm3b5tCYRERERKSS/ZLkXBzJOedExQfqbJKTnJxMRkYGmzZtcnQoNVpoaCjHjh2jdevWv9u2piRE06ZNIy4uDg8PD3x9fS/b5tChQ/Tp0wcPDw8aNGjA2LFjKSoqKtNm7dq1tG/fHnd3d5o3b868efMuuU5qaipNmjTBbDbTuXNnNm7cWAVPJCIiIlKJLk5Zu5jkaCRH6hoXFxeCgoJwda09r1SyWq3079+f4cOHX/Z4cXExffr0wWq1sm7dOubPn8+8efNISUmxt9m/fz99+vShW7dubNu2jVGjRvHYY4+xatUqe5vFixczZswYJk2axNatW2nbti2JiYmcOHGiyp9RRERE5HoZXF3hV9/tvP2cr4y0kpxysNls5BXlkVuYW+0f2y+Z9rVISEhg5MiRjBs3Dn9/f4KCgpg8efJ1PfNvR2eys7MZNGgQgYGBWCwWwsPDmTt3LgBhYWEAREdHYzAYSEhIAEpHRDp16oSnpye+vr506dKFgwcPXlc812LKlCmMHj2aNm3aXPb4Z599RkZGBu+++y7t2rWjd+/ePPPMM6SmpmK1WgGYPXs2YWFhzJgxg1atWjFixAjuvfdeXnnlFft1Xn75ZYYOHcojjzxCREQEs2fPxsPDg7fffrvKnk1ERESkolr+8D2ttv+Aa2Ag8L8y0s70QtDa8+P5GiCvKI9ey3o55N4bHtiAh5vHNbefP38+Y8aMYcOGDaSnpzNkyBC6dOlCz549KxTHxIkTycjIYMWKFQQEBLBnzx7y8vIA2LhxI506deLzzz8nMjISk8lEUVERSUlJDB06lPfeew+r1crGjRsvqezxa5GRkVdNguLj41mxYsV1P0N6ejpt2rShYcOG9n2JiYkMHz6cHTt2EB0dTXp6Oj169ChzXmJiIqNGjQJKR4u2bNnC+PHj7ceNRiM9evQgPT39umMTERERqW5eTjiSoyTHSUVFRTFp0iQAwsPDmTVrFmlpaRVOcg4dOkR0dDQdOnQAoEmTJvZjgb/8NKB+/foEBQUBkJWVxdmzZ7nzzjtp1qwZAK1atbrqPZYvX05hYeEVj1ssloo8ApmZmWUSHMC+nZmZedU2OTk55OXlkZ2dTXFx8WXb7Ny5s0LxiYiIiFSni4UHrHlFWPOKMFlqf4pQ+5+gGllcLXzW5zO8vb0xGqt3pp/FtXxf7KOiospsBwcHV8pakeHDh3PPPfewdetWevXqRVJSEnFxcVds7+/vz5AhQ0hMTKRnz5706NGDAQMGEBwcfMVzGjduXOE4RUREROTamMyuuHu4UpBbxLnsfOpbvBwdUoVpTU45GAwGLK4WPNw8qv1zteldl+Pm5nZJ7CUlJRX+PejduzcHDx5k9OjRHD16lO7du/PEE09c9Zy5c+eSnp5OXFwcixcvpkWLFqxfv/6K7SMjI/Hy8rrip3fv3hV6hqCgII4fP15m38XtiyNQV2rj4+ODxWIhICAAFxeXy7a5eA0RERGR2sLZpqxpJEfKLTAwkMGDBzN48GDi4+MZO3Ys06dPx2QyAaXVy34rOjqa6Ohoxo8fT2xsLAsXLuTmm2++7PWrerpabGws06ZN48SJEzRo0ACA1atX4+PjQ0REhL3N8uXLy5y3evVqYmNjATCZTMTExJCWlkZSUhIAJSUlpKWlMWLEiArFJyIiIlLdvP3dOX3kvNOUkVaSI+WSkpJCTEwMkZGRFBQU8Omnn9rX2DRo0ACLxcLKlStp1KgRZrOZrKws5syZQ9++fQkJCWHXrl3s3r2bhx9++Ir3qOh0tUOHDpGVlcWhQ4coLi62V4Zr3rw5Xl5e9OrVi4iICB566CFeeuklMjMzmTBhAsnJybi7l85J/dOf/sSsWbMYN24cf/zjH1mzZg3vv/8+y5Yts99nzJgxDB48mA4dOtCpUydeffVVLly4wCOPPFKh+EVERESqm0ZypE4zmUyMHz+eAwcOYLFYiI+PZ9GiRQC4uroyc+ZMpk6dSkpKCvHx8SxevJidO3cyf/58Tp8+TXBwMMnJyTz++ONVFmNKSgrz58+3b0dHRwPwxRdfkJCQgIuLC59++inDhw8nNjYWT09PBg8ezNSpU+3nhIWFsWzZMkaPHs1rr71Go0aNePPNN0lMTLS3GThwICdPniQlJYXMzEzatWvHypUrLylGICIiIlLTOVsZaYOtPC9gcUI5OTnUq1ePs2fP4uPjU+ZYfn4++/fvJywsDLPZTElJCTk5Ofj4+FR74QGp/S72p0aNGrFmzRruuOOOS9ZOiVyLwsJCli9frj4kFaJ+JJVB/ch5ZHx9lC/e3UmTqAD6/Dnq90+oRL/tR1f7fn6t9E1dRERERKSuu1jjyknGP5Tk1DELFiy4YtWyyMhIR4cnIiIiIlJhWpNTx/Tt25fOnTtf9piGmUVERETqNucYx1GSU+d4e3vj7e3t6DBEREREpAaxv5LRSbIcTVcTEREREanzyvfi+ZquziY5qampRERE0LFjR0eHIiIiIiJSIzhJ3YG6m+QkJyeTkZHBpk2bHB2KiIiIiIhD2aerOcl8tTqb5IiIiIiIiHNSkiMiIiIiUtep8IA4O4PBwNKlSwE4cOAABoOBbdu2OTQmEREREak6TpbjKMmRqwsNDeXYsWO0bt36d9vWhITowIEDPProo4SFhWGxWGjWrBmTJk3CarWWaff9998THx+P2WwmNDSUl1566ZJrLVmyhJYtW2I2m2nTpg3Lly8vc9xms5GSkkJwcDAWi4UePXqwe/fuKn0+EREREfl9SnLkqlxcXAgKCsLVtXa8Umnnzp2UlJTwxhtvsGPHDl555RVmz57NU089ZW+Tk5NDr169aNy4MVu2bOEf//gHkydPZs6cOfY269at4/777+fRRx/l22+/JSkpiaSkJLZv325v89JLLzFz5kxmz57Nhg0b8PT0JDExkfz8/Gp9ZhEREZEKu1h5wEnKqynJKQebzUZJXh4lubnV/rGVo8MlJCQwcuRIxo0bh7+/P0FBQUyePPm6nvm3ozPZ2dkMGjSIwMBALBYL4eHhzJ07F4CwsDAAoqOjMRgMJCQkALB27Vo6deqEp6cnvr6+dOnShYMHD15XPL/n9ttvZ+7cufTq1YumTZvSt29fnnjiCT766CN7mwULFmC1Wnn77beJjIzkvvvuY+TIkbz88sv2Nq+99hq33347Y8eOpVWrVjzzzDO0b9+eWbNmAaV94dVXX2XChAn069ePqKgo3nnnHY4ePWqf6iciIiJS2zhJjkPt+PF8DWHLy+N4t9s47oB737R1CwYPj2tuP3/+fMaMGcOGDRtIT09nyJAhdOnShZ49e1YojokTJ5KRkcGKFSsICAhgz5495OXlAbBx40Y6derE559/TmRkJCaTiaKiIpKSkhg6dCjvvfceVquVjRs3YjBc+YVTkZGRV02C4uPjWbFixTXHfPbsWfz9/e3b6enpdO3aFZPJZN+XmJjIiy++SHZ2Nn5+fqSnpzNmzJgy10lMTLQnMPv37yczM5MePXrYj9erV4/OnTuTnp7Offfdd83xiYiIiEjlUpLjpKKiopg0aRIA4eHhzJo1i7S0tAonOYcOHSI6OpoOHToA0KRJE/uxwMBAAOrXr09QUBAAWVlZnD17ljvvvJNmzZoB0KpVq6veY/ny5RQWFl7xuMViueZ49+zZwz//+U+mT59u35eZmWkfdbqoYcOG9mN+fn5kZmba9/26TWZmpr3dr8+7XBsRERGR2uIqP3+ulZTklIPBYqHhF2vw8fbGaKzemX6Gcnyxh9Ik59eCg4M5ceJEheMYPnw499xzD1u3bqVXr14kJSURFxd3xfb+/v4MGTKExMREevbsSY8ePRgwYADBwcFXPKdx48YVjhPgyJEj3H777fTv35+hQ4dWyjVFREREnJJzLcnRmpzyMBgMGC0WjB4e1f652vSuy3Fzc7sk9pKSkgr/HvTu3ZuDBw8yevRojh49Svfu3XniiSeues7cuXNJT08nLi6OxYsX06JFC9avX3/F9pGRkXh5eV3x07t379+N8+jRo3Tr1o24uLgyBQUAgoKCOH687KTDi9sXR6Cu1ObXx3993uXaiIiIiIhjaCRHyi0wMJDBgwczePBg4uPjGTt2LNOnT7evcSkuLr7knOjoaKKjoxk/fjyxsbEsXLiQm2+++bLXr+h0tSNHjtCtWzdiYmKYO3fuJaNusbGxPP300xQWFtqTwdWrV3PTTTfh5+dnb5OWlsaoUaPs561evZrY2FigtMhCUFAQaWlptGvXDiit2rZhwwaGDx9+1fhEREREahqDk70pR0mOlEtKSgoxMTFERkZSUFDAp59+al9j06BBAywWCytXrqRRo0aYzWaysrKYM2cOffv2JSQkhF27drF7924efvjhK96jItPVjhw5QkJCAo0bN2b69OmcPHnSfuziCMsDDzzAlClTePTRR3nyySfZvn07r732Gq+88oq97V//+lduvfVWZsyYQZ8+fVi0aBGbN2+2jwoZDAZGjRrFs88+S3h4OGFhYUycOJGQkBCSkpKuO34RERERh3KOHEdJjpSPyWRi/PjxHDhwAIvFQnx8PIsWLQLA1dWVmTNnMnXqVFJSUoiPj2fx4sXs3LmT+fPnc/r0aYKDg0lOTubxxx+vkvhWr17Nnj172LNnD40aNSpz7GIZ7nr16vHZZ5+RnJxMTEwMAQEBpKSkMGzYMHvbuLg4Fi5cyIQJE3jqqacIDw9n6dKlZV6KOm7cOC5cuMCwYcM4c+YMt9xyCytXrsRsNlfJs4mIiIhUGScrPGCwlecFLE4oJyeHevXqcfbsWXx8fMocy8/PZ//+/YSFhWE2mykpKSEnJwcfH59qLzwgtd/F/tSoUSPWrFnDHXfcccnaKZFrUVhYyPLly9WHpELUj6QyqB85j92bj/PZmzsICffl7r+1r9Z7/7YfXe37+bXSN3UREREREXEqSnLqmAULFlyxallkZKSjwxMRERERByhvJd+aTmty6pi+ffvSuXPnyx7TMLOIiIhI3eYsK1mU5NQx3t7eeHt7OzoMEREREZEqo+lqIiIiIiJ1nJPNVlOSIyIiIiJS5znXu0CV5IiIiIiISCknWZJTd5Oc1NRUIiIi6Nixo6NDERERERFxKIOTvQ20ziY5ycnJZGRksGnTJkeHIiIiIiLiWPYcxzmGcupskiNXZjAYWLp0KQAHDhzAYDCwbds2h8YkIiIiIlVP09WkTggNDeXYsWO0bt36d9vWlISoSZMmGAyGMp8XXnihTJvvv/+e+Ph4zGYzoaGhvPTSS5dcZ8mSJbRs2RKz2UybNm1Yvnx5meM2m42UlBSCg4OxWCz06NGD3bt3V+mziYiIiMjvU5IjV+Xi4kJQUBCurrXrlUpTp07l2LFj9s9f/vIX+7GcnBx69epF48aN2bJlC//4xz+YPHkyc+bMsbdZt24d999/P48++ijffvstSUlJJCUlsX37dnubl156iZkzZzJ79mw2bNiAp6cniYmJ5OfnV+uzioiIiEhZSnLKwWazUWQtprCg+j/leftsQkICI0eOZNy4cfj7+xMUFMTkyZOv65l/OzqTnZ3NoEGDCAwMxGKxEB4ezty5cwEICwsDIDo6GoPBQEJCAgBr166lU6dOeHp64uvrS5cuXTh48OB1xXOtvL29CQoKsn88PT3txxYsWIDVauXtt98mMjKS++67j5EjR/Lyyy/b27z22mvcfvvtjB07llatWvHMM8/Qvn17Zs2aBZT2hVdffZUJEybQr18/oqKieOeddzh69Kh9qp+IiIhIbXHxPTnOMl2tdv143sGKrCUsTvnOIfce9tqtuLm7XHP7+fPnM2bMGDZs2EB6ejpDhgyhS5cu9OzZs0JxTJw4kYyMDFasWEFAQAB79uwhLy8PgI0bN9KpUyc+//xzIiMjMZlMFBUVkZSUxNChQ3nvvfewWq1s3LgRw1XeOBUZGXnVJCg+Pp4VK1ZcNc4XXniBZ555hhtvvJEHHniA0aNH20ej0tPT6dq1KyaTyd4+MTGRF198kezsbPz8/EhPT2fMmDFlrpmYmGhPYPbv309mZiY9evSwH69Xrx6dO3cmPT2d++6776rxiYiIiNQoTvY2UCU5TioqKopJkyYBEB4ezqxZs0hLS6twknPo0CGio6Pp0KEDULr+5aLAwEAA6tevT1BQEABZWVmcPXuWO++8k2bNmgHQqlWrq95j+fLlFBYWXvG4xWK56vkjR46kffv2+Pv7s27dOsaPH8+xY8fsIzWZmZn2UaeLGjZsaD/m5+dHZmamfd+v22RmZtrb/fq8y7URERERqXWcZChHSU45uJqMDJzaFm9vH4zG6p3p52oq3/2ioqLKbAcHB3PixIkKxzF8+HDuuecetm7dSq9evUhKSiIuLu6K7f39/RkyZAiJiYn07NmTHj16MGDAAIKDg694TuPGjSsU469HYKKiojCZTDz++OM8//zzuLu7V+jaIiIiIs7IucZxtCanXAwGA64mF9zcq/9zteldl+Pm5nZJ7CUlJRX+PejduzcHDx5k9OjRHD16lO7du/PEE09c9Zy5c+eSnp5OXFwcixcvpkWLFqxfv/6K7SMjI/Hy8rrip3fv3uWKuXPnzhQVFXHgwAEAgoKCOH78eJk2F7cvjkBdqc2vj//6vMu1EREREak1nCzL0UiOlFtgYCCDBw9m8ODBxMfHM3bsWKZPn25f41JcXHzJOdHR0URHRzN+/HhiY2NZuHAhN99882WvX9Hpar+1bds2jEYjDRo0ACA2Npann36awsJCezK4evVqbrrpJvz8/Oxt0tLSGDVqlP06q1evJjY2FigtshAUFERaWhrt2rUDSqu2bdiwgeHDh5crPhEREZGawklmqynJkfJJSUkhJiaGyMhICgoK+PTTT+1rbBo0aIDFYmHlypU0atQIs9lMVlYWc+bMoW/fvoSEhLBr1y52797Nww8/fMV7VGS6Wnp6Ohs2bKBbt254e3uTnp7O6NGjefDBB+0JzAMPPMCUKVN49NFHefLJJ9m+fTuvvfYar7zyiv06f/3rX7n11luZMWMGffr0YdGiRWzevNleZtpgMDBq1CieffZZwsPDCQsLY+LEiYSEhJCUlHTd8YuIiIg4gtFowNXNiIurc0z0UpIj5WIymRg/fjwHDhzAYrEQHx/PokWLAHB1dWXmzJlMnTqVlJQU4uPjWbx4MTt37mT+/PmcPn2a4OBgkpOTefzxx6skPnd3dxYtWsTkyZMpKCggLCyM0aNHl1mnU69ePT777DOSk5OJiYkhICCAlJQUhg0bZm8TFxfHwoULmTBhAk899RTh4eEsXbq0zEtRx40bx4ULFxg2bBhnzpzhlltuYeXKlZjN5ip5NhEREZGqcmNkfR7/Z4Kjw6g0Blt5XsDihHJycqhXrx5nz57Fx8enzLH8/Hz2799PWFgYZrOZkpIScnJy8PGp/sIDUvtd7E+NGjVizZo13HHHHZesnRK5FoWFhSxfvlx9SCpE/Ugqg/qRVIbf9qOrfT+/VvqmLiIiIiIiTkVJTh2zYMGCK1Yti4yMdHR4IiIiIiIVpjU5dUzfvn3p3LnzZY9pmFlEREREnIGSnDrG29sbb29vR4chIiIiIlJlNF3tGtTx2gxSSS72o/K+2FVEREREykdJzlW4uLgAYLVaHRyJOIPc3FygtNS2iIiIiFQdfdu6CldXVzw8PDh58qR9vYrVaiU/P18lpOWa2Ww2cnNzOXHiBL6+vvbkWURERESqhpKcqzAYDAQHB7N//34OHjyIzWYjLy8Pi8WiKUdSbr6+vgQFBVFUVOToUEREREScmpKc32EymQgPD8dqtVJYWMiXX35J165dVYlMysXNzU0jOCIiIiLVREnONTAajZjNZlxcXCgqKsJsNivJERERERGpobSwREREREREnIqSHBERERERcSpKckRERERExKnU+TU5F1/QmJOT87ttCwsLyc3NJScnR2ty5LqpH0lFqQ9JZVA/ksqgfiSV4bf96OL38ovf069HnU9yzp07B0BoaKiDIxERERERkYvOnTtHvXr1rutcg60iKZITKCkp4ejRo3h7e//uu29ycnIIDQ3l8OHD+Pj4VFOE4mzUj6Si1IekMqgfSWVQP5LK8Nt+ZLPZOHfuHCEhIRiN17e6ps6P5BiNRho1alSuc3x8fPQ/slSY+pFUlPqQVAb1I6kM6kdSGX7dj653BOciFR4QERERERGnoiRHREREREScipKccnB3d2fSpEm4u7s7OhSpxdSPpKLUh6QyqB9JZVA/kspQFf2ozhceEBERERER56KRHBERERERcSpKckRERERExKkoyREREREREaeiJEdERERERJyKkpzfSE1NpUmTJpjNZjp37szGjRuv6bxFixZhMBhISkqq2gClVihPP5o3bx4Gg6HMx2w2V2O0UhOV9++iM2fOkJycTHBwMO7u7rRo0YLly5dXU7RSU5WnHyUkJFzyd5HBYKBPnz7VGLHUROX9++jVV1/lpptuwmKxEBoayujRo8nPz6+maKWmKk8/KiwsZOrUqTRr1gyz2Uzbtm1ZuXJl+W5oE7tFixbZTCaT7e2337bt2LHDNnToUJuvr6/t+PHjVz1v//79thtuuMEWHx9v69evX/UEKzVWefvR3LlzbT4+PrZjx47ZP5mZmdUctdQk5e1DBQUFtg4dOtjuuOMO29dff23bv3+/be3atbZt27ZVc+RSk5S3H50+fbrM30Pbt2+3ubi42ObOnVu9gUuNUt5+tGDBApu7u7ttwYIFtv3799tWrVplCw4Oto0ePbqaI5eapLz9aNy4cbaQkBDbsmXLbHv37rW9/vrrNrPZbNu6des131NJzq906tTJlpycbN8uLi62hYSE2J5//vkrnlNUVGSLi4uzvfnmm7bBgwcryZFy96O5c+fa6tWrV03RSW1Q3j70r3/9y9a0aVOb1WqtrhClFrief9N+7ZVXXrF5e3vbzp8/X1UhSi1Q3n6UnJxsu+2228rsGzNmjK1Lly5VGqfUbOXtR8HBwbZZs2aV2feHP/zBNmjQoGu+p6ar/cJqtbJlyxZ69Ohh32c0GunRowfp6elXPG/q1Kk0aNCARx99tDrClBruevvR+fPnady4MaGhofTr148dO3ZUR7hSA11PH/rPf/5DbGwsycnJNGzYkNatW/Pcc89RXFxcXWFLDXO9fxf92ltvvcV9992Hp6dnVYUpNdz19KO4uDi2bNlin4q0b98+li9fzh133FEtMUvNcz39qKCg4JKp+xaLha+//vqa76sk5xenTp2iuLiYhg0bltnfsGFDMjMzL3vO119/zVtvvcW///3v6ghRaoHr6Uc33XQTb7/9Np988gnvvvsuJSUlxMXF8fPPP1dHyFLDXE8f2rdvHx988AHFxcUsX76ciRMnMmPGDJ599tnqCFlqoOvpR7+2ceNGtm/fzmOPPVZVIUotcD396IEHHmDq1KnccsstuLm50axZMxISEnjqqaeqI2Spga6nHyUmJvLyyy+ze/duSkpKWL16NR999BHHjh275vsqyblO586d46GHHuLf//43AQEBjg5HarHY2Fgefvhh2rVrx6233spHH31EYGAgb7zxhqNDk1qipKSEBg0aMGfOHGJiYhg4cCBPP/00s2fPdnRoUku99dZbtGnThk6dOjk6FKll1q5dy3PPPcfrr7/O1q1b+eijj1i2bBnPPPOMo0OTWuS1114jPDycli1bYjKZGDFiBI888ghG47WnLq5VGF+tEhAQgIuLC8ePHy+z//jx4wQFBV3Sfu/evRw4cIC77rrLvq+kpAQAV1dXdu3aRbNmzao2aKlxytuPLsfNzY3o6Gj27NlTFSFKDXc9fSg4OBg3NzdcXFzs+1q1akVmZiZWqxWTyVSlMUvNU5G/iy5cuMCiRYuYOnVqVYYotcD19KOJEyfy0EMP2UcB27Rpw4ULFxg2bBhPP/10ub6kinO4nn4UGBjI0qVLyc/P5/Tp04SEhPD3v/+dpk2bXvN91dN+YTKZiImJIS0tzb6vpKSEtLQ0YmNjL2nfsmVLfvjhB7Zt22b/9O3bl27durFt2zZCQ0OrM3ypIcrbjy6nuLiYH374geDg4KoKU2qw6+lDXbp0Yc+ePfYftAD89NNPBAcHK8Gpoyryd9GSJUsoKCjgwQcfrOowpYa7nn6Um5t7SSJz8QcwNput6oKVGqsifx+ZzWZuuOEGioqK+PDDD+nXr9+13/i6SiQ4qUWLFtnc3d1t8+bNs2VkZNiGDRtm8/X1tZfzfeihh2x///vfr3i+qquJzVb+fjRlyhTbqlWrbHv37rVt2bLFdt9999nMZrNtx44djnoEcbDy9qFDhw7ZvL29bSNGjLDt2rXL9umnn9oaNGhge/bZZx31CFIDXO+/abfccott4MCB1R2u1FDl7UeTJk2yeXt729577z3bvn37bJ999pmtWbNmtgEDBjjqEaQGKG8/Wr9+ve3DDz+07d271/bll1/abrvtNltYWJgtOzv7mu+p6Wq/MnDgQE6ePElKSgqZmZm0a9eOlStX2hdKHTp0SMOs8rvK24+ys7MZOnQomZmZ+Pn5ERMTw7p164iIiHDUI4iDlbcPhYaGsmrVKkaPHk1UVBQ33HADf/3rX3nyyScd9QhSA1zPv2m7du3i66+/5rPPPnNEyFIDlbcfTZgwAYPBwIQJEzhy5AiBgYHcddddTJs2zVGPIDVAeftRfn4+EyZMYN++fXh5eXHHHXfwf//3f/j6+l7zPQ02m8YORURERETEeWhYQkREREREnIqSHBERERERcSpKckRERERExKkoyREREREREaeiJEdERERERJyKkhwREREREXEqSnJERERERMSpKMkRERERERGnoiRHRETqpMmTJ9OuXTv79pAhQ0hKSnJYPCIiUnmU5IiIiIiIiFNRkiMiIjWO1Wp1dAgiIlKLKckRERGHS0hIYMSIEYwaNYqAgAASExPZvn07vXv3xsvLi4YNG/LQQw9x6tQp+zklJSW89NJLNG/eHHd3d2688UamTZtmP/7kk0/SokULPDw8aNq0KRMnTqSwsNARjyciItVMSY6IiNQI8+fPx2Qy8c033/DCCy9w2223ER0dzebNm1m5ciXHjx9nwIAB9vbjx4/nhRdeYOLEiWRkZLBw4UIaNmxoP+7t7c28efPIyMjgtdde49///jevvPKKIx5NRESqmcFms9kcHYSIiNRtCQkJ5OTksHXrVgCeffZZvvrqK1atWmVv8/PPPxMaGsquXbsIDg4mMDCQWbNm8dhjj13TPaZPn86iRYvYvHkzUFp4YOnSpWzbtg0oLTxw5swZli5dWqnPJiIi1c/V0QGIiIgAxMTE2H/93Xff8cUXX+Dl5XVJu71793LmzBkKCgro3r37Fa+3ePFiZs6cyd69ezl//jxFRUX4+PhUSewiIlKzKMkREZEawdPT0/7r8+fPc9ddd/Hiiy9e0i44OJh9+/Zd9Vrp6ekMGjSIKVOmkJiYSL169Vi0aBEzZsyo9LhFRKTmUZIjIiI1Tvv27fnwww9p0qQJrq6X/lMVHh6OxWIhLS3tstPV1q1bR+PGjXn66aft+w4ePFilMYuISM2hwgMiIlLjJCcnk5WVxf3338+mTZvYu3cvq1at4pFHHqG4uBiz2cyTTz7JuHHjeOedd9i7dy/r16/nrbfeAkqToEOHDrFo0SL27t3LzJkz+fjjjx38VCIiUl2U5IiISI0TEhLCN998Q3FxMb169aJNmzaMGjUKX19fjMbSf7omTpzI3/72N1JSUmjVqhUDBw7kxIkTAPTt25fRo0czYsQI2rVrx7p165g4caIjH0lERKqRqquJiIiIiIhT0UiOiIiIiIg4FSU5IiIiIiLiVJTkiIiIiIiIU1GSIyIiIiIiTkVJjoiIiIiIOBUlOSIiIiIi4lSU5IiIiIiIiFNRkiMiIiIiIk5FSY6IiIiIiDgVJTkiIiIiIuJUlOSIiIiIiIhT+f/yvCq/QSYDNQAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots(1, 1, figsize=plt.figaspect(1/2))\n", - "fig.suptitle(\n", - " f'Effects of n_list on QPS/recall trade-off ({DATASET_FILENAME})\\n' + \\\n", - " f'k = {k}, pq_dim = {pq_dim}, search = {search_label}')\n", - "labels = []\n", - "for i, n_lists in enumerate(n_list_variants):\n", - " ax.plot(bench_recall_nl[i, :], bench_qps_nl[i, :])\n", - " labels.append(f\"n_lists = {n_lists}\")\n", - "\n", - "ax.legend(labels)\n", - "ax.set_xlabel('recall')\n", - "ax.set_ylabel('QPS')\n", - "ax.set_yscale('log')\n", - "ax.grid()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This chart demonstrates that for the given data set (SIFT-128) and the selected parameters, the QPS/recall curves are rather close to each other.\n", - "Yet, two lines, which correspond to 100- and 5000-cluster indices, lag below the others.\n", - "This suggests that 5000 clusters is probably too many and 100 clusters is probably too few for this dataset. In the range of 500-2000 the algorithm performs very similar though.\n", - "Hence, you shouldn't worry about finding the exact single best value of `n_lists`, but rather make sure it's within a reasonable range.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### kmeans_trainset_fraction\n", - "\n", - "This parameter defines how much of the original data should be fed into training.\n", - "This is useful when in conjunction with `add_data_on_build = True`.\n", - "For example, having a 100M-record dataset, it's reasonable to set `kmeans_trainset_fraction = 0.1` to train the index (i.e. run the k-means clustering) using 10M records only (10% of data), and then add the whole dataset to the index.\n", - "Hence, this parameter directly affects the training speed, but can indirectly affect the search performance (depending on how well the training set represents the full dataset).\n", - "\n", - "Note, if `add_data_on_build = False`, setting the trainset fraction less than one is identical to passing a smaller dataset to the `ivf_pq.build`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### kmeans_n_iters\n", - "\n", - "This parameter is passed directly to the k-means algorithm during training. It's set to a reasonable default of 20, which works for most datasets. However, once in a while you may see a warning complaining that the trained clusters are imbalanced. You can try to fix that by increasing the number of iterations." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Indexing parameters affecting the fine search / product quantization\n", - "\n", - "In the IVF-PQ index, a database vector y is approximated with two level quantization:\n", - "\n", - "$ y = Q_1(y) + Q_2(y - Q_1(y)) $\n", - "\n", - "The first level quantizer ($Q_1$), maps the vector y to the nearest cluster center. The number of\n", - "clusters is `n_lists`.\n", - "\n", - "The second quantizer encodes the residual, and it is defined as a product quantizer\n", - "(see [_\"Product quantization for nearest neighbor search\" by Herve Jegou, Matthijs Douze, Cordelia Schmid_](https://www.researchgate.net/publication/47815472_Product_Quantization_for_Nearest_Neighbor_Search)).\n", - "\n", - "A product quantizer encodes a `dim` dimensional vector with a `pq_dim` dimensional vector.\n", - "First we split the input vector into `pq_dim` subvectors (denoted by u), where each u vector\n", - "contains `pq_len` distinct components of y\n", - "```\n", - "y_1, y_2, ... y_{pq_len}, y_{pq_len+1}, ... y_{2*pq_len}, ... y_{dim-pq_len+1} ... y_{dim}\n", - " \\___________________/ \\____________________________/ \\______________________/\n", - " u_1 u_2 u_{pq_dim}\n", - "```\n", - "Then each subvector encoded with a separate quantizer $q_i$, end the results are concatenated\n", - "\n", - "$ Q_2(y) = q_1(u_1),q_2(u_2),...,q_\\mathtt{pq\\_dim}(u_\\mathtt{pq\\_dim}) $\n", - "\n", - "Each quantizer $q_i$ outputs a code with `pq_bit` bits. The second level quantizers are also defined\n", - "by k-means clustering in the corresponding sub-space: the reproduction values are the centroids,\n", - "and the set of reproduction values is the codebook.\n", - "\n", - "During the search, for every query and probed list, a look-up table (LUT) is constructed using appropriate codebooks and the query coordinates.\n", - "The size of the LUT has profound effect on the performance; here it is one more time:\n", - "\n", - "$ \\mathtt{lut\\_size} = \\mathtt{pq\\_dim} \\cdot \\mathtt{sizeof(lut\\_dtype) \\cdot 2^{\\mathtt{pq\\_bits}}} $\n", - "\n", - "If possible, the LUT is stored fully in GPU L1 (shared) memory during search;\n", - "otherwise, a slower version of the kernel is used, which stores the LUT in the global memory.\n", - "\n", - "\n", - "#### codebook_kind\n", - "\n", - "The second-level quantizers are trained either for each subspace or for each cluster, controlled by parameter `codebook_kind`:\n", - "\n", - " 1. \"subspace\" (C++ api: `codebook_gen::PER_SUBSPACE`): \\\n", - " creates `pq_dim` second-level quantizers - one for each slice of the data along features;\n", - " 2. \"cluster\" (C++ api: `codebook_gen::PER_CLUSTER`): \\\n", - " creates `n_lists` second-level quantizers - one for each first-level cluster.\n", - "\n", - "In either case, the centroids are found using k-means clustering interpreting the data as having `pq_len` dimensions.\n", - "\n", - "There's no definitive way to tell in advance, which of the two options yields better performance for a particular use case.\n", - "A few observations, however, may help:\n", - "\n", - " - A per-cluster codebook tends to take more time to train, since `n_lists` is usually much higher than `pq_dim` - more codebooks to train.\n", - " - Search with a per-cluster codebook usually utilizes L1 cache of the GPU better than with a per-subspace codebook; this may result in a faster search when the LUT is big and occupies a large part of the GPU L1 memory.\n", - " - However, in practice, the recall is slightly higher with a per-subspace codebook.\n", - "\n", - "\n", - "#### pq_dim, pq_bits\n", - "\n", - "`pq_dim` parameter is the main way to control the compression in the database.\n", - "You should choose it depending on your expectations about the sparsity of the information in the data.\n", - "As an experiment, you could start with `pq_dim` in the range of the data dimensionality `[dim / 2, dim]`.\n", - "\n", - "`pq_bits` is the number of bits in a single PQ code.\n", - "Hence, it controls the codebook size - $2^{\\mathtt{pq\\_bits}}$ - the number of possible values a code can take.\n", - "IVF-PQ supports the codebooks sizes from 16 to 256, or the `pq_bits` in the range of `[4, 8]`.\n", - "\n", - "`pq_bits` affects the compression: a database with `pq_bits = 4` is twice smaller than with the `pq_bits = 8`.\n", - "Though much stronger `pq_bits` affects the LUT size, as the LUT size is proportional to $2^{\\mathtt{pq\\_bits}}$ (see the formula above).\n", - "This also means a drastic effect on the recall.\n", - "\n", - "A few observations:\n", - "\n", - " - It's required that `(pq_dim * pq_bits) % 8 == 0`; in general, keeping `pq_dim` in powers of two improves the search performance due to better data alignment.\n", - " - Keeping `pq_dim * pq_bits >= 128` and `(pq_dim * pq_bits) % 32 == 0` maximizes the GPU memory bandwidth utilization.\n", - " - Generally `pq_bits = 8` is a good starting point.\n", - " - The recall loss due to smaller `pq_bits` can be compensated by enabling refinement.\n", - " - For high-dimensional data and large `pq_dims`, lowering `pq_bits` can yield a drastic search speedup due to enabling the faster kernel that keeps the LUT in L1.\n", - " - Alternatively, setting the search parameter `lut_dtype` to `uint8` may be enough to keep the LUT in L1.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "8.25 ms ± 10.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "15.5 ms ± 24.8 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "36.7 ms ± 468 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "71.8 ms ± 222 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "9.4 ms ± 16.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "16.2 ms ± 32.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "38.2 ms ± 520 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "74.4 ms ± 291 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "160 ms ± 48.4 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "168 ms ± 393 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "191 ms ± 139 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "228 ms ± 590 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "12.2 ms ± 24.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "25.2 ms ± 73.1 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "59.8 ms ± 167 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "117 ms ± 84.6 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "14.3 ms ± 19.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "25.2 ms ± 2.93 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "59.6 ms ± 29.3 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "116 ms ± 17.6 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "165 ms ± 757 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "176 ms ± 168 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "212 ms ± 245 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "270 ms ± 283 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "6.47 ms ± 20.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "11 ms ± 13.1 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "24.5 ms ± 285 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "46.2 ms ± 460 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "8.25 ms ± 19.3 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "13.2 ms ± 3.08 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "28.7 ms ± 3.21 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "53.4 ms ± 6.59 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "158 ms ± 135 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "164 ms ± 137 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "180 ms ± 114 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "206 ms ± 322 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "6.29 ms ± 3.05 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "10.7 ms ± 10.8 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "23.8 ms ± 5.83 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "44.6 ms ± 126 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "8.17 ms ± 6.97 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "13 ms ± 35.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", - "28.4 ms ± 11.9 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "52.6 ms ± 69.9 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "159 ms ± 205 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "164 ms ± 121 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "181 ms ± 256 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "207 ms ± 2.57 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "# Let's try a few build configurations.\n", - "# Warning: this will take some time\n", - "\n", - "k = 10\n", - "n_probes_variants = [10, 20, 50, 100]\n", - "n_lists = 1000\n", - "\n", - "build_configs = {\n", - " '64-8-subspace': ivf_pq.IndexParams(n_lists=n_lists, metric=metric, pq_dim=64, pq_bits=8, codebook_kind=\"subspace\"),\n", - " '128-8-subspace': ivf_pq.IndexParams(n_lists=n_lists, metric=metric, pq_dim=128, pq_bits=8, codebook_kind=\"subspace\"),\n", - " '128-6-subspace': ivf_pq.IndexParams(n_lists=n_lists, metric=metric, pq_dim=128, pq_bits=6, codebook_kind=\"subspace\"),\n", - " '128-6-cluster': ivf_pq.IndexParams(n_lists=n_lists, metric=metric, pq_dim=128, pq_bits=6, codebook_kind=\"cluster\"),\n", - "}\n", - "\n", - "bench_qps_ip = np.zeros((len(build_configs), len(search_configs), len(n_probes_variants)), dtype=np.float32)\n", - "bench_recall_ip = np.zeros_like(bench_qps_ip, dtype=np.float32)\n", - "\n", - "for i, index_params in enumerate(build_configs.values()):\n", - " index = ivf_pq.build(index_params, dataset, handle=resources)\n", - " for l, search_fun in enumerate(search_configs):\n", - " for j, n_probes in enumerate(n_probes_variants):\n", - " r = %timeit -o search_fun(n_probes); resources.sync()\n", - " bench_qps_ip[i, l, j] = (queries.shape[0] * r.loops / np.array(r.all_runs)).mean()\n", - " bench_recall_ip[i, l, j] = calc_recall(search_fun(n_probes), gt_neighbors)" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABToAAAhnCAYAAAATE7MkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd1xV9ePH8fdl7yGCoiJuFETAmXuvHFmppaVoZpaalT+1snJkWWmmZmllfdVMKxs23am5cgtqLtw7EZUNMs7vD+LmFVAwja69no+Hj+Lcz/mczzn3cC/3fT/DZBiGIQAAAAAAAACwYjbF3QAAAAAAAAAA+LsIOgEAAAAAAABYPYJOAAAAAAAAAFaPoBMAAAAAAACA1SPoBAAAAAAAAGD1CDoBAAAAAAAAWD2CTgAAAAAAAABWj6ATAAAAAAAAgNUj6AQAAAAAAABg9Qg6AeA/KCkpSY8//rhKly4tk8mkZ599VpL0xx9/qHv37vLx8ZHJZNK0adOKtZ23W0xMjNq1aydPT0+ZTCZ99913+ZY7fvy4TCaT5s6de9uOfSfqBHD7jBs3TiaTyWJbhQoV1K9fv+Jp0E3069dPFSpUKLbjL1u2TOHh4XJycpLJZNKVK1ckSfPnz1f16tVlb28vLy+vm9YzePBgtW3btkjHnjt3rkwmk44fP26xffLkyapUqZJsbW0VHh5epDqR19q1a2UymbR27VrztsLed9bwnpf7O3/x4sVbrsNkMmncuHEW27Zt26ZGjRrJ1dVVJpNJUVFRha4vLi5Orq6uWrJkyS23CQD+6wg6AeAukfvBr6B/mzdvNpedOHGi5s6dq6eeekrz589Xnz59JEnPPfecli9frhdffFHz589Xhw4dbns7J06cWGDAeKdFRkZqz549ev311zV//nzVrVu3WNqBf86+ffs0bty4PIHIf4lhGJo/f76aNWsmLy8vubi4KDQ0VK+99ppSUlLylG/RooXFa0eJEiVUr149/e9//1N2drZF2R9//FHNmzeXn5+fXFxcVKlSJfXs2VPLli3Lty0PPvig7r333jtynsUlJSVF48aNswiD7nZxcXHq2bOnnJ2d9f7772v+/PlydXXVgQMH1K9fP1WuXFmzZ8/WRx99dMN6jh07po8//lijR4/+221asWKFRo0apcaNG2vOnDmaOHGizp49q3HjxhUpaNq6dasGDx6sOnXqyN7ePk/4nevUqVMaP3686tevL29vb5UsWVItWrTQqlWr8i2/Y8cOde7cWaVLl5abm5tq1aqld999V1lZWbdyuviXysjIUI8ePXTp0iVNnTpV8+fPV2Bg4A3/Rjt//rx5fx8fHz3++ON65ZVXivEsAMC62RV3AwAAt9err76qihUr5tlepUoV8/+vXr1a99xzj8aOHWtRZvXq1brvvvs0YsSIO9a+iRMnqnv37urWrdsdO0Z+UlNT9dtvv+mll17S0KFDb1g2MDBQqampsre3/4dahztl3759Gj9+vFq0aFGsvd+KS1ZWlnr37q1FixapadOmGjdunFxcXLR+/XqNHTtWixYt0qpVq+Tn52exX7ly5fTGG29IkmJjY/Xpp59qwIABOnTokN58801J0ttvv62RI0eqefPmevHFF+Xi4qLDhw9r1apV+uKLL/J8UZKRkaGVK1ea671bpKSkaPz48ZJyQuL/gm3btikxMVETJkxQmzZtzNvXrl2r7OxsTZ8+3eI9pyDTp09XxYoV1bJlyyIdv0+fPnr44Yfl6Oho3rZ69WrZ2Njok08+kYODgyRp+/btGj9+vCpUqFDoHp5LlizRxx9/rFq1aqlSpUo6dOhQvuW+//57vfXWW+rWrZsiIyOVmZmpTz/9VG3bttX//vc/9e/f31x2x44datSokapWrarnn39eLi4uWrp0qZ555hkdOXJE06dPL9L5F6fZs2fn+cIDfzly5IhOnDih2bNn6/HHH8/zeH5/o13f8/nJJ5/Uu+++q9WrV6tVq1Z3srkAcFci6ASAu0zHjh1v2lPxwoULCg4Oznd7YYYaWqPY2FhJeT9Q5MdkMsnJyekOt+juYRiG0tLS5OzsXNxN+cckJyfL1dW1uJtxU5MmTdKiRYs0YsQITZ482bz9iSeeUM+ePdWtWzf1799fP//8s8V+np6eevTRR80/Dxo0SEFBQXrvvfc0YcIEmUwmTZgwQW3bttWKFSvyHPfChQt5tq1fv16JiYnq1KlTge21luv6d9wN55j7/F7/elrQ9vxkZGRowYIFevLJJ4t8fFtbW9na2uY5trOzsznkvFVPPfWUnn/+eTk7O2vo0KEFBp0tW7bUyZMnVbJkSfO2J598UuHh4RozZoxF0Pnhhx9KktatW6cSJUpIyvmdat68uebOnWtVQSdfAN7YzX4HCvM3Wo0aNVSzZk3NnTuXoBMAbgFD1wHgPyR3vq1jx47p559/Ng+byh1SZRiG3n//ffP2XFeuXNGzzz6rgIAAOTo6qkqVKnrrrbfy9OrI7ckTGhoqJycn+fr6qkOHDtq+fbuknAAxOTlZ8+bNMx8jd/67xMREPfvss6pQoYIcHR3l5+entm3baufOnTc9r127dqljx47y8PCQm5ubWrdubTFUf9y4cQoMDJQkjRw5UiaT6Ya9+/KbW6xfv35yc3PTmTNn1K1bN7m5ucnX11cjRozIM/TwypUr6tevnzw9PeXl5aXIyEjz/HXXO3DggLp3764SJUrIyclJdevW1Q8//GB+/MKFC/L19VWLFi1kGIZ5++HDh+Xq6qqHHnrohtcmdw6yAwcOqGfPnvLw8JCPj4+eeeYZpaWlWZSdM2eOWrVqJT8/Pzk6Oio4OFizZs3KU2eFChXUuXNnLV++XHXr1pWzs7P5g3xR61i7dq25jtDQUPPw32+//dZ8H9WpU0e7du0q8rWbO3euevToISknlMi9564dYrx06VI1bdpUrq6ucnd3V6dOnfT7779bHCf3uT9y5Ijuvfdeubu765FHHpGUM+/rgw8+qNKlS8vJyUnlypXTww8/rPj4+Bs+L5L01VdfqU6dOnJ2dlbJkiX16KOP6syZM/keuzD33fVSU1M1efJkVatWLd9elF26dFFkZKSWLFmirVu33rAuFxcX3XPPPUpOTlZsbKwuXryohIQENW7cON/y1/cQlaSff/5ZwcHB5t+9G13X7OxsTZs2TSEhIXJyclKpUqU0aNAgXb58OU+9S5cuVfPmzeXu7i4PDw/Vq1dPCxcuND++fv169ejRQ+XLl5ejo6MCAgL03HPPKTU19YbnXBjHjx+Xr6+vJGn8+PHmeyx3zr4bnWNR2vXdd9+pZs2acnJyUs2aNbV48eJ821OU61aQm92XLVq0UGRkpCSpXr165tfxChUqmEcJ+Pr65jt34bU2bNigixcvWvQIzTVjxgyFhITIxcVF3t7eqlu3rsVzev0cnSaTSXPmzFFycrLF+1q9evUkSf3797fYfiOlSpUq1Jc2ISEhFiGnJDk6Ouree+/V6dOnlZiYaN6ekJAgJyenPOGXv79/ob8gSk9P19ixY1WlShXz/TJq1Cilp6eby9xoXsz8no8zZ85owIABKlOmjBwdHVWxYkU99dRTunr1aoHtyG+Oztv5nidJly5d0ogRIxQaGio3Nzd5eHioY8eOio6OtiiX+zfNokWL9Prrr6tcuXJycnJS69atdfjw4QLP4Xq57ffy8pKnp6f69++fZ1qP9PR0Pffcc/L19ZW7u7u6du2q06dP57k2zZs3lyT16NFDJpMp317eiYmJN339btu2rX788UeL930AQOHQoxMA7jLx8fF5JtY3mUzy8fFRjRo1NH/+fD333HMqV66c/u///k+SFBERYZ6rs23bturbt69535SUFDVv3lxnzpzRoEGDVL58eW3atEkvvviizp07Z7Fg0YABAzR37lx17NhRjz/+uDIzM7V+/Xpt3rxZdevW1fz58/X444+rfv36euKJJyRJlStXlpTTE+brr7/W0KFDFRwcrLi4OG3YsEH79+9X7dq1Czzf33//XU2bNpWHh4dGjRole3t7ffjhh2rRooV+/fVXNWjQQA888IC8vLz03HPPqVevXrr33nvl5uZW5GublZWl9u3bq0GDBnr77be1atUqTZkyRZUrV9ZTTz0lKad343333acNGzboySefVI0aNbR48WJzMHB92xs3bqyyZcvqhRdekKurqxYtWqRu3brpm2++0f333y8/Pz/NmjVLPXr00IwZMzRs2DBlZ2erX79+cnd318yZMwvV9p49e6pChQp64403tHnzZr377ru6fPmyPv30U3OZWbNmKSQkRF27dpWdnZ1+/PFHDR48WNnZ2RoyZIhFfQcPHlSvXr00aNAgDRw4UEFBQUWu4/Dhw+rdu7cGDRqkRx99VG+//ba6dOmiDz74QKNHj9bgwYMlSW+88YZ69uypgwcPysbGptDXrlmzZho2bJjeffddjR49WjVq1JAk83/nz5+vyMhItW/fXm+99ZZSUlI0a9YsNWnSRLt27bL4MJ+Zman27durSZMmevvtt+Xi4qKrV6+qffv2Sk9P19NPP63SpUvrzJkz+umnn3TlyhV5enoW+HzMnTtX/fv3V7169fTGG2/ojz/+0PTp07Vx40bt2rXLIhApzH2Xnw0bNujy5ct65plnZGeX/598ffv21Zw5c/Tjjz+qfv36BdYlSUePHpWtra28vLzk5OQkZ2dn/fjjj3r66afNvdRuZMmSJercubPFtvyuq5TT2y33Gg0bNkzHjh3Te++9p127dmnjxo3mXmVz587VY489ppCQEL344ovy8vLSrl27tGzZMvXu3VtSTnCXkpKip556Sj4+Ptq6datmzJih06dP66uvvrppu2/E19dXs2bN0lNPPaX7779fDzzwgCSpVq1aNz3HwrZrxYoVevDBBxUcHKw33nhDcXFx6t+/v8qVK5enPYW9bgUpzH350ksvKSgoSB999JF5GG7lypXVrVs3ffrpp1q8eLFmzZplnoeyIJs2bZLJZFJERITF9tmzZ2vYsGHq3r27+QuZ3bt3a8uWLebn9Hrz58/XRx99pK1bt+rjjz+WJFWtWlWvvvqqxowZoyeeeEJNmzaVJDVq1OiG1+DvOn/+vFxcXMzPs5QTDn/55ZcaNGiQhg8fbh66/u2331r0tC5Idna2unbtqg0bNuiJJ55QjRo1tGfPHk2dOlWHDh26pXmvz549q/r16+vKlSt64oknVL16dZ05c0Zff/21UlJSCt0z9na/50k5rzXfffedevTooYoVK+qPP/7Qhx9+qObNm2vfvn0qU6aMRb1vvvmmbGxsNGLECMXHx2vSpEl65JFHtGXLlkKdQ8+ePVWxYkW98cYb2rlzpz7++GP5+fnprbfeMpd5/PHH9dlnn6l3795q1KiRVq9enad3+qBBg1S2bFlNnDhRw4YNU7169VSqVCmLMi1btlRSUpIcHBzUvn17TZkyRVWrVs3Tpjp16mjq1Kn6/fffVbNmzUKdBwDgTwYA4K4wZ84cQ1K+/xwdHS3KBgYGGp06dcpThyRjyJAhFtsmTJhguLq6GocOHbLY/sILLxi2trbGyZMnDcMwjNWrVxuSjGHDhuWpNzs72/z/rq6uRmRkZJ4ynp6eeY5dGN26dTMcHByMI0eOmLedPXvWcHd3N5o1a2beduzYMUOSMXny5JvWmVt2zpw55m2RkZGGJOPVV1+1KBsREWHUqVPH/PN3331nSDImTZpk3paZmWk0bdo0T52tW7c2QkNDjbS0NPO27Oxso1GjRkbVqlUtjtOrVy/DxcXFOHTokDF58mRDkvHdd9/d9FzGjh1rSDK6du1qsX3w4MGGJCM6Otq8LSUlJc/+7du3NypVqmSxLTAw0JBkLFu2LE/5otaxadMm87bly5cbkgxnZ2fjxIkT5u0ffvihIclYs2aNeVthr91XX32VZ1/DMIzExETDy8vLGDhwoMX28+fPG56enhbbc5/7F154waLsrl27DEnGV199leecb+Tq1auGn5+fUbNmTSM1NdW8/aeffjIkGWPGjMlz7Jvdd/mZNm2aIclYvHhxgWUuXbpkSDIeeOAB87bmzZsb1atXN2JjY43Y2Fhj//79xrBhwwxJRpcuXczlxowZY0gyXF1djY4dOxqvv/66sWPHjnyPc/To0TzPQ0HXdf369YYkY8GCBRbbly1bZrH9ypUrhru7u9GgQQOL62gYlq85+d2Tb7zxhmEymSzus9zflWsFBgbm+3p1rdjYWEOSMXbs2DyPFXSORWlXeHi44e/vb1y5csW8bcWKFYYkIzAw0LytsNetIEW5L3Pfb7Zt22ZRR+41jI2NveGxDMMwHn30UcPHxyfP9vvuu88ICQm54b65xz927Jh5W2RkpOHq6mpRbtu2bXled4tiyJAhee6JG4mJiTGcnJyMPn36WGzPzMw0hg4datjb25vfl21tbY1Zs2YVqt758+cbNjY2xvr16y22f/DBB4YkY+PGjYZh5P/elev6e7Rv376GjY1NnufQMP76/VmzZk2+v7fX3nd34j0vLS3NyMrKsmjTsWPHDEdHR4vXwtz21ahRw0hPTzdvnz59uiHJ2LNnT55zu1bu/frYY49ZbL///vst7s2oqChDkjF48GCLcr17985zXXPbdP37wpdffmn069fPmDdvnrF48WLj5ZdfNlxcXIySJUua/4661qZNmwxJxpdffnnDcwAA5MXQdQC4y7z//vtauXKlxb+lS5fecn1fffWVmjZtKm9vb128eNH8r02bNsrKytK6deskSd98841MJlOeBY4kFbhq7bW8vLy0ZcsWnT17ttBty8rK0ooVK9StWzdVqlTJvN3f31+9e/fWhg0blJCQUOj6CuP6+eSaNm2qo0ePmn9esmSJ7OzsLHra2dra6umnn7bY79KlS1q9erV69uypxMRE83WNi4tT+/btFRMTYzFc9L333pOnp6e6d++uV155RX369NF9991X6HZf35sytz1Lliwxb7t2CGVuz+DmzZvr6NGjeYZiV6xYUe3bt89znKLUERwcrIYNG5p/btCggSSpVatWKl++fJ7tude5qNcuPytXrtSVK1fUq1cvi/va1tZWDRo00Jo1a/Lsc33vydwem8uXL8939fKCbN++XRcuXNDgwYMt5oLt1KmTqlevnme+TOnm911+cofOuru7F1gm97Frh9lKOcNLfX195evrqxo1amjGjBnq1KmT/ve//5nLjB8/XgsXLlRERISWL1+ul156SXXq1FHt2rW1f/9+i/p+/vlneXp6qkmTJnnacP11/eqrr+Tp6am2bdtaPDd16tSRm5ub+blZuXKlEhMT9cILL+SZU/fa15xr78nk5GRdvHhRjRo1kmEY+U6JcCfk1/O2MO06d+6coqKiFBkZadFDuG3btnnmWS7sdSvIrdyXf0dcXJy8vb3zbPfy8tLp06e1bdu223q8Oy0lJUU9evSQs7OzecGuXLa2tqpcubLat2+vefPm6csvv1SXLl309NNPF6o35ldffaUaNWqoevXqFs9t7vyNN3tur5edna3vvvtOXbp0yXe+yMK8Z+e6E+95jo6O5t77WVlZiouLk5ubm4KCgvKdzqZ///4WPVBze+/e7DUyV36vr3Fxcea/H3LfJ4cNG2ZR7tlnny1U/VJOr9E5c+aob9++6tatmyZMmKDly5crLi5Or7/+ep7yub8b14/QAQDcHEPXAeAuU79+/ZtOdF8UMTEx2r17t3keuuvlTrx/5MgRlSlTplBDWPMzadIkRUZGKiAgQHXq1NG9996rvn37WgSY14uNjVVKSop52PS1atSooezsbJ06dUohISG31Kbr5c47ei1vb2+L+e9OnDghf3//PEPjr2/j4cOHZRiGXnnlFb3yyiv5Hu/ChQsqW7asJKlEiRJ699131aNHD5UqVUrvvvtukdp+/dC4ypUry8bGxjzHnSRt3LhRY8eO1W+//ZYnuIuPj7cIWq5fNfZW6rg2zJT+Cg4DAgLy3Z57nYt67fITExMjSQUu9ODh4WHxs52dXZ6hwhUrVtTw4cP1zjvvaMGCBWratKm6du2qRx999IbD1k+cOCEp7z0hSdWrV9eGDRssthXmvstPQSHmtXIfu35OzQoVKmj27NnmhbmqVq2a77ybvXr1Uq9evZSQkKAtW7Zo7ty5Wrhwobp06aK9e/eaA7Off/5Z7dq1yzOEPr/rGhMTo/j4+HyPJ1m+5ki66bDOkydPasyYMfrhhx/yXLPCzKX6d+V3joVtV+69kt/Q1utDn8Jet/j4eIt5QB0cHFSiRIki35e3g5HP/IPPP/+8Vq1apfr166tKlSpq166devfuXeB8sLciKSlJSUlJ5p9tbW0LfI8rjKysLD388MPat2+fli5dmu/Q6unTpysmJsb83tCzZ0+1bNlSQ4YMUefOnWVnZ6fY2FiLuRvd3Nzk5uammJgY7d+//6bvw4UVGxurhISE2zIk+k685+XO9z1z5kwdO3bM4pr4+Pjk2e/695LckLCwc9PeaH8PDw+dOHFCNjY25ql2CjrHomrSpIkaNGigVatW5Xks93ejKKEzACAHQScA4Iays7PVtm1bjRo1Kt/Hq1WrdluO07NnTzVt2lSLFy/WihUrNHnyZL311lv69ttv1bFjx9tyjL/r+lV+/47chZxGjBiRb89ISapSpYrFz8uXL5eU8+Hr9OnThVrZuCDXf3g6cuSIWrdurerVq+udd95RQECAHBwctGTJEk2dOjXPwlP5LaBR1DoKup4Fbc/94Hcr1+56uXXMnz9fpUuXzvP49YHctT2MrjVlyhT169dP33//vVasWKFhw4aZ50HNL9y6Fbd63+X2+Nu9e7e6deuWb5ndu3dLUp4vFFxdXfNdJKYgHh4eatu2rdq2bSt7e3vNmzdPW7ZsUfPmzZWSkqK1a9fmuyhVftc1Oztbfn5+WrBgQb7HKkoglZWVpbZt2+rSpUt6/vnnVb16dbm6uurMmTPq169fnnvyTsjvHO9Euwp73Z555hnNmzfPvL158+YWC3T9U3x8fPINomrUqKGDBw/qp59+0rJly/TNN99o5syZGjNmjMaPH39bjv32229b1BUYGGjxpU9RDRw4UD/99JMWLFiQ75cnM2fOVKtWrfKEgV27dtXw4cN1/PhxValSRfXq1TMHzpI0duxYjRs3TtnZ2QoNDdU777yT7/FzvxwqKBS72cI3/4SivG5PnDhRr7zyih577DFNmDBBJUqUkI2NjZ599tl8fzdu9p5xM393/78jICBABw8ezLM993fj+gWvAAA3R9AJALihypUrKykp6aahR+XKlbV8+XJdunTphr06b9Q7wd/fX4MHD9bgwYN14cIF1a5dW6+//nqBQaevr69cXFzy/ZBw4MAB2djY5OkdeKcFBgbql19+UVJSksWH2uvbmBss2dvbFypQWrZsmT7++GONGjVKCxYsUGRkpLZs2VLgIjPXi4mJseiFefjwYWVnZ5sX3Pnxxx+Vnp6uH374waJ3S1GGRN6OOgqjKNeuoPstt2eOn59fkQK9/ISGhio0NFQvv/yyNm3apMaNG+uDDz7Qa6+9lm/5wMBASTn3xPWhyMGDB82P/12NGzeWl5eXFi5cqJdeeinfD/O5i1Hlrk5/O9StW1fz5s3TuXPnJEmrV69Wenp6ob+wqFy5slatWqXGjRvfcEXq3Odw7969BQbbe/bs0aFDhzRv3jyLRdZWrlxZ2NO5qVvpcVXYduXeC7k9kK91/WtKYa/bqFGj9Oijj5p/zu299k/dl7mqV6+uBQsW5OnpLeUE7Q899JAeeughXb16VQ888IBef/11vfjii3mmKbiRgp6bvn37WkyjUNiVz/MzcuRIzZkzR9OmTVOvXr3yLfPHH3/kGzZmZGRIylmwSpIWLFhg0ds297WucuXKio6OVuvWrW94v+U+l9eveH5teCrlvHd6eHho7969Nzm7m7sT73lff/21WrZsqU8++cRi+5UrV4ol+AsMDFR2draOHDli0Yszv789iuro0aP5fnlz7NgxSX8tngcAKDzm6AQA3FDPnj3122+/mXsTXuvKlSvmD2gPPvigDMPIt8fNtb0iXF1d83wIy8rKyjOE1M/PT2XKlFF6enqBbbO1tVW7du30/fffW/TG+eOPP7Rw4UI1adIkzxDkO+3ee+9VZmamRe+1rKwszZgxw6Kcn5+fWrRooQ8//NAcCF0rNjbW/P9Xrlwxr1Y/ceJEffzxx9q5c6cmTpxY6Ha9//77Fj/ntic3fMoNwa59ruLj4zVnzpxCH+N21FEYRbl2rq6ukvJ+8G/fvr08PDw0ceJEc9hQUB0FSUhIMN//uUJDQ2VjY3PD+7Zu3bry8/PTBx98YFFu6dKl2r9/f56VfG+Vi4uLRo0apYMHD+qll17K8/jPP/+suXPnqkuXLgoNDS1S3SkpKfrtt9/yfSx3TuDcQGDJkiWqW7duntWHC9KzZ09lZWVpwoQJeR7LzMw0P5ft2rWTu7u73njjDaWlpVmUy70H87snDcPQ9OnTC9WWwshdXfv6e+xGCtsuf39/hYeHa968eRavkStXrtS+ffssyhb2ugUHB6tNmzbmf3Xq1JH0z92XuRo2bCjDMLRjxw6L7XFxcRY/Ozg4KDg4WIZh5Pu7eiMF/f5XqlTJ4hrc6rD4yZMn6+2339bo0aP1zDPPFFiuWrVqWrlypcW5ZWVladGiRXJ3dzeH9o0bN7ZoV2442LNnT505c0azZ8/OU3dqaqqSk5Ml5fSsLlmypHnu7FwzZ860+NnGxkbdunXTjz/+qO3bt+epsyg9Ge/Ee56trW2eNnz11Vc3nXv5Ri5evKgDBw4UaT7lXLnvk9dPGTNt2rRC15Hfe8qSJUu0Y8cOdejQIc9jO3bskKen522begcA/kvo0QkAd5mlS5fqwIEDebY3atTohvNdFmTkyJH64Ycf1LlzZ/Xr10916tRRcnKy9uzZo6+//lrHjx9XyZIl1bJlS/Xp00fvvvuuYmJi1KFDB2VnZ2v9+vVq2bKlhg4dKkmqU6eOVq1apXfeeUdlypRRxYoVFRQUpHLlyql79+4KCwuTm5ubVq1apW3btmnKlCk3bN9rr72mlStXqkmTJho8eLDs7Oz04YcfKj09XZMmTSry+f5dXbp0UePGjfXCCy/o+PHjCg4O1rfffpvvXIDvv/++mjRpotDQUA0cOFCVKlXSH3/8od9++02nT59WdHS0pJyhpnFxcVq1apVsbW3VoUMHPf7443rttdd03333KSws7KbtOnbsmLp27aoOHTrot99+02effabevXub923Xrp0cHBzUpUsXDRo0SElJSZo9e7b8/Pzy/VCan9tRR2EV9tqFh4fL1tZWb731luLj4+Xo6KhWrVrJz89Ps2bNUp8+fVS7dm09/PDD8vX11cmTJ/Xzzz+rcePGeu+9927YhtWrV2vo0KHq0aOHqlWrpszMTM2fP1+2trZ68MEHC9zP3t5eb731lvr376/mzZurV69e+uOPPzR9+nRVqFBBzz333G27TqNGjVJUVJTeeust/fbbb3rwwQfl7OysDRs26LPPPlNISIjmzp1b5HpTUlLUqFEj3XPPPerQoYMCAgJ05coVfffdd1q/fr26deumiIgISTkf5vv371/oups3b65BgwbpjTfeUFRUlNq1ayd7e3vFxMToq6++0vTp09W9e3d5eHho6tSpevzxx1WvXj317t1b3t7eio6OVkpKiubNm6fq1aurcuXKGjFihM6cOSMPDw998803hZ67rzCcnZ0VHBysL7/8UtWqVVOJEiVUs2bNG85/WJR2vfHGG+rUqZOaNGmixx57TJcuXdKMGTMUEhJiMc9kYa9bQf7J+1LKmZ/Qx8dHq1atsuhB2q5dO5UuXVqNGzdWqVKltH//fr333nvq1KnTDRfWyk/lypXl5eWlDz74QO7u7nJ1dVWDBg0KnGNYyun9OH/+fEkyh4C5vbMDAwPVp08fSdLixYs1atQoVa1aVTVq1NBnn31mUU/btm3N4f4LL7ygRx99VA0aNNATTzwhZ2dnff7559qxY4dee+012dvb3/A8+vTpo0WLFunJJ5/UmjVr1LhxY2VlZenAgQNatGiRli9fbp6b+/HHH9ebb76pxx9/XHXr1tW6det06NChPHVOnDhRK1asUPPmzfXEE0+oRo0aOnfunL766itt2LCh0FOj3In3vM6dO+vVV19V//791ahRI+3Zs0cLFiy4pb9hcr333nsaP3681qxZoxYtWhRp3/DwcPXq1UszZ85UfHy8GjVqpF9++UWHDx8udB2NGjVSRESE6tatK09PT+3cuVP/+9//FBAQoNGjR+cpv3LlSnXp0oU5OgHgVvyDK7wDAO6gOXPmGJIK/Ddnzhxz2cDAQKNTp0556pBkDBkyJM/2xMRE48UXXzSqVKliODg4GCVLljQaNWpkvP3228bVq1fN5TIzM43Jkycb1atXNxwcHAxfX1+jY8eOxo4dO8xlDhw4YDRr1sxwdnY2JBmRkZFGenq6MXLkSCMsLMxwd3c3XF1djbCwMGPmzJmFOvedO3ca7du3N9zc3AwXFxejZcuWxqZNmyzKHDt2zJBkTJ48+ab15Za99ppFRkYarq6uecqOHTvWuP7tNC4uzujTp4/h4eFheHp6Gn369DF27dqVp07DMIwjR44Yffv2NUqXLm3Y29sbZcuWNTp37mx8/fXXhmEYxvfff29IMqZMmWKxX0JCghEYGGiEhYVZPAcFtW/fvn1G9+7dDXd3d8Pb29sYOnSokZqaalH2hx9+MGrVqmU4OTkZFSpUMN566y3jf//7nyHJOHbsmLlcQffP7agjv3uwoOfuZtcu1+zZs41KlSoZtra2hiRjzZo15sfWrFljtG/f3vD09DScnJyMypUrG/369TO2b99uLlPQc3/06FHjscceMypXrmw4OTkZJUqUMFq2bGmsWrUq32tzvS+//NKIiIgwHB0djRIlShiPPPKIcfr0aYsyRbnvCpKdnW3MnTvXaNy4seHu7m5+TWjTpo2Rnp6ep3zz5s2NkJCQG9aZkZFhzJ492+jWrZsRGBhoODo6Gi4uLkZERIQxefJkc7179+41JBlbt27NU0dB55bro48+MurUqWM4Ozsb7u7uRmhoqDFq1Cjj7NmzFuV++OEHo1GjRoazs7Ph4eFh1K9f3/j888/Nj+/bt89o06aN4ebmZpQsWdIYOHCgER0dnef3Mb9rGhgYaERGRt7wWhiGYWzatMmoU6eO4eDgYEgyxo4de9NzLGy7DMMwvvnmG6NGjRqGo6OjERwcbHz77bdGZGSkERgYeMvXrSCFuS9z32+2bdtmsT33GsbGxhbqWMOGDTOqVKlise3DDz80mjVrZvj4+BiOjo5G5cqVjZEjRxrx8fF5jn/ta0pB1/r77783goODDTs7u3yv7fXWrFlT4Pto8+bN85xrQf+ufZ0xDMNYtmyZ0bx5c6NkyZKGg4ODERoaanzwwQeFuk6GYRhXr1413nrrLSMkJMRwdHQ0vL29jTp16hjjx4+3uDYpKSnGgAEDDE9PT8Pd3d3o2bOnceHCBYv7MteJEyeMvn37Gr6+voajo6NRqVIlY8iQIebf39xrce255Hff3c73PMMwjLS0NOP//u//DH9/f8PZ2dlo3Lix8dtvvxnNmze3eA5y2/fVV19ZHCO/9/Dc5+vacynofs3v/kpNTTWGDRtm+Pj4GK6urkaXLl2MU6dO5bmuBbXppZdeMsLDww1PT0/D3t7eKF++vPHUU08Z58+fN663f/9+Q1Kh30sAAJZMhvEPzLIMAACKxbhx4zR+/HjFxsayqAEk5cwL2KVLF/3yyy/68ccf8x02ebtMmjRJ77zzjs6dO0fPJORx9OhRVa9eXUuXLlXr1q2LuznAv8Kzzz6rdevWaceOHbxuAsAtYI5OAACA/xB7e3t98803Cg8PV48ePbRz5847dqwKFSpo6tSpfFhHvipVqqQBAwbozTffLO6mAP8KcXFx+vjjj/Xaa6/xugkAt4g5OgEAAP5jXF1dtW3btjt+nJ49e97xY8C6XbuIDfBf5+PjYzH3LgCg6OjRCQAAAAAAAMDqMUcnAAAAAAAAAKtHj04AAAAAAAAAVo+gEwAAAAAAAIDVI+gEAAC4DcaNGyeTyaSLFy8Wd1PuGv369VOFChUstplMJo0bN65Y2gMAAIB/N4JOAACAu8zrr7+url27qlSpUjcNBs+cOaOePXvKy8tLHh4euu+++3T06NF/rrH/gIULF2ratGnF3YwCffnll3r00UdVtWpVmUwmtWjRosCy6enpev7551WmTBk5OzurQYMGWrlyZb5lN23apCZNmsjFxUWlS5fWsGHD8l3RuSh1AgAA/JsRdAIAANxlXn75ZW3btk0RERE3LJeUlKSWLVvq119/1ejRozV+/Hjt2rVLzZs3V1xc3D/U2qJJTU3Vyy+/XKR9/u1B56xZs/T9998rICBA3t7eNyzbr18/vfPOO3rkkUc0ffp02dra6t5779WGDRssykVFRal169ZKSUnRO++8o8cff1wfffSRevTocct1AgAA/NvZFXcDAAAAcHsdO3ZMFSpU0MWLF+Xr61tguZkzZyomJkZbt25VvXr1JEkdO3ZUzZo1NWXKFE2cOPGfanKhOTk5FXcTbrv58+erbNmysrGxUc2aNQsst3XrVn3xxReaPHmyRowYIUnq27evatasqVGjRmnTpk3msqNHj5a3t7fWrl0rDw8PSVKFChU0cOBArVixQu3atStynQAAAP929OgEAAC4Q06cOKEqVaqoZs2a+uOPP/6x414/r2VBvv76a9WrV88cckpS9erV1bp1ay1atOiWjt2vXz+5ubnpzJkz6tatm9zc3OTr66sRI0YoKyvrluq81vVD8RMTE/Xss8+qQoUKcnR0lJ+fn9q2baudO3dKklq0aKGff/5ZJ06ckMlkkslksrg+M2bMUEhIiFxcXOTt7a26detq4cKFf7udRREQECAbm5v/Wf7111/L1tZWTzzxhHmbk5OTBgwYoN9++02nTp2SJCUkJGjlypV69NFHzSGnlBNgurm5WTy3ha0TAADAGtCjEwAA4A44cuSIWrVqpRIlSmjlypUqWbJkgWUzMjIUHx9fqHpLlChRqFDsZrKzs7V792499thjeR6rX7++VqxYocTERLm7uxe57qysLLVv314NGjTQ22+/rVWrVmnKlCmqXLmynnrqqb/d9ms9+eST+vrrrzV06FAFBwcrLi5OGzZs0P79+1W7dm299NJLio+P1+nTpzV16lRJkpubmyRp9uzZGjZsmLp3765nnnlGaWlp2r17t7Zs2aLevXvf8LiFXXTK3d1djo6Of+8k/7Rr1y5Vq1bNIryUcp4vKWe4ekBAgPbs2aPMzEzVrVvXopyDg4PCw8O1a9euItcJAABgDQg6AQAAbrMDBw6odevWKlu2rJYvX37TeRc3btyoli1bFqru3GHpf9elS5eUnp4uf3//PI/lbjt79qyCgoKKXHdaWpoeeughvfLKK5JywsjatWvrk08+ue1B588//6yBAwdqypQp5m2jRo0y/3/btm1VtmxZXb58WY8++miefUNCQvTVV18V+bg3mhLgWnPmzFG/fv2KXH9+zp07d9PnK7fctduvL7t+/foi1wkAAGANCDoBAABuo7179+qhhx5SlSpVtHTp0jw95fITFhZW6FWuS5cu/XebKClnUR9J+fY2zJ0HM7fMrXjyySctfm7atKnmz59/y/UVxMvLS1u2bNHZs2dVpkyZIu97+vRpbdu2zWL4fmEU9vkKCQkpUr03kpqaWqjn62bP7bXPa2HrBAAAsAYEnQAAALdRly5dVKpUKS1fvtw8RPpmvL291aZNmzvcMkvOzs6SpPT09DyPpaWlWZQpKicnpzw9Hr29vXX58uVbqu9GJk2apMjISAUEBKhOnTq699571bdvX1WqVOmm+z7//PNatWqV6tevrypVqqhdu3bq3bu3GjdufNN9/+nnS8p5PgrzfN3sub32eS1snQAAANaAxYgAAABuowcffFBHjhzRggULCr3P1atXdf78+UL9ux0L+kg5c306OjqahzlfK3dbUXtI5rK1tf1bbSuKnj176ujRo5oxY4bKlCmjyZMnKyQkREuXLr3pvjVq1NDBgwf1xRdfqEmTJvrmm2/UpEkTjR079qb7Fvb5up09Iv39/Qv1fOUOOy+o7LXPa2HrBAAAsAYEnQAAALfR5MmTNWDAAA0ePLjQq3dv2rRJ/v7+hfp3u1bBtrGxUWhoqLZv357nsS1btqhSpUq3tBBRcfD399fgwYP13Xff6dixY/Lx8dHrr79uftxkMhW4r6urqx566CHNmTNHJ0+eVKdOnfT666+bezTe6JiF+ffll1/etvMMDw/XoUOHlJCQYLF9y5Yt5sclqWbNmrKzs8vz3F69elVRUVHmckWpEwAAwBowdB0AAOA2MplM+uijj5SYmKjIyEi5ubmpa9euN9ynOObolKTu3bvrhRde0Pbt280rdB88eFCrV6/WiBEjbttx7pSsrCwlJSXJ09PTvM3Pz09lypSxGI7t6uqa76r2cXFx8vHxMf/s4OCg4OBgLV26VBkZGeZ5KvNTHHN0du/eXW+//bY++ugj8/OTnp6uOXPmqEGDBubV0T09PdWmTRt99tlneuWVV8yB9fz585WUlKQePXoUuU4AAABrQNAJAABwm9nY2Oizzz5Tt27d1LNnTy1ZskStWrUqsPztnqNz/vz5OnHihFJSUiRJ69at02uvvSZJ6tOnjwIDAyVJgwcP1uzZs9WpUyeNGDFC9vb2euedd1SqVCn93//9n0WdLVq00K+//irDMG5bO/+uxMRElStXTt27d1dYWJjc3Ny0atUqbdu2zWIV9jp16ujLL7/U8OHDVa9ePbm5ualLly5q166dSpcurcaNG6tUqVLav3+/3nvvPXXq1OmmvVlv5/O1bt06rVu3TpIUGxur5ORk8/PVrFkzNWvWTJLUoEED9ejRQy+++KIuXLigKlWqaN68eTp+/Lg++eQTizpff/11NWrUSM2bN9cTTzyh06dPa8qUKWrXrp06dOhgLleUOgEAAP71DAAAAPxtY8eONSQZsbGx5m0pKSlG8+bNDTc3N2Pz5s3/WFuaN29uSMr335o1ayzKnjp1yujevbvh4eFhuLm5GZ07dzZiYmLy1FmnTh2jdOnSNz12ZGSk4erqmmd77vUpisjISCMwMNBimyRj7NixhmEYRnp6ujFy5EgjLCzMcHd3N1xdXY2wsDBj5syZFvskJSUZvXv3Nry8vAxJ5jo//PBDo1mzZoaPj4/h6OhoVK5c2Rg5cqQRHx9fpHb+XbnXJr9/ueeaKzU11RgxYoRRunRpw9HR0ahXr56xbNmyfOtdv3690ahRI8PJycnw9fU1hgwZYiQkJOQpV5Q6AQAA/s1MhvEv+loeAAAA/zqJiYkqUaKEpk2bpiFDhhR3cwAAAIB8sRgRAAAAbmjdunUqW7asBg4cWNxNAQAAAApEj04AAAD8oy5duqSrV68W+Litra18fX3/wRYBAADgbkDQCQAAgH9U7sJGBQkMDNTx48f/uQYBAADgrkDQCQAAgH/Ujh07dPny5QIfd3Z2VuPGjf/BFgEAAOBuQNAJAAAAAAAAwOqxGBEAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAAAAAAAAwOoRdAIAAAAAAACwegSdAAAAAAAAAKweQScAAAAAAAAAq0fQCQAAAAAAAMDqEXQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6hF0AgAAAAAAALB6BJ0AAAAAAAAArB5BJwAAAAAAAACrR9AJAACAu06LFi1Us2bN4m4GAAAA/kEEnQAAAEAhzZo1Sz169FD58uVlMpnUr1+/G5ZftWqVWrVqJU9PT7m7u6tOnTr68ssv8y37f//3fwoODpYkJSUlaezYserQoYNKlCghk8mkuXPnFnic7OxszZo1S+Hh4XJ2dpaPj49atWql6OjoWz1VAAAAq2NX3A0AAAAArMVbb72lxMRE1a9fX+fOnbth2Tlz5mjAgAFq27atJk6cKFtbWx08eFCnTp3Kt/zPP/+sLl26SJIuXryoV199VeXLl1dYWJjWrl17w2M99thjWrBggfr27auhQ4cqOTlZu3bt0oULF27pPAEAAKwRQScAAACsQnJyslxdXYu1Db/++qu5N6ebm1uB5Y4fP64hQ4bo6aef1vTp029a79GjR3Xw4EF98MEHkiR/f3+dO3dOpUuX1vbt21WvXr0C9120aJHmzZunb7/9Vvfff3/RTwoAAOAuwdB1AAAA3FBiYqKeffZZVahQQY6OjvLz81Pbtm21c+dOi3JbtmxRhw4d5OnpKRcXFzVv3lwbN260KHPixAkNHjxYQUFB5iHWPXr00PHjxy3KzZ07VyaTSb/++qsGDx4sPz8/lStXzvz40qVL1bx5c7m7u8vDw0P16tXTwoUL87R93759atmypVxcXFS2bFlNmjQpT5mTJ0/qwIEDhboWgYGBMplMNy33wQcfKCsrS6+++qqknKHohmEUWP7nn3+Wp6enmjRpIklydHRU6dKlC9Wmd955R/Xr19f999+v7OxsJScnF2o/AACAuw1BJwAAAG7oySef1KxZs/Tggw9q5syZGjFihJydnbV//35zmdWrV6tZs2ZKSEjQ2LFjNXHiRF25ckWtWrXS1q1bzeW2bdumTZs26eGHH9a7776rJ598Ur/88otatGihlJSUPMcePHiw9u3bpzFjxuiFF16QlBOCdurUSZcuXdKLL76oN998U+Hh4Vq2bJnFvpcvX1aHDh0UFhamKVOmqHr16nr++ee1dOlSi3J9+/ZVjRo1bucl06pVq1S9enUtWbJE5cqVk7u7u3x8fPTKK68oOzs7T/klS5aobdu2srMr2oCrhIQEbd26VfXq1dPo0aPl6ekpNzc3VapUSYsWLbpdpwMAAGAVGLoOAACAG/r55581cOBATZkyxbxt1KhR5v83DENPPvmkWrZsqaVLl5p7PA4aNEghISF6+eWXtWLFCklSp06d1L17d4v6u3TpooYNG+qbb75Rnz59LB4rUaKEfvnlF9na2kqS4uPjNWzYMNWvX19r166Vk5OTRTuudfbsWX366afmOgcMGKDAwEB98skn6tix49+9LDcUExMjW1tb9e/fX6NGjVJYWJi+/fZbvfbaa8rMzNQbb7xhLpuSkqK1a9dq1qxZRT7OkSNHZBiGvvjiC9nZ2WnSpEny9PTU9OnT9fDDD8vDw0MdOnS4nacGAADwr0WPTgAAANyQl5eXtmzZorNnz+b7eFRUlGJiYtS7d2/FxcXp4sWLunjxopKTk9W6dWutW7fO3IvR2dnZvF9GRobi4uJUpUoVeXl55RkKL0kDBw40h5yStHLlSiUmJuqFF16wCDkl5RlS7ubmpkcffdT8s4ODg+rXr6+jR49alFu7du0Nh5XfiqSkJF2+fFnjx4/Xq6++qgcffFALFixQhw4dNH36dCUmJprLrl69Wunp6bcUviYlJUmS4uLi9P333+upp55S79699csvv8jHx0evvfbabTsnAACAfzuCTgAAANzQpEmTtHfvXgUEBKh+/foaN26cRVgYExMjSYqMjJSvr6/Fv48//ljp6emKj4+XJKWmpmrMmDEKCAiQo6OjSpYsKV9fX125csVc5loVK1a0+PnIkSOSpJo1a9603eXKlcsTfnp7e+vy5ctFuwC3IDfQ7dWrl8X2Xr16KTU1Vbt27TJv+/nnn1W3bl2VKlXqlo9TsWJFNWjQwLzdzc1NXbp00datW5WZmXkrpwAAAGB1GLoOAACAG+rZs6eaNm2qxYsXa8WKFZo8ebLeeustffvtt+rYsaO5t+bkyZMVHh6ebx25K5Q//fTTmjNnjp599lk1bNhQnp6eMplMevjhh/Odu/LaHqBFdW1P0Gvd7t6b+SlTpoxiYmLyhJd+fn6SZBG2LlmyRP3797/l40jKNyT18/NTRkaGkpOT5enpeUv1AwAAWBOCTgAAANyUv7+/Bg8erMGDB+vChQuqXbu2Xn/9dXXs2FGVK1eWJHl4eKhNmzY3rOfrr79WZGSkxXyfaWlpunLlSqHakXusvXv3qkqVKrd2Mv+AOnXqKCYmRmfOnFGlSpXM23OH//v6+krKOY+TJ0+qU6dOt3ScMmXKqHTp0jpz5kyex86ePSsnJye5u7vfUt0AAADWhqHrAAAAKFBWVlaeIeV+fn4qU6aM0tPTJeWEepUrV9bbb79tnjPyWrGxseb/t7W1zdOjcsaMGcrKyipUe9q1ayd3d3e98cYbSktLs3jsVntqnjx5UgcOHLilfQvy0EMPSZI++eQT87bs7GzNmTNHJUqUUJ06dSTl9OYsVaqU6tat+7eOderUKa1cudK87eLFi/r+++/VqlUr2djwJz8AAPhvoEcnAAAACpSYmKhy5cqpe/fuCgsLk5ubm1atWqVt27aZe2Xa2Njo448/VseOHRUSEqL+/furbNmyOnPmjNasWSMPDw/9+OOPkqTOnTtr/vz58vT0VHBwsH777TetWrVKPj4+hWqPh4eHpk6dqscff1z16tVT79695e3trejoaKWkpGjevHlFPse+ffvq119/LVRQ+uOPPyo6OlpSzmJKu3fvNi/407VrV9WqVUuSdN9996l169Z64403dPHiRYWFhem7777Thg0b9OGHH8rR0VFSzvycHTt2zDOXqCS99957unLlirkX6I8//qjTp09LypkCIHc4+osvvqhFixbpwQcf1PDhw+Xp6akPPvhAGRkZmjhxYpGvBwAAgLUi6AQAAECBXFxcNHjwYK1YsULffvutsrOzVaVKFc2cOVNPPfWUuVyLFi3022+/acKECXrvvfeUlJSk0qVLq0GDBho0aJC53PTp02Vra6sFCxYoLS1NjRs31qpVq9S+fftCt2nAgAHy8/PTm2++qQkTJsje3l7Vq1fXc889d1vPPT/ffPONRZi6a9cu88JC5cqVMwedJpNJ3333nV5++WV9+eWXmjt3roKCgvTZZ5/pkUcekSTFx8dr06ZNGjp0aL7Hevvtt3XixAnzz99++62+/fZbSdKjjz5qDjpLlSqlDRs2aMSIEZo6daoyMjLUsGFDffbZZwoLC7v9FwEAAOBfymT8E7OxAwAAALCwaNEiPfLII7p48SKLBQEAANwGTNgDAAAAFAMvLy+9++67hJwAAAC3CT06AQAAAAAAAFg9enQCAAAAAAAAsHoEnQAAAAAAAACsHkEnAAAAAAAAAKtH0AkAAAAAAADA6tkVdwPuZtnZ2Tp79qzc3d1lMpmKuzkAAAAAAACAVTEMQ4mJiSpTpoxsbG7cZ5Og8w46e/asAgICirsZAAAAAAAAgFU7deqUypUrd8MyBJ13kLu7u6ScJ8LDw+MfOWZGRoZWrFihdu3ayd7e/h85JnAncC/jbsL9jLsJ9zPuJtzPuFtwL+Nuwv2M6yUkJCggIMCcs90IQecdlDtc3cPD4x8NOl1cXOTh4cELAqwa9zLuJtzPuJtwP+Nuwv2MuwX3Mu4m3M8oSGGmhWQxIgAAAAAAAABWj6ATAAAAAAAAgNUj6AQAAAAAAABg9ZijEwAAAAAAAJKkrKwsZWRkFNvxMzIyZGdnp7S0NGVlZRVbO/DPsre3l62t7d+uh6ATAAAAAAAASkpK0unTp2UYRrG1wTAMlS5dWqdOnSrU4jO4O5hMJpUrV05ubm5/qx6CTgAAAAAAgP+4rKwsnT59Wi4uLvL19S22kDE7O1tJSUlyc3OTjQ0zLv4XGIah2NhYnT59WlWrVv1bPTsJOgEAAAAAAP7jMjIyZBiGfH195ezsXGztyM7O1tWrV+Xk5ETQ+R/i6+ur48ePKyMj428FndwxAAAAAAAAkCSGi6NY3K77jqATAAAAAAAAgNUj6AQAAAAAAAD+Bfr166du3boVdzOsFkEnAAAAAAAArNaZM2f06KOPysfHR87OzgoNDdX27dvzLfvkk0/KZDJp2rRpN61327Ztat26tby8vOTt7a327dsrOjr6NrcetxNBJwAAAAAAAKzS5cuX1bhxY9nb22vp0qXat2+fpkyZIm9v7zxlFy9erM2bN6tMmTI3rTcpKUkdOnRQ+fLltWXLFm3YsEHu7u5q3769MjIy7sSp4DYg6AQAAAAAAIBVeuuttxQQEKA5c+aofv36qlixotq1a6fKlStblDtz5oyefvppLViwQPb29jet98CBA7p06ZJeffVVBQUFKSQkRGPHjtUff/yhEydOFLhfdHS0WrZsKXd3d3l4eKhOnTrm3qXjxo1TeHi4Rflp06apQoUKeeoZP368fH195eHhoSeffFJXr141P/b1118rNDRUzs7O8vHxUZs2bZScnCzpr6HvN9p/2bJlatKkiby8vOTj46POnTvryJEjFsc/ffq0evXqpRIlSsjV1VV169bVli1bzI9///33ql27tpycnFSpUiWNHz9emZmZN72ud5pdcTcAAAAAAAAA/y6GYSg1I+sfP252drYMwyh0+R9++EHt27dXjx499Ouvv6ps2bIaPHiwBg4caFFnnz59NHLkSIWEhBSq3qCgIPn4+OiTTz7R6NGjlZWVpU8++UQ1atTIN5jM9cgjjygiIkKzZs2Sra2toqKiChWsXuuXX36Rk5OT1q5dq+PHj6t///7y8fHR66+/rnPnzqlXr16aNGmS7r//fiUmJmr9+vUW1+xG+0tScnKyhg8frlq1aikpKUljxozR/fffr6ioKNnY2CgpKUnNmzdX2bJl9cMPP6h06dLauXOnsrOzJUnr169X37599e6776pp06Y6cuSInnjiCUnS2LFji3SutxtBJwAAAAAAACykZmQpeMzyYjn2b8PvkWchyx49elSzZs3S8OHDNXr0aG3btk3Dhg2Tg4ODIiMjJeX0+rSzs9OwYcMK3QZ3d3etXbtW3bp104QJEyRJVatW1fLly2VnV3CcdvLkSY0cOVLVq1c371NUDg4O+t///icXFxeFhITo1Vdf1ciRIzVhwgSdO3dOmZmZeuCBBxQYGChJCg0NLfT+NjY2evDBBy3K/+9//5Ovr6/27dunmjVrauHChYqNjdW2bdtUokQJSVKVKlXM5cePH68XXnjBfH0rVaqkCRMmaNSoUcUedDJ0HQAAAAAAAFYpOztbtWvX1sSJExUREaEnnnhCAwcO1AcffCBJ2rFjh6ZPn665c+fKZDLlW0fHjh3l5uYmNzc3c4/P1NRUDRgwQI0bN9bmzZu1ceNG1axZU506dVJqaqokmfdxc3PTk08+KUkaPny4Hn/8cbVp00ZvvvlmniHhhREWFiYXFxfzzw0bNlRSUpJOnTqlsLAwtW7dWqGhoerRo4dmz56ty5cvF3p/SYqJiVGvXr1UqVIleXh4mHuonjx5UpIUFRWliIgIc8h5vejoaL366qsW5z9w4ECdO3dOKSkpRT7f24kenQAAAAAAALDgbG+rfa+2/8ePm52drYzU5EKX9/f3V3BwsMW2GjVq6JtvvpGUM8z6woULKl++vPnxrKws/d///Z+mTZum48eP6+OPPzaHl7nDzBcuXKjjx4/rt99+k42NjXmbt7e3vv/+ez388MOKiooy1+nh4SEpZx7O3r176+eff9bSpUs1duxYffHFF7r//vtlY2OTZ1h+URc2srW11cqVK7Vp0yatWLFCM2bM0EsvvaQtW7aoYsWKhaqjS5cuCgwM1OzZs1WmTBllZ2erZs2a5nk8nZ2db7h/UlKSxo8frwceeCDPY05OTkU6n9uNoBMAAAAAAAAWTCaTXBz++dgoOztbCWn597zMT+PGjXXw4EGLbYcOHTIP6+7Tp4/atGlj8Xj79u3Vp08f9e/fX5JUtmzZPPWmpKTIxsbGohdo7s+5c1VeO5z7WtWqVVO1atX03HPPqVevXpozZ47uv/9++fr66vz58zIMw1zvtWFprujoaKWmppoDx82bN8vNzU0BAQGScp6bxo0bq3HjxhozZowCAwO1ePFiDR8+/Kb7x8XF6eDBg5o9e7aaNm0qSdqwYYPF8WvVqqWPP/5Yly5dyrdXZ+3atXXw4MECz784MXQdAAAAAAAAVum5557T5s2bNXHiRB0+fFgLFy7URx99pCFDhkiSfHx8VLNmTYt/9vb2Kl26tIKCggqst23btrp8+bKGDBmi/fv36/fff1f//v1lZ2enli1b5rtPamqqhg4dqrVr1+rEiRPauHGjtm3bpho1akiSWrRoodjYWE2aNElHjhzR+++/r6VLl+ap5+rVqxowYID27dunJUuWaOzYsRo6dKhsbGy0ZcsWTZw4Udu3b9fJkyf17bffKjY21nyMm+3v7e0tHx8fffTRRzp8+LBWr15tDkhz9erVS6VLl1a3bt20ceNGHT16VN98841+++03SdKYMWP06aefavz48fr999+1f/9+ffHFF3r55ZeL9uTdAQSdAAAAAAAAsEr16tXT4sWL9fnnn6tmzZqaMGGCpk2bpkceeeRv1Vu9enX9+OOP2r17txo2bKimTZvq7NmzWrZsmfz9/fPdx9bWVnFxcerbt6+qVaumnj17qmPHjho/fryknCH1M2fO1Pvvv6+wsDBt3bpVI0aMyFNP69atVbVqVTVr1kwPPfSQunbtqnHjxknKGSK/bt063XvvvapWrZpefvllTZkyRR07dizU/jY2Nvriiy+0Y8cO1axZU88995wmT55scXwHBwetWLFCfn5+uvfeexUaGqo333xTtra2knJ6xP70009asWKF6tWrp3vuuUdTp04196ItTibj+skBcNskJCTI09NT8fHx5rka7rSMjAwtWbJE9957r3leCaAwriz+Tilbt8o5PFzOEeFyrFJFJpvi+y6Eexl3E+5n3E24n3E34X7G3YJ7GbdDWlqajh07pooVKxbrPIvZ2dlKSEiQh4eHeW5MFF6/fv105coVfffdd8XdlCK50f1XlHyNOToBSJKSVq9W4sqVil+8WJJk4+Ym57AwOUdE5ISfYbVk6+5ezK0EAAAAAADIH0EnAEmS9yOPyKFKZaXuilLq7t3KTkpS8saNSt64MaeAySTHKlX+Cj4jwuVQoYLFxMwAAAAAAADFhaATgCTJ9Z4Gcr2ngSTJyMxUekyMUnbtUmpUlFJ3RSnj1Cmlx8QoPSZGVxYtkiTZennlhJ7h4TkBaGhN2bi4FOdpAAAAAADwnzV37tzibkKxIugEkIfJzk5ONWrIqUYNqXdvSVLmxYtKjYr6M/yMVtqePcq6ckVJa9cqae3anB1tbeUUFHRNr88I2ZctQ6/Pf4Gkq0lyc3Ar7mYAAAAAAHDHEHQCKBS7kiXl3qaN3Nu0kSQZV68q7cABpe7apZQ/e31mnj+vtH37lLZvny4vWCBJsvUtKZfwv4JPp5Bg2Tg6Fuep/OfEp8er6RdNVdGzosL9whXmG6Zw33BV8KwgGxOTewMAAAAA7g4EnQBuicnBQc61asm5Vi2ViIyUJGWcO5cz1D0qSim7opS2b5+yYi8qceVKJa5cmbOfvb2cgoMte32W8ivOU7nrHbh0QIYMHY0/qqPxR/VtzLeSJA8HD4X5huUEn37hCi0ZKhd7ph4AAAAAAFgngk4At429v7/s/f3l0bGjJCk7LU1pv/9u0eszKy5OqdHRSo2ONu9nV8bfstdn9aDiOoW7UgP/Bvr1oV+1O3a3oi5EKTo2Wnsv7lXC1QStP7Ne68+slyTZmGxUzbuaOfgM8w1TObdyTD0AAAAAALAKBJ0A7hgbJye51Kkjlzp15CPJMAxlnD6t1D8XOUrZFaX0gweVefacEs6eU8KSJZIkk5OTHENCVNLdXcnOznKrW1d2JUoU78lYuRJOJdQioIVaBLSQJGVkZ+jQpUOKio1S9IVoRcVG6VzyOR24dEAHLh3Qlwe/lCT5OPko3C9c4b7hCvMLU7BPsBxtmXoAAAAAAPDvQ9AJ4B9jMpnkEBAgh4AAeXbtKknKSkpW2t49f/X6jIpWdny80nbsUAlJ5/5c6Mg+sHxOr8+InF6fjlWqyGRrW3wnY+XsbewVUjJEISVD9EiNRyRJfyT/oejYaHP4ue/SPsWlxemXk7/ol5O/SJLsbOwU7BOcE3z+2fPTz4WpBwAAAAAAxY+gE0CxsnVzles998j1nnskSUZ2tq4eP66k7dsV8+NP8r10SVePHFHGiZOKP3FS8d9/L0mycXWVc1gt83B357Aw2Xp4FOepWL1SrqXUzrWd2lVoJ0lKz0rXvrh9iroQZR7yHpcWp92xu7U7drd5vzKuZXLm+vTLCT6reVeTvY19cZ0GAAAAANzVWrRoofDwcE2bNq24m/KvQ9AJ4F/FZGMjx0qVZBMQoD8cHVXn3ntlk5Ki1N27zUPeU6OilZ2crORNvyl502/mfR2qVJbLNYscOVSoIJMNq4rfKkdbR0X4RSjCL0JSztQDp5NOm0PP6NhoHbp8SGeTz+ps8lktPb5UkuRk66SaJWuah7zX8q0lbyfv4jwVAAAAAHexdevWafLkydqxY4fOnTunxYsXq1u3bpKkjIwMvfzyy1qyZImOHj0qT09PtWnTRm+++abKlCljruPQoUMaOXKkNm7cqKtXr6pWrVqaMGGCWrZsecNjL1++XGPHjtXvv/8uJycnNWvWTFOmTFGFChXu4BmjIASdAP71bD095da0qdyaNpUkGVlZSj98OCf43BWllKhdyjhxUlcPH9HVw0d05auvJUk2np5yDg+TS26vz9BQ2bi6FuepWDWTyaQA9wAFuAeoS+UukqTkjGTtvbg3p9dnbE4Amng1Udv/2K7tf2w371vBo8JfvT59w1XZq7JsTITQAAAAAP6+5ORkhYWF6bHHHtMDDzxg8VhKSop27typV155RWFhYbp8+bKeeeYZde3aVdu3//WZpXPnzqpatapWr14tZ2dnTZs2TZ07d9aRI0dUunTpfI977Ngx3XfffRo+fLgWLFig+Ph4Pffcc3rggQe0c+fOO3rOyB9BJwCrY7K1lVNQkJyCguT98MOSpMzc1dz/DD9T9+xRdny8kn9dp+Rf1+XsaGMjx6AguUSEm3t92pdjVfG/w9XeVQ38G6iBfwNJUraRrePxxxUV+9dw96PxR3U84biOJxzX90dyph5wt3dXqG+oeZGjWiVryc3BrThPBQAAAICV6tixozp27JjvY56enlq5cqXFtvfee0/169fXyZMnVb58eV28eFExMTH65JNPVKtWLUnSm2++qZkzZ2rv3r0FBp07duxQVlaWXnvtNdn8OZpwxIgRuu+++5SRkSF7+/yn9Fq7dq1GjRql33//Xfb29goJCdHChQsVGBiofv366cqVK/ruu+/M5Z999llFRUVp7Z9rWEhSZmamhg4dqvnz58ve3l5PPfWUXn31VfPn25kzZ2rq1Kk6deqUPD091bRpU339dU6noBYtWqhmzZqSVOD+8+fP1/Tp03Xw4EG5urqqVatWmjZtmvz8/lqj4ffff9fzzz+vdevWyTAMhYeHa+7cuapcubIk6eOPP9aUKVN07NgxVahQQcOGDdPgwYPzvSa3C0EngLuCnY+P3Fu1knurVpIkIyNDaQcO/rXCe9QuZZ49p/T9+5W+f78uL/xckmTr4yPniHBzr0+nkBDZODkV56lYNRuTjSp5VVIlr0p6oGrON6nx6fHmoe7RF6K1++JuJWYkatPZTdp0dpMkySSTqnhXUbhvuML9chY6Ku9enhAaAAAAKC6GIWWk/PPHzc7OOfYdFB8fL5PJJC8vL0mSj4+PgoKC9Omnn6p27dpydHTUhx9+KD8/P9WpU6fAeurUqSMbGxvNmTNH/fr1U1JSkubPn682bdoUGHJmZmaqW7duGjhwoD7//HNdvXpVW7duLfJnn3nz5mnAgAHaunWrtm/frieeeELly5fXwIEDtX37dg0bNkzz589Xo0aNdOnSJa1fv77Q+0s5Q/4nTJigoKAgXbhwQcOHD1e/fv20ZMkSSdKZM2fUrFkztWjRQqtXr5aHh4c2btyozMxMSdKCBQs0ZswYvffee4qIiNCuXbs0cOBAubq6KjIyskjnWhQEnQDuSiZ7ezmH1pRzaE2pbx9JUsYff+T09syd63PfPmXFxSlp1S9KWpWzqrjs7eVUo4Zlr88Cvr1D4Xg6eqpZuWZqVq6ZJCkzO1Mxl2PMK7xHXYjSmaQzirkco5jLMfrq0FeSJG9Hb/NQ9zDfMIWUDJGznXNxngoAAADw35GRIk0sc/Nyt5mNJA3ZL8nzjtSflpam559/Xr169ZLHnwvamkwmrVq1St26dZO7u7tsbGzk5+enZcuWydu74PUGKlasqBUrVqhnz54aNGiQsrKy1LBhQ3MYmJ+EhATFx8erc+fO5p6PNWrUKPJ5BAQEaOrUqTKZTAoKCtKePXs0depUDRw4UCdPnpSrq6s6d+4sd3d3BQYGKiIiotD7S9Jjjz1mLlupUiW9++67qlevnpKSkuTm5qb3339fnp6e+uKLL8yhbrVq1cz7jB07VlOmTDFPJVCxYkXt27dPH374IUEnANwO9qVKyb5De3l0aC9Jyk5PV9rv+yx6fWbFXlTa7t1K271bmvepJMmudGnLXp/Vq8vk4FCcp2LV7GzsVMOnhmr41NDD1XOmHriYelHRF/4KPvfF7dPl9Mtae2qt1p5am7OfyU7VS1Q3h5/hfuEq7UoIDQAAAKBwMjIy1LNnTxmGoVmzZpm3G4ahIUOGyM/PT+vXr5ezs7M+/vhjdenSRdu2bZO/v79CQkJ04sQJSVLTpk21dOlSnT9/XgMHDlRkZKR69eqlxMREjRkzRt27d9fKlSt16tQpBQcHm48zevRojR49Wv369VP79u3Vtm1btWnTRj179pS/v3+RzuWee+6x6AXasGFDTZkyRVlZWWrbtq0CAwNVqVIldejQQR06dND9998vFxeXQu1va2urHTt2aNy4cYqOjtbly5eVnZ0tSTp58qSCg4MVFRWlpk2b5ttzNTk5WUeOHNGAAQPMwamU05vV0/POBNi5CDoB/GfZODrKpXaEXGr/tap4xpkzFr0+0w4eVOb580pcukyJS5dJkkyOjnKqWfOvXp/h4bIrWbI4T8XqlXQuqdaBrdU6sLUk6WrWVe2/tN88z2fUhSjFpsZqb9xe7Y3bqwX7F0iS/Fz8LIa71yhRQ/a2+Q8RAQAAAFAE9i7S6LP/+GGzs7Ol1MzbXm9uyHnixAnzUOtcq1ev1k8//aTLly+bt8+cOVMrV67UvHnz9MILL2jJkiXKyMiQJDk754w0y+3VOGnSJHNdn332mQICArRlyxbVrVtXUVFR5sdKlCghSZozZ46GDRumZcuW6csvv9TLL7+slStX6p577pGNjY2M64bu5x63sNzd3bVz506tXbtWK1as0JgxYzRu3Dht27bNPFz/RpKTk9W+fXu1b99eCxYskK+vr06ePKn27dvr6tWrFtcgP0lJSZKk2bNnq0GDBhaP2draFulcioqgEwD+ZDKZ5FCunBzKlZNnl86SpOyUFKXu2fvXcPddu5QVH6/UHTuUumOHeV/7gAA5/xl8ukREyLFqVZnseIm9VQ62DjmrtPuGScoJoc8lnzOHnlGxUTp46aAupFzQihMrtOLECkmSo62jQnxCFOYXZt6/pDMhNAAAAFBkJpPk4PrPHzc7W0pLuK1V5oacMTExWrNmjXx8fCweT0nJmYs0d0GhXDY2NuaejIGBgXnqTUlJybNPbpCXnZ0tOzs7ValSJd82RUREKCIiQi+++KIaNmyohQsX6p577pGvr6/27t1rUTYqKipPz8ktW7ZY/Lx582ZVrVrVfHw7Ozu1adNGbdq00dixY+Xl5aXVq1ebh5LfaP8DBw4oLi5Ob775pgICAiTJYoV6SapVq5bmzZuX76JLpUqVUpkyZXT06FE98sgj+Z7/ncKncAC4ARsXF7k2qC/XBvUl5QRuV48fz+n1+WfwmX74sDJOnVLGqVNK+OFHSZLJxUXOtWrJOTxMLhERcg4Lk20hvjlD/kwmk8q4lVEZtzLqWDFnNcWUjBT9Hve7OfyMjo3WlfQr2nlhp3Ze2GneN8A9wDzPZ7hfuKp4VZGtzZ39FhEAAADAPycpKUmHDx82/3zs2DFFRUWpRIkS8vf3V/fu3bVz50799NNPysrK0vnz5yXl9LB0cHBQw4YN5e3trcjISI0ZM0bOzs6aPXu2jh07pk6dOhV43E6dOmnq1Kl69dVXzUPXR48ene+cmNe27aOPPlLXrl1VpkwZHTx4UDExMerbt68kqVWrVpo8ebI+/fRTNWzYUJ999pn27t2bp76TJ09q+PDhGjRokHbu3KkZM2ZoypQpkqSffvpJR48eVbNmzeTt7a0lS5YoOztbQUFBhdq/fPnycnBw0IwZM/Tkk09q7969mjBhgsXxhw4dqhkzZujhhx/Wiy++KE9PT23evFn169dXUFCQxo8fr2HDhsnT01MdOnRQenq6tm/frsuXL2v48OGFfWqLjKATAIrAZDLJsWJFOVasKK8H7pckZSUmKjV691+9PqOjlZ2UpJTNm5WyebPi/tzXoVIli16fDpUqyXTdt38oPBd7F9UrXU/1SteTlBNCn0g4YZ7nMzo2WkeuHNGpxFM6lXhKPx7NCaFd7FwU6htqHvJey7eWPBw8bnQoAAAAAP9i27dvV8uWLc0/5wZpkZGRGjdunH744QdJUnh4uMV+a9asUYsWLVSyZEktW7ZML730klq1aqWMjAyFhITo+++/V1hYWIHHbdWqlRYuXKhJkyZp0qRJcnFxUcOGDbVs2bICh3a7uLjowIEDmjdvnuLi4uTv768hQ4Zo0KBBkqT27dvrlVde0ahRo5SWlqbHHntMffv21Z49eyzq6du3r1JTU1W/fn3Z2trqmWee0RNPPCFJ8vLy0rfffqtx48YpLS1NVatW1eeff66QkJBC7e/r66u5c+dq9OjRevfdd1W7dm29/fbb6tq1q3l/Hx8frV69WiNHjlTz5s1la2ur8PBwNW7cWJL0+OOPy8XFRZMnT9bIkSPl6uqq0NBQPfvsswVez9vBZFw/8B+3TUJCgjw9PRUfH28x98OdlJGRoSVLlujee+/Nd0JYwFpY871sZGUp/cgRi7k+rx4/nqecjYeHnMPCzAsdOdWqJVs3t3++wXexhKsJ2hO7x9zrc/fF3UrOSM5TrrJnZfM8n2F+YaroUdFiYu6/y5rvZ+B63M+4m3A/427BvYzbIS0tTceOHVPFihXl5ORUbO3Izs5WQkKCPDw88gwLx+3RokULhYeHa9q0acXdFLMb3X9Fydfo0QkAt5nJ1lZO1arJqVo1eT/UU5KUefnyn0Pd/xzyvmePshMSlLx+vZLXr8/Z0cZGjlWrWvT6tC9f/rYGbv81Hg4ealy2sRqXzflWMSs7S0fij1gscnQy8aSOxB/Rkfgj+ibmG0mSp6OneY7PcN9w1SxZUy72Ljc6FAAAAACgmBF0AsA/wM7bW+4tW8r9z+EURmam0g4etJjrM+PMGaUfPKj0gwd15YsvJUm2JUqYV3Z3iQiXU82asrnB6na4MVsbW1XzrqZq3tXUMygnhI5LjdPu2N2Kis0JP/de3Kv49HitO71O606vy9nPlLNf7jyf4X7hKuNahhAaAAAAAP5FCDoBoBiY7OzkHBIi55AQ6dGcVegyLlyw6PWZtnevsi5dUtLq1UpavTpnRzs7OVWvLueICPNCR3b+/gRuf4OPs49alm+pluVzQuiMrAwdvHzwr16fsVE6n3xe+y/t1/5L+/XFwS8kSSWdS5rn+QzzDVMNnxpytHUszlMBAAAAgJtau3ZtcTfhjiHoBIB/CXs/P9m3ayePdu0kSdlXryp93z6lXNPrM/PCBaXt3au0vXt1ef58SZKdn9+fwWdOr0/H4GDZODgU56lYNXtbe9UsWVM1S9bUo3pUknQ++bzF6u77L+3XxdSLWnVylVadXJWzn429gn2Cc1Z498sZ8u7r4lucpwIAAAAA/ykEnQDwL2Xj4GAeti7lrCqeefasUq7t9bl/vzIvXFDi8uVKXL5ckmRycJBTSIi516dzeLjs/fyK8UysX2nX0irtWlrtK7SXJKVlpmlf3D6LFd4vpV1SdGy0omOjpX05+5V1K6tavrUUWiJUSZlJyszOlL1YIAAAAAAA7gSCTgCwEiaTSfZly8qzbFl5duokScpOTVXa3r05vT7/XOE96/LlnP/ftcu8r33ZsuZen84R4XIKCpLJjreAW+Vk56TapWqrdqnaknJC6NOJp83zfEZdiFLMlRidSTqjM0lntPTYUknS/776n0J9Q81D3muVrCUvJ69iPBMAAAAAuHvwKRcArJiNs7Nc6tWTS716knICt4wTJyx6faYfOqSMM2eUceaMEn76SZJkcnaWc2ioOfh0Dg+Xnbd3cZ6KVTOZTArwCFCAR4C6VO4iSUrOSNaei3sUdSFKu/7YpZ3ndiotK03bzm/TtvPbzPtW8Khgnucz3DdclbwqycZkU1ynAgAAAABWi6ATAO4iJpNJDhUqyKFCBXl16yZJykpKUtru3UrZtUupUdFKjYpSdmKiUrZuVcrWreZ9HSpUsOj16Vilikw2BG63ytXeVff436N7/O9RRkaGfvr5JwU3CdbeS3vNPT+PxR/T8YTjOp5wXN8d/k6S5G7vrlp+tczBZ2jJULk5uBXvyQAAAACAFSDoBIC7nK2bm1wbNZJro0aSJCM7W1ePHs0JPv/s9Xn16FFdPX5cV48fV/zixZIkGzc3OYeF/RV+htWSrbt7cZ6KVbMx2aiSZyUFlQzSg9UelCRdSbui3Rd3m+f53HNxjxIzErXxzEZtPLPRvF8VryoWK7wHuAfIZDIV5+kAAAAAwL8OQScA/MeYbGzkWKWKHKtUkXePHpKkrCtXlBod/Vevz927lZ2UpOSNG5W8ceOfO5rkWKWKRa9PhwoVCNz+Bi8nLzUr10zNyjWTJGVmZ+rQ5UMWK7yfSTqjQ5cP6dDlQ1p0aJEkqYRTiZwen38GnyE+IXKycyrOUwEAAADwD2nRooXCw8M1bdq04m7Kvw5BJwBAtl5ecmveXG7Nm0uSjMxMpR86ZDHXZ8apU0qPiVF6TIyuLFpk3i93ZXjniAg5h9aUjYtLcZ6KVbOzsVOwT7CCfYLVq3ovSVJsSqw5+IyKjdK+uH26lHZJa06t0ZpTa3L2M9mphk8NhfmGKcwvZ8h7adfSxXkqAAAAwD9m3bp1mjx5snbs2KFz585p8eLF6vbnVF4ZGRl6+eWXtWTJEh09elSenp5q06aN3nzzTZUpU8Zcx6FDhzRy5Eht3LhRV69eVa1atTRhwgS1bNnyhsc2DENTpkzRRx99pBMnTqhkyZIaPHiwXnrppTt5yigAQScAIA+TnZ2cgoPlFBws9e4tScq8eFGpUVHmXp9pe/Yo68oVJa1dq6S1a3N2tLWVU1DQNb0+I2Rftgy9Pv8GXxdftQlsozaBbSRJV7Oual/cPovw82LqRe25uEd7Lu7RZ/s/kySVdi1tnucz3C9cQSWCZG9jX5ynAgAAANwRycnJCgsL02OPPaYHHnjA4rGUlBTt3LlTr7zyisLCwnT58mU988wz6tq1q7Zv324u17lzZ1WtWlWrV6+Ws7Ozpk2bps6dO+vIkSMqXbrgTgTPPPOMVqxYobfffluhoaG6dOmSLl26dMfOFTdG0AkAKBS7kiXl3qaN3NvkBG7G1atK27//z/AzSqm7dinzjz+Utm+f0vbt0+UFCyRJtr4l5RL+V/DpFBIsG0fH4jwVq+Zg66Bwv5zwMjIkUoZh6GzyWfNQ96gLUTp0+ZDOJ5/X+eTzWn58uSTJ0dZRIT4h5uHuYb5h8nH2KeazAQAAAP6+jh07qmPHjvk+5unpqZUrV1pse++991S/fn2dPHlS5cuX18WLFxUTE6NPPvlEtWrVkiS9+eabmjlzpvbu3Vtg0Ll//37NmjVLe/fuVVBQkCSpYsWKN23v2rVrNWrUKP3++++yt7dXSEiIFi5cqMDAQPXr109XrlzRd999Zy7/7LPPKioqSmtzO5hIyszM1NChQzV//nzZ29vrqaee0quvvmruZDJz5kxNnTpVp06dkqenp5o2baqvv/5aUs7Q95o1a0pSgfvPnz9f06dP18GDB+Xq6qpWrVpp2rRp8vPzM7fh999/1/PPP69169bJMAyFh4dr7ty5qly5siTp448/1pQpU3Ts2DFVqFBBw4YN0+DBg296ff4Ogk4AwC0xOTjkLFYUFqYSkZGSpIxz5yx7fe7bp6zYi0pcuVKJf/5xYbK3l1NwsGWvz1J+NzoUbsBkMqmsW1mVdSurTpU6SZJSMlL0e9zv5h6f0bHRik+P184LO7Xzwk7zvuXdy1sEn1W8qsjWxra4TgUAAAD/IoZhKDUz9R8/bnZ2tgzDuKPHiI+Pl8lkkpeXlyTJx8dHQUFB+vTTT1W7dm05Ojrqww8/lJ+fn+rUqVNgPT/++KMqVaqkn376SR06dJBhGGrTpo0mTZqkEiVK5LtPZmamunXrpoEDB+rzzz/X1atXtXXr1iKPgps3b54GDBigrVu3avv27XriiSdUvnx5DRw4UNu3b9ewYcM0f/58NWrUSJcuXdL69esLvb+UM+R/woQJCgoK0oULFzR8+HD169dPS5YskSSdOXNGzZo1U4sWLbR69Wp5eHho48aNyszMlCQtWLBAY8aM0XvvvaeIiAjt2rVLAwcOlKurqyL//Px4JxB0AgBuG3t/f9n7+8vjz29Ts9PSlPb770rdtcs832dWXJxSo6OVGh1t3s+ujL9lr8/qQTLZM8z6VrnYu6he6XqqV7qepJw/Uo8nHDf3+oyOjdbhK4d1MvGkTiae1A9HfpAkudq7KrRkaE6PUd9whfqGysPBozhPBQAAAMUkNTNVDRY2KJZjr+i0Qp7yvCN1p6Wl6fnnn1evXr3k4ZHzt67JZNKqVavUrVs3ubu7y8bGRn5+flq2bJm8vb0LrOvo0aM6ceKEvvrqK3366afKysrSc889p+7du2v16tX57pOQkKD4+Hh17tzZ3POxRo0aRT6PgIAATZ06VSaTSUFBQdqzZ4+mTp2qgQMH6uTJk3J1dVXnzp3l7u6uwMBARUREFHp/SXrsscfMZStVqqR3331X9erVU1JSktzc3PT+++/L09NTX3zxhez//OxWrVo18z5jx47VlClTzFMJVKxYUfv27dOHH35I0AkAsE42Tk5yqVNHLnXqyEc5gVvGqVMWvT7TDx5U5tlzSjh7Tgl/fjtocnKSc82aOb0+I3IWO7Ir4BtR3JzJZFJFz4qq6FlR91e9X5IUnx6vPRf3mIe7747dreSMZG0+t1mbz23O2U8mVfaqbLHCewWPCsy5CgAAAKuUkZGhnj17yjAMzZo1y7zdMAwNGTJEfn5+Wr9+vZydnfXxxx+rS5cu2rZtm/z9/RUSEqITJ05Ikpo2baqlS5cqOztb6enp+vTTT80h3yeffKI6dero4MGDcnZ2VnBwsPk4o0eP1ujRo9WvXz+1b99ebdu2VZs2bdSzZ0/5+/sX6Vzuuecei7/LGzZsqClTpigrK0tt27ZVYGCgKlWqpA4dOqhDhw66//775XLNwrE32t/W1lY7duzQuHHjFB0drcuXLys7O1uSdPLkSQUHBysqKkpNmzY1h5zXSk5O1pEjRzRgwABzcCrl9Gb19LwzAXYugk4AwD/GZDLJoXx5OZQvL8+uXSVJWUnJStu7J6fX565dSo3erez4eKVs366UayYHtw8sn9PrMyKn16djlSoy2TLM+lZ5OnqqSdkmalK2iSQpKztLh68ctljk6FTiKR2+cliHrxzWNzHfSJK8HL3MQ93D/cIV4hMiF3uXGx0KAAAAVsjZzllbem/5x4+bnZ2tjJSM215vbsh54sQJ81DrXKtXr9ZPP/2ky5cvm7fPnDlTK1eu1Lx58/TCCy9oyZIlysjIaZezs7Mkyd/fX3Z2dhY9GXN7Z548eVItW7ZUVFSU+bHc4exz5szRsGHDtGzZMn355Zd6+eWXtXLlSt1zzz2ysbHJM3Q/97iF5e7urp07d2rt2rVasWKFxowZo3Hjxmnbtm3m4fo3kpycrPbt26t9+/ZasGCBfH19dfLkSbVv315Xr161uAb5SUpKkiTNnj1bDRpY9gq2vcOf4Qg6AQDFytbNVa733CPXe+6RJBnZ2bp67Ng1vT6jdPXwEWWcOKn4EycV//33kiQbV1c5h9UyD3d3DguTrQfDrG+VrY2tgkoEKahEkHoG9ZQkxaXG5QSfsVGKvhCt3+N+15X0K/r19K/69fSvOfuZcva7doV3f1d/en0CAABYOZPJVCxfaGdnZyvBlHBb68wNOWNiYrRmzRr5+FguypmSkiJJsrGxsdhuY2Nj7skYGBiYp97GjRsrMzNTR44cMQ9DP3TokLm8nZ2dqlSpkm+bIiIiFBERoRdffFENGzbUwoULdc8998jX11d79+61KBsVFZWn5+SWLZYh9ObNm1W1alVzkGhnZ6c2bdqoTZs2Gjt2rLy8vLR69WrzUPIb7X/gwAHFxcXpzTffVEBAgCRZrFAvSbVq1dK8efOUkZGRp22lSpVSmTJldPToUT3yyCP5nv+dQtAJAPhXMdnYyLFyZTlWriyvBx+UJGXFxyt1925zr8+06N3KTk5W8qbflLzpN/O+DlUqy+WaRY4cKlSQ6bo/VlB4Ps4+alW+lVqVbyVJysjK0IFLB8wLHO26sEsXUi5oX9w+7Yvbp88PfC5J8nX2tVjkKNgnWA62DsV5KgAAALiLJSUl6fDhw+afjx07pqioKJUoUUL+/v7q3r27du7cqZ9++klZWVk6f/68pJwelg4ODmrYsKG8vb0VGRmpMWPGyNnZWbNnz9axY8fUqVOnAo/bpk0b1a5dW4899pimTZum7OxsDRkyRG3btrXo5XmtY8eO6aOPPlLXrl1VpkwZHTx4UDExMerbt68kqVWrVpo8ebI+/fRTNWzYUJ999pn27t2bZ47NkydPavjw4Ro0aJB27typGTNmaMqUKZKkn376SUePHlWzZs3k7e2tJUuWKDs727wy/M32L1++vBwcHDRjxgw9+eST2rt3ryZMmGBx/KFDh2rGjBl6+OGH9eKLL8rT01ObN29W/fr1FRQUpPHjx2vYsGHy9PRUhw4dlJ6eru3bt+vy5csaPnx4YZ/aIiPoBAD869l6esqtaVO5NW0qSTKyspQeE6PUqCjzQkcZJ07q6uEjunr4iK589bUkycbTU87hYXLJ7fUZGiobV9fiPBWrZm9rr1DfUIX6hqqP+kiSziefN/f4jLoQpQOXDig2NVYrT6zUyhMrc/azsVeIT4g5/Az3C1dJ55LFeSoAAAC4i2zfvl0tW7Y0/5wbpEVGRmrcuHH64YecxTfDw8Mt9luzZo1atGihkiVLatmyZXrppZfUqlUrZWRkKCQkRN9//73CwsIKPK6NjY1+/PFHPf3002rWrJlcXV3VsWNHc2CYHxcXFx04cEDz5s1TXFyc/P39NWTIEA0aNEiS1L59e73yyisaNWqU0tLS9Nhjj6lv377as2ePRT19+/ZVamqq6tevL1tbWz3zzDN64oknJEleXl769ttvNW7cOKWlpalq1ar6/PPPFRISUqj9fX19NXfuXI0ePVrvvvuuateurbfffltd/5x+TMpZqX716tUaOXKkmjdvLltbW4WHh6tx48aSpMcff1wuLi6aPHmyRo4cKVdXV4WGhurZZ58t8NrcDibj+oH/uG0SEhLk6emp+Ph4i7kf7qSMjAwtWbJE9957b74TwgLWgnsZRZWZu5r7rl1K3RWl1D17ZKSnWxaysZFjUJBc/lzgyDkiQvblyt3xYdb/pfs5NTNV++L2mef53B27W5fSLuUpV9at7F/Bp2+4qnpXlZ0N379ag//S/Yy7H/cz7hbcy7gd0tLSdOzYMVWsWFFOTk7F1o7s7GwlJCTIw8Mjz1By3B4tWrRQeHi4pk2bVtxNMbvR/VeUfI1PFACAu4Kdj4/cW7WSe6ucYdbG1atKO3gwJ/SMyun1mXn2nNL371f6/v26vDBnmLWtj4+cI8LNvT6dQkJkU4x/2Fk7Zztn1SlVR3VK1ZGUs4LlqcRTf/X6jI1SzOUYnUk6ozNJZ/Tz0Z/N+4WWDLVY4d3T8c6uyAgAAADg7kLQCQC4K5kcHOQcGirn0FCpb84w64w//sgJPnftUkrULqXt26+suDglrfpFSat+ydnR3l5ONWpY9vosXboYz8S6mUwmlfcor/Ie5dW1cs5Ql6SrSdp9cbeiY6MVfSFa0bHRSspI0tbzW7X1/FbzvhU9Kyrc96/h7hU9K8rGxLf6AAAAAPJH0AkA+M+wL1VK9h3ay6NDe0lSdnq60n7//a9en7uilHXxotJ271ba7t3SvE8lSXalS1v2+qxeXSYHFte5VW4ObmpUppEalWkkSco2snX0ylFFxUYp6kLOQkfHE47rWPwxHYs/psWHF0uS3B3czQschfuFK7RkqFztmXMVAAAAKIq1a9cWdxPuGIJO3LLoU1f08nd71TLIVy2q+ymsnJdsbe7sPHcAcDvZODrKpXZtudSuLSlnmHXGmTPmXp+pUVFKO3hQmefPK3HpMiUuXSZJMjk6yqlmzb96fYaHy64ki+vcKhuTjap4V1EV7yrqXq27JOly2mXtjt1tXuF978W9SryaqA1nNmjDmQ3m/ap6VbVY5Kic252fcxUAAADAvxNBJ27ZmoMXtOdMvPacide7qw/L28Vezav5qkWQn5pV81UJV3o7AbAuJpNJDuXKyaFcOXl26SxJyk5JUeqevX8ucpQTfmbFxyt1xw6l7thh3tc+IEDOfwafLhERcqxaVSY73mZvlbeTt5oHNFfzgOaSpIzsDB26fMjc4zP6QrTOJp/VwcsHdfDyQX158EtJUgmnEgr3DTeHn8E+wXKyY85VAAAA4L+AT2C4ZY80CFRZL2etPRirdTGxupySoe+izuq7qLMymaTwAC+1DPJTyyA/hZTxkA29PQFYIRsXF7k2qC/XBvUl5fT6vHrsuFKj/ur1mX74sDJOnVLGqVNK+OFHSZLJxUXOtWrJOTxMDqG1ZJOSUpynYfXsbewV4hOiEJ8QPVLjEUnShZQLio6NNq/wvj9uvy6lXdLqU6u1+tRqSZKdjZ2CSwSrlm8thfuFK9w3XKVcSxXnqQAAAAC4Qwg6cct83R3Vo26AetQNUEZWtnaeuKy1h2K15sAFHTifqF0nr2jXySt6Z+UhlXRzVIsgX7UM8lOTqiXl6Wxf3M0HgFtiMpnkWKmiHCtVlNcD90uSshISlBq9+6/wc/duZSclKWXzZqVs3ixJqiLpxKfz5VI7wtzr06FSJZlsWFznVvm5+KltYFu1DWwrSUrPStf+uP3mXp+7LuxSXFqcdl/crd0Xd+uz/Z9Jkvxd/c1D3cN9w1WtRDXZ2/C+BAAAAFg7gk7cFva2NmpQyUcNKvno+Q7VdS4+VWsP5oSeGw5f1MWkdH2947S+3nFatjYm1Qn0VssgP7UI8lX10u7MpwbAqtl6eMitaRO5NW0iSTKyspR++Ig5+EyJ2qWM4yeUceyY4o8dU/w330qSbDw85BwWJufwMLlERMipVi3ZurkV56lYNUdbx5zw0i9cUk7v2zNJZ3Lm+fxzdfeDlw/qXPI5nUs+p2XHc+ZcdbJ1UkjJEIsh795O3sV4JgAAAABuBUEn7gh/T2f1ql9eveqXV3pmlrYfv6w1By5ozcELOhKbrK3HLmnrsUt6a9kBlfZwUsvqOXN7Nq5SUm6O3JYArJvJ1lZOQdXkFFRN3g/1VEZGhpYvWqQmfn66mjvf5969yk5IUPL69Upevz5nRxsbOVatajHXp3358nwZdItMJpPKuZdTOfdy6lwpZ87VlIwU7b2412KF94SrCdrxxw7t+OOvOVcDPQItVniv7FlZtja2xXUqAAAAAAqBRAl3nKOdrRpXKanGVUrq5c7BOnUpRWsPXtCag7HadOSiziek6fOtp/T51lOytzWpfsUSf/b29FNlX1c+4AO4K2S5/T979x1W9Xn+cfz9PYs9ZCkCgoKCE9wxjkj2Mjsam2GSZptp9mxWs2Nsotm7WZo0TX4x00Rwxa2IiyXTyd4bzu+PL2Js00z1MD6v63quFj0H7qecIny4n+f2xmvyZPxPMI9ZO5uaqE/POOiuz6Zdu2hIT6chPZ3yj8zhOtaAgPbJ7p7DE3AfMgSLh4crt9Kpedo9GRM6hjGh5p2rrc5Wcitz2VS4qb3zc0fFDvIq88irzOP/dvwfAN52b4YGDW0/7j40eCg+Dh9XbkVEREREuijDMPj3v//NWWed5epSOh0FnXLERQR4cvG4KC4eF0V9UwursktITi9icVoh+aW1rMgqYUVWCY9+uZ3wHh7mQKO4YMb1C8LDoW4aEekaDLsdjyGD8RgyGC4yh+s0FRa2BZ9m+Fm/dSstpaVUL15M9WJzuA42G+5xcXgMH95+5N0WGqpfCv1BFsNCP79+9PPrx9n9zTtXKxoqSC1KNQcdFaWwuWgz1U3VrNyzkpV7VgJgYBDtH90efMYHxxPpG6nPg4iIiIgLLF26lKeffpr169ezZ8+eg0LCpqYm7rvvPr766iuys7Px8/Pj+OOP54knnqB3797t7yMjI4Pbb7+dFStW0NjYyLBhw3jkkUdITEz8xY/tdDp59tlnefXVV8nLyyMoKIjrrruOe++993Bu+TdLTk4mMTGRsrIy/P39XV3OYaegU1zK3W5lclv35t+mDCKnuIak9CKS0wtZnV3KzrI6/rkqj3+uysNhszCuXyCJscEkxoUQGejl6vJFRA4pe0gI9hNPxPfEEwFobWykfutW6lI2mV2fGzfSXFRE/ZYt1G/ZQtk//wmALSSkLfg0uz7dBg3C4nC4ciudmp+bHxPDJzIxfCIAza3NZJVntXd9phSmsLN6J1nlWWSVZ/FJxicA9HDrYR53DzGPvA8JGoKHTd23IiIiIodbTU0N8fHxXH755ZxzzjkH/V1tbS0bNmzg/vvvJz4+nrKyMm666SbOOOMM1q1b1/64008/nf79+7N48WI8PDyYM2cOp59+Ojt27KBXr17/82PfdNNNfPfddzzzzDMMHTqU0tJSSktLD9teXcXpdNLS0oLN1rGjxI5dnXQrhmHQL9ibfsHe/HVCX2oamlm5o4Sk9EKS0grZXVHPkowilmQU8eAX2+gX5MXktm7PMX0DcLOp21NEuhaLw4Hn8OF4Dh8Ol12K0+mkefduajemtB95r09Lo7mwkKpvv6Xq228BMBwO3AcPbu/69EhIwB4S4uLddF42i424gDjiAuKYFjcNgOK6YjYVbWofcrSleAtlDWUk70wmeWey+TzDRmxA7EET3nt59VLXp4iIiMghdsopp3DKKaf87N/5+fmxaNGig/5s7ty5jBkzhvz8fPr06UNxcTGZmZm88cYbDBs2DIAnnniCF198kS1btvzPoHP79u289NJLbNmyhdjYWAD69u37m2p+8803efbZZ8nKyiIgIIBzzz2XuXPn/tfjfq4jMyUlheHDh5OTk0NUVBR5eXlcf/31LF++nMbGRqKionj66acZNGhQe0dqjx7msM0ZM2bw9ttv09raypNPPsmrr77K3r17GTBgAPfffz/nnXfeQR/3q6++4r777mPz5s189913TJ48+Tftz1UUdEqH5eVm4/hBPTl+UE+cTieZhdXtA43W5ZaRXVxDdnEOb67IwcNu3gM6ua3bM8xfHTQi0vUYhoE9LAy/sDD8Tj8NgNbaWuq2bDnQ9ZmSQktZWXsH6H72sLD2rk+P4Qm4x8ZidPDfxnZkQR5BHNfnOI7rcxwATS1NbC/d3j7gKKUwhcK6QraWbGVryVY+SPsAgBDPEDP4bJvwPjBgIHar3ZVbEREREflZTqcTZ13dEf+4ra2tOJ3Ow/oxKioqMAyjPTgMDAwkNjaWd999lxEjRuDm5sYrr7xCSEgII0eO/J/v54svvqBfv34sXLiQk08+GafTyfHHH89TTz1FQEDA/3zeSy+9xKxZs3jiiSc45ZRTqKioYMWKFX94PzNnzqSxsZGlS5fi5eXFtm3b8Pb2JiIign/961+ce+65pKen4+vri0fbff+PP/447733Hi+//DL9+/dn6dKlXHTRRQQHB3PMMce0v++77rqLZ555hn79+rWHpR2ZfsKRTsEwDAb09GFATx+uPiaayvomVmQWk5ReSHJ6EYVVDXy/fR/fb98HwICe3u0DjUZF9cButbh4ByIih4fF0xOvMWPwGmMO13E6nTTl5VH7k7s+GzIzadq1i6Zdu6hcuBAAw8MDj6FD24NPj4QEbJ3gG5eOym61Myx4GMOCzQ4Ap9PJ3pq95oCjtuAzrTSNwtpCFuUtYlGe2VXgsDgYHDS4/Z7P+JB4gjyCXLkVEREREQCcdXWkj/jfId/h1DNpMfj5HZb3XV9fz5133sn06dPx9fUFzMzh+++/56yzzsLHxweLxUJISAjffPPNL4Z72dnZ5OXl8fHHH/Puu+/S0tLCLbfcwnnnncfi/Xfs/4xHH32UW2+9lZtuuqn9z0aPHv2H95Sfn8+5557L0KFDAejXr1/73+0PXENCQtqD3YaGBh577DG+//57xo0b1/6c5cuX88orrxwUdD788MOc0DZQtTNQ0Cmdkq+7nVOGhnLK0FCcTifb9lSSnF5EUlohG/LLyNhXTca+al5Zmo2Pm40J/YNIjA3hmNhgevq6u7p8EZHDxjAMHFFROKKi8G+7gL2lupq6TZsODDratInWqipq16yhds2a9uc6oqIO6vp0i4nBsOgXRX+EYRiEeocS6h3KKX3NY1R1zXVsLd7aPt19U9EmyhrK2Fi4kY2FB7pvw73DSQhJaD/yHuMfg82ib9lERERE/qympiamTp2K0+nkpZdeav9zp9PJzJkzCQkJYdmyZXh4ePD6668zZcoU1q5dS2hoKIMHDyYvLw+AiRMn8vXXX9Pa2kpDQwPvvvsuAwYMAOCNN95g5MiRpKen4+HhwaBBg9o/zj333MMVV1zB7t27Oe644w7Zvm688UauvfZavvvuO44//njOPffc9iP4PycrK4va2tr/CjAbGxsZPnz4QX82atSoQ1bnkaDvmqXTMwyDwb39GNzbj5mJMZTXNrI0s5jktEKSM4oorWnk6y17+XrLXgAGhfqSGBdMYmwICRH+2NTtKSJdnNXbG+/x4/EePx4AZ2srjTt2HOj6TEmhMTubxtxcGnNzqfj3vwGweHvjER9/IPyMH4bVx8eVW+nUPGwejOo1ilG9zG8WnU4n+VX5B467F6WQVZbFzuqd7KzeycJss/vW0+bJ0KChxIeYR96HBQ/Dz+3wdDiIiIiI7Gd4eBC7Yf0R/7itra1UNTUd8ve7P+TMy8tj8eLF7d2cAIsXL2bhwoWUlZW1//mLL77IokWLeOedd7jrrrv46quvaGqra//x79DQUGw2W3vICTBw4EDA7LJMTEwkJSWl/e8CAgKw23/ftUWWtsaDnx7nb/qP/32uuOIKTjrpJL788ku+++47Hn/8cZ599lluuOGGn32f1dXVAHz55ZeEhYUd9Hdubm4Hve3l1bkGQSvolC7H39PBGfG9OSO+N62tTlJ3VZCcXkhSehGpO8vZtqeSbXsqmZe0Az8PO5MGBJMYG8ykAcEEebv9+gcQEenkDIsFt/79cevfnx7nnw9Ac1nZwV2fmzfTWl1NzYoV1Oy/L8gwcIuJOajr0xEVpeE6f5BhGET6RhLpG8mZMWcCUNVYxeaize3BZ2pRKtVN1azeu5rVe1e3P7efX7/2AUfxwfFE+UVhMfSLOxERETl0DMPA8PQ88h+4tRWjsvKQvsv9IWdmZiZJSUkEBgYe9Pe1tbXAgVBxP4vFQmtrKwCRkZH/9X7Hjx9Pc3MzO3bsIDo6GoCMjIz2x9tsNmJiYv7reVFRUfzwww/tg4J+SXBwMAB79uxpP0b/0/B0v4iICK655hquueYa7r77bl577TVuuOEGHA4HAC0tLe2PHTRoEG5ubuTn5x90TL0rUNApXZrFYpAQ4U9ChD83Hz+A4uoGlmYUkZRexNKMIirqmvhi026+2LQbw4Bh4f4kxprdnkPD/LBY9MO7iHQPth498Jk8GZ+2KYrO5mYaMjIOuuuzaedOGjIzacjMpHzBAgCs/v5m6JmQYAagQ4dgccU3xF2Ej8OHo8OO5uiwowFoaW1hR8WO9ns+NxVtIq8yj+yKbLIrsvk081MAfB2+5h2fbcfdhwYNxdOuz4OIiIh0D9XV1WRlZbW/nZOTQ0pKCgEBAYSGhnLeeeexYcMGFi5cSEtLC3v3mic+AwICcDgcjBs3jh49ejBjxgweeOABPDw8eO2118jJyeG00077nx/3+OOPZ8SIEVx++eXMmTOH1tZWZs6cyQknnHBQl+d/evDBB7nmmmsICQnhlFNOoaqqihUrVvxsB2ZMTAwRERE8+OCD/P3vfycjI4Nnn332oMfcfPPNnHLKKQwYMICysjKSkpLaO0sjIyMxDIOFCxdy6qmn4uHhgY+PD7fddhu33HILra2tTJgwoX0gkq+vLzNmzPhd//t3JAo6pVsJ8nbjnBHhnDMinOaWVlIKytsHGm3dXcmmgnI2FZQz5/tMAr0cHDMgmMlxIUzqH4S/p8PV5YuIHDGGzYb7oEG4DxoEf/kLAM1FRWbw2RZ+1m/ZQkt5OdXJyVQnJ5tPtFpxj439SdfncOxhvdX1+QdZLVYG9BjAgB4DOH+A2X1bWl9KalFqe/C5pXgLlY2VLNu1jGW7lgFgMSwM6DGgPfhMCE4gzDtMnwcRERHpktatW3dQd+SsWbMAmDFjBg8++CD/93//B0BCQsJBz0tKSmLy5MkEBQXxzTffcO+993LsscfS1NTE4MGD+fzzz4mPj/+fH9disfDFF19www03MGnSJLy8vDjllFP+K4j8TzNmzKC+vp7nnnuO2267jaCgIM4777yffazdbufDDz/k2muvZdiwYYwePZpHH32U89tOZoHZrTlz5kx27tyJr68vJ598Ms899xwAYWFhPPTQQ9x1111cdtllXHLJJbz99ts88sgjBAcH8/jjj5OdnY2/vz8jRozgnnvu+cXaOzrD+dND/nJIVVZW4ufnR0VFxUF3PxxOTU1NfPXVV5x66qm/+96H7m5fZT1L0otISi9kWWYx1Q3N7X9nMWBEnx4kxoUwOTaYQaG++mHxMNNrWbqSrvp6djY2Ur99O3UpKdS2dX0279v3X4+zBgfhmXAg+HQfPAiLm64KOVSaWpvIKM1oH3KUUpTCnpo9//W4QPfAA8fdQ+IZFDgIN+vv/zx01dezdE96PUtXodeyHAr19fXk5OTQt29f3N1dN8S3tbWVyspKfH19/+souXRdv/T6+z35mjo6Rdr09HVn6ugIpo6OoLG5lfV5ZW13exaSsa+adXllrMsr4+lv0wnxcSMx1gw9x/cPwtdd30yISPdjOBzmsKL4eALajrc07dlD3caN7Ufe67dvp6WomKpFi6hatMh8nt2O+6BBB3d99gxx5VY6NbvFzuCgwQwOGsyFAy8EYF/NvvZ7PjcVbmJb6TZK6kv4If8Hfsj/AQCbxcagwEHt93wmhCQQ4qnPg4iIiIh0Xgo6RX6Gw2ZhXHQg46IDufvUgewsqyU5vYjk9EJWZJVQWNXA/HUFzF9XgM1iMCqqB4mxISTGhdA/xFvdniLSbdlDQ7GHhuJ76qkAtNbXU79ly4Guz5QUWkpKzMFHmza1P8/WO/Tgrs+4WAx1pPxhPb16cqLXiZwYdSIADS0NbCvZRkphSvuR95L6ElKLUkktSm1/Xm+v3uZdnyFm8DmgxwDsFn0eRERERKRzUNAp8huE9/DkoqMiueioSOqbWlibW0pSmhl8ZhfXsCq7lFXZpTz+dRph/h5MbhtodHRMIJ4O/d9MRLovi7s7nqNG4TlqFIGA0+mkqaDgoK7PhowMmnfvoXL3Hiq/+goAw90djyFDzK7P4eawI1tAgGs304m5Wd0YHjKc4SHDAfPzsLN6Z3voualoExllGeyu2c3umt18nfs1AB42DwYHDm4/8j4seBjeVm9XbkVERERE5H9SAiPyO7nbrUzsH8zE/sE8MGUQucU1bUfci1iZXcKu8jreX53P+6vzcVgtjO0X0N7t2TfIy9Xli4i4lGEYOPr0wdGnD35nnglAS3UN9ZtT27o+N1KXsonWykpq162jdt269ufaI/uYXZ/Dza5Pt5gYDKvVVVvp1AzDIMInggifCKZETwGgpqmGLcVbzK7PIjMArWqsYt2+dazbd+DzEOkTSUBDAHVZdYzsNZJo/2gshu7PEhERERHXU9Ap8idFBXlxaVBfLh3fl7rGFlZmF5OcXsTitEJ2ltWxLLOYZZnFPLxwG5GBnu13ex7VLxB3u35AFxGxenvhNW4cXuPGAeBsbaUxJ+egrs/GHTtoysunIi+fis8/B8Di5YVH/LD24+4e8fFYj9Dwv67Iy+7F2NCxjA0dC0Crs5XcilxSig4cd8+uyCavKo888ti4ZiMAPnYfhgUPaz/yPixoGN4OdX2KiIiIyJGnoFPkEPJwWDk2rifHxvXkoTOc7CiqaR9otCanlLySWt7+MZe3f8zF3W7h6OggEmODmRwbQkSAp6vLFxHpEAyLBbfoaNyio/E/7zwAWioqzHs927o+6zel0lpTQ82PK6n5cWX7cx0x0Xj+ZMiRIyoKQ9M6/xCLYaGffz/6+ffjnP7nAFDRUMGGPRv4dNWn1PjXsKVkC1VNVazYvYIVu1cAYGAQ0yOGhOCE9iPvET4Rur9aRESkk3A6na4uQbqhQ/W6U9ApcpgYhkFMiDcxId5cMbEf1Q3NrMgqNoPPtCL2VtazOK2QxWmFwFZiQryZPCCYxLgQRkcF4LDpB3MRkf2sfn54T5qE96RJADhbWmjIzKRu48b2QUdN+fk0Zu2gMWsH5R9/AoDFzw+PhHg893d9Dh2KxUvXiPxRfm5+TAibQKVHJacedyqG1SCzLLN9wntKYQq7qneRWZZJZlkmH2d8DEAPtx7mgKO2Ce+DgwbjYfNw8W5ERETkp6xtVwI1Njbi4aF/p+XIamxsBA68Dv8oBZ0iR4i3m42TBvfipMG9cDqdpO+rIimtiKT0QtbnlZFVWE1WYTWvL8/By2FlfEwQiXHmMfdQP/0jIyLyU4bVintcHO5xcfSYPh2A5pIS6lJSDnR9bt5Ca0UFNUuWUrNkqflEiwW32Fg82wYceQwfjj08XN2Gf5DNYmNg4EAGBg7kgrgLACiuK2ZT4YHgc1vJNsoaykguSCa5INl8nmEjLiCuPfxMCEmgl1cvl+1DREREwGaz4enpSVFREXa7HYuLTsW0trbS2NhIfX29y2qQI6u1tZWioiI8PT2x2f5cVKmgU8QFDMMgrpcvcb18uXZyNBV1TSzPLCYpvZDk9CKKqxv4bts+vtu2D4C4Xj4kxoWQGBvCiD7+2Kz6Yi8i8p9sgYH4HHccPscdB4CzsZH69PSDuj6b9+yhYft2GrZvp+yDDwGwBgbiMTyhvevTffBgLO7urtxKpxbkEcRxkcdxXKT5eWhsaWR76fb2ez5TClMoqitiS8kWtpRs4f3t7wMQ4hly0HH3uIA47Fa7K7ciIiLSrRiGQWhoKDk5OeTl5bmsDqfTSV1dHR4eHvpldDdisVjo06fPn/6cK+gU6QD8POycNiyU04aF0trqZOvuSpLa7vZMKSgnbW8VaXureCl5Bz7uNib1D2ZybDDHxAYT4qMfxkVEfo7hcOAxdCgeQ4fCJZcA0LR3r9n1uTGF2pSN1G/bTktJCdXf/0D19z+YT7TbcR848OCuz17qNvyjHFaHOagoOB4wf3jZU7OnPfRMKUohvTSdwtpCvsv7ju/yvgPAzerG4MDBxIfEtz8/yCPIlVsRERHp8hwOB/37928/RuwKTU1NLF26lEmTJmG365ee3YXD4TgkHbwKOn9FTk4Ol19+Ofv27cNqtbJq1Sq8dLeXHEYWi8HQcD+Ghvtx43H9Ka1pZFlmEUlphSzJKKKstokvN+/hy817ABga5mcONIoLIT7cH6tFv/ESEflf7L16YT/5ZHxPPhmA1oYG6rdupW5jCnUpG6ndmEJLcTH1qanUp6bCO+8CYOvV6+Cuz7g4DIfDlVvptAzDoLd3b3p79+aUvqcAUNtUy9aSre3h56aiTZQ3lLOhcAMbCje0PzfCJ6L9ns+EkARi/GOwWv7cPU4iIiJyMIvFgrsLT7dYrVaam5txd3dX0Cm/m4LOX3HppZfy6KOPMnHiREpLS3Fzc3N1SdLNBHg5ODMhjDMTwmhpdbJpZznJaYUkpRexeVdF+3p+cRY9PO0c0zbQaFL/YHp46YdwEZFfYnFzw3PECDxHjADMbsOmXbvM4+5tXZ8N6Rk0791L1dffUPX1NwAYbm64DxlyoOszIQFbkLoN/yhPuyeje41mdK/RgPl5yKvMa7/nc1PRJnaU76CgqoCCqgK+yP7CfJ7Nk6HBQ9uPvA8LHoavw9eVWxERERERF1LQ+Qu2bt2K3W5n4sSJAAQEBLi4IunurBaDEX16MKJPD2adGEthVT1L0otITi9iaabZ7flZym4+S9mNYUBChD+JsebdnoN7+2JRt6eIyC8yDANHeDiO8HD8pkwBoLWmhrrNW9qOvJv3fbZUVFC3fj1169e3P9ceEYFHW/DpOXw4bv37Y/zJy9S7K8MwiPKLIsovirNizgKgsrGSzUWb27s+U4tTqWmqYfWe1azes7r9udF+0SSEmF2f8SHx9PXtq/u9RERERLqJLv3d99KlS3n66adZv349e/bs4d///jdnnXXWQY+ZN28eTz/9NHv37iU+Pp4XXniBMWPGAJCZmYm3tzdTpkxh165dnHfeedxzzz0u2InIzwvxcef8URGcPyqCppZWNuaXm3d7phWStreKjfnlbMwvZ/aiDIK83ZgcG0xibAgT+gfh56EjACIiv4XFywuvo8biddRYwOw2bMzJbQ8961I20pC1g6aCApoKCqj8P7Pb0PD0xGPYMDwS4vEcPhyP+His/v4u3Enn5uvwZXzYeMaHjQegpbWFHRU7DhpylF+Vz46KHeyo2MG/Mv8FgJ+bX/sdnwnBCQwJGoKn3dOVWxERERGRw6RLB501NTXEx8dz+eWXc8455/zX38+fP59Zs2bx8ssvM3bsWObMmcNJJ51Eeno6ISEhNDc3s2zZMlJSUggJCeHkk09m9OjRnHDCCS7Yjcgvs1stjOkbwJi+Adx5chx7KupITjfv9lyRVUxxdQOfrN/JJ+t3YrUYjIzsQWJsCJNjg4nr5aNuFxGR38gwDNz69cWtX1/8zzW/v2iprKRuU+qB8HPTJlpraqhdtYraVasoaXuuo1+/g7o+Hf36YRyCS9e7I6vFyoAeAxjQYwBTY6cCUFJXQmpRKilFZvi5pXgLFQ0VLN25lKU7l5rPM8zn7b/nMyEkgd5evfXvoIiIiEgX0KWDzlNOOYVTTjnlf/797NmzufLKK7nssssAePnll/nyyy958803ueuuuwgLC2PUqFFEREQAcOqpp5KSkvI/g86GhgYaGhra366srATMiWFNTU2Halu/aP/HOVIfTzquIE8b5w0P5bzhoTQ0t7I+r4wlGcUsySxmR1ENa3JKWZNTypPfpNHT143JA4I4pn8w46ID8HZz/ZcGvZalK9HruRvw8MDtqLG4HTUWf8DZ0kLjjh3Up2yifpO5mvLyaMzOpjE7m4p/fQqAxccH92HDcI+Pxz0hAfehQ7B4e7t0K7+mI7+efW2+TAidwITQCQA0tTSRUZ7BpqJNpBanklqcyt7avWwv3c720u18lP4RAEHuQQwLHkZ8UDzDgoYxMGAgDqvuue4OOvLrWeT30GtZuhK9nuU//Z7XguF0Op2HsZYOwzCMg46uNzY24unpySeffHLQcfYZM2ZQXl7O559/TnNzM6NHj2bx4sX4+flx5plncvXVV3P66af/7Md48MEHeeihh/7rzz/44AM8PXVESjqOknrYVm6wrcwgs9KgqfVAF4vVcBLt62SQv5NBPZyEuIOaXERE/jxrdTXu+fm45+fjkZeHe8FOLP/xTZvTMGjs1ZO6yEjq+vShPjKSpsBAfSE+hCpaKyhoLiC/JZ/85nz2tOyhhZaDHmPFSm9rb/rY+tDH2oc+tj74WHxcVLGIiIhI91ZbW8tf/vIXKioq8PX95cGTrm/bcpHi4mJaWlro2bPnQX/es2dP0tLSALDZbDz22GNMmjQJp9PJiSee+D9DToC7776bWbNmtb9dWVlJREQEJ5544q9+Ig6VpqYmFi1axAknnIDdrjsY5dfVN7WwJreM5IxiktOLKCirI6PCIKMCPsuD8B4eZrfngCDGRgXg4bAekbr0WpauRK9n+TnOpiYaMjKpT0lp7/ps3r0btz17cduzF/9V5oAda0AP3IfFt3V9xuM2eDAWDw+X1d3VXs/1zfVsL93OpuIDXZ+l9aUUtBRQ0FLAClYA0NurN8OChjEsaBjxwfH09++PzdJtv5XuMrra61m6L72WpSvR61n+0/4T07+Fvjv7Fb92/P2n3NzccHNz+68/t9vtR/z/nK74mNI52e12jhsUynGDQnE6neQU15CUXkRyeiGrs0vZWVbHe6sLeG91AW42C0f1CyQxNpjEuBAiA72OSH16LUtXodezHMRux5EQj09CfPsfNe0rbBtwZE54r9+6lZbSMmqSk6lJTjYfZLPhHheHx/Dh7YOObKGhR/yOya7yerbb7YwJG8OYMHMYpdPpZGfVzvZ7PlMKU8gsz2R3zW521+zmm7xvAPCweTAkaAgJweY9n8OChuHv7u/Cncif0VVezyJ6LUtXotez7Pd7XgfdNugMCgrCarWyb9++g/5837599OrVy0VVibiWYRj0C/amX7A3f53Ql5qGZlbuKCEpvZDk9CJ2ldexJKOIJRlFPPjFNvoFeTE5NoTEuGDG9A3AzXZkuj1FRLoqe88Q7CediO9JJwLQ2thI/dat1G08EH42FxVRv2UL9Vu2UPbPfwJgCwlpCz4T8ByegNugQVgcumPyjzAMgwjfCCJ8I5gSPQWAmqYaNhdvJqUwhZSiFFILU6lqqmLt3rWs3bu2/blRvlHmgKPgBOKD4+nn3w+LoWFTIiIiIkdKtw06HQ4HI0eO5Icffmi/o7O1tZUffviB66+/3rXFiXQQXm42jh/Uk+MH9cTpdJJZWE1SWiFJ6YWsyy0ju7iG7OIc3lyRg4fdyviYICa3dXuG+bvuWKWISFdhcTjwHD4cz+HDAbPbsHn3bmp/EnzWp6XRXFhI1bffUvXttwAYDgfugwe3d316JCRgDwlx5VY6NS+7F0eFHsVRoUcB0OpsJacipz343FS0iZyKHHIrc8mtzOWzrM8A8LH7MCzEPOqeEJzA0KCheDs69rApERERkc6sSwed1dXVZGVltb+dk5NDSkoKAQEB9OnTh1mzZjFjxgxGjRrFmDFjmDNnDjU1Ne1T2EXkAMMwGNDThwE9fbj6mGiq6ptYkVVMUloRSemFFFY18P32fXy/3eySHtDTm8TYECbHhjAqqgd2qzpaRET+LMMwsIeF4RcWht/ppwHQWltL3ZYtB3V9tpSXU7dxI3UbN7Y/1x4W1t716TE8AffYWAxbl/5W8LCxGBai/aOJ9o/m3AHnAlBeX05qcSophWbwubl4M1VNVazYtYIVu1a0Py/GP6b9uHtCcALhPuFH/NoBERERka6qS393u27dOhITE9vf3j8oaMaMGbz99ttMmzaNoqIiHnjgAfbu3UtCQgLffPPNfw0oEpH/5uNu5+QhoZw8xLzbc9ueSpLTi0hKK2RDfhkZ+6rJ2FfNK0uz8XGzMaF/EImxIRwTG0xPX3dXly8i0mVYPD3xGjMGrzEH7phsyss7qOuzITOTpl27aNq1i8qFCwEwPDzwGDq0Pfj0SEjA1qOHK7fSqfm7+zMpfBKTwicB0NzaTEZZRvs9n5uKNrGrehcZZRlklGWwIGMBAAHuAWbHZ4h53H1w4GDcbfp3UkREROSP6NJB5+TJk3E6nb/4mOuvv15H1UX+JMMwGNzbj8G9/ZiZGEN5bSNLM4tJTiskOaOI0ppGvt6yl6+37AVgcG/ftm7PYBIi/LGp21NE5JAxDANHVBSOqCj8zz4LgJaqKupSUw90faak0FpdTe2aNdSuWdP+XEdU1EFdn24xMRgWfY3+I2wWG4MCBzEocBDT46YDUFRb1B58phSlsK1kG6X1pSQVJJFUkGQ+z7AxMHAg8cHxxIeYR957een+eBEREZHfoksHnSLiGv6eDs6I780Z8b1pbXWSuquC5PRCktKLSN1ZztbdlWzdXcncpCz8POxMGhBMYmwwxwwIJtDbzdXli4h0OVYfH7zHj8d7/HgAnK2tNO7YQe3GjdSlbKJu40Yac3JozM2lMTeXin//GwCLtzce8fEHws/4YeCubsM/KtgzmOMjj+f4yOMBaGxpZFvJtoPCz+K6YjYXb2Zz8Wbe2/4eAL28erXf85kQkkBsQCx2i6bQioiIiPwnBZ0iclhZLAYJEf4kRPhz8/EDKK5uYGlGEUnpRSzNKKKirokvNu3mi027MQwYFu5PYmwwE6MDaP3lhmwREfmDDIsFt/79cevfnx5TpwLQXFZG3aZNB7o+U1Npra6mZsUKalasaHuigSM6mpCAACobm/AeNRJHVJTumPyDHFaHeVdnSAIzBs/A6XSyu2Z3+1H3lMIUMsoy2Fuzl701e/k21xw25W51Z1DgoAMT3kPiCXAPcPFuRERERFxPQaeIHFFB3m6cMyKcc0aE09zSSkpBOUnphSSnF7F1dyWbCsrZVFDOnO/B22ZlSd1mjh3Ui0n9g/D3dLi6fBGRLsvWowc+kyfjM3kyAM7mZhoyMg7q+mzauZPGrCz8gcI1aygErP7+eCQk0GP6BXgfc4wLd9D5GYZBmHcYYd5hnNbPHDZV21TL1pKtB014r2ioYEPhBjYUbmh/bh+fPu33fMYHxxPjH4PVYnXVVkRERERcQkGniLiMzWphVFQAo6ICuP2kOPZV1rMk3ZzivjSziOqGFj7btIfPNu3BYsCIPj1IjDPv9hwU6qsOIhGRw8iw2XAfNAj3QYPgwgsBaC4qomr9erZ9+m96V1XRsHUrLeXlVCcnU52cTPDNNxF49dX6+nwIedo9Gd1rNKN7jQbMYVO5lbntXZ+bijaRVZ5FflU++VX5/N+O/wPAy+7F0KCh7V2fQ4OH4uvwdeVWRERERA47BZ0i0mH09HVn6ugIpo6OoKaugZc+/pb6gGiWZhaTsa+adXllrMsr4+lv0wnxcWsfaDS+fxC+7rqrTETkcLMFB+N93HEUNzQw5tRTsTqdNGzfTvmn/6Z8/nyK5vyDhqwdhD76CBbd5XlYGIZBX7++9PXry9n9zwagoqGCzcWb24+7pxalUtNUw6o9q1i1Z5X5PAyi/aMPmvAe5atrB0RERKRrUdApIh2Sw2ahv5+TU08awH2nD2ZXeZ050CitiBVZxRRWNTB/XQHz1xVgsxiMiupBYmwIiXEh9A/x1g9uIiJHgMXhMIcVxcfjPnAgex99lMqFC2nMzyd87gvYQ0JcXWK34Ofmx4SwCUwImwBAS2sLWeVZBw05KqgqIKs8i6zyLP6V+S8A/N38Dwo+BwcOxtPu6cqtiIiIiPwpCjpFpFMI8/fgwrGRXDg2kvqmFtbmlpKUVkRyeiHZxTWsyi5lVXYpj3+dRpi/B5Njg0mMDeHomEA8HfpSJyJyuPW4YBqOqCh23nQT9amp5E6dRvi8uXgMHuzq0rodq8VKbEAssQGxTI01h02V1JWYwWdRCpsKN7G1ZCvlDeUs2bmEJTuXmM8zzOfFB8dzRvQZDAka4sptiIiIiPxu+ulfRDodd7uVif2Dmdg/mAemDCK3uMbs9kwvYmV2CbvK63h/dT7vr87HYbUwtl9Ae7dn3yAvV5cvItJleR01lr4L5lNw7XU0ZmeTd+FF9H7ySXxPOtHVpXV7gR6BHNvnWI7tcywATS1NpJWmtQ842li4kcLaQraVbGNbyTY+Tv+YW0fdyoUDL9QpCREREek0FHSKSKcXFeTFpUF9uXR8X+oaW1iZXUxyehGL0wrZWVbHssxilmUW8/DCbUQGerbf7XlUv0Dc7ZpIKyJyKDkiI4n66EN2zbqVmuXL2XXTTTTceANB116rwKwDsVvtDA0eytDgoVzMxQDsrdlLSlEK3+R8ww/5P/Dk2idJLU7lwXEP6ki7iIiIdAoKOkWkS/FwWDk2rifHxvXkoTOc7Cja3+1ZyJqcUvJKann7x1ze/jEXd7uFo6ODSIwNZnJsCBEB+iFORORQsPr6EvHyS+x76inK3v0nxc+/QGPWDkIf+7uGFHVgvbx6cbLXyZwUeRIfpH3AM2uf4eucr8ksy2RO4hwifSNdXaKIiIjIL1LQeRjMmzePefPm0dLS4upSRLo1wzCICfEmJsSbKyb2o7qhmRVZxe1DjfZW1rM4rZDFaYXAVmJCvJk8IJjEuBBGRwXgsFlcvQURkU7LsNnodc89uEXHsPeRR6j86isaCwoInzsXe08NKerIDMPgwoEXMjBgILcuuZWs8iwuWHgBj014jMQ+ia4uT0REROR/UtB5GMycOZOZM2dSWVmJn5+fq8sRkTbebjZOGtyLkwb3wul0kr6viqS0IpLSC1mfV0ZWYTVZhdW8vjwHL4eV8TFBJMaZx9xD/TxcXb6ISKfUY9pUHFFR7LrxRuo3byZ36lTCX5ynIUWdwIieI1hw+gJuW3IbGwo3cGPSjVw59EpmJszEatHVLyIiItLxKOgUkW7JMAzievkS18uXaydHU1HXxPLMYpLSC0lOL6K4uoHvtu3ju237AIjr5UNiXAiJsSGM6OOPzapuTxGR38pr7BiiFsyn4LqZNO7YYQ4peuIJfE8+ydWlya8I9gzm9ZNe59l1z/L+9vd5bfNrbC3ZypMTn8Tf3d/V5YmIiIgcREGniAjg52HntGGhnDYslNZWJ1t3V7aFnoVsLCgnbW8VaXureCl5Bz7uNiYNCGbygGCOiQ0mxEf3zYmI/Jr2IUW33krN0mXsuvlmGm64nqDrrtOQog7ObrFz15i7GBo0lIdWPsSPu39k2sJpzE6czeBAdeaKiIhIx6GgU0TkP1gsBkPD/Rga7seNx/WntKaRZZlFJKUVsiSjiLLaJr5M3cOXqXsAGBrmZw40igshPtwfq0U/sIuI/Byrjw8RL71E4VNPU/rOOxS/MJfGHTsIfewxDSnqBE7rdxr9e/Tn5qSbKagq4JKvLuG+o+7j7P5nu7o0EREREUBBp4jIrwrwcnBmQhhnJoTR0upk085yktMKSUovYvOuivb1/OIsenjaOaZtoNGk/sH08HK4unwRkQ7FsFrpefddOGKi2fvQw1R+9TWN+QWEz5unIUWdwIAeA/jo9I+4d9m9JO9M5oEfHyC1OJW7x9yNw6p/80RERMS1FHSKiPwOVovBiD49GNGnB7NOjKWwqp4l6UUkpxexNNPs9vwsZTefpezGMCAhwp/EWPNuz8G9fbGo21NEBIAe55+PIzKSXTfeRP2WLeSefz7h8+bhMXSIq0uTX+Hr8OUfx/6D11JfY17KPD7J+IT00nRmT55NL69eri5PREREujFN0xAR+RNCfNw5f1QE8y4cwYb7T2DB1eO4dnI0cb18cDphY345sxdlMGXucsY89gO3fbyJL1P3UFHX5OrSRURczmvMGKI+XoAjJprmwkLyLr6Yyq+/dnVZ8htYDAtXx1/Ni8e/iK/Dl83Fm5n6xVRW71nt6tJERESkG1PQKSJyiNitFsb0DeDOk+P45uZJrLz7WB4/ZygnDuqJl8NKcXUDn6zfycwPNjDikUVMfWUlLyXvIG1vJU6n09Xli4i4hCMigqiPPsJr0kSc9fXsumUWRXPn6etiJzEhbALzT5/PwICBlDWUcdWiq3hzy5v6/ImIiIhLKOgUETlMQv08mD6mD69eMooND5zA+1eM5cqJfYkJ8aal1cmanFKe/CaNk+cs4+gnFnP3p6l8u3Uv1Q3Nri5dROSIsnp7E/HSSwRceikAxXPnsmvWLFrr6lxbmPwm4T7hvHvKu5wZfSatzlaeW/8cs5JnUd1Y7erSREREpJvRHZ0iIkeAm83K+JggxscEce9pUFBaS3K6OdDoxx3F7Kmo58M1BXy4pgC71WBM3wASY0OYHBtCdLAXhqG7PUWkazOsVnredSduMdHseehhqr7+hryCnYTPm4u9Z09Xlye/wt3mziPjH2FY8DAeX/M43+d/z46KHcyZPId+/v1cXZ6IiIh0Ewo6RURcICLAk4vHRXHxuCjqm1pYlV1CcnoRSemF5JXUsiKrhBVZJTz65XYiAjzaBxod1S8QD4fV1eWLiBw2/uedhyMykp033GgOKTrvfMJfnIfH0KGuLk1+hWEYTI2dSlxAHLck30JORQ7Tv5zOI+Mf4cSoE11dnoiIiHQDOrouIuJi7nYrk2NDePCMwSy5PZGk2ybzwOmDmNg/CIfVQkFpHe+uzOOyt9eS8PB3zHhzDW+vyCGvpMbVpYuIHBaeo0cfGFJUVETeRRdT+dVXri5LfqNhwcNYcPoCxvQaQ21zLbcuuZXZ62bT3KqrWUREROTwUkeniEgH0zfIi74T+nL5hL7UNjbzY1YJSemFJKcXsau8jiUZRSzJKOLBL7bRL8iLybEhJMYFM6ZvAG42dXuKSNewf0jRrltvpWbJUnbNupWGrB0EXT8Tw6Lf1Xd0gR6BvHLCKzy/4Xne2voWb219iy0lW3h60tMEegS6ujwRERHpohR0ioh0YJ4OG8cP6snxg3ridDrJLKwmKa2QpPRC1uWWkV1cQ3ZxDm+uyMHDbt4DmhgXzOTYEML8PVxdvshvs28rBMeBRUG9HMzq7U3Eiy9S+MyzlL71FsUvvkjDjh30fuJxLB76GtfR2Sw2Zo2axZCgIdy/4n7W7l3L1IVTmT15NvHB8a4uT0RERLogBZ0iIp2EYRgM6OnDgJ4+XH1MNFX1TazIKiYpzbzbs7Cqge+37+P77fsAGNDTu32g0aioHtit6oCSDqhqH7x0NLj5QeTREDUB+k6EnkNBXXtC25CiO+8whxQ9+BBV335LXkEB4S/Ow96rl6vLk9/gxKgTifGP4ebkm8mpyOHSby7l7jF3c/6A8zVsT0RERA4pBZ0iIp2Uj7udk4eEcvKQUJxOJ9v2VJoDjdIK2ZBfRsa+ajL2VfPK0mx83GxM6B9EYmwIx8QG09PX3dXli5hKssyQs6ECMr42F4C7vxl6Rk2AqIkQMkjBZzfnf+65B4YUbdtG7vlTCZ83F49hw1xdmvwG/fz78eFpH3L/ivtZlLeIR1Y9wqaiTdx/1P242/RvkoiIiBwaCjpFRLoAwzAY3NuPwb39mJkYQ3ltI0szi0lOL2RJehElNY18vWUvX2/ZC8Dg3r5t3Z7BJET4Y1O3p7hK1Hi4Mwf2bILc5ZC7DPJWQn05pC00F4BnIESOh76TzPAzOA7UCdbteI4aRdTHC9h57XU0ZGaSd/ElhD72d/xOO83Vpclv4GX34tljnuXtrW8zZ8Mc/m/H/5FZlsnsybMJ9wl3dXkiIiLSBSjoFBHpgvw9HZwR35sz4nvT2upk864KktILSUovInVnOVt3V7J1dyVzk7Lw87AzaUAwibHBHDMgmEBvN1eXL92NxQphI8w1/kZoaYY9KZCz1Aw/81dBbQls/z9zAXgFH+j2jJoIQf0VfHYTjvBwIj/8gN233U51cjK7b72Nxh07CLr+eg0p6gQMw+CyIZcxKHAQty+5ne2l25m2cBpPTnqSCWETXF2eiIiIdHIKOkVEujiLxSA+wp/4CH9uPn4AxdUNLM0oIim9iKUZRVTUNfHFpt18sWk3hgHDwv1JjA0mMTaEoWF+WCwKj+QIs9ogfJS5Js6ClibYtQFyl0LOMihYDTVFsPXf5gLw7nXgfs+oiRDQT8FnF2b19iZ83lwKZ8+m9I03KX7xJRqy2oYUeXq6ujz5DcaGjmXBlAXMSp7F5uLNXPf9dVyXcB1XDbsKi6HAWkRERP4YBZ2Hwbx585g3bx4tLS2uLkVE5L8EebtxzohwzhkRTnNLK5t2lrcPNNq6u5JNBeVsKihnzveZBHo5OGZAMJPjQpjUPwh/T4ery5fuyGqHPmPNNel2aG6AXevN0DN3GRSsgeq9sOUTcwH49D4QevadCD2iXLoFOfQMq5Wet9+OW3QMe/72N6q++47cnQVEvPiihhR1Er28evH2yW/zxJon+DjjY+alzGNL8RYem/gYvg5fV5cnIiIinZCCzsNg5syZzJw5k8rKSvz8/FxdjojI/2SzWhgZGcDIyABuOymWfZX1LEk3Q89lmcWU1DTy6cZdfLpxFxYDRvTpQWKcebfnoFBfTcsV17C5mRPaI48G7oSmeti51gw9c5aZ/71qN6TONxeAX5+DOz79I1y6BTl0/M85G0dkH3ZefwMN27aTc/75RMydi0d8vKtLk9/AYXXwwLgHGBo0lEdXPcqSnUu4YOEFPDf5OWIDYl1dnoiIiHQyCjpFRKRdT193po6OYOroCBqbW1mfV0ZyeiFJ6YVk7KtmXV4Z6/LKePrbdHr6ujF5QAiJccGMjwnCx93u6vKlu7K7mwFm34mQCDTWws41Bzo+d62HinzY9IG5wOzwjJoAUZPM5/n2duUO5E/yHDmSqI8/Zue11/5kSNFj+J2uIUWdxdn9zyY2IJZbkm6hoKqAi766iL8d/TdO73e6q0sTERGRTkRBp4iI/CyHzcK46EDGRQdy96kD2VVeZ4aeaUWsyCpmX2UD89cVMH9dATaLweioACbHBpMYF0L/EG91e4rrODyh32RzATTWmAON9nd87t4IZbnm2vie+ZiA6LaOz0lmx6dPT9fULn+YIzyMyA8/ZPftt1OdlMTu226jYUcWwTfcoCFFncSgwEHMP30+dy67kx93/8jdy+5mc9Fmbht1G3arfpkmIiIiv05Bp4iI/CZh/h5cODaSC8dGUt/UwtrcUpLSikhOLyS7uIaV2SWszC7h8a/TCPP3MEPP2BCOjgnE06F/bsSFHF4Qc5y5AOorzYFGOUvN8HPPJijdYa4N75iPCRrQNtG9bbK7d7Dr6pffzOrtRfjcFyh67jlKXn+DkpdepjFrB72ffEJDijoJf3d/XjzuRV7c9CKvpr7KB2kfsL10O88e8yzBnvr/oYiIiPwy/eQpIiK/m7vdysT+wUzsH8wDUwaRW1zTdsS9iFXZJewqr+P91fm8vzofh9XC2H4BJMaGkBgXQt8gL1eXL92duy/0P8FcAHXlkL8Scpeb4efezVCcYa51b5iPCR7Ydr/nBIicAF6BLitffplhtRJy2204omPY+8ADVC1aRO6unUTMm4c9NNTV5clvYLVYuWH4DQwJHMI9y+9hY+FGpi6cyjPHPMPIniNdXZ6IiIh0YAo6RUTkT4sK8uLSoL5cOr4vdY0trMouISm9kMVphewsq2NZZjHLMot5eOE2IgM9SYw1Bxod1S8Qd7vV1eVLd+fhD7GnmAugthTyfjSDz9xlsG8LFG0315pXzcf0HPKTjs/x4NHDZeXLz/M/+6z/GFI0lYh5GlLUmST2SeSj0z/i5qSbySrP4opvr+DWUbdy4cALdT2KiIiI/CwFnSIickh5OKwkxpndmw+d4WRHUU37QKM1OaXkldTy9o+5vP1jLu52C0dHB5EYG8zk2BAiAnS0VDoAzwAYeLq5AGpKIG9523Cj5WbguW+LuVa/BBjQa+iB+z0jx4G7n0u3ICbPESPo+/ECCq69joaMDHNI0d//jt8UDbjpLCJ9I3n/1Pd5cOWDfJ3zNU+ufZLU4lQeHPcgnnb9myEiIiIHU9ApIiKHjWEYxIR4ExPizRUT+1Hd0MyKrGKS0827PfdU1LM4zez8hK3EhHiT2Ha356ioABw2DRCRDsArEAadaS6A6iKz0zO3LfgszoC9qeZaORcMC4TGm6Fn30nQ5yhw83HtHroxe1gYkR98wO477qB68WJ23347DVlZBN90o4YUdRKedk+enPgkw4KG8ey6Z/k652syyzKZkziHSN9IV5cnIiIiHYiCThEROWK83WycNLgXJw3uhdPpJH1fFUlpRSSlF7I+r4yswmqyCqt5bVkOXg4r42OCSIwzj7mH+nm4unwRk3cwDDnHXABVew/c75m73BxqtHujuX58Hgwr9B7edsfnRDP4dOiu2iPpwJCiOZS89holr7xCY/YOej/xBBYvfS46A8MwuGjQRQwMHMhtS24jqzyLCxZewOMTH2dyxGRXlyciIiIdhIJOERFxCcMwiOvlS1wvX66dHE1FXRPLM4tJSi8kOb2I4uoGvtu2j++27QMgrpePeSQ+NoQRffyxWdWJJR2ETy8Yep65ACp2td3v2RZ8luXCrnXmWv4cWGwQNrKt43MiRIwFu4L8w82wWAi5dRZuMdHsue9+qhZ9T27BRUS8OA97796uLk9+o5E9R7Lg9AXcuuRWNhZu5IbFN3DVsKu4Lv46rBbd+SwiItLdKegUEZEOwc/DzmnDQjltWCitrU627q5sCz0L2VhQTtreKtL2VvFS8g583G1MGhDM5AHBHBMbTIiPu6vLFznALwzip5kLoDy/reOz7bh7RQEUrDbXsmfA6oCwUQc6PsNHg12v6cPF78wzsUf0YecNN9CQlkbO1GmEv/A8nsOHu7o0+Y2CPYN548Q3eHb9s7y//X1eTX2VLcVbeHLik/i7+7u6PBEREXEhBZ0iItLhWCwGQ8P9GBrux43H9ae0ppFlmUUkpRWyJKOIstomvkzdw5epewAYGuZnDjSKCyE+3B+rRdN4pQPx7wMJfzGX02l2eO6f6J6zDKp2Q/6P5lryJFjdIGLMgY7PsFFgc7h6F12K54jh9F0wn4LrZtKQnk7+jEsJffQR/M44w9WlyW9kt9q5a8xdDAkawkM/PsSPu39k2sJpPJf4HIMCB7m6PBEREXERBZ0iItLhBXg5ODMhjDMTwmhpdbJpZznJaYUkpRexeVdF+3p+cRY9PO0cMyCYxLgQJvUPpoeXAiLpQAwDAvqaa8TFZvBZmn0g9MxdBtX7Dgw7SgZsHtBnLERNgKhJEDYCrHZX76TTs4eFEfXB++y6806qv/+B3XfcSUPWDoJvvklDijqR0/udTn///tySfAsFVQVc/NXF3HfUfZzd/2xXlyYiIiIuoKBTREQ6FavFYESfHozo04NZJ8ZSWFXP0gzzbs+lbd2en6Xs5rOU3VgMSIjwJzE2hMS4EAaF+mJRt6d0JIYBgdHmGnmpGXwWZx481b2mCLKTzQVg9zIHGkVNMKe6hyaAVd/S/REWLy/Cn3+eojn/oOTVVyl59VUasncQ9uSTGlLUicQGxPLR6R9xz7J7WLJzCQ/8+ACpxancPeZuHFb9sktERKQ70XfFIiLSqYX4uHPeyHDOGxlOc0srG/LLSUovJCmtkLS9VWzIL2dDfjnPLsogyNuNybHBJMaGMKF/EH4e6oqTDsYwIHiAuUb/1Qw+i9IOdHvmLoe6Utjxg7kAHD4QOc486h41AULjQUNZfjPDYiFk1i3tQ4qqv/+B3As1pKiz8XX48vyxz/Nq6qu8mPIin2R8QnppOrMnz6aXVy9XlyciIiJHiIJOERHpMmxWC2P6BjCmbwB3nhzHnoo6ktOLSE4vZHlmMcXVDXyyfiefrN+J1WIwMrJHW7dnMLE9fTAMdXtKB2MYEDLQXGOvgtZWKNx2IPTMXQ715ZD5nbkA3Pwg8ugDw416DgEdxf5VfmecgT0igp3Xtw0pOn8q4XNf0JCiTsRiWLgm/hqGBA3hzqV3srl4M1O/mMrTxzzN2NCxri5PREREjgAFnSIi0mWF+nkwfUwfpo/pQ2NzK+tyS81uz/QisgqrWZNTypqcUp78Jo1QP3cmxwYzOTaE8TFBeLvpn0jpgCwW6DXEXEddC60tsG9LW8fncshbAQ0VkPG1uQDc/dvu92wbbhQ8UMHn/+A5fDh9P15gDilKSyP/khnmkKIzz3R1afI7TAibwPzT5zMreRbbS7dz1aKruGnETVw2+DL9QktERKSL009xIiLSLThsFo6OCeLomCDuPQ0KSmtJbgs9f9xRzJ6Kej5cU8CHawqwWw3G9A0gMTaEybEhRAd76Ydj6ZgsVvOoemg8HH29GXzu2XRguFH+SrPjM22huQA8AyFyvHm/Z9RECI41O0cFAHvv3kS9/96BIUV33kVDVhbBt9yiIUWdSLhPOO+e8i6PrnqUz3d8znPrn2Nz0WYeGf8I3g5vV5cnIiIih4mCThER6ZYiAjy5eFwUF4+Lor6phVXZJSSnF5GUXkheSS0rskpYkVXCo19uJyLAwzziHhvCUf0C8XDo/kPpoCxWcyp72AgYfxO0NJnBZ85SM/zMXwW1JbD9/8wF4BX8k47PSRAY0+2Dz/YhRf94npJXXqHktddpyM4h7CkNKepM3G3uPDL+EYYFD+PxNY/zff737KjYwZzJc+jn38/V5YmIiMhhoKBTRES6PXe7lclt3ZsPMpic4hqS0gpJSi9kdXYpBaV1vLsyj3dX5uFms3BUv0ASY4NJjAshMlChh3RgVjuEjzLXxFnQ3Ai7N0LuUrPjs2C1OdV967/NBeDdq22ie9sdnwH9umXwaVgshNxyszmk6N77qP7hB3L/cqE5pCgszNXlyW9kGAZTY6cSGxDLrORZ5FTkMP3L6Twy/hFOjDrR1eWJiIjIIaagU0RE5D/0DfKi74S+XD6hL7WNzfyYVUJSeiHJ6UXsKq9jSUYRSzKKePCLbfQL8mJy20CjMX0DcLOp21M6MJsD+ow116TbobkBdq0/MNW9YA1U74Utn5gLwDfs4Ds+vbtXyOc3ZQqOiAgKrr+BhvR0cqZOI/yFF/AcoSFFnUl8cDwLTl/A7UtvZ+3etdy65FYuK76Ma4de6+rSRERE5BBS0HkYzJs3j3nz5tHS0uLqUkRE5E/ydNg4flBPjh/UE6fTSWZhtXm3Z1oRa3NLyS6uIbs4hzdX5ODpsHJ0dBCJceZQozB/D1eXL/LLbG7mhPbIo4E7oakedq49cMfnzrVQuQtS55sLsPlFMNwahZFaCdGTwT/CpVs4EjwSEg4MKdq+nfwZM+j1yMP4n3WWq0uT3yHQI5BXT3iV5zc8z1tb3+KtrW+xuXgzx7ce7+rSRERE5BBR0HkYzJw5k5kzZ1JZWYmfn5+ryxERkUPEMAwG9PRhQE8frpoUTVV9EyuyiklKM+/2LKxq4Pvt+/h++z4ABvT0JjE2hIkxAbS0urh4kd/C7m52bfadCIlAY615vD13uRl+7lqPUVFAHwrgi2Xmc3pEHbjfM2oC+PZ25Q4OG3toKFHvv8fuO++iatEi9tx1N437hxRZ1cndWdgsNmaNmsWQoCHcv+J+1u1bR4aRwcDigYwIHeHq8kRERORPUtApIiLyB/m42zl5SCgnDwnF6XSybU+lOdAorZAN+WVk7KsmY181ryzNxt1q5duqFI4b2ItjYoPp6evu6vJFfp3DE6ITzQXQUE1zzgqyk94lxroHy54UKMs118Z/mo8JiD5wv2fURPDp6aLiDz2Lpydh/5hD0QsvUPLSy5S8/gYNO7Lp/fTTWL11X29ncmLUicT4x3BT0k3kVuby1+//yt1j7ub8AedjdMM7aUVERLoKBZ0iIiKHgGEYDO7tx+DefsxMjKG8tpFlmcVtd3sWUlrTxLfbCvl2WyEAg3v7khgbwuTYYBIi/LFZLS7egchv4OaNM/pYtqfX0/fUU7G01JmT3HOXml2fezZB6Q5zrX/bfE7QgAP3e0ZOAO9gl27hzzIsFkJuugm3ftHsufdeqpOSyPvLXwh/8UUc4d3r/tLOrp9/P9496V2u+ewatjVt45FVj5BalMp9R92Hu02/jBIREemMFHSKiIgcBv6eDqbE92ZKfG8aGhp55ZOvaQ6KZUlWCak7y9m6u5KtuyuZm5SFn4edSQOCSYwN5pgBwQR6u7m6fJHfxt0XBpxoLoC6cshf2TbcaCns3QLFGeZa94b5mOCBP+n4nACeAS4r/8/wm3I6jj4RFFx/PQ0ZGeROnUr4C8/jOXKkq0uT38Hb7s10z+kU9S1i7qa5fL7jczLKMpg9eTbhPuGuLk9ERER+JwWdIiIih5nFYhDpDaceG82sk+Iorm5gaUYRSelFLM0ooqKuiS827eaLTbsxDBgW7k9ibDCJsSEMDfPDYtExSukkPPwh9hRzAdSWQt6PB4YbFW6Fou3mWvOq+ZieQ37S8Xk0ePRwWfm/l0d8PH0//piC666jYdt28i69jNCHHsL/nLNdXZr8DoZhcOmgSxkaMpQ7ltzB9tLtTFs4jacmPcX4sPGuLk9ERER+BwWdIiIiR1iQtxvnjAjnnBHhNLe0smlneftAo627K9lUUM6mgnLmfJ9JoJeDY9pCz0n9g/HztLu6fJHfzjMABp5uLoCaEshb3tbxuQyK0mDfFnOtfgkwoNfQtsFGEyFyHLh37MGO9l69iHrvPXbfdTdV333HnnvuoWFHFiGzZmlIUSdzVOhRzD99PrOSZ7GlZAvXfn8tMxNmcuWwK7EYul5ERESkM1DQKSIi4kI2q4WRkQGMjAzgtpNi2VdZz5J0M/RclllMSU0jn27YxacbdmExYESfHiTGmXd7Dgr11dAM6Vy8AmHQmeYCqC48MNE9ZxmUZMLeVHOtnAuGBUITzCPufSdBn6PAzcelW/g5Fk9PwuY8R/HcuRS/+BKlb7xJ445sej/zjIYUdTKh3qG8c8o7PLHmCT7O+Ji5KXPZUryFv0/8O74OX1eXJyIiIr9CQaeIiEgH0tPXnamjI5g6OoKmllbW5ZaRnF5IcnoR6fuqWJdXxrq8Mp7+Np2evm5MHhBCYlww42OC8HFXt6d0Mt4hMOQccwFU7TWDz5ylZvhZmg27N5jrx+fBsELYCDP4jJpoBp+OjhEkGhYLwTfeiCM6mj333Et1cjJ506cT/tJLGlLUyTisDh4Y9wBDg4by6KpHSd6ZzPSF03ku8TkG9Bjg6vJERETkFyjoFBER6aDsVgvjogMZFx3I3acOZFd5HcnphSSlFbEiq5h9lQ3MX1fA/HUF2CwGo6MCmBwbTGJcCP1DvNXtKZ2PTy8Yep65ACp2tXV8LjU7PsvzYOdacy1/Dix2CBvZNtxoAkSMBbuHS7fgd9ppOCIi2DnzehoyM8k9/3zC576gIUWd0Nn9z2ZAwABmJc0ivyqfC7+8kAePfpDT+p3m6tJERETkf1DQKSIi0kmE+Xtw4dhILhwbSUNzC2tySklKKyI5vZDs4hpWZpewMruEx79OI8zfwww9Y0M4OiYQT4f+yZdOyC8M4qeZC6A8v+1+z7bj7hUFULDKXEufBqsDwkcfmOgePhrs7ke8bI9hw4j6eAE7r5tJ/bZt5pCiBx/E/9xzjngt8ucMDhzM/NPnc8fSO1i5ZyV3LbuLzcWbuXXUrdgt6qIXERHpaPRTj4iISCfkZrMysX8wE/sH88CUQeSV1JDcdrfnyh0l7Cqv4/3V+by/Oh+H1cLYfgEkxoaQGBdC36COcdRX5Hfz7wPDLzSX0wlluWbgmds24KhqN+StMNcSwOZuhp37hxuFjQSb44iUau/Vi8j324YUffste+69l4asLEJuu1VDijoZf3d/Xjr+JealzOO1za/x/vb32VayjWePeZZgz2BXlyciIiI/oaBTRESkC4gM9GLG0V7MODqKusYWVmWXkJReyOK0QnaW1bEss5hlmcU8vHAbUYGeTI41Bxod1S8Qd7tCF+mEDAMC+pprxCVm8FmafeB+z9zlUL2v7b8vM59j84A+Y83Qs+8k6D0crIevK8/i4UHYc7MpnjuP4hdfpPStt2jMzqb3s89g9fY+bB9XDj2rxcqNI25kaNBQ7ll+DxsLNzJ14VSePeZZRvQc4eryREREpI2CThERkS7Gw2ElMc7s3nzoDCc7imrMuz3TC1mTU0puSS1v/5jL2z/m4m63cHR0ENNGR3DS4F6uLl3kjzMMCIw216jLzOCzOPPA/Z65y6G2GLKTzQVg9zIHGvWdCFGTIDQerIf222NzSNENuMVEs/vue6hessQcUvTiizgiIg7px5LDL7FPIh+d/hE3J91MVnkWf/32r9w2+jb+EvcX3YssIiLSASjoFBER6cIMwyAmxJuYEG+umNiP6oZmVmQVk5xu3u25p6KexWlm5+fDZw7mknFRri5Z5NAwDAgeYK7RV5jBZ1FaW+i5FHJXQF0p7PjBXAAOH4gc19bxORF6DQPLoel49j31VOwREey8biYNmVnkTp1G+AvP4zlq1CF5/3LkRPpG8v6p7/Pgjw/yde7XPLHmCVKLUvnbuL/hafd0dXkiIiLdmoJOERGRbsTbzcZJg3tx0uBeOJ1O0vdV8c+Veby/Op8HPt8KoLBTuibDgJCB5hp7FbS2QuE281h7zjLIWw71FZD5nbkA3Pwg8ui2js+J0HMIWCx/uASPoUOJ+uRjc0jR1q3kXXY5oQ/+Df9zzz1Em5QjxdPuyZOTnmRY8DCeWfcMX+V8RUZZBnMS5xDpG+nq8kRERLotBZ0iIiLdlGEYxPXy5dGzhuDtbuOVJdk88PlWDOBihZ3S1Vks0GuIuY66FlpbYN+Wto7PZZD3IzRUQMbX5gLw6AGR4w90fAYP/N3Bp71nTyLf+ye777mHqq+/Yc+999GQmUXI7bdpSFEnYxgGFw26iIGBA7k1+VayyrOYvnA6j018jMkRk11dnoiISLekoFNERKSbMwyDu06OAye8sjSb+z/fCobBxUepK0m6EYvVvKMzNB6Ovh5ammFv6oGOz/yVUFcGaQvNBeAZCFETzOAzaiIEx5qdo7/2oTw8CJs9m+LoGIrnzqX07bdpyMkm7NlnNaSoExrZcyQLpizgtiW3sbFwIzcsvoGrhl3FdfHXYT1EVx+IiIjIb6OgU0RERMyw85Q4nMCrS7O5/7MtGMBFCjulu7LaIGyEucbfBC1NsDvlwBT3/FVQWwLbPjcXgFdIW/A5wZzqHhjzP4NPwzAIvn6mOaTorrupWbKU3AsuIOKllzSkqBMK8QzhjRPf4Jl1z/BB2ge8mvoqW4u38sTEJ/B393d1eSIiIt2Ggk4REREBzODl7lPicDqdvLYsh/s+24JhwIVjFXaKYLVDxGhzTZwFzY2we8OBjs+C1VBTCFs/NReAd68D93tGTYCAfv8VfPqefDL2sHB2zpxJY9YOcs+fStjz/8BrzBgXbFL+DLvVzt1j72ZI0BAeXvkwK3av4IIvL2D25NkMChzk6vJERES6hT9+m7qIiIh0OYZhcM+pA7lyYl8A7v33Fj5Yne/iqkQ6IJsD+hwFk26HGf8Hd+XDpV/B5LvNYNPqBtV7YfPH8MWN8MIIeG4wfHo1bHwPyvLa35XH0CFEffwx7kOG0FJeTv7lf6Xs449duDn5M6ZET+G9U98jwieCXdW7uPiri/l35r9dXZaIiEi3oI5OEREROcj+sNPphNeX53DPvzcD8JexfVxcmUgHZnODqPHmAmiqg51r24YbLTf/e+UuSP3IXAB+fdo7Pu19JxL5z3fZc++9VH71NXvvf4DGrCxC7rhDQ4o6odiAWD487UPuXX4vS3Yu4YEfH2Bz8WbuGnMXDqvD1eWJiIh0WQo6RURE5L8YhsG9pw3ECbzRFnYaBkwfo7BT5Dexe5j3dPadZL7dWGseb89tCz53rYeKfEh531yApUcUvSdNwOGdSPGCJErfeZeG7BzCZj+L1cfHhZuRP8LPzY/nj32eV1Nf5cWUF/k442PSStOYPXk2vbx6ubo8ERGRLklH10VERORnGYbBfacN5PLx5jH2uz/dzEdrdIxd5A9xeEJ0Ihz3APz1O7gzDy76F4y/GcJGgWGFslyMlPcItrxP2NGlGDaoWbaM3LNOo3H7BlfvQP4Ai2HhmvhrmHfcPHwdvmwu3sy0hdNYvWe1q0sTERHpktTRKSIiIv+TYRjcf/pAnDh5a0Uud31qdnZOG63OTpE/xc0bYo43F0B9pTnJPXcp5CzD10jF7l3EzmUBNO4qInfadMJO9cJrwuQDA468gly6BfntJoZP5KPTP2JW8izSStO4atFV3DziZi4dfCnGfwyoEhERkT9OQaeIiIj8IsMweOB0c2Jwe9iJwdTRES6uTKQLcfeFASeaC6CuHI/8lUSN/46dLy6ifh/kf15Lr10f0iP6DfMxIYPMae77p7p7BriufvlVET4R/POUf/LIqkf4vx3/x+z1s9lcvJlHxj+Cl93L1eWJiIh0CTq6fhjMmzePQYMGMXr0aFeXIiIickjsDzsvPToKpxPu/DSVBWsLXF2WSNfl4Q+xp2Cf9hyR367D96TjwWmwd60/e9P64mwFCrfBmldhwcXwVD94aQJ8czekfQl1Za7egfwMd5s7j45/lPvG3ofNYmNR3iKmfzmd7PJsV5cmIiLSJSjoPAxmzpzJtm3bWLt2ratLEREROWQMw+BvU/4j7FynsFPkcLO4u9N7zvME3XgDAGUpDRQUnEXLaa/C6CshOA5wwr7NsOpF+Ogv8GRfeGUSfHsvpH8D9RWu3YS0MwyDaXHTePvktwnxDCGnIofpX07nu9zvXF2aiIhIp6ej6yIiIvKb7Q87nU4n76zM485/pWIA54/SMXaRw8kwDIKvuw636Bh233UXNSvXkLuvmIiXXsRx2jNQXXhgonvOMijJhD2bzLVyLhgWCE1ou99zEvQZC26a5O5K8cHxzD99PncsvYO1e9dy65Jbuaz4Mm4ccSM2i35MExER+SP0L6iIiIj8LoZh8OAZg3EC767M445/pWIYBueNDHd1aSJdnu9JJ2IPD2PndTNpzM4md+o0wv7xD7yOGgtDzjUXQOUeM/TMXWau0mzYvcFcK/5hTnkPG2He79l3IkSMBYfuiTzSgjyCePWEV/nHhn/w9ta3eWvrW2wt2cpTk54i0CPQ1eWJiIh0Ojq6LiIiIr+bYRg8dMZgLj4qEqcTbv9kE5+s3+nqskS6BY/Bg4n6eAHuw4bRUlFB/hVXUDZ/wcEP8g2FYefDGc/DjRvhlq1w9isw/CLwjwRnC+xcC8tnwz/Phici4Y2TYPGjkL0Emupcs7luyGaxceuoW3nmmGfwsHmwZu8api2cRmpRqqtLExER6XTU0SkiIiJ/iGEYPHzmYAD+uSqP2z/ZhAGcq85OkcPOHhJC5LvvsOfe+6j88kv2/u1vNGRl0fPOOzBsP/Mtvl84xF9gLoDyfPOIe+4y8z8rd0LBKnMtfRqsDggffaDjM3w02NyO7Ca7mZOiTiLGP4abk24mtzKXS7+5lLvG3MX5A87HMAxXlyciItIpKOgUERGRP2x/2OnEyXur8rntk00YBpwzQmGnyOFmcXen9zNP49Y/hqI5/6Dsn/+kMSeHsNnPYvX1/eUn+/eB4Reay+mEstwDoWfuMqjaA3krzLXkCbC5m2Fn30lm+Bk2EmyOI7LP7iTaP5oPT/uQ+1bcxw/5P/DIqkdILUrlvqPuw93m7uryREREOjwFnSIiIvKnGIbBw2cMwemE91fnc+vHZth59nCFnSKHm2EYBF1zDY5+/dh9513ULF9O7rQLiHj5JRyRkb/1nUBAX3ONuMQMPkuzIWfpgfCzpvDAfZ8ANg9zoFHURDP87D0crPbDt9FuxNvhzXOTn+PNLW/y/Mbn+XzH52SUZTB78mzCffR1VURE5Jco6BQREZE/zWIxeOTMITiBD1bnc+uCTYDCTpEjxffEE3GEh1Nw3Uwac3LImTqN8H/Mweuoo37/OzMMCIw216jLzOCzOBNyl7Z1fC6H2mLITjYXgN0LIsdB1ARzqntoPFj1o8YfZRgGfx36VwYHDeaOJXewvXQ7F3x5AU9OfJLxYeNdXZ6IiEiHpWFEIiIickhYLAaPnjmE6WP60OqEWxds4rONu1xdlki34T5oEH0/XoB7/DBaKyrIv+JKyj766M+/Y8OA4AEw+gqY+g7cngXXrYJTnoaBU8CjBzTVQNb38P2D8Pqx8FRfeH8q/PgC7N4IrS1/vo5u6KjQo5h/+nyGBA6hoqGCa7+/llc2vUKrs9XVpYmIiHRI+jWriIiIHDIWi8HfzxoCOPlwTQGzFqRgGHBmQpirSxPpFmzBwUS++y577rufyi++YO+DD9GQmUXPu+/6+SFFf4RhQMhAc429ClpboXCr2emZswzylkN9BWR+ay4Adz+IHG8edY+aAD2HgEU9F79FqHcob5/yNk+seYJPMj5hbspcthRv4e8T/46v41fuYhUREelmFHSKiIjIIWWGnUNxOuGjtQXcMj8FUNgpcqRY3Nzo/dSTuEVHUzRnDmXvv28OKXpuNlY/v8PwAS3Qa6i5jrrW7N7cu9kMPnOXQd6PZvCZ/pW5wOwCjRx/YLhRcJyCz1/gZnXjb+P+xrCgYTy66lGSdyYzfeF0nkt8jgE9Bri6PBERkQ5DQaeIiIgcchaLwWNnDwUUdoq4gjmk6Goc0f3Yfced1Pz444EhRVFRh/eDW6zQO8FcR18PLc2wd9OB+z3zV0JdGaQtNBeAZ2Db/Z5tw42CBpido3KQs/ufzYAeA7gl+Rbyq/K58MsLefDoBzmt32muLk1ERKRDUNApIiIih8X+sNPphPnrzLDTMAzOiO/t6tJEug3fE07A8UHbkKLcXHKmXUD4nOfwGjfuyBVhtUHYSHNNuBlammB3yoHhRgWrobYEtn1uLgCvEDP47DvRHG4UGK3gs83goMHMP30+dy69k5V7VnLXsrvYXLyZW0fdit1id3V5IiIiLqXzISIiInLYWCwGj58zlKmjwml1ws0fbeSLTbtdXZZIt+I+cCB9F8zHIz7+wJCiDz90XUFWO0SMhom3wiWfwZ15cPm3kHif2c1pc4eaQtj6KSy8BeaOhNkD4V9XwPp3oDTbnATfjfVw78FLx7/ElUOvBOD97e9zxbdXUFRb5OLKREREXEtBp4iIiBxWFovBE+cMOxB2zk9hYarCTpEjyRYcTJ9338H3jCnQ0sLehx5m78OP4GxudnVpYHNAn6PgmNthxhdwVz5c+hVMvhsiJ4DVAVV7YPPH8MWN8PxweG4IfHo1bHwPyvJcvQOXsFqs3DjiRv6R+A+87d5sKNzA1IVT2bBvg6tLExERcRkFnSIiInLY7Q87zx8ZTkurk5s+SuHL1D2uLkukW7G4udH7yScJnjULDIOyDz6g4KqraKmocHVpB7O5QdR4mHwXXPalGXzO+AIm3QF9xoHFDpU7IfUj+Hwm/GMYzBkKn82ElA+hYqerd3BEHdvnWD487UNi/GMorivmr9/+lfe3v4+zm3e9iohI96Q7OkVEROSIsFgMnjx3GE7gk/U7ufGjjQCcNizUtYWJdCOGYRB01ZW4Rfdj1+13UPPjSnKnXUD4Sy/i1revq8v7eXYP80h730nm24215r2eucvMOz53b4DyfEh5z1wAPfq23e/Ztny79teZKL8o3j/1ff7249/4JvcbnljzBKlFqfxt3N/wtHu6ujwREZEjRkGniIiIHDHtYacT/rXBDDsNA04d2rVDCJGOxue444j64H0KrruOxtxcM+yc8xxeRx/t6tJ+ncMTohPNBdBQDQWr2qa6L4PdG6Esx1wb3jUfExjTFnq2TXb36em6+g8TT7snT016imHBw3h23bN8lfMVmeWZzJk8hz6+fVxdnoiIyBGhoFNERESOKKvF4KnzhuHEyacbdnHDh2Znp8JOkSPLPS6OvgsWsPP6G6hLSSH/yqvoee89BPzlL64u7fdx84aY480FUF8J+SsPdHzuTYWSLHOtf8t8TFAs9J2IEXE0jqZa19V+iBmGwcWDLmZgwEBuW3IbmWWZXLDwAh6b+BiTIya7ujwREZHDTnd0ioiIyBFntRg8fV485wwPo6XVyQ0fbuTrzbqzU+RIswUF0eedt/E78wxoaWHfw4+w9+GHcTY1ubq0P87dFwacBCc+ClcvgTty4IIP4aiZ0GsoYEBxOqx9Hdunl3PKluuxvToRvroDtn8BtaWu3sGfNqrXKBZMWUBCcAJVTVXcsPgGXtj4Ai2tLa4uTURE5LBSR6eIiIi4hNVi8PT58QB8utHs7JxrwMlD1NkpciRZ3NwIfeIJHDExFM1+jrIPPqQhJ4fwOXOw+vm5urw/z8Mf4k41F5hBZt4KyFmGM3cZRuE2jKLtULQd1rwCGNBzyIE7PiOPNt9HJxPiGcKbJ73J0+ue5sO0D3k19VW2Fm/liYlP4O/u7+ryREREDgt1dIqIiIjL7A87zx4eRnOrk+s/2Mg3W/a6uiyRbscwDIKuvJLweXMxPD2pXbmK3KnTaMjOcXVph55nAAycAqc+RfOVS/l6yFyaz3kTRl8BwXGAE/ZthlUvwkfT4ckoeGUSfHsvZHxrHo3vJOxWO/eMvYfHJjyGu9WdFbtXcMGXF7CtZJurSxMRETksFHSKiIiIS1ktBs+cH89ZCb3bws4NCjtFXMTn2GOJ+vADbL1DaczLI/eCC6hescLVZR1WjXZfnAPPgNOehZmr4bZMOO9NGHkZBPYHnLBnE6ycCx9MhScj4dVEWPQAZH5vDkPq4KZET+G9U98j3DucXdW7uOTrS/gs6zNXlyUiInLIKegUERERl7NaDJ6dmnBQ2PntVoWdIq7gHhtL348/xmP4cForKym46mpK33/f1WUdOd4hMORcmDIHblgHs9LgnNdhxCUQ0A+crbB7A6z4B7x/rhl8vn4CfP8Q7FgMjR1zuFFsQCwfnf4Rk8In0dDSwP0r7ufhlQ/T2NLo6tJEREQOGQWdIiIi0iHsDzvPbAs7Z76/ge8Udoq4hC0wsG1I0ZnmkKJHHmXPQw917iFFf5RvKAw7H854AW7cCLdshbNfgYSLwL8PtDbDzjWwfDb882x4og+8eTIsfhSyl0BTnat30M7PzY8Xjn2B6xKuw8Dg44yPueyby9hbo6+1IiLSNSjoFBERkQ7DajF49vx4zohvCzs/2MCibftcXZZIt2RxOAh94nFCbr8NDIPyDz8i/8qraCkvd3VpruUXDvEXwFnz4ObNcFMqnPkixE8H33BobYL8lbD0aXj3DDP4fOs0SHoccpdDc4NLy7cYFq6Nv5Z5x83Dx+FDanEq0xZOY/We1S6tS0RE5FBQ0CkiIiIdis1qYfbUeKbE96apxcl1769X2CniIoZhEPjXvxI+by4WT09qV60iZ9o0GrKzXV1ax9EjEoZfCGe/DLdsMbs+z3gBhk4Fn1BoaYS85bDkCXj7NDP4fGcKLHka8lZCs2uOjk8Mn8j80+cTFxBHaX0pVy26ire2vIXT6XRJPSIiIoeCgk4RERHpcGxWC8/9R9j5vcJOEZfxOfZYIj/8EHtYGE15+eROu4Dq5V17SNEfYhjmPZ4jLoFzX4NZ2+H69XD6HPPeT68QaK6HnKWQ9Ci8dbJ5x+e7Z8GyZ6FgDbQcuesBInwiePeUdzkj+gxana3MXj+bW5fcSk1TzRGrQURE5FBS0CkiIiId0v6w8/RhoTS1OLlWYaeIS7nHDiBqwXw8RoygtaqKgquuovSf76kD8JcYBgTFwKjLzEnut2XAzDVw6jMw6CzwDIKmWshOgh8ehjdOgCej4L1zYfkc2LUeWpoPa4keNg8eHf8o9429D5vFxqK8RUz/cjrZFeraFRGRzkdBp4iIiHRYNquFOdMSOO0nYecP2xV2iriKLTCQPm+/hd/ZZ0NrK/v+/nf2PthNhxT9EYYBwbEw5kqY+g7cngXXroRTnoKBU8CjBzRWQ9b38P3f4LVj4am+8P5U+PEF2J0CrS2HoSyDaXHTeOuktwjxCCGnIofpC6ezKG/RIf9YIiIih5OCzsNg3rx5DBo0iNGjR7u6FBERkU7PZrXwj2kJnDa0Lex8bwOL0xR2iriKxeEg9LG/E3L77eaQovnzyb/iSprLylxdWudjGNBzEIy9Gqa9B7dnwzXL4aTHIfY0cPeDhkrI/Ba+uw9ePcYMPj+cDitfhL2bobX1kJWTEJLA/CnzGdVzFLXNtcxKnsXsdbNpbj28XaUiIiKHioLOw2DmzJls27aNtWvXuroUERGRLsFmtTDnggROHdqLxpZWrvnnBpLSCl1dlki3ZQ4pupzwF+eZQ4pWryZ32gU07Njh6tI6N4sFeg2FcdfB9A/gjhy4agmc+CgMOBncfKG+AtK/gm/vhpcnwNP94KMLYfUrsG8b/MmrBII8gnjtxNeYMWgGAG9tfYurF11NSV3JodihiIjIYaWgU0RERDoFu9XCPy4Y3h52Xv3P9Qo7RVzMJzGRyI/ahhTltw0pWrbc1WV1HRYr9E6Ao2+Av8w3g88rF8PxD0HM8WD3groySFsIX98BL42Dp2NgwSWw5jUoSv9DwafNYuO20bfx9DFP42HzYM3eNUxbOI3UotRDv0cREZFDSEGniIiIdBr7w85Thvwk7ExX2CniSu4DBhD18QI8Ro2ktbqagquvpvTddzWk6HCw2iBsJEy4GS76F9yVB3/9Ho57APolgt0Taoth2+fw1W0wbww8MwA+vgzWvQnFWb8r+Dw56mQ+PO1Donyj2Fe7j0u/uZQF6Qv0uRURkQ5LQaeIiIh0KnarheenHxx2JivsFHEpW0AAkW++id8555hDih57nL1/e1BDig43qx0iRsPEW+GSz+DOPLj8W0i8D/pOAps71BTC1k9h4S0wdyTMHgj/uhLWvwOl2b8afEb7R/PhaR9yXJ/jaGpt4pFVj/DAjw9Q31x/ZPYoIiLyOyjoFBERkU5nf9h58uBeNDa3cpXCThGXMxwOQv/+KCF33GEOKVqwgPy/XqEhRUeSzQF9joJjbocZX8Bd+XDpVzD5boicAFYHVO2BzQvgixvh+eHw3BD49zWw8X0oy/vZd+vt8Oa5yc9x84ibsRgWPsv6jEu+voRd1buO8AZFRER+mYJOERER6ZTsVgsv/GU4Jw3u2R52LskocnVZIt2aYRgEXn4Z4S+9iMXLi9o1azSkyJVsbhA1HibfBZd9aQafM76ASXdAn3FgsUPlTtj0IXx+HfxjGMwZCp/NhE0fQcXO9ndlGAZ/HfpXXj7+ZXq49WB76XamLZzGil0rXLhBERGRgynoFBERkU7LbrXwwvQRnDjIDDuvfHedwk6RDsBn8mSiPvoQe3j4T4YULXN1WWL3MI+0H3svXP6Necfnxf82j76HjwGLDcrzIeU9+PfV8Nxg+EcC/N8NkPoxVO5hXO9xzD99PoMDB1PRUMG131/LK5teodXZ6urdiYiIKOgUERGRzs1hszD3LyM44Sdh51KFnSIu59a//38MKbqG0nfe0SCbjsThBdHHmsOMrlhk3vF54b9g/M3m0CPDAmU5sOFd+PQKmB0HL4wkNOlJ3ulzFudGnYYTJ3NT5nLT4puobKx09Y5ERKSbU9ApIiIinZ7DZmHef4SdyzIVdoq4mq1HD3NI0XnnmkOKHn+CvQ88gLOx0dWlyc9x84b+x8MJD8GVi83g8y8LYNz1EJoAGFCSBevfwu3Tq3kw6SUeqrfjwELyzmSmfzGVjLIMV+9CRES6MQWdIiIi0iXsDzuPH9iThuZWrnhnHcszi11dlki3ZzgchD7yCCF33QkWC+Uff6IhRZ2Fuy8MOAlO+jtcvQTuzIULPoSjroOeQwE4Z88O3t21m9DmZvKrd3HR5+fy1acXwfYvoLbUtfWLiEi3o6BTREREugyHzcKLF47g+IEhNDS38td31irsFOkADMMg8NJLiXj5JXNI0dq15E6dRkNWlqtLk9/Dwx/iToWTH4drl8MdOTDtPQYnXMb8Bj/G1dVRZ8CdVZt4YtENND3VD16eAN/cDWlfQV25q3cgIiJdnIJOERER6VLMsHPkQWHniiyFnSIdgfekSUTN/wh7RARNBQXmkKIlS1xdlvxRngEwcAqc+hQ9rlvFSxet4MpekwB438+HK0KDKSraCqtehI+mw1N94ZVj4Nt7IeNbqNedniIicmgp6BQREZEux2GzMO/CERwXdyDs/FFhp0iH4BYTQ9SC+XiOHk1rTQ0F115Hydtva0hRF2D17smNJ81jTuIcvO3ebHB3Z2q/ODYMOxsCY8DZCntSYOVc+GAqPBkFrx0Li/4Gmd9DQ7WrtyAiIp2cgk4RERHpktxsVl68aATHxoVQ39TK5Qo7RToMW48e9HnjdfzPPw9aWyl84kn23H+/hhR1Ecf1OY4PT/uQaL9oipur+Wv1Jt4/fhbOW7bDOa/DiEugR19wtsCu9bBiDrx/LjwZCa+fAD88DDsWQ2Otq7ciIiKdjIJOERER6bLcbFZe+s+wc4fCTpGOwHA46PXww/S8+y6wWKj45F/kX/5XDSnqIqL8ovjgtA84Oepkmp3NPLHmCe7a9A9qB54GZ7wAN6XALVvh7Fcg4SLw7wOtzbBzDSx7Fv55NjzRB948GRb/HXKWQlOdq7clIiIdnIJOERER6dL2h52JscFm2Pm2wk6RjsIwDAJmzDCHFHl7U7tuHbnnT6UhM9PVpckh4Gn35KlJT3HH6DuwGla+yvmKi76+iPzKfPMBfuEQfwGcNQ9u3gw3pcKZ8yB+OviGQ2sT5K+EpU/BO1PgiUh46zRIfgJyV0Bzg2s3KCIiHY6CThEREenyzLBzJJN/Enau3FHi6rJEpM1BQ4p27iT3gulUJSe7uiw5BAzD4OJBF/P6ia8T6B5IZlkmFyy8gOSC5P9+cI9IGH4RnP0y3LIFbtwIU56HoVPBJxRaGiBvOSQ/Dm+fanZ8vjMFljwN+augWVcfiIh0dwo6RUREpFtwt1t5+aKRHDPgQNi5Klthp0hH4RYdbQ4pGjOG1poadl57HSVvvqUhRV3EqF6jmH/6fOKD46lqquKGxTfwwsYXaGlt+fknGAYE9IORM+Dc12DWdrh+PZz+HAw+B7xCoLnePNKe9Ci8eZJ5x+e7Z5lH3wvWQkvTEd2jiIi4noJOERER6Tbc7VZeudgMO+uaWrjsLYWdIh2JrUcP+rz+Gv7nnw9OJ4VPPcWee+/TkKIuoqdXT9466S0uiL0AgFdTX2XmDzOpaKj49ScbBgTFwKjL4fy34LYMmLkGTn0GBp0JnoHQVAvZSeYwozeON6e6v3cuLJ9jDj1qaT6s+xMREddT0CkiIiLdyv6wc9JPws7VCjtFOgxzSNFD9LznHnNI0aefknf55TSXlrq6NDkE7FY79x51L49NeAx3qzsrdq9g2sJpbCvZ9vvekWFAcCyMuRKmvgu3ZcG1K+GUpyDudPDoAY3VkPU9fP83eO1YeKovfDANfnwBdqfA/+omFRGRTktBp4iIiHQ77nYrr/407Hx7LWtyFKKIdBSGYRBwycVEvPIyFm9v6tatJ/f8qdRnZLi6NDlEpkRP4b1T3yPcO5xd1bu45OtL+Czrsz/+Di0W6DkIxl4NF7wPt2fDNcvhpMch9lRw84OGSsj4Br67D149xgw+P/wLrHwR9m6G1tZDtj8REXENBZ0iIiLSLe0POyf2D6K2sYVL31qjsFOkg/GeONEcUtSnD027dpF3wXSqkpJcXZYcIrEBsXx0+kdMDJtIQ0sD96+4n0dWPkJjyyG4qsBigV5DYdx1MP1DuDMHrloCJz4K/U8Chw/UV0D6l/Dt3fDyBHg6GuZfBKtfgX3bQPfDioh0Ogo6RUREpNtyt1t57ZJRB4Wda3MVdop0JG7R0UTN/wjPsWNpra1l53UzKXnjTQ0p6iL83PyYe9xcrou/DgODBRkLuOyby9hbs/fQfiCLFXonwNE3wIUL4M5cuGIxHP8QxBwPdi+oK4XtX8DXd8BL4+DpGFgwA9a+DkXpCj5FRDoBBZ0iIiLSrf1X2Pmmwk6RjqZ9SNG0aeaQoqefZs+999GqIUVdgsWwcG3Ctcw9bi4+Dh9Si1OZtnAaa/asOXwf1GqD8JEw4Wa46F9wVx78dREc9wD0SwSbB9QWw7bP4MtbYd4YeDYWPrkc1r0FxVkKPkVEOiAFnSIiItLt7Q87J8QEUdMWdq5T2CnSoRh2O70e/Bs97723fUhR/mUaUtSVTAqfxPzT5xPbI5bS+lKuXHQlb21568h071rtEDEGJt4Kl3wGd+XDZd9A4n3QdxLY3KF6H2z5Fyy8GeaOhNmD4F9XwoZ3oTRHwaeISAegoFNERESEA2Hn+JhAahpbmKGwU6TDMQyDgIsvIuLVV7H4+FC3fj25551PfbqGFHUVET4R/PPUfzKl3xRana3MXj+bW5fcSk1TzZEtxOaAyHFwzO0w4wu4Mw8u/RKOuQsiJ4DVAVW7YfMC+L8b4PkEmDMU/n0NbHwfyvOPbL0iIgIo6BQRERFp5+Gw8volozk6+kDYuT5PYadIR+M9Ybw5pCiyD027d5M3fTpVizWkqKvwsHnw9wl/596x92Kz2FiUt4jpX04nuyLbdUXZ3SFqAiTeDZd9aXZ8XvJ/MOl26DMOLHaoKIBNH8Ln15mh55xh8NlM2PQRVOxyXe0iIt2Igk4RERGRn/BwWHljxk/DzrWszytzdVki8h/c+vWj7/z5eB51lDmkaOZMyhYscHVZcogYhsEFcRfw1klvEeIRQk5FDtMXTmdR3iJXl2aye0C/Y+DY++Dyb8w7Pi/+N0yYBeFjwGKD8jxIeQ/+fTU8NwieH252f6Z+DFWHeNiSiIgACjpFRERE/sv+sHNcv0CqG5rbOjsVdop0NFZ/f/q89mr7kKKiOf/QNPYuJiEkgflT5jOq5yhqm2uZlTyL2etm09za7OrSDubwguhj4fi/wRWLzKPuF/4Lxt8EvUeAYYHSbPM+z0+vMAcbvTAKvrjZvPezutDVOxAR6RIUdIqIiIj8DA+HlTcuHcVR/QLaw84N+Qo7RToaw26n5913gcVCS2kpLSUlri5JDrEgjyBePfFVLhl0CQBvbX2LaxZdQ0ldB/5cu3lD/+PhhIfhqiS4Mxf+sgDGXQ+h8YABJZmw/i1zkvsz/WHeWPjyNtj6GdR04L2JiHRgCjpFRERE/gdPh403Lx19IOx8Yw0bFXaKdDgWd3ccffoA0JChwURdkd1i5/bRt/P0pKfxsHmweu9qpi2cxuaiza4u7bdx94MBJ8FJf4erl8KdOXDBh3DUddBzqPmYojRY+xp8PAOe7gcvHg1f3wnbv4Ba3RctIvJbKOgUERER+QU/DTurGpq5RGGnSIfk1r8/AA2ZmS6uRA6nk/uezAenfkCUbxT7avcx45sZfJzxcee7ssCjB8SdCic/DtcuhztyYNp7MOZqCBlkPqZwK6x+GeZfBE/1g5cnwDf3QNpXUFfu0vJFRDoqBZ0iIiIiv2J/2Dm274GwM6Wg3NVlichPuA0YAEC9Ojq7vJgeMXxw2gccG3EsTa1NPLzyYV7Y+IKry/pzPANg4BQ49Sm4biXcvgPOfwdGXwFBsYAT9m6GVfPgo+nwVF945Rj47j7I+A4aqly9AxGRDkFBp4iIiMhv4Omw8dZloxnTFnZe/PpqhZ0iHciBjs4sF1ciR4KPw4c5iXO4PuF6AN7b/h4trS0uruoQ8gqCwWfBac/C9Wvg1gw49w0YeRkExoCzFfakwI8vwAfnY3s2hknpD2JZ/DBkfQ8N1a7egYiISyjoFBEREfmNPB023rp0NGOi2sLON1azSWGnSIewv6OzISsLZ2uri6uRI8EwDK4YegUeNg/qmuvIq8xzdUmHj09PGHoeTJkDN6yHWdvhnNdgxCXQoy+Gs4UetdlYVz4P750LT0bC6yfADw/DjiRorHX1DkREjggFnSIiIiK/g5dbW2dnVABV9c1cpLBTpENw9InAcDhw1tbStGuXq8uRI8RqsTKghxlyby/d7uJqjiDf3jBsKpzxAtyUQtMNm9jQ5ypah/0F/PtAazPsXAPLnoV/ngVP9IE3T4bFf4ecpdBU7+odiIgcFgo6RURERH6n/WHn6Kge7WFn6s5yV5cl0q0ZNhuOmGhAk9e7m4EBAwHYXtKNgs7/5BtGQeAEWqY8DzdvhptS4cx5MOwC8A2D1ibIXwlLn4J3ppjB51unQfITkLsCmhtcvQMRkUNCQedhMG/ePAYNGsTo0aNdXYqIiIgcJmbYOYZRkW1h5+ur2byzwtVliXRr7pq83i0NCjSnlHerjs5f0yMShl8E57wCt2yFGzfClOdh6Png3QtaGiBvOSQ/Dm+fCrMHwdbPXF21iMifpqDzMJg5cybbtm1j7dq1ri5FREREDiNvNxtvX26GnZX1zVz4+iqFnSIu1H5Ppzo6u5WBgQc6Op1Op4ur6YAMAwL6wcgZcO7rcGsaXL8eTn8OBp8DXsFQWwwfz4B/XQl1Za6uWETkD+sUQWdeXh7btm2jVZeKi4iISAezP+wc2RZ2XvTGarbsUtgp4gr7J6/XK+jsVqL9orFb7FQ1VbGzeqery+n4DAOCYmDU5XD+W3DLNph4KxgW2LwAXjwasn5wdZUiIn9Ihwo633zzTWbPnn3Qn1111VX069ePoUOHMmTIEAoKClxUnYiIiMjP83az8U5b2FlR18SFryvsFHGF/R2djbl5tDY2urgaOVLsVjsx/jEApJWmubiaTsjmgOMegMu/g4BoqNoN750DC2dBY42rqxMR+V06VND56quv0qNHj/a3v/nmG9566y3effdd1q5di7+/Pw899JALKxQRERH5ed5uNt6+bDQj+vgr7BRxEVvPnlh8faG5mcacHFeXI0dQ+z2d3Xkg0Z8VMRquWQZjrjLfXvcGvDQe8le5ti4Rkd+hQwWdmZmZjBo1qv3tzz//nDPPPJMLL7yQESNG8Nhjj/HDD2qhFxERkY7Jx93OO5ePaQ87dYxd5MgyDKP9+HpDhgYSdSf7J69vK93m4ko6OYcXnPo0XPyZOa29LAfeOgUW/U2T2UWkU+hQQWddXR2+vr7tb//4449MmjSp/e1+/fqxd+9eV5QmIiIi8pvsDzuH9/GnvNYMO7fuVtgpcqS4DdgfdOqezu5EA4kOsehEuPZHiJ8OzlZYMQdeTYQ9qa6uTETkF3WooDMyMpL169cDUFxczNatWxk/fnz73+/duxc/Pz9XlSciIiLym+wPOxMizLDzwtcVdoocKe0dnZnq6OxOBvQYgNWwUlpfSmFtoavL6Ro8/OHsl2Hae+AZBIVb4bVjYekz0NLs6upERH5Whwo6Z8yYwcyZM3nkkUc4//zziYuLY+TIke1//+OPPzJkyBAXVigiIiLy2/i623n3rweHndt2V7q6LJEuz71tIJE6OrsXd5s7ff36AvD/7N13fJXl+cfxzznZARJG2CNsMEy1gIgoKoq7aK2K1i1W0Vq1tVatu8Ofo9qqaN3WOlqtq05cKIIoQ/aQjbLDTsgiOb8/DjkaQQVJ8uQkn/frxYvknPOccz14l8KX676vuRs8p7NS7XM8jJoI3Y+DshJ4/1Z4/CjIXRh0ZZK0kxoVdP7ud79j5MiRvPjii6SmpvL8889XeH78+PGMGDEioOokSZL2THnY2ScWdk407JSqWHlHZ8nKlZTm5QVcjapT+TmdBp1VoH7TaGfn8AchJQO+mgQPHgSfPgRlZUFXJ0kxNSroDIfD3HLLLXz++ee8+eab7LPPPhWef/755zn//PMDqk6SJGnPZaQm8c/z+tOnTSYbd4Sdc1cZdkpVJSEzk8TmzQG3r9c13zynU1UgFIK+I6Jnd3Y4BLYXwJtXwVPDYfNXQVcnSUANCzoB/v3vf3PGGWfw85//nAcffDDociRJkvZaZloS/zx/wDfCzk+Zt9qwU6oqTl6vm+zorCYN20ansh99BySmwZIPYfSBMO1ZcBCUpIDVqKDzgQceYMSIEUyePJkFCxZwySWXcNVVVwVdliRJ0l4rDzt7t8lkQ34xpz9s2ClVlZTyczrt6KxTujfuDsDq/NVsKNwQcDW1XDgMAy6Eiz6G1j+Bos3w8kXw719A3rqgq5NUh9WooPO+++7jxhtvZP78+UybNo0nn3yS0aNHB12WJElSpchMS+Kpb4Wd81dvDbosqdb5uqPTgUR1Sf3k+mRnZAMwb/28gKupI7I6w3lvw2HXQzgJ5r0Gow+Aua8FXZmkOqpGBZ2LFy/m7LPPjn1/+umns337dlatWhVgVZIkSZUnMy2Jp84bQK/W5WHnRMNOqZKldP066Iy4lbZOcft6ABIS4eDfwsj3oVkObMuFf58BL10MhZuDrk5SHVOjgs6ioiLq1asX+z4cDpOcnExBQUGAVUmSJFWuzPQk/nV+NOxcb9gpVbqUTp0gHKZ00yZKc3ODLkfVqHz7ukFnAFr2hgvHwqBfAyGY/kz07M7FYwMuTFJdkhh0Ad92/fXXk56eHvu+uLiYP/3pT2RmZsYe++tf/xpEaZIkSZWmPOw849GJzFqxhdMfnsizFx5A1+YNgi5Ninvh1FSS27WjeOlSihYsILFp06BLUjVx8nrAElPgiFug2zHw0kWwcQn886fQ/0IYejMkp//we0jSXqhRHZ0HH3ww8+fP5/PPP4/9OPDAA1m8eHHs+2nTpgVdpiRJUqUoDzt7tMqIdXYuWGNnp1QZygcSFXpOZ51SvnV9+dblbC3299PAtDsgOqjoJ+dHv//sIfjHYPhqcrB1Sar1alRH59ixYyt8n5ubS3JyMhkZGcEUJEmSVMUapifz9AUDOOORT5m9cgsjHp7IsyMPoIudndJeSenSha1jxjh5vY5plNqIlvVasip/FfM2zKNfi35Bl1R3pdSH4/4K3Y+BVy6F9Qvh0SPgoCvhkKshMTnoCiXVQjWqoxNg06ZNXHLJJWRlZdG8eXMaNWpEixYtuOaaa9i2bVvQ5UmSJFW68rAzp2UGuXnFjHj4UxautRNJ2hvlHZ1FXxh01jWxgURuX68ZOg+FUZ9Ar59DpAzG3QmPHAZrZgddmaRaqEYFnRs2bGDAgAE8+eST/OxnP+Ouu+7irrvu4oQTTuDee+/l4IMPprCwkM8++4y///3vQZcrSZJUaSqGnUWc9pBhp7Q3UrrsmLy+cCGRsrKAq1F16t4kOpBo3oZ5AVeimLRG8LNH4OdPQlpjWD0THhoCH98DZaVBVyepFqlRQectt9xCcnIyixYt4h//+AeXX345l19+OQ899BALFy6kuLiYM888kyOOOKLCcCJJkqTaoFG9aNi5T4WwMy/osqS4lJzdjlByMpGCAkq++iroclSNchrnAE5er5F6DIdRE6HrUVBaDO/eCI8fAxsWB12ZpFqiRgWdL7/8MnfeeSfNmzff6bkWLVpw++2389///pcrr7ySs88+O4AKJUmSqta3w84RD0807JR+hFBCAsmdOwFQ5ECiOqV88vrizYsp2F4QcDXaSYPmMOI5OOE+SK4PX06EBw6CSY9CJBJ0dZLiXI0KOletWkWPHj2+8/mePXsSDoe58cYbq7EqSZKk6tV4R9jZvUUD1m2Nhp2L1hl2SnsqtcuOczodSFSnNE1rSpPUJpRFyvhioyF3jRQKwX5nwsUTIPsgKMmH16+Ef/0MtqwMujpJcaxGBZ1ZWVksXbr0O59fsmQJzZo1q76CJEmSAtK4XjLPjDzg67DzIcNOaU+ldI2e01loR2edEgqFYl2dDiSq4Rplw9n/g2F/gcRUWPQejD4AZjxvd6ekH6VGBZ3Dhg3juuuuo7i4eKfnioqKuP766znqqKMCqEySJKn6fTPsXGvYKe2x2OR1OzrrnPLJ6w4kigPhMAwcBb/8CFrtC4Wb4cUL4PmzIX990NVJijM1Kui85ZZbmD9/Pl26dOH222/n1Vdf5ZVXXuG2226jS5cuzJ07l5tuuinoMiVJkqrNN7exl4ediw07pd1SHnQWL1lK2S6aKVR7lXd0zlk/J+BKtNuadoPz34Eh10A4Eea8Eu3unP9W0JVJiiM1Kuhs06YNn3zyCTk5OVxzzTUMHz6cE088keuuu46cnBzGjx9Pu3btgi5TkiSpWjWpn8LTFwygW/MdYefDE1mSmx90WVKNl9isGeGMDCgtpXjJkqDLUTUq7+hcsGkBJaUlAVej3ZaQBEN+Dxe8C027Q/5aePZUeOVSKNwSdHWS4kCNCjoBOnTowJtvvklubi4TJ05k4sSJrFu3jrfeeovOnTsHXZ4kSVIgmtRP4emRA+javD5rthRx2kOfGHZKPyAUCsXO6XTyet3Sun5rGiQ3YHvZdhZuWhh0OdpTrfaFCz+EgZcCIfj8KXhgECwZF3Rlkmq4Ghd0lmvUqBH9+/enf//+NG7cOOhyJEmSApdVP4VnRh4QCztHPDSRpYad0vdK6VIedHpOZ10SCoXIaZwDwNwNDiSKS0mpMOxPcM7r0LAdbF4OTx4Hb10DJQVBVyephqqxQackSZJ2Vh52dmlWn9VbCjnNsFP6XqnlA4ns6KxzPKezlmg/CC6eAPudFf1+4mj4x8GwYmqwdUmqkQw6JUmS4sy3w84RD09k2XrDTmlXYh2dTl6vc5y8XoukNIAT7oXT/wP1m0PuF/DIUPjgL+AZrJK+waBTkiQpDjVt8HXYuWpztLPTsFPaWXnQWbJyJaV5eQFXo+rUvUl3AOZvmE9pWWnA1ahSdB0GoyZCjxMhUgof3hYNPNcaZkuKMuiUJEmKU+VhZ+cdYecIw05pJwmZmSQ2bw54Tmddk90gm7TENApLC1m6ZWnQ5aiypDeGnz8BP3sUUhvCqmnRrewT7gUDbanOM+iUJEmKY9GwcwCdmtZj5Y6wc/n6bUGXJdUoKeXndLp9vU5JCCfQvXG0q9NzOmuhXidHuzs7D4XSIhjzB3jyeNi4NOjKJAXIoFOSJCnONWuQyrMXHhALO0976BPDTukbvp687kCiuqb8nE4nr9dSGS3hjBfguHsgqR4sGw8PDIIpT0IkEnR1kgJg0ClJklQLNGuQyrMjD6BjeWfnwxP5coNhpwSQ0tWBRHVV+eR1BxLVYqEQ/ORcuHg8tBsIxXnwv8vgmVNh6+qgq5NUzQw6JUmSaolmGak8tyPsXLGpgNMeMuyUAFLLt65/8QURu7zqlFhH5/q5lEXKAq5GVapxBzjndTjiVkhIhgVvw+gDYNaLQVcmqRoZdEqSJNUisbAzy7BTKpfcsSOEw5Ru2kRpbm7Q5agadWzYkeRwMnkleazYuiLoclTVwgkw6DK48ENo0RsKNsIL58IL58G2DUFXJ6kaGHRKkiTVMs0yomd2GnZKGUvo5QABAABJREFUUeHUVJKzswEo9JzOOiUpnESXRtGjC+ZscCBRndE8By54Dw7+HYQSYNZ/YfRAWPBO0JVJqmIGnZIkSbVQ8x1hZ4cdYeeIhyfy1UbDTtVdsYFEntNZ55Sf0zl3vQOJ6pTEZDjsOjj/HWjSBfJWw9Mnw/9+DUV5QVcnqYoYdEqSJNVSzTOiA4o6ZNXjq43Rzk7DTtVVKbFzOg066xonr9dxbfaHX34EAy6Ofj/lCXjgQFg2IdCyJFUNg05JkqRarEVmxbDTzk7VVXZ01l05TXKA6OR1h1HVUcnpcPRtcNarkNkWNi2Dx4+BMX+AksKgq5NUiQw6JUmSarnysLN9k3S+3BANO1dsKgi6LKlapXTdEXQuXEikzOnbdUmXRl1ICCWwoXADa7atCbocBanjIXDxeOj7CyACE+6Fh4bAqulBVyapkhh0SpIk1QEtMqNndmbvCDtPe+gTw07VKcnt2hFKSSFSUEDJl18GXY6qUUpCCh0bdgQ8p1NAaiYMvx9OexbqNYV1c+Hhw+DD26F0e9DVSdpLBp2SJEl1RMvMNJ77Rtg54qGJrDTsVB0RSkggpVMnwO3rdZHndGon3Y+BURNhn+OhbDt88Cd47EjI9fcHKZ4ZdEqSJNUhLTPTeHbkAbRrnM7yDds4zbBTdUj5OZ2FX3wRcCWqbuXndNrRqQrqZcEpT8GJD0FKJqyYAg8eBBMfBI+4kOKSQackSVId06phtLOzPOwc8fBEVm027FTtF5u8bkdnnWNHp75TKAR9ToVRn0DHQ2F7Ibx1NfzzBNi0POjqJO0hg05JkqQ6qFXDNJ698ADaNk5j2fpoZ6dhp2q72ECiLww665pujbsRIsSabWtYX7A+6HJUE2W2hjNfgmPuhKR0WDoORh8In/8LIpGgq5O0mww6q8D9999PTk4O/fr1C7oUSZKk79S6YRrPXTgwFnaOeGgiqzcXBl2WVGXKOzqLly6lrLg44GpUneol1SM7IxuAeRvmBVyNaqxQCPqPhIs+hjb9oXgrvHIJPHc65K0NujpJu8GgswpccsklzJkzh0mTJgVdiiRJ0vdq3TB6ZmebRmksXb+N0x76xLBTtVZis2aEMzOhtJTixYuDLkfVzO3r2m1NOsF5b8HhN0I4Cea/AaMPgDmvBF2ZpB9g0ClJklTHtWmUznMXfh12jnjYzk7VTqFQiJQunQHP6ayL9mkSDTrnrJ8TcCWKC+EEGHwlXPgBNO8J29bDf86CFy+Egk1BVyfpOxh0SpIkqULYuSQ3nxEPT2TNFsNO1T6p5QOJnLxe55QHnU5e1x5p0QtGvg8HXQmhMMz4N4weCIveD7oySbtg0ClJkiQgGnY+O/IAWjeMhp2nPWTYqdonpYsDieqq8q3rX+V9xZbiLQFXo7iSmAJDb4Tz3obGHWHrSnjqRHj9N1CcH3R1kr7BoFOSJEkxbRtHOzvLw84Rhp2qZcoHEhUusKOzrslMyaR1/dYAzN8wP+BqFJfa9o8OKuo3Mvr9pEfgwYNg+afB1iUpxqBTkiRJFXwz7Fy8I+xca9ipWiKlc/SMzu0rV1G6dWvA1ai6lXd1ek6nfrTkenDsnXDmS9CgFWxYDI8fBe/eBNuLgq5OqvMMOiVJkrSTb4edpz1s2KnaISEzk8QWLQAoXrQo4GpU3bo37g44eV2VoNNhMOoT6H0aRMrg47vh4cNg9aygK5PqNINOSZIk7VLbxtEzO1tlprJ4XXRA0bqtdqso/pWf01nsOZ11jgOJVKnSGsJJ/4BTnoL0JrBmFjw0BMbdBaXbg65OqpMMOiVJkvSd2jVJ57kLB9IqM5VF6/L5xWOT2VIcdFXS3knpuiPoXLgw4EpU3XKa5ACwdMtStpVsC7ga1Ro5J8CoidDtWCgrgfdugcePhvV2jUvVzaBTkiRJ36tdk3SevfAAWmamsjg3n/vmJJCbZ2en4lds8voCOzrrmqy0LJqmNaUsUsYXGx1IpUpUvxmc9jT8dDSkZMBXn0UHFX32MJSVBV2dVGcYdEqSJOkHZTepx3MXHkCLjBTWFIT4xWOT3cauuJW6Y/J68cKFEIkEXI2qW2z7uud0qrKFQrDvGXDxBOhwMJRsgzd+C/86CTavCLo6qU4w6JQkSdJuyW5Sj3+d34+GyREWrcvndM/sVJxK7tQJwmHKNm0iwcnrdU5sIJHndKqqNGwLZ74CR98Oiamw+AMYPRCmP+c/rkhVzKBTkiRJuy27cTqX5pTSPCOFBWvzDDsVl8IpKSRnZwOQsnpNwNWouuU0jp7TaUenqlQ4DAN+Cb8cB633h6LN8NIv4T9nQn5u0NVJtZZBpyRJkvZI0zR4+rx+tMhIjYWdntmpeJOyY/t68urVAVei6la+dX3hxoUUlzpdTVWsaVc4bwwc+gcIJ8Lc/8HoA2De60FXJtVKBp2SJEnaY9k7BhSVd3aO/OfkoEuS9kj5QKKUNQaddU3Lei3JTMlke2Q7CzctDLoc1QUJiXDIVTDyfWiWA/nr4LnT4eVRULg56OqkWsWgU5IkST9Kh6x6PDvyAAA+X76Jjfl2Ril+pHTdEXS6db3OCYVC7NN4x0Aiz+lUdWrZB0Z+AAdeBoRg2tPwwCBY/GHQlUm1hkGnJEmSfrSOTevTKjMVgMW5eQFXI+2+8o7O5DVriJSWBlyNqpuT1xWYpFQ48lY49w1o1B42fwn/PAHevBqKtwVdnRT3DDolSZK0Vzo2rQ/AorX5AVci7b7kdu0IpaQQLimhZMWKoMtRNbOjU4HLPhAuGg/7nxv9/tMH4R+D4aspwdYlxTmDTkmSJO2Vjk3rAbDIjk7FkVBCAskdOwJQvGBBwNWoupUHnfM3zmd72faAq1GdlVIfjr8HzvgvNGgJ6xfCo0fA+3+E7R4HI/0YBp2SJEnaK512dHQuXmdHp+JL8o7t68ULHEhT17TLaEd6YjpFpUUs3bw06HJU13UZChdPgJ4nQ6QUProDHjkc1swJujIp7hh0SpIkaa+Ud3QuXmdHp+JLcpfOgB2ddVE4FKZ74+6A53SqhkhvDCc/Cic/DmmNYPUMeOgQGP83KPMcYWl3GXRKkiRpr5Sf0bl8wza2l5YFXI20+8o7OosW2tFZF+U0yQFgznq75lSD9DwJRk2ELsOgtBjeuQGeOBY2LAm6MikuGHRKkiRpr7TMSCU1KUxJaYQvNxYEXY6028onr5csW0ZZsefh1TVOXleN1aAFnP5vOOFeSK4Pyz+BBwbB5McgEgm6OqlGM+iUJEnSXgmHQ3TIKp+87vZ1xY+Epk0pTUuD0lKKFy8OuhxVs/Kt6/M2zKMsYje6aphQCPY7Cy4eD9mDoCQfXrsCnj4ZtqwMujqpxjLolCRJ0l6LndPp5HXFkVAoRFGLFgAUffFFwNWounXM7EhKQgr5Jfl8ufXLoMuRdq1Rezj7NTjyT5CQAgvfhdEDYeYLQVcm1UgGnZIkSdprTl5XvCpu0RyAIgcS1TmJ4US6NuoKuH1dNVw4DAdeCr/8CFr2hcJN8N/z4flzYNuGgIuTahaDTkmSJO21TrHJ6wadii/lHZ2FdnTWSfs03nFO53qDTsWBZt3hgndhyDUQSoDZL8HoA+CLt4OuTKoxDDolSZK01zruOKPTreuKN0XNy7eu29FZF8UGEhl0Kl4kJMGQ30cDz6xukLcGnjkFXv0VFG0NujopcAadkiRJ2msddnR05uYVs3lbScDVSLuvfOv69lWrKN1qSFDXfHPyesRp1oonrfeDX34IB1wChGDqP+GBA2Hpx0FXJgXKoFOSJEl7rX5KIs0zUgBYZFen4khZWhqJzT2ns67q0rALiaFENhVtYnX+6qDLkfZMUhoc9Wc45zXIbAeblsMTx8Hb10FJYdDVSYEw6JQkSVKliG1f95xOxZnkLl0At6/XRckJyXRq2AmAORvmBFyN9CO1PwguHg/7nglE4JP74B8Hw4qpQVcmVTuDTkmSJFWKjrGBRHZ0Kr4kd+kMQJEDieqk8u3r8zbMC7gSaS+kZsBP74MR/4Z6zSB3PjwyFMbeBqUeKaO6w6BTkiRJlaJTUzs6FZ9iHZ1uXa+TnLyuWqXbUTBqIuQMh0gpjP0LPHoErJsfdGVStTDolCRJUqWIdXR6RqfiTEps6/oXDqSpg3Ka5AAGnapF6jWBnz8BP3sUUjNh5efw4GD45H4oKwu6OqlKGXRKkiSpUpR3dC7N3UZpmWGR4kdShw6QkEDp5s1sX7cu6HJUzbo26kqIEGsL1pJbkBt0OVLlCIWg18nR7s5Oh0NpEbx9LTx5PGxcFnR1UpUx6JQkSVKlaNUwjeTEMMWlZXy1cVvQ5Ui7LZySQnJ2NuBAorooPSmd9pntAbs6VQtltIJf/BeOuxuS6sGyj+GBA2HqP8EOdtVCBp2SJEmqFAnhEB2alA8k8pxOxZdvbl9X3VN+TqcDiVQrhULwk/Pg4o+h7QFQnAev/gqePQ22rgm6OqlSGXRKkiSp0pSf07nIyeuKMyldHUhUl8XO6dxgR6dqscYd4dw34IhbICEZvngLRh8As18KujKp0hh0SpIkqdLEJq/n2tGp+GJHZ91W3tE5Z/2cgCuRqlg4AQb9Gi4cCy16QcEGeP4c+O8FULAx6OqkvWbQKUmSpEoTm7xuR6fiTGrXrgAULVpEpLQ04GpU3bo36Q7AirwVbC7aHHA1UjVo3gMueB8OvgpCYZj5PIweCAvfDboyaa8YdEqSJKnSdNzR0bnIMzoVZ5LatiWUmkqksJCSL78MuhxVs4zkDNrUbwN4TqfqkMRkOOwPcP470KQzbF0F//oZvHYFFPkPlopPBp2SJEmqNOUdneu2FrG1sCTgaqTdF0pIIKVTJwAKPaezTtqnSXT7upPXVee0+Qn8chz0/2X0+8mPwYODYNknwdYl/QgGnZIkSao0GalJZNVPAZy8rvjjOZ11W/k5nQ4kUp2UnA7H3A5nvQIZbWDjUnj8aHjnBtheFHR10m4z6JQkSVKlip3Tmeu2N8WXlPJzOhcsDLgSBSHW0WnQqbqs4xAYNQH6ngFEYPzf4KEhsGpGwIVJu8egU5IkSZUqNnndjk7FmVjQaUdnndS9cXQg0dLNS9lWsi3gaqQApWbC8NFw2jOQngVr58DDh8JHd0Dp9qCrk76XQackSZIqVafY5HWDTsWX8q3rxcuWUVbkVs26Jisti2bpzYgQYf7G+UGXIwWv+7EwaiJ0Pw7KtsP7f4THhkGu5xir5jLolCRJUqUq37q+aJ1b1xVfEps1JSEzE0pLKV68OOhyFICcxjkAzFk/J+BKpBqiflM49V9w4j8gJRNWTIYHB8On/4CysqCrk3Zi0ClJkqRK1TErunV9SW4+ZWWRgKuRdl8oFHIgUR3XvUl0+/q8DfMCrkSqQUIh6HNa9OzOjkNgewG8+Tt46qew6cugq5MqMOiUJElSpWrTKI2khBBF28tYsakg6HKkPZLWtw+pvXsTSkkJuhQFIDZ5fb0DiaSdZLaBX7wEx9wJiWmw5CN44ECY9gxE/IdN1QwGnZIkSapUiQlhspuUT173nE7Fl2a//S0d/vNvMo46KuhSFICcJtGt64s2LaKo1HNapZ2Ew9B/JFz0MbTpB0Vb4OWL4bkzIG9d0NVJBp2SJEmqfF8PJPKcTknxo3l6cxqlNGJ7ZDsLNy4Muhyp5srqDOe+BYffAOEkmP86jD4A5v4v6MpUxxl0SpIkqdJ1bBo9p9PJ65LiSSgUYp8m0e3rczY4kEj6XgmJMPg3cOEH0KwHbMuFf/8CXvwlFGwKujrVUQadkiRJqnQds5y8Lik+dW8cHUjkOZ3SbmrRKxp2HnQFhMIw47no2Z2LPgi6MtVBBp2SJEmqdHZ0SopX5R2dTl6X9kBiCgy9KbqdvVEH2LICnhoOb1wFxduCrk51iEGnJEmSKl35GZ2rtxSSX7Q94GokafflNM6hYUpDmqQ2IeIkaWnPtBsAF4+HfhdEv//sIXjwIPhyUrB1qc4w6JQkSVKla5ieTON6yQAscfK6pDjStkFbPjr1I+49/F5CoVDQ5UjxJ7keHHsX/OJFaNAKNiyCx46Ed2+G7cVBV6dazqBTkiRJVaK8q9NzOiXFk1AoZMApVYbOh8OoCdD7VIiUwcd/hYcPg9Wzgq5MtZhBpyRJkqpExyzP6ZQkqU5LawQnPQSn/BPSGsOamfDQEPj4bigrDbo61UIGnZIkSaoSHe3olCRJADk/hUs+ha5HQ1kJvHsTPH40rF8UdGWqZQw6JUmSVCWcvC5JkmLqN4MRz8JP74fkBvDlp9FBRZMeAQd/qZIYdEqSJKlKlHd0LsnNp6zMv8BIklTnhUKw7y+iZ3e2Hwwl2+D138C/ToLNK4KuTrWAQackSZKqRLvG6SSGQxSUlLJ6S2HQ5UiSpJqiYTs461U46jZITIVF78MDA2HGf+zu1F4x6JQkSVKVSEoI065xOuD2dUmS9C3hMBxwMfxyHLTaDwo3w4sjSXjxPJJLtgRdneKUQackSZKqTOyczlwHEkmSpF1o2hXOfwcOvQ7CiYTn/Y9D511L6Is3g65MccigU5IkSVWmU/nk9bUGnZIk6TskJMIhv4ML3iOS1Y3U7VtIfP5MePkSKLS7U7vPoFOSJElVpnwg0eJct65LkqQf0Kov289/jwXNjiZCCKb9Cx4YBEs+CroyxQmDTkmSJFWZ2NZ1z+iUJEm7IzGVOa1HUHrmq9AwGzYvhyePhzd/DyUFQVenGs6gswrcf//95OTk0K9fv6BLkSRJClTHrGhH54pNBRQUlwZcjSRJiheRdgPh4vGw/znRBz59AP5xMKyYEmhdqtkMOqvAJZdcwpw5c5g0aVLQpUiSJAWqcb1kMtOSAFji9nVJkrQnUhrA8X+D05+H+i0g9wt45Ah4/09QWhJ0daqBDDolSZJUZUKhUGwgkZPXJUnSj9L1SBj1CfT8GURK4aPb4ZHDYe3coCtTDWPQKUmSpCpVfk7norV2dEqSpB8pvTGc/Fj0R1ojWDUd/nEITLgXyjweR1EGnZIkSapSHe3olCRJlaXnz2DUROhyJJQWwZg/wBPHwYYlQVemGsCgU5IkSVWqY5aT1yVJUiVq0AJO/0/0/M7k+rB8AjwwCCY/DpFI0NUpQAadkiRJqlKxMzrX5RHxLx+SJKkyhELRiewXj4d2B0JJPrx2OTz9c9iyKujqFBCDTkmSJFWpdk3SCYcgv7iUtVuLgi5HkiTVJo3awzmvwZF/hIQUWPgOjD4AZv036MoUAINOSZIkVamUxATaNU4HYNE6z+mUJEmVLJwAB/4KfvkhtOwDhZvghfPg+XNh24agq1M1MuiUJElSlYtNXvecTkmSVFWa7QMXvAeHXA2hBJj9YrS784sxQVemamLQKUmSpCrXMevrczolSZKqTEISHHotXPAOZHWFvDXwzM/h1cugaGvQ1amKGXRKkiSpypV3dDp5XZIkVYvW+8MvP4IDRkW/n/pkdDL7sgnB1qUqZdApSZKkKtexfPJ6rh2dkiSpmiSlwVF/gbNfg8x2sGkZPH4MvH0dlBQGXZ2qgEGnJEmSqlx50PnVxgIKS0oDrkaSJNUpHQbDxeNh318AEfjkPnjoEFj5edCVqZIZdEqSJKnKNa2fQoPURCIRWLZ+W9DlSJKkuiY1A356P4x4Duo1g3Xz4JGhMPb/oLQk6OpUSQw6JUmSVOVCodA3Jq+7fV2SJAWk29EwaiLk/BTKtsPYP8OjR8K6L4KuTJXAoFOSJEnVopOT1yVJUk1Qrwn8/Ek46RFIzYSVU+Efg+GT0VBWFnR12gsGnZIkSaoWsYFETl6XJElBC4Wg98+j3Z2dDoPthfD2NfDPE2DT8qCr049k0ClJkqRqEdu6nmvQKUmSaoiMVvCLF+HYuyApHZaOg9EHwuf/gkgk6Oq0hww6JUmSVC2+7ujMI+JfHCRJUk0RCkG/C+Cij6HtACjeCq9cAs+OgK1rgq5Oe8CgU5IkSdWifZN6hEKwtXA7uXnFQZcjSZJUUZNOcO6bMPRmSEiGL96E0QfA7JeDrky7yaBTkiRJ1SI1KYE2jdIAJ69LkqQaKpwAB10OF46F5r2gYAM8fzb8dyQUbAy6Ov0Ag05JkiRVm45Z0XM6HUgkSZJqtOY9YOT7MPg3EArDzP9Ez+5c+F7Qlel7GHRKkiSp2nzznE5JkqQaLTEZDr8BzhsDjTvB1pXwr5PgtSuh2H+0rYkMOiVJklRtyievL3byuiRJihdt+0UHFfW/MPr95EfhgUGwfGKwdWknBp2SJEmqNp2y7OiUJElxKDkdjrkDznwZMlrDxiXw+NHwzo2wvSjo6rSDQackSZKqTXlH55cbCyjaXhpwNZIkSXuo06Fw8QToMwIiZTD+HnjoUFg9M+jKhEGnJEmSqlHzjBTqJSdQWhZh+fptQZcjSZK059IawokPwqlPQ3oWrJ0dDTs/uhNKtwddXZ1m0ClJkqRqEwqFYl2di5y8LkmS4tk+x8GoidD9OCgrgfdvhcePgvWLgq6szjLolCRJUrWKTV7P9ZxOSZIU5+o3hVP/BcMfhJQM+GpSdFDR1KeCrqxOMuiUJElSteqYtWPyuh2dkiSpNgiFoO+I6NmdHQ6B7QXwv8tgy8qgK6tzDDolSZJUrWIdnU5elyRJtUnDttGp7C16RwcVLf046IrqHINOSZIkVavyoHPRunwikUjA1UiSJFWicBg6HBz92qCz2hl0SpIkqVqVb13fXFDChvzigKuRJEmqZO0Piv68bHywddRBBp2SJEmqVmnJCbRumAbA4lzP6ZQkSbVMu4FACNYvhK2rg66mTjHolCRJUrXznE5JklRrpTWEFr2iX9vVWa0MOiVJklTtOmaVB512dEqSpFqofPv6UoPO6mTQKUmSpGrXsWn0nM5FBp2SJKk2yh4U/dmBRNXKoFOSJEnVLrZ1Pdet65IkqRbKPjD6c+58yFsXbC11iEGnJEmSql2nHR2dy9dvo6S0LOBqJEmSKll6Y2jWI/q153RWG4NOSZIkVbsWGamkJSWwvSzC8g3bgi5HkiSp8rXfsX3doLPaGHRKkiSp2oXDITo4kEiSJNVmsXM6DTqri0GnJEmSAhE7p3Od53RKkqRaqDzoXDsbtm0ItpY6wqBTkiRJgSifvG5HpyRJqpXqN4WsbtGvl00ItpY6wqBTkiRJgejk5HVJklTbtT8o+rPndFYLg05JkiQFonzy+iI7OiVJUm1VPpBo6bhg66gjDDolSZIUiPJhRBvyi9m0rTjgaiRJkqpA9o6OztWzoGBjsLXUAQadkiRJCkS9lERaZKQCdnVKkqRaqkFzaNIZiMDyiUFXU+sZdEqSJCkwTl6XJEm1Xvn09aUfB1tHHWDQKUmSpMDEgs5cOzolSVIt5UCiamPQKUmSpMB0zIoOJLKjU5Ik1VrlHZ2rpkPhlmBrqeUMOiVJkhSY8o5Oz+iUJEm1VmZraNQeImXw5adBV1OrGXRKkiQpMJ2aRjs6l63PZ3tpWcDVSJIkVZHy7etLxwVbRy1n0ClJkqTAtG6YRkpimJLSCF9tLAi6HEmSpKqRXR50ek5nVTLolCRJUmDC4RAdssoHEnlOpyRJqqXa7zinc+XnUOSfeaqKQackSZICFZu87jmdkiSptmrYDjLbQaTUczqrkEGnJEmSAlU+ed2BRJIkqVYr7+pc5vb1qmLQKUmSpEB9PXndbVySJKkWy94RdHpOZ5Ux6JQkSVKgyievu3VdkiTVauUdnSumQPG2YGuppQw6JUmSFKjyjs7cvCK2FJYEXI0kSVIVadQBGrSCshL4alLQ1dRKBp2SJEkKVIPUJJo2SAHs6pQkSbVYKATtD4p+vfTjYGuppQw6JUmSFLiOWeWT1z2nU5Ik1WIOJKpSiUEXIEmSJJ0+oB3DerSgb9uGQZciSZJUdbJ3dHR+NRlKCiEpNdh6ahmDTkmSJAXup31bB12CJElS1WvSCeo3h7w1sGLy11vZVSncui5JkiRJkiRVh1AIsndsX1/q9vXKZtApSZIkSZIkVZfYOZ0OJKpsBp2SJEmSJElSdSk/p/PLSbC9ONhaahmDTkmSJEmSJKm6NO0G6VmwvQBWTg26mlrFoFOSJEmSJEmqLqHQ19vXl44LtpZaxqBTkiRJkiRJqk7l29cdSFSpDDolSZIkSZKk6lTe0fnlZ1BaEmwttYhBpyRJkiRJklSdmu4DaY2gJB9WTgu6mlrDoFOSJEmSJEmqTuEwZO/o6lz2cbC11CIGnZIkSZIkSVJ1Kw86Paez0hh0SpIkSZIkSdWt/JzO5ROhdHuwtdQSBp2SJEmSJElSdWveE1IyoXgrrJ4RdDW1gkGnJEmSJEmSVN3CCZB9YPTrpZ7TWRkMOiVJkiRJkqQglG9fX+Y5nZXBoFOSJEmSJEkKQmzy+idQVhpsLbWAQackSZIkSZIUhBa9IbkBFG2GNbOCribuGXRKkiRJkiRJQUhIhHYHRL9e6vb1vWXQKUmSJEmSJAXFczorjUGnJEmSJEmSFJTsg6I/LxsPZWXB1hLnDDolSZIkSZKkoLTqC0n1oGAjrJsbdDVxzaBTkiRJkiRJCkpCErTtH/166cfB1hLnDDolSZIkSZKkILXfsX3doHOvGHRKkiRJkiRJQSoPOpdNgEgk2FrimEGnJEmSJEmSFKRW+0FiGmzLhXXzg64mbhl0SpIkSZIkSUFKTIa2/aJfL3P7+o9l0ClJkiRJkiQFLbv8nM7xwdYRxww6JUmSJEmSpKC1HxT9edl4z+n8kQw6JUmSJEmSpKC1/gkkpEDeGli/KOhq4pJBpyRJkiRJkhS0pFRo85Po10vHBVtLnDLolCRJkiRJkmqC9jvO6VzmOZ0/hkGnJEmSJEmSVBNk7zinc6nndP4YBp2SJEmSJElSTdCmH4STYOtK2Lgk6GrijkGnJEmSJEmSVBMkp0Pr/aNfL3X7+p4y6JQkSZIkSZJqivY7tq97TuceM+iUJEmSJEmSaopvntOpPWLQKUmSJEmSJNUUbQdAKAE2L4eNy4KuJq4YdEqSJEmSJEk1RUp9aLVv9Gu3r+8Rg05JkiRJkiSpJml/UPRnt6/vEYNOSZIkSZIkqSYpDzqXfRxsHXHGoFOSJEmSJEmqSdoOgFAYNi6FzSuCriZuGHRKkiRJkiRJNUlqBrTsE/3aczp3m0GnJEmSJEmSVNNkD4r+vNTt67vLoFOSJEmSJEmqaWLndNrRubsMOiVJkiRJkqSapt1AIATrF8LW1UFXExcMOiVJkiRJkqSaJq0htOgZ/drt67vFoFOSJEmSJEmqidoPjv7s9vXdYtApSZIkSZIk1USxgUQGnbvDoFOSJEmSJEmqibIPjP6cOx/y1gVbSxww6PwB7du3p3fv3vTt25dDDz006HIkSZIkSZJUV6Q3hmY9ol+7ff0HJQZdQDyYMGEC9evXD7oMSZIkSZIk1TXtB8Ha2dGgs8fwoKup0ezolCRJkiRJkmoqz+ncbbU66Pzoo484/vjjadWqFaFQiJdffnmn19x///20b9+e1NRUBgwYwGeffVbh+VAoxCGHHEK/fv14+umnq6lySZIkSZIkia+DzrWzYduGYGup4Wp10Jmfn0+fPn24//77d/n8v//9b6688kpuvPFGpk6dSp8+fRg2bBhr166Nvebjjz9mypQpvPrqq/z5z39mxowZ1VW+JEmSJEmS6rr6TSGrW/Rrz+n8XrX6jM6jjz6ao48++juf/+tf/8rIkSM599xzAXjwwQd5/fXXeeyxx/j9738PQOvWrQFo2bIlxxxzDFOnTqV37967fL+ioiKKiopi32/ZsgWAkpISSkpKKuWefkj551TX50lVxbWs2sT1rNrE9azaxPWs2sK1rNrE9bxr4XYHkpA7n9LF4yjrfFTQ5VSrPVkLtTro/D7FxcVMmTKFa665JvZYOBxm6NChfPLJJ0C0I7SsrIwGDRqQl5fH+++/zymnnPKd7/mXv/yFm2++eafHx4wZQ3p6euXfxPd45513qvXzpKriWlZt4npWbeJ6Vm3ielZt4VpWbeJ6rqjVxjT6AVtnvcmH2wcFXU612rZt226/ts4Gnbm5uZSWltK8efMKjzdv3px58+YBsGbNGk488UQASktLGTlyJP369fvO97zmmmu48sorY99v2bKFtm3bcuSRR5KRkVEFd7GzkpIS3nnnHY444giSkpKq5TOlquBaVm3ielZt4npWbeJ6Vm3hWlZt4nr+Dnn7w99Gk1nwJccceiCkNQy6ompTvmN6d9TZoHN3dOzYkenTp+/261NSUkhJSdnp8aSkpGr/H2cQnylVBdeyahPXs2oT17NqE9ezagvXsmoT1/O3NGoDTToTWr+QpFWTodt3H9VY2+zJOqjVw4i+T1ZWFgkJCaxZs6bC42vWrKFFixYBVSVJkiRJkiTtQvn09aUfB1tHDVZng87k5GT2339/3nvvvdhjZWVlvPfeewwcODDAyiRJkiRJkqRvaX9Q9Gcnr3+nWr11PS8vj4ULF8a+X7JkCdOmTaNx48a0a9eOK6+8krPPPpuf/OQn9O/fn3vuuYf8/PzYFHZJkiRJkiSpRijv6Fw1HQq3QGr1zIOJJ7U66Jw8eTKHHnpo7PvyQUFnn302TzzxBKeeeirr1q3jhhtuYPXq1fTt25e33nprpwFFkiRJkiRJUqAyW0Oj9rBxKSyfCF2PDLqiGqdWB51DhgwhEol872suvfRSLr300mqqSJIkSZIkSfqR2h8UDTqXfWzQuQt19oxOSZIkSZIkKa5k7zinc6nndO6KQackSZIkSZIUD9rvOKdz5edQlBdsLTWQQackSZIkSZIUDxq2g8x2ECmFLz8Nupoax6BTkiRJkiRJihflXZ3L3L7+bQadkiRJkiRJUrzI3hF0ek7nTgw6JUmSJEmSpHhR3tG5YgoUbwu2lhrGoFOSJEmSJEmKF406QINWUFYCX30WdDU1ikGnJEmSJEmSFC9Coa+7Ot2+XoFBpyRJkiRJkhRP2h8U/dmBRBUYdFaB+++/n5ycHPr16xd0KZIkSZIkSaptsncEnV9NhpLCYGupQQw6q8All1zCnDlzmDRpUtClSJIkSZIkqbZp0gnqN4fSIlgxOehqagyDTkmSJEmSJCmehEKQ7Tmd32bQKUmSJEmSJMWb8oFEyz4Oto4axKBTkiRJkiRJijfl53R+OQm2FwVbSw1h0ClJkiRJkiTFm6bdID0LthfAiqlBV1MjGHRKkiRJkiRJ8SYUguwDo1+7fR0w6JQkSZIkSZLiU/vB0Z8dSAQYdEqSJEmSJEnxqXwg0ZefQWlJsLXUAIlBFyBJkiRJkiTpR2i6D+xzArTqGx1IlJAUdEWBMuiUJEmSJEmS4lE4DKc+FXQVNYZb1yVJkiRJkiTFPYNOSZIkSZIkSXHPoFOSJEmSJElS3DPolCRJkiRJkhT3DDolSZIkSZIkxT2DTkmSJEmSJElxz6BTkiRJkiRJUtwz6JQkSZIkSZIU9ww6JUmSJEmSJMU9g84qcP/995OTk0O/fv2CLkWSJEmSJEmqEww6q8All1zCnDlzmDRpUtClSJIkSZIkSXWCQackSZIkSZKkuGfQKUmSJEmSJCnuGXRKkiRJkiRJinsGnZIkSZIkSZLinkGnJEmSJEmSpLhn0ClJkiRJkiQp7hl0SpIkSZIkSYp7Bp2SJEmSJEmS4p5BpyRJkiRJkqS4Z9ApSZIkSZIkKe4lBl1AbRaJRADYsmVLtX1mSUkJ27ZtY8uWLSQlJVXb50qVzbWs2sT1rNrE9azaxPWs2sK1rNrE9axvK8/VynO272PQWYW2bt0KQNu2bQOuRJIkSZIkSYpfW7duJTMz83tfE4rsThyqH6WsrIyVK1fSoEEDQqFQtXzmli1baNu2LV9++SUZGRnV8plSVXAtqzZxPas2cT2rNnE9q7ZwLas2cT3r2yKRCFu3bqVVq1aEw99/CqcdnVUoHA7Tpk2bQD47IyPD3xBUK7iWVZu4nlWbuJ5Vm7ieVVu4llWbuJ71TT/UyVnOYUSSJEmSJEmS4p5BpyRJkiRJkqS4Z9BZy6SkpHDjjTeSkpISdCnSXnEtqzZxPas2cT2rNnE9q7ZwLas2cT1rbziMSJIkSZIkSVLcs6NTkiRJkiRJUtwz6JQkSZIkSZIU9ww6JUmSJEmSJMU9g05JkiRJkiRJcc+gs4a7//77ad++PampqQwYMIDPPvvsO187ZMgQQqHQTj+OPfbY2GsikQg33HADLVu2JC0tjaFDh7JgwYLquBWpUtdzSUkJV199Nb169aJevXq0atWKs846i5UrV1bX7aiOq+zfn7/poosuIhQKcc8991RR9dLXqmItz507lxNOOIHMzEzq1atHv379WL58eVXfilTp6zkvL49LL72UNm3akJaWRk5ODg8++GB13Iq0R+sZ4J577qFbt26kpaXRtm1brrjiCgoLC/fqPaXKUtnr+S9/+Qv9+vWjQYMGNGvWjOHDhzN//vyqvg3Fg4hqrOeeey6SnJwceeyxxyKzZ8+OjBw5MtKwYcPImjVrdvn69evXR1atWhX7MWvWrEhCQkLk8ccfj73mtttui2RmZkZefvnlyPTp0yMnnHBCpEOHDpGCgoJquivVVZW9njdt2hQZOnRo5N///ndk3rx5kU8++STSv3//yP7771+Nd6W6qip+fy734osvRvr06RNp1apV5O67767aG1GdVxVreeHChZHGjRtHrrrqqsjUqVMjCxcujLzyyivf+Z5SZamK9Txy5MhIp06dIh988EFkyZIlkX/84x+RhISEyCuvvFJNd6W6ak/X89NPPx1JSUmJPP3005ElS5ZE3n777UjLli0jV1xxxY9+T6myVMV6HjZsWOTxxx+PzJo1KzJt2rTIMcccE2nXrl0kLy+vum5LNZRBZw3Wv3//yCWXXBL7vrS0NNKqVavIX/7yl926/u677440aNAg9j/0srKySIsWLSJ33HFH7DWbNm2KpKSkRJ599tnKLV76lspez7vy2WefRYDIsmXL9rpe6ftU1Xr+6quvIq1bt47MmjUrkp2dbdCpKlcVa/nUU0+N/OIXv6j0WqUfUhXruUePHpFbbrmlwuv222+/yHXXXVc5RUvfYU/X8yWXXBI57LDDKjx25ZVXRgYNGvSj31OqLFWxnr9t7dq1ESDy4YcfVk7RiltuXa+hiouLmTJlCkOHDo09Fg6HGTp0KJ988sluvcejjz7KaaedRr169QBYsmQJq1evrvCemZmZDBgwYLffU/oxqmI978rmzZsJhUI0bNhwb0uWvlNVreeysjLOPPNMrrrqKnr06FHpdUvfVhVruaysjNdff52uXbsybNgwmjVrxoABA3j55Zer4hakmKr6vfnAAw/k1VdfZcWKFUQiET744AO++OILjjzyyEq/B6ncj1nPBx54IFOmTIltB168eDFvvPEGxxxzzI9+T6kyVMV63pXNmzcD0Lhx40qsXvHIoLOGys3NpbS0lObNm1d4vHnz5qxevfoHr//ss8+YNWsWF1xwQeyx8ut+7HtKP1ZVrOdvKyws5Oqrr2bEiBFkZGTsdc3Sd6mq9fx///d/JCYmctlll1VqvdJ3qYq1vHbtWvLy8rjttts46qijGDNmDCeeeCInnXQSH374YaXfg1Suqn5vvvfee8nJyaFNmzYkJydz1FFHcf/993PwwQdXav3SN/2Y9Xz66adzyy23cNBBB5GUlESnTp0YMmQI11577Y9+T6kyVMV6/raysjIuv/xyBg0aRM+ePSv9HhRfDDprqUcffZRevXrRv3//oEuR9toPreeSkhJOOeUUIpEIDzzwQDVXJ+2ZXa3nKVOm8Le//Y0nnniCUCgUYHXS7tvVWi4rKwPgpz/9KVdccQV9+/bl97//Pccdd5wDXFSjfdefNe69914mTpzIq6++ypQpU7jrrru45JJLePfddwOqVNq1sWPH8uc//5nRo0czdepUXnzxRV5//XVuvfXWoEuT9tierudLLrmEWbNm8dxzz1VzpaqJEoMuQLuWlZVFQkICa9asqfD4mjVraNGixfdem5+fz3PPPcctt9xS4fHy69asWUPLli0rvGffvn0rp3BpF6piPZcrDzmXLVvG+++/bzenqlxVrOdx48axdu1a2rVrF3ustLSU3/zmN9xzzz0sXbq00uqXylXFWs7KyiIxMZGcnJwKj++zzz58/PHHlVO4tAtVsZ4LCgq49tpreemll2KT2Hv37s20adO48847K2zDlCrTj1nP119/PWeeeWasK7lXr17k5+dz4YUXct111+3V/0akvVEV6zkc/rpn79JLL+W1117jo48+ok2bNlV3I4obdnTWUMnJyey///689957scfKysp47733GDhw4Pde+/zzz1NUVMQvfvGLCo936NCBFi1aVHjPLVu28Omnn/7ge0p7oyrWM3wdci5YsIB3332XJk2aVHrt0rdVxXo+88wzmTFjBtOmTYv9aNWqFVdddRVvv/12ldyHVBVrOTk5mX79+jF//vwKj3/xxRdkZ2dXXvHSt1TFei4pKaGkpKTCX6gBEhISYt3LUlX4Met527Ztu1yrAJFIZK/+NyLtjapYz+U/X3rppbz00ku8//77dOjQoYruQHEn0FFI+l7PPfdcJCUlJfLEE09E5syZE7nwwgsjDRs2jKxevToSiUQiZ555ZuT3v//9TtcddNBBkVNPPXWX73nbbbdFGjZsGHnllVciM2bMiPz0pz+NdOjQIVJQUFCl9yJV9nouLi6OnHDCCZE2bdpEpk2bFlm1alXsR1FRUZXfj+q2qvj9+ducuq7qUBVr+cUXX4wkJSVFHnroociCBQsi9957byQhISEybty4Kr0XqSrW8yGHHBLp0aNH5IMPPogsXrw48vjjj0dSU1Mjo0ePrtJ7kfZ0Pd94442RBg0aRJ599tnI4sWLI2PGjIl06tQpcsopp+z2e0pVpSrW88UXXxzJzMyMjB07tsLfBbdt21bt96eaxaCzhrv33nsj7dq1iyQnJ0f69+8fmThxYuy5Qw45JHL22WdXeP28efMiQGTMmDG7fL+ysrLI9ddfH2nevHkkJSUlcvjhh0fmz59flbcgxVTmel6yZEkE2OWPDz74oIrvRKr835+/zaBT1aUq1vKjjz4a6dy5cyQ1NTXSp0+fyMsvv1xV5UsVVPZ6XrVqVeScc86JtGrVKpKamhrp1q1b5K677oqUlZVV5W1IkUhkz9ZzSUlJ5Kabbop06tQpkpqaGmnbtm1k1KhRkY0bN+72e0pVqbLX83f9XfDxxx+vvptSjRSKRHb0/UqSJEmSJElSnPKMTkmSJEmSJElxz6BTkiRJkiRJUtwz6JQkSZIkSZIU9ww6JUmSJEmSJMU9g05JkiRJkiRJcc+gU5IkSZIkSVLcM+iUJEmSJEmSFPcMOiVJkiRJkiTFPYNOSZIkaQ/cdNNN9O3bN/b9Oeecw/DhwwOrR5IkSVEGnZIkSZIkSZLinkGnJEmSao3i4uKgS5AkSVJADDolSZIUt4YMGcKll17K5ZdfTlZWFsOGDWPWrFkcffTR1K9fn+bNm3PmmWeSm5sbu6asrIzbb7+dzp07k5KSQrt27fjTn/4Ue/7qq6+ma9eupKen07FjR66//npKSkqCuD1JkiTtAYNOSZIkxbUnn3yS5ORkxo8fz2233cZhhx3Gvvvuy+TJk3nrrbdYs2YNp5xySuz111xzDbfddhvXX389c+bM4ZlnnqF58+ax5xs0aMATTzzBnDlz+Nvf/sbDDz/M3XffHcStSZIkaQ+EIpFIJOgiJEmSpB9jyJAhbNmyhalTpwLwxz/+kXHjxvH222/HXvPVV1/Rtm1b5s+fT8uWLWnatCn33XcfF1xwwW59xp133slzzz3H5MmTgegwopdffplp06YB0WFEmzZt4uWXX67Ue5MkSdKeSQy6AEmSJGlv7L///rGvp0+fzgcffED9+vV3et2iRYvYtGkTRUVFHH744d/5fv/+97/5+9//zqJFi8jLy2P79u1kZGRUSe2SJEmqPAadkiRJimv16tWLfZ2Xl8fxxx/P//3f/+30upYtW7J48eLvfa9PPvmEM844g5tvvplhw4aRmZnJc889x1133VXpdUuSJKlyGXRKkiSp1thvv/3473//S/v27UlM3PmPul26dCEtLY333ntvl1vXJ0yYQHZ2Ntddd13ssWXLllVpzZIkSaocDiOSJElSrXHJJZewYcMGRowYwaRJk1i0aBFvv/025557LqWlpaSmpnL11Vfzu9/9jn/+858sWrSIiRMn8uijjwLRIHT58uU899xzLFq0iL///e+89NJLAd+VJEmSdodBpyRJkmqNVq1aMX78eEpLSznyyCPp1asXl19+OQ0bNiQcjv7R9/rrr+c3v/kNN9xwA/vssw+nnnoqa9euBeCEE07giiuu4NJLL6Vv375MmDCB66+/PshbkiRJ0m5y6rokSZIkSZKkuGdHpyRJkiRJkqS4Z9ApSZIkSZIkKe4ZdEqSJEmSJEmKewadkiRJkiRJkuKeQackSZIkSZKkuGfQKUmSJEmSJCnuGXRKkiRJkiRJinsGnZIkSZIkSZLinkGnJEmSJEmSpLhn0ClJkiRJkiQp7hl0SpIkSZIkSYp7Bp2SJEmSJEmS4p5BpyRJkiRJkqS4Z9ApSZIkSZIkKe4ZdEqSJEmSJEmKewadkiRJkiRJkuKeQackSZIkSZKkuGfQKUmSJEmSJCnuGXRKkiRJkiRJinsGnZIkSZIkSZLinkGnJEmSJEmSpLhn0ClJkiRJkiQp7hl0SpIkSZIkSYp7Bp2SJEmSJEmS4p5BpyRJkiRJkqS4Z9ApSZIkSZIkKe4ZdEqSJEmSJEmKewadkiRJkiRJkuKeQackSZIkSZKkuGfQKUmSJEmSJCnuGXRKkiRJkiRJinsGnZIkSZIkSZLinkGnJEmSJEmSpLhn0ClJkiRJkiQp7hl0SpIkSZIkSYp7Bp2SJEmSJEmS4p5BpyRJkiRJkqS4Z9ApSZIkSZIkKe4ZdEqSJEmSJEmKewadkiRJkiRJkuKeQackSZIkSZKkuGfQKUmSJEmSJCnuGXRKkiRJkiRJinsGnZIkSZIkSZLinkGnJEmSJEmSpLhn0ClJkqQ6YciQIfTs2TPoMiRJklRFDDolSZKkH6mgoIDzzz+fnj17kpmZSf369enTpw9/+9vfKCkpqfDa9957j/POO4+uXbuSnp5Ox44dueCCC1i1atV3vv+9995LZmZm7L1WrVrFhRdeSIcOHUhLS6NTp05ceeWVrF+/vkrvU5IkKR4kBl2AJEmSFK8KCgqYPXs2xxxzDO3btyccDjNhwgSuuOIKPv30U5555pnYa6+++mo2bNjAz3/+c7p06cLixYu57777eO2115g2bRotWrTY6f1ff/11jjzySJKSksjLy2PgwIHk5+czatQo2rZty/Tp07nvvvv44IMPmDJlCuGwfQySJKnuMuiUJElS3MrPz6devXqBfX7jxo2ZOHFihccuuugiMjMzue+++/jrX/8aCzD/+te/ctBBB1UII4866igOOeQQ7rvvPv74xz9WeJ9t27bx4Ycf8sADDwDw6quvsmzZMl577TWOPfbYCjXccsstTJ8+nX333beqblWSJKnG8598JUmStMe2bt3K5ZdfTvv27UlJSaFZs2YcccQRTJ06tcLrPv30U4466igyMzNJT0/nkEMOYfz48RVes2zZMkaNGkW3bt1IS0ujSZMm/PznP2fp0qUVXvfEE08QCoX48MMPGTVqFM2aNaNNmzax5998800OOeQQGjRoQEZGBv369avQUVluzpw5HHrooaSnp9O6dWtuv/32nV6zfPly5s2b96N/fdq3bw/Apk2bYo8dfPDBO3VcHnzwwTRu3Ji5c+fu9B7vvfceRUVFHH300QBs2bIFgObNm1d4XcuWLQFIS0v70fVKkiTVBnZ0SpIkaY9ddNFFvPDCC1x66aXk5OSwfv16Pv74Y+bOnct+++0HwPvvv8/RRx/N/vvvz4033kg4HObxxx/nsMMOY9y4cfTv3x+ASZMmMWHCBE477TTatGnD0qVLeeCBBxgyZAhz5swhPT29wmePGjWKpk2bcsMNN5Cfnw9EQ9DzzjuPHj16cM0119CwYUM+//xz3nrrLU4//fTYtRs3buSoo47ipJNO4pRTTuGFF17g6quvplevXrFAEeCss87iww8/JBKJ7NavR3FxMVu2bKGgoIDJkydz5513kp2dTefOnb/3ury8PPLy8sjKytrpuTfeeIP9998/FmyWB6W//vWvueuuu2jTpg0zZszgT3/6E8OHD6d79+67VaskSVJtFYrs7p/eJEmSpB0aNmzIL37xC+67775dPh+JROjWrRsdO3bkzTffJBQKAdEzLXv06EHnzp0ZM2ZM7LFvdyNOnDiRgQMH8s9//pMzzzwTiIaZ5557LgcddBBjx44lISEBgM2bN9O2bVtycnIYO3YsqampFeoo/+whQ4bw4YcfVnjP4uJisrOzGTRoEC+88ELsuvLX7u4flZ977jlGjBgR+/4nP/kJjz32GL169fre6/74xz9y/fXX895773HYYYdVeC47O5tzzz2Xm266KfbYo48+ym9/+9sKnaJnn302jzzyCImJ9jBIkqS6zT8NSZIkaY81bNiQTz/9lJUrV9KqVaudnp82bRoLFizgD3/4w04TwQ8//HCeeuopysrKCIfDFULOkpIStmzZQufOnWnYsCFTp06NhZLlRo4cGQs5Ad555x22bt3K73//+wohJxALOcvVr1+fX/ziF7Hvk5OT6d+/P4sXL67wurFjx+7eL8QOhx56KO+88w6bNm3ivffeY/r06bFu0+/y0UcfcfPNN3PKKafsFHLOmjWL5cuXVziLE6B169b079+fY445huzsbMaNG8ff//53srKyuPPOO/eoZkmSpNrGoFOSJEl77Pbbb+fss8+mbdu27L///hxzzDGcddZZdOzYEYAFCxYA0W7D77J582YaNWpEQUEBf/nLX3j88cdZsWJFhS7KzZs373Rdhw4dKny/aNEiAHr27PmDdbdp02an8LNRo0bMmDHjB6/9Ps2bN49tMT/55JP585//zBFHHMGCBQt2OU193rx5nHjiifTs2ZNHHnlkp+dff/11mjdvzk9+8pPYY+PHj+e4445j4sSJsceHDx9ORkYGN998M+eddx45OTl7dR+SJEnxzGFEkiRJ2mOnnHIKixcv5t5776VVq1bccccd9OjRgzfffBOAsrIyAO644w7eeeedXf6oX78+AL/61a/405/+xCmnnMJ//vMfxowZwzvvvEOTJk1i7/NNezN055udoN9U2ac5nXzyyeTl5fHKK6/s9NyXX37JkUceSWZmJm+88QYNGjTY6TVvvPEGRx11VIVQ9h//+MdO4SfACSecQCQSYcKECZV6D5IkSfHGjk5JkiT9KC1btmTUqFGMGjWKtWvXst9++/GnP/2Jo48+mk6dOgGQkZHB0KFDv/d9XnjhBc4++2zuuuuu2GOFhYUVzqH8PuWfNWvWrB8c/lNdCgoKgJ07UtevX8+RRx5JUVER7733Xmxi+jdt2rSJCRMmcOmll1Z4fM2aNZSWlu70+pKSEgC2b99eWeVLkiTFJTs6JUmStEdKS0t3CvCaNWtGq1atKCoqAmD//fenU6dO3HnnneTl5e30HuvWrYt9nZCQsFNH5b333rvLUG9XjjzySBo0aMBf/vIXCgsLKzz3Yzs1ly9fzrx5837wdbm5ubv8jPLt6N/svszPz+eYY45hxYoVvPHGG3Tp0mWX71k+pOnII4+s8HjXrl1Zs2bNTueHPvvsswDsu+++P1ivJElSbWZHpyRJkvbI1q1badOmDSeffDJ9+vShfv36vPvuu0yaNCnWlRkOh3nkkUc4+uij6dGjB+eeey6tW7dmxYoVfPDBB2RkZPC///0PgOOOO46nnnqKzMxMcnJy+OSTT3j33Xdp0qTJbtWTkZHB3XffzQUXXEC/fv04/fTTadSoEdOnT2fbtm08+eSTe3yPZ5111m5NXf/Xv/7Fgw8+yPDhw+nYsSNbt27l7bff5p133uH444+vMGTojDPO4LPPPuO8885j7ty5zJ07N/Zc/fr1GT58OBA9n/Oggw4iMzOzwmddeumlPP744xx//PH86le/Ijs7mw8//JBnn32WI444ggEDBuzxfUqSJNUmBp2SJEnaI+np6YwaNYoxY8bw4osvUlZWRufOnRk9ejQXX3xx7HVDhgzhk08+4dZbb+W+++4jLy+PFi1aMGDAAH75y1/GXve3v/2NhIQEnn76aQoLCxk0aBDvvvsuw4YN2+2azj//fJo1a8Ztt93GrbfeSlJSEt27d+eKK66o1Hv/toMOOogJEybw7LPPsmbNGhITE+nWrRt//etf+dWvflXhtdOmTQPgscce47HHHqvwXHZ2NsOHDycSifDWW2/x29/+dqfP6tatG1OmTOEPf/gD//rXv1i9ejWtWrXit7/9LTfffHOV3aMkSVK8CEUq++R1SZIkST/KZ599xoABA5g9e7YT1CVJkvaQZ3RKkiRJNcif//xnQ05JkqQfwY5OSZIkSZIkSXHPjk5JkiRJkiRJcc+gU5IkSZIkSVLcM+iUJEmSJEmSFPcMOiVJkiRJkiTFvcSgC6jNysrKWLlyJQ0aNCAUCgVdjiRJkiRJkhRXIpEIW7dupVWrVoTD39+zadBZhVauXEnbtm2DLkOSJEmSJEmKa19++SVt2rT53tcYdFahBg0aANH/EBkZGQFXE6ySkhLGjBnDkUceSVJSUtDlKM65nlSZXE+qTK4nVSbXkyqT60mVyfWkyuaa0vfZsmULbdu2jeVs38egswqVb1fPyMgw6CwpIT09nYyMDH/T0l5zPakyuZ5UmVxPqkyuJ1Um15Mqk+tJlc01pd2xO8dCOoxIkiRJkiRJUtwz6JQkSZIkSZIU9ww6JUmSJEmSJMU9z+iUJEmSJEkSAGVlZRQXF1frZ5aUlJCYmEhhYSGlpaXV+tmqGZKTkwmH974f06BTkiRJkiRJFBcXs2TJEsrKyqr1cyORCC1atODLL7/crYEzqn3C4TAdOnQgOTl5r97HoFOSJEmSJKmOi0QirFq1ioSEBNq2bVsp3XW7q6ysjLy8POrXr1+tn6uaoaysjJUrV7Jq1SratWu3V2G3QackSZIkSVIdt337drZt20arVq1IT0+v1s8u3y6fmppq0FlHNW3alJUrV7J9+3aSkpJ+9Pu4eiRJkiRJkuq48rMx93brsPRjlK+7vT2j1aBTkiRJkiRJAJ6RqUBU1roz6JQkSZIkSZIU9ww6JUmSJEmSpBrgnHPOYfjw4UGXEbcMOiVJkiRJkhS3VqxYwS9+8QuaNGlCWloavXr1YvLkybt87UUXXUQoFOKee+75wfedNGkShx9+OA0bNqRRo0YMGzaM6dOnV3L1qkwGnZIkSZIkSYpLGzduZNCgQSQlJfHmm28yZ84c7rrrLho1arTTa1966SUmTpxIq1atfvB98/LyOOqoo2jXrh2ffvopH3/8MQ0aNGDYsGGUlJRUxa2oEhh0SpIkSZIkKS793//9H23btuXxxx+nf//+dOjQgSOPPJJOnTpVeN2KFSv41a9+xdNPP01SUtIPvu+8efPYsGEDt9xyC926daNHjx7ceOONrFmzhmXLln3nddOnT+fQQw+lQYMGZGRksP/++8e6S2+66Sb69u1b4fX33HMP7du33+l9br75Zpo2bUpGRgYXXXQRxcXFsedeeOEFevXqRVpaGk2aNGHo0KHk5+cDX299/77r33rrLQ466CAaNmxIkyZNOO6441i0aFGFz//qq68YMWIEjRs3pl69evzkJz/h008/jT3/yiuvsN9++5GamkrHjh25+eab2b59+w/+ula1xKALkCRJkiRJUs0SiUQoKCmtls8qKyujoLiUxOLthMNh0pISdnsK96uvvsqwYcP4+c9/zocffkjr1q0ZNWoUI0eOrPD+Z555JldddRU9evTYrfft1q0bTZo04dFHH+Xaa6+ltLSURx99lH322WeXwWS5M844g3333ZcHHniAhIQEpk2btlvB6je99957pKamMnbsWJYuXcq5555LkyZN+NOf/sSqVasYMWIEt99+OyeeeCJbt25l3LhxRCKR3boeID8/nyuvvJLevXuTl5fHDTfcwIknnsi0adMIh8Pk5eVxyCGH0Lp1a1599VVatGjB1KlTKSsrA2DcuHGcddZZ/P3vf2fw4MEsWrSICy+8EIAbb7xxj+61shl0SpIkSZIkqYKCklJybng7kM+ec8sw0pN3L7JavHgxDzzwAFdeeSXXXnstkyZN4rLLLiM5OZmzzz4biHZ9JiYmctlll+12DQ0aNGDs2LEMHz6cW2+9FYAuXbrw9ttvk5j43bUtX76cq666iu7du8eu2VPJyck89thjpKen06NHD2655Rauuuoqbr31VlatWsX27ds56aSTyM7OBqBXr167fX04HOZnP/tZhdc/9thjNG3alDlz5tCzZ0+eeeYZ1q1bx6RJk2jcuDEAnTt3jr3+5ptv5ve//33s17djx47ceuut/O53vws86HTruiRJkiRJkuJSWVkZ++23H3/+85/Zd999ufDCCxk5ciQPPvggAFOmTOFvf/sbTzzxxHd2iR599NHUr1+f+vXrxzo+CwoKOP/88xk0aBATJ05k/Pjx9OzZk2OPPZaCggKA2DX169fnoosuAuDKK6/kggsuYOjQodx22207bQnfHX369CE9PT32/cCBA8nLy+PLL7+kT58+HH744fTq1Yuf//znPPzww2zcuHG3rwdYsGABI0aMoGPHjmRkZMQ6VJcvXw7AtGnT2HfffWMh57dNnz6dW265pcL9jxw5klWrVrFt27Y9vt/KZEenJEmSJEmSKkhLSmDOLcOq5bPKysrYumUrDTIaxLau766WLVuSk5NT4bF99tmH//73v0B0m/XatWtp165d7PnS0lJ+85vfcM8997B06VIeeeSRWHhZvs38mWeeYenSpXzyySeEw+HYY40aNeKVV17htNNOY9q0abH3zMjIAKLncJ5++um8/vrrvPnmm9x4440899xznHjiiYTD4QpbzIE9HmyUkJDAO++8w4QJExgzZgz33nsv1113HZ9++ikdOnTYrfc4/vjjyc7O5uGHH6ZVq1aUlZXRs2fP2DmeaWlp33t9Xl4eN998MyeddNJOz6Wmpu7R/VQ2g05JkiRJkiRVEAqFdnv7+N4qKytje3IC6cmJsVBxdw0aNIj58+dXeOyLL76Ibes+88wzGTp0aIXnhw0bxplnnsm5554LQOvWrXd6323bthEOhyt0gZZ/X35W5Te3c39T165d6dq1K1dccQUjRozg8ccf58QTT6Rp06asXr2aSCQSe99vhqXlpk+fTkFBQSxwnDhxIvXr16dt27ZA9L/NoEGDGDRoEDfccAPZ2dm89NJLXHnllT94/fr165k/fz4PP/wwgwcPBuDjjz+u8Pm9e/fmkUceYcOGDbvs6txvv/2YP3/+d95/kNy6LkmSJEmSpLh0xRVXMHHiRP785z+zcOFCnnnmGR566CEuueQSAJo0aULPnj0r/EhKSqJFixZ069btO9/3iCOOYOPGjVxyySXMnTuX2bNnc+6555KYmMihhx66y2sKCgq49NJLGTt2LMuWLWP8+PFMmjSJffbZB4AhQ4awbt06br/9dhYtWsT999/Pm2++udP7FBcXc/755zNnzhzeeOMNbrzxRi699FLC4TCffvopf/7zn5k8eTLLly/nxRdfZN26dbHP+KHrGzVqRJMmTXjooYdYuHAh77//fiwgLTdixAhatGjB8OHDGT9+PIsXL+a///0vn3zyCQA33HAD//znP7n55puZPXs2c+fO5bnnnuMPf/jDnv3HqwIGnZICt23KFDY+/zyF878gUlo9U/0kSZIkSfGvX79+vPTSSzz77LP07NmTW2+9lXvuuYczzjhjr963e/fu/O9//2PGjBkMHDiQwYMHs3LlSt566y1atmy5y2sSEhJYv349Z511Fl27duWUU07h6KOP5uabbwaiW+pHjx7N/fffT58+ffjss8/47W9/u9P7HH744XTp0oWDDz6YU089lRNOOIGbbroJiG6R/+ijjzjmmGPo2rUrf/jDH7jrrrs4+uijd+v6cDjMc889x5QpU+jZsydXXHEFd9xxR4XPT05OZsyYMTRr1oxjjjmGXr16cdttt5GQED1SYNiwYbz22muMGTOGfv36ccABB3D33XfHumiDFIp8+3AAVZotW7aQmZnJ5s2bY2c11FUlJSW88cYbHHPMMbHzLqRyq264kU3/+Q8AofR00nJySO3Tm7RevUnr3YvEli0rbBdwPakyuZ5UmVxPqkyuJ1Um15Mqk+updiosLGTJkiV06NCh2s9ZLCsrY8uWLWRkZOzx1nVVdM4557Bp0yZefvnloEvZI9+3/vYkX/OMTkmBS+nWlfQDDqBw5kzK8vPZNnky2yZPjj2f0DQrFnqm9upFYvfuAVZbM6zdtpYxS8fQM6sn+zTZh5SElKBLkiRJkiQpUAadkgLX+IwzaHzGGURKSylesoSCGTMpmDGdwhkzKfziC0rX5ZL3/vvkvf9+7Jr2TZuy5uOPSe/bl7TevUnp1o1wcnKAd1G9Jq+ezP9N+j8AEsOJdGvUjZ5ZPendtDc9s3rSPqM94ZD/EipJkiRJqjsMOiXVGKGEBFI6dyalc2cannQiAGWFhRTOnUvhjBnRAHTmTEqWLyd53Tq2/u81tv7vtei1SUmk5OxTofMzuX37Clvea5OGqQ0Z0mYIM3JnsKFwA7PXz2b2+tn8e/6/AWiQ1IAeWT3oldUr+qNpL7LSsgKuWpIkSZJUlZ544omgSwiUQaekGi2cmkr6vvuSvu++sccK165l3ONP0DstlaLZsymcPoPSTZsonD6Dwukz2Fh+bWYmaT17ktq7F2m9e5PWuzeJTZoEcyOV7MBWB3JgqwOJRCKsyl/FjNwZzFo3i5m5M5mzfg5bS7YycdVEJq6aGLumZb2WFYLPfRrvQ3pSeoB3IUmSJElS5THolBR3Eho1Ylv3bjTecfh5JBKh5KuvKJgxI9b5WThnDmWbN5M/fjz548fHrk1q1arCoKPUnBzC6fEb9oVCIVrVb0Wr+q04qv1RAGwv287CTQuZmTuTmetmMjN3Jos2LWJV/ipW5a9izLIxACSEEujcsDO9mvaKBaAdMzuSEE4I8pYkSZIkSfpRDDolxb1QKERy27Ykt21L5rHHAhApKaFowQIKyoPPmTMoWriIkpUrKVm5kq1vvhW9OCGBlC5dSOvVa0fnZx9SOncilBC/YV9iOJHujbvTvXF3ft715wDkl+QzO3d2NPzc8WPttrXM3zif+Rvn88IXLwCQnphOj6we0fM+s6Lnfbao1yLI25EkSZIkabcYdEqqlUJJSaTm5JCak0Oj004DoDQvj8JZsymY+XXn5/Y1ayiaN4+iefPg+eej16ank5aTQ+qO7e5pvXuR2LJlXJ/3WS+pHv1b9qd/y/6xx9bkr2FW7qzotvfcWczKncW27duYtHoSk1ZPir2uWVozemb1jHV+9mjSg/rJ9YO4DUmSJEmSvpNBp6Q6I6F+feodMIB6BwyIPVayZg2FM2dSMH0GBTNnUjhzJmX5+WybPJltkyd/fW1WFmm9epHWpzepvXqR1qsXCRkZQdxGpWlerznN6zXn8OzDASgtK2XJ5iUVuj4XbFzA2oK1vP/l+7z/ZXTqfYgQnRp2ioafO7a8d27UmaRwUpC3I0mSJEmq4ww6JdVpSc2bk9S8OQ2GDgUgUlZG8eLFOya8R4cbFX7xBaW5ueR98AF5H3wQuza5Q4cdE957k9anNynduhFOTg7qVvZaQjiBzo0607lRZ07sEp16X7C9gLnr534dfq6bycr8lSzctJCFmxby8sKXAUhNSGWfJvtUGHbUql6ruO6ClSRJkiTFF4NOSfqGUDhMSufOpHTuTMOTomFfWWEhhXPnVuj8LFm+nOIlSyhesoTNr7wavTYpiZR99qnQ+ZmcnU0oHA7ylvZKWmIa+zXfj/2a7xd7LLcgl1m5s2LB56zcWWwt2crnaz/n87Wfx17XOLUxvbJ6xc777JHVg8yUzCBuQ5IkSZJqjSFDhtC3b1/uueeeoEupcQw6JekHhFNTSd93X9L33Tf22PaNG6PB5zc6P0s3baJwx+T3jU8/Hb02I+PrQUc7Jr0nZmUFdSuVIistiyFthzCk7RAAyiJlLNuyLHre57roeZ/zNs5jQ+EGPvzqQz786sPYte0z2lfY8t6tcTeSE+K3C1aSJElS8D766CPuuOMOpkyZwqpVq3jppZcYPnw4ACUlJfzhD3/gjTfeYPHixWRmZjJ06FBuu+02WrVqFXuPL774gquuuorx48dTXFxM7969ufXWWzn00EO/97PffvttbrzxRmbPnk1qaioHH3wwd911F+3bt6/CO9Z3MeiUpB8hsVEj6h98MPUPPhiASCRCyVdfUTBjBoUzZkZ/njOHsi1byB8/nvzx42PXJrVqFR10VN75mZNDOD09qFvZa+FQmA6ZHeiQ2YHjOx0PQFFpEfM3zGdm7sxY+Ll863KWblnK0i1LeW3xawAkhZPo3rh7bLt7r6xetGvQzi3vkiRJknZbfn4+ffr04bzzzuOkk06q8Ny2bduYOnUq119/PX369GHjxo38+te/5oQTTmDyN+YyHHfccXTp0oX333+ftLQ07rnnHo477jgWLVpEixYtdvm5S5Ys4ac//SlXXnklTz/9NJs3b+aKK67gpJNOYurUqVV6z9o1g05JqgShUIjktm1JbtuWzGOPBSBSUkLRggUU7JjwXjhzBkULF1GyciUlK1ey9a23oheHw6R06UJa797Rzs/evUnp1IlQYvz+Fp2SkELvpr3p3bQ3Z+xzBgCbCjcxa/0sZq77etjRpqJNsa+ZF702IzmjQvDZM6snjVMbB3g3kiRJkmqyo48+mqOPPnqXz2VmZvLOO+9UeOy+++6jf//+LF++nHbt2pGbm8uCBQt49NFH6d27NwC33XYbo0ePZtasWd8ZdE6ZMoXS0lL++Mc/Et5xZNlvf/tbfvrTn1JSUkJS0q4Hto4dO5bf/e53zJ49m6SkJHr06MEzzzxDdnY255xzDps2beLll1+Ovf7yyy9n2rRpjB07NvbY9u3bufTSS3nqqadISkri4osv5pZbbok1jYwePZq7776bL7/8kszMTAYPHswLL7wARLe+9+zZE+A7r3/qqaf429/+xvz586lXrx6HHXYY99xzD82aNYvVMHv2bK6++mo++ugjIpEIffv25YknnqBTp04APPLII9x1110sWbKE9u3bc9lllzFq1Khd/ppUlvj9W7Qk1XChpCRSc3JIzcmh0WmnAVCal0/hrFnR7e47Oj+3r1lD0fz5FM2fD88/H702PZ20nJxo5+eO8DOxZcu47nRsmNqQg1ofxEGtDwKiXbBf5X1VIficu34uW4q3MH7leMav/LoLtnX91vTO6h0977Npb7o37k5qYmpQtyJJkiTVfpEIlGyrns8qK4t+VnEChMOQlA5V+HefzZs3EwqFaNiwIQBNmjShW7du/POf/2S//fYjJSWFf/zjHzRr1oz999//O99n//33JxwO8/jjj3POOeeQl5fHU089xdChQ78z5Ny+fTvDhw9n5MiRPPvssxQXF/PZZ5/t8d/1nnzySc4//3w+++wzJk+ezIUXXki7du0YOXIkkydP5rLLLuOpp57iwAMPZMOGDYwbN263r4folv9bb72Vbt26sXbtWq688krOOecc3njjDQBWrFjBwQcfzJAhQ3j//ffJyMhg/PjxbN++HYCnn36aG264gfvuu499992Xzz//nJEjR1KvXj3OPvvsPbrXPWHQKUnVKKF+PeodMIB6BwyIPVayZk2FQUeFM2dSlp/PtsmT2faNrRQJWVkVBh2l9epFQkZGELdRKUKhEG0btKVtg7Yc0/EYAEpKS/hi0xfMWjeLGbnRLe+LNy9mRd4KVuSt4M2lbwKQGEqkS6Mu9G7aOzbsqH1me8Kh+B38JEmSJNUoJdvgz61++HWVIAw0/OYD166E5HpV8lmFhYVcffXVjBgxgowdf58KhUK8++67DB8+nAYNGhAOh2nWrBlvvfUWjRo1+s736tChA2PGjOGUU07hl7/8JaWlpQwcODAWBu7Kli1b2Lx5M8cdd1ys83GfffbZ4/to27Ytd999N6FQiG7dujFz5kzuvvtuRo4cyfLly6lXrx7HHXccDRo0IDs7m32/MXPih64HOO+882Kv7dixI3//+9/p168feXl51K9fn/vvv5/MzEyee+65WKjbtWvX2DU33ngjd911V+wogQ4dOjBnzhz+8Y9/GHRKUm2W1Lw5Sc2b02DoUAAiZWUUL1myI/iMdn4Wzp9PaW4ueR98QN4HH8SuTW7ffkfwGe38TOnenXBy/A73SUpIokeTHvRo0oNTORWArcVbmZU7KzrsKHcGM9fNZH3heuZumMvcDXP59/x/A1A/qT49snrEBh31yupF0/SmQd6OJEmSpBqkpKSEU045hUgkwgMPPBB7PBKJcMkll9CsWTPGjRtHWloajzzyCMcffzyTJk2iZcuW9OjRg2XLlgEwePBg3nzzTVavXs3IkSM5++yzGTFiBFu3buWGG27g5JNP5p133uHLL78kJycn9jnXXnst1157Leeccw7Dhg3jiCOOYOjQoZxyyim0bNlyj+7lgAMOqNAFOnDgQO666y5KS0s54ogjyM7OpmPHjhx11FEcddRRnHjiiaR/YzbE912fkJDAlClTuOmmm5g+fTobN26krKwMgOXLl5OTk8O0adMYPHjwLjtX8/PzWbRoEeeff34sOIVoN2tmZuYe3eeeMuiUpBomFA6T0qkTKZ060fCkEwEoKyykcO7crye9z5hByfLlFC9dSvHSpWx+5dXotUlJpOyzT4XOz+TsbELh+O10bJDcgIGtBjKw1UAg+oeQ1fmrY9vdZ+bOZM76OeSV5PHpqk/5dNWnsWtb1GsRCz17ZvWkR5MepCfF7+AnSZIkqdokpUc7K6tBWVkZW7ZuJWNHNyVV8Gf28pBz2bJlsa3W5d5//31ee+01Nm7cGHt89OjRvPPOOzz55JP8/ve/54033qCkpASAtLQ0gFhX4+233x57r3/961+0bduWTz/9lJ/85CdMmzYt9lzjxtHZA48//jiXXXYZb731Fv/+97/5wx/+wDvvvMMBBxxAOBwmEonsVPueaNCgAVOnTmXs2LGMGTOGG264gZtuuolJkybFtut/n/z8fIYNG8awYcN4+umnadq0KcuXL2fYsGEUFxdX+DXYlby8PAAefvhhBgwYUOG5hISEPbqXPWXQKUlxIJyaSvq++5L+je0G2zdujJ73OWNGbNp76caNFM6YQeGMGWx8+unotRkZpPXsSWqf3qTt6PxMzMoK6lb2WigUomX9lrSs35Ij2x8JwPay7SzatIiZuTNjnZ+LNi1idf5qVuev5p1l0cPHw6EwnRp2ip332SurF+3qtQvydiRJkqSaKRSqsu3jOykrg6TS6OdVQZNGeci5YMECPvjgA5o0aVLh+W3bomeRhr/12eFwONbJmJ2dvdP7btu2badryoO8srIyEhMT6dy58y5r2nfffdl333255pprGDhwIM888wwHHHAATZs2ZdasWRVeO23atJ06Jz/99NMK30+cOJEuXbrEPj8xMZGhQ4cydOhQbrzxRho2bMj7778f20r+fdfPmzeP9evXc9ttt9G2bVuAChPqAXr37s2TTz65y6FLzZs3p1WrVixevJgzzjhjl/dfVQw6JSlOJTZqRP3Bg6k/eDAQ7XQsWbGCgunTo4OOZs6kcPZsyrZsIX/CBPInTIhdm9SqVXTQUa9epPXuRWqPHoTT47fTMTGcSLfG3ejWuBsndz0ZgG0l25i9fvbX4ee6GazZtoYFGxewYOMC/rvgvwCkJabRPNKcLz7/gj7N+tC7aW+apzeP68FPkiRJUl2Sl5fHwoULY98vWbKEadOm0bhxY1q2bMnJJ5/M1KlTee211ygtLWX16tVAtMMyOTmZgQMH0qhRI84++2xuuOEG0tLSePjhh1myZAnHHnvsd37usccey913380tt9wS27p+7bXX7vJMzG/W9tBDD3HCCSfQqlUr5s+fz4IFCzjrrLMAOOyww7jjjjv45z//ycCBA/nXv/7FrFmzdnq/5cuXc+WVV/LLX/6SqVOncu+993LXXXcB8Nprr7F48WIOPvhgGjVqxBtvvEFZWRndunXbrevbtWtHcnIy9957LxdddBGzZs3i1ltvrfD5l156Kffeey+nnXYa11xzDZmZmUycOJH+/fvTrVs3br75Zi677DIyMzM56qijKCoqYvLkyWzcuJErr7xyd//T7jGDTkmqJUKhEMlt2pDcpg2ZO/7POFJSQtGCBdHt7jOjnZ5FCxdRsnIlJStXsvWtt6IXh8OkdOkSDT179yatd29SOnUilBi//zeRnpROvxb96NeiX+yxtdvWxoLPmetmMmv9LPJL8lnKUpbOXQpzo6/LSsv6+qzPpr3o0aQHDZIbBHMjkiRJkr7X5MmTOfTQQ2PflwdpZ599NjfddBOvvho96qtv374Vrvvggw8YMmQIWVlZvPXWW1x33XUcdthhlJSU0KNHD1555RX69OnznZ972GGH8cwzz3D77bdz++23k56ezsCBA3nrrbe+c2t3eno68+bN48knn2T9+vW0bNmSSy65hF/+8pcADBs2jOuvv57f/e53FBYWct5553HWWWcxc+bMCu9z1llnUVBQQP/+/UlISODXv/41F154IQANGzbkxRdf5KabbqKwsJAuXbrw7LPP0qNHj926vmnTpjzxxBNce+21/P3vf2e//fbjzjvv5IQTTohd36RJE95//32uuuoqDjnkEBISEujbty+DBg0C4IILLiA9PZ077riDq666inr16tGrVy8uv/zy7/z1rAyhyLc3/qvSbNmyhczMTDZv3lzh7Ie6qKSkhDfeeINjjjlmlwfVSnvC9bR3SvPyKZw9m4IZX3d+bt/xL5rfFEpLI61Hjx3BZ3TKe2KrVrWq07EsUsaC9Qt45v1nCLcOM3vDbBZsXMD2yPYKrwsRokNmh9iE955Ne9K1UVeSwq4/VeTvT6pMridVJteTKpPrqXYqLCxkyZIldOjQgdTU1Gr97LKyMrZs2UJGRsZOW8FV+YYMGULfvn255557gi4l5vvW357ka/HbqiNJ+lES6tej3oD+1BvQP/ZYyZq1FM6c8XXn58xZlOXlsW3yZLZ94yyWhKysr7e79+pNWq+eJFTx1LyqFA6F6ZjZkf1S9uOY/tE/qBduL2TuhrnMXPf1sKMVeStYvHkxizcv5tVF0X8NTklIoXvj7hU6P9vUb1OrgmBJkiRJiicGnZIkkpo3I6n5UBoMHQpApKyM4iVLdkx4j3Z+Fs6fT2luLnkffEDeBx/Erk1u357U3r1I692HtN69SOnenXByclC3stdSE1PZt9m+7Nvs6zNw1hesZ/b62cxYNyO67T13JluKtzB93XSmr5see12jlEbRIUdNe8UC0MyU+A2CJUmSJCmeGHRKknYSCodJ6dSJlE6daHjicADKiooomjt3x5T3aOdnybLlFC9dSvHSpWx59X/Ri5OSSO3enbQdW95Te/UmuX02oTjegtIkrQkHtzmYg9scDEQHPy3furxC8Dlvwzw2Fm1k3IpxjFsxLnZtuwbtKgSf3Rt3JzkhfoNgSZIkSfFt7NixQZdQZQw69aNtyC9m0tINDOzUhIxUz2WRartwSgppffuS9o0DvLdv3EjhrFk7ws8ZFM6YSenGjRTOnEnhzJlsfHrHtRkZpPXsWaHzMzErK5gbqQShUIjsjGyyM7I5vtPxABSXFjN/w/zYdveZuTNZtmUZy7cuZ/nW5by++HUgOiG+e6Pu0fM+m/amV1Yv2mW0IxyK3yBYkiRJkmoCg079aGPnr+XK/0wnIRxiv3YNGdylKYO7ZNG7TUMSwp5RJ9UFiY0aUX/wYOoPHgxEOx1LVqygcMYMCqbPoGDmTApnz6ZsyxbyJ0wgf8KEr69t1ZK0Xr2/7vzMySFcr15Qt7LXkhOSo52bTXvFHttctDnW8TkzdyYz181kY9FGZq2fxaz1s3hu/nMANEhuQK+sXl8PO8rqSZO0JkHdiiRJkiTFJYNO7ZWOWfVYnJvPpKUbmbR0I3995wsy05I4qHMWg7tkMbhrU1o3TAu6TEnVJBQKkdymDclt2pBxzDEAREpKKFqw4OtBRzNmULRwEdtXrmLrylVsffvt6MXhMClduuzY7t6LtD59SOnUiVBi/P5fVWZKJoNaD2JQ60FANAhekbeCWbmzmJEb3fY+Z/0cthZvZcLKCUxY+XUQ3Lp+6+h5nzu2vO/TZB/SEv39VJIkSZK+S/z+7VGBO2m/Npy0Xxu+3LCNcQtyGbdgHeMX5rK5oITXZ67i9ZmrAOjYtB6DOjUhbVOIQ4q20zDJbe5SXRJKSiI1J4fUnBwanXYqAKV5+RTOnh2d9L6j83P76tUUzZ9P0fz58PwL0WvT0kjtkRPt/OzTm7RevUhs1SpuJ5uHQiHaNGhDmwZtOKrDUQCUlJWwcOPCCl2fizcvZkXeClbkreDtpdEgOCGUQJdGXb6e8p7Viw6ZHUgIJwR5S5IkSZJUYxh0aq+1bZzO6QPacfqAdmwvLWPGis2M+yKXjxasY9qXm1i8Lp/F6/KBBB7/ywfsn92IwV2acnCXpvRolUHYbe5SnZNQvx71BvSn3oD+scdK1qyNBp/lnZ8zZ1GWl0fB5CkUTJ7y9bVZWaT16hUbdJTWqycJmfE72TwpnMQ+TfZhnyb7cEq3UwDIK85j9vrZseBzZu5M1hWsY96GeczbMI/nv3gegHpJ9ejRpEeFLe/N6zUP8nYkSZIkKTAGnapUiQlh9mvXiP3aNeLXQ7uwuaCETxat58P5axgz40vWF8HExRuYuHgDd7w9n8b1kr/e5t6lKS0yU4O+BUkBSWrejKTmQ2kwdCgAkbIyipcsoWDGzFjnZ+H8+ZTm5pL3wQfkffBB7Nrk9u2jg452dH6mdO9OODl+J5vXT67PgJYDGNByABDd8r5m25oKXZ+z188mvySfz1Z/xmerP4td2yy9WSz07N20NzlNcqiXFL9nn0qSJEnS7jLoVJXKTEviqJ4tOLxbEwYkLKXnAUP4ZMlGPlqQyyeL1rMhv5hXp6/k1ekrAejavD4Hd2nK4K5N6d++MWnJbsmU6qpQOExKp06kdOoEJw4HoKyoiKK5c3dMeY92fpYsW07x0qUUL13Kllf/F704KYnU7t2/HnTUqzfJ7bMJheNzsnkoFKJFvRa0qNeCI7KPAKC0rJRFmxdFz/tcFz3vc8GmBazdtpZ3l7/Lu8vfBSAcCtMxs2N0u3vT6Jb3zg07kxj2jwCSJElSPBoyZAh9+/blnnvuCbqUGse/5fyAJUuWcN5557FmzRoSEhKYOHEi9eJ4KnCQQiHIbpJO5xaZnDmwPSWlZXy+fBPjFqzjowW5zPhqE1+syeOLNXk88vESkhPD9G/fmIO7Rrs9u7doELfn8kmqHOGUFNL69iWtb9/YY9s3bqRw1iwKZsygcMZMCmbMoHTjRgpnzqRw5kw2Pr3j2owM0nr2jHZ+9o5Oe0/MygrmRipBQjiBro260rVRV07qchIA20q2MWf9nArDjlblr2LhpoUs3LSQlxa+BEBqQio5TXKik96bRre9t6zX0t9jJUmSFJc++ugj7rjjDqZMmcKqVat46aWXGD58OAAlJSX84Q9/4I033mDx4sVkZmYydOhQbrvtNlq1ahV7jy+++IKrrrqK8ePHU1xcTO/evbn11ls59NBDv/ezI5EId911Fw899BDLli0jKyuLUaNGcd1111XlLes7GHT+gHPOOYc//vGPDB48mA0bNpCSkhJ0SbVGUkKY/h0a079DY35zZDc25hczYdF6PvpiHeMWrGPl5kI+XpjLxwtzgXk0bZDC4M5ZDO6axUGdm9K0gf8tJEFio0bUHzyY+oMHA9E/aJSsWEFhedfnjBkUzp5N2ZYt5E+YQP6EryebJ7ZqSUqPnjRKSqKgWTMSevcmHMf/mJWelM5PWvyEn7T4Seyx3ILc2DmfM3NnMit3FnkleUxdO5Wpa6fGXtcktUms67NnVk96ZvUkIzkjiNuQJEmS9kh+fj59+vThvPPO46STTqrw3LZt25g6dSrXX389ffr0YePGjfz617/mhBNOYPLkybHXHXfccXTp0oX333+ftLQ07rnnHo477jgWLVpEixYtvvOzf/3rXzNmzBjuvPNOevXqxYYNG9iwYUOV3au+n0Hn95g9ezZJSUkM3vGX58aNGwdcUe3WqF4yx/ZuybG9WxKJRFi0Lp9xC9Yxbsc293Vbi3jx8xW8+PkKAHJaZjC4axYHd2nK/tmNSE1ym7uk6Dbv5DZtSG7ThoxjjgEgUlJC0cKFOya8Rzs/ixYuZPvKVWxfuYqmwIo33oBwmJTOnUnr05vUXtHOz5TOnQklxu//XWalZXFou0M5tF30X6LLImUs3bK0Qvj5xYYvWF+4nrFfjWXsV2Nj17bPaE/vpr1jw466NupKUkJSQHciSZKk6hSJRCjYXlAtn1VWVkbB9gISSxIJh8OkJabt0W6jo48+mqOPPnqXz2VmZvLOO+9UeOy+++6jf//+LF++nHbt2pGbm8uCBQt49NFH6d27NwC33XYbo0ePZtasWd8ZdM6dO5cHHniAWbNm0a1bNwA6dOjwg/WOHTuW3/3ud7HcqUePHjzzzDNkZ2dzzjnnsGnTJl5++eXY6y+//HKmTZvG2LFjY49t376dSy+9lKeeeoqkpCQuvvhibrnlltiv2+jRo7n77rv58ssvyczMZPDgwbzwwgtAdOt7z549Ab7z+qeeeoq//e1vzJ8/n3r16nHYYYdxzz330KxZs1gNs2fP5uqrr+ajjz4iEonQt29fnnjiCTp16gTAI488wl133cWSJUto3749l112GaNGjfrBX5+9Eb9/c9sN39e6XO7+++/njjvuYPXq1fTp04d7772X/v2jU4AXLFhA/fr1Of7441mxYgUnn3wy1157bQB3UveEQiE6N6tP52b1OXdQB4q2lzJl2UbGLchl3IJ1zFqxhTmroj/+8eFiUpPCHNCxyY5p7ln/z959x0dVZg0c/90pmZn0MumF9EIqJQGlCGIBxNVdAXUt2Bs2WHt3RUUFxIK9NwR1d1VE14IQQHpLIZBCekJ678nM+8cNg6zltQCThPPdz/PZ5Xpv5jzZMSQn55yHSB9nacEUQtgoej3GuDiMcXF4XHA+AH2tbXRmZ9O2exdF33yLR00NvVVVdOXm0pWbCx+p3wQoJhPG+OHqQUf9be+6gIBB+zXm0MzOcLdwzok8B4Cuvi5y6nJsLe+ZNZmUtZZR1FxEUXMRnxV8BoCDxoFYr1i18rN/BbsED9rPhRBCCCGE+GUdvR2M+WCMXV57y9+34Kh3PGYfv6mpCUVRcHd3B8DLy4uYmBjeeecdRo4cicFg4OWXX8bHx4dRo0b94sf5/PPPCQ8PZ9WqVUydOhWr1cppp53Gk08++YvFcr29vZx77rlcffXVLF++nO7ubrZu3fq7v6d+++23ufLKK9m6dSvbt2/nmmuuISQkhKuvvprt27dz88038+6773LyySdTX1/P+vXrf/PzoLb8P/LII8TExFBdXc38+fO57LLLWL16NQDl5eVMnDiRSZMmsWbNGlxdXdm4cSO9vb0AvP/++zzwwAM8//zzjBgxgl27dnH11Vfj5OTEnDlzftdef48hnej8tdJlgBUrVjB//nxeeuklxowZw9KlSznzzDPZv38/Pj4+9Pb2sn79enbv3o2Pjw9Tp04lNTWV008/3Q67ObEZdFpOjjBzcoSZO6fGUtvaxcb8WtJz1cRndUsXa/fXsHZ/DQD+bkbbSe7jIs14Og3e05eFEMeG1tkJpzFpOIwcwSY/P0ZMnw71DXRmZfa3vO+hMzMLS2srHdt30LF9x+FnvbwwJSb2z/tMxpSYgNbNzY67+XMMWgMpPimk+KTYrjV0Ntha3Q/N+2zqaiKjJoOMmgzbfW4GN1vFZ6JZbXv3MHrYYRdCCCGEEEL8/zo7O7nzzju58MILcXVVRzUpisK3337Lueeei4uLCxqNBh8fH7766is8PH75e9sDBw5QXFzMRx99xDvvvENfXx/z5s1j5syZrFmz5mefaW5upqmpiRkzZtgqH+Pi4n73PoKDg3n66adRFIWYmBgyMzN5+umnufrqqykpKcHJyYkZM2bg4uLCsGHDGDFixG9+HuCKK66w3RseHs6zzz5Lamoqra2tODs7s2zZMtzc3Pjwww/R69Wur+joaNszDz74IIsXL7bl48LCwti7dy8vv/yyJDr/qF8rXQZYsmQJV199NZdffjkAL730El988QVvvPEGd911F4GBgYwePZrg4GAApk+fzu7du38x0dnV1UVXV5ftz83NzYCaBe/p6Tla2xqUDu3/aH0e3Awapsf7MD3eB6vVSl51K+vz69iQX8e2ogYqmzpZub2MldvLUBRICHBlfIQX46O8SAlyx0E3OE9eFqqj/X4SJ7Yfv5/0nh4YJ07EOHEiHoDVYqGnqJjOzAy6MrPozMqia/9++urqaF27ltYftY7ohw3DmJiIISEBY2IChthYFIfB+0sWZ60zJ/mexEm+JwFq61JpaynZddlk1WaRVZfF/ob9NHU1sbF8IxvLN9qeDXIOIsErgQSvBOK94on1jMWgPTHmKsvXJ3E0yftJHE3yfhJHk7yfhqaenh6sVisWiwWLxYJBY2DTBZuO2+u3tLTg4uICgEFjwGKx/OGPdWgP/6unp4dZs2ZhtVpZtmyZ7R6r1coNN9yAt7c369atw2Qy8frrr3P22WezZcsW/P39SUxMpLi4GIDx48ezevVq+vr66Orq4q233rIl+V599VVSU1PJycnBZDLZWsQB7r77bu6++27mzJnDmWeeyWmnncZpp53GrFmz8Pf3t8Vy6P+HQ6xWq21fh4wZM8Z276E/L168mJ6eHqZMmcKwYcMIDw/nzDPP5Mwzz+Svf/0rjo6Ov+l5rVbLjh07ePjhh8nIyKChocH22kVFRQwfPpxdu3Yxfvx4tFrtTz7XbW1tFBQUcOWVV9oSp6BWs7q5uf3s/zcWi0U9b6H/9f/3/7ffakgnOn9Nd3c3O3bs4O6777Zd02g0nHbaaWzapP6LnJqaSnV1NQ0NDbi5uZGens611177ix/z8ccf5+GHH/7J9a+//vqIN9OJ7H/nYhxN/sAsbzjHEw60KOxrVNjXpFDZrpBZ3kxmeTMvphdi0FiJcrMS42Yl1t2Kt1E9EV4MPsfy/SROPL/6ftLrYeQIGDkCpacHQ0UlxrJSjKWlGEtKcairo6e4mJ7iYlpWrQLAqtXS5e9PZ3AwHcFBdIaE0OPlBZrB/4uW+P7/9Lr0UtVXRWlfKWW9ZZT1lVFrqaWstYyy1jK+Kv4KAA0a/LR+BGuDCdQFEqwNxkvjhUYZ/J+LXyJfn8TRJO8ncTTJ+0kcTfJ+Glp0Oh1+fn60trbS3d193F/fpDPR26G2PbfQ8qc+VkdHh6347JCenh4uv/xyioqK+OwzdTTToXvWrVvHF198QWFhoa3K8/HHH+frr7/mlVdeYd68eSxfvtzWlm00GmlubsbT09P2eTv0sQIDAwHYt28fEyZMID093RaDh4cHzc3NLF26lCuuuIJvv/2WDz74gPvvv59//etfpKam0tfXR09PzxHxt7W10dvba7vW29v7k3s6Ojpse9JqtaxZs4YNGzawZs0aHnjgAR566CHWrFmDm5vb//t8Z2cnU6dO5dRTT+Wll17CbDZTVlbGeeedR0NDA83Nzej1+p98jEOqq6sBWLp0KaNHjz7in2m12p99pru7m46ODtLT022f50Pa29t/cv8vOWETnbW1tfT19eHr63vEdV9fX/bt2weo/5I/9thjTJw4EavVyhlnnMGMGTN+8WPefffdzJ8/3/bn5uZmgoODOeOMM2z/opyoenp6+Oabbzj99NNtJc3HS1VzJz8U1LMhv44NBbXUt/WQ1aCQ1aD+8yB3I+MizYyP9OKkcE/cTHLQxkBnz/eTGHqOxvupr7GRzuxsujIy6czKojMzE0tDA8ayMoxlZbj3/yJc4+KCIT4eY2KiWvWZmIjObD6Ku7G/lu4WteqzLsu26jvrqeiroKKvAvq/Z3bWOxPvFU+8VzyJXonEe8VjNg3+z4V8fRJHk7yfxNEk7ydxNMn7aWjq7OyktLQUZ2dnjEbjcX1tq9Vqq+g8GvPfTSbTEXmYnp4eLrvsMoqKivjuu+/w9vb+2efc3d1xdna2/Vmn0+Hg4ICrq+sRlZmHTJ48mSeffJKamhpbG3phYSGgtqN7enr+4qzO8ePHM378eB566CHGjRvHZ599xpQpUwgICCA3N/eI+HNyctDr9bZrOp2OXbt2HXFPRkYGUVFRR7Ta/+Uvf+Evf/kLjz76KJ6enmzbto2//e1v/+/zO3bsoL6+nkWLFtm6nA/lypycnHB1dWXkyJG88847mEymn3wdcHV1JSAggIMHD5KSkvKz+/9fnZ2dmEwmJk6c+JP3388lRn/JCZvo/K3+v/b3HzMYDBgMP23N0+v18sW/nz0+F0FeemZ7uTA7bRgWi5W9lc22Q422FzVQ1tjJiu1lrNhehkaBlGB39VCjaDPJQe7otEO34miwk3+3xNH0Z95Pem9vjJMmwaRJgPqNWk95OZ0ZGeq8z8xMOrOzsbS00LF5Mx2bN9ue1QX42w46MiYmYoqPR+PkdBR2ZB+eek8mOE1gQsgEQP1cVLZVqnM+a7LIrM1kb91eWnta2XJwC1sObrE96+/kf3jep3cicZ5xx3QI/bEkX5/E0STvJ3E0yftJHE3yfhpa+vr6UBQFjUaD5jh3IR1qZT70+r9Xa2sr+fn5tj8XFxeTkZGBp6cn/v7+zJ49m507d7Jq1SqsVqut4tDT0xMHBwfGjRuHh4cHl19+OQ888AAmk4lXX32VwsJCZsyY8YsxnXHGGYwcOZKrrrqKpUuXYrFYmDt3LqeffjqxsbE/+0xhYSGvvPIKf/nLXwgICGD//v3k5eVx6aWXotFomDJlCosWLeK9997jpJNO4r333iMrK4sRI0YcEUdJSQm33XYb1157LTt37uT5559n8eLFaDQaVq1axYEDB5g4cSIeHh6sXr0ai8VCXFyc7WP82vOhoaE4ODiwbNkyrrvuOrKysnj00UcBbO+Pm266ieeff56///3v3H333bi5ubF582bS0tKIiYnh4Ycf5uabb8bd3Z2pU6fS1dXF9u3baWhoOKJI8BCNRoOiKD/7deX3fJ05YROdZrMZrVZLVVXVEderqqrw8/OzU1TiWNNoFBIC3UgIdOP6SRG0d/ey5UA96Xk1rM+rJb+6lZ0ljewsaeSZ7/JwMeo4OcKLidHeTIzyJthzcP7ALYQ4vhRFwSEoCIegIFynTwfA2tNDV37+4YOOMjLpys+nt6KSlopKWv77X/VhjQZDZGT/QUdJmJKSMERGougG51/ZiqIQ4BxAgHMAU0OnAtBr6SW/MZ/M2kwyazLJrM2koLGAyrZKKtsq+aZYbYPTKloi3SPV5Kd3EgnmBCLcItBqtL/2kkIIIYQQ4gSzfft2Jk+ebPvzoUTanDlzeOihh2yt6v9bXfj9998zadIkzGYzX331Fffeey+nnnoqPT09xMfH8+mnn5KcnPyLr6vRaPj888+56aabmDhxIk5OTkybNo3Fixf/4jOOjo7s27ePt99+m7q6Ovz9/Zk7d65tVOKZZ57J/fffzx133EFnZydXXHEFl156KZmZmUd8nEsvvZSOjg7S0tLQarXccsstXHPNNYBamfqvf/2Lhx56iM7OTqKioli+fDnx8fG/6Xlvb2/eeust7rnnHp599llGjhzJokWL+Mtf/mJ73svLizVr1nD77bdzyimnoNVqSUlJYdy4cQBcddVVODo68tRTT3H77bfj5OREYmIit9566y9+bo4GxXpo6ugQpygK//73vzn33HNt18aMGUNaWhrPPfccoP4GISQkhBtvvJG77rrrT79mc3Mzbm5uNDU1Set6Tw+rV69m+vTpA/o3fhWNHWzIq2VdXg0b82tpbD9y4G2olyMToryZEGXmpAgvXIwDdy9D2WB5P4nBwZ7vp77WNjr3Zh9R+dlbWfmT+xSjEWN8fH/iMxFjYhL6wICj0tYzULT1tJFdm60mP/tXdXv1T+4z6Uxqu7t3Iolmdfk5DZxfUMrXJ3E0yftJHE3yfhJHk7yfhqbOzk4KCwsJCws77q3rFouF5uZmXF1dj3s16Ylo0qRJpKSksHTpUnuHYvNr77/fk18bnOUhv9H/li4XFhaye/duPD09CQkJYf78+cyZM4fRo0eTlpbG0qVLaWtrs53CLk48Ae4mZqcGMzs1mD6LlazyJtbn1ZCeW8vOkgaK6topqivm3c3F6DQKI0M8mBBlZkK0N4mBbmg1QyfpIIQ49rTOTjilpeGUlma71lNdTWdm5uHKz8wsLK2tdOzYQceOHYef9fLClJjYX/mZjCkxAa2bmz22cVQ46Z1I808jzf/w56KqrYqs2iy17b02i6zaLNp729letZ3tVdtt9/mYfEgwJ9iSn/Fe8Tg7OP/cywghhBBCCCGGsCGd6Py10uW33nqL888/n5qaGh544AHbgNSvvvrqJwcUiROTVqOQHOxOcrA7N54aRUtnD5sP1LO+v829sLaNrUX1bC2qZ/E3ubg76hkXaWZilJkJUd4EuJvsvQUhxCCk9/FBP2UKLlOmAGC1WOguKqIjI8NW+dm5fz99dXW0rl1L69q1tmcdhg3D2N/ubkpKxBAXh8bBwU47+fN8nXzxdfJlyjD1c9Fn6aOwqfCIqs+8hjyqO6pZU7qGNaVrAFBQCHcLP6LqM9IjEr1GKk6EEEIIIYQYyoZ0onPSpEn8f535N954IzfeeONxikgMZi5GPacP9+X04WoivLS+XZ3tmVvLxgK1zf2LjEq+yFDbTiO8nWyzPceEe+LoMKT/dRNCHCOKRoMhPBxDeDj0j1+xdHXRlZNja3fvyNhDT3EJ3cXFdBcX0/z55+rDej3G2NgjKj8dQoehDNJ2IK1GS6RHJJEekfw16q8AdPR2kFOXczj5WZNJRVsFBU0FFDQV8J/8/wBg1BqJ84qzHXaUYE4g0DlwSLX/CyGEEEII8Vus/VGxxFAjmRch/qBgT0cuGjOMi8YMo7fPwp6yRtJz1dPcd5c2UlDTRkFNG29uLMJBq2HUMA8mRqvzPYf7u6KRNnchxB+kMRgwpaRg+tEw9b7GRjoys+jIzKBzTwYdmZn01dfTmZlJZ2YmfND/rIsLpsQEjIlJmJKTMCUmovP2ts9GjgKTzsRI35GM9B1pu1bbUUtWbZYt8ZlVm0VLTwu7qnexq3qX7T5Po6fa8t5f9ZlgTsDNMHjb/4UQQgghhDjRSaJTiKNAp9Uwapgno4Z5Mu/0aJo6ethUUMu63FrSc2sob+xg04E6Nh2o44mvwMvJgfH9Le4To8z4uB7fQc9CiKFH6+6O84TxOE8YD4DVaqWnvILOjD22ys/O7GwsLS20/bCJth822Z7V+fv/6KCjREzx8WicnOy1lT/NbDIzKXgSk4InAWCxWihuLlbnfdao8z73NeyjvrOe9LJ00svSbc8Ocx1mS3wmmhOJ8YzBQTt42/+FEEIIIYQ4kUiiU4hjwM2kZ2qCP1MT/LFarRTVtdsONdpUUEtdWzef7q7g090VAMT6uaiHGkV5kxbmiVGvtfMOhBCDnaIoOAQF4hAUiOv06QBYe3vpysvrT3xm0JmRSVd+Pr2VlbRUVtLy3/+qD2s0GCIj1Xb3xP55n1FRKLrB+W2DRtEQ5hZGmFsYZ0ecDUBXXxf76/eTWZtpS36WtJRQ3FxMcXMxqw6sAkCv0RPrGWur+EzyTiLEJURa3oUQQgghhBiABudPLEIMIoqiEGZ2IszsxKUnhdLda2FXSQPr89Q294zyJvYdbGHfwRZeXV+IQachLcyTiVHeTIg2E+PrIj9QCyGOCkWnwxgXhzEuDo/zZwPQ19pG595s9aT3/pb33spKunJz6crNpenjT9RnjUaM8fGYEhMxJSdhTExCHxgwaL8+GbQGkryTSPJO4qK4iwBo7Gwkqy6LzJrDhx01djXa/vchrg6uRyQ+E8wJeBo97bUVIYQQQgghRD9JdApxnDnoNIwJ92JMuBe3nRlDQ1s3G/Jrbae5VzZ19idBa2E1+LgYGB9l5pRob8ZFmjE7G+y9BSHEEKJ1dsIpLQ2ntDTbtZ7qajXxmZFJZ6Z60rultZWOHTvo2LHj8LNeXocPOkpMwpSYgNbd3Q67ODrcje6MDxzP+MDD7f9lrWVHJD5z6nJo7m5mY8VGNlZstD0b6BxIojmR4Z7Dae1tpbO3E71eTnkXQgghhBDieJJEpxB25uHkwNnJAZydHIDVaqWgppV1/YcabT5QR3VLF//aWc6/dpYDEB/gqs72jDYzapgHBp20uQshji69jw/6KVNwmTIFAKvFQndRER0Zart7R0YGnfv301dXR+vatbT+6NRGh2HDMCYl2So/DbGxaAyD8xc0iqIQ7BJMsEsw08PV9v+evh5yG3PJqskio1ZteT/QdIDy1nLKW8v5qugrAN746A2iPKLUWZ/e6rzPMLcwNMrgPPFeCCGEEEKIwUASnUIMIIqiEOnjQqSPC1eOD6Ort48dRQ2k56mHGu2tbCa7Ql0vrSvApNcyNtzTlviM8HYetG2kQoiBS9FoMISHYwgPh3PPBcDS1UXXvn22dvfOjAy6i4ttq/nzz9WH9XqMMTGYkpLUys+kJBxCQ1E0gzPhp9fqifeKJ94rnvM5H4CW7hay67LJrMlkT/UedlTsoNXaSk59Djn1OazMXQmAk96JBK8EEr37297NSXg7Dt4T74UQQgghxLGhKAr//ve/Obf/e2/x20miU4gBzKDTcnKkmZMjzdw1LZaali425teS3t/mXtPSxff7a/h+fw0AAW5GJvTP9hwXYcbDSU4KFkIcGxqDAVNyMqbkZNu1vsZGOjKzbAcddWRk0FdfT2dWFp1ZWfBB/7MuLpgSEzD2H3RkSkpC5z14E34uDi6M9R/LWP+x9PT08MUXXzBy0kj2Ne6ztbzvrdtLW08bWw5uYcvBLbZnfR19bXM+E82JxHvF46h3tONuhBBCCCEGn/T0dJ566il27NhBZWXlEUnCnp4e7rvvPlavXs2BAwdwc3PjtNNOY+HChQQEBNg+Rm5uLrfffjsbN26ku7ubpKQkHnnkESZPnvyrr221Wlm8eDGvvPIKxcXFmM1mbrjhBu69995jueXfbO3atUyePJmGhgbcB/GYqd9KEp1CDCLeLgbOHRHIuSMCsVqt7DvYYpvtuaWwnoqmTlZsL2XF9lIUBZKC3JnYf5r7iBB39NrBWUElhBgctO7uOE8Yj/OEwzMue8orbHM+OzIy6MzOxtLSQtsPm2j7YZPtWZ2/v9runpSotr7Hx6NxcrLXVv4URVHwd/InxD2EM0LPAKDX0ktBYwGZtZlk1apt7wWNBVS1V/FN8Td8U/wNoJ4QH+EeQZL5cPIzwj0CnUa+ZRNCCCGE+CVtbW0kJydzxRVX8Le//e2If9be3s7OnTu5//77SU5OpqGhgVtuuYW//OUvbN++3XbfjBkziIqKYs2aNZhMJpYuXcqMGTMoKCjAz8/vF1/7lltu4euvv2bRokUkJiZSX19PfX39MdurvVitVvr6+tDpBvb3pQM7OiHEL1IUhTh/V+L8XblmYgSdPX1sKaxnfa6a+Nxf1cKe0kb2lDby3Jp8nA06xoZ7cUq0mvgc5uUobe5CiGNKURQcggJxCArEddo0AKy9vXTl5/e3vKuVn135+fRWVtJSWUnL11+rD2s0GCIiMCYnqQcdJSViiIpCGeDfWP0SnUZHjGcMMZ4xzIyeCUB7T7va8n4o+VmTQVV7FXkNeeQ15PFJnnrivUlnYrjXcHXeZ//yc/KTr+FCCCGEOKasVivWjo7j8loWiwVLRwcWnQ40GhST6Xd9rzNt2jSm9X+/+b/c3Nz45ptvjrj2/PPPk5aWRklJCSEhIdTW1pKXl8frr79OUlISAAsXLuSFF14gKyvrFxOdOTk5vPjii2RlZRETEwNAWFjYb4r5jTfeYPHixeTn5+Pp6cl5553H888//5P7fq4ic/fu3YwYMYLCwkJCQ0MpLi7mxhtvZMOGDXR3dxMaGspTTz3F8OHDbRWpHh4eAMyZM4e33noLi8XCE088wSuvvMLBgweJjo7m/vvvZ+bMmUe87urVq7nvvvvIzMzk66+/ZtKkSb9pf/YyOH9aEEL8hFGv5ZRob06JVts/q5rV09vTc2vYkF9LfVs33+ZU8W1OFQDBniZ1tmeUmZMizLiZ5HRgIcSxp+h0GGNjMcbG4nH+bAAsbW10ZGfbTnrvyMigt7KSrrw8uvLyaPpYTfgpRiPG+PgfVX4mow8MGLQJP0e9I6l+qaT6pdquVbdX2xKfmTWZZNVl0dbTxo6qHeyoOnzivdlkts35TDAnkGBOwMXBxR7bEEIIIcQQZe3oYP/IUcf1Nav6/ztm5w4Ux2M3zqepqQlFUWyJQy8vL2JiYnjnnXcYOXIkBoOBl19+GR8fH0aN+uXPweeff054eDirVq1i6tSpWK1WTjvtNJ588kk8PT1/8bkXX3yR+fPns3DhQqZNm0ZTUxMbN278w/uZO3cu3d3dpKen4+TkxN69e3F2diY4OJhPPvmE8847j/379+Pq6orJZALg8ccf57333uOll14iKiqK9PR0Lr74Yry9vTnllFNsH/uuu+5i0aJFhIeH25KlA5kkOoUYonxdjcwcFcTMUUFYLFb2Vjarsz1za9leXE9pfQcfbCnhgy0laDUKKcHuTOhvc08OckMnbe5CiONE4+SEU1oaTmlptms91dV0ZmXRsSdDbX3PzMLS0kLHjh107Dic8NN6emJKTDxc+ZmYgHYQzx7ycfRhSsgUpoSoJ95brBYKmwrVWZ816rzPvIY8ajtqWVu6lrWla23PhrmFHa769E4k2j0avVZ+iSWEEEII8WOdnZ3ceeedXHjhhbi6ugJqJ9K3337Lueeei4uLCxqNBh8fH7766qtfTe4dOHCA4uJiPvroI9555x36+vqYN28eM2fOZM2aNb/43IIFC/jHP/7BLbfcYruWmpr6i/f/f0pKSjjvvPNITEwEIDw83PbPDiVcfXx8bIndrq4uHnvsMb799ltOOukk2zMbNmzg5ZdfPiLR+c9//pPTTz/9D8d2vEmi8xhYtmwZy5Yto6+vz96hCAGARqOQEOhGQqAbN0yKpK2rly2FdaTn1rI+r4aCmjZ2FDewo7iBpd/m4WrUMS5STXpOiDIT7CkHYwghji+9jw/6U0/F5dRTAbBaLHQXFalzPjMy1ZPe9+2jr76e1nXraF23zvasw7Bh6pzP/spPQ1wcGoPBXlv5Uw7N7Ixwj+DcyHMB6OztJKc+x5b4zKzNpLy1nMKmQgqbCvms4DMAHDQOxHnFHZH8DHIOGrQVsEIIIYQ4vhSTiZidO/7/G48Ci8VCc0sLrv1JRqW/6vBo6+npYfbs2VitVl588UXbdavVyty5c/Hx8WH9+vWYTCZee+01zj77bLZt24a/vz/x8fEUFxcDMGHCBL788kssFgtdXV288847REdHA/D6668zatQo9u/fj8lkYvjw4bbXueeee7jqqquoqKhgypQpR21fN998M9dffz1ff/01p512Guedd56tBf/n5Ofn097e/pMEZnd3NyNGjDji2ujRo49anMeDJDqPgblz5zJ37lyam5txc3OzdzhC/ISTQcepsb6cGusLQFlDOxvyalmfV8uG/FqaOnr4MusgX2YdBCDM7GQ71GhshBfOBvnSIYQ4vhSNBkN4OIbwcOg/QdPS1UXXvn2HDzrKyKC7uNi2mj//XH1Yr8cYE3P4oKOkJBxCQ1E0g7Ny3agzMsJnBCN8Dn8TWtdRR3ZdNhk1GWrbe20mzd3N7KnZw56aPbb7PAwetkOOEr0TSfBKwN3oboddCCGEEGKgUxTlmLaPH8FiQdPbi8bREc0x+h7tUJKzuLiYNWvW2Ko5AdasWcOqVatoaGiwXX/hhRf45ptvePvtt7nrrrtYvXo1PT09ALb2b39/f3Q6nS3JCRAXFweoVZaTJ09m9+7dtn/m6emJXv/7Om4OfT6sVusRe/mxq666ijPPPJMvvviCr7/+mscff5zFixdz0003/ezHbG1tBeCLL74gMDDwiH9m+J8CAadBdkCoZCuEEAR5OHJBWggXpIXQZ7GSWd5Eem4N6/Nq2FnSSGFtG4W1bby9qRidRmHkMA8mRpmZGO1NfIAbWo1UBwkhjj+NwYApORlTcrLtWl9jIx1Z2XRk7FErPzMy6KuvpzMri86sLPhgufqsszPGxARMSclqAjQxEb2Pj7228qd5mbyYGDSRiUETAfUb4ZKWkiMSn/vq99HQ1cD68vWsL19vezbEJUSd9+mdRKI5kRjPGAzawVkBK4QQQgjxcw4lOfPy8vj+++/x8vI64p+3t7cD/CTJqtFosFgsAAwbNuwnH3fcuHH09vZSUFBAREQEALm5ubb7dTodkZGRP3kuNDSU7777znZQ0K/x9lbP4aisrLS10f84eXpIcHAw1113Hddddx133303r776KjfddBMODg4AR3QdDx8+HIPBQElJyRFt6kOBJDqFEEc4NK8zJdidm6dE0dLZw6aCOvVgo7waiuva2VpYz9bCehZ9nYuHo55xkWYmRnkzIdqMv9uxaTEQQojfQuvujvP4cTiPHweoCb+e8gp1zuehys/sbCytrbRv2kz7ps22Z3X+/ocPOkpMwpQQj2aQ/Qb7EEVRGOY6jGGuwzg74mwAuvu62V+/39bunlWbRVFzESUtJZS0lLC6cDWgnhAf6xFrS34mmBMY5joMjTI4K2CFEEIIMfS1traSn59v+3NhYSG7d+/G09MTf39/Zs6cyc6dO1m1ahV9fX0cPKh2L3p6euLg4MBJJ52Eh4cHc+bM4YEHHsBkMvHqq69SWFjIWWed9Yuve9pppzFy5EiuuOIKli5disViYe7cuZx++ulHVHn+r4ceeojrrrsOHx8fpk2bRktLCxs3bvzZCszIyEiCg4N56KGHePTRR8nNzWXx4sVH3HPrrbcybdo0oqOjaWho4Pvvv7dVlg4bNgxFUVi1ahXTp0/HZDLh4uLCbbfdxrx587BYLIwfP952IJKrqytz5sz5XZ//gUQSnUKIX+Vi1HNGvB9nxPsBUFzXxvo8dbbnD/l1NLT3sCqjklUZlQBE+Tirsz2jzYwJ88TRQb7MCCHsR1EUHIICcQgKxHXaNACsvb105efTkZFhm/nZlZ9Pb2UlLZWVtHz9tfqwRoMhIgJjUqKt8tMQFYWiG5xf1xy0DiR6qy3rhzR1NdkqPg8deNTQ1UBWXRZZdVl8uP9DAFwcXEjwSlCf75/56WXy+qWXEkIIIYQ4rrZv335EdeT8+fMBmDNnDg899BCffabOME9JSTniue+//55JkyZhNpv56quvuPfeezn11FPp6ekhPj6eTz/9lOQfdQ/9L41Gw+eff85NN93ExIkTcXJyYtq0aT9JRP6vOXPm0NnZydNPP81tt92G2Wxm5syZP3uvXq9n+fLlXH/99SQlJZGamsqCBQuYNWuW7Z6+vj7mzp1LWVkZrq6uTJ06laeffhqAwMBAHn74Ye666y4uv/xyLr30Ut566y0eeeQRvL29efzxxzlw4ADu7u6MHDmSe+6551djH+gU64+b/MVRdWhGZ1NT0xGzH05EPT09rF69munTp//ueRRi4Orps7CntJH0/sTnntJGLD/6iuKg1ZAa5mE71CjOzxXNUWhzl/eTOJrk/SQALG1tdO7d25/8zKQjM4Peisqf3KcYjRiHD8eUlGSb+akPDLQd8DPY309Wq5Xy1nKyarPIqFXb3vfW7aWrr+sn9wY4BRyR+IzzisOkk6r+o2mwv5/EwCLvJ3E0yftpaOrs7KSwsJCwsDCMRuNxfW2LxUJzczOurq7HbEanGNh+7f33e/Jrg7MkQQgxIOi1GkaHejI61JP5p0fT1N7DxgI16ZmeW0t5Ywcb8+vYmF/Hwi/B7OzA+Eh1tuf4KDM+Lsf3L08hhPglGicnHFNTcUxNtV3rqa6mMyvLdtBRR2YWlpYWOnbupGPnTtt9Wk9PTImJGJMS0cfHo+mf8TQYKYpCkEsQQS5BTA2bCkCPpYf8hvwjqj4PNB2goq2CirYK/lv0XwC0ipYojyi15d2szvsMcwtDq9Hac0tCCCGEEOIEIolOIcRR4+aoZ3qiP9MT/bFarRTWqm3u6bk1bDpQR21rN//ZXcF/dlcAEOvnwsRobyZGeTM61AOjXn4YFkIMHHofH/SnnorLqacCYLVY6C4qVud97smgIzOTzn376Kuvp3XdOlrXrQMgEih+863+qk+18tMQF4fGMDgP+NFr9MR5xRHnFcfsmNkAtHa3kl2XbUt8ZtZmUtNRw776feyr38fHuR8D4KR3It4r3pb8TDAn4Ovka8/tCCGEEEKIIUwSnUKIY0JRFMK9nQn3dmbOyaF091rYWdLA+rwa1ufVklnexL6DLew72MIr6Qcw6DSMCfdiYpSZCVHeRPs621pBhRBiIFA0GgzhYRjCw3A75xwALN3ddOXk2NrdOzIy6CkqpqekhJ6SEppXrVIf1usxxsQcPugoKRGHsDCUQdqa5ezgzBj/MYzxHwOoLe9V7VVHVH1m12XT1tPG1oNb2Xpwq+1ZH0cfW7t7ojmReHM8TvrBeeiTEEIIIYQYWCTRKYQ4Lhx0GsaGezE23Ivbz4S61i42FtSxPreG9Lwaqpq7SM+tIT23BsjB19Vgm+05PtKMl/PgrIQSQgxtGgcHTMnJmPqH1Pf09PDVxx8zMSCAnr171crPjAz66uvpzMqiMysLWK4+6+yMMTEBU2ISpuQkjImJ6H187LibP05RFPyc/PBz8uP0YacD0Gfpo6CpQJ33WaPO+8xrzKO6vZrvSr7ju5Lv1GdRiHCPINGcaDvpPdI9Ep1Gvk0VQgghhBC/j3wHKYSwCy9nA39JDuAvyQFYrVbyqltJz1WrPbcU1lHV3MXHO8r4eEcZAAmBrkyM8mZClDdJAc52jl4IIX6ZxdERx5NPRn/KKYBa7dhbUXHEQUed2XuxtLbSvmkz7Zs2257V+fkdPugoMQljfDxa58FZ7ajVaIn2iCbaI5q/Rf0NgPaedvbW7T3isKPKtkryG/PJb8zn3/n/BsCoNTLca7ia/PRW2979nfyl0l8IIYQ4DuTMamEPR+t9J4lOIYTdKYpCtK8L0b4uXDUhnM6ePnYUN6gVnnm15FQ2k1WurhfWFuDooCXUUUONRzGT4/wINzvJD79CiAFLURT0gYHoAwNxnTYNAGtvL135+epBR5mZdOzJoCs/n96DB2k5eJCWr79WH9ZoMEREYExKtFV+GqKiUHSD81s4R70jo/1GM9pvtO1abUetbc5nZm0mWbVZtPa0srN6JzurDx/65Gn0tM35TPRWqz9dHX791E0hhBBC/HZarXpmQnd3NyaTyc7RiBNNd3c3cPh9+EcNzu+ShRBDmlGvZVykmXGRZu4Gqls62ZhfS3queqJ7bWs3e7s17F29nwWr9xPobmJC/2zPcZFeuDs62HsLQgjxqxSdDmNsLMbYWJitHvBjaWujc+/eIyo/eysq6crLoysvj6ZP/qU+azRiHD78cOVnUhL6wMBB+wsfs8nM5JDJTA6ZDIDFaqGoueiI5GdufS71nfWsLVvL2rK1tmdDXUPVWZ/e6rzPGI8Y9Fq9nXYihBBCDG46nQ5HR0dqamrQ6/VojuMscYvFQnd3N52dncf1dcXAYLFYqKmpwdHREd2f/IW+JDqFEAOej4uRv44I4q8jgrBYrGSVNfDaqg3U6r3ZXtxIeWMHH24r5cNtpWgUSApyVw81ivYmJdgdvVb+ohRCDHwaJyccU1NxTE21XeutqaEjM1Ot/MzIpCMzE0tLCx07d9Kx83C1o9bTE1Niolr5mZSEKTERrbu7HXbx52kUDeFu4YS7hXNOpHroU1dfFzl1ObaW98yaTMpayyhqLqKouYjPD3wOgIPGgViv2CMOOwp2CR60SWAhhBDieFIUBX9/fwoLCykuLj6ur221Wuno6MBkMsnf2ycojUZDSEjIn/7/XxKdQohBRaNRiPN3YUqglenTR9Nr1bClsI71eWq1Z25VK7tLG9ld2siza/JxMeg4KcKLCdHeTIwyM8xrcM66E0KcmHTe3riceioup54KgNViobuomM7M/qrPjAw69+2jr76e1nXraF23zvasfliI2u7en/w0xMWhMQzOg90MWgMpPimk+KTYrjV0Ntha3Q/N+2zqaiKjJoOMmgzbfW4GN/WQo0Nt7+ZEPIwedtiFEEIIMfA5ODgQFRVlayM+Xnp6ekhPT2fixIno9dKdcSJycHA4KtW8kugUQgxqJgctk2J8mBSjnlR8sKmT9Dz1UKMNeTU0tPfw9d4qvt5bBUCIpyMTo9U295MivHA1yl+iQojBQ9FoMISHYQgPw+0ctdrR0t1N17596gnvmWrlZ3dRET3FJfQUl9C8apX6sE6HMSam/4R3NQHqEBaGMkjbwzyMHkwMmsjEoImAWglS2lJqa3fPrM1kX90+mrqa2Fi+kY3lG23PBjkH2drdE82JxHrGYtQZ7bUVIYQQYkDRaDQYjcf370WtVktvby9Go1ESneJPkUSnEGJI8XMzMnt0MLNHB2OxWMmuaCY9r4b03Bp2FDdQUt/Oe5tLeG9zCVqNwohgdyZEeTMx2kxSkDtajbRJCCEGF42DQ/+8ziTbtb7GRjqyso+o/Oyrq6MzO5vO7GxgufqsszPGxARb5acxKQm9j4+ddvLnKIpCiGsIIa4hnBV+FgA9fT3kNuTaKj4zajIoai6irLWMstYyviz8EgCdoiPaM/qIlvdQt1A0yuBMAgshhBBCnKgk0SmEGLI0GoXEIDcSg9yYOzmS1q5ethxQ29zTc2s4UNvG9uIGthc38PS3ubgadYzvP9RoQpSZIA9He29BCCH+EK27O87jx+E8fhygVjv2VlSo8z4PVX5m78XS2kr7ps20b9pse1bn5/ejeZ/JGOPj0ToPzrEfeq2eeHM88eZ427Xm7mayarPIrDnc9l7fWc/eur3srdvLiv0rAHDWOxNvjifJnGQ78MhsMttrK0IIIYQQ4jeQRKcQ4oThbNAxJc6XKXG+AJTWt7MhX53tuSGvlubOXlZnHmR15kEAwr2dmNif9Bwb7oWTQb5kCiEGJ0VR0AcGog8MxHXqVACsvb105eercz4zM+nIyKQrL4/egwdpOXiQlm++OfQwhsgIjElJtspPQ1QUyiBtK3N1cOXkgJM5OeBkQE0CV7ZVqlWfNVlk1mayt24vrT2tbKncwpbKLbZn/Z38j5j3OdxrOI56+aWYEEIIIcRAIT+1CyFOWMGejlyYFsKFaSH0WazsKWtkfa6a+NxV2siBmjYO1LTx1g9F6LUKI0M8mBjtzcQob+IDXNFIm7sQYhBTdDqMsbEYY2Nh9mwALG1tdO7dq7a7Z2bSkbGH3opKuvLy6crLp+mTf6nPGo0Yhw8/XPmZnIw+MHBQnpKqKAoBzgEEOAcwNVRNAvdaeslvzFdnfdao8z4LGguobKuksq2Sb4rVJLBG0RDpHnm45d07kQi3CLQarT23JIQQQghxwpJEpxBCAFqNmsgcGeLBLadF0dzZww/5dazPqyE9r4bS+g62FNazpbCep/67H08nB8ZFmpnY3+ru5yaHWAghBj+NkxOOqak4pqbarvXW1NCRmUVHxh46+xOglpYWOnbupGPnTtt9Wg+P/nZ3dV6oMSEBncfgPN1cp9ER6xlLrGcss6JnAdDW00Z2bfYRhx1Vt1eT25BLbkMun+R9AoBJZyLeK/6Iw458HX0HZRJYCCGEEGKwkUSnEEL8DFejnqkJfkxN8AOguK6N9P7ZnpsK6qhv6+bzPRV8vqcCgGhfZ9tszzFhXpgcpJpHCDE06Ly9cTl1Mi6nTgbAarHQXVxMZ0aGrfKzKyeHvoYG2tal07Yu3fasPiSkP/GZiDExEePw4WgMBntt5U9x0juR5p9Gmn+a7VpVW5VtzmdWbRZZtVm097azvWo726u22+7zNnnbKj4TzYnEe8Xj7OBsj20IIYQQQgxpkug8BpYtW8ayZcvo6+uzdyhCiKNkmJcTl3g5ccnYYfT0Wdhd2sj63BrS82rZU9ZIblUruVWtvL6hEAedhrRQTyb0V3vG+btIJY8QYshQNBoMYWEYwsJwO+ccACzd3XTt29d/wrta+dldVERPSQk9JSU0r1qlPqzTYYyJsR10ZEpKxCEsDEUzOE8393XyxdfJlynDpgDQZ+mjsKnwiKrPvIY8ajpqWFO6hjWlawBQUAh3C1fnfXqr8z6jPKLQawbn3FMhhBBCiIFCEp3HwNy5c5k7dy7Nzc24ubnZOxwhxFGm12pIDfUkNdST+WfE0NjezcZDbe65NVQ0dbIhv5YN+bU8/uU+zM4GtcU92sz4SG+8XQZnNZMQQvwSjYODrWUdLgKgr6mJjqysw5WfGRn01dXRmZ1NZ3Y2jcs/VJ91dsaYkPCjys8k9L4+dtzNH6fVaIn0iCTSI5K/Rv0VgI7eDnLqcg4nP2syqWiroKCpgIKmAj4t+BQAg9ZAnGfcES3vPobB+XkQQgghhLAXSXQKIcSf5O7owFlJ/pyV5I/VauVAbRvpuTWsz6tlU0Edta1d/GtXOf/aVQ5AnL8rE6PNTIzyZtQwD4x6aXMXQgw9Wjc3nMeNw3ncOEA93by3oqL/kKP+ys/svVhaW2nfvJn2zZttz+r8/A4fdJSUjDE+Hq2zk7228qeYdCZG+o5kpO9I27XajlqyarNsic+s2ixaelrYXbOb3TW7bfd5GDzw6fOhPLOcZN9kEswJuBnkl+hCCCGEEL9EEp1CCHEUKYpChLczEd7OXD4ujK7ePnYWN5KeV8P6vBqyypvJqVTXy+sOYNRrGBPmxYQoM6dEexPp4yxt7kKIIUlRFPSBgegDA3Gdqp5ubu3tpauggI6MDFvlZ1deHr0HD9Jy8CAt33xz6GEMkREYE5NslZ+GqCgU/eBs9TabzEwKnsSk4EkAWKwWipuL1XmfNeq8z30N+2joaqCBBvZn7odM9dlhrsNINCeqbe/mJGI8Y3DQOthvM0IIIYQQA4gkOoUQ4hgy6LScFOHFSRFe3Dk1lrrWLjbk17I+r5b1eTVUNXexLreGdbk1LPgiBz9XozrbM9qb8ZFmPJ3kh1chxNClHJrZGRMDs9TTzS3t7XRmZ9sOOurMyKCnooKuvHy68vJp+te/1GeNRozDh/+o8jMJfVDQoPxlkUbREOYWRphbGGdHnA1AV18X2dXZrFy/Eouvhey6bEpaSihuLqa4uZhVB9S5p3qNnljPWBLMCbaW92Guwwbl50EIIYQQ4s+SRKcQQhxHXs4GzkkJ5JyUQKxWK7lVrepsz7xathyo42BzJx/tKOOjHWUoCiQGutkONRoZ4oGDbnAe2CGEEL+VxtERx9RUHFNTbdd6a2royMyiIzODzj0ZdGRmYmlpoWPnTjp27rTdp/XwUJOeiUmYkpMwJiSg8/Cwxzb+NIPWQKI5kVJDKdNPno5er6exs5Gsuiwyaw4fdtTY1Wj738tZDoCrg+sRic9E70Q8jZ523pEQQgghxLEniU4hhLATRVGI8XMhxs+FqyaE09nTx7aietbn1ZKeW8O+gy1klDWRUdbEsu8LcHJQq0MnRHkzIcpMmNlJKnaEECcEnbc3LqdOxuXUyQBYLRa6i4sPH3SUmUlXTg59DQ20rUunbV267VmX00/D/7HH0Lq42Cv8o8bd6M74wPGMDxwPqHNPy1rLjkh85tTl0NzdzA8VP/BDxQ+2ZwOdA49IfMZ5xmHUGe21FSGEEEKIY0ISnUIIMUAY9dr+JKY390yPo7q509biviG/ltrWbr7NqebbnGoAAt1NtkONTo4w4+Y4OGfVCSHE76VoNBjCwjCEheF2zjkAWLq76dq3j46MTDozM+jYk0F3UREt33xLV14+QS8swxAebufIjy5FUQh2CSbYJZjp4dMB6OnrIbcxl6yaLDJq1XmfB5oOUN5aTnlrOV8VfQWAVtES7RF9eN6ndxJhbmFoFOkcEEIIIcTgJYlOIYQYoHxcjZw3KojzRgVhsVjJOdhsq/bcXtRAeWMHy7eWsnxrKRoFkoPdmRDlzcQoMynB7ui08sOqEOLEoXFw6D+oKAm4CICOrGzKbryR7qIiimafT8BTT+IyebJ9Az3G9Fo98V7xxHvFcz7nA9DS3UJ2XTaZNZlk1GaQWZNJXWcdOfU55NTnsDJ3JQBOeicSvBLUtnfvRMb6j8VJPzhPuxdCCCHEiUkSnUIIMQhoNArxAW7EB7hx3SkRtHf3sqWwnvW5asVnXnUru0oa2VXSyLPf5eFi0HFypFd/4tObEC9He29BCCGOO1NCPGEff0TZLbfSsWMHZTfMxfuWW/C69poTavSHi4MLY/3HMtZ/LKC2vB9sO2hrd8+szWRv3V7aetrYcnALWw5uAcDX0ZfFkxaT7J1sz/CFEEIIIX4zSXQKIcQg5OigY3KMD5NjfACoaOxgQ14t6f1t7o3tPfw3u4r/ZlcBEOrlaJvteVKEFy5GaXMXQpwYdGYzw958g4OPP07j8g+pWbqUzn37CHjsUTSOJ+YvgRRFwd/ZH39nf84IPQOAXksvBY0FZNZmklWbxcaKjRxsO8hlX13GHal3cEHMBSdUclgIIYQQg5MkOoUQYggIcDcxOzWY2anB9FmsZJU32U5z31ncQFFdO0V1xby7uRitRmFkiDsTo7yZEO1NYqAbWo388CqEGLoUBwf8H3wQY2wcBxcsoOWrrygqLCRo2fM4BAXZO7wBQafREeMZQ4xnDDOjZ9LW08b9G+/nm+JveGzLY+yu3s2DJz2Io/7ETA4LIYQQYnCQRKcQQgwxWo1CcrA7ycHu3HhqFK1dvWwuqCM9r4b1ebUU1raxraiBbUUNLP4mFzeTnvGRZiZGm5kQ5U2Au8neWxBCiGPC4/zZGCIjKLvlVrr276do5iwClz6N09ix9g5twHHSO7H4lMW8u/ddluxYwurC1eQ25LJk0hLC3MLsHZ4QQgghxM+SRKcQQgxxzgYdpw335bThvgCU1rcfcZp7U0cPX2RW8kVmJQAR3k7qbM9oM2PDvXB0kL8qhBBDh+OoUerczhtvojMri5Irr8L3zjvxuORiac3+H4qicGn8pcSb47l93e3kN+Zz4RcX8si4Rzh92On2Dk8IIYQQ4ifkp1chhDjBBHs68vcxIfx9TAi9fRb2lKlt7uvzatlV0kBBTRsFNW289UMReq3C6GGeTIg2MzHKm+H+rmikzV0IMcjp/fwY9t67HHzwQZo+/Yyqxx6jMycHv4ceRGMw2Du8AWeU7yhWnr2S29fdzvaq7cxfO585w+dwy6hb0Gtk5rMQQgghBg5JdAohxAlMp9UwapgHo4Z5cOtp0TR19LCpoJb0vFrSc2soa+hg04E6Nh2o48mv9uPl5MD4KLPtYCNfV6O9tyCEEH+IxmjEf+FCDHFxVD/5FE3//jddBQUEPfccel8fe4c34JhNZl4941We3fUsb2a9ydt73yazNpNFpyzC29Hb3uEJIYQQQgCS6BRCCPEjbiY9UxP8mZrgj9VqpbiunfS8GtJza9lUUEtdWzef7q7g090VAMT4ujAhyszEaG/Swjwx6rV23oEQQvx2iqLgddllGKKiKJ//DzozMiiceR5Bzz6L44gR9g5vwNFpdMwfNZ9kczL3bryXndU7mfX5LBadsojRfqPtHZ4QQgghhCQ6hRBC/DxFUQg1OxFqduLSk0Lp6bOwq6SR9Nwa1ufVkFHexP6qFvZXtfDahkIcdBrGhHnaEp8xvi4y704IMSg4jxtH2EcrKZt7I115eZRcOge/Bx/AfeZMe4c2IE0ZNoUI9wjmrZ1HfmM+V319FbeOvJU58XPk674QQggh7EoSnUIIIX4TvVZDWpgnaWGe3HZmDA1t3WwsqGV9bi3peTVUNnX2H3JUy2Or9+HtYlCTnlHejI8yY3aWuXdCiIHLISSE0A+XU3HX3bR88w2V991PZ84+fO+6E0Uvcyj/V6hbKO9Pf59HNj/CqgOrWLxjMXtq9vDPcf/ExcHF3uEJIYQQ4gQliU4hhBB/iIeTAzOSApiRFIDVaqWgppX0XPU0980H6qlp6eJfO8v5185yAOIDXNXT3KPMjAr1wKCTNnchxMCicXIi8Jml1L70ErXPPkfD++/TlZdH4NKn0Xl62ju8AcdR78hj4x8jxTuFhdsW8m3Jt+Q15vH0pKeJ8oiyd3hCCCGEOAFJolMIIcSfpigKkT4uRPq4cMX4MLp6+9hR1EB6npr4zK5otq2X1hVg0msZG+6pJj6jzUR4O0u7oxBiQFA0GrxvuAFjTAwVt99B+9atFM2cRdCy5zHGxdk7vAFHURTOjz2f4V7Dmb9uPsXNxVy0+iIeOOkBZoTPsHd4QgghhDjBSKJTCCHEUWfQaTk50szJkWbumhZLbWsXG/NrWZdbw/q8Wmpauvh+fw3f768BwN/NaJvtOS7CjIeTg513IIQ40blMmULoyhWUzp1LT3EJRRf+nYDHHsV1+nR7hzYgJXonsnLGSu5afxc/VPzA3evvZnf1bu5IvQMHrXxNF0IIIcTxIYlOIYQQx5zZ2cA5KYGckxKI1Wplf1WLbbbnlsJ6Kps6Wbm9jJXby1AUSAp0Y0KUNxOizIwc5oFeq7H3FoQQJyBDZCRhK1dS/o/baNuwQT2ZPScH71tvRdHK+I3/5WH04IUpL/BSxku8tOclVuxfwd66vSw+ZTH+zv72Dk8IIYQQJwBJdAohhDiuFEUh1s+VWD9Xrp4YTmdPH1sL61mfp1Z77jvYwp6yJvaUNfH89/k4OWg5KcLMxGgzE6K8CfVylDZ3IcRxo3VzI/jll6h5+mnqXnuduldfo3P/fgIXLULr6mrv8AYcrUbL3JS5JJoTuXv93WTWZjJ71WyemPAEJweebO/whBBCCDHESaJTCCGEXRn1WiZGezMx2huAquZDp7fXsCGvlrq2br7NqeLbnCoAgjxM6v1RZk6KMONmktOQ7aZkM+R/B0GpEDQaHOWwFjE0KVotPrfdhiE2jsp776UtfT1Fs2YT9MIyDBER9g5vQJoYNJGVZ69k/tr57K3by3XfXsfclLlcnXQ1GkWq9IUQQghxbEii8xhYtmwZy5Yto6+vz96hCCHEoOPramTmqCBmjgrCYrGyt7KZ9Lwa1ufWsr24nrKGDj7YUsIHW0rQKJAS7M7EaG8mRHmTHOSGTtrcj5/9q2HjM4f/7BV5OOkZlAY+w0Er32qIocNtxlk4hIVSduNNdBcXUzT7fAKeegqXUyfbO7QBKdA5kHemvcPCrQv5OPdjnt/9PHtq9vD4hMdxM7jZOzwhhBBCDEHy08cxMHfuXObOnUtzczNubvJNnBBC/FEajUJCoBsJgW7cMCmStq5ethbW9x9qVENBTRs7SxrZWdLI0m/zcDHqGBdhZkK0mYlR3gR7Otp7C0Nb8FhIroGyrVCXf3jtWa7+c70jBIxUE5/BaRA4Glx87RuzEH+SKT6esI8/ovyWW2nfvp2yuXPxvvkmvK67TsZq/AyD1sCDJz1IsncyCzYvYH35es5fdT6LJy0m3ive3uEJIYQQYoiRRKcQQohBw8mgY3KsD5NjfQAob+xgQ14N6Xm1bMirpamjh6+yD/JV9kEAwsxOTIhSZ3ueFOGFs0H+2juqYqerC6C9Hsp3QNk2KN2q/u+uZijeoK5D3EP6qz7T1P/2SwSdnMgsBhedlxchb75B1eOP0/DBcmqeeZbOnH0EPP4YGicne4c3IJ0beS6xnrHM+34eZa1lXLr6Uu4Zcw/nRZ9n79CEEEIIMYTIT3xCCCEGrUB3E+enhnB+agh9FiuZ5U2sz1UPNdpZ0kBhbRuFtW28s6kYnUZh5DAPJvYnPhMC3dBqpPrqqHH0hKjT1QVgsUBtrpr4LNsKZduhOgcaS9SV9Yl6n9YA/slq0jM4Vf1v10CQyjgxwCl6PX4PPIAhNpaDjyyg5euvKSoqImjZ8zgEB9s7vAEp1jOWFWev4N7197K2bC0PbXqI3TW7uXfMvRh1RnuHJ4QQQoghQBKdQgghhgStRiEl2J2UYHdumhJFS2cPmwrqbAcbFdW1s7Wwnq2F9Sz6OhcPRz3jIs2cHO5Jd5e9ox+CNBrwiVXXyEvUa53NULETSrf1J0C3QUd9fyJ0K2zuf9bF//Ccz6BUCEgBvcleOxHiV3nMno0hMpKym2+hKzeXopmzCFz6NE4nnWTv0AYkVwdXnjn1Gd7IeoPndj3Hf/L/w776fSw5ZQnBrpIgFkIIIcSfI4lOIYQQQ5KLUc8Z8X6cEe8HQEldu3qoUV4NP+TX0dDew6qMSlZlVAI63i3daDv9fUyYJ44O8lfkUWd0hfBJ6gKwWqH+gFrtWbZVTXwezIKWSsj5XF0AGh34JqhzPg8dduQRJlWfYsBwHDmSsI8/ouymm+nMzKTkqqvxveN2PC69VOZ2/gyNouGqxKtIMCdwZ/qd7Kvfx/mrzuexCY8xKXiSvcMTQgghxCAmP8UJIYQ4IYR4OXKx1zAuHjuM3j4Le8oaSc+tZV1uNXtKGymoaaOgpo03NxbhoNUwOtSDCVHeTIgyM9zfFY20uR99igJeEepKPl+91t0OlbvVOZ+Hqj5bq9Rrlbth6yvqfY5e/UnP/hU4EgwudtqIEKD382PYe+9y8IEHafr0U6oeX0hnzj78Hn4IjcFg7/AGpLH+Y1kxYwW3rbuNPTV7uGnNTVydeDVzU+ai1WjtHZ4QQgghBiFJdAohhDjh6LQaRg3zZNQwT26cFMbHn63GOWIUPxxoID23hvLGDn4oqOOHgjqe+ArMzg6MjzTbEp8+rjJL7phxcIRhJ6sL1KrPprLDcz7LtkHlHmivg9yv1AWgaMA77vCcz6BU8IpSW+iFOE40BgP+Cx/HODyOqiefouk//6GroICg559D7+tr7/AGJD8nP948800W71jM+znv82rmq2TUZvDEhCfwMnnZOzwhhBBCDDKS6BRCCHHCc9TB1Hhfzk4Jwmq1UljbZpvt+UNBHbWt3fxndwX/2V0BQKyfCxOj1aRnaqgnRr1UHh0zigLuwepK6D+dubcLDmb+qOpzOzSVQHW2una8pd5ndIPA0T+q/BwFJg+7bUWcGBRFwXPOHAzR0ZTfOo/OzEwKZ84k6JlncRw5wt7hDUh6rZ670u4i2TuZB394kC2VW5i9ajaLT1lMik+KvcMTQgghxCAiiU4hhBDiRxRFIdzbmXBvZ+acHEp3r4WdJQ2sz1NPc88sb2LfwRb2HWzhlfQDGHQaxoR72U5zj/Z1lpl8x5rO0H9Y0ejD11oOHm51L9sO5TuhswkKvlPXIV5R/bM++xOg3nGglW+HxNHndNJJhH78EWU3zKUrL4/iOXPwe+B+PGbNsndoA9a0sGlEe0Qzb+08CpsKufyry7kt9Tb+Hvt3+boqhBBCiN9EvrMXQgghfoWDTsPYcC/Ghntx+5lQ39bNxvxa0nPVxOfB5k7Sc2tIz60BcvBxMTAhypuJ0WbGR5rxcpbZfMeFix/Ena0ugL4eqMo+nPgs2wb1BVCXp67d76v36Z3U+Z4/nvfp7G2/fYghxSE4mNAPl1Nx9z20fP01B+9/gK6cHHzvvhtFr7d3eANShHsEy89azgMbH+Dr4q9ZuHUhe2r28NBJD+God7R3eEIIIYQY4CTRKYQQQvwOnk4OnJ0cwNnJAVitVvKrW0nvb3PffKCO6pYuPtlZxic7ywBICHS1zfYcPcwTB53MjDwutHoISFFX2tXqtbY6KN/+o8rPHdDdAkXr1XWIR+iPEp+jwTcRdA522IQYCjROTgQ+s5S6l16i5plnafhgOV25eQQ+sxSdl8yg/DlOeicWnbKI93LeY8n2JXxZ+CW59bksmbyEcLdwe4cnhBBCiAFMEp1CCCHEH6QoClG+LkT5unDl+DA6e/rYUdxAel4N63Nr2VvZTFa5ul5cW4Cjg5ax4V5M6G9zj/B2knbM48nJC6LPVBeApQ9q9h/Z8l6zDxqK1JX5kXqfzgj+KYfb3YPTwDXATpsQg5GiKJivvx5DTCwVt99O+/btFM6aRdBzz2GKj7d3eAOSoihcMvwS4r3iuW3dbRQ0FXDhqgt5ZNwjnBF6hr3DE0IIIcQAJYlOIYQQ4igx6rWMizQzLtLM3dOguqWTjfm1rM+tJT2vltrWLtbsq2bNvmoAAt1NtqTnuEgv3B2lavC40mjBd7i6Rs1Rr3U2QfmOw+3uZdugowFKN6vrENfAw4nPoFQ1Eao32mUbYvBwOXUyoStXUHbDXLqLiym+6GL8FyzAbcZZ9g5twBrpO5KVZ6/kjvQ72HZwG/9Y9w8urbmUW0fdil4j7f9CCCGEOJIkOoUQQohjxMfFyF9HBPHXEepp7vsOtthme24tqqe8sYMPt5Xy4bZSFAWSgtw5JcrMhGhvUoLd0Wulzf24M7pBxKnqArBaoa7gR1Wf29TZn83lsLcc9n6q3qfRg18imoBRBNZroTEezBHqqfFC/IghIoLQj1ZS/o/baFu/norbbqNrXw7e8+ahaLX2Dm9AMpvMvHL6Kzy36zneyHqDd/a+Q1ZtFk+d8hQ+jj72Dk8IIYQQA4gkOoUQQojjQFEU4vxdifN35dpTIujo7mNrUT3rc2tIz6sht6qVPaWN7Clt5Nk1+TgbdJwUoZ7mPjHam2FeTvbewolJUcAcqa6UC9Vr3W1Qsetwu3vpVmirhoqdaCt2Mhpg2Uvg5H14zmdQGgSMAIOzPXcjBgitqyvBL71IzdKl1L36GnWvvU7n/lwCFz2F1s3N3uENSDqNjnmj5pHkncR9G+5jZ/VOZn8+m6dOeYpUv1R7hyeEEEKIAUISnUIIIYQdmBy0nBLtzSnR6gnfB5s6WZ+nVntuyK+lvq2bb/ZW8c3eKgBCPB1tbe4nR3rhapSWTbtxcILQ8eoCteqzsQTKttFXspWm7G/x6CxBaauB/avVBaBowCf+yFmfnhGgkcrdE5Gi1eLzj39giI2l8t77aFu/nsLZswletgxDZKS9wxuwpoRMIXJGJPPWziOvIY+rv76aW0bewmXxl8nMYyGEEEJIolMIIYQYCPzcjMwaHcys0cFYLFayK5rVQ43yathR3EBJfTvvbynh/S0laDUKI4Ld1dPco80kBbqhkzZ3+1EU8BgGHsOwxJ7D+t5xTD/jVPS1OYfb3Uu3QXMZVGWqa8eb6rNG9yNnfQaOApO7PXcjjjO3s87CEBZG6Y030lNcQtH5FxDw1JO4nHqqvUMbsIa5DuP96e/zyKZH+PzA5yzZsYQ9NXt4ZNwjuDi42Ds8IYQQQtiRJDqFEEKIAUajUUgMciMxyI25kyNp6+pl84E61ufVkp5Xw4GaNrYXN7C9uIGnv83F1ahjXKTa4j4hykyQh6O9tyB0RrViMzjt8LXmiiMPOarYBZ2NkP+tug4xx/RXfPYnP71j1YOTxJBlHD6csI8/pvyWW2nfto2yG+Zivvkm3K680t6hDVgmnYlHxz9Kik8KC7cu5LuS78hryGPJpCXEeMbYOzwhhBBC2IkkOoUQQogBzsmgY0qcL1PifAEoa2hnQ3/Sc0NeLc2dvXyZdZAvsw4CEG52YkL/bM+x4V44GeSv+wHBNQCG/0VdAH09UJV1eM5n2TZoKITa/era/Z56n4MzBI5U53wemvnpZLbfPsQxofP0JOSN16la+AQN779P7bPP0bF3L8rEifYObcBSFIXZMbMZ7jWc+WvnU9JSwsWrL+aBkx7g7Iiz7R2eEEIIIexAfvIRQgghBpkgD0cuSAvhgrQQ+ixWMsoa1WrP3Bp2lTZyoLaNA7VtvL2pGL1WYWSIh63aMyHADY1G5tgNCFq9ekBRwAhIu1q91lbbX/XZn/gs3wndrVCYrq5DPMIOz/kMGg2+CerHE4Oaotfjd/99GONiqXz4n7R9+x0hWVn0jE5FHx5m7/AGrARzAitmrOCu9XfxQ8UP3LPhHnZX7+bOtDtx0DrYOzwhhBBCHEeS6BRCCCEGMa1GYUSIByNCPLh5ShTNnT1sKqhjfV4N6bm1lNS3s6Wwni2F9Tz13/14OOoZH+Xdf7CRGX83k723IH7MyQwxU9UFYOmDmn2H53yWbVOrPRsK1ZW5Ur1PZ1QTpodmfQalgqu//fYh/hT3mTNxiIig7KabMRysovSCCwha+jROJ59s79AGLA+jBy9MeYGXM17mpT0vsTJ3JXvr9rJk0hL8neXfBSGEEOJEIYlOIYQQYghxNeo5M96PM+P9ACiuayM9r5b1uTX8UFBHQ3sPn++p4PM9FQBE+zozIUo9/X18pFmqPQcajRZ849U16jL1WkcjlO84POuzbBt0NkHJJnUd4hp0eM5nUCr4JYHeaI9diD/AccQIgj9cTs7lV2AqLaXkqqvxueN2POfMkdPFf4FWo+WGlBtINCdy94a7yarLYvaq2SycsJBxgePsHZ4QQgghjgNJdAohhBBD2DAvJy7xcuKSscPo6bOwp7SR9Nwa0vNqyShrJLeqldyqVl7fUMhZif48c0GKnOA+0JncIXKKugAsFqgvODzns2w7VGerp7xnl0H2v9X7NHrwT+qf9dl/0rt7iHpqvBiQdL6+lF17DSO3bqPls8+oXvgEXTk5+D38MBqjJK1/yYSgCayYsYL5a+ezt24v1397PdenXM+1SdeiUeTrmxBCCDGUSaJTCCGEOEHotRpGh3oyOtST+WfE0NjezQ8FdaTn1vCvneV8kVmJRqPw9OxkSXYOJhoNmKPUNeIi9VpXq3qqe9nWw4cdtdeqlaDlO2BL/7POvocPOApKVdvfHZzsthXxU1a9Hp8Fj+CYEE/VE0/S9OlndBUcIOj559D7+dk7vAEr0DmQd6a9w8KtC/k492Ne2P0CGTUZLJywEDeDm73DE0IIIcQxIolOIYQQ4gTl7ujA9ER/pif6c1qcL9e/v4PP91SgVWDx7BS00sY+eBmcIWyCugCsVmgsPjzns2wbHMyA1irYt0pdAEp/q/yPZ316RUjVp50pioLnpZdiiI6m/JZb6czKonDmLIKefQbHkSPtHd6AZdAaePCkB0n2TmbB5gVsKN/A7M9ns2TyEuK94u0dnhBCCCGOAUl0CiGEEILThvvy3IUjufGDnfxndwVajYanZibJzM6hQlHAI1RdSbPUaz0dULnnR7M+t0NzuZoAPZgB219X7zN5/CjxORoCR4FRKuLswWnsWEI/+ZiyuTfStX8/xXMuw+/++/CYPdveoQ1o50aeS5xnHPPWzqO0pZRLVl/CPWPu4byo82TeqRBCCDHESKJTCCGEEABMTfDj2QtHcNPyXXyyswydRuHxvyVKsnOo0psgZKy6DmkqPzLxWbELOhog72t1AaCAd+zhdvegVPXPGhl3cDw4BAURuvwDKu65l5avvuLgAw/SmZOD3913ozg42Du8ASvGM4YPZ3zIvRvuZW3pWh7e9DC7q3dz39j7MOpk3qkQQggxVEiiUwghhBA20xP96bNYueXDXazYXopGo/DouQmS7DxRuAWqK/5c9c+93VCVqSY9DyVAG4qgJkddu95V7zO4QuDII1veHT3ttYshT+PoSODTS6iLjaXmmWdoXP4hXXl5BD3zDDovL3uHN2C5OrjyzORneDPrTZ7d9SyfFnzKvvp9PD3paYJdg+0dnhBCCCGOAkl0HgPLli1j2bJl9PX12TsUIYQQ4nc7OzkAi9XKvBW7Wb61BJ1G4Z/nxEuL54lI56C2qgeOgjHXqtdaq49MfJbvhK5mOLBWXYd4Rhxudw9OA5940Mq3nkeLoiiYr7sWQ0w0FbfdTsf2Herczueew5Qg8yd/iUbRcGXilSSaE7k9/Xb2N+zn/FXn8+j4R5kcMtne4QkhhBDiT5LvNo+BuXPnMnfuXJqbm3FzkxlWQgghBp9zUgLp7bNy28d7eHdzMVqNwoNnD5dkpwBnH4idri6Avl61uvNQu3vZNqjNhfoCdWV8qN6nd1RPdbe1vKeBi6/99jFEuEyeTOhHKym7YS7dRUUUX3QR/gsW4Hb2DHuHNqCl+aexcsZK/rHuH+yp2cPN39/MVYlXMTdlLjqN/IgkhBBCDFbyt7gQQgghftZ5o4Los1i545MM3vqhCK1G4b6z4iTZKY6k1YFforpGX6Fea69XKz1/PO+zqwmKN6rrELeQw4nP4DT1Y+gM9tnHIGYIDyd05QrKb7+dtnXpVNx+O505Ofj8Yz6KVmvv8AYsXydf3jzzTZbsWMJ7Oe/xWuZrZNZk8sTEJ/AyyQgAIYQQYjCSRKcQQgghftHs1GB6LVbu+Xcmr28oRKdRuGtarCQ7xa9z9ISo09QFYLFAXd6Ric/qvdBUoq7sf6n3aR3AP/nIWZ9uQeqp8eJXaV1dCX7hBWqeeZa6V16h/o036Nq/n8Ali9FKh9Ev0mv13Jl2J8neyTzwwwNsObiF2atms/iUxaT4pNg7PCGEEEL8TpLoFEIIIcSv+vuYEPqsVu7/TxYvpx9Ap1W47YwYSXaK306jAe8YdY24WL3W1fKjqs/tULYV2usOJ0MPcfY7surTPwUcHO2yjYFO0WrxmT8PY1wsFffcS9vGjRTOmk3wsucxREXZO7wBbWrYVKI8opi3dh6FTYVc/tXl3JZ6G3+P/bt8rRNCCCEGEUl0CiGEEOL/dcnYYfT1WXjo870s+74AnUbDvNOj7R2WGMwMLhB+iroArFZoKDw857N0K1RlQetB2LdKXQCKFvwSDs/5DBoNnuFS9fkjrtOm4RAaStncG+kpKaHo/AsIeOpJXKZMsXdoA1qEewTLz1rOgz88yH+L/svCrQvZU72Hh05+CEe9JNeFEEKIwUASnUIIIYT4TS4bF0avxcqCL3J45rs8tBqFm6dIlZg4ShRFTVh6hkPSbPVadztU7umv8twKpdvUxGflHnVte029z+TZX/HZ3+4eMBKMrvbbywBgjIsj9OOPKL91Hu1bt1I290bMN96I+YbrUTQae4c3YDnpnXhq4lOkeKewePtiviz6kv0N+3l68tOEu4XbOzwhhBBC/D8k0SmEEEKI3+yqCeFYrFYeW72PJd/kotUozJ0cae+wxFDl4AjDTlIXqFWfzeX9FZ/9Le6Vu6GjHvL+qy4AFPCJO3LWpzlabaE/geg8PQl5/TWqnniShvfeo/b55+ncl0PAwifQOjvZO7wBS1EULh5+MfHmeG5bexsHmg5w4aoLeXjcw0wNnWrv8IQQQgjxKyTRKYQQQojf5ZqJEfRarDz51X6e+u9+dBqFa0+JsHdY4kSgKOrhRG5BEP9X9VpvFxzMOlz1WbYNGkvUw46q98LOt9X7DG4QNOpw4jNwlHpo0hCn6PX43Xcvxrg4Dj70EK3ffkfxhRcQtGwZDiEh9g5vQBvhM4IVZ6/gjvQ72HZwG7evu5091XuYP3o+eo3e3uEJIYQQ4mdIolMIIYQQv9sNkyLp67Oy+JtcHv9yH1qNwlUTpK1T2IHO0J/AHAVcp15rqYLy7eqcz7LtULETupqgYI26DvGKPDznMygVfIaDdmh+e+x+3t8wRIRTdtPNdOXlUzhrNoFLFuM8bpy9QxvQzCYzr5z+Cs/teo43st7gvZz3yK7LZtEpi/Bx9LF3eEIIIYT4H0PzOzkhhBBCHHM3TYmix2Ll2e/yWPBFDjqNwmXjwuwdlhDg4guxZ6kLoK9Xre4s23r4sKO6/MNrzwfqfXonCBx5OPEZlArOQyeZZUpJIfTjjym7+SY692RQevU1+Nx2G56XXyYni/8KnUbHvFHzSPJO4r4N97GrehezPp/FolMWkeqXau/whBBCCPEjkugUQgghxB8277Qo+iwWln1fwEOf70Wr1XDJ2GH2DkuII2l14J+krtSr1Gvt9VC+o7/qc5v6v7uaoWi9ug5xD+mv+uxPfPolgs7BPvs4CvS+Pgx75x0OPvxPmv71L6qffJLOfTn4//OfaIxGe4c3oE0JmULUjCjmrZ1HbkMuV319FbeMvIXL4y+XRLEQQggxQEiiUwghhBB/mKIo3HZGDL0WKy+vO8D9/8lCp1G4ME1m/4kBztETok5XF4DFArW5h+d8lm2H6hx13mdjCWR9rN6nNUBASn/is7/y0y3Ibtv4IzQGA/6PLsAYF0fVwoU0f/Y53QUHCHr+OfT+/vYOb0ALcQ3hvenvsWDzAj4r+IyndzzNnuo9LBi/ABcHF3uHJ4QQQpzwJNEphBBCiD9FURTumhpLX5+V1zYUcve/MtEqCrNTg+0dmhC/nUYDPrHqGnmpeq2zWa30PNTuXrZNPeG9dIu6DnEJOLLdPSAF9Ca7bOO3UhQFz0suxhAVRfmtt9KZnU3hzFkEPfsMjqNG2Tu8Ac2kM7Fg3AJSfFJ4fMvjrCldwwWrLmDJpCXEeMbYOzwhhBDihCaJTiGEEEL8aYqicO9ZcfRarLz1QxF3/isDrUbhvFGDq9JNiCMYXSFisroArFaoP3A46Vm2TT3xvaUCcj5TF4BGp7a4H0p8BqWCR6h6avwA4zR2jDq3c+5cuvbvp/iyy/G79148Ljjf3qENaIqiMCt6FsM9hzN/7XxKWkq4ePXFPHDSA5wdcba9wxNCCCFOWJLoFEIIIcRRoSgKD549nD6LlXc3F3Pbx3vQahTOHRFo79CEODoUBbwi1JV8gXqtuw0qdh+Z/Gytgopd6tr6inqfo/nIdvfAkWAYGK3ODkGBhC7/gIp776Xly684+NBDdObk4HfvPSgOg3ce6fEQb45nxYwV3LX+LjZWbOSeDfewu3o3d6bdiYNWPndCCCHE8SaJTiGEEEIcNYqi8PBf4umzWvlgSwnzV+5Gq1E4OznA3qEJcWw4OEHoOHWBWvXZVHp4zmfZNqjcA+21kPulugAUDfgM/1HLexp4Raot9HagcXQkcMkS6mLjqFm6lMYVK+jKzyfomaXozGa7xDRYuBvdWTZlGa9kvMKLe15kZe5KsuuyWTJpCQHO8rVPCCGEOJ4k0SmEEEKIo0qjUVhwTgJ9fVZWbC/l1hVqsnN6ohxyIk4AiqKe1O4eAgnnqdd6u6Ay40dVn9uhqQSqstS14y31PqMbBPYnPoNTwSf5OIeuYL72Ggwx0VTcdjsdO3aoczufew5TYsJxjWWw0Wq0XJ9yPYneidy1/i6y67KZvWo2T0x4gnGB4+wdnhBCCHHCkESnEEIIIY46jUbh8b8l0mux8snOMm5evguNojA1wc/eoQlx/OkMauIyOPXwteZKKN9+OPFZvhM6m6DgO3UBeuBUgz/avi8hZIyaAPWJA432mIbrMmkSoStXUjZ3Lt2FhRRffDH+Cx7B7WyZPfn/GR84npUzVjJ/7Xyy67K5/tvruT75eq5NvhaNYp9qXSGEEOJEIolOIYQQQhwTGo3CkzOT6LNY+M/uCm5avpMXLxrFacN97R2aEPbn6g+uZ0Ncf/Kwrweqso9sea8vwKWrEjKWqwvAwRkCRvRXfaapFaDO3kc9PEN4GKErV1Bx2+20rltHxe130Lk3B59/zEfRyY8QvybAOYB3pr3Dwq0L+Sj3I17Y8wJ7avewcPxC3I3u9g5PCCGEGNLkuxQhhBBCHDNajcKiWcn0WeHzPRXc8P5OXr5kFJNjfewdmhADi1YPASnqSrsagJ6mg+z49GVS/RW0Ff1Vn90tULReXYd4hB6e8xk0GnwTQPfnD8LRurgQ9MIyap59jrqXX6b+zTfp2r+fwCWL0bq7/+mPP5Q5aB144KQHSPZO5pHNj7CxfCPnrzqfJZOWEG+Ot3d4QgghxJAliU4hhBBCHFM6rYanZydjsVj5IrOSa9/bwauXjuaU6KNfhSbEkOLoRZVbCpZJ09Hq9WDpg5r9R57wXrMPGorUlfmR+pzOCP4patIzOE1Ngrr+sUNxFK0Wn3m3YoyLpeLue2j74QcKZ59P0PPPYYyOPlo7HbLOiTyHWM9Y5q2dR2lLKZd8eQl3j7mbmVEzURTF3uEJIYQQQ44kOoUQQghxzOm0GpZekEKvxcJ/s6u45p3tvD4nlfFRcpqzEL+ZRgu+w9U1ao56raMRKnaq7e6lW9XkZ2cjlG5W16b+Z10D+0947098+ieD3vibX9p16lQcQkMpm3sjPSUlFF1wIQFPLMT19NOP9i6HnBjPGD6c8SH3bbiP70u/55+b/snu6t3cN/Y+TDqTvcMTQgghhhSZiC2EEEKI40Kv1fDchSM5Lc6Xrl4LV72zjR8Kau0dlhCDm8kdIk6FU+6Aiz+GO4vgxh1w7ksw+grwSwRFA83lsPdT+PpeeOMMeDwIXpkMX94JmR+rFaFW66++lDE2ltCPP8JxzBis7e2U33QzNc89j9ViOR47HdRcHVxZOnkpt468FY2i4bOCz7h49cWUNJfYOzQhhBBiSJFEpxBCCCGOGwedhmUXjeDUWB86eyxc+dZ2thyos3dYQgwdigLmSEi5EGY8DddtgLtK4bIv4LSHIOYscPIGS49aCbrlJfjkSngmGRZFw/K/w/olULgeulp/8uF1Hh6EvPYqHpdeAkDtsmWU3Xwzfa1tx3mjg49G0XBl4pW8evqreBo9yW3I5YJVF7CmZI29QxNCCCGGDEl0CiGEEOK4Mui0vHDRSCZGe9PR08flb21je1G9vcMSYugyOEPoeBg/Dy78AG7Lg1sy4LzXYcx1EDgKNHpoq4b9X8B3D8PbM2BhMLw0HlbNg90fQG0eWK0oej1+99yD/2OPoej1tH77HUUXnE93cbG9dzoopPmnsXLGSlK8U2jpaeGW729h6Y6l9Fp67R2aEEIIMehJolMIIYQQx51Rr+WVS0YxPtJMe3cfc97Yyo7iBnuHJcSJQVHAYxgkzoRpT8DVa+DuMrjyGzjjURh+LrgGgdUCBzNh+xvwn+vh+dHwRCi8NxPWPoF7kivDXn8RnY8P3fkFFM6aTev6Dfbe3aDg6+TLG1Pf4OK4iwF4Pet1rv3mWmo7ZJyHEEII8WdIolMIIYQQdmHUa3n10tGcFO5FW3cfl72xld2ljfYOS4gTk96ontB+8o0w+22Ynw3zc2D2u3DyTRByknqae2cj5H8Dax+D9/6GafVZhM5oxxTiiqW5mdJrr6Xutdew/j/zPgXoNXruTLuTp055CpPOxNaDWzn/8/PZXb3b3qEJIYQQg5YkOoUQQghhNyYHLa9fNpq0ME9aunq59PUtZJY12TssIQSAawAM/wucsQCu+Eqt+rxmLUx7ChJng0cYYEXfnkvImH24hbeBxUL1osVUnDcGy5cPwv6voE2qFH/N1NCpfHjWh4S7hVPdUc3lX13O+znvS7JYCCGE+AMk0SmEEEIIu3J00PHmZamMHuZBc2cvF7++hewKSXYKMeBo9RAwAsZcA+e9CrfshtsL4MIP0Uz6B/6zk/BN6wDFSvPeFor/+R49r/0dnoqAZ0fAv66Bra9CxS7o67H3bgaUcPdwlp+1nKmhU+m19rJw60LuSL+D9p52e4cmhBBCDCo6ewcghBBCCOFk0PHWFWlc+voWdpY0cvFrW/jg6rHE+bvaOzQhxK9xMkPMNIiZhgJ4XtqH4b8fU/7AE3Q2QOG3fgSdVIMjB6D+AGSsUJ/TmdSkadBoCEpV2+Zd/Oy6FXtz1Dvy5MQnSfFJYdG2RXxV9BW5Dbk8Pelpwt3D7R2eEEIIMShIRacQQgghBgTn/mRncrA7De09XPTaFvYfbLF3WEKI30OjxWna+YT953MMcXH0dVgpTvelwXMeTLobIk8Doxv0dkDJD/DDs7DyElgcA08nwEeXwaZlULoNervsvZvjTlEULoq7iDenvomPyYcDTQe44IsL+KroK3uHJoQQQgwKkugUQgghxIDhatTzzhVpJAa6Ud/WzUWvbSa/WpKdQgw2+sBAQj94H9fp06C3j4MvrKByTQfW2cvhjiKYuw3OeQFGXQ6+iaBooKkUsv8N/70HXj8NHg+CV6fAl3dB5sfQWAInyNzKFJ8UVp69kjS/NDp6O7h93e08sfUJeqTlXwghhPhV0rp+DCxbtoxly5bR19dn71CEEEKIQcfNpOfdK9P4+6tb2FvZzIWvbuHDa8YS4e1s79CEEL+DxmQiYPFiDHFx1Cx5msaVK+nKyyPo2WfQeUeDdzSMuEi9uatFnd1Ztg3KtkPpVmivhfLt6trS/0GdfdVW96DREJQGASng4GSvLR5TXiYvXj79ZZ7f9TyvZ73OeznvkVWbxaJTFuHr5Gvv8IQQQogBSSo6j4G5c+eyd+9etm3bZu9QhBBCiEHJ3dGB968aQ6yfCzUtXVz4ymYKa9vsHZYQ4ndSFAXz1VcT/NKLaFxc6Ni1i8KZs+jIzDryRoMLhE2ECf+AC5fD7flw827422uQdi0EjASNDlqrYN8q+PYheGs6PB4ML02AVfNhz4dQVzCkqj51Gh23jrqVZyY/g4vehd01u5m9ajZbK7faOzQhhBBiQJJEpxBCCCEGJA8nNdkZ7etMdUsXf391MyV1cgKxEIOR8ymnELpyBQ7h4fRWVVF80UU0ffrpLz+gKOAZBkmzYPqTcM33cHcZXPFfOGMBDD8HXALA2gcHM2D76/Dva+G5kfBkOLw/C9Y9BQXfQ2fT8dvoMXJqyKl8OONDoj2iqe+s5+pvrub1zNexDqGkrhBCCHE0SOu6EEIIIQYsL2cD7181lgtf3Ux+dSsXvrqZD68ZS7Cno71DE0L8ToawMEJXrqDi9jto/f57Ku68i869OfjcfhuK7jf8WKI3QchYdR3SVN7f7t6/KnZDRz3kfa0uABTwjlXb3YPT1NZ3cwxoBlfNR4hrCO9Nf48FmxfwWcFnLN25lD01e1gwfgGuDq72Dk8IIYQYECTRKYQQQogBzdvFwAdXj+GCVzZzoKaNC1/dzIprTyLQ3WTv0IQQv5PW2ZmgZc9T89xz1L34EvVvv01XXi6BS5agdXf//R/QLVBd8eeqf+7thqpMdc5n2TZ11mdjMdTkqGvXu+p9BlcIHKnO+Tw089PR82ht85gx6UwsGLeAFJ8UHt/yON+Xfs8Fqy7g6UlPE+MZY+/whBBCCLsbXL/GFEIIIcQJycfFyPKrxxJmdqKsoYMLX9lMZVOHvcMSQvwBikaDzy23ELh0KYqjI20/bKJw1mw69+f++Q+uc4DAUTDmWjjvNbg1A27LgwuWw/j5EDoB9E7Q1QwH1kL6k/DBLHgyDJ4dCf++Dra9BpV7oK/3z8dzDCiKwqzoWbw77V0CnAIobSnlotUX8Wn+r4wCEEIIIU4QUtEphBBCiEHB19XIB1eP4fyXN1NS386Fr6iVnb6uRnuHJoT4A1ynnolDWBhlc+fSU1pK0YUXErDwcVzPOOPovpCzD8ROVxeoCcyanP6Kz/6W97o8qC9Q157l6n16R/UQpKDR/VWfqeAycE47jzfHs2LGCu7acBcbyzdy38b72FOzhzvT7sSgNdg7PCGEEMIupKJTCCGEEIOGv5uJ5deMJcjDRFGdmuysbu60d1hCiD/IGBNN6EcrcRw7Fmt7O+U330LNs89itViO3YtqdeCXCKOvgL++CDdthzsK4aJP4JS7IOJUMLhBTzsUb4CNS2HFRbA4Gp5OhI+vgM0vqu3xvV3HLs7fwN3ozgtTXuCGlBtQUPgo9yPmfDmH8tZyu8YlhBBC2ItUdAohhBBiUAl0N7H86rHqzM7aNv7+2haWXz0WbxepYBJiMNJ5eBDy2qtUP/UU9W+/Q+0LL9K5bz8BTz6B1tn5+ATh6AlRp6kLwGJRqzwPzfks2w7Ve6GpRF1Zn6j3aQ3gn3x4zmdQKrgFqafGHycaRcP1ydeTZE7izvV3kl2XzfmrzmfhhIWMDxx/3OIQQgghBgKp6BRCCCHEoBPs6cgHV4/B381IfnUrF722mbpW+1ZWCSH+OEWnw/fuu/Ff+DiKgwOta9ZQdP4FdBcV2ScgjQa8Y2DExfCXZ+GGH+CuErj0Mzj1PoieCo5e0NcFZVth8zL4+HJYmgBL4mDFxbDxGSj+Abrbj0vI4wLHsXLGShK8EmjqauKGb2/gxd0vYrEew+pYIYQQYoCRik4hhBBCDErDvJz44OqxXPDKJnKrWrmov7LTw8nB3qEJIf4g93PPxRAeTtmNN9FdUEDh7PMJXLwI5wkT7B0aGF0h/BR1AVit0FCoVnuWblWrP6uyoKUScj5XF4BGB74Jh+d8Bo0Gz/BjUvUZ4BzA29Pe5sltT7Ji/wpe2PMCe2r3sHD8Qpy0Tkf99YQQQoiBRio6hRBCCDFohZnVZKe3i4F9B1u46LUtNLZ32zssIcSfYEpKIvTjjzClpGBpbqb02uuoe+01rFarvUM7kqKoCcuk2XDWIrh2HdxVCpd/Baf/E+LOBmc/sPRC5W7Y9ir8+xp4biQ8FQEfnA/pT6mnv3e1HLWwHLQO3Df2Ph4b/xhGrZGN5RuZvWo22XXZR+01hBBCiIFKKjqFEEIIMahFeDuz/OoxXPDKZvZWNnPJ61t576oxuJn09g5NCPEH6X18CHnnbaoeeYTGjz6metFiOnP24b/gETQmk73D+2UOjjDsJHWBWvXZXH54zmfZNjXp2V4HuV+pCwAFfIYfnvMZnAZeUWoL/R90dsTZRHtEM3/tfEpaSrjimyuYZpjGNOu0P71NIYQQYqCSRKcQQgghBr1IH5f+NvbNZJY3cekbW3n3yjRcjZLsFGKw0jg44PfPf2KIi6Pqscdp/uILugoPEPz88+gDAuwd3m+jKOrhRG5BkPA39VpvFxzMUmd7lm1TV2MJVGera+fb6n0GNwga1d/ungaBI9VDk36HGM8YPpzxIfdtuI81pWv4rOMzLJstPHDyA5h0AzhhLIQQQvxBkugUQgghxJAQ7evC+1eN4cJXN7OntJE5b2zlnSvScJFkpxCDlqIoeP797xgiIym/dR5de3MonDmLoGeW4piaau/w/hidoT+BOQq4Xr3WUnU46Vm2HSp2QlcTFKxR1yFeUYfnfAangXccaH/9RzoXBxeWTl7Kaxmv8dzu51hVuIrcxlyWTFrCMNdhx26fQgghhB3IjE4hhBBCDBlx/q68d6Xatr6rpJHL39xGW1evvcMSQvxJTmlphH38EYbhcfTV11N8+RXUf/DBwJvb+Ue5+ELcDDj9Ybj8C3XW57XpcNZiSL4QvCLV++ryYM8H8MV8eGk8LAyBt2bAtw/BvtXQWv2zH15RFC4bfhmXO12Op9GT3IZcLlh1Ad+VfHf89iiEEEIcB5LoFEIIIcSQkhDoxntXjsHFqGN7cQOXv7WN9m5Jdgox2OkDAgh9/31czzoLenup+ucjHHzgASzdQ/AAMq0O/JMh9Sr460tw0w64oxD+/hFMvAPCJ4PBFXraoGg9bHgaPrwQFkXB0iT4+ErY/BKU74Dew5+fcH04y6cuZ4TPCFp7Wrn1+1tZsmMJvRb5GimEEGJokNZ1IYQQQgw5iUFuvHvlGC55bQtbC+u58q3tvHFZKiYHrb1DE0L8CRqTiYBFT2EcHkf1osU0fvQxXfkFBD6zFL2Pj73DO7YcPSH6DHUBWCxQu//IlvfqHGgsVlfWx+p9WgMEpKAJGIl/gxbv3hReP/N1nt7xNO/ufZc3s94kqzaLJyc+idlktt/+hBBCiKNAKjqFEEIIMSSlBLvz9pVpOBt0bDpQx9XvbKezp8/eYQkh/iRFUfC68kqCX3kZjYsLHbt2UTRzFh0ZGfYO7fjSaMAnDkZeCn95Dm7YBHcVwyX/gcn3QdSZYPKEvi4o3YJ2y4ukFT2P/rkk9E8ncceBTBb5nYaj1sC2g9uY/flsdlXvsveuhBBCiD9FEp1CCCGEGLJGhnjw1uWpODpo2ZBfy7Xv7pBkpxBDhPOECYR9tBKHiAh6q6spvvgSGv/zH3uHZV9GN4iYDKfcDhethDsOwE074a8v0zfqChpNw7AqWmipgJzPOHPTGywvLiS8u4eajhqu+HIO7377D6x1B2CozD8VQghxQpFEpxBCCCGGtNGhnrx5WSomvZZ1uTXc8P5Ounol2SnEUOAQGkroig9xPvVUrN3dVN51N1WPP461V2ZOAqAo4BUByRdgmfok62Ifofe2A3DZajjtYYidQbjBi+UVB5nW2kYvVp4s/5rbPzyNtkWR8MEFsH4xFKZDV6u9dyOEEEL8v2RGpxBCCCGGvDHhXrxxWSqXv7WVNfuqmfv+Ll64aCQOOvmdrxCDndbZmaDnn6P2+WXUvvAC9W+/Q2duLoFLlqDz8LB3eAOPgxOEjlMXgNWKY1MpT5RuJTnvYxY1Z/NfZydyHXp4uvAbInK/VO9TNOAzHIJSDy+vSLWFXgghhBgg5G8lIYQQQpwQTorw4rVLUzHoNHybU8VNy3fS02exd1hCiKNA0WjwvvkmAp99BsXRkfZNmymaNZvO/bn2Dm3gUxRwD0FJnMlFf/uQN6e/i4/Jh0IHPRcGh/BlzCngFgxWC1RlwY434dMbYFkqPBkG750HaxdC/rfQ0WDv3QghhDjBSaJTCCGEECeM8VFmXrl0NA5aDf/NruKWD3fRK8lOIYYM1zPOIHT5cvTBwfSUlVF04YU0//dre4c1qKT4pLDy7JWM8RtDh7WXO7oLWTjuYnpuzYTz34Nxt0DIyaAzQWejmuBc+7ia8HwiFJ5Pg//Mhe1vwsEssMioECGEEMePtK4LIYQQ4oRySrQ3L18yimve3c7qzINoNXt4enYyOq38/leIocAYE03YRyspnz+fth82UX7LLXRefx3eN92EIm3Wv4mXyYuXT3+ZZbuX8Wrmq7yf8z7ZtdksOmURvnFnqzf19UBVNpRtO7zqD0DtfnXtfk+9z8EZAkce2fLuZLbf5oQQQgxpkugUQgghxAlncqwPL140iuvf38HneyrQKrB4dgpajWLv0IQQR4HW3Z3gV16hetFi6t96i7oXX6Jr334CnnoSrbOzvcMbFLQaLTePvJlEcyL3briX3TW7mb1qNk9OfJIx/mNAq4eAFHWlXa0+1FYH5duhdKua+CzfCd0t6mFGhemHP7hH2I8Sn6PBL1H9eEIIIcSfJIlOIYQQQpyQThvuy3MXjuTGD3byn90VaDUanpqZhEaSnUIMCYpOh+9dd2KMi6Xy/gdo/f57imafT9Cy5zGEhdk7vEFjcshkVsxYwby189jfsJ9rvrmGm0bcxBUJV6BR/qdC1skLos9UF6ht6zX7+ys+t0LZdqjZBw2F6spcqd6nM0LACDXpeSgB6hpwfDcqhBBiSJBEpxBCCCFOWFMT/Hj2whHctHwXn+wsQ6dRePxviZLsFGIIcTvnHBzCwym78Sa6DxygaPb5BC5ehPPEifYObdAIdg3mvenvsWDzAj4t+JRndj7Dnpo9PDr+UVwdXH/5QY0WfIera9Qc9VpHI1TshNIftbx3NkLJJnUd4hp0ZOLTPxn0xmO5TSGEEEOAJDqFEEIIcUKbnuhPn8XKLR/uYsX2UjQahUfPTZBkpxBDiCkxkbCPP6Lsllvp2LmT0muvw3vePLyuvgpFkX/Xfwujzsgj4x4hxSeFx7Y8xtrStVyw6gKWTFpCrGfsb/9AJneIOFVdAFYr1BX0V3z2Jz6rsqG5DPaWwd7/qPdp9OCfdOSsT/cQ9dR4IYQQop8kOoUQQghxwjs7OYA+i5V5K3ezfGsJOo3CP8+JlwSIEEOIztubYW+9ycEFj9K4ciU1S5bQtS8H/0cfRWMy2Tu8QUFRFGZGzyTOM475a+dT2lLKxasv5r6x93Fu5Ll/9IOCOVJdKX9Xr3W1QuXu/lmf29UkaFsNlO9Q15aX1PucfA7P+QxOU9vfHZyOxlaFEEIMUpLoFEIIIYQAzh0RSK/Fyu0f7+HdzcVoNQoPnj1ckp1CDCGKgwP+/3wY4/A4Di54lObVX9JVWETw88+hDwy0d3iDRrw5npVnr+Su9XexoXwD92+8n93Vu7l7zN0YtIY//wIGZwgdry5Qqz4bS4484b0yA9qqYf8X6gJQ+lvlg1IhKE39b68IqfoUQogTiCQ6hRBCCCH6zRwVhMVi5Y5PMnjrhyK0GoX7zoqTZKcQQ4zHBRdgiIig7JZb6crJoXDmLAKXLsVpTJq9Qxs03AxuLJuyjFcyXuGF3S/wSd4n5NTnsGTSEgKdj3LSWFHAY5i6Emeq13o6oXLPj5Kf29V294OZ6tr+hnqfyQMC+2d9BqdC4Cgwuh3d+IQQQgwYkugUQgghhPiR2anB9Fqs3PPvTF7fUIhOo3DXtFhJdgoxxDimpqpzO2+8ic69eym54gp8774bj4v+Lv++/0YaRcN1ydeRZE7izvV3srduL7M/n83CCQuZEDTh2L643gghY9R1SHPFkYnPil3Q0QD536gLAAW8Y/oPOuqv+vSOUQ9OEkIIMehJolMIIYQQ4n/8fUwIfVYr9/8ni5fTD6DTKtx2RowkP4QYYvQBAQx7/z0q73+A5lWrqFqwgM59Ofg98AAaBwd7hzdonBx4MitmrOAfa/9BVl0Wc7+by7XJ13Jd0nVoj2cC0TUAhp+jLoC+HrW6s2z74QRoQyHU7FPXrvfU+xxcIHCkOuczKFWtAHXyOn5xCyGEOGok0SmEEEII8TMuGTuMvj4LD32+l2XfF6DTaJh3erS9wxJCHGUak4mAp57EGBdH9eLFNH38Cd35BQQ++wx6Hx97hzdoBDgH8Pa0t3ly25Os2L+Cl/a8REZNBgsnLMTD6GGfoLR6NYEZOBLGXKNea62B8h8lPst3QncLFK5T1yGe4Uee8O4br348IYQQA5okOoUQQgghfsFl48LotVhZ8EUOz3yXh1ajcPOUKHuHJYQ4yhRFwevKKzBER1P+j3/QsXs3RTNnEfTcs5iSk+0d3qDhoHXgvrH3keydzD83/ZMfKn7g/FXns2TSEhLMCfYOT+XsDTHT1AVg6YPqnMPt7mVboTYX6g+oK2OFep/OpJ7qHvyj5KeLn/32IYQQ4mdJolMIIYQQ4ldcNSEci9XKY6v3seSbXLQahbmTI+0dlhDiGHCeMJ6wj1ZSOncu3fkFFF98CX4P/x979x0fdX34cfx1lx1Iwt4bke0AwYF7UVxF6164cAC16s/VqnVVa23dA/fede9FVRQQGSIIgspQZMqG7OTu98eRSAQV8JLvXfJ6/h55JDfzvvPT/MI7n3E1DY44POhoSeXQzofStVFXzv/gfL5f+z0nv3Uyl/a/lKO2PSrxtgAJp0CLXrGPnU6NXVe4EhZMqrrkvWg1fD829lEhr23VvT5bbgepcTh1XpK01Sw6JUmSfsOZe3amtDzKv9+Zxb/fmUVqOMRZe3UOOpakapDevj0dnnmWhZdewrr3R7Hob3+j6KuvaH7xRYTSXLq8ubZtuC3PHPIMV4y5glHfj+LaT69lytIpXLHrFWSlZgUd79dlNYRt9o99AEQisPzbDQ46mgBLZ8Dq+bGP6S/F7peSDi23Xz/jc/1J73ltY6fGS5JqhEWnJEnSZhi+zzaUR6Lc/N7X/POtmaSEQ5yxR6egY0mqBin169Hm9ttZdtfdLLvrLlY+/jjFX39N61tvIbVhQPtNJqGc9Bxu2fsWHp3+KLdOvpXX5rzGzJUzuWXvW2if2z7oeJsvHIam28Y+djwhdl3x2tip7j9MgPnry8+CZT8VoRXqt/ip9GzTD1rtAOn1AnkZklQXWHRKkiRtpnP360JZJMrto77hH298RWo4xCkDOgYdS1I1CIXDNP3zCDK6dWXRJZdSMH58bN/Ou+8is2vXoOMljVAoxCm9TqFnk55c9NFFfLPyG459/Vj+sfs/2K/dfkHH23oZOdBxz9gHQDQKK+dtsNz9s9iJ7+sWw8zXYx8AofVL5Tc86KhRJ2d9SlKcWHRKkiRtgfP370J5JMJdH8zmqtdmkBIOcdKuHYKOJama5B5wABkdOjB/+AhKv/+eecceR6t/Xk/uH/4QdLSk0q9FP5479Dku+ugiJi+dzHkfnMepvU7l3B3PJTVcC/5ZGgpBo46xj+2Oil1XWgiLvlg/6/Oz2Oe1i2LXLfoCJjwQu19Wow2Kz52gdV/IzA3utUhSEqsF/x9FkiSp5oRCIS48sCtlkSj3fjSHK16ZTko4zPE7tws6mqRqktGlCx2fe5YFF/wf+WPHsuC88yk6ayZN/3IuoXA46HhJo1l2Mx4Y+AC3TrqVx2Y8xsNfPsy0H6fx773+TZOsJkHHi7+0LGi3S+yjwuoFVff6XDgFClfAN+/EPgAIQbPuVZe8N+kaW0IvSfpVFp3V4K677uKuu+6ivLw86CiSJKkahEIhLv1DN8rLozzwyVz+9tI0UsMhDt+hRdDRJFWTlAYNaHvfvSy96WZWPPwwy++9l+JZs2j17xtJyckJOl7SSAuncVG/i9i+6fZcMeYKJi6ZyNGvHc1/9voPfZr3CTpe9ctrHfvoOTh2uawElkyLLXmvmPW56rvYYUdLZ8Dkx2L3y8iNzfRs0y9WnHbax+JTkjbBorMaDB8+nOHDh7NmzRry8vKCjiNJkqpBKBTisoO7UxaJ8sjYeVzy4lSIRsgIOpikahNKTaX5JReT2aM7iy6/gnUffsi8o4+hzV13kdHJ/Xq3xIEdDqRLwy6c/8H5zF49m9PeOY0L+l7AST1OIlSX9qtMTY8VmK37ws5nxa5bt3T9Xp+fxT4vmAzFa2DOB7EPgPa7w+EjoYGrCSRpQ/4JSJIkaSuFQiGuPLQHJ+3SnmgULnnpSyb+WIf+gS7VUXmHHkr7J58ktUULSubOZd7RR7Puo4+CjpV0OuZ15KmDn2JQx0GUR8v598R/838f/R/5pflBRwtW/WbQ7SDY/yo45XW49Hs4+xM45BbY/nhIqwfffQIjB8CUp2MHIUmSAItOSZKk3yUUCnH1YT05rn87olF44tswb0xbHHQsSdUsq1dPOj7/X7L69iWybh3zzz6HZffeR9TSaYtkp2Xzrz3+xV/7/5XUcCrvffcex75+LLNXzQ46WuJISYUWvWGn02KzOM/5BNruHJvl+fLZ8NzJkL886JSSlBAsOiVJkn6ncDjEdYN7cWSf1kQJ8X/PT+PNaYuCjiWpmqU2aUL7hx+iwbHHQDTKj7fcwoILLiBSUBB0tKQSCoU4vvvxPDzwYZplN2Pemnkc98ZxvDnnzaCjJaZGneDUt2C/v0M4Fb56FUbuCt+8F3QySQqcRackSVIchMMhrvtjD/o3jVAeiXLu05/z9pfO7JRqu1B6Oi2vuooWV10Fqamsfett5h1/AiU/LAg6WtLZodkOPHfIc+zcYmcKywq55ONL+Of4f1JaXhp0tMQTToE9/g/OGBU7kX3dEnjySHj9Aiip40v/JdVpFp2SJElxEg6HOK5zhMO2a0lZJMqIpybz3owlQceSVAMaHnsM7R99hJTGjSmeOZN5Rx5J/vjPgo6VdBpnNebeA+5laO+hADw18ylOfedUFuf7h6NNarUDnPUR7HxO7PLEB+GePWKHGElSHWTRKUmSFEfhEPzriJ4cun0ryiJRhj05iQ9mLg06lqQakN23Lx2f/y+ZPXtSvmoV3592Gisef8J9O7dQSjiFc/ucyx373kFOWg5f/PgFx7x+DJ8u+jToaIkpLQsG3QAnvQy5rWHFbHjwQPjgenA2rKQ6xqJTkiQpzlJTwtxy9PYc3LslpeVRznpiEh99/WPQsSTVgLSWLWn/5BPkHnYolJez5LrrWHTZ5URKSoKOlnT2brs3zx76LN0adWNF0QrOeu8sHpj2AJFoJOhoianzPnDOGOh9FETL4aN/xQrPZd8EnUySaoxFpyRJUjVITQlz67E7MLBnc0rKIpz52EQ++WZZ0LEk1YBwZiat/vUvml18MYTDrH7xRb476SRKlzi7e0u1zWnL44MeZ/A2g4lEI9w2+Tb+8r+/sLp4ddDRElNWQ/jTA/CnByEzDxZOji1l/+x+cGaxpDrAolOSJKmapKWEueO4PuzfvRnFZRHOeGwCY2dbdkp1QSgUovFpp9L2/vsI5+VR9MVU5h15JIVTpgQdLelkpmZy7YBruXq3q0kPp/PhDx9y7OvHMnPFzKCjJa7eR8I546DTPlBWCG9eCE/8CdYsCjqZJFUri05JkqRqlJ4a5q4T+rBP16YUlUY4/ZGJjJ+zPOhYkmpI/QED6Pjf58josg1lP/7IdyedzKoXXgw6VlI6ossRPHbQY7Su35of1v3AiW+eyMvfvhx0rMSV1xpOfBEG3QipmTB7FIzcFaa/FHQySao2Fp2SJEnVLCM1hZEn9mXPbZtSWFrOqY9MYOK8FUHHklRD0tu1o/3Tz5BzwP5ES0tZdNllLP7HdURLPShmS/Vs3JNnD3mWPVrvQXF5MVeMuYKrxl5FcXlx0NESUzgMO58FZ42GlttD4Ur47ynw4plQuCrodJIUdxadkiRJNSAzLYX7TurL7ts0oaCknCEPfcak71YGHUtSDUmpX4/Wt91Gkz+PAGDlE0/w/RlDKVvpz4EtlZeRx5373cmIHUYQIsQL37zASW+exA9rfwg6WuJq2hVOfx/2vAhCYZj6LIwcAHNHB51MkuLKolOSJKmGZKalcP/JO7Frp8bkl5RzykOfMWX+qqBjSaohoXCYpsOH0+bOOwhnZ1Mwfjzz/nQkRV99FXS0pBMOhTlr+7O454B7aJDRgK9WfMUxrx/D6B8s7n5Rajrsezmc9g407AhrfoBHD4V3LoPSoqDTSVJcWHRKkiTVoKz0FB48ZSf6d2zE2uIyTn5wPNN+8PRgqS7J2X9/Ojz7DGnt21G6cCHzjjueNW++GXSspLRbq9147pDn6N2kN2tK1jB81HDu/PxOyiPlQUdLXG37w9mfQN9TYpfH3Qn37Q2LpgaZSpLiwqJTkiSphmWnp/LwKf3YqX1D1hSVceKD45m+0LJTqksyunSh43PPUW/AAKJFRSy44P9YevMtRMst6LZUy/oteeQPj3BM12MAuHfqvZzz/jmsLHJbgF+UUR8OvQ2OexbqNYUfv4L794VPbgFLYklJzKJTkiQpAPUyUnnktP7s2K4BqwtLOfGB8Xy1aE3QsSTVoJS8PNredy+NTj8NgOX33cf8YcMoX7s24GTJJz0lnct3uZzrd7+ezJRMxi0ax9GvH820H6cFHS2xdf0DDPsUuh0CkVJ4/yp45GBYOS/oZJK0VSw6JUmSAlI/I5VHT+vP9m3yWFlQygkPjGfWYgsOqS4JpaTQ/KKLaPXvfxPKyCD/o9HMO+poiufMCTpaUjq086E8efCTtM9tz+L8xZz89sk8O/NZotFo0NESV70mcMwT8Me7IT0Hvh8XO6jo8yfA901SkrHolCRJClBuZhqPnb4zvVvnsSK/hBMe+JRvl1p2SnVN3qGH0P7JJ0lt2ZKSefOYd/QxrP3gg6BjJaVtG27L0wc/zX7t9qMsUsY/xv+Dv33yNwpKC4KOlrhCIdjxBDjnE2i3K5Ssg1eGw7MnQv6yoNNJ0maz6JQkSQpYXlYaj5/enx4tc1m2roTj7h/P7B/XBR1LUg3L6tWTjs//l6yd+hJZt44fhg1n2T33OhtxK+Sk53DL3rfwf33/j5RQCq/PeZ0T3jyBeavnBR0tsTXsAKe8AftfDeE0mPk63L0LzHo76GSStFksOiVJkhJAg+x0njxjZ7q1yOHHtcUcd9+nzF2WH3QsSTUstXFj2j/0EA2OOxaiUX689VYWnHc+kXx/HmypUCjEKb1O4f4D76dxZmO+XfUtx75xLO9/937Q0RJbOAV2Pw/O/ACa9YD8H+HpY+DVc6HYP8JJSmwWnZIkSQmiYb1Y2blt8/osXVvM8fd/yvfLXWop1TWh9HRaXnklLa65GtLSWPvOO8w77nhKlywNOlpS6teiH/899L/0adaH/NJ8zv/wfG6eeDNlkbKgoyW2Fr1h6Aew6wggBJMfhXt2h/mfBZ1Mkn6RRackSVICaVw/gyfP2IVtmtVn0eoijrv/U+avsOyU6qKGRx9N+0cfIaVJE4q//ppld98ddKSk1TS7KQ8MfIAhPYYA8PD0hznj3TNYVuj+k78qLRMGXgdDXoXcNrByLjw0kPCH/yQUtSiWlHgsOiVJkhJM05wMnhq6M52a1mPBqkKOu/9TFqwqDDqWpABk9+lDiyv/DkDh558HnCa5pYXTuLDfhdy0103US6vHpCWTOOq1o5i0ZFLQ0RJfxz3hnDGw3bEQjZAy5ib2nHUNLPs66GSSVIVFpyRJUgJqlpPJ00N3oUPjbH5YWchx933KotWWnVJdlLXd9gAUf/ute3XGwYEdDuTpg59mmwbbsKxwGcNHDWdNyZqgYyW+rAZwxL1w1CNEsxrSoHAeqQ/uC5/eA5FI0OkkCbDolCRJSljNczN5+sxdaNcom+9XFHDcfZ+yZE1R0LEk1bC05s1Ibd4cIhGKZswIOk6t0DGvI08e9CSt6rUivzSfaT9OCzpS8uh5OGVDR7MkpzehsiJ4+xJ44nBYvSDoZJKUHEXnd999x4wZM4j4VyJJklTHtMzL4ukzd6FNwyzmLY+VnUstO6U6J2u73gAUTrWQi5fstGx2aLYDANOW+b5ukZyWfNr5QsoH3gipWTDnQxi5K0x7Puhkkuq4hCo6H3roIW6++eYq15155pl06tSJ3r1706tXL+bPnx9QOkmSpGC0bpDF00N3oXWDLOYsy+f4B8bz49rioGNJqkGZvbcDoHCahVw89W4SK5CnL5secJIkFAoR2ek0OPtjaNUHilbDC6fD86dD4cqg00mqoxKq6Lzvvvto2LBh5eW3336bhx9+mMcee4wJEybQoEEDrr766gATSpIkBaNto2yeGrozLfMy+XbpOk544FOWr7PslOqKihmdRRadcdWrSS8gNqMzGo0GnCZJNekCp78Le10KoRT48nm4ezeY/UHQySTVQQlVdH7zzTfstNNOlZdfeeUV/vjHP3LCCSfQp08frr/+ekaNGhVgQkmSpOC0b1yPp4buQvPcDL5eso4THhjPyvySoGNJqgGZPXsCULpgAWXLlwecpvbo1qgbKaEUlhctZ3H+4qDjJK+UNNjnr7HCs1FnWLsQHh8Mb10KpR6kJ6nmJFTRWVhYSG5ubuXlsWPHsueee1Ze7tSpE4sX+/98JElS3dWxSazsbJqTwczFaznhgfGsKrDslGq7lJwc0jt1Aly+Hk+ZqZls23BbAL5c/mXAaWqBNjvFlrLvdHrs8viRcO9esHBKoLEk1R0JVXS2b9+eSZMmAbBs2TKmT5/OgAEDKm9fvHgxeXl5QcWTJElKCJ2b1ufpoTvTpH46Mxat4aQHP2N1YWnQsSRVs6ze65eveyBRXPVsEpst64FEcZJeDw65GU54Huo3h2Wz4IH9YPR/oLws6HSSarmEKjqHDBnC8OHDufbaaznqqKPo1q0bffv2rbx97Nix9OrVK8CEkiRJiWGbZjk8ecYuNKqXzrQFqzn5oc9YU2TZKdVmmRUnrzujM64qDiT6cpkzOuOqywFwzjjofhhEyuB/18IjB8GKOUEnk1SLJVTRefHFFzN06FBefPFFMjMz+e9//1vl9jFjxnDccccFlE6SJCmxdG2Rw5Nn7EyD7DS+mL+KIQ99xlrLTqnWytoudvJ60dSpHpwTRxUHEs1YPoPySHnAaWqZeo3h6Mfg8HshIxfmj4eRu8OkR8AxLKkaJFTRGQ6Hueaaa/j8889566236N69e5Xb//vf/3L66acHlE6SJCnxdG+ZyxOn70xeVhqff7+KUx+eQH6xSwOl2iija1dCaWmUr15N6fz5QcepNTrldSIrNYv80nzmrZkXdJzaJxSC7Y+Fc8ZA+92hNB9e+ws8fSysWxp0Okm1TEIVnQDPPvssJ5xwAkcddRT33HNP0HEkSZISXq/WeTxx+s7kZKYy8buVnPrIBApKLDul2iacnk7G+skghe7TGTep4VS6N4q9r+7TWY0atIMhr8GB/4CUdPj6bbh7V5j5RtDJJNUiCVV0jhw5kuOOO46JEyfyzTffMHz4cC666KKgY0mSJCW83m3yePz0ncnJSOWzuSs4/ZGJFJa4BFOqbSoPJJo2NeAktYv7dNaQcBh2+zOc+SE07wUFy+CZ4+GVEVC8Nuh0kmqBhCo677zzTq688kpmzZrFlClTePTRR7n77ruDjiVJkpQUdmjbgEdP70/9jFTGzVnO0McmUlRq2SnVJlmVBxJZyMVTxT6dFp01pHlPGPo/GPAXIASfPw4jB8B344JOJinJJVTROWfOHIYMGVJ5+fjjj6esrIxFixYFmEqSJCl59GnXkEdO7Ud2egqffLuMsx6fZNkp1SKZvdcfSDRjBtFSDx+Ll4qic9bKWZSUlwScpo5IzYADroFT3oC8drDqO3h4ELx/FZT530DS1kmoorO4uJh69epVXg6Hw6Snp1NYWBhgKkmSpOSyU4dGPHxKP7LSUvjo6x8Z9uRkisssO6XaIL1De8L16xMtKqL422+DjlNrtK7fmoYZDSmLlDFrxayg49QtHQbEDira4QQgCp/cAg/sC0u/CjqZpCSUGnSAn7viiivIzs6uvFxSUsJ1111HXl5e5XU333xzENEkSZKSxs6dGvPgKTtx2iMT+N/MpQx/8nPuPqEP6akJ9XduSVsoFA6T2bsXBeM+pXDqNDLXH06k3ycUCtGzSU8+WfAJ05ZNo3fT3kFHqlsyc2Hw3dB1UOxE9sXT4N69YP8rYedzYnt7StJmSKifFnvuuSezZs3i888/r/zYbbfdmDNnTuXlKVOmBB1TkiQpKezWuQkPnNyPjNQw73+1hD8/PZnS8kjQsST9Tlnrl68XeiBRXFUcSDR9+fSAk9Rh3Q+Fc8ZBl4FQXgzv/A0eOwxWzQ86maQkkVAzOj/88MMql5ctW0Z6ejq5ubnBBJIkSUpyu3dpwn0n78TQRyfyzvQl/OWZz7n92B1JTUmov3dL2gIVBxIVTZ0WcJLapWKfzmnLfF8DldMcjn8WJj0M71wG8z6OHVR08H+g91EQCgWdUFICS7jfcFetWsXw4cNp0qQJzZs3p2HDhrRo0YK//vWvFBQUBB1PkiQp6ey1bVPuPakvaSkh3py2mPOf+4IyZ3ZKSaviQKLib78lkp8fcJrao6LonLt6LmtL1gacpo4LhWCn0+DsT6D1TlC8Gl4cCs+fCgUrgk4nKYElVNG5YsUKdt55Zx599FH+9Kc/cdNNN3HTTTdx2GGHcccdd7DnnntSVFTEZ599xu233+lPqsgAAQAASURBVB50XEmSpKSxT7dmjDwhVna+9sVCLvzvF5RHokHHkrQV0po3I7V5c4hEKJoxI+g4tUajzEa0rt8agBnLfV8TQuPOcNo7sM9lEE6F6S/ByN3g2/eDTiYpQSVU0XnNNdeQnp7O7NmzuffeeznvvPM477zzuO+++/j2228pKSnhpJNO4oADDqhyOJEkSZJ+2/49mnPHcX1IDYd4ecpCLn5+KhHLTikpVSxfL3T5ely5fD0BpaTCXhfD6e9B4y6wdhE88Sd48yIocdWnpKoSquh8+eWX+c9//kPz5s03uq1FixbceOONvPDCC1xwwQUMGTIkgISSJEnJ7Q+9WnD7cTuSEg7xwuQf+OuL0yw7pSRUsXy98EsLuXjq1ThWdH657MuAk2gjrfvAWaOh/1mxy5/dB/fuCQsmB5tLUkJJqKJz0aJF9OzZ8xdv79WrF+FwmCuvvLIGU0mSJNUuB/VuyS3H7EA4BM9OnM9lL39p2SklGQ8kqh4VMzotOhNUejYcdCOc+CLktITl38CDB8BHN0J5WdDpJCWAhCo6mzRpwrx5837x9rlz59KsWbOaCyRJklRLHbZ9K24+egdCIXj6s++58tXpRKOWnVKyyFw/QaR0wQLKli8POE3t0aNxD8KhMEsKlrC0YGnQcfRLttkPzhkLPQ+HSBl8cB08NBCWzw46maSAJVTROXDgQC677DJKSko2uq24uJgrrriCP/zhDwEkkyRJqn0G79iafx+5PaEQPP7pd1z92gzLTilJpOTkkN6pEwCF05zVGS/Zadl0you9r87qTHDZjeDIh+GIByAjDxZMhHt2h4kPgf+/TKqzEqrovOaaa5g1axZdunThxhtv5NVXX+WVV17hhhtuoEuXLnz11VdcddVVQceUJEmqNY7s24Z/HRHb6++RsfP4xxtfWXZKSSKrt8vXq0PvJrH31aIzCYRCsN1RMGwsdNwTSgvg9fPhqaNh7ZKg00kKQEIVnW3atGHcuHH06NGDv/71rwwePJjDDz+cyy67jB49ejBmzBjatWsXdExJkqRa5eh+bbn+8Ng/7B/8ZC43vDXTslNKApkVJ687ozOu3KczCeW1gZNegYH/hJQM+OZduHsXmPFq0Mkk1bDUoAP8XMeOHXnrrbdYuXIl33zzDQDbbLMNjRo1CjiZJElS7XX8zu0oj0a54uUvuXf0HFJTQlx4YFdCoVDQ0ST9gqztYrOxi6ZOJRqN+r/XOKksOpd/6fuaTMJh2HUYdN4HXhwKi6fBcyfB9sfDoBsgMy/ohJJqQELN6NxQw4YN6d+/P/3790+6kvOuu+6iR48e9OvXL+gokiRJm+2kXdpz1aE9ALjrg9nc+v43ASeS9GsyunYllJZG+erVlM6fH3ScWqNLwy6kh9NZW7KW79d+H3Qcbalm3eGM/8HuF0AoDF88BSN3h3ljgk4mqQYkbNGZzIYPH86MGTOYMGFC0FEkSZK2yCkDOnL5wd0BuG3UN9w+yrJTSlTh9HQyusf+9+ry9fhJC6fRrXE3AKYt831NSqnpsP+VcOpb0LADrP4eHjkY3r0CyoqDTiepGll0SpIkqYoz9ujEXwfF/pF/83tfc9cH3wacSNIv8UCi6uGBRLVEu13g7E+gz8lAFMbeDvfvC4v97yrVVhadkiRJ2shZe3XmooFdAfj3O7O496PZASeStClZHkhULTyQqBbJyIHD7oBjn4bsJrDkS7h/HxhzO0TKg04nKc4sOiVJkrRJw/fZhgsO2BaAf741kwc+nhNwIkk/l1kxo3PGDKKlpQGnqT16NY4VnTNXzKQ04vtaK3Q7CIaNg20HQXkJvHcFPHoYrHIfVqk2seiUJEnSLzp3vy6cu18XAP7xxlc8MmZuwIkkbSi9QwfC9esTLSqi+Fu3mYiXdrntyEnPobi8mG9X+r7WGvWbwXFPw6G3Q1o9+O4TGDkApjwN0WjQ6STFgUWnJEmSftX5+3dh+D6dAbjqtRk8Pm5esIEkVQqFw2T2js0+LHSfzrgJh8KVszo9kKiWCYWg7xA45xNouzMUr4GXz4bnTob85UGnk/Q7WXRKkiTpV4VCIS48sCtn7dUJgCtemc5T413qJyWKrN7bAVA4bWrASWoX9+ms5Rp1ip3Kvt/fIZwKX70KI3eFb94LOpmk38GiU5IkSb8pFApx6R+6ccbuHQH420vTeG7C/IBTSYKfDiTy5PX4qiw6l1t01lrhFNjj/+CMUdC0G6xbAk8eCa9fACX5QaeTtBUsOiVJkrRZQqEQlx3cnVN26wDAJS9O5YVJPwQbShKZ62d0Fn/7LZGCgoDT1B69m8QK5NmrZlNQ6vtaq7XaAc78EHYZFrs88UG4Zw/4YWKQqSRtBYtOSZIkbbZQKMSVh/bgxF3aEY3Chc9/wcufLwg6llSnpTVvRmrz5hCJUDRjRtBxao2m2U1plt2MSDTCjOW+r7VeWhb84Z9w8iuQ2xpWzIYHD4QProfy0qDTSdpMFp2SJEnaIqFQiGsO68Vx/WNl5wXPTeG1LxYGHUuq0yqWr3sgUXxVzOqcvnx6wElUYzrtDeeMgd5HQbQcPvpXrPBc9k3QySRtBotOSZIkbbFwOMR1g3tx9E5tiEThvGen8Oa0RUHHkuqsTA8kqhYV+3R68nodk9UQ/vQA/OlByMyDhZNjS9k/ux+i0aDTSfoVFp2SJEnaKuFwiBuO2I4/9WlDeSTKuU9/zttfLg46llQneSBR9fDk9Tqu95Ew7FPotA+UFcKbF8ITf4I1/mFPSlQWnZIkSdpq4XCIG4/cjsE7tKIsEmXEU5N5b8aSoGNJdU5mz54AlC5YQNny5QGnqT16No69rwvWLWBF0YqA0ygQua3gxBdh0I2QmgmzR8HIXWH6S0Enk7QJFp2SJEn6XVLCIf5z1PYcun2s7Bz25CQ+mLk06FhSnZKSk0N6p04AFE5zVme85KTn0DGvI+CszjotHIadz4KzRkPLHaBwJfz3FHjxTChcFXA4SRuy6JQkSdLvlpoS5pajt+eg3i0oLY9y1hOT+OjrH4OOJdUpWb1dvl4dejV2+brWa9oVzngf9rwYQmGY+iyMHABzRwedTNJ6Fp2SJEmKi9SUMLcduyMDezanpCzCmY9N5JNvlgUdS6ozMitOXndGZ1y5T6eqSEmDfS+D096Bhh1hzQ/w6KHwzmVQWhR0OqnOs+iUJElS3KSlhLnjuD7s370ZxWURznhsAmNnW3ZKNSFru9jJ60VTpxL1ZOi46d0kViB/uexL31f9pG1/OPsT6HtK7PK4O+G+vWHR1CBTSXWeRackSZLiKj01zF0n9GGfrk0pKo1w+iMTGT/Hw1Gk6pbRtSuhtDTKV6+m9Icfgo5Ta3Rt1JXUcCori1eyYN2CoOMokWTUh0Nvg+OehXrN4Mev4P594ZNbIFIedDqpTrLolCRJUtxlpKYw8sS+7LltUwpLyzn1kQlMnOeJxVJ1Cqenk9G9OwCFU51VFi/pKel0bdgVgC+Xu3xdm9D1DzBsHHQ7BCKl8P5V8MjBsHJe0MmkOseiU5IkSdUiMy2F+07qy+7bNKGgpJwhD33GpO9WBh1LqtU8kKh6VO7T+aNFp35BvSZwzBPwx7shPQe+Hxc7qOjzJ8AtD6QaY9EpSZKkapOZlsL9J+/Erp0ak19SzikPfcaU+auCjiXVWlkeSFQtKorOact8X/UrQiHY8QQ45xNotxuUrINXhsOzJ0K++1VLNcGiU5IkSdUqKz2FB0/Zif4dG7G2uIyTHhzPtB9WBx1LqpUyK2Z0zphBtLQ04DS1R8WBRF+t+IqySFnAaZTwGnaAU16H/a+GcBrMfB3u3gVmvR10MqnWs+iUJElStctOT+XhU/qxU/uGrC0qY8jDn7Gu2LJAirf0Dh0I169PtKiI4m+/DTpOrdEhtwPZqdkUlhUyZ/WcoOMoGYRTYPfz4MwPoFkPyP8Rnj4GXj0XitcFnU6qtSw6JUmSVCPqZaTy8Kn9aJ6bwYr8Er5wCbsUd6FwmMzesWXWhe7TGTcp4RR6NukJwPRl0wNOo6TSojcM/QB2HQGEYPKjcM/uMP+zoJNJtZJFpyRJkmpMTmYaO7ZtCMCMhWsCTiPVTlm9twOgcJonr8eT+3Rqq6VlwsDrYMirkNsGVs6FhwbC//4B5W4xIcWTRackSZJqVM9WuQBMX+g+nVJ1qDiQyJPX46tX4/Unry/z5HVtpY57wrCxsN2xEI3A6H/DA/vBj7OCTibVGhadkiRJqlE9W1cUnc7olKpD5voZncXffkukoCDgNLVHxYFE36z8hqKyooDTKGll5sER98JRj0BWQ1j0Bdy7J3x6D0QiQaeTkp5FpyRJkmpUz1Z5AMz+cR2FJeUBp5Fqn7TmzUht3hwiEYpmzAg6Tq3Rol4LGmc2pixaxswVM4OOo2TX83A4Zxx03g/KiuDtS+CJw2H1gqCTSUnNolOSJEk1qllOBk3qpxOJwszFzuqUqkPF8nUPJIqfUChUuU+ny9cVF7kt4cQX4OCbIDUL5nwII3eFac8HnUxKWhadkiRJqlGhUIge62d1unxdqh6ZHkhULSqLzuUWnYqTUAj6nQFnfwyt+kDRanjhdHj+dChcGXQ6KelYdEqSJKnG/XQgkUWnVB2yescKOQ8kiq+KfTqd0am4a9IFTn8X9roUQinw5fNw924w+4Ogk0lJxaJTkiRJNa5Hy1jROcOT16VqkdkrVnSWLlhA2fLlAaepPXo27gnAd2u+Y3WxP78UZylpsM9f4fT3oFFnWLsQHh8Mb10KpYVBp5OSgkWnJEmSalzFjM6Zi9dSVu4ps1K8peTkkN6pEwCF05zVGS8NMhvQNqctANOXTw84jWqtNn1jS9n7nRG7PH4k3LsXLJwSaCwpGVh0SpIkqcZ1aFyPeukpFJdFmLMsP+g4Uq2U1Tu2zNrl6/HlgUSqEen1YocUnfA81G8Oy2bBA/vB6P9AeVnQ6aSEZdEpSZKkGhcOh+jesmKfTpd/StUhs+LkdWd0xlWvxrGic9oy31fVgC4HwDnjoPthECmD/10LjxwEK+YEnUxKSBadkiRJCkTlgUQLPJBIqg5Z28VOXi+aNo1oNBpwmtqjd9OfDiTyfVWNqNcYjn4MDr8XMnJh/ngYuTtMegQcg1IVFp2SJEkKRM9WeYAnr0vVJaNrV0JpaZSvWkXZDwuCjlNrdGvUjZRQCssKl7GkYEnQcVRXhEKw/bFwzhhovzuU5sNrf4Gnj4V1S4NOJyUMi05JkiQFokern5auOytKir9wejoZ3bsDUPSly6zjJSs1i20abAO4T6cC0KAdDHkNDvwHpKTD12/D3bvCzDeCTiYlBItOSZIkBWLb5jmkpYRYU1TGDysLg44j1UqVBxJNs5CLJw8kUqDCYdjtz3Dmh9C8FxQsg2eOh1dGQPHaoNNJgbLolCRJUiDSU8N0aZYDuHxdqi5Z6w8kKv7SQi6eejf5aZ9OKTDNe8LQ/8GAvwAh+PxxGDkAvhsXdDIpMBadkiRJCkzF8vUZnrwuVYvM9TM6i7/6CsrLA05Te1TM6Jy+fDqRaCTgNKrTUjPggGvglDcgrx2s+g4eHgTvXwVlJUGnk2qcRackSZICU3nyujM6pWqR3qED4fr1iRYVkbHEg3PipXODzmSmZLKudB3z1swLOo4EHQbEDira4QQgCp/cAg/sC0u/CjqZVKMsOiVJkhSYipPXZyyy6JSqQygcJrN3bPZh5vz5AaepPVLDqfRo3ANw+boSSGYuDL4bjnkCshvD4mlw714w7i6IOPNYdYNFpyRJkgLTvWVsj85Fq4tYke8SO6k6ZPXeDrDojLeeTXoCFp1KQN0PhXPGQZeBUF4M7/wNHv8jrPJngGo/i05JkiQFJiczjQ6NswGY7j6dUrWoOJAo84cfAk5Su3ggkRJaTnM4/lk45FZIy4a5o2MHFU19DqLRoNNJ1caiU5IkSYGqWL7uPp1S9chcP6MzffESIgUFAaepPXo1jm0JMHPFTErLSwNOI21CKAQ7nQpnfwJt+kHxanhxKDx/KhSsCDqdVC0sOiVJkhSoHh5IJFWrtObNSGnWjFA0Gjt9XXHRJqcNeRl5lEZK+Xrl10HHkX5Z485w6tuwz+UQToXpL8HI3eDb94NOJsWdRackSZIC9dPJ6y5dl6pLZu/YMuuiL11mHS+hUIheTWKzOqctmxZwGuk3pKTCXhfB6e9B4y6wdhE88Sd48yIocaa3ag+LTkmSJAWqYun63GX55BeXBZxGqp0ye8UKueJpFp3xVLF83aJTSaN1HzhrNPQ/K3b5s/vg3j1hweRgc0lxYtEpSZKkQDXNyaBpTgbRKMxc7PJ1qTpk9IqdEF70pYVcPFUcSDR92fSAk0hbID0bDroRTnwRclrC8m/gwQPgoxuh3D84KrlZdEqSJClwPd2nU6pWGT1jRWfZgoWULV8ecJrao2eT2Ps6Z/Uc1pWsCziNtIW22Q/OGQs9D4dIGXxwHTw0EJbPDjqZtNUsOiVJkhS4iqJzhkWnVC1ScnIobtoUgMJpzuqMlyZZTWhZryVRosxYPiPoONKWy24ERz4MRzwAGXmwYCLcsztMfAii0aDTSVvMolOSJEmBq9in0xmdUvUpats29nmqRWc8VRxI9OVy9z9VkgqFYLujYNhY6LgnlBbA6+fDU0fD2iVBp5O2iEWnJEmSAlcxo3PW4rWUlkcCTiPVThVFZ6H7dMZVxT6dXy6z6FSSy2sDJ70CA/8JKRnwzbtw9y4w49Wgk0mbzaJTkiRJgWvbMJucjFRKyiN8u9R97qTqUNS2Tezz1GlEXZIaNxUzOj15XbVCOAy7DoOzPoIWvaFwBTx3Erx0DhStDjqd9JssOiVJkhS4cDhEdw8kkqpVScuWkJZG+apVlP7wQ9Bxao0ejXsQIsTi/MUsK1wWdBwpPpp1hzP+B7tfAKEwfPEUjNwd5o0JOpn0qyw6JUmSlBB+OnndGSNSdYimppLRrSsAhVOnBpym9qiXVo/ODToDLl9XLZOaDvtfCae+BQ07wOrv4ZGD4d0roKw46HTSJll0SpIkKSF4IJFU/TJ7xfaT9ECi+OrZuCfg8nXVUu12gbM/gT4nA1EYezvcvy8stthX4rHolCRJUkLo0TI2o/OrhWuIRNw/UKoOGb1ihVzhNAu5eKo4kGj6sukBJ5GqSUYOHHYHHPs0ZDeBJV/C/fvAmNshUh50OqmSRackSZISQpfm9UlPCbO2uIwfVhYGHUeqlSpndM6YQbS0NOA0tUevpj8dSORBT6rVuh0Ewz6FrgdBeQm8dwU8ehis+j7oZBJg0SlJkqQEkZYSZtsW9QH36ZSqS1qH9oTr1ydaVETxt98GHafW2LbBtqSF01hTsob5a+cHHUeqXvWbwrFPxWZ4ptWD7z6BkQNgytNg0a+AWXRKkiQpYfRs6T6dUnUKhcNk9o7NPix0n864SUtJo3uj7oAHEqmOCIVie3ae8wm03RmK18DLZ8NzJ0P+8qDTqQ6z6JQkSVLC6Nnak9el6pbVezsAir606IynXk1+Wr4u1RmNOsVOZd/v7xBOha9ehZG7wjfvBZ1MdZRFpyRJkhJGz1YVRaczOqXqkrVdbJ9OZ3TGV0XROX25BxKpjgmnwB7/B0P/B027wbol8OSR8PoFUJIfdDrVMRadkiRJShjdWuQSCsHStcX8uLY46DhSrZS5fkZn8TffECkoCDhN7VFRdH61/CtKIx70pDqo5fZw5oewy7DY5YkPwj17wA8TA42lusWiU5IkSQmjXkYqHZvUA1y+LlWXtObNSG3eHCIRimbMCDpOrdE+tz05aTkUlRcxe9XsoONIwUjLgj/8E05+BXJbw4rZ8OCB8MH1UO4fAFT9LDolSZKUUHq28kAiqbq5fD3+wqEwPZr0ADyQSKLT3nDOWOh9NETL4aN/xQrPZd8EnUy1nEWnJEmSEkqPlrF9OmdYdErVpmL5euG0qQEnqV16N4kVyBadEpDVAP50Pxz5EGTmwcLJsaXsn90P0WjQ6VRLWXRKkiQpoVQcSDRjkUWnVF2yesf2kyxyRmdc9WrsyevSRnr9CYZ9Cp32gbJCePNCeOJPsGZR0MlUC1l0SpIkKaFUFJ1zl+Wzrrgs4DRS7ZTZK1bIlS5YQNny5QGnqT0qDiSavWo2BaUe9CRVym0FJ74Ig26E1EyYPQpG7grTXwo6mWoZi05JkiQllMb1M2iRmwnAV87qlKpFSk4O6Z06AVA4zdmH8dK8XnOaZTWjPFrOzBUzg44jJZZwGHY+C876GFruAIUr4b+nwItnQpEHECo+LDolSZKUcCpmdU5f4D98pOqS1Tu2n2TRNPeTjKeeTXoCLl+XflHTbeGM92HPiyEUhqnPknr/njRZOyPoZKoFLDolSZKUcCqLTg8kkqpNZsXJ6x5IFFcVBxJNXzY94CRSAktJg30vg9PegUadCK1ZwIBvbyD8/hVQWhR0OiUxi85qcNddd9GjRw/69esXdBRJkqSk1KNVHmDRKVWnrO1iJ68XTZ1G1BOQ46Zin05ndEqboW1/OOtjynccAkDK+JFw396wyD/AaOtYdFaD4cOHM2PGDCZMmBB0FEmSpKRUMaPzm6VrKSmLBJxGqp0yunYllJZG+apVlP7wQ9Bxao2Kpes/rPuBlUUrA04jJYGM+kQOuolPO11AtF4z+PEruH9f+OQWiJQHnU5JxqJTkiRJCadNwyxyM1MpLY/y9ZK1QceRaqVwejoZ3bsDUDjV2VPxkpueS4fcDgBMX+7ydWlzLcnbgbKho6HbIRAphfevgkcOhpXzgo6mJGLRKUmSpIQTCoXosX5W5wyXr0vVpvJAoqkus44nl69LW6leEzjmCfjj3ZCeA9+Pg5ED4PMnwC02tBksOiVJkpSQeq7fp3PGIotOqbpk9o4VcoXTLOTiqaLo/HKZJ9pLWywUgh1PgHM+gXa7Qck6eGU4PHsi5C8LOp0SnEWnJEmSEtJPJ6+vDjiJVHtVHkg0YwbR0tKA09QeGxadHvQkbaWGHeCU12H/qyGcBjNfh7t3gVlvB51MCcyiU5IkSQmpckbnwjVEIhYFUnVI79CBcP36RIuKKP7226Dj1BrdGnUjNZTKiqIVLMpfFHQcKXmFU2D38+DMD6BZD8j/EZ4+Bl49F4rXBZ1OCciiU5IkSQmpc9N6ZKSGyS8p57sVBUHHkWqlUDjs8vVqkJGSQZeGXQD36ZTiokVvGPoB7PZnIASTH4V7dof5nwWdTAnGolOSJEkJKTUlTLcWOYDL16XqlNV7/fJ1i8646t0kdtDT9GWevC7FRVomHPgPGPIa5LWFlXPhoYHwv39AuVtvKMaiU5IkSQmrx/rl69M9eV2qNlnbxQq5Qk9ejytPXpeqScc94JwxsN2xEI3A6H/DA/vBj7OCTqYEYNEpSZKkhPXTgUQWnVJ1yey9HfX32Yfcgw7y4Jw4qig6ZyyfQXmkPOA0Ui2TmQdH3AtHPQpZDWHRF3DvnvDpPRCJBJ1OAbLolCRJUsKqKDpnLFxtASNVk7TmzWg78m6anHUmoVAo6Di1Rqe8TmSlZlFQVsDc1XODjiPVTj0Hw7BPYZv9oawI3r4EnjgcVi8IOpkCYtEpSZKkhNWtRS7hECxbV8LStcVBx5GkzZYSTqFn456Ay9elapXTAk54Hg6+CVKzYM6HMHJXmPZ80MkUAItOSZIkJays9BQ6Na0PwAyXr0tKMqf2OpWb976ZPdvsGXQUqXYLhaDfGXD2x9CqDxSthhdOh+dPh8KVQadTDbLolCRJUkL7aZ9OT16XlFz2bLMnB7Q/gMZZjYOOItUNTbrA6e/C3n+FUAp8+TzcvRvM/iDoZKohFp2SJElKaB5IJEmSNltKGux9KZz+HjTqDGsXwuOD4a1LobQw6HSqZhadkiRJSmg9W+UBFp2SJGkLtOkbW8re74zY5fEj4d69YOGUQGOpell0SpIkKaFVzOj8fkUBa4pKA04jSZKSRnq92CFFJzwP9ZvDslnwwH4w+j9QXhZ0OlUDi05JkiQltAbZ6bRukAV4IJEkSdoKXQ6AYZ9C98MgUgb/uxYeOQhWzAk6meLMolOSJEkJr4f7dEqSpN8juxEc/Rgcfi9k5ML88TByd5j0CESjQadTnFh0SpIkKeH1aOnJ65Ik6XcKhWD7Y+GcMdBhDyjNh9f+Ak8fC+uWBp1OcWDRKUmSpIRXsU+nS9clSdLv1qAdnPwqHHgdpKTD12/D3bvCzDeCTqbfyaJTkiRJCa9n69jJ698uXUdxWXnAaSRJUtILh2G3EXDmh9C8FxQsg2eOh1dGQPHaoNNpK1l0SpIkKeG1ysukQXYaZZEoXy9eF3QcSZJUWzTvCUP/BwP+AoTg88dh5AD4blzQybQVLDolSZKU8EKhUOXydffplCRJcZWaAQdcA6e8EVvWvuo7eHgQvH8VlJUEnU5bwKJTkiRJSaFnq9jydU9elyRJ1aLDADh7DOxwIhCFT26BB/aFpV8FnUybyaJTkiRJScEZnZIkqdpl5sLgu+CYJyC7MSyeBvfuBePugkgk6HT6DRadkiRJSgoVRedXi9ZSHokGnEaSJNVq3Q+Fc8ZBl4FQXgzv/A0e/yOsmh90Mv0Ki05JkiQlhY5N6pOZFqawtJy5y/KDjiNJkmq7nOZw/LNwyK2Qlg1zR8cOKpr6HET9o2sisuiUJElSUkgJh+jWwuXrkiSpBoVCsNOpcPYn0KYfFK+GF4fC86dCwYqg0+lnLDolSZKUNCqWr89Y5IFEkiSpBjXuDKe+DftcDuFUmP4SjNwNvn0/6GTagEWnJEmSkkbFyeszPHldkiTVtJRU2OsiOP09aLItrF0ET/wJ3rwISgqCTicsOiVJkpREfjp5fQ1R98aSJElBaN0HzvwI+p8Vu/zZffDKsGAzCbDolCRJUhLp2iKHlHCIFfklLF5TFHQcSZJUV6Vnw0E3wrFPxy7PfBNKC4PNJItOSZIkJY/MtBS2aVofgOkLXL4uSZIC1nUQ5LSC8mKYPz7oNHWeRackSZKSyobL1yVJkgIVCkHHPWNfzx0dbBZZdEqSJCm59KgsOlcHnESSJAnotFfs85yPgs0hi05JkiQll4qT153RKUmSEkLFjM6Fk6HIP8QGyaJTkiRJSaVHy9iMzgWrCllVUBJwGkmSVOfltYFGnSEage/GBZ2mTrPolCRJUlLJy06jTcMsAGYsclanJElKAJX7dLp8PUgWnZIkSUo6FQcSzXD5uiRJSgQeSJQQLDolSZKUdNynU5IkJZSKonPJl7Dux2Cz1GEWnZIkSUo6PT15XZIkJZJ6TaB5r9jX8z4ONksdZtEpSZKkpFMxo3P2j/kUlZYHnEaSJAnouFfss8vXA2PRKUmSpKTTPDeDxvXSKY9Embl4bdBxJEmSPJAoAVh0SpIkKemEQiF6uHxdkiQlkva7QSgFVsyBVfODTlMnWXRKkiQpKXkgkSRJSiiZudC6T+xrl68HwqJTkiRJSemnGZ0WnZIkKUFULl+36AyCRackSZKSUsXJ6zMXraGsPBJwGkmSJDY4kOgjiEaDzVIHWXRKkiQpKXVsXI/s9BSKyyLMXZYfdBxJkiRo2x9SMmDtIlj+bdBp6hyLTkmSJCWlcDhE95YuX5ckSQkkLQva7Rz72tPXa5xFpyRJkpJWT09elyRJiaZin845Fp01zaJTkiRJSaunBxJJkqREU7FP57yPIeI+4jXJolOSJElJq2erPCBWdEbd8F+SJCWCVn0gPQcKV8KSaUGnqVMsOiVJkpS0ujSvT2o4xOrCUhasKgw6jiRJEqSkQvvdYl/PHR1sljrGolOSJElJKyM1hS7NcwCXr0uSpATSaf3ydYvOGmXRKUmSpKTWY/3J6zMsOiVJUqKoOJDou7FQXhpsljrEolOSJElJzQOJJElSwmnWE7IbQ8k6WDA56DR1hkWnJEmSklpF0Tlj4eqAk0iSJK0XDkOHPWJfz/0o2Cx1iEWnJEmSklqP9UXnwtVFrMwvCTiNJEnSehXL192ns8ZYdEqSJCmp5WSm0b5xNuDydUmSlEA67R37PH88lBQEGqWusOiUJElS0vtpn06Xr0uSpATRqBPktobykljZqWpn0SlJkqSk17NVHuCMTkmSlEBCIZev1zCLTkmSJCW9Hs7olCRJiajjXrHPHkhUIyw6JUmSlPQqlq7PWZZPQUlZwGkkSZLWq5jRufBzKPIPstXNolOSJElJr1lOJk3qZxCNwszFa4OOI0mSFJPXGhpvA9EIzBsTdJpaz6JTkiRJtcJPBxK5T6ckSUog7tNZYyw6JUmSVCtUFJ0z3KdTkiQlksp9Oi06q5tFpyRJkmoFT16XJEkJqcMesc9Lp8O6H4PNUstZdEqSJKlWqJjROXPxWkrLIwGnkSRJWq9eY2jeO/b1PGd1VieLTkmSJNUK7RplUz8jlZKyCLN/XBd0HEmSpJ90Wr98fc5Hweao5Sw6JUmSVCuEwyF6tFx/INECl69LkqQE4oFENcKiU5IkSbVGD09elyRJiaj9bhBKgZVzYdX3QaeptSw6JUmSVGv0rCw6PXldkiQlkIwcaN039rWzOquNRackSZJqjYoZnXOX5RONRgNOI0mStAGXr1c7i05JkiTVGts2z+Hd8/dk7KX7EgqFgo4jSZL0kw0PJPIPstXColOSJEm1RlpKmG2b55Ca4q+5kiQpwbTpD6mZsG4xLPsm6DS1kr8BSpIkSZIkSdUtLRPa7hz7eu5HwWappSw6JUmSJEmSpJpQuU+nRWd1sOiUJEmSJEmSakKnvWOf534MkUigUWoji05JkiRJkiSpJrTcAdJzoGgVLJ4adJpax6JTkiRJkiRJqgkpqdBhQOzruaODzVILWXRKkiRJkiRJNaXjXrHP7tMZdxadkiRJkiRJUk2pOJDou3FQVhJsllrGolOSJEmSJEmqKc16QHYTKM2HBZOCTlOrWHRKkiRJkiRJNSUcho57xL52n864suiUJEmSJEmSalLF8nWLzriy6JQkSZIkSZJqUsWBRD98BiUFwWapRSw6JUmSJEmSpJrUqBPktoHyEpj/adBpag2LTkmSJEmSJKkmhULQaf2szjkfBZulFrHolCRJkiRJkmqa+3TGnUWnJEmSJEmSVNMqis5FU6BwVZBJag2LTkmSJEmSJKmm5baCxl0gGoHvxgSdplaw6JQkSZIkSZKC4PL1uLLolCRJkiRJkoLggURxZdEpSZIkSZIkBaHDHkAIfvwK1i0NOk3Ss+iUJEmSJEmSgpDdCFr0jn3t8vXfzaJTkiRJkiRJCkrlPp0uX/+9LDolSZIkSZKkoHRcv0+nMzp/N4tOSZIkSZIkKSjtd4VwKqycByu/CzpNUrPolCRJkiRJkoKSkQOt+8a+dlbn72LRKUmSJEmSJAWpcvm6+3T+HhadkiRJkiRJUpAqDyQaDdFosFmSmEWnJEmSJEmSFKQ2/SA1E9YtgWVfB50maVl0SpIkSZIkSUFKy4R2u8S+nuPy9a1l0SlJkiRJkiQFrXL5ukXn1rLolCRJkiRJkoLWce/Y53kfQ6Q8yCRJy6JTkiRJkiRJClrL7SEjF4pWw+KpQadJShadkiRJkiRJUtBSUqH9gNjX7tO5VSw6JUmSJEmSpETQaa/Y57mjg82RpCw6JUmSJEmSpERQcSDR9+OgrCTYLEnIolOSJEmSJElKBM16QHYTKC2ABRODTpN0LDolSZIkSZKkRBAK/TSr0+XrW8yiU5IkSZIkSUoUFft0eiDRFrPolCRJkiRJkhJFxYzOHyZASX6wWZKMRackSZIkSZKUKBp2hLy2ECmF7z8NOk1SseiUJEmSJEmSEkUoBB3XL1+f6/L1LWHRKUmSJEmSJCUSDyTaKhadkiRJkiRJUiKpKDoXToHClYFGSSYWnZIkSZIkSVIiyW0JTbYFojBvTNBpkoZFpyRJkiRJkpRoKpevu0/n5rLolCRJkiRJkhJN5YFE7tO5uSw6JUmSJEmSpETTYXcgBD/OhLVLgk6TFCw6f0OHDh3Ybrvt2GGHHdhnn32CjiNJkiRJkqS6ILsRtNwu9rWzOjdLatABksHYsWOpX79+0DEkSZIkSZJUl3TcExZ9Edunc7ujgk6T8JzRKUmSJEmSJCWijnvHPnsg0Wap1UXn6NGjOfTQQ2nVqhWhUIiXX355o/vcdddddOjQgczMTHbeeWc+++yzKreHQiH22msv+vXrx5NPPllDySVJkiRJklTntdsFwqmw6ntYOS/oNAmvVhed+fn5bL/99tx1112bvP3ZZ5/lggsu4Morr2Ty5Mlsv/32DBw4kKVLl1be55NPPmHSpEm8+uqrXH/99UydOrWm4kuSJEmSJKkuy6gPrXeKfe0+nb+pVu/ROWjQIAYNGvSLt998880MHTqUU089FYB77rmHN954g4ceeohLL70UgNatWwPQsmVLDjroICZPnsx22223yecrLi6muLi48vKaNWsAKC0tpbS0NC6vKVlVvP66/j4oPhxPiifHk+LJ8aR4cjwpnhxPiifHk+LNMfXrwu13J2X+p0Rmf0B57+OCjlPjtmRchKLRaLQasySMUCjESy+9xODBgwEoKSkhOzub559/vvI6gCFDhrBq1SpeeeUV8vPziUQi5OTksG7dOvbaay/uuece+vXrt8nvcdVVV3H11VdvdP1TTz1FdnZ2dbwsSZIkSZIk1WKN185k92+vpyg1j3d63Q6hUNCRalRBQQHHH388q1evJjc391fvW6tndP6aZcuWUV5eTvPmzatc37x5c2bOnAnAkiVLOPzwwwEoLy9n6NChv1hyAvz1r3/lggsuqLy8Zs0a2rZty4EHHvib/yFqu9LSUt577z0OOOAA0tLSgo6jJOd4Ujw5nhRPjifFk+NJ8eR4Ujw5nhRvjqnfULYf0ZtuIbNsNQf17wxNuwWdqEZVrJjeHHW26NwcnTp14osvvtjs+2dkZJCRkbHR9Wlpaf4PdT3fC8WT40nx5HhSPDmeFE+OJ8WT40nx5HhSvDmmfkFaWuxQojkfkDZ/LLTqHXSiGrUlY6JWH0b0a5o0aUJKSgpLliypcv2SJUto0aJFQKkkSZIkSZKkn+m4Z+yzBxL9qjpbdKanp9O3b19GjRpVeV0kEmHUqFHsuuuuASaTJEmSJEmSNtBpr9jneR9DpDzYLAmsVi9dX7duHd9++23l5blz5zJlyhQaNWpEu3btuOCCCxgyZAg77bQT/fv359ZbbyU/P7/yFHZJkiRJkiQpcC13gIw8KFoNi76A1n2CTpSQanXROXHiRPbZZ5/KyxUHBQ0ZMoRHHnmEY445hh9//JG///3vLF68mB122IG33357owOKJEmSJEmSpMCEU6DD7jDrDZj7kUXnL6jVRefee+9NNBr91fuMGDGCESNG1FAiSZIkSZIkaSt03HN90Tkadj8/6DQJqc7u0SlJkiRJkiQljYoDib4bB2XFwWZJUBadkiRJkiRJUqJr1h3qNYWyQvhhYtBpEpJFpyRJkiRJkpToQqGfZnXOHR1slgRl0SlJkiRJkiQlg457xT7P/SjYHAnKolOSJEmSJElKBhUzOn+YACX5wWZJQBadkiRJkiRJUjJo1BEatINIWexQIlVh0SlJkiRJkiQli8p9Ol2+/nMWnZIkSZIkSVKyqNyn0wOJfs6iU5IkSZIkSUoWFTM6F30BBSuCzZJgLDolSZIkSZKkZJHTApp0BaLw3Zig0yQUi05JkiRJkiQpmXRav3x9jvt0bsiiU5IkSZIkSUomlQcSuU/nhiw6q8Fdd91Fjx496NevX9BRJEmSJEmSVNu0HwCEYNksWLMo6DQJw6KzGgwfPpwZM2YwYcKEoKNIkiRJkiSptsluBC23j3097+NgsyQQi05JkiRJkiQp2VQuX3efzgoWnZIkSZIkSVKyqTyQaDREo8FmSRAWnZIkSZIkSVKyabcrhFNh9fewcl7QaRKCRackSZIkSZKUbNLrQZv1B2G7fB2w6JQkSZIkSZKSU8f1y9fnjg42R4Kw6JQkSZIkSZKSUeWBRO7TCRadkiRJkiRJUnJq0w/qNYNWfaBoddBpApcadABJkiRJkiRJWyE1Hf5vFoSdywjO6JQkSZIkSZKSlyVnJd8JSZIkSZIkSUnPolOSJEmSJElS0rPolCRJkiRJkpT0LDolSZIkSZIkJT2LTkmSJEmSJElJz6JTkiRJkiRJUtKz6JQkSZIkSZKU9Cw6JUmSJEmSJCU9i05JkiRJkiRJSc+iU5IkSZIkSVLSs+iUJEmSJEmSlPQsOqvBXXfdRY8ePejXr1/QUSRJkiRJkqQ6waKzGgwfPpwZM2YwYcKEoKNIkiRJkiRJdYJFpyRJkiRJkqSkZ9EpSZIkSZIkKelZdEqSJEmSJElKehadkiRJkiRJkpKeRackSZIkSZKkpGfRKUmSJEmSJCnpWXRKkiRJkiRJSnqpQQeozaLRKABr1qwJOEnwSktLKSgoYM2aNaSlpQUdR0nO8aR4cjwpnhxPiifHk+LJ8aR4cjwp3hxT+jUVvVpFz/ZrLDqr0dq1awFo27ZtwEkkSZIkSZKk5LV27Vry8vJ+9T6h6ObUodoqkUiEhQsXkpOTQygUCjpOoNasWUPbtm2ZP38+ubm5QcdRknM8KZ4cT4onx5PiyfGkeHI8KZ4cT4o3x5R+TTQaZe3atbRq1Ypw+Nd34XRGZzUKh8O0adMm6BgJJTc31x9aihvHk+LJ8aR4cjwpnhxPiifHk+LJ8aR4c0zpl/zWTM4KHkYkSZIkSZIkKelZdEqSJEmSJElKehadqhEZGRlceeWVZGRkBB1FtYDjSfHkeFI8OZ4UT44nxZPjSfHkeFK8OaYULx5GJEmSJEmSJCnpOaNTkiRJkiRJUtKz6JQkSZIkSZKU9Cw6JUmSJEmSJCU9i05JkiRJkiRJSc+iU1vlrrvuokOHDmRmZrLzzjvz2Wef/er9b731Vrp27UpWVhZt27bl/PPPp6io6Hc9p2qPeI+nf/7zn/Tr14+cnByaNWvG4MGDmTVrVnW/DCWI6vj5VOGGG24gFApx3nnnVUNyJarqGFMLFizgxBNPpHHjxmRlZdG7d28mTpxYnS9DCSLe46m8vJwrrriCjh07kpWVRefOnbn22mvxvNG6YUvGU2lpKddccw2dO3cmMzOT7bffnrfffvt3Padql3iPJ38nr9uq4+dTBX8n16+KSlvomWeeiaanp0cfeuih6PTp06NDhw6NNmjQILpkyZJN3v/JJ5+MZmRkRJ988sno3Llzo++88060ZcuW0fPPP3+rn1O1R3WMp4EDB0Yffvjh6JdffhmdMmVK9KCDDoq2a9cuum7dupp6WQpIdYynCp999lm0Q4cO0e222y76l7/8pZpfiRJFdYypFStWRNu3bx895ZRTouPHj4/OmTMn+s4770S//fbbmnpZCkh1jKfrrrsu2rhx4+jrr78enTt3bvS///1vtH79+tHbbrutpl6WArKl4+niiy+OtmrVKvrGG29EZ8+eHb377rujmZmZ0cmTJ2/1c6r2qI7x5O/kdVd1jKcK/k6u32LRqS3Wv3//6PDhwysvl5eXR1u1ahX95z//ucn7Dx8+PLrvvvtWue6CCy6IDhgwYKufU7VHdYynn1u6dGkUiH700UfxCa2EVV3jae3atdEuXbpE33vvvehee+3lL1V1SHWMqUsuuSS6++67V09gJbTqGE8HH3xw9LTTTqtynyOOOCJ6wgknxDG5EtGWjqeWLVtG77zzzirX/Xys+Dt53VUd4+nn/J287qiu8eTv5NocLl3XFikpKWHSpEnsv//+ldeFw2H2339/xo0bt8nH7LbbbkyaNKlyqvqcOXN48803Oeigg7b6OVU7VMd42pTVq1cD0KhRozimV6KpzvE0fPhwDj744CrPrdqvusbUq6++yk477cRRRx1Fs2bN2HHHHbn//vur98UocNU1nnbbbTdGjRrF119/DcAXX3zBJ598wqBBg6rx1ShoWzOeiouLyczMrHJdVlYWn3zyyVY/p2qH6hhPm+Lv5HVDdY4nfyfX5kgNOoCSy7JlyygvL6d58+ZVrm/evDkzZ87c5GOOP/54li1bxu677040GqWsrIyzzz6bv/3tb1v9nKodqmM8/VwkEuG8885jwIAB9OrVK+6vQYmjusbTM888w+TJk5kwYUK15lfiqa4xNWfOHEaOHMkFF1zA3/72NyZMmMC5555Leno6Q4YMqdbXpOBU13i69NJLWbNmDd26dSMlJYXy8nKuu+46TjjhhGp9PQrW1oyngQMHcvPNN7PnnnvSuXNnRo0axYsvvkh5eflWP6dqh+oYTz/n7+R1R3WNJ38n1+ZyRqeq3Ycffsj111/P3XffzeTJk3nxxRd54403uPbaa4OOpiS0peNp+PDhfPnllzzzzDM1nFTJ4LfG0/z58/nLX/7Ck08+udFfmaVN2ZyfUZFIhD59+nD99dez4447cuaZZzJ06FDuueeeAJMrEW3OeHruued48skneeqpp5g8eTKPPvoo//nPf3j00UcDTK5EdNttt9GlSxe6detGeno6I0aM4NRTTyUc9p+E2nJbOp78nVy/5rfGk7+Ta0s4o1NbpEmTJqSkpLBkyZIq1y9ZsoQWLVps8jFXXHEFJ510EmeccQYAvXv3Jj8/nzPPPJPLLrtsq55TtUN1jKcNf7kaMWIEr7/+OqNHj6ZNmzbV90KUEKpjPE2aNImlS5fSp0+fyseUl5czevRo7rzzToqLi0lJSam+F6VAVdfPqJYtW9KjR48qj+vevTsvvPBC9bwQJYTqGk8XXXQRl156Kccee2zlfb777jv++c9/OkO4Ftua8dS0aVNefvllioqKWL58Oa1ateLSSy+lU6dOW/2cqh2qYzxtyN/J65bqGE/+Tq4t4Z/vtEXS09Pp27cvo0aNqrwuEokwatQodt11100+pqCgYKO/7FX8EIpGo1v1nKodqmM8VXweMWIEL730Ev/73//o2LFjNb0CJZLqGE/77bcf06ZNY8qUKZUfO+20EyeccAJTpkzxF6parrp+Rg0YMIBZs2ZVuc/XX39N+/bt4xlfCaa6xtMv3ScSicQzvhLM7/n9OTMzk9atW1NWVsYLL7zAH//4x9/9nEpu1TGewN/J66rqGE/+Tq4tEswZSEpmzzzzTDQjIyP6yCOPRGfMmBE988wzow0aNIguXrw4Go1GoyeddFL00ksvrbz/lVdeGc3JyYk+/fTT0Tlz5kTffffdaOfOnaNHH330Zj+naq/qGE/nnHNONC8vL/rhhx9GFy1aVPlRUFBQ469PNas6xtPPecJj3VIdY+qzzz6LpqamRq+77rroN998E33yySej2dnZ0SeeeKLGX59qVnWMpyFDhkRbt24dff3116Nz586Nvvjii9EmTZpEL7744hp/fapZWzqePv300+gLL7wQnT17dnT06NHRfffdN9qxY8foypUrN/s5VXtVx3jyd/K6qzrG08/5O7l+iUWntsodd9wRbdeuXTQ9PT3av3//6Kefflp521577RUdMmRI5eXS0tLoVVddFe3cuXM0MzMz2rZt2+iwYcM2+qH1a8+p2i3e4wnY5MfDDz9ccy9KgamOn08b8pequqc6xtRrr70W7dWrVzQjIyParVu36H333VdDr0ZBi/d4WrNmTfQvf/lLtF27dtHMzMxop06dopdddlm0uLi4Bl+VgrIl4+nDDz+Mdu/ePZqRkRFt3Lhx9KSTToouWLBgi55TtVu8x5O/k9dt1fHzaUP+Tq5fEopG1697kSRJkiRJkqQk5R6dkiRJkiRJkpKeRackSZIkSZKkpGfRKUmSJEmSJCnpWXRKkiRJkiRJSnoWnZIkSZIkSZKSnkWnJEmSJEmSpKRn0SlJkiRJkiQp6Vl0SpIkSZIkSUp6Fp2SJEnSFrjqqqvYYYcdKi+fcsopDB48OLA8kiRJirHolCRJkiRJkpT0LDolSZJUa5SUlAQdQZIkSQGx6JQkSVLS2nvvvRkxYgTnnXceTZo0YeDAgXz55ZcMGjSI+vXr07x5c0466SSWLVtW+ZhIJMKNN97INttsQ0ZGBu3ateO6666rvP2SSy5h2223JTs7m06dOnHFFVdQWloaxMuTJEnSFrDolCRJUlJ79NFHSU9PZ8yYMdxwww3su+++7LjjjkycOJG3336bJUuWcPTRR1fe/69//Ss33HADV1xxBTNmzOCpp56iefPmlbfn5OTwyCOPMGPGDG677Tbuv/9+brnlliBemiRJkrZAKBqNRoMOIUmSJG2NvffemzVr1jB58mQA/vGPf/Dxxx/zzjvvVN7nhx9+oG3btsyaNYuWLVvStGlT7rzzTs4444zN+h7/+c9/eOaZZ5g4cSIQO4zo5ZdfZsqUKUDsMKJVq1bx8ssvx/W1SZIkacukBh1AkiRJ+j369u1b+fUXX3zBBx98QP369Te63+zZs1m1ahXFxcXst99+v/h8zz77LLfffjuzZ89m3bp1lJWVkZubWy3ZJUmSFD8WnZIkSUpq9erVq/x63bp1HHroofzrX//a6H4tW7Zkzpw5v/pc48aN44QTTuDqq69m4MCB5OXl8cwzz3DTTTfFPbckSZLiy6JTkiRJtUafPn144YUX6NChA6mpG/+q26VLF7Kyshg1atQml66PHTuW9u3bc9lll1Ve991331VrZkmSJMWHhxFJkiSp1hg+fDgrVqzguOOOY8KECcyePZt33nmHU089lfLycjIzM7nkkku4+OKLeeyxx5g9ezaffvopDz74IBArQr///nueeeYZZs+eze23385LL70U8KuSJEnS5rDolCRJUq3RqlUrxowZQ3l5OQceeCC9e/fmvPPOo0GDBoTDsV99r7jiCv7v//6Pv//973Tv3p1jjjmGpUuXAnDYYYdx/vnnM2LECHbYYQfGjh3LFVdcEeRLkiRJ0mby1HVJkiRJkiRJSc8ZnZIkSZIkSZKSnkWnJEmSJEmSpKRn0SlJkiRJkiQp6Vl0SpIkSZIkSUp6Fp2SJEmSJEmSkp5FpyRJkiRJkqSkZ9EpSZIkSZIkKelZdEqSJEmSJElKehadkiRJkiRJkpKeRackSZIkSZKkpGfRKUmSJEmSJCnpWXRKkiRJkiRJSnoWnZIkSZIkSZKSnkWnJEmSJEmSpKRn0SlJkiRJkiQp6Vl0SpIkSZIkSUp6Fp2SJEmSJEmSkp5FpyRJkiRJkqSkZ9EpSZIkSZIkKelZdEqSJEmSJElKehadkiRJkiRJkpKeRackSZIkSZKkpGfRKUmSJEmSJCnpWXRKkiRJkiRJSnoWnZIkSZIkSZKSnkWnJEmSJEmSpKRn0SlJkiRJkiQp6Vl0SpIkSZIkSUp6Fp2SJEmSJEmSkp5FpyRJkiRJkqSkZ9EpSZIkSZIkKelZdEqSJEmSJElKehadkiRJkiRJkpKeRackSZIkSZKkpGfRKUmSJEmSJCnpWXRKkiRJkiRJSnoWnZIkSZIkSZKSnkWnJEmSJEmSpKRn0SlJkiRJkiQp6Vl0SpIkSZIkSUp6Fp2SJEmSJEmSkp5FpyRJkiRJkqSkZ9EpSZIkSZIkKelZdEqSJEmSJElKehadkiRJkiRJkpKeRackSZIkSZKkpGfRKUmSpKS1995706tXr6BjSJIkKQFYdEqSJEkbKCws5PTTT6dXr17k5eVRv359tt9+e2677TZKS0ur3HfUqFGcdtppbLvttmRnZ9OpUyfOOOMMFi1a9IvPf8cdd5CXl1f5XIsWLeLMM8+kY8eOZGVl0blzZy644AKWL1++yce/9tprhMNhFi9e/KuvY+bMmVx88cXssMMO5OTk0LJlSw4++GAmTpy4he+IJElSckgNOoAkSZKUSAoLC5k+fToHHXQQHTp0IBwOM3bsWM4//3zGjx/PU089VXnfSy65hBUrVnDUUUfRpUsX5syZw5133snrr7/OlClTaNGixUbP/8Ybb3DggQeSlpbGunXr2HXXXcnPz2fYsGG0bduWL774gjvvvJMPPviASZMmEQ6HN3p83759N/ncG3rggQd48MEH+dOf/sSwYcNYvXo19957L7vssgtvv/02+++/f3zeMEmSpAQRikaj0aBDSJIkSRXy8/OpV6/eZt137733ZtmyZXz55ZfVnAr+/Oc/c+edd7Jo0aLKknH06NHsvvvuVcrI0aNHs9dee3HZZZfxj3/8o8pzFBQU0LhxY0aOHMkpp5zCU089xQknnMDrr7/OwQcfXHm/K6+8kmuuuYbJkyez4447VnmOdu3acdppp3HVVVdtMmdRURHp6el8/vnndO3alfr161fetnz5crp37862227LJ5988nvfEkmSpITi0nVJkiQBsHbtWs477zw6dOhARkYGzZo144ADDmDy5MlV7jd+/Hj+8Ic/kJeXR3Z2NnvttRdjxoypcp/vvvuOYcOG0bVrV7KysmjcuDFHHXUU8+bNq3K/Rx55hFAoxEcffcSwYcNo1qwZbdq0qbz9rbfeYq+99iInJ4fc3Fz69etXZUZlhRkzZrDPPvuQnZ1N69atufHGGze6z/fff8/MmTO3+v3p0KEDAKtWraq8bs8999xoxuWee+5Jo0aN+OqrrzZ6jlGjRlFcXMygQYMAWLNmDQDNmzevcr+WLVsCkJWVVeX6adOmMX/+/MpS9MMPPyQUCvHMM89w+eWX07p1a7Kzs1mzZg19+/atUnICNG7cmD322GOT2SRJkpKdS9clSZIEwNlnn83zzz/PiBEj6NGjB8uXL+eTTz7hq6++ok+fPgD873//Y9CgQfTt25crr7yScDjMww8/zL777svHH39M//79AZgwYQJjx47l2GOPpU2bNsybN4+RI0ey9957M2PGDLKzs6t872HDhtG0aVP+/ve/k5+fD8RK0NNOO42ePXvy17/+lQYNGvD555/z9ttvc/zxx1c+duXKlfzhD3/giCOO4Oijj+b555/nkksuoXfv3pWFIsDJJ5/MRx99xOYuaCopKWHNmjUUFhYyceJE/vOf/9C+fXu22WabX33cunXrWLduHU2aNNnotjfffJO+fftWFpsVRelf/vIXbrrpJtq0acPUqVO57rrrGDx4MN26ddvo8c2aNWOnnXaqcv21115Leno6F154IcXFxaSnp/9ivsWLF28ymyRJUrKz6JQkSRIQ2/tx6NCh3HTTTZXXXXzxxZVfR6NRzj77bPbZZx/eeustQqEQAGeddRY9e/bk8ssv59133wXg4IMP5sgjj6zy/Iceeii77rorL7zwAieddFKV2xo1asSoUaNISUkBYPXq1Zx77rn079+fDz/8kMzMzCo5NrRw4UIee+yxyuc8/fTTad++PQ8++GCVonNLvfjiixx33HGVl3faaSceeughUlN//VfoW2+9lZKSEo455piNbnvzzTc59dRTKy/36NGD++67jwsvvJBdd9218vohQ4bwwAMPbPT4N954g0GDBlW+9xWKioqYOHHiRjNAf+7jjz9m3LhxXH755b96P0mSpGRk0SlJkiQAGjRowPjx41m4cCGtWrXa6PYpU6bwzTffcPnll290Ivh+++3H448/TiQSIRwOVyncSktLWbNmDdtssw0NGjRg8uTJGxWdQ4cOrSw5Ad577z3Wrl3LpZdeWqXkBDYq+erXr8+JJ55YeTk9PZ3+/fszZ86cKvf78MMPN++NWG+fffbhvffeY9WqVYwaNYovvviicrbpLxk9ejRXX301Rx99NPvuu2+V27788ku+//77KntxArRu3Zr+/ftz0EEH0b59ez7++GNuv/12mjRpwn/+85/K+61atYpx48bx5z//eaPvO2TIkN8sOZcuXcrxxx9Px44dqxTYkiRJtYVFpyRJkgC48cYbGTJkCG3btqVv374cdNBBnHzyyXTq1AmAb775BoiVar9k9erVNGzYkMLCQv75z3/y8MMPs2DBgiqzMFevXr3R4zp27Fjl8uzZswHo1avXb+Zu06bNRuVnw4YNmTp16m8+9tc0b968con5kUceyfXXX88BBxzAN998s8kTz2fOnMnhhx9Or169fnE2ZvPmzassOx8zZgyHHHIIn376aeX1gwcPJjc3l6uvvprTTjuNHj16APDOO+8AcOCBB2703D9//34uPz+fQw45hLVr1/LJJ59stHenJElSbeBhRJIkSQLg6KOPZs6cOdxxxx20atWKf//73/Ts2ZO33noLgEgkAsC///1v3nvvvU1+VBRof/7zn7nuuus4+uijee6553j33Xd57733aNy4ceXzbOi3ZiP+mg1ngm5oc/fi3FxHHnkk69at45VXXtnotvnz53PggQeSl5fHm2++SU5Ozkb3efPNN/nDH/5QpZS99957Nyo/AQ477DCi0Shjx46t8vgBAwaQl5e30XP/2vtXUlLCEUccwdSpU3nllVc2qzyWJElKRs7olCRJUqWWLVsybNgwhg0bxtKlS+nTpw/XXXcdgwYNonPnzgDk5uay//77/+rzPP/88wwZMqTKfp9FRUVVTiz/NRXf68svv/zNw39qSmFhIbDxjNTly5dz4IEHUlxczKhRoypPTN/QqlWrGDt2LCNGjKhy/ZIlSygvL9/o/qWlpQCUlZUBsdL27bff5sILL9yizJFIhJNPPplRo0bx3HPPsddee23R4yVJkpKJMzolSZJEeXn5RgVes2bNaNWqFcXFxQD07duXzp0785///Id169Zt9Bw//vhj5dcpKSkbzai84447NlnqbcqBBx5ITk4O//znPykqKqpy29bO1Pz++++ZOXPmb95v2bJlm/weFcvRN5x9mZ+fz0EHHcSCBQt488036dKlyyafs+KQpp8vO992221ZsmTJRvuHPv300wDsuOOOQOwU+6VLl260v+dv+fOf/8yzzz7L3XffzRFHHLFFj5UkSUo2zuiUJEkSa9eupU2bNhx55JFsv/321K9fn/fff58JEyZUzsoMh8M88MADDBo0iJ49e3LqqafSunVrFixYwAcffEBubi6vvfYaAIcccgiPP/44eXl59OjRg3HjxvH+++/TuHHjzcqTm5vLLbfcwhlnnEG/fv04/vjjadiwIV988QUFBQU8+uijW/waTz75ZD766KPfLEqfeOIJ7rnnHgYPHkynTp1Yu3Yt77zzDu+99x6HHnpolUOGTjjhBD777DNOO+00vvrqK7766qvK2+rXr8/gwYOB2P6cu++++0bLzkeMGMHDDz/MoYceyp///Gfat2/PRx99xNNPP80BBxzAzjvvXPn4Dh06VO7XuTluvfVW7r77bnbddVeys7N54oknqtx++OGHU69evc1+PkmSpERn0SlJkiSys7MZNmwY7777Li+++CKRSIRtttmGu+++m3POOafyfnvvvTfjxo3j2muv5c4772TdunW0aNGCnXfembPOOqvyfrfddhspKSk8+eSTFBUVMWDAAN5//30GDhy42ZlOP/10mjVrxg033MC1115LWloa3bp14/zzz4/ra/+53XffnbFjx/L000+zZMkSUlNT6dq1KzfffPNGJ55PmTIFgIceeoiHHnqoym3t27dn8ODBv7rsvGvXrkyaNInLL7+cJ554gsWLF9OqVSsuvPBCrr766sr7vfnmmxx00EFb9Doqso0bN45x48ZtdPvcuXMtOiVJUq0SisZ7l3ZJkiRJlT777DN23nlnpk+fvkUzMissWbKEli1b8vrrr29x2SlJklSXuEenJEmSVM2uv/76rSo5IXb40d///nf22WefOKeSJEmqXZzRKUmSJEmSJCnpOaNTkiRJkiRJUtKz6JQkSZIkSZKU9Cw6JUmSJEmSJCW91KAD1GaRSISFCxeSk5NDKBQKOo4kSZIkSZKUVKLRKGvXrqVVq1aEw78+Z9OisxotXLiQtm3bBh1DkiRJkiRJSmrz58+nTZs2v3ofi85qlJOTA8T+Q+Tm5gacRjWhtLSUd999lwMPPJC0tLSg40hbzbGs2sBxrNrCsazawrGs2sKxrNoiWcbymjVraNu2bWXP9mssOqtRxXL13Nxci846orS0lOzsbHJzcxP6h4T0WxzLqg0cx6otHMuqLRzLqi0cy6otkm0sb862kB5GJEmSJEmSJCnpWXRKkiRJkiRJSnoWnZIkSZIkSZKSnnt0SpIkSZIkCYBIJEJJSUnQMVQDSktLSU1NpaioiPLy8kCzpKenEw7//vmYFp2SJEmSJEmipKSEuXPnEolEgo6iGhCNRmnRogXz58/frIN+qlM4HKZjx46kp6f/ruex6JQkSZIkSarjotEoixYtIiUlhbZt28Zldp0SWyQSYd26ddSvXz/Q/96RSISFCxeyaNEi2rVr97tKV4tOSZIkSZKkOq6srIyCggJatWpFdnZ20HFUAyq2KcjMzAy82G7atCkLFy6krKyMtLS0rX4e63lJkiRJkqQ6rmKPxt+7dFjaGhXj7vfuFWrRKUmSJEmSJIDA92pU3RSvcWfRKUmSJEmSJCnpWXRKkiRJkiRJCeCUU05h8ODBQcdIWhadkiRJkiRJSloLFizgxBNPpHHjxmRlZdG7d28mTpy4yfueffbZhEIhbr311t983gkTJrDffvvRoEEDGjZsyMCBA/niiy/inF7xZNEpSZIkSZKkpLRy5UoGDBhAWloab731FjNmzOCmm26iYcOGG933pZde4tNPP6VVq1a/+bzr1q3jD3/4A+3atWP8+PF88skn5OTkMHDgQEpLS6vjpSgOLDolSZIkSZKUlP71r3/Rtm1bHn74Yfr370/Hjh058MAD6dy5c5X7LViwgD//+c88+eSTpKWl/ebzzpw5kxUrVnDNNdfQtWtXevbsyZVXXsmSJUv47rvvfvFxX3zxBfvssw85OTnk5ubSt2/fytmlV111FTvssEOV+99666106NBho+e5+uqradq0Kbm5uZx99tmUlJRU3vb888/Tu3dvsrKyaNy4Mfvvvz/5+fnAT0vff+3xb7/9NrvvvjuNGjWiU6dOHHroocyePbvK9//hhx847rjjaNSoEfXq1WOnnXZi/Pjxlbe/8sor9OnTh8zMTDp16sTVV19NWVnZb76v1S016ACSJEmSJElKLNFolMLS8kC+d1Zaymafwv3qq68ycOBAjjrqKD766CNat27NsGHDGDp0aOV9IpEIJ510EhdddBE9e/bcrOft2rUrjRs35sEHH+Rvf/sb5eXlPPjgg3Tv3n2TxWSFE044gR133JGRI0eSkpLClClTNqtY3dCoUaPIzMzkww8/ZN68eZx66qk0btyY6667jkWLFnHcccdx4403cvjhh7N27Vo+/vhjotHoZj0eID8/nwsuuIBevXqxZMmSyueaMmUK4XCYdevWsddee9G6dWteffVVWrRoweTJk4lEIgB8/PHHnHzyydx+++3ssccezJ49mzPPPBOAK6+8cotea7xZdEqSJEmSJKmKwtJyevz9nUC+94xrBpKdvnmV1Zw5cxg5ciQXXHABf/vb35gwYQLnnnsu6enpDBkyBIjN+kxNTeXcc8/d7Aw5OTl8+OGHDB48mGuvvRaALl268M4775Ca+svZvv/+ey666CK6detW+ZgtlZ6ezkMPPUR2djY9e/bkmmuu4aKLLuLaa69l0aJFlJWVccQRR9C+fXsAevfuvdmPD4fD/OlPfwJiBXCzZs148MEHad68OTNmzKBXr1489dRT/Pjjj0yYMIFGjRoBsM0221Q+/9VXX82ll15a+f526tSJa6+9losvvjjwotOl65IkSZIkSUpKkUiEPn36cP3117Pjjjty5plnMnToUO655x4AJk2axG233cYjjzzyi7NEBw0aRP369alfv37ljM/CwkJOP/10BgwYwKeffsqYMWPo1asXBx98MIWFhQCVj6lfvz5nn302ABdccAFnnHEG+++/PzfccMNGS8I3x/bbb092dnbl5V133ZV169Yxf/58tt9+e/bbbz969+7NUUcdxf3338/KlSs3+/EA33zzDccddxzbbLMN7dq1o1OnTkCspAWYMmUKO+64Y2XJ+XNffPEF11xzTZXXP3ToUBYtWkRBQcEWv954ckanJEmSJEmSqshKS2HGNQMD+96bq2XLlvTo0aPKdd27d+eFF14AYsusly5dSrt27SpvLy8v5//+7/+49dZbmTdvHg888EBleVmxzPypp55i3rx5jBs3jnA4XHldw4YNeeWVVzj22GOZMmVK5XPm5uYCsX04jz/+eN544w3eeustrrzySp555hkOP/xwwuFwlSXmwBYfbJSSksJ7773H2LFjeffdd7njjju47LLLGD9+PB07dtys5zj00ENp37499957L7m5uWRnZ7PddttV7uOZlZX1q49ft24dV199NUccccRGt2VmZm7R64k3i05JkiRJkiRVEQqFNnv5eJAGDBjArFmzqlz39ddfVy7rPumkk9h///2r3D5w4EBOOukkTj31VABat2690fMWFBQQDoerzAKtuFyxV+WGy7k3tO2227Ltttty/vnnc9xxx/Hwww9z+OGH07RpUxYvXkw0Gq183g3L0gpffPEFhYWFlYXjp59+Sv369Wnbti0Q+28zYMAABgwYwN///nfat2/PSy+9xAUXXPCbj1++fDmzZs3i/vvvZ8CAAaxZs4apU6dW+f7bbbcdDzzwACtWrNjkrM4+ffowa9asX3z9QXLpuiRJkiRJkpLS+eefz6effsr111/Pt99+y1NPPcV9993H8OHDAWjcuDG9evWq8pGWlkaLFi3o2rXrLz7vAQccwMqVKxk+fDhfffUV06dP59RTTyU1NZV99tlnk48pLCxkxIgRfPjhh3z33XeMGTOGCRMm0L17dwD23ntvfvzxR2688UZmz57NXXfdxVtvvbXR85SUlHD66aczY8YM3nzzTa688kpGjBhBOBxm/PjxXH/99UycOJHvv/+eF198kR9//LHye/zW4xs2bEjjxo257777+Pbbbxk9ejQXXnhhle9/3HHH0aJFCwYPHsyYMWOYM2cOL7zwAuPGjQPg73//O4899hhXX30106dP56uvvuKZZ57h8ssv37L/eNXAolO12rzV8/hh7Q+sLFpJcXnxRlPEJUmSJElS8urXrx8vvfQSTz/9NL169eLaa6/l1ltv5YQTTvhdz9utWzdee+01pk6dyq677soee+zBwoULefvtt2nZsuUmH5OSksLy5cs5+eST2XbbbTn66KMZNGgQV199NRBbUn/33Xdz1113sf322/PZZ59tVDIC7LfffnTp0oU999yTY445hsMOO4yrrroKiC2RHz16NAcddBDbbrstl19+OTfddBODBg3arMeHw2GeeeYZJk2axHbbbcff/vY3/vWvf1X5/unp6bz77rs0a9aMgw46iN69e3PDDTeQkhLbUmDgwIG8/vrrvPvuu/Tr149ddtmFW265pXIWbZBCUZufarNmzRry8vJYvXp15V4Nqln7/Xc/lhYsrbycGkolOy2b7LRs6qXWo15aPbLSsqiXWi92XVrsc3bq+q9TN75uw8tZqVlVprGXlpby5ptvctBBB1Xu6yElI8eyagPHsWoLx7JqC8eyaovaOpaLioqYO3cuHTt2DHyfRW29U045hVWrVvHyyy//5n0jkQhr1qwhNze3ch/SoPza+NuSfi3xN1uQfofMlEyyUrMoLIttKlwWLWNNyRrWlKyJy/OHCFWWptlpseKzcG0h73z4DvUz6lctR1OzNypZKy5vWKymhDd/02VJkiRJkiTFWHSqVnvjiDcAKI+UU1hWSH5pPvll+RSWrv+6NJ+CsgLyS/N/un3D60oLyS9bf11pAQWlBeSXxb6Orv+/isdQ+NP3nbdw3lZnzkzJ/Gl26foCtGLW6YazSSvus2GJWmUW6vrPaSm15y+MkiRJkiRJv8SiU3VCSjiF+un1qZ9ePy7PF41GKSwrpKBsffm5vhxdU7iGMRPG0LV3V4oiRVWK0Q3vt+HXFSVqWbQMgKLyIorKi1hRtCIuWdPCaVVmnW749S+WpBvOMv3ZYzNTMqss15ckSZIkSYnhkUceCTpCoCw6pa0QCoUqiz+yfrq+tLSUNV+s4aDOW75XS0l5yUbl54ZF6a+VpBVfbzgztbi8OJYpUsrq4tWsLl4dl9ceDoWpl7p+lukvlaMbXvcrS/XrpdUjKzWLcMhz0SRJkiRJ0u9j0SkliPSUdNJT0mlIw7g8X1mkrLIY/Xk5umEhuqlidcMCdcPHAkSiEdaWrmVt6dq45ATISs3aqPysnE36s1mnG84u/aU9UNPCLteXJEmSJKmusehUwvrxjjtZ8cgjhLOzCWVnEc6uRzgri3B29vrPWYSyswlnZf90Xb3Y51DW+vtn/3T/yuuyMgml1P4Df1LDqeSm55Kb/usnkm2uSDRCUVnRLxamFeVolT1QN1i2v+FjKgrU8mg5AIVlhRSWFbK8aHlcsqaH0zeaUVpRlFaUqFXK0Q0L058v20+rR3o43eX6kiRJkiQlOItOJazIunVE8vOJ5OfH/blDmZk/labZWYQ2LEvXXxfOzo6VoxW3/cp1FY+jFpdh4VD4p+X6cRCNRikuL97kEvzNXapf5fbSAkoiJQCUREooKS5hZfHKuGRNDaVuvFR/c/Y2/YX7ZaVmWZxKkiRJkhRnFp1KWE2GnUPD444lUlBApLCQSEHh+q8LiBQUEP2l6/LX37/wp9ui668jGgUgWlREeVER5SvjU4RVSk2lc2oqc2+6mZTs7A1mo66febphubrhdfV+uq3KbNSK6zIzCYVr1z6WoVCIzNRMMlMzaZTZKC7PWRop3eRS/Z+Xo5tbrBaWFQJQFi1jbcla1pbEZ7l+iFCVmaVVlupveGjUL5SoP1+qn52aTWrYH+eSJEmSpLrNfxkrYaXk5ZGSlxe354tGo0SLijYoTfPXl6UFm76uYIPbKorUDa9b/zlaUEC0tDT2TcrKSCkri5WocUseE6qYObrh8v162VVno1Zet0G5WjHzNGuDcrXeT/cPbeGhSYksLZxGXkYeeRnxGTflkXIKywp/uRzdYGn+5izVLygrIBKNECUau66sgB8Lf4xL1syUzMrSs7Ik/aUDon6hMN1wD9QQzjiVJEmSJCUXi07VGaFQaP2y8yyIzwTCStHSUiKFhZSsXs0H77zDHv36kVJS8lMhusHM0yqzTivK0sKCja6rmKFa+T0KCigvKIh/gZqWFitAN1y6n5VVdSbqBvuibjTr9Od7oFbcJyMj6Zdnp4RTqJ9en/rp9ePyfNFolKLy2D6nhaWFVQ5+qrK36QYzT6vsgbqJpfxlkTIAisqLKCovYgUr4pI1NZxKWjSNO1++k/rp9Tdagr/h/qcb7oe6qRK1Xlo9MlMyk348SJIkSVIi2Hvvvdlhhx249dZbg46ScCw6pTgIpaWRkpZGalYWpU2bktmjB2lxmCkZjURis1A3nEVasMEy/Sqlaf7PlvP/bFl/QWGV56A8VplGS0uJrl5NZPXq3523inD4p8J0E3ugbnLW6a/sgRraYCZqsh4mFQrFlqxnpWZBVnyes7S8tMps0s3a23QTs1ErHlNUXgRAWaSMMsooLCiEgt+fMxwKx4rPn804rZdar3L/0w0Pjfqt2ahZqVmkhJNzHEiSJEmKr9GjR/Pvf/+bSZMmsWjRIl566SUGDx4MQGlpKZdffjlvvvkmc+bMIS8vj/33358bbriBVq1aVT7H119/zUUXXcSYMWMoKSlhu+2249prr2Wfffb51e/9zjvvcOWVVzJ9+nQyMzPZc889uemmm+jQoUM1vmL9EotObbXS8gghIDWldu0dmUhC4XDlbMt4ikajsYKzojj92X6n0V/bF/Vne6D+fDZqtLg49k0ikdhBUvn58Z+FmpHx0/6lmyhSf2lf1F/cAzUri3C9erHZrUk26zAtJY0GKQ1oQIO4PF9ZpIzCskJWF67mrVFv0W+3fhRHizcuRzcsVn/pgKj1X0eJEolGWFe6jnWl66Dwt3NsjqzUrE3ONP3VA6I23AP1Z8VqWrj2bOMgSZIk1SX5+flsv/32nHbaaRxxxBFVbisoKGDy5MlcccUVbL/99qxcuZK//OUvHHbYYUycOLHyfocccghdunThf//7H1lZWdx6660ccsghzJ49mxYtWmzy+86dO5c//vGPXHDBBTz55JOsXr2a888/nyOOOILJkydX62vWpll0aqu9/PkCLnp+KjkZqeRmpdEgO428DT7nZqXRICudvKyq1+dlpZGXnUZORmrSlUq1RSgUIpSeDunppDRoENfnjpaXEyks2uI9UKMbXbf+/htcRyQS+x7FxZQXF1O+alVcs5OSUnX5fr2fLd3f5B6oGxaoG++BWnFbshwmlRpOJSc9h8xQJs1SmtGzcc/fNTs5Eo1QVFb0mwc//bwwrbK36fpiteI+5dFYdV5YVkhhWSHLi5bH5bWnhdM2PiDqlwrTnxWrVZb0r79fRkryb98gSZIkJYNBgwYxaNCgTd6Wl5fHe++9V+W6O++8k/79+/P999/Trl07li1bxjfffMODDz7IdtttB8ANN9zA3XffzZdffvmLReekSZMoLy/nH//4B+H1/+a78MIL+eMf/0hpaekv/lvqww8/5OKLL2b69OmkpaXRs2dPnnrqKdq3b88pp5zCqlWrePnllyvvf9555zFlyhQ+/PDDyuvKysoYMWIEjz/+OGlpaZxzzjlcc801lf8Gufvuu7nllluYP38+eXl57LHHHjz//PNAbOl7r169iEajPP7446Snp2/0+Mcff5zbbruNWbNmUa9ePfbdd19uvfVWmjVrVplh+vTpXHLJJYwePZpoNMoOO+zAI488QufOnQF44IEHuOmmm5g7dy4dOnTg3HPPZdiwYZt8T+LFolNbbXVh7ACetcVlrC0uY8GqLZumFQ7xU/GZlUZedqwUbZBVtRD9eVHaICudzLSwBUKCCqWkkFK/Hin168X1eaPRKNHi4soDoKou5//ZHqibsS9qbDn/+j1SS0pi36S8nMi6dUTWrYtrdqByf9jf2gM19PPrKpbrb2Jf1HBWVqywTmDhULhyBmWTrCa/+/mi0SglkZKfDoDaxD6mG123vkDd1KFR+aX5lERi//1LI6WsKl7FquJVvzsnQEooZZP7mf58qf6vHhC1wR6oWalZhP+fvfsOj6O8+j7+ne1FvUvuvcuWreICxjXuhBZCyYMJnZgAIRVCCAl5E55QQw0t9PaQhAC2cWxs07GKmyz3XtW7drV93j9GWmtVbK0tWy7nk2suy6udnXuFImt/e+5zlLMjMBdCCCGEEOcAVQVvF/SxOhFGG5zC1/y1tbUoikJMU+FPfHw8Q4YM4Y033mDs2LGYzWZeeOEFkpKSGDduXIePM27cOHQ6Ha+++irXX389DQ0NvPnmm8yYMaPDkNPn83HJJZdw88038+677+LxeMjLyws743j99de58cYbycvLo6CggFtuuYXevXtz8803U1BQwJ133smbb77JxIkTqaqq4quvvmpz/g033MDKlSvZtm0bt912W/B80Lb8P/TQQwwZMoSysjLuuecerr/+epYuXQrA4cOHmTx5MlOmTGHVqlVERUXxzTff4PNpcyTefvttHnjgAZ555hkyMjJYv349N998M3a7nYULF4b1XMMhQac4YddP7MulGT2obfRS2+ilptFLXaOXGqf36G3Bjz0ht7l9AQIqVDu9VDu9YV/bpNeFVJE2B6RtbmtZRdpUXWoySFBwNlIUBcViQWexQGxslz626vOdVA/UYDVqO0Fq8BqNjfgbG/FXdc2woCCjMSRADQ1S7e1v3T9OD1SdzYZ6hvZBVRQFs96MWW8m1tI13wfegBen19luYNruVv12tue3DlkB/Kqfem899d76LlkncNyt+s2VqG0+385WfZvBhkEnvwYIIYQQQogOeJ3w57Tj3+9UuO8ImLq2eKaZy+Xi17/+NVdffTVRUVGA9jrjs88+45JLLiEyMhKdTkdSUhLLli0j9hivP/v168fy5cu58sorufXWW/H7/UyYMCEYBranrq6O2tpa5s+fH6x8HDZsWNjPo1evXjzxxBMoisKQIUPYtGkTTzzxBDfffDMHDhzAbrczf/58IiMj6dOnDxkZGW3Of/zxx6mvr2fcuHFs3rw5eD7ADTfcELxv//79eeqpp8jKyqKhoYGIiAieffZZoqOjee+994Kh7uDBg4Pn/P73v+exxx4LthLo168fW7Zs4YUXXpCgU5yZDHod8RFm4iPMYZ/r8vq1ULRNIOql1ukJCU+1245+3hdQ8fgDVDS4qWhwh31tm0kfWknaOhS1mULC0+aPo6xG9DqpIj0XKQYD+shI9JGRXfq4qqqGDpNyNPU07XQP1NCt+yHDpJreJcPrJeD1Eqir69K1oygMNBrZ+9dHmkJR2zH7ooZUo7aoRG2vL6piOLP+6THqjESbo4k2R3fJ4wXUQDA07WjwU3u9TVsHqw6vg0ZvIw6fg4CqtW1w+rRzKhorumStZr25c71NO9kD1aQzSbW9EEIIIYQ4Y3m9Xq688kpUVeX5558P3q6qKosWLSIpKYmvvvoKq9XKyy+/zIIFC8jPzyc1NZURI0awf/9+AC688EI+/fRTSkpKuPnmm1m4cCFXX3019fX1PPDAA1xxxRWsWLGCgwcPMnz48OB17rvvPu677z6uv/56Zs2axcyZM5kxYwZXXnklqampYT2X8ePHh/zuPWHCBB577DH8fj8zZ86kT58+9O/fn9mzZzN79mwuvfRSbC3mfxzrfL1ez9q1a3nwwQfZuHEj1dXVBJpayR04cIDhw4ezYcMGLrzwwnYrVx0OB7t37+bGG28MBqegVbNGR3fN666OnFmvNsV5w2LUYzHqSYqyhHWeqqo4PP5g+FnT6GlbRdr0Z8vba5we6t0+VBWcHj9Oj5/iWlfY6460GFpVkZpCqkgjTDp2VyrE7qkkLsJ69HbpR3peUhQluGW9q6keT6sK0nB7oDaFq60qVFVX0/8vVBWdx4O/shJ/Zdf0wGymmEyhPVBbVqTaWwSjTdv52/RAba5GDQav2m2K6cwI2XSKLlhV2RVUVcXtdx87HG3Z2/QYW/Wb7+cNaJX0br8bt99Ntbu6S9ZqUAztb9Vvp5q0o8C05d+tBusZ8d9UCCGEEOK8ZLRplZXdde0u1hxy7t+/P7jVutmqVatYvHgx1dXVwdufe+45VqxYweuvv85vfvMbli5diter/R5tbXqN11zV+Ne//jX4WG+99Ra9evUiNzeXzMxMNmzYEPxcXFwcAK+++ip33nkny5Yt4/333+f+++9nxYoVjB8/Hp1Oh6qqbdYejsjISNatW8fnn3/O8uXLeeCBB3jwwQfJz88Pbtc/FofDwaxZs5g1axZvv/02iYmJHDhwgFmzZuFpav1mPcbr3IamdnAvvfQSOTk5IZ/Tn+LdgxJ0irOKoihEmA1EmA30iAkvPPIHVBpcPmpabaMPVpK2CE+1v/uC1aUOjzb8pN7lo97l4+Axx0breXXH2tBbdMqxq0hDbgsd4GQxnplbiEX3Ukwm9CYT+i5+N0wNBFAbG3HX1bH600+ZnJ2NzuM9qR6owWFSfu3/R6rHg9/jgdraLl07Ol1o1WnLrfsd9UC1tVN1amvVF9VqQenGrfyKomAxWLAYLMRb47vkMb1+b5vBTy0rSFsOjmq9pb+9rfrN2/V9qo86Tx11nq6pMFZQQrbdt64gPV5vU5NiotxfTqmzlGhrNDaDDb1OfqYKIYQQQnSKopyy7eOnW3PIuXPnTlavXk18fOjv1c6mtmO6VkNkdTpdsJKxT58+bR7X6XS2Oac5yAsEAhgMBgYOHNjumjIyMsjIyODee+9lwoQJvPPOO4wfP57ExESKiopC7rthw4Y2lZO5ubkhf1+zZg2DBg0KXt9gMDBjxgxmzJjB73//e2JiYli1alVwK/mxzt+2bRuVlZU8/PDD9OrVCyBkQj1Aeno6r7/+ertDl5KTk0lLS2PPnj1ce+217T7/U0WCTnHe0OsUbbiRLfwJ0h5fgDrX0XC0rvFopWht49HwtMbhYe+RMvSWCOpcPmoavXh8AfwBlSqHhyqHJ+xrmwy6kC30MTZtG31zRWm01RAMR1v3KDXqpR+pCI+i06HY7RhMJrzx8ZiHDDmpqevNVFXVqlCdLcLSYEDaYut+p3qgtqhGbWxEdTe1sAgECDgcBByOk15va4rFErpNv72t++1VnR6jL2pzFWp3MOqNROu7bru+P+A/GoL6mgLTYw2Iam/7fqt+qGrT/5rPOeb7S8fxt//8LfixRW9p29u0qeq0ZYDafJ8Ot+03/WnUn/z/P4QQQgghxMlpaGhg165dwb/v3buXDRs2EBcXR2pqKldccQXr1q1j8eLF+P1+SkpKAK3C0mQyMWHCBGJjY1m4cCEPPPAAVquVl156ib179zJv3rwOrztv3jyeeOIJ/vjHPwa3rt93333t9sRsubYXX3yRiy++mLS0NLZv387OnTu57rrrAJg2bRqPPPIIb7zxBhMmTOCtt96iqKiozeMdOHCAe+65h1tvvZV169bx9NNP89hjjwGwePFi9uzZw+TJk4mNjWXp0qUEAgGGDBkScv7Pf/5zrrnmGnbs2BFyfu/evTGZTDz99NPcdtttFBUV8dBDD4Vc/4477uDpp5/mqquu4t577yU6Opo1a9aQnZ3NkCFD+MMf/sCdd95JdHQ0s2fPxu12U1BQQHV1Nffcc09n/9OGTYJOITrBZNCREGEm4Tj9SL1eL0uXLmXu3EnBcMjl9bca0OQJrSJtr7K06fAHVDy+AGX1bsrqw+9Ham/uR2rTAtFgOGrruLI0xmoi0mJAJ/1IRRdSFAXFbEZnNp+aYVIuV5g9UDvqi9oUrjbdRtOWEdXlwu9y4a/umi3eQQZDm2FSJ9YDtVUfVYsFRXf63ujQ6/REmiKJNHVNn1tVVWn0NbZbQdo6HG2vB2pw277XQW1jLV68+FStr63L78Lld1Hl6prBYEadsU3VachW/c70QG1xrkVvke36QgghhBBhKigoYOrUqcG/NwdpCxcu5MEHH+Tjjz8GYMyYMSHnrV69milTppCQkMCyZcv47W9/y7Rp0/B6vYwYMYKPPvqI0aNHd3jdadOm8c477/DXv/6Vv/71r9hsNiZMmMCyZcs63Npts9nYtm0br7/+OpWVlaSmprJo0SJuvfVWAGbNmsXvfvc7fvWrX+Fyubjhhhu47rrr2LRpU8jjXHfddTQ2NpKdnY1er+euu+7illtuASAmJoZ///vfPPjgg7hcLgYNGsS7777LiBEj2pw/ffp0DAZDyPmJiYm89tpr3HfffTz11FOMHTuWRx99lIsvvjh4fnx8PKtWreKXv/wlF110EXq9njFjxjBp0iQAbrrpJmw2G4888gi//OUvsdvtjBo1irvvvrvDr2dXUNTWG/9Fl6mrqyM6Opra2tqQ3g/i3HU06Jx70lVwqqrS4Pa1rSJtFY5qQ51Ct+PXu3wndW1FgUjz0UrR0CrSttWlLcNTu0kvL9LPAV35vXy2Cg6Taq8Haif6orbXAzXQ2IjqdKKG2WPnRCjNwWebHqgtQtGQvqjN1agttu8332Y/en/lLPp+aP4+njNnDujpcPBTRyFp899bbvNvrkx1+8N/86kzdIoOu6GpyrSjcLTlbS226rfe3m832rEarOgUqe4/28nPZHGukO9lca44V7+XXS4Xe/fupV+/flgs4c3TEGeXKVOmMGbMGB5//HHq6uqIiopqswX/dDvW9184+ZpUdApxhlIUhUiLkUiLkZ5hFsD5A2owGA2ZXt9iqn1HVaROjx9VhTqXj7oTCEwNLfuR2tqGo9GtepA2fz7KKv1IxZklZJhUXNc+tur1Hg0/nS0GRnW6B6ojuHU/pC9q49H93arTid/pxN+1S0cxGoMDoFoGqYqt1db9zvZAbb6P2XzK3iRRFAWj3ohJbyKWrqko9gV8R4PQVoOfOupt2hystrtt36f1hQqoAeq99dR767tknQBWg7VN+BmsJm1VddpeP9TWPVCNunPnBZ0QQgghhDi3SNApxDlIr1OItZuItYff/8/jCzSFnu0PbWquLm0Znjbf5vEH8AVUKh0eKk+gH6nZoGs1oMnUdmt9B9WlBulHKs4iitGI3mhE38XV/mogoFWhhoSmLbbph4SmrapRO+iB2nx+cJiU14taW0vgVAyTslhQ7K3D0hPrgapYbQRMRmhqHt/VDDoDUaYookxd898woAZw+VzH7m3aeqt+O9v2WwarflX7b9Y8MKrSVdklazXpTG0qSpuD0uYQNSQcbRmYtt62b7Rj0plkJ4AQQgghhOgSEnQKIUKYDDoSI80kRh67H2lrqqri8gaODmZqWTHqbKe6tFWv0oAKbl+A0jo3pXXhbwmNMBs6nmrfqgdpy4rTSLP0IxXnDkWnC1ZbdiVVVbWA09l+D1S1nds66oGqNjpDKlRDhkk5ndDFVaiDgd0P/uGYPVA76osaEqS27otqt2vVrV0U0OkUXTA4TCTxpB9PVVXcfnfHvU1bbdVv734tg1an14knoL2B5Ql48Lg9VLu7pmetQTG03arfmd6mHdzParBKcCqEEEIIcQyff/45QHCi/LlEgk4hRJdQFAWrSY/VZCU1uv2myx0JBFQaPL7QQDQkHPW0mHQfWkVa79a21ze4fTS4fRyuCW80s04hWCHaUVAaYzUdrSK1Hf3TapR+pOL8oCiKNiHeZEIfE9Olj636/QQaXWH3QFVb3xYMXY/e1lzNqbrd+N1u/DU1Xbp29PqQHqjBatQ2fVHb9kDtqC9q8+dOdpiUoihYDBYsBgtxlq7pveD1ezsVjrbX27T57y3PbfRpP699qo96Tz31nq7Zrq+ghFSWhmzVbzk0qoMQtfVWfZvBhkEnvzILIYQQQpwN5Lc2IUS30+kUoixGoixGeoV5rs8foM7VPLSp1UR7Z+sq0tDw1OUNEFChxqkFp+Ey6pV2wlFTx5WlLapLzQbpRyoEgKLXo4+wo4+wd+njqqqKp6GB5Z98wvRJk9B7PB33QO1UX9Sjlauqp6k1h99PoKGBQENDl64dCPaHPV4PVCWMvqg6q1ULrE+QUW8kWh9NtDm6S56jP+A/WjHaXjjaQQ/UjoZIOX1OAmoAFVW7zeekvLG8S9Zq0VuCoWcwJO0oHO0gMG3ZA9WkP/H/DkIIIYQQomMSdAohzmoGvY44u4k4uwkILyhx+/yhgWirLfZHq0g9bW7z+lW8fpWKBg8VDeH3I7Ua9W3Cz5Z9R0N6kbYIT6MsBulHKkQnKIqCzmIhYLdjTEvr0omoqs8XRg/U9qpOW/dFPRqkBq/R2Ii/sbHLh0lhNIZWnDZXo9qagtFO9EVt7oEasp3fYgm7wl2v0xNhiiDCFNElT01VVVx+rc9po7cxZPCTw9d0W6seqCFb+tupVvUFtF0DLr8Ll99FFVVdslaDznDMrfot+5/ajXbMOjPbPduJOhxFlDWqzTkWffhffyGEEEKIc5EEnUKI85bZoCcpUk9SpCWs81RVxenxt7PNvv0BTiFDm1xeVBUavX4avX5K6lxhrzvSbAgNR5s+jmrRgzR0qNPRfqTyQliIk6cYDOgjI9FHRnbp4waHSbUITdXG9vuitt8D1RGydb/ln/i0wA6vl4DXS6CurkvXjqI0Baa2NkFqRz1Qg9WoTVv8Q3qg2o7eXzF07tdVRdG2rFsNVgivg0qHvH5vSDVpe+Fouz1Q2xkU5fQ6cfm1n/m+gI9ady217vCGer3/xfvt3q5TdFrw2ari1G6wB/ufthwaFdLvtJ3t+1aDFb1Odh4IIYQQ4uwjQacQQoRJURTsZgN2s4G0mPD7kda7fW220bfsO9pRUNrQ1I+03u2j3u3jUHX4/UiPBp+mNlWkLatLI0wKRxxQXOsiMUqHxaiTkFSIU+xUDZMCUFtu3e+iHqgBpxPV1fRmjaqekmFSAIrJ1IkeqPZ2tu530AO16WusmI4/7d2oNxKjjyGGmC55Lr6AL2TAU4fhaItqVKfXSYOngYOlB7FEWYJVq83hqYpKQA3Q4G2gwdsA4f3T0CGrwdpupekxB0S17IHaVI3a/HejruuqqoUQQgghOiJBpxBCnEY63dG+nuHy+gNHg9CmP+taDWg6GoyGVpe6fVo/0mqnl2qnFyqdx78gBv638EsATHqdVjFqCw1Ho1pVjx79+GiQajLIVnshuptiMqE3mdBHd01/zWZqINBqiNTJ90ANDpPya5Gp6vHg93igNrzqx+PS6Vr1QLWH9EUNtwfq0cDVgqJvvxrSoDMQaYok0hReNbDX62Xp0qXMnTM3pA1DQA3g8rmOOfjpeD1QW1em+lXt697oa6TR10ilq/LEv8YtGHXGtgOiOhoG1aofans9UM16s7wBJ4QQQog2JOgUQoizhFGvIz7CTHyEOexzXV5/m230zf1HW4entY1eahweymsdNAZ0+AMqHn+AigY3FQ3usK9tM+lDt9G3GdDUtro0xmYk0mJEr5MXsUKcyRSdDsVuR2fv+mFSqscTGoAGA9IWW/c71QO16f5Nt6nupp9jgQABh4OAw9GlawdQLJajoandpvU0bb11v4MeqNogqtDb/EYjis+Hqqoh19EpumDFZII14aTXraoqnoDnaJWp1xGsQD3WVv1jbdv3BLQ+1t6Alxp3DTXumpNeJ4Be0bfbz7T1Vv1jDohqUXVqNVjRKfLGnBBCiLPDlClTGDNmDE8++WR3L+WMI0GnEEKcByxGPRajnuSozvUjba4emjPne3hUXchQppAq0nam3Dfft97tQ1XB6fHj9Pgprj2BfqQWQ2jFqNXUtoq0nerSCOlHKsRZTVEUFLMZndkMsbFd+tiqz0fA5QqzB2pHfVGbwtWm22gKIlWXC7/Lhb+6usvWPQjY/fsHO98DteWW/mP0QA0Ok9JpLUrMejNmvZk4S1yXrNsb8AaDz3Z7m7YIR1tXmrYMWpvPafRpe/P9qp96bz313vouWSdw3K36zZWobT7fYtt+y4pUg05eagkhxOny5Zdf8sgjj7B27VqKi4v58MMPueSSSwDttc3999/P0qVL2bNnD9HR0cyYMYOHH36YtLS04GPs2LGDX/7yl3zzzTd4PB7S09N56KGHmDp16jGvraoqjz32GC+++CL79+8nISGBn/zkJ/z2t789lU9ZdED+9RVCCNEhRVGIMBmIMBvoEWY/Un9Apd7Vtoo0tLI0dIt9c3Wpw6Ntnax3+ah3+TgYZtM5fYsWAR1Wkraaat/8OYtRBnAIcS5TDAb0ERHoI7pm2nszVVWPDpNq3QO1RTVqR31Rg1Wnrbbwq04nqterXcTnI1BfT6C+68K9ZsFBUm16oHbUF7VtD1SlnduMRiPR5miizV3TNiGgBkL6nLauIG359/a26rfsgdrobcThcxBQAwDafXxOKhorumStZr05GIimRaSRmZxJVkoW6YnpmPXh784QQgjRMYfDwejRo7nhhhu47LLLQj7ndDpZt24dv/vd7xg9ejTV1dXcddddXHzxxRQUFATvN3/+fAYNGsSqVauwWq08+eSTzJ8/n927d5OSktLhte+66y6WL1/Oo48+yqhRo6iqqqKqquqUPVdxbBJ0CiGEOCX0OoUYm4kYmynscz2+AHWuo+FoXfPQJqeX2kZfcIBT6FAn7U+PL4A/oFLl8FDl8IR9bZNBFzKgKcYWOtE+2moIhqOtq0uNetn2KMT5SlGUpm3nVuiaYsggj9PJfz/+mBmTJqHz+pqqS4/RF7VND1RHcOt+SF/UxqNvIqlOJ/5TMUzKaAwOgGoZpCq2Vlv3O+qBam17m9Vmw2ZNRLGdfOW+qqq4/K5jh6PH6oHqa1uZ6g1owbTb78btd1PtruZww2HyS/J5fuPzGHVG0hPTyUrJIjM5k9GJo7EYOrfjQgghRPvmzJnDnDlz2v1cdHQ0K1asCLntmWeeITs7mwMHDtC7d28qKirYuXMnr7zyCunp6QA8/PDDPPfccxQVFXUYdG7dupXnn3+eoqIihgwZAkC/fv2Ou97PP/+cX/3qV2zevBmj0ciIESN455136NOnD9dffz01NTX85z//Cd7/7rvvZsOGDXz++efB23w+H3fccQdvvvkmRqOR22+/nT/+8Y/BnW3PPfccTzzxBAcPHiQ6OpoLL7yQf/7zn4C29X3kyJGoqsqbb76JyWRqc/6bb77J3/72N7Zv347dbmfatGk8+eSTJCUlBdewefNmfv3rX/Pll1+iqipjxozhtddeY8CAAQC8/PLLPPbYY+zdu5e+ffty55138pOf/OS4X5+TIUGnEEKIM47JoCMhwkzCCfYjbdmDtHUVaW2rrfd1LW73B1Q8vgBl9W7K6sPvR2pv7kdqMxFtNQS320fbOq4sjbGaiLQY0Ek/UiFEBxSjkYDViiElJWQY0clSAwGtCjVkmFSLbfohoakjtBq1gx6ozecHh0l5vai1tQS6epiUomhVpPZWYWm7VafH7ouqt1qJstqIsceii07rcJhUZ3n93pBw1OF1sKtmF/kl+RSUFFDWWMba0rWsLV0LaIOaRiWM0oLPFC34tBrC20UhhBCngqqqwZYhp5vVYD2lrahqa2tRFIWYmBgA4uPjGTJkCG+88QZjx47FbDbzwgsvkJSUxLhx4zp8nE8++YT+/fuzePFiZs+ejaqqzJgxg7/+9a/ExbX/zqfP5+OSSy7h5ptv5t1338Xj8ZCXlxf283399de58cYbycvLo6CggFtuuYXevXtz8803U1BQwJ133smbb77JxIkTqaqq4quvvmpz/g033MDKlSvZtm0bt912W/B80Lb8P/TQQwwZMoSysjLuuecerr/+epYuXQrA4cOHmTx5MlOmTGHVqlVERUXxzTff4PP5AHj77bd54IEHeOaZZ8jIyGD9+vXcfPPN2O12Fi5cGNZzDYcEnUIIIc4pFqOelGg9KdHhVceoqkqD29eqirS9rfeeNrfVu7R/zB0ePw6PnyNh9iNVFIiytK0ibT2gKVhF2iI8tZv00o9UCHFCFJ0uWG3ZlVRV1QJOZ/s9UNWQ7fzH7oGqNjpDKlSDw6RUVQtUT0UVqtnctBW/bdWpITERW1YmtpwcjMnJ7Z5v1BuJ1odu1x+TNIYrBl+BqqocrD9Ifkk++aX55JfkU+YsY13ZOtaVreOFwhcw6AykJ6QzLnkcWSlZjEkaI8GnEKJbNPoayXknp1uunXtNLjZj1/771MzlcvHrX/+aq6++mqioKEDbmfHZZ59xySWXEBkZiU6nIykpiWXLlhF7jH7he/bsYf/+/XzwwQe88cYb+P1+fvazn3HFFVewatWqds+pq6ujtraW+fPnBysfhw0bFvbz6NWrF0888QSKojBkyBA2bdrEE088wc0338yBAwew2+3Mnz+fyMhI+vTpQ0ZGRpvzH3/8cerr6xk3bhybN28Ong9www03BO/bv39/nnrqKbKysmhoaCAiIoJnn32W6Oho3nvvveAbsYMHDw6e8/vf/57HHnss2EqgX79+bNmyhRdeeEGCTiGEEOJUUxSFSIs27b1XmOf6A2q70+trnW1D0ZZVpDVOL41eP6pK8H7hMjT3I21nQFPLqfYxLe4T0xSYSj9SIcSpoCgKiskEJhP6pkqZrqL6/QQaXWH3QFVb3xYMXY/eRkDr1am63fjdbqipaXcNNR98AICpb19sOTnYx+dgy87GEB9/3PUrikLvqN70jurN5YMvR1VVDtUfCoae+SX5lDpLg8HnS5tewqAzMCphFJnJmWSmZDImccwpe/EvhBDnOq/Xy5VXXomqqjz//PPB21VVZdGiRSQlJfHVV19htVp5+eWXWbBgAfn5+aSmpjJixAj2798PwIUXXsinn35KIBDA7XbzxhtvBEO+V155hXHjxrF9+3asVivDhw8PXue+++7jvvvu4/rrr2fWrFnMnDmTGTNmcOWVV5KamhrWcxk/fnxIwcOECRN47LHH8Pv9zJw5kz59+tC/f39mz57N7NmzufTSS7G1eHPzWOfr9XrWrl3Lgw8+yMaNG6muribQ9O/kgQMHGD58OBs2bODCCy9sd7eJw+Fg9+7d3HjjjcHgFLRq1ujorunb3REJOoUQQoiTpNcpxNpNxNrD70fq9vmpa/QFK0VrnG2rSFtWl2rb8X3UNXrx+AP4AiqVDg+VJ9CP1GzQBStFmyfat9laH+xRGrr93iD9SIUQ3UDR69FH2NFH2Lv0cVVVRXW7gwOgQrfzHw1GPfv24czNw7VlC559+/Ds20fN++8DYB40EFvOeGw52dizsjoV8iqKQq+oXvSK6sVlgy7Tgs+GQxSUFASrPkscJawvW8/6svVa8KkYGJkwksyUTLKStYpPCT6FEKeC1WAl95rcbrt2V2sOOffv3x/cat1s1apVLF68mOrq6uDtzz33HCtWrOD111/nN7/5DUuXLsXbNCDQatXWl5qaisFgCKlkbK7OPHDgAFOnTmXDhg3BzzVvZ3/11Ve58847WbZsGe+//z73338/K1asYPz48eh0OlRVbbP2cERGRrJu3To+//xzli9fzgMPPMCDDz5Ifn5+cLv+sTgcDmbNmsWsWbN4++23SUxM5MCBA8yaNQuPxxPyNWhPQ0MDAC+99BI5OaFVwfqTbBFzPBJ0CiGEEN3IbNCTGKknMTK8fqSqqtLo9bfdWh8yoEkLRVtWlzZXlAZUcPsClNa5Ka0Lvx9phNnQ8UT7Vj1IWwamkWbpRyqEOPMoioJisaCzWOAYWxSb+evqcBYU4FizBmduHu7t23Hv3IV75y6q33oLFAXzsKHYs3Owjc/BlpmJPiKiU+voFdmLXpG9uHTQpaiqGhxkVFCqhZ/FjmI2lG9gQ/kGXt70MgbFwPCE4WQlZ5GVkkVGUoYEn0KILqEoyjnz86Q55Ny5cyerV68mvlUVvtPpBECnC30zX6fTBSsZ+/Tp0+ZxJ02ahM/nY/fu3cFt6Dt27Aje32AwMHDgwHbXlJGRQUZGBvfeey8TJkzgnXfeYfz48SQmJlJUVBRy3w0bNrSpnMzNDQ2h16xZw6BBg4JBosFgYMaMGcyYMYPf//73xMTEsGrVquBW8mOdv23bNiorK3n44Yfp1Uvb79ZyQj1Aeno6r7/+Ol6vt83akpOTSUtLY8+ePVx77bXtPv9TRYJO0X12r4LiQrAnQkTS0T9tCWAIvypKCCHOJ4qiYDMZsJkMpEaH9453IKDS4PGFhqLOluFoUxVpq8/XNXqpd2v9SBvcPhrcPg7XhNegXqcQrBw91oCm1hPtY2xGrEbpRyqEODPoo6KInDaNyGnTAPBVVeHMy8eZl4sjNw/P7t24t2zFvWUrVa+9Bno9lhEjsOfkYMvJwTY2o1O9URVFoWdkT3pG9uTSQZcCBIPP/JJ81pau5XDDYQrLCyksL+SVolfQK3pGxI/QKj6bgk+7sWsrYIUQ4kzT0NDArl27gn/fu3cvGzZsIC4ujtTUVK644grWrVvH4sWL8fv9lJSUAFqFpclkYsKECcTGxrJw4UIeeOABrFYrL730Env37mXevHkdXnfGjBmMHTuWG264gSeffJJAIMCiRYuYOXNmSJVnS3v37uXFF1/k4osvJi0tje3bt7Nz506uu+46AKZNm8YjjzzCG2+8wYQJE3jrrbcoKipq02PzwIED3HPPPdx6662sW7eOp59+msceewyAxYsXs2fPHiZPnkxsbCxLly4lEAgEJ8M3n//zn/+ca665hh07doSc37t3b0wmE08//TS33XYbRUVFPPTQQyHXv+OOO3j66ae56qqruPfee4mOjmbNmjVkZ2czZMgQ/vCHP3DnnXcSHR3N7NmzcbvdFBQUUF1dzT333NPZ/7Rhk6BTdJ9tSyD/5fY/Z4lpCj+TwJ7Q/scRiVo4apJf3IQQIhw6nUKUxUjUCfQj9fkD1Ll87U+1d7buURoanrq8AQIq1Di14DRcRr3SKhw1tQlMW4amdqNCnUerXO3CQdVCCNGGIS6OqNmziJo9CwBvWZkWfOauwZGbh/fAAVyFhbgKC6l86SUwGrGmp2PPycaWMx7rmNHozJ2r7O8R0YMeA3twycBLAC34LCgpCFZ8Hm44TGFFIYUVhfyj6B/oFT3D44cHt7pnJGUQYTp+dakQQpxNCgoKmDp1avDvzUHawoULefDBB/n4448BGDNmTMh5q1evZsqUKSQkJLBs2TJ++9vfMm3aNLxeLyNGjOCjjz5i9OjRHV5Xp9PxySef8NOf/pTJkydjt9uZM2dOMDBsj81mY9u2bbz++utUVlaSmprKokWLuPXWWwGYNWsWv/vd7/jVr36Fy+Xihhtu4LrrrmPTpk0hj3PdddfR2NhIdnY2er2eu+66i1tuuQWAmJgY/v3vf/Pggw/icrkYNGgQ7777LiNGjGhz/vTp0zEYDCHnJyYm8tprr3Hffffx1FNPMXbsWB599FEuvvji4Pnx8fGsWrWKX/7yl1x00UXo9XrGjBnDpEmTALjpppuw2Ww88sgj/PKXv8RutzNq1CjuvvvuDr82XUFRW2/8F12mrq6O6OhoamtrQ3o/iCYb34M9n0NDGTjKoKEcHOWghjk702hvCj2bq0KbPo5oCkaDHyeCJVobb3yKeL1eli5dyty5c9ttyCvE2UK+l8Wp4PL6g4OYWg9oOlpF2mKAU4vKUl/g5H5dsRr1bbbWx1jb9iJtGaA2D23Sy1Z70c3kZ/LZz3vkCI7cPJy5uThyc/EVF4d8XjGZsGZkaIONcnKwjhypDXQ6AUcajlBQWhDs83mo4VDI53WKjuFxw8lKySIzJZOMpAwiTZEn/NzCId/L4lxxrn4vu1wu9u7dS79+/bBYLN29HHEKTZkyhTFjxvD4449TV1dHVFRUm237p9uxvv/CydekolN0n9FXaUdLgQC4ao6Gn47ypgC0rOm2ihahaBn4XOB1QLUDqvcd/5p6kxZ4ttwuH/y4VcWoLQ50MpFYCCG6isWox2LUkxQV3i/Oqqri9PjbDGtqb4BTbUiQ6qGu0YuKQqPXT6PXT0mdK+x1R5oNoX1Hgx+b2t163xymRpoNstVeCAGAMS2NmEsvIebSS1BVFe/Bgzhyc3GuycWRl4u/vAJnbi7Opn5pitWKbdw4bbDR+PFYhg1DMXTupVtaRBoXR1zMxQO0qpsSR0lIj8+D9QcpqiyiqLKIVze/ik7RMSxumBZ8JmcyNnnsaQs+hRBCiK4mQac4s+h0WsBoiwOGHvu+qgqehqYAtPzon8GPm4LR5tvddeD3QN1h7TgeRQe2+Bbb5I9RMWpPlL6iQghxiiiKgt1swG42kBbT+X6kXq+XxUuWcuG0mTi9hGyjb9l3NKQXaYvKUodH22FQ7/ZR7/ZxqDq8fqR6nUKUpWloU4tt9jHtVZGGVJqasBh1EpIKcY5SFAVT796Yevcm9gc/QFVVPHv3BgcbOfPy8FdX4/j6axxff005oIuIwJaZiW18DvacHMxDhqB0svImxZ7CggELWDBgAaAFny0rPg/UH2Bz5WY2V27mtc2voVN0DI0bSlayVvE5NnksUSbZnSaEEOLsIEFnmJxOJ8OGDeMHP/gBjz76aHcv5/ymKGCO1I74Ace/v9fVFIS2qAjtqGLUWQVq4GhwWtaJ9VhiMNgTmOTWo//3vyAyueOKUekrKoQQp4VOgWirkYSo8LeVef0BLfRsr+9oO5WlLW93+wL4AyrVTi/VTi9UOsO6tkmvazW9Xvuz5ZCm0CrSo0GqydC9246EEOFRFAVz//6Y+/cn7pprUAMB3Dt3atvc1+TizM8nUF9Pw+ef0/D55wDoo6OxZWdjy8nBPj4H04ABnX5zJMWewvz+85nffz4ApY7SYLVnQWkB++v2s6VyC1sqt/D6ltdRUBgaNzTY43Ns8liizdGn6sshhBDiNPi86d+T5ony5xIJOsP0//7f/2P8+PHdvQxxIowWiOmlHcfj94GzIrRStE3FaPnRI+ADVw2Kq4YEgK3bj7MWe/uDldqrHrXEnNK+okIIIdpn1OuIjzATH9G5ASEtubz+NuFny+30NY3eNpWlzRWl/oCKxx+gvN5Neb077GvbTPrQbfStQ9EWPUhbfj7SIv1IhTgTKDodliFDsAwZQtx116H6/bi2bmsabJRLY8Fa/LW11K9YQf2KFQDoExKwZ2dhyxmPPScbY58+nQ4+k+3JzOs/j3n9tanCZc4yrdqzNJ+CkgL21e1ja9VWtlZt5c0tbwaDz3HJ48hKyWJc8jgJPoUQQpwxJOgMw86dO9m2bRsLFiygqKiou5cjTiW9ASJTtON4mvuKOsrx1R5h/dfLGTu4J/rGqvarR32NWl/RGgfU7O/EWpr7irYarNSyz2jzn7Z46SsqhBBngOZ+pMkn0I/U4fGHTrRvtbW+7W0eap1e6t0+VBWcHj9Oj5/i2hPoR2oxHK0YbaoSjWoVlAYD0ha3RUg/UiFOGUWvxzpyBNaRI4i/8UZUr5fGoiKcuXk4ctfQuG49/ooK6pZ+St3STwEwJCdrg42ytYpPY48enb5eki2Juf3nMrf/XADKneUhFZ97a/cGg8+3tr6FgsLg2MHB4UaZyZkSfAohhOg2Z0TQefjwYX7961/z6aef4nQ6GThwIK+++iqZmZld8vhffvkljzzyCGvXrqW4uJgPP/yQSy65pM39nn32WR555BFKSkoYPXo0Tz/9NNnZ2cHP/+IXv+CRRx7h22+/7ZJ1iXNEi76iakx/jmyuY0zWXPTtTd8L6Sta0WLLfHk71aMV4K498b6ix60YTQBD+FVKQgghTh1FUYgwG4gwG+gZG965/oBKvaudKtLGFhWjztABTs3Vpc7mfqQuH/UuHwcJvx/pMatIW020b/l5i1HeoBMiHIrRiC0jA1tGBgm33UrA48G1caO2zT03l8aNG/GVllL70cfUfvQxAMaePYODjWzZORiTkzp9vURbInP6zWFOvzkAVDRWBPt7FpQWsKd2D9urt7O9ensw+BwUO4islCyykrWKzxhLzKn4UgghhBBtdHvQWV1dzaRJk5g6dSqffvopiYmJ7Ny5k9jY9n+7/+abb8jOzsbYKkTasmUL8fHxJCcntznH4XAwevRobrjhBi677LJ2H/f999/nnnvu4e9//zs5OTk8+eSTzJo1i+3bt5OUlMRHH33E4MGDGTx4sASd4sSdTF/R4GCl5irRVv1GW/cV7QxLdNvBSu1VjNoTwRxxcs9dCCHEKaXXKcTYTMTYTPSJD+9cjy9AnatlQOoJVo6226O0xd89fq0faZXDQ5XDE/a6TQZdyICmYBVpsO+o4WhA2io8NeqlH6kQOpMJW1YWtqws+OkdBBobaVy/HkduHs41a2gsKsJ76BC1hw5R+69/A2Dq2zc42MiWnY0hvvM/NBKsCczuN5vZ/WYDTcFn03CjgpICdtfuZkf1DnZU7+DtrW8DaMFnchYZiRk4Ao6u/yIIIYQQTbo96Pzf//1fevXqxauvvhq8rV+/fu3eNxAIsGjRIgYNGsR7772HXq9VAGzfvp1p06Zxzz338Ktf/arNeXPmzGHOnDnHXMfjjz/OzTffzI9//GMA/v73v7NkyRL+8Y9/8Jvf/IY1a9bw3nvv8cEHH9DQ0IDX6yUqKooHHnjgRJ+6EMcXdl/RyraDlTrqLRrwgatWOyp3dmIttrbhZ+shS80Vo9JXVAghziomg46ECDMJYfYjVVUVlzcQ0oO0bRVpaDha1+J+AVULWcvq3ZSdQD9Se3M/UptJC0St7QeirbfjR1oM6KQfqThH6axW7BMnYp84EQB/g4PGdWuDFZ+uLVvw7NuHZ98+at57HwDzoEHBwUa2rCz00Z3fep5gTWB239nM7ns0+FxbulYLPksL2FWzi53VO9lZvZN3tr0DwAdLPiArNYvM5EwyUzKJs8R18VdBCCHE+arbg86PP/6YWbNm8YMf/IAvvviCHj168JOf/ISbb765zX11Oh1Lly5l8uTJXHfddbz55pvs3buXadOmcckll7QbcnaGx+Nh7dq13HvvvSHXmjFjBt999x0Af/nLX/jLX/4CwGuvvUZRUVGHIeezzz7Ls88+i9/vP6H1CHFC9AZt0ntk26rmNlr0FW0bhLZTPeprBK9T6ynamb6iOuPRYUotByu1Vz0qfUWFEOKspSgKVpMeq0lPSnT4/Ugb3L5WE+w7nmrf8rZ6lw8Ah8ePw+PnSJj9SBUFoiyhQWhUOwOaQqpLbdrnbSa99CMVZxV9hJ2IyZOJmDwZAH9tLc6CAhy5uTjX5OLesQP3zp24d+6k+q23QFEwDxuKPWc8tpxsbJmZ6CM6v7MnwZrArL6zmNV3FgBVrirWlq4lvySf/OJ8dtXuCh7vbnsXgIExA4OhZ2ZyJvHWMMvShRBCiCbdHnTu2bOH559/nnvuuYf77ruP/Px87rzzTkwmEwsXLmxz/7S0NFatWsWFF17INddcw3fffceMGTN4/vnnT3gNFRUV+P3+Ntvek5OT2bZtW9iPt2jRIhYtWkRdXR3RYbwbKsRp06KvKIlDjn1fVQWPo/3BSm2qR8u1vqIBL9Qf0Y7jUrSws/VgpXYrRhOlr6gQQpwjFEUh0qJNe+/EvoUQPn+AepcvpFK0eaJ9Rz1Km29r9PpRVYKfD5ehuR9puwOaTKG32Y5+HCX9SMUZQh8dTeT06UROnw6Ar6oKZ14+jtw1OHPz8OzZg3vLVtxbtlL16qug12MZOQJ7dg628TnYxo5FZ7V2+npxljhm9pnJzD4z8Xq9fLD4A+JGx7G+fD35pfnsrN7Jrppd7KrZxXvb3wNgQPQALfRsCj4TrAmn5GshhBBnKkVROpwvI46t24POQCBAZmYmf/7znwHIyMigqKiIv//97+0GnQC9e/fmzTff5KKLLqJ///688sorp/Wd9euvv/60XUuIbqcoWn9OcwTE9T/+/b0ucFa0GqzUQcWosxJQtfs7Kzq3Hkt0O4OVOqgYlb6iQghxTjLodcTaTcTaTWGf6/b5g1voQ7bWt1NZenQ7vo/aRg9ev4ovoFLp8FB5Av1ILUZdiyDURFTrrfUhVaRHg1Rbt//GLs5lhrg4ombPImq2VoHpLSvDmZuHMy8Xx5pcvAcP4tpYiGtjIZUvvQRGI9b0dK2/Z04O1jGj0Zk7/0a0XWdneq/pzO6vbXWvdlWzrnQd+aX55Jfks6N6B7trd7O7djfvb9e21veP7k9mcmZwsrsEn0KI1o41hNrr9XL//fezdOlS9uzZQ3R0NDNmzODhhx8mLS0t+Bg7duzgl7/8Jd988w0ej4f09HQeeughpk6desxrq6rKY489xosvvsj+/ftJSEjgJz/5Cb/97W9P5VPutM8//5ypU6dSXV1NTExMdy/nlOv2X5tSU1MZPnx4yG3Dhg3jX//6V4fnlJaWcsstt7BgwQLy8/P52c9+xtNPP33Ca0hISECv11NaWtrmOikpKSf8uEKcl4wWiO6pHccT7Cta3omK0XKtUjTYV3RXJ9Zia3+wUnvVo9ZY6SsqhBDnAbNBT1KknqTI8LfaN3r9bUPRYA/SpgFOjb7Q6tKmUDWggssbwOV1U1oXfj9Ss17PX7d+SXRTn9GQqfatepC23I4faZZ+pCI8xqQkohfMJ3rBfAC8R44EBxs58vLwFRfTuHYtjWvXwnPPoZjNWDMysOdka8HnqFEorQbHHkusJZbpfaYzvY9WYVrjqmFt2drgZPcd1TvYU7uHPbV7+L8d/wdAv+h+R4PP5EwSbYld/4UQQpxVjjWE2ul0sm7dOn73u98xevRoqqurueuuu7j44ospKCgI3m/+/PkMGjSIVatWYbVaefLJJ5k/fz67d+8+ZjZ01113sXz5ch599FFGjRpFVVUVVVVVp+y5dhdVVfH7/RgM3R4lHlO3r27SpEls37495LYdO3bQp0+fdu9fUVHB9OnTGTZsGB988AE7duxgypQpmM1mHn300RNag8lkYty4caxcuTKY+AcCAVauXMkdd9xxQo8phOiEcPqKqio0Vh97yFLL27zOE+sr2nqwUnvVo/YE6SsqhBDnGUVRsJkM2EwGUqM7v20XIBBQqXf7Otha7wkJTFt+vq7RS71b60fq9iscrnFxuCa8fqQ6haOVo03Voq2D0ubq0pDbbEasRulHKsCYlkbMpZcQc+klqKqK9+BBHGu0be6O3Fz8FRU416zBuWYNAIrNhm3sWG2wUU4OluHDUfSd/70pxhLD9N7Tmd5bCz5r3bXBHp9rS9eyrWobe2v3srd2Lx/s+ACAvlF9yUzJJCtZq/hMsiV1/RdCCHFGO9YQ6ujoaFasWBFy2zPPPEN2djYHDhygd+/eVFRUsHPnTl555RXS09MBePjhh3nuuecoKirqMOjcunUrzz//PEVFRQwZorWF62jAdmv/+Mc/eOyxx9i1axdxcXFcfvnlPPPMM23u115F5oYNG8jIyGDv3r307duX/fv3c8cdd/D111/j8Xjo27cvjzzyCMOHDw9WpMbGxgKwcOFCXnvtNQKBAA8//DAvvPACZWVlDB48mN/97ndcccUVIdddunQp999/P5s2bWL58uVMmTKlU8+vu3R70Pmzn/2MiRMn8uc//5krr7ySvLw8XnzxRV588cU29w0EAsyZM4c+ffrw/vvvYzAYGD58OCtWrGDatGn06NGDn/3sZ23Oa2hoYNeuo9Vfe/fuZcOGDcTFxdG7d28A7rnnHhYuXEhmZibZ2dk8+eSTOByO4BR2IUQ3U5QWfUUHH//+7ob2Byu1Vz3qOpm+oi0GK3VUMSp9RYUQ4ryma+7raT2xfqSV9Y18vOwzxmRPxOFV2x3aVONssR2/KTx1eQMEVKhxap/vxNt+IYz6o+vWwk9TyN/bBKa2o0Gq2SBvCJ6LFEXB1Ls3pt69ib3ySlRVxbNnT3CwkTMvD39NDY6vv8bx9dcA6CIisGVlYcvJxjxunDYUMwzR5mim9Z7GtN7TAC34bN7qXlBSwLaqbeyr28e+un38c8c/AegT1Sek4jPZ3ok31YUQbaiqitrY2C3XVqzWU/pmW21tLYqiBIPD+Ph4hgwZwhtvvMHYsWMxm8288MILJCUlMW7cuA4f55NPPqF///4sXryY2bNno6oqM2bM4K9//StxcXEdntc8q+bhhx9mzpw51NbW8s0335zw81m0aBEej4cvv/wSu93Oli1biIiIoFevXvzrX//i8ssvZ/v27URFRWFt6rP8l7/8hbfeeovHH3+c0aNH8/XXX/OjH/2IxMRELrroouBj/+Y3v+HRRx+lf//+wbD0TNbtQWdWVhYffvgh9957L3/84x/p168fTz75JNdee22b++p0Ov785z9z4YUXYjId7ck0evRoPvvsMxIT29+yUFBQENJT4Z577gGOptgAP/zhDykvL+eBBx6gpKSEMWPGsGzZsjYDioQQZ4lw+or63C0qQo9TMXoifUXN0S0qQo9VMZoIpgjZQi+EECLIoNcRZzeRZIUxvWIwhrEl2OX1h2yhP7rNvnnbvadFZenRKtIapxdfQMXrV6lo8FDREH4/UqtR3yb8bD3VPqpVeNpccaqXrfZnDUVRMA8YgHnAAOKuuQY1EMC9c6e2zT03D2d+PoH6ehpWr6Zh9WoABthsFK9cScSECdhzcjANGBBWmBFtjmZq76lM7a29vqvz1LGudJ221b00n21V29hft5/9dfv5106tHVrvyN7B/p6ZyZmk2KU9mRCdoTY2sn1sxyHfqTRk3VoUm+2UPLbL5eLXv/41V199NVFRUYD28+yzzz7jkksuITIyEp1OR1JSEsuWLTtmuLdnzx7279/PBx98wBtvvIHf7+dnP/sZV1xxBatWrerwvD/96U/8/Oc/56677grelpWVdcLP6cCBA1x++eWMGjUKgP79j74Obg5ck5KSgsGu2+3mz3/+M8uXL2fEiBFERUUxcOBAvv76a1544YWQoPOPf/wjM2fOPOG1nW7dHnSC1gdh/vz5nbpvR1/cjIyMDs+ZMmUKqqoe97HvuOMO2aouxPnIYO58X9GAXws7Ww9WarditKmvqLtWOzrTV9RgPf6QpeZKUekrKoQQ4hgsRj0Wo56kqPD7kTo9/pBwtLbR06aStLn/aMh2e5dX6zbj9dPo9VNSF95We4BIs+Fo39GWvUhb9SANqS61af1IZat991J0OixDhmAZMoS4hQtR/X5cW7Zqg41yc3HmF6B3OnF8thLHZysB0CckYM/W+nvac7Ix9ukT1n/HKFMUU3pNYUqvKYAWfK4vXU9Bqdbjc2vVVg7UH+BA/YFg8Nkrslew2jMrJUuCTyHOI16vlyubKtKff/754O2qqrJo0SKSkpL46quvsFqtvPzyy8HZMKmpqYwYMYL9+7X9ERdeeCGffvopgUAAt9vNG2+8weDB2s7DV155hXHjxrF9+3asVmvIXJr77ruPm266iSNHjjB9+vQue1533nknt99+O8uXL2fGjBlcfvnlwS347dm1axdOp5NZs2aF3O7xeNrka5mZmV22ztPhjAg6hRDirKHTa0FjRCd6P6kquGraGazUQcWo1wm+Rqg5oB3HXYvhGMOWWlWP2uK1nqhCCCHEcSiKgt1swG420CPmBPqRunytBjSF9h2tCRng5Gu6zYPD4weg3u2j3u3jUHV42yX1OoUoi6Ep+DQRYzWS3jOaeempDEmOlBC0Gyh6PdZRI7GOGkn8jTficTpZ/fLLjNHrcRUU0LhuPf6KCuqWLqVu6VIADCkpTYONxmvBZ48eYV0zyhTFRb0u4qJeWjVSvaee9WXrg8ONtlRt4WD9QQ7WH+TfO/8NQM+InsGKz6zkLFIjUrv2CyHEWUqxWhmybm23XburNYec+/fvZ9WqVcFqToBVq1axePFiqqurg7c/99xzrFixgtdff53f/OY3LF26FK/XCxDc/p2amorBYAiGnKAN2AatynLq1Kls2LAh+Lm4uLiwdmeAtrsZCCnga15Hs5tuuolZs2axZMkSli9fzl/+8hcee+wxfvrTn7b7mA0NDYC29T46OpqIiIjgdczm0NZrdrs9rPV2N3nVK4QQp4qiaFWX1tgw+oqWtx2s1F71qKsWAj6oL9aO4y9G62/aerBSRxWj6E722QshhDgP6XSKVo1pM9Kb8LYcev0BLfRsNdG+zaT7pgC1ZWWpxxfAH1Cpdnqpdnqh0gnAFzvKeXrVLgYmRTBvVCoLRqcyMCnyVDx10QmK0YirTx/i5s7FuGgRAY+Hxg0bcObm4czNxblxI76SEmo/+pjajz4GwNizJ7bxOdhzcrBl52BMDm/QUKQpksk9JzO552QAGjwNrC9bH+zxuaVyC4caDnFo1yE+3PUhAD0ieoRUfKZFpHXtF0KIs4SiKKds+/jp1hxy7ty5k9WrVxMfHx/yeadT+3ejOexrptPpCDT1Fm5vaPakSZPw+Xzs3r2bAQMGANqA7eb7GwwGBg4c2Oa8vn37snLlypA2ix1pbtNYXFwc3EbfMjxt1qtXL2677TZuu+027r33Xl566SV++tOfBls/+v3+4H2HDx+O2WzmwIEDfP/73ycqKqrNcz9bSdAphBBnimBf0U5M6fO5W1SHthis1F71qLMS1ID2p7MSyrce9+EN5iimY0Vf8dzRCtaOKkalr6gQQoguYNTriI8wEx8R/hA/l9ffJhAtq3exels5X+4oZ1dZA39buZO/rdzJkORI5qWnMj89lf6JEafgmYjO0plM2LOzsWdnw0/vINDYSOP69TjW5OLMzaWxqAjvoUPU/vMQtf/Utp2b+vXDlpONffx4bNnZGI4x7KM9EaYILux5IRf2vBAAh9ehBZ8lWvC5uXIzhxsOc3jXYf6z6z+AFnyOSx5HVkoWWSlZ9IgIr8pUCHHqHWsIdWpqKldccQXr1q1j8eLF+P1+SkpKAK3C0mQyMWHCBGJjY1m4cCEPPPAAVquVl156ib179zJv3rwOrztjxgzGjh3LDTfcwJNPPkkgEGDRokXMnDkzpMqztQcffJDbbruNpKQk5syZQ319Pd988027FZgDBw6kV69ePPjgg/y///f/2LFjB4899ljIfe6++27mzJnD4MGDqa6uZvXq1cHK0j5NLUEWL17M3LlzsVqtREZG8otf/IKf//znOJ1OZsyYEVxDVFQUCxcuDOvrfyaRoFMIIc5GBjNE99CO42nuK9p6sFJHFaMBL4q7jgjq4GBpJ9ZibX+wUpvq0SSwxMA58k6hEEKIM0dzP9LkVv1Ir83pQ53Ly4rNpSzZVMxXO8vZXlrP9hX1PL5iB8NSo5ifnsq8Uan0TTi7tuadi3RWK/aJE7FPnAiAv8FB49oCbbBRbi6uLVvw7N2LZ+9eat57HwDzoEHYxmvb3G1ZWeijo8O6pt1o54IeF3BBjwsALfjcULaB/JJ88kvz2VKxRQs+Gw7z8W6tyjTNnhYcbNQcfEprBCG617GGUD/44IN8/LH2/98xY8aEnLd69WqmTJlCQkICy5Yt47e//S3Tpk3D6/UyYsQIPvroI0aPHt3hdXU6HZ988gk//elPmTx5Mna7nTlz5rQJIltbuHAhLpeLJ554gl/84hckJCRwxRVXtHtfo9HIu+++y+233056ejpZWVn86U9/4gc/+EHwPn6/n0WLFnHo0CGioqKYPXs2TzzxBAA9evTgD3/4A7/5zW/48Y9/zHXXXcdrr73GQw89REJCAk888QR33XUXMTExjB07lvvuu++Yaz/TKWpnpvSIE1JXV0d0dDS1tbUhvR/Eucvr9bJ06VLmzp0bdt8NIc4ITX1FvTXF5K76hPGjBmBorGpbMdr8sdcR3uMH+4q2GqzUss9o85+2BOkrKk6K/EwW5wr5Xu46tU4v/91SwpLCYr7ZVYEvcPSl0MgeUcwblcb89FR6xZ0bWzXPNCf7veyvrcVZUBCs+HQ3bQ8NUhQsw4Zpg43G52Adl4k+4uQCbKfXqQWfTVvdiyqK8Km+kPuk2lODoWdmSiY9I3pK8HmOO1d/LrtcLvbu3Uu/fv2wWMIbZCfOToFAgLq6ujNi6/qxvv/CydfkFaQQQoijmvuKGiKojBiKOmwuHOuXN4/j+EOWmv901Zx4X9GWg5U6qhg1yi9jQgghji3aZuTKzF5cmdmLaoeH/24uYcmmYr7dXUnR4TqKDtfxv8u2MbppiNG89LSwBzKJU0cfHU3k9OlENk0q9lVV4czL0ya65+bh2bMH15YtuLZsoerVV0GvxzJyBPac8dhysrGNHYsuzAEnNqONiT0mMrGHVmXq9DrZUL6BgpICCkoL2FSxiWJHMZ/s+YRP9nwCQLItObjNPSs5i56REnwKIcTpIkGnEEKIE2eyaz1FO9VX1NMUgrYarNRQfvT25orRNn1FO7EWc1SLitDjVIxKX1EhhDjvxdpNXJXdm6uye1PZ4GbZZq3Sc82eSjYeqmXjoVr+vHQbGb1jmDcqlXnpqaRGS+h5JjHExRE1ezZRs2cD4C0tawo+1+DMzcN78CCujYW4NhZS+eKLYDRiHZ2OPTsH2/gcrGPGoGsa0tFZNqONiWkTmZh2NPjcWL6R/JJ81paupbCikFJnKYv3LGbxnsUAJNmSgqFnVkoWvSJ7SfAphBCniASdQgghTg+DKcy+olVtByt1VDEa8IK7TjuqdndiLZZW1aHtDFlqDkqlr6gQQpzz4iPMXJvTh2tz+lBW7+K/RSUsLiwmb18V6w/UsP5ADX9aspXMPrHMS09l7qjUNv1ARfczJicRvWA+0QvmA+A9fDjY39ORm4uvpITGgrU0FqyF555DMZuxZmRgH69NdLeOGokS5jZkm9HGhLQJTEibAECjr5GN5RspKCkgvySfwopCypxlLNmzhCV7lgCQZE0iMyUzONm9T1QfCT6FEKKLSNAphBDizKPTayFkRCIkjzj2fVUVXLXtD1Zqr2LU6wCfC2oPaMdx12LQ+oWGDFZKbFEx2qJ6VPqKCiHEWS8p0sL/TOjL/0zoS2mdi083FbNkUzH5+6op2K8df1y8hay+ccxPT2XOyFQSI8OfFC9OPWOPHsRcdikxl12Kqqp4DxwIbnN35Obir6jAuWYNzjVrAFBsNmzjxmmDjXLGYxk+DEWvD+uaVoOV8anjGZ86HtCCz8LyQgpKm4LP8kLKGstYuncpS/cuBSDRmhgSfPaN6ivBpxBCnCB5NSaEEOLspihgjdGOhEHHv7/H0f5gpfaqR5v7ijaUaEdnWOPaDlYKqRhtEZRKX1EhhDijJUdZuH5SP66f1I/i2kaWbiphSeER1h2oIW9vFXl7q3jw483k9ItnXnoqc0amEB8hoeeZSFEUTH36YOrTh9grr0RVVTx79uBYo21zd+bl4a+pwfHVVzi++goAXWQktsxMreIzJwfz4MEoYe7ysBqs5KTmkJOaA4DL52oTfJY3lvPp3k/5dO+nACRYE8hK1gYbZaZk0i+qnwSf4rSSmdWiO3TV950EnUIIIc4vJrt2xPY9/n19HnBWtB2s1F71qLNC6yvaWKUd5duO//jmqBYVoR1VjDYFpeZI6SsqhBDdKDXayo0X9OPGC/pxuKaRpYXFLN5UzMaDNXy3p5Lv9lTy+483M6G/FnrOHpFCrD28/o/i9FEUBfOAAZgHDCDu2mtRAwHcO3Y0bXPPw5mfT6C+nobVq2lYvRoAfUwMtuxsbDnZ2MePx9S/f9gBpMVgITs1m+zUbADcfrcWfJYUkF+az8ayjVQ0VvDpvk/5dJ8WfMZb4oPVnlkpWfSLluBTnBr6pgpmj8eDNczBXUKcLI/HAxz9PjxREnQKIYQQHTGYICpNO44n2Fe01WCljipG/Z4WfUX3dGItllY9RNurGG0KSK2x0ldUCCFOoR4xVm6e3J+bJ/fnYJWTJZuKWVJYzKbDtXy9q4Kvd1Vw/3+KmDQwgfnpqcwankK0Lbzej+L0UnQ6LEOHYhk6lLiFC1H9flxbtuLMXaMFn2vX4q+poX75cuqXLwdAn5iAPSsb2/gc7Dk5GHv3DjuANOvNwQntt3M7br+bTeWbyC/Np6CkgI3lG6l0VbJs3zKW7VsGQJwlLhh6ZqVk0T86/MBViPYYDAZsNhvl5eUYjUZ08vvkOS8QCODxeHC5XN363zsQCFBeXo7NZsNgOLmoUoJOIYQQoiu07CvK8GPft2Vf0daDldqrGPU0nFhf0ZDq0MSjQWjL6lF7AujlxbcQQpyoXnE2brtoALddNID9lQ4WF2qh55biOr7cUc6XO8r5rX4TFwxMYH56GjNHJBNlkZ+7ZzpFr8c6aiTWUSOJv+kmVK+Xxk1FOPO0wUaN69bjL6+gbulS6pZqvTYNKSnYc7Rt7vacbIw9OjGAsRWz3hzcss5o8Pg9bKrYRH6JFnxuKN9AlauK5fuXs3y/FrjGWeIYlzwuONl9QMwACT7FCVEUhdTUVPbu3cv+/fu7ezniNFBVlcbGRqxWa7f/3NDpdPQ+gTeMWpOgUwghhDjdwu4r6jz+kKXm6tHG6tC+oqWdWE/LvqId9RZtDkqNso1JCCE60ifezqKpA1k0dSB7yhtYUqgNMtpWUs/q7eWs3l6O6d86Jg9OYF56KjOGJRMpoedZQTEasY3NwDY2g4TbbiPgdtO4cSPONbk48nJp3FiIr6SE2o8+ovajjwAw9uqlbXPPGY8tJxtjUlLY1zXpTYxLHse45HHB4LOookgLPksL2FCmBZ8r9q9gxf4VAMSaY7WwtKnqc0DMAHSKVOaJzjGZTAwaNCi4jVic27xeL19++SWTJ0/GaOzef49MJlOXVJVK0CmEEEKc6Uw2MPU9gb6iLQYrtVc9eiJ9RU2RLSpCE9oGoS17i0pfUSHEeax/YgQ/nT6In04fxK6yehYXFrO4sJhdZQ18trWMz7aWYTLomDI4MRh62s3y8uxsoTObsWdnY8/OJpGfEmhsxLlunTbYKDeXxqIivAcPUnvwILX//BcApn79gtvcbdnZGOLiwr6uSW9ibPJYxiaP5VZuxev3UlRZFFLxWe2uDgk+Y8wxZCZnBie7D4wZKMGnOCadTofFIkMzzwd6vR6fz4fFYun2oLOryL+kQgghxLkkrL6iTSFn68FKHVWM+j3gqYeq+jD6irasDE1oO2SpOSiVvqJCiHPYwKRI7p4Ryd0zBrO9pJ4lhUdYXFjMngoHy7eUsnxLKWaDjmlDk5iXnsq0oUnYTPJS7Wyis1qJmDSJiEmTAPA3NNC4di2ONbk4c3Nxbd2KZ+9ePHv3UvPuewCYBw/WtrmPz8GWmYk+Ojrs6xr1RjKSMshIyuCW9Fvw+r1srtwcrPhcX7aeGncNnx34jM8OfAZAtDk6WO2ZmZzJoNhBEnwKIc4Z8q+nEEIIcb7S6ZrCxwQ61VfUXddqsFJ5B71Fy1v0FT2oHcej6FtNoG8VhLbsNyp9RYUQZ7EhKZEMSRnCz2YOZmtxPUs2aaHn/konnxaV8GlRCVajnmnDkpg/KpWpQ5OwGE9uAq04/fQREURcdBERF10EgL+2Fmd+vjbYKDcX944dwaP6zTdBUbAMG4Zt/HjsOdlYx2Wij7CHfV2j3siYpDGMSRrDzdyMN+Blc8VmCkoLKCgpYF3ZOmrdtaw8sJKVB1YCWvA5Lknr8ZmZksng2MESfAohzloSdAohhBDi+BQFLNHakTDw+Pf3ONsZrNRyAn2LitHGalD90FCqHZ3qKxrbTnVo663zsegC0l9KCHFmUhSF4WlRDE+L4hffG8LmI3XaIKNNRzhY1aj19ywsxmbSM2NYMvPSU7locKKEnmcpfXQ0kTNmEDljBgC+qiqceXk41qzBmZuHZ+9eXFu24Nqyhap//AP0eqwjRwYrPq0ZGeis4ffJNuqOBp83jboJb8DLlsotFJQUkF+az/rS9dS6a1l1cBWrDq4CIMoUxbjkccGqz8Gxg9Hr5PtOCHF2kKBTCCGEEF3PZANTH4jtc/z7NvcVbRmEtplGX3F0e73q18LRxmqo2N7hwxqBBYC69e7QIUvtbZ1vDkrNUdJXVAhx2imKwsge0YzsEc2vZw+h8FAtSzZpQefhmkY+3niEjzceIcJsYObwZOaNSuXCwQmYDRI+na0McXFEzZ5N1OzZAHhLy4IT3Z25eXgPHqRx40YaN26k8sUXwWjEOjo9ONjIOmYMOpMp7OsadUZGJ45mdOJobhx1I76Aj62VW8kvzSe/JJ91peuo89Sx+uBqVh9cDUCkKTIk+BwSO0SCTyHEGUuCTiGEEEJ0rxPpK9omCG09gb4CtaEMxe9G8TRoW+mr9x7/8fXmVv1EE1sEoa0CUmuc9BUVQnQ5RVEY3SuG0b1iuHfOUNYfrAlWd5bUufhw/WE+XH+YSIuB7w1PYX56KpMGJmAyyM+js5kxOYnoBQuIXrAAAO/hw03b3NfgyM3DV1JCY8FaGgvWwrPPopjNWMdmaIONcnKwjhyJcgKDRAw6A6MSRzEqcRQ3jLwBX8DHtqpt5Jc0BZ9l66j31PP5wc/5/ODnAEQaIxmbPDa41X1o7FAJPoUQZwwJOoUQQghx9mjZVzRp2DHv6vN4WL74X3xv4hiM7ppjT6B3VGiDlvzuE+grmthxpWjLfqPSV1QIESZFURjbO5axvWP57dxhrDtQzeLCYpZuKqas3s2/1h3iX+sOEW01MmtEMvPT05gwIB6jXkLPs52xRw9iLruUmMsuRVVVvAcOBAcbOfLy8FdU4PxuDc7v1gCg2GzYxo3TBhtl52AZPgxFH374aNAZGJkwkpEJI/nxyB/jC/jYXrVdCz5LtYrPem89Xxz6gi8OfQFAhDFCCz6Ts7SKz7ghGHQSNQghuof89BFCCCHEuUlR8OltED8QOlPl0rKvaMveog3t3Na6r2hnNPcVDekn2hyEtuo3arKd3HMXQpxzdDqFzL5xZPaN44H5w8nfV8WSTcUs3VRCRYOb/ys4xP8VHCLWZmT2yBTmp6eR0y8Og4SeZz1FUTD16YOpTx9if3glqqri2b1b2+a+JhdnXh7+2locX32F46uvANBFRmLLysKek40tJwfz4MEoJ7ALwaAzMCJhBCMSRnD9yOvxB/xsq96m9fgsORp8fnnoS7489CWgBZ8ZSRlkpWjB59C4oRJ8CiFOG/lpI4QQQggB4fUV9Xtb9A0tbzWNvqLNNvrO9hU9upaIDqpE26kYlb6iQpx3dDqFnP7x5PSP5/cLRpC3t4rFhUdYVlRCpcPDu3kHeTfvIPF2UzD0zO4Xh14nPyvOBYqiYB44EPPAgcRdey1qIIB7x47gYCNnfj6B+noaVq2iYZU2YEgfE4MtOxvb+BzsOTmY+vdHOYF/O/Q6PSPiRzAifgQLRyzEH/CzvVqr+CwoLWBt6VrqPfV8dfgrvjqsha52o/1o8JmcxbD4YRJ8CiFOGfnpIoQQQggRLr0RolK143gCAS3gbHfrfFMQ2nIbvd+t9RQNp69oyyrRNhPoE45+LH1FhTjn6HUKEwbEM2FAPH+4eAS5rULPt3MP8HbuARIizMwdpYWemX1i0Unoec5QdDosQ4diGTqU+OuvR/X5cG3dqm1zz83DuXYt/poa6pcvp375cgD0iQnYs3Ow5WRjz8nB2Lv3CQefw+OHMzx+eDD43FG9Ixh8FpQWUO+p5+vDX/P14a8BsBlsZCRnkJWs9fgcHj8co07auwghuoYEnUIIIYQQp5JOB/Z47ThOX1FUFdz1rYLQY1SMNvcVrTukHcej6MCW0H6laHu3SV9RIc4qBr2OSQMTmDQwgT9+fyTf7q5kSeER/ru5lIoGN298t583vttPcpSZOSNTWTA6lYxeEnqeaxSDAeuoUVhHjSL+pptQvV4aNxUFBxs1rl+Pv7yCuiVLqFuyBABDair27Gxs48djz8nGmNaJAYHt0Ov0DIsfxrD4YVw34jr8AT87a3YGt7oXlBZQ56njm8Pf8M3hbwCwGqyMTRpLZkommcmZjEgYIcGnEOKESdAphBBCCHGmUBSwRGlH/IDj39/b2E4Q2mrYUvPHjVWgBprC07LOrccS085gpdbT6Js+lr6iQpxRjHodFw1O5KLBifzpkgDf7KpgcWExy7eUUFrn5rVv9/Hat/tIjbYwd1Qq89NTGdMr5oSq+sSZTTEasY3NwDY2g4TbbyfgdtO4YWPTYKNcGjcW4isupvajj6j96CMAjL16BQcb2XKyMSYlndC19To9Q+OGMjRuKD8a/iMCaoCd1TspKD0afNa6a/nmyDd8c+Ro8JmRlEFmciZZKVmMiB+BUd54E0J0kgSdQgghhBBnK6MVYnprx/EE+4qWt+ohWt7Ox+VaX1FXjXZU7Dj+45siQrfJH6ti1BItfUWFOI1MBh1ThyYxdWgSbt9IvtpRwZJNxazYUkpxrYtXvt7LK1/vpUeMlXnpWug5qke0hJ7nKJ3ZjD0nG3tONon8lIDTiXP9epxrtODTVbQZ78GD1Bw8SM0H/wTA1L9/0zb38dhysjHExp7YtRUdQ+KGMCRuCNcOu5aAGmBXzS4t9CzRtrrXuGv49si3fHvkW0ALPsckjiEzRQs+R8aPlOBTCNEhCTqFEEIIIc4HJ9RX9BiVoi230ftcLfqK7uvEWkxNFaEJx6gYbfrYFgc6/Uk/fSGExmzQM2N4MjOGJ+Py+vliRzlLCov5bGsph2saefHLPbz45R56xVmZNyqN+empjEiLktDzHKaz2YiYNImISZMA8Dc04Cwo0AYb5ebi2roVz549ePbsoebd9wAwDx4cHGxky8pCHxV1YtdWdAyOHczg2MHB4HN3ze6jPT5LCqh2V/Nd8Xd8V/wdABa9hTFJY4IVnyMTRmLSm7rmiyGEOOtJ0CmEEEIIIUK17CvK0GPft2Vf0Za9RZsrQ1tPoHfXgd8Tfl/RYw1Zaq4UtSeCQV7sCtFZFqOeWSNSmDUiBZfXz+ptZSzeVMyqrWUcrGrk71/s5u9f7KZvvK2p0jONoSmREnqe4/QREUROmULklCkA+GtqcBYU4FiTizM3F/fOnbh37MC9YwfVb7wJioJl+HBsOTnYx+dgHTsOfYT9hK6tU3QMih3EoNhBXDPsGgJqgD01e8gvzSe/JJ+1pWupclWxpngNa4rXAFrwOTpxdLDic1TCKAk+hTiPSdAphBBCCCFO3En1FT1OxWjrvqKdaS0a7Cvacut8B9WjphN7IS7Euchi1DNnVCpzRqXi9PhYta2MJYXFrNpWxr5KJ8+u3s2zq3fTP9HO/FGpzB+dxuDkyO5etjgN9DExRM6YQeSMGQD4Kitx5uXhyM3FmZuHZ+9eXJs349q8map//AP0eqwjRwYHG1kzMtBZrSd0bZ2iY2DsQAbGDuTqoVejqip7aveQX5IfrPqsclWRW5JLbkkuAGa9+WjwmZzFqMRRmPXmLvt6CCHObBJ0CiGEEEKI0yesvqI+cFa0mEBfcYyK0RPoK2q0tz9Yqb2KUekrKs4jNpOB+elpzE9Pw+H28dnWUpYUFvP5jnL2lDt4atUunlq1i0FJEcxPT2NeeioDkyK6e9niNDHExxM1Zw5Rc+YA4C0t1YLPNWtw5ubhPXSIxo0bady4kcoXXkAxGrGOHh2s+LSMHo3OdGIVl4qiMCBmAANiBnDV0KtQVZW9tXuDoWd+ST6VrkrySvLIK8kDwKQzMTppNFnJWWSmZJKemC7BpxDnMAk6hRBCCCHEmUlvgMgU7TieQEALOIOhaOtp9BWh2+h9LvA6oNoRRl/RxLaDldqrGJW+ouIcYjcb+P6YHnx/TA/qXd5g6PnFjnJ2ljXwxGc7eOKzHQxNiWTeqFTmpafSP1FCz/OJMTmZ6AULiF6wAADPocM4c3Nx5uXiWJOLr7RU6/lZUEDFs8+iWCxYM8YEBxtZR45EMZ7YcCFFUegf05/+Mf354dAfasFn3V5tsFFJAfml+VQ0VgQrQNmoBZ/pielkpWSRmawFnxaDpSu/JEKIbiRBpxBCCCGEOPvpdFrAaIujU31FPQ2tBiu16jEarB4tb9FX9LB2HI+iA1t8i23yrSpGW38sfUXFWSLSYuTSjJ5cmtGT2kYvK7aUsqTwCF/trGBbST3bSup5bMUOhqdGBae394mXFhHnG1PPHph6XkbM5Zehqire/ftx5ObhzF2DIzcPf2Ulzu/W4PxO67Gps9mwZo7TBhvljMcybCiK/sTeLFIUhf7R/ekf3Z8rh1yJqqrsq9sXrPYsKCmgvLFcG3RUWgCAUWcMCT5HJ46W4FOIs5gEnUIIIYQQ4vyiKGCO1I5O9RV1tTNYqYOKUWdzX9Gm4LRTfUWj2w5Wav44IgnFHIvNXQYeBxhjTvbZC9Eloq1GrhjXkyvG9aTG6WH55lIWbyrmm10VbCmuY0txHY/8dzujekQzLz2VeaNSSYk8sao9cfZSFAVT376Y+vYl9oda8OjZvTs42MiZl4e/thbHl1/h+PIrAHRRUdgyM7GPz8GWk4N50CAUne6Er98vuh/9ovvxg8E/QFVV9tftDwk+yxrLWFu6lrWlawEt+ByVMCo43Gh04mishhPrMSqEOP0k6BRCCCGEEOJYjBaI6aUdx9PcV7T1YKX2KkYd5RDwgatWOyp3tvuQBmAmwJZfaH1F2xus1F71qCVG+oqK0yLGZuLKrF5cmdWLKoeH/24uYUlhMd/urmDT4Vo2Ha7l4U+3kd4zin56hTE1jfRJlNDzfKQoCuaBAzEPHEjcj65FDQRwb98eHGzkzM8nUFdHw6pVNKxaBYA+NhZbdja2nGzs48dj6tcP5QR/timKQt/ovvSN7ssVg69AVVUO1h/UtrY3TXYvc5axrmwd68rW8WLhixh0BtIT0hmXPI6slCzGJI3BIFGKEGcsRVVVtbsXca6qq6sjOjqa2tpaoqKiuns54jTwer0sXbqUuXPnYjzBPjNCnAnke1mcC+T7WJzxQvqKthqs1KJ6VG0ox19XgkH1hPf4wb6iCcesGMWeqG21l76iootVNLhZVqSFnrl7Kwm0eOU5tncM89LTtErPaNkmLDSqz4dr69bgYCPn2rWojY0h99EnJmDPzsE2Pgd7Tg7GXr1OOPhsc31V5VD9oWDomV+ST6mzNOQ+Bp2BkfEjia6L5ocTf8i41HHYjLYuub4Qp9vZ8vtyOPmaBJ2nkASd55+z5YeEEMcj38viXCDfx+Jc4fV6WbpkCXNnTMborm4xWKl1pWj50W307trwLtKyr+gxK0abDoNMLBbhKat3sWTjYd76Yit7GhRavgrN6hvLvFGpzB2VSlKUhJ7iKNXjobGoCGduLo7cPBrXrUP1hL7pY0hNbervmYM9JxtjWlrXXV9VOdRwSBts1FT1WeIoCb2+YmBEwgiyUrLIStYqPiX4FGeLs+X35XDyNam3FkIIIYQQ4kzX3Fc0Ii78vqLBwUrtV4y26SvaGc19RVsOVopIar961CwTuAUkRVr4UU5v4iqLGHfBNFZsq2BJYTEF+6vJ36cdf1i8hey+ccxPT2X2yFQSIyVQP98pJhO2sWOxjR1Lwu23E3C7adywMTjYqLGwEF9xMbX/+Q+1//kPAMbevbHnZGPLGY89JxtDYuKJX19R6BXZi16Rvbh00KWoqsrhhsOsObyGj9Z9RImxhBJnCRvLN7KxfCMvb3oZg2JgeMJwspKzyErJIiMpQ4JPIU4jCTqFEEIIIYQ414TdV7Sy7WCl9ipGO9lXNHQtttBt8sGP26kelb6i54XkKAs/ntSPH0/qR3FtI0sKi1myqZj1B2rI3VtF7t4qfv/xZsb3j2deeiqzR6QQHyGhpwCd2Yw9Jxt7TjaJQMDpxLluvVbxmZeLa1MR3gMHqDlwgJoP/gmAqX9/bbBRdg62nGwMsbEnfH1FUegZ2ZPvD/g+xu1G5s6dS5m7LLjNfW3pWg43HKawvJDC8kJeKXoFvaJnRPyI4HCjjKQM7EZ7F31FhBCtSdAphBBCCCHE+UxvgMhk7Tie5r6irQcrdVQx6msErxNq9mvH8eiMoVWix6oYlb6i54TUaCs3Xdifmy7sz6FqJ0s3FbOksJiNh2r5dncl3+6u5IGPNjNxQDzz01P53vAUYu2m7l62OEPobDYiLphExAWTAPA3NOAsKMC5Rgs+3Vu34dmzB8+ePVS/8y4A5iFDgoONbJmZ6E+yzVyPiB70GNiDSwZeAsDhhsMUlBQEJ7sfbjhMYUUhhRWF/KPoH+gVPcPjh5OZkklmciZjk8YSYZLKdyG6igSdQgghhBBCiM7R6cAWpx2JQ459X1UFjyN0m7yj/OjHIdWj5Vpf0YAX6o9ox3EpWth5rCFLLT+WvqJnvJ6xNm6ZPIBbJg/gYJWTxYXFLNl0hKLDdXy1s4Kvdlbw2w+LmDQwIRh6RtvO3J5y4vTTR0QQOWUKkVOmAOCvqcGRn68NNsrNxb1zJ+7t23Fv3071G2+CTodl2LDgYCPr2HHoI06u2rI5+Pz+wO8DcKThCAWlBcE+n4caDrGpYhObKjbxatGr6BQdw+OGk5WSRWZKJhlJGUSaIk/2SyHEeUuCTiGEEEIIIUTXUxStP6c5AuL6H//+Xhc4K1oNVipvv3rUWQmo2v2dFZ1bjzm6RXVo6yFLrapHpa9ot+sVZ+P2KQO4fcoA9lU4WLKpmMWFxWwtruOLHeV8saOc+/SbuHBQIvPTU5kxPJkoi4SeIpQ+JoaomTOJmjkTAF9lJc68PBxrcnHm5uLZtw/X5s24Nm+m6pV/gMGAdeRIbbDR+BysGRnoLCc3ICstIo2LIy7m4gEXA1DiKCG/JD9Y8Xmw/iBFlUUUVRbx6mYt+BwWN0wLPpMzGZs8VoJPIcIgQacQQgghhBCi+xktEN1TO44n2Fe0vBMVo+Vapai7Vjsqd3ViLba2g5Vi+8KYazu3xV90qb4JdhZNHciiqQPZXd6g9fQsLGZ7aT2rtpWxalsZJr2OyYMTWTA6lenDkokwy0td0ZYhPp6oOXOImjMHAG9padNE91ycuXl4Dx2iccMGGjdsoPKFF1CMRqyjR2Mbrw02sowejc50cq0TUuwpLBiwgAUDFgBa8Nmy4vNA/QE2V25mc+VmXtv8GjpFx9C4oWQmaz0+xyaPJcp0ctvthTiXyU9/IYQQQgghxNklnL6iqgqN1W0HK3U0bMnrbOorekA7Wvr8YRhzNUy8E+IHnJrnJo5pQGIEd04fxJ3TB7GztJ7FhcUsLjzC7nIHn20t5bOtpZgMOqYOSWR+ehrThiZhl9BTdMCYnEz0xRcTfbFWbek5dLgp+FyDMzcPX2mp1vOzoICKZ0CxWLCNzcCcmYnF50f1esF4cpXEKfYU5vefz/z+8wEodZQGqz0LSgvYX7efLZVb2FK5hTe2vIGCogWfKZlkJWvBZ7Q5+qS/FkKcK+QnvhBCCCGEEOLcpSid7ysK4G5of8jS7pVwMBfWvgZrX4fhF8Oku6HH2FP9DEQHBiVH8rOZkdw9YxDbS+tZUqhtb99b4eC/m0v57+ZSLEYd04YmMT89jalDkrCaZICV6JipZw9MPS8j5vLLUFUV7/792jb3vFwcuXn4KytxfPsdjm+/ozew57XXsGWOw54zHltODpZhQ1H0J/c9lmxPZl7/eczrPw+AMmeZVu1Zmk9BSQH76vaxtWorW6u28uaWN1FQGBI3JFjxOS55nASf4rwmQacQQgghhBBCNOuor+iUX8P+7+DrJ2Dnf2HLR9rRb7IWeA6YpoWq4rRTFIWhKVEMTYninpmD2Vpcz+LCIyzZVMz+SidLN5WwdFMJVqOe6cO00HPKkEQsRgk9RccURcHUty+mvn2JveqHqKqKZ9cuHLl5NHz3HXXffove6cTx5Vc4vvwKAF1UFLasLOw52dhyxmMeNBBFpzupdSTZkpjbfy5z+88FoNxZHqz4zC/JZ1/dPrZVbWNb1Tbe2voWCgqDYwcHhxtlJmdK8CnOKxJ0CiGEEEIIIURn9JmgHaVb4Ju/QdE/Ye+X2pGSDpPuguGXaFvrRbdQFIXhaVEMT4vil7OGsPlIHZ8UHmFJYTGHqhubtroXYzfpmTE8mfnpaUwenIDZIKGnODZFUTAPGoR50CAif3gl6xcvZvrAgbgL1uLMzcVZUECgro6GlStpWLkSAH1sLLbsbOzjc7Dl5GDq1w/lJN8QSbQlMqffHOb00/qMVjRWBPt7FpQWsKd2D9urt7O9ejtvbX0LIBh8ZiVrFZ8xlpiTWoMQZzL5F1gIIYQQQgghwpE8HC57AabdD989C+teh5JC+NeNsOohmHAHZPwIjNbuXul5TVEURvaIZmSPaH4zeyiFh2q1Ss/CYo7UuvhowxE+2nCESLOBmcOTmT86lQsGJmIynFwFnjhP6HSYhw4lYtQo4n98ParPh2vLFm2w0ZpcnOvW4a+upv6//6X+v/8FwJCYiC0nB1tONvacHIy9ep108JlgTWB2v9nM7jcbaAo+m4YbFZQUsLt2Nzuqd7Cjegdvb30bgEGxg8hK1io+xyWPI84Sd3JfCyHOIBJ0CiGEEEIIIcSJiOkFcx6Gi34F+S9D7t+heh8s/YU2uCjnNsi6UesPKrqVoiiM7hXD6F4x3DtnGBsO1bB4YzFLNxVTUufi3+sP8+/1h4myGPjeiBTmp6cyaWACRr2EnqJzFIMBa3o61vR0uPlmVI+HxqIiHGu0wUaN69fjKy+nbvFi6hYvBsCQloo9Owfb+Bwt+ExNPel1JFgTmN13NrP7Hg0+15au1YLP0gJ21exiZ/VOdlbv5J1t7wAwMGagttU9OZPMlEwJPsVZTYJOIYQQQgghhDgZtjgt7JxwB2x4G759SpvYvvpPWk/PcdfDhJ9AdM/uXqkAdDqFsb1jGds7lvvnDWPdgWoWFxazZFMx5fVu/rn2EP9ce4gYm5FZw1OYPzqVCf3jMUjoKcKgmEzYxo7FNnYs/OQnBNxuGtdv0AYbrcmlsbAQ35Fiav/zH2r/8x8AjL17Y8/Rtrnbc7IxJCae9DoSrAnM6juLWX1nAVDZWKkFn019PnfV7Aoe7257F9CCz+bQMzM5k3hr/EmvQ4jTRYJOIYQQQgghhOgKJhtk3wzjfgybP9T6eJZugjXPQt4LMOpKrY9n0tDuXqlootMpZPaNI7NvHL+bP5yCfVUsLizm06JiKho8vF9wkPcLDhJnNzFrRAoL0lPJ7hcnoacIm85sxj4+B/v4HBLvhIDTiXPdepy5a3Dk5uEqKsJ74AA1Bw5Q88EHAJgGDAgONrJlZ2GIjT3pdcRb4/le3+/xvb7fA6DKVRWs+MwvzWdn9c5g8Pne9vcAGBA9QAs9m4LPBGvCSa9DiFNFgk4hhBBCCCGE6Ep6A6T/AEZdAbtWwjdPwr6vYOM72jF4DlxwN/Qe390rFS3odQo5/ePJ6R/PgxePIHdvJYsLi1lWVEKVw8O7eQd4N+8ACREmZo9MYX56Gll949DrTq7Hojg/6Ww2Ii6YRMQFkwDw19fjLCjAmZuHIy8X99ZteHbvxrN7N9XvaJWW5iFDgoONbJmZ6KOiTnodcZY4ZvaZycw+MwGodlWzrnQd+aXaVPcd1TvYXbub3bW7eX/7+wD0j+5PZnJmcLK7BJ/iTCJBpxBCCCGEEEKcCooCg2Zox6G1WuC59RPY8al29BqvBZ6DZoFOKgTPJHqdwsQBCUwckMAfLx7Bmj1VLC48wrLNJVQ0eHhrzQHeWnOAxEgzc0emMH90GuN6x6KT0FOcIH1kJJFTpxI5dSoA/poaHPn52mCjvFzcO3fh3r4d9/btVL3+Buh0WIYP1wYbjR+PbexYdHb7Sa8j1hLL9D7Tmd5nOgA1rhrWlq0NTnbfUb2DPbV72FO7h//b8X8A9I3qq011b+rzmWg7+S33QpwoCTqFEEIIIYQQ4lTrOQ5++CZU7NJ6eG58Fw6ugXevgsSh2pb2kVeAwdTdKxWtGPQ6LhiUwAWDEnjokpF8s6uCJYXF/HdzCeX1bl7/bj+vf7ef5Cgzc0elMj89jYxeMRJ6ipOij4khauZMomZqlZa+igqceXk4cvNw5ubi2bcPV1ERrqIiql75BxgMWEeODA42smZkoLNYTnodMZYYpveezvTeWvBZ665lbela8kvyKSgtYHvVdvbV7WNf3T4+2KFtue8b1ZfMlMzgZPckW9JJr0OIzpKgUwghhBBCCCFOl4SBcPFTMPU+WPM8FPwDyrfBf26HVX+CCYtg7EIwR3T3SkU7jHodU4YkMWVIEv/v0lF8vaucxYXFrNhcSmmdm1e/2cer3+wjLdqihZ6j0xjdMxpFkdBTnBxDQgJRc+cSNXcuAN7SUpy52mAjZ24u3sOHadywgcYNG6j8+wsoRiPWMWO0wUbjc7Cmp6OYTv6NlGhzNNN6T2Na72mAFnw2b3UvKClgW9W2YPD5zx3/BKBPVJ+jW92TM0m2J5/0OoToiASdQgghhBBCCHG6RabAzD/AhfdAwauw5jmoOwz/vQ+++Ctk3QQ5t0GEbAE9U5kMOqYNTWba0GTcPj9f7qhgSeERVmwp5Uiti5e/3svLX++lR4yV+elapefIHlESeoouYUxOJvrii4m++GIAPIcOacFnbi7ONbn4yspw5ufjzM+n4plnUCwWbGMzsOWMx56TjWXkSBTDyUdC0eZopvaeytTe2pb7Ok+dFnw2VXxuq9rG/rr97K/bz792/guA3pG9g/09M5MzSbGnnPQ6hGgmQacQQgghhBBCdBdLtNanc/ztsPE9bVt75S746lH47hnI+BFMuAPi+nX3SsUxmA16Zg5PZubwZFxeP59vL2fJpmJWbi3lcE0jL3y5hxe+3EPvOBvz0lOZn57K8FQJPUXXMfXsialnT2IuvxxVVfHs26cNNspdgzM3D39VFY5vv8Px7XeUow1DsmZlYs/OwTY+B8vQoSh6/UmvI8oUxZReU5jSawqgBZ/rS9dTUKr1+NxatZUD9Qc4UH8gGHz2iuwVrPbMSsmS4FOcFAk6hRBCCCGEEKK7GcwwbqEWbG5bog0uOrwW8l/WtrePuFTr45k6urtXKo7DYtQze2QKs0em0Ojx8/n2MhYXFrNyWykHqpw8//lunv98N/0S7Mwblcr80akMSY6U0FN0GUVRMPfrh7lfP2Kv+qEWfO7apW1zz8vFkZdPoLYWxxdf4vjiSwB0UVHYsrKw52hT3c2DBqJ0wZC0KFMUF/W6iIt6XQRAvaee9WXrg8ONtlRt4WD9QQ7WH+TfO/8NQM+InsGKz6zkLFIjUk96HeL8IUGnEEIIIYQQQpwpdHoYfjEMWwD7vtYCz12fQdG/tGPANJh0N/SbrE11F2c0q0nPnFGpzBmVitPjY9W2MhZvLGb19jL2Vjh4ZvUunlm9iwGJdualp7EgPZVByZHdvWxxjlEUBfOgQZgHDSLuf36EGgjg3rZNG2y0Zg3OggICdXU0rFxJw8qVAOhjY7X+njnZ2HLGY+rXt0vC+EhTJJN7TmZyz8kANHgaWFe2joLSAgpKCthSuYVDDYc4tOsQH+76EIAeET2C1Z5ZKVmkRaSd9DrEuUuCTiGEEEIIIYQ40ygK9LtQO0o2wTd/g6J/w+5V2pGWoVV4DrtYC0fFGc9mMjA/PY356Wk0uH2s3FrK4sJivthezu5yB0+t3MlTK3cyODmC+elpzEtPZUCiDKUSXU/R6bAMH45l+HDif3w9qs+Ha8uW4GAj57p1+KurqV+2jPplywAwJCYGBxvZcnIw9uzZJcFnhCkiJPh0eB2sL1uv9fgsKWBz5WYONxzmcMNhPtr9EaAFn+OSxwWDzx4RPU56HeLcIUGnEEIIIYQQQpzJUkbB5S/DtPvhu2dh3ZtwZD18cD3E9YeJP4XR14DR0t0rFZ0UYTbw/TE9+P6YHtS5vFroubGYL3eWs6O0gcdX7ODxFTsYmhLJgtFpzBuVSt8Ee3cvW5yjFIMBa3o61vR0uOVmVI+Hxk2btMFGuXk0rl+Pr7ycusWLqVu8GABDWir2nPHYcrKx5+RgTO2a7eV2o50LelzABT0uALTgc0PZBvJL8skvzWdLxZZg8Pnx7o8BSLOnBQcbNQef0gri/CVBpxBCCCGEEEKcDWL7wtxH4KJfQ96L2lG1Bxb/DFb/BcbfBpk3gjWmu1cqwhBlMXJpRk8uzehJbaOXFVtKWVx4hK93VrCtpJ5tJdt55L/bGZEWpVV6jkqld7ytu5ctzmGKyYRt3Dhs48bBT35CwO2mcf2G4GCjxsJCfEeKqf3wQ2o/1LaXG/v0Dg42smdnY0hM7JK12I12JvWYxKQekwBwep1a8FmaT35JPpsrNnPEcYSPd38cDD5T7ClkJWcFKz57RvbskrWIs4MEnUIIIYQQQghxNrEnwNT7YOKdsP5N+PYZqDsEK/8IXz0BmT+G8T+BKBngcbaJthq5YlxPrhjXkxqnh+WbS/mk8Ajf7q5k85E6Nh+p43+XbSO9ZzTz01OZOyqVnrESeopTS2c2Yx+vbVsHCDgcONet1wYb5ebhKirCu/8ANfsPUPPBBwCYBgwIDjayZWdhiI3tkrXYjDYm9pjIxB4Tgabgs3wDBSUFFJQWsKliEyWOEj7Z8wmf7PkEBYWHJj3E9wd+v0uuL858EnQKIYQQQgghxNnIHAHjb4esm7RBRd/8Dcq2wLdPwZrnYfQPYeJdkDi4u1cqTkCMzcSVWb24MqsXVQ4Py4pKWLLpCN/trqTwUC2Fh2r589JtjOkVEww902Ks3b1scR7Q2e1EXHgBERdq28v99fU4CwpwrsnFkZeHe9s2PLt349m9m+p33gHAPHRocLCRLSsTfWTXDN2yGW1MTJvIxLSjwefG8o3kl+Tz3ZHvKKos4pkNzzCv/zwMOonAzgfyX1kIIYQQQgghzmZ6I4y+CtJ/CDuXw9dPwoFvYf1bsP5tGDpPm9TeK6u7VypOUJzdxDU5vbkmpzcVDW4+LSphSeERcvdWseFgDRsO1vCnJVsZ1yc2GHomR0nPVnF66CMjiZw6lcipUwHwVVfjzM/HmZuHMy8X985duLdtw71tG1WvvwFNw5CaBxvZxo5FZ++aHrQ2o40JaROYkDaBW0ffyswPZlLiKOGLg18wvc/0LrmGOLNJ0CmEEEIIIYQQ5wJFgcGztONgnhZ4bl8C2xZrR59JWuA5aKZ2X3FWSogw8z/j+/A/4/tQVu/i000lLCksJn9/FWv3V7N2fzV/XLyFrD5xzB+dyuyRKSRFSugpTh9DbCxR3/seUd/7HgC+igqceXnBqe6e/ftxFRXhKiqi8uVXwGDAOmqUNtho/HisY8ags5z896xZb+bywZfz8qaXeXfbuxJ0nick6BRCCCGEEEKIc02vbLj6HSjfDt88BYXvw/5vtCNpBEy6C0ZeplWDirNWUqSFhRP7snBiX0pqXSzdVMySTcWs3V9N3r4q8vZV8fuPN5PTL4556WnMGZlCQoS5u5ctzjOGhASi5s4lau5cALwlJThztf6ezjVr8B45QuP69TSuX0/l319AMRqxjhmjDTbKycGano5iMp3Qta8cfCX/KPoHuSW57KrexcDYgV351MQZSIJOIYQQQgghhDhXJQ6BS57VhheteQ7WvgZlm+HDW2DVn2DCIhj7P2Dqmm2jovukRFu44YJ+3HBBP47UNLJ0UzGLC4vZcLCGNXuqWLOnit9/VMSEAfHMG5XG7JEpxNlPLDwS4mQYU1KI/v73if6+NiDIc+iQFnw2VXz6ysq0re/5+VQ8/QyK1YotIwNbTg6RM6ZjHjCg09dKjUhlWq9pfHbgM97b/h73j7//VD0tcYaQoFMIIYQQQgghznXRPWDW/4PJv4D8VyD371B7AJb9Gr74X8i+RTvs8d29UtEF0mKs3HRhf266sD8Hq5zBSs/CQ7V8s6uSb3ZV8ruPipg4IJ4F6Wl8b0QyMTYJPUX3MPXsialnT2IuvxxVVfHs29dU8ZmLMzcPf1UVjm+/xfHtt1Q8+yx933sXy/DhnX78q4dezWcHPuPj3R9z19i7iDR1zSAkcWaSoFMIIYQQQgghzhfWWC3snLAINrwD3z4N1Xvhi4e1ae0Z/wMT74CY3t29UtFFesXZuPWiAdx60QAOVDpZvOkISwqL2Xykjq92VvDVzgru+1DhgkEJzE9PY+bwZKKt0tJAdA9FUTD364e5Xz9ir7oKVVVx79yJMzePmg//jXvLVipf+Qc9Hnu004+ZlZLFgOgB7K7dzce7P+baYdeewmcgupuuuxcghBBCCCGEEOI0M1oh60b46Vq44lVIHQ1eJ+S9AH8bA/+6GUqKunuVoov1jrfxkykDWXLnhaz+xRR+OWsIQ1Mi8QVUPt9ezi8+2Ejmn1Zw42v5fLj+EPUub3cvWZznFEXBMngwcf/zI9L+9CcA6v77X7wlJWE9xtVDrwbg3W3vElADp2St4swgQacQQgghhBBCnK90em0o0S1fwP/8B/pPAdUPm/4P/j4J3roC9n0NqtrdKxVdrF+CnUVTB7Ls7sl8ds9F3DNzMIOTI/D6VVZuK+Nn729k3J8+4+Y3Cvhow2Ea3L7uXrI4z1mGD8eWnQ0+H9Vvvx3WuQsGLCDCGMH+uv18d+S7U7RCcSaQoFMIIYQQQgghzneKAgOmwnUfwS2fw4hLQdHBrhXw2jx4eQZs/QQCUgl1LhqYFMGd0wex/GcXsfxnk7lz+iD6J9rx+AKs2FLKXe9tYNxDK7jtzbUsLjyC0yOhp+gecdcvBKD6/f8j4HB0+jyb0cYlAy8BtKpOce6SoFMIIYQQQgghxFFpGfCD17Rt7Zk3gN4Mhwvg/R/Bs9mw7g3wubt7leIUGZwcyT0zB7PynotYdveF3DF1IH3jbbh9AZZtLuGOd9Yz9qEVLHp7HZ9uKqbR4+/uJYvzSMSUKRj79CZQV0fNf/4T1rk/HPJDAL489CUH6w+egtWJM4EEnUIIIYQQQggh2orrD/OfgJ8VwYU/B0s0VO6Ej38KfxsN3/wNXHXdvUpxiiiKwtCUKH4xawirfzGFJXdewO1TBtA7zobLG2DJpmJuf3sd4/60gp++u57/bi7B5ZXQU5xaik5H3HXXAVD1xhuoYVSZ943uy6S0SaiovL/t/VO1RNHNJOgUQgghhBBCCNGxiCSY/gD8bDN8708QmQb1xbDiAXhiJHz2INSXdvcqxSmkKAoj0qL59eyhfPHLKXxyxwXcOrk/PWKsOD1+Ptl4hFvfXEvmnz7j7vfW89mWUtw+CT3FqRFzySXooqLw7j9Aw+efh3XuNcOuAeDfu/5No6/xFKxOdDcJOoUQQgghhBBCHJ85Eib+FO7aCN9/FhIGg7sWvn4CnhwFn9wFlbu7e5XiFFMUhVE9o7l37jC+/vVU/rNoEjdd0I/UaAsNbh//2XCEm94oIPOhz7jn/zawelsZHp/0dhVdR2e3E3vlDwCoeu31sM6dlDaJnhE9qffUs3TP0lOxPNHNJOgUQgghhBBCCNF5BhNk/Ah+kgtXvQu9csDvhrWvwdPj4P+ug8PrunuV4jRQFIUxvWK4f/5wvvn1NP51+0R+PKkvyVFm6t0+/r3uMD9+LZ/MP63glx9s5Isd5Xj9EnqKkxf7ox+BwYAzLw/Xli2dPk+v03PV0KsAeGfbO6iqeqqWKLqJBJ1CCCGEEEIIIcKn08HQuXDjcvjxMhg0C1Bhy0fw0lR4fQHsWgkSJJwXdDqFcX1i+f2CEXz3m+l8cNsEFk7oQ2KkmTqXjw/WHmLhP/LI/n+fce+/C/l6ZwU+CT3FCTKmpBA1axYAVa+HV9V5ycBLsOgt7KjewboyeVPmXCNBpxBCCCGEEEKIk9NnAlz7f3D7d5B+FegMsPdLeOsyeGEybPonBHzdvUpxmuh0Cll94/jD90ey5t7pvHfLeH40vjfxdhPVTi/v5h3kR6/kkvPnlfz2w018u7sCf0ACcRGeuOuvB6B26ad4y8o6fV60OZp5/ecB8O62d0/F0kQ3kqBTCCGEEEIIIUTXSB4Ol70Ad26AnNvBaIOSQvjXjRiez6Fv+WfglQEg5xO9TmF8/3j+dMkocu+bzjs35XB1dm9ibUYqHR7ezj3ANS9poecDHxWRu6dSQk/RKdZRI7GOGwdeL9XvvBPWuVcPvRqAlftXUuqQYWrnEgk6hRBCCCGEEEJ0rZheMOdhbVL71N+CLR6lZj+jD72B4ZkM+OIRcFZ19yrFaWbQ65g4MIG/XDaK/N/O4M0bs7kqqxcxNiMVDW7e+G4/P3xxDRP+spIHP95Mwb4qAhJ6imOIW3gdADXvvkegsfNvogyJG8LYpLH4VB8f7PjgVC1PdAMJOoUQQgghhBBCnBq2OLjoV3B3Ef5Z/4vDlIDirIDVf4InRsKy+6D2UHevUnQDg17HhYMSefjydPJ/O4PXfpzFFeN6EmkxUFbv5rVv93HF379j0v+u4qHFW1h3oFoGx4g2IqdPx9izJ/7aWmo/+jisc68Zdg0AH+z4AI/fcyqWJ7qBBJ1CCCGEEEIIIU4tk41A5o2sHP4IvktegORR4HXAmmfhb6Phw9uhbFt3r1J0E6Nex5QhSTz6g9GsvX8m/7g+k8syehBpNlBc6+KVr/dy2XPfcsH/rubPS7ey8WCNhJ4CAEWvJ+66/wGg6o03UAOdH3A1rfc0kqxJVLmqWLF/xalaojjNJOgUQgghhBBCCHFaqIoedcTlcNtXcO2/oO+F2pCije/AcznwzlWw/7vuXqboRiaDjmlDk3n8h2PIv38GL12XyffHpGE36Tlc08iLX+7h+89+w4V/Xc1fPt1K0eFaCT3Pc9GXXY4uIgLPnj04vvqq0+cZdUZ+MOQHALyzLbwen+LMJUGnEEIIIYQQQojTS1Fg0Ay4fjHctAqGXQwosONTeHU2vDILtn8KYVRniXOPxahn5vBk/nZVBmt/N5O//2gc89NTsRr1HKpu5IUv9jD/6a+Z8ujn/HXZNrYcqZPQ8zykj7AT8wMtsKx6/fWwzr1i8BUYdAYKywvZXLH5VCxPnGYSdAohhBBCCCGE6D49x8EP34Q7CmDsQtCb4OAaePcqeH4CbHgHfNI/73xnMeqZPTKFZ64Zy7rfzeS5a8cyd1QKFqOO/ZVOnvt8N3Of+orpj33BY8u3s72kXkLP80jcj64FnQ7Ht9/h2r690+clWBOY1XcWAO9ue/dULU+cRhJ0CiGEEEIIIYTofgkD4eKn4O5NMOluMEdB+Tb4z+3w1Bj47llwN3T3KsUZwGrSM3dUKs9dO46198/k6aszmDUiGZNBx54KB0+v2sWsJ79k5hNf8uRnO9hVVt/dSxanmLFHDyK/9z0Aql5/I6xzrx56NQCf7v2Uald1l69NnF4SdAohhBBCCCGEOHNEpsDMP8DPimDGHyAiGeoOw3/vgydGwMqHoKG8u1cpzhB2s4EFo9N44X8yWfe7mfztqjHMGJaMSa9jV1kDT362kxmPf8msJ77k6ZU72VMuYfm5Kv76hQDUffIJvoqKTp+XnpDO8PjheAIe/rXzX6dqeeI0kaBTCCGEEEIIIcSZxxINF9ytVXgueAriB4KrBr56FJ4cCYvvgaq93b1KcQaJMBv4/pgevLwwk4LfzeDxK0czbWgSRr3C9tJ6Hluxg2mPffH/27vvKKnKw//j75mt9N57kSZFiiiioogiKipGVEBEY/QXg4nol2gSoyYmlm++xo4lRgULggULqCiioCBIR+lIE5G+9LJ1fn8MbCSi7sIOd2f2/Tpnz87euc/sZzz3bJhP7vM89HrkM4Z98jWrt+wJOrKKUKkTTqBUu3ZEsrPZNrLg09BDoRD9W/QH4NWlr5KTlxOriDoGLDolSZIkScVXchp0HASDZ8BlL0KdjpCzH2Y9C491gNeugfXzg06pYqZ8egqXdKjLc1efyKzbz+b/Lm1Lt2bVSA6HWLx+J//3wVLOeGASFzz2GU9NXsHajL1BR1YRqHzgrs5to0aRl5lZ4HHnNjqXimkVWb9nPZO/nRyreDoGLDolSZIkScVfOAlaXQi/mgiDxkHTHhDJg4Vj4OnT4cU+sHIyuAGN/kuF0in07VSPEb/szMzbe3D/JW047biqJIVDLFi3k/vfX8Jp//iEix6fwjOfrmTd9n1BR9YRKnf22STXrkVuRgY7x44t8Li0pDR+cdwvADclincWnZIkSZKk+BEKQaPT4Mo34NdToE1fCCXBio/hhQvhmTNh4ZuQlxt0UhVDlcqkckXn+rx47UnM+NNZ3NunDac0qUI4BPO/3cE97y2m6/0f0+eJqTw7ZRXrd1h6xpNQcjKVrxwIQMaIEUQK8X98XNb8MsKhMF+s/4IV21fEKqJizKJTkiRJkhSfaraBX/wbfjcHOl8PyaXgu7nw2tXweCeY9Rxk7w86pYqpKmXT6H9SfUZedzJf/KkHf7u4NSc1qkwoBHO/2c7fxi2iy30fc+mTnzN86io27vRaigcV+15KuHRpMpd/zZ6pnxd4XO2ytTmz3pmAd3XGM4tOSZIkSVJ8q9QQzvu/6E7t3W6DUpUgYyWMuxkebgOf/RP2bQ86pYqxauXSGHhyA0b/vy588cez+OuFx3Niw0oAzFqzjb+MXcTJ903ksqen8fIX37AzK+DA+lFJ5cpR4dLoNPSMESMKNbZfi34AvLPiHXZl7SrybIo9i05JkiRJUmIoUxXO/BMMWQDn3g/l68KeTTDxbnioNXz4Z9i5PuiUKuaql09n0CkNee3XpzD9j2dx5wWt6FC/IpEIzFiVwV/GLeHO2UkMfG4mL3+xhq27C77pjY6NygMHQijEns8+I/Prrws8rnPNzjSp0IR9Oft4Z8U7MUyoWLHolCRJkiQllrSycPINcNM86PM0VG8FWbvg88eid3i+PRg2Lws6peJAzQrp/PLURoz5TVem/qE7fz6/JW3rlidCiOmrtnH7mwvofO9Ervz3F4ya8Q3b9nirZ3GQWq8e5XqcBUDGiBcKPC4UCuXf1TlqySjyInkxyafYseiUJEmSJCWmpBRodwXc8Dn0fxXqnwJ52TD3JRjWGUYNgLUzg06pOFGnYil+dVpj3vh/J3Nn+xxu7XkcbepUIDcvwpSvt/CHMV9x4j0fMei5Gbw6ay079mYHHblEq3z11QDseOcdcjIyCjyud5PelE0py+qdq5n+3fQYpVOsWHRKkiRJkhJbKATNesIv34drJ0Dz84EILBkHz/aA58+DZR9CIXZoVslWJR2uO7URY397KpN/fwa3ntucVrXKk5MXYfKyzdz6+pd0umcCvxw+kzdmf8vO/Zaex1qpDh1Ib92aSGYm20aNKvC40imluajpRYCbEsUji05JkiRJUslRrzP0GwmDZ8AJV0I4BdZMhZF94cmuMH805FpKqeAaVCnDb85oyns3ncbH/9ONoec0o0XNcmTnRvh4ySb+57X5dPrbR/xqxCzemruOXZaex0QoFKLyoEEAbBv5CnlZBV9W4IrmVwAw+dvJrN21Nib5FBsWnZIkSZKkkqdac7h4GNw0H7rcCKllYdNCePN6eLQ9TH8KsvYEnVJxpnG1stzY/TjGDzmdj245nZt7NOO46mXJys3jo8UbGTJ6Hh3//hH/78VZvDP/O/Zk5gQdOaGVP7cnyTVqkLtlCzvffa/A4xpWaEjX2l2JEOHVpa/GMKGKmkWnJEmSJKnkqlAHet4DNy+A7ndAmWqwYy2Mvy26U/sn98GerUGnVBxqWr0cN/U4jgm3dOPDm0/nd92b0rhqGbJy8vhg4UZ+98pcOvxtAje8NJt3v1zP3ixLz6IWSkmh0pUDAMgYPpxIIZanOLgp0ZjlY9iXsy8m+VT0LDolSZIkSSpVCU4fCkO+gvMfhEqNYF8GTL4fHm4N790K278JOqXiVLMa5bjlnOZM/J9uvH/TaQw+swkNq5QmMyeP9xdsYPDIOXT820cMHjmH8QvWsz87N+jICaPSZZcRKlWKzKVL2fvFFwUed2qdU6lTtg47s3by/qr3Y5hQRcmiU5IkSZKkg1JKwYnXwm9nw6XPQ612kL0XZjwNj5wAb1wHGxYEnVJxKhQK0bJWeX7fswWfDD2Dcb89lV93a0K9yqXYl53Lu1+u59cvzaHj3ybwu1fm8uHCDZaeRympQgUq9ukDQMbwEQUfF07Kv6tz5OKRhbobVMGx6JQkSZIk6b+Fk6D1JXD9ZBj4FjQ+AyK58NWr8FRXeOlSWD3Fndp1xEKhEK3rVOAPvVrw6e/P5J0bu/L/Tm9MnYql2JOVyzvzv+P6F2fT6e8fcfPoeUxcvJHMHEvPI1H5qoEQCrF70iQyV64q8LiLm15MelI6S7ctZe6muTFMqKJi0SlJkiRJ0o8JhaDJmXDV23D9JDi+D4TC8PUEGH4+/LsHLB4LeXlBJ1UcC4VCtK1bkT+e15Ipt53Jm785hV+d2ohaFdLZnZnDm3PXce2IWXT6+0f8z6vz+WTpJrJyvOYKKrVhQ8qecQYAGS++UOBxFdIqcH7j8wF4ZckrsYimImbRKUmSJElSQdRuD32HR6e1d/olJKXBulkw+koY1hnmvAA5mUGnVJwLhUK0r1+JP1/Qiqm3deeNG7pwTdeG1Cifxq79Obwx51uueX4mJ97zEbe9/iWfLttMdq6l58+pfPXVAOx48y1yt28v8LiD09c/WvMRm/ZuikEyFSWLTkmSJEmSCqNyY7jgoehO7af9D6RXgK3L4Z3fwiPtYOojsH9n0CmVAMLhEB0bVOau3scz7Q9n8er/68KgLg2oWjaNHfuyGT1rLVc9N4PO93zEH8d8xdSvt5Bj6XlYpTufSFrLlkT272fb6FcLPK555eZ0qN6BnEgOry17LYYJVRQsOiVJkiRJOhJlq8NZd8LNC+Gcv0O52rBrPUy4Ex5qDR/9BXZtDDqlEkQ4HKJzo8r89aLWfPGns3jlupO58uT6VCmTyra92bwy4xsG/PsLTr5vIn9+6yumrdhKbp5ryB4UCoWoPOgqALa9/DKRrKwCj+3XMnpX52tLXyM7Nzsm+VQ0LDolSZIkSToaaeXglN/CTfPhomFQtRlk7oApD8HDbWDsTbB1RdAplUCSwiG6NKnC3y9uwxd/OouXf3US/TrXp1LpFLbszuKl6d/Q75npnHzfRO56ewEzVmWQZ+lJhfPOI7laNXI2bWLnBx8UeNxZ9c+iWqlqbN2/lQlrJsQwoY6WRackSZIkSUUhORXaXwm/+QKueAXqnQS5mTB7ODzWEV69CtbNCTqlEkxyUpiuTaty3yVtmHF7D174ZWcu61SXCqVS2LwrkxHT1nDZ09Pocv9E/jp2IbPXlNzSM5SaSqUB/QHIeH44kUjB/jukhFPo27wvACOXjIxZPh09i05JkiRJkopSOAwtzoNrP4RrxkOzc4EILHobnjkTRvSGrydCAUsWqaBSksKc3qwa/7i0HTNv78Hz15zILzrUpVx6Mht3ZvL81NX84slpnPq/H/P3cYuY+822Apd9iaLi5ZcTSktj/6JF7Js1q8Dj+jbrS3I4mfmb57Nw68IYJtTRsOiUJEmSJClWGnSB/qPhhmnQrh+Ek2HVp/DSJfD06fDV65CbE3RKJaDU5DBnNq/OPy9rx6w/9+DZQZ3o074OZdOS+W7Hfv49ZRV9nvicU//3E+57bzFffru9RJSeyZUqUeHiiwHYOmJEgcdVLVWVcxqcA8CoJaNiEU1FwKJTkiRJkqRYq9EK+jwFv5sHJ90AKaVhw5fwxrXwWAeY8Qxk7ws6pRJUWnISZ7WswUOXn8CsP/fgXwM7cmG72pROTWLd9n08/elKLnx8Kt3+bxL/O34JC9btSOjS8+CmRLsnfkzWmjUFHtevRXRTovdWvse2/dtikk1Hx6JTkiRJkqRjpWI96HV/dKf2M2+H0lVg+xp4b2h0p/bJ/4C9GUGnVAJLT0ninONr8mi/9sy542yeurID57etRamUJL7J2MuTk1ZwwWNT6P7PyTzwwVIWr9+ZcKVnWuPGlDn9NIhEyHjxpQKPa1etHa2qtCIrL4sxy8fEMKGOlEWnJEmSJEnHWunK0O1WGLIAznsAKtaHvVvgk3uihef4P8GOb4NOqQSXnpLEua1rMax/B2bf0YNh/TvQq3VN0pLDrNqyh8c/+Zpej3xGjwcn8+CEZSzbuCvoyEWmytVXA7B9zBhyd+4s0JhQKJR/V+fopaPJzcuNVTwdIYtOSZIkSZKCkloaOl8Hv50Lv3gWarSB7D0wfRg80g7evAE2LQk6pUqA0qnJnN+2Fk9e2ZE5d5zNo/3a0/P4GqQmh1mxeQ+PTlzOOQ99yjkPTeaRj5bz9abdQUc+KqW7dCGtWTMie/ey/bXXCjzu3IbnUjGtIuv3rGfyt5NjmFBHwqJTkiRJkqSgJSVDm0vh15/BgDeg4WmQlwPzR8ITJ8HIK2DNtKBTqoQok5bMhe1q8/TATsz+cw8evvwEerSsQWpSmGUbd/PQR8vo8eBkzn34Ux7/eDmrtuwJOnKhhUKh/LU6M156mUh2doHGpSenc8lxlwAwcsnImOXTkbHolCRJkiSpuAiF4LgecPU4+NXH0PJCIATL3ofnz4Vnz4El70FeXtBJVUKUS0/h4vZ1+PegTsz8cw/+2bcd3VtUJyUpxJINu3jgw2Wc+cAkLn96Gjv2FawsLC7KX3ABSVWqkLN+PTs//LDA4y5vfjnhUJgv1n/Byu0rY5hQhWXRKUmSJElScVS3I1z+Itw4CzoMgqRUWPsFjOoHT3aBuS9DTlbQKVWCVCiVwi861uW5q09k1u1n849L29KtWTWSwyG+WJXBqzPXBh2xUMJpaVTqF11zM2PECwXedKl22dqcUfcMAF5Z8kqs4ukIWHRKkiRJklScVW0KFz4KQ76CrkMgrTxsXgJv/wYePQE+fxwyE2eTGMWHCqVTuKxTPUb8sjN39W4FwNgvvws4VeFV6ncFodRU9n/5JfvmzivwuH4towXpOyveYXdWfK9XmkgsOiVJkiRJigflasLZf4WbF0CPv0LZGrBzHXx4e3Sn9ol/g92bg06pEqhXm1okhUN8+e0OVsfZep3JVapQ/sLeAGQMH17gcSfVPInGFRqzN2cvb694O0bpVFgWnZIkSZIkxZP0CnDqkOgdnr0fhSpNYf92+OwBeLg1jLsFMlYFnVIlSNWyaZzSpAoA4+Lwrs7KV0U3Jdr10UdkffttgcaEQiH6tYje1TlqySjyIq6bWxxYdEqSJEmSFI+S06DjIBg8Ay57Eep0hJz9MOtZeKwDvHYNrJ8fdEqVEL3b1gZg3JfrA05SeOnNmlHmlFMgL49tL75U4HG9m/SmTEoZVu9czfT102OYUAVl0SlJkiRJUjwLJ0GrC+FXE2HQOGjaAyJ5sHAMPH06vHAxrJwEBdxoRToSPY+vmb8T+7KN8bdmbOVrrgZg++uvk7u7YGtulkkpw0VNLgLglcVuSlQcWHRKkiRJkpQIQiFodBpc+Qb8egq06QuhJFj5CbxwEfzrDFj4JuTlBp1UCahC6RS6NasGwLj58Td9vcypp5LapAl5e/aw/fXXCzzuihZXADD528l8u6tg094VOxadkiRJkiQlmppt4Bf/ht/Nhc7XQ3IpWD8PXrsaHu8Es56D7P1Bp1SC6d0uOn197JfricTZHcShUCh/rc5tL75EJCenQOMaVWjEKbVPIUKEV5e+GsuIKgCLTkmSJEmSElWlBnDe/0V3au92G5SqBBkrYdzN8HAb+OyfsG970CmVIHq0rEF6SphVW/aw8LudQccptAoXXUhSxYpkr1vHro8mFnjcwU2J3lj+Bvty9sUqngrAolOSJEmSpERXpiqc+ScYsgDOvR/K14U9m2Di3fBQa/jwz7Az/qYbq3gpk5bMWS1qADA2Dqevh9PTqdgvOhU9Y8SIAo87rc5p1Clbh51ZO3l/1fuxiqcCsOiUJEmSJKmkSCsLJ98AN82DPk9D9VaQtQs+fwwebgtvD4bNy4JOqTjWu10tILr7el5efE1fB6jUrx+kpLBv7lz2zZ9foDFJ4SSuaB4tSF9Z8krcTdtPJBadkiRJkiSVNEkp0O4KuOFz6P8q1D8F8rJh7kswrDOMGgBrZwadUnHojObVKZuWzLrt+5i7dlvQcQotpXp1Kpx/PlC4uzr7HNeHtKQ0lmQsYd7meTFKp59j0SlJkiRJUkkVCkGznvDL9+HaCdD8fCACS8bBsz3g+fNg2YfgHWoqoPSUJM5pdXD6+vqA0xyZylcPAmDnBx+S/V3BpuBXSKvA+Y2jBenIxSNjlk0/zaJTkiRJkiRBvc7QbyQMngEnXAnhFFgzFUb2hSe7wvzRkJsddErFgQu+N309Nw6nr6e3aEHpk06C3FwyXn65wOMObkr00ZqP2LR3U6zi6SdYdEqSJEmSpP+o1hwuHgY3zYcuN0JqWdi0EN68Hh5tD9Ofgqw9QadUMXZq02pUKJXClt2ZfLFya9BxjsjBuzq3v/oaeXsKdr23qNyCDtU7kBPJ4fVlr8cynn6ERackSZIkSfqhCnWg5z1w8wLofgeUqQY71sL426I7tX9yH+yJzxJLsZWaHKZX65oAjP0yPqevl+3WjdSGDcnbtYvtY94s8LiDd3W+tuw1sr0D+piz6JQkSZIkST+uVCU4fSgM+QrOfxAqNYJ9GTD5fnjoeHjvVtj+TdApVcz0blcbgPcXrCc7Ny/gNIUXCoepdNVAADJefJFIbm6Bxp3V4CyqlarGln1bmLBmQiwj6jAsOiVJkiRJ0s9LKQUnXgu/nQ2XPg+12kHOPpjxNDxyArxxHWxYEHRKFRMnN65C1bJpbN+bzZSvtwQd54hUvPhiwhUqkP3NN+z+5JMCjUkJp9C3eV8AXlnySizj6TAsOiVJkiRJUsGFk6D1JXD9ZBj4FjQ+AyK58NWr8FRXeOlSWD3FndpLuKRwiPPbHJi+Pr9gO5cXN+HSpal02WUAZAwfUeBxfZv1JTmczLzN81i0dVGs4ukwLDolSZIkSVLhhULQ5Ey46m24fhIc3wdCYfh6Agw/H/7dAxaPhbz4m7asonFw+vqHCzeyP7tgU7+Lm0pXDoDkZPbOmsW+BQsLNKZqqaqc3eBsAEYtGRXLePovFp2SJEmSJOno1G4PfYdHp7V3+iUkpcG6WTD6ShjWGea8ADmZQafUMdahfiVqV0hnd2YOk5ZuDjrOEUmpUYPyvXoBkDGi4Hd19m/RH4D3Vr3H9v3bYxFNh2HRKUmSJEmSikblxnDBQ9Gd2k/7H0ivAFuXwzu/hYfbwtRHYP/OoFPqGAmHQ1xw4K7OsV/G5/R1gMqDBgGw8/33yd64sUBj2lVrR8vKLcnMzWTM12NiGU/fY9EpSZIkSZKKVtnqcNadcPNCOOfvUK427N4AE+6Eh1rDR3+BXQUrjBTfereNFp0TF29kT2ZOwGmOTKnWx1OqU0fIyWHbyyMLNCYUCtGvRT8ARi8ZTW5efE7djzcWnZIkSZIkKTbSysEpv4Wb5sNFw6BqM8jcAVMegofbwNibYOuKoFMqhlrXKU/DKqXZn53HR4vjt9yucvXVAGwbPZq8vXsLNKZXo15UTKvId3u+Y/K3k2OYTgdZdEqSJEmSpNhKToX2V8JvvoArXoF6J0FuJsweDo91hFevgnWzg06pGAiFQlxw4K7OsfPXB5zmyJU980xS6tUjb8cOdrz9doHGpCenc8lxlwDwypJXYhlPB1h0SpIkSZKkYyMchhbnwbUfwjXjodm5QAQWvQ3PdIcRveHriRCJBJ1URejg7uufLtvMjn3ZAac5MqGkJCoPHAhAxogXiOTlFWjcZc0vIxwKM339dFZuXxnLiMKiU5IkSZIkBaFBF+g/Gm6YBu36QTgZVn0KL10CT58OX70OufG5pqMO1bxmOZrVKEtWbh4fLtwQdJwjVuGSSwiXK0fW6tXsnlywqeh1ytahW91uAIxaOiqW8YRFpyRJkiRJClKNVtDnKfjdPDjpBkgpDRu+hDeuhcc6wIxnIHtf0Cl1lA5uSjT2y/idvp5UtgwV+/YFond1FtTBTYne/vptdmftjkk2RVl0SpIkSZKk4FWsB73uj+7UfubtULoKbF8D7w2N7tQ++R+wNyPolDpCFxyYvj716y1s3Z0ZcJojV/nKAZCUxN7p09m/eHGBxpxc62QaVWjE3py9vLPinRgnLNksOiVJkiRJUvFRujJ0uxWGLIDzHoCK9WHvFvjknmjhOf6PsOPboFOqkBpVLUObOhXIzYvw/oL4nb6eUrs25XueAxT8rs5QKJR/V+crS14h4hq0MWPRKUmSJEmSip/U0tD5OvjtXPjFs1CjDWTvgelPwCPt4M1fw6aC3VGn4qF3u1oAjJ3/XcBJjk7lQYMA2PHuu2Rv2lSgMRc2uZAyKWVYvXM109ZPi2W8Es2iU5IkSZIkFV9JydDmUvj1Z3DlG9DwNMjLgfmvwBMnw8grYI3FUTw4/8A6nTNWZ7Bhx/6A0xy5Uu3aUap9e8jOZtsrrxRoTJmUMlzU5CIgelenYsOiU5IkSZIkFX+hEDTtAVePg199DC0vBEKw7H14/lx49hxY8h7k5QWdVD+iTsVSdGpQiUgE3v0qfjclgv/c1bl91Gjy9hestL28xeUATF47mXW718UsW0lm0SlJkiRJkuJL3Y5w+Ytw4yzoMAiSUmHtFzCqHzzZBea+DDlZQafUYfQ+sClRvE9fL9fjLFJq1yZ32zZ2vFOwDYYaV2hMl1pdiBBh9NLRMU5YMll0SpIkSZKk+FS1KVz4KAz5CroOgbTysHkJvP0bePQE+PwR3pTyAABAwklEQVRxyNwVdEp9T682NQmHYN7a7azN2Bt0nCMWSk6m0lUDgeimRAXdYOjgpkRjlo9hf078Tt8vriw6C2nv3r00aNCAoUOHBh1FkiRJkiQBlKsJZ/8Vbl4APf4KZWvCznXw4e3w0PEw8W+we3PQKQVUL5fOyY2rADD2y/i+q7PipZcSLlOGrBUr2DNlSoHGnF73dOqUrcOOzB28v+r9GCcseSw6C+mee+7h5JNPDjqGJEmSJEn6b+kV4NQhMORL6P0oVGkK+3fAZw/Aw61h3C2QsSrolCXewenr4+bH9zqdSWXLUvHSXwCQMXxEwcaEk7i8eXStzpFLRhb4TlAVjEVnISxfvpwlS5bQq1evoKNIkiRJkqQfk5wGHQfB4Blw2YtQpyPk7IdZz8JjHeC1a2D9/KBTlljnHl+T5HCIRet38vWm3UHHOSqVBg6EcJg9U6eyf9myAo3p07QPaUlpLMlYwvzNXodFKfCi8y9/+QuhUOiQrxYtWhTp7/j000/p3bs3tWvXJhQK8dZbbx32vGHDhtGwYUPS09M56aSTmDFjxiHPDx06lPvuu69Is0mSJEmSpBgJJ0GrC+FXE2HQuOiu7ZE8WDgGnj4dXrgYVk4C76o7piqVSeW046oCMC7Op6+n1q1LuR49AMh44YUCjamYXpHzGp0HwMjFI2OWrSQKvOgEOP7441m/fn3+15SfWNdg6tSpZGdn/+D4okWL2Lhx42HH7Nmzh3bt2jFs2LAffd3Ro0dzyy23cNdddzFnzhzatWtHz5492bRpEwBvv/02zZo1o1mzZoV8d5IkSZIkKVChEDQ6Da58A349Bdr0hVASrPwEXrgI/nUGLHwT8nKDTlpifH/39Xifvl356kEA7HxnLDlbtxZozMFNiSasmcDmva4fW1SKRdGZnJxMzZo187+qVq162PPy8vIYPHgw/fv3Jzf3P398li5dSvfu3Rkx4vDrIfTq1Yu///3v9OnT50czPPjgg1x33XVcc801tGrViqeeeorSpUvz3HPPATB9+nRGjRpFw4YNGTp0KM888wx33333UbxrSZIkSZJ0zNVsA7/4N/xuLnS+HpJLwfp58NrV8HgnmPUcZLsbdqyd3aoGqclhVmzew+L1u4KOc1RKtW9Petu2RLKy2PbKqAKNaVmlJe2rtycnksPry16PccKSo1gUncuXL6d27do0btyYAQMG8M033xz2vHA4zHvvvcfcuXO56qqryMvLY8WKFXTv3p2LL76YW2+99Yh+f1ZWFrNnz6bHgVuND/6uHj16MG3aNADuu+8+1q5dy+rVq3nggQe47rrruPPOOw/7esOGDaNVq1aceOKJR5RHkiRJkiTFWKUGcN7/RXdq73YblKoEGSth3M3wcBv47J+wb3vQKRNWufQUujevDsT/7uuhUIjKg64CYNsrr5CXmVmgcQfv6nx12atk5/5w9rIKL/Ci86STTmL48OGMHz+eJ598klWrVnHaaaexa9fh2/zatWvz8ccfM2XKFPr370/37t3p0aMHTz755BFn2LJlC7m5udSoUeOQ4zVq1GDDhg2Ffr3BgwezaNEiZs6cecSZJEmSJEnSMVCmKpz5JxiyAM69H8rXhT2bYOLd8FBr+PDPsDO+i7jiKpGmr5c/5xySa9Ykd+tWdo57t0BjetTvQbVS1diybwsfffNRjBOWDIEXnb169aJv3760bduWnj178t5777F9+3ZeffXVHx1Tv359XnzxRUaPHk1ycjLPPvssoVDomGW++uqreeCBB47Z75MkSZIkSTGWVhZOvgFumgd9nobqrSBrF3z+GDzcFt4eDJsLtqu2CqZ7i+qUTk3i2237mLd2e9BxjkooJYXKA68EIGPEiAIVtylJKfRt1heAV5a8EtN8JUXgRed/q1ixIs2aNePrr7/+0XM2btzI9ddfT+/evdm7dy8333zzUf3OqlWrkpSU9IPNjDZu3EjNmjWP6rUlSZIkSVIcSUqBdlfADZ9D/1eh/imQlw1zX4JhnUl67Soq7fnxzkIFVyo1iR4to7Nrx85fH3Cao1exb19CpUuTuWwZew8shfhzLm12KcmhZOZumsvirYtjnDDxFbuic/fu3axYsYJatWod9vktW7Zw1lln0bJlS8aMGcPEiRMZPXo0Q4cOPeLfmZqaSseOHZk4cWL+sby8PCZOnEiXLl2O+HUlSZIkSVKcCoWgWU/45ftw7QRofj4QIbzsPU5fdjdJL/aGZR9CnE+5DtrB6evvfvUdeXnx/d8yqXx5Kh7YCHvrj2yY/d+qla7G2Q3OBryrsygEXnQOHTqUyZMns3r1aj7//HP69OlDUlIS/fr1+8G5eXl59OrViwYNGuRPW2/VqhUTJkzg+eef56GHHjrs79i9ezfz5s1j3rx5AKxatYp58+YdsunRLbfcwjPPPMOIESNYvHgxN9xwA3v27OGaa66JyfuWJEmSJElxol5n6DcSBs8gr21/8kJJhL+ZBiP7wpNdYf5ocDOZI3J6s6qUS09m485MZq7OCDrOUat81UAIhdgz+VMyV6wo0Jj+LfsD8N6q99i+f3sM0yW+wIvOb7/9ln79+tG8eXMuu+wyqlSpwvTp06lWrdoPzg2Hw9x777288cYbpKam5h9v164dH330EX379j3s75g1axbt27enffv2QLTUbN++/SG7pl9++eU88MAD3HnnnZxwwgnMmzeP8ePH/2CDIkmSJEmSVEJVa05u70eZ0Oqf5J70G0gtC5sWwpvXw6PtYfpTkLUn6JRxJS05iXOPjy4bGO+7rwOkNmhA2e7dAch44cUCjWlXrR0tK7ckMzeTN79+M5bxEl7gReeoUaP47rvvyMzM5Ntvv2XUqFE0adLkR88/++yzSU9P/8Hx9u3bU7du3cOOOeOMM4hEIj/4Gj58+CHn3XjjjaxZs4bMzEy++OILTjrppKN6b5IkSZIkKfHsT61MXo+74eYF0P0OKFMNdqyF8bdFd2r/5D7YszXomHHj4PT1977aQE5uXsBpjl7lQVcBsOPtt8nZtu1nzw+FQvRrEZ3ZPHrpaHLzcmOaL5EFXnRKkiRJkiTFpVKV4PShMOQrOP9BqNQI9mXA5PvhoePhvVth25qgUxZ7pzSpQuUyqWTsyeLzFfFfEJc+8UTSW7Uisn8/20ePLtCYXo16USGtAut2r+PTbz+NccLEZdEpSZIkSZJ0NFJKwYnXwm9nw6XPQ612kLMPZjwdndL+xnWwYUHQKYut5KQw57U5MH19fvxPXw+FQlS+ehAA214eSSQr62fHpCenc8lxlwBuSnQ0LDolSZIkSZKKQjgJWl8C10+GgW9B4zMgkgtfvQpPdYWXLoXVU9yp/TB6t41OXx+/cAOZOfE/dbv8ueeSXK0aOZs3s/P99ws05vLmlxMixLT101i5Y2WMEyYmi05JkiRJkqSiFApBkzPhqrfh+klwfB8IheHrCTD8fPh3D1g8FvLifz3KonJiw8rUKJ/Grv05fLpsS9BxjlooNZVKV14JwNbhI4gUoNyuU7YO3ep1A2DUklExzZeoLDolSZIkSZJipXZ76Ds8Oq290y8hKQ3WzYLRV8KwzjDnBcjJDDpl4MLhEBccuKszEaavA1S6/DJC6elkLl7M3hkzCzSmf4v+ALz99dvsztody3gJyaJTkiRJkiQp1io3hgseiu7UftpQSK8AW5fDO7+Fh9vClIdh/86gUwbqgra1AJiwaCN7s3ICTnP0kipWpMLFFwGQMWJEgcacXOtkGpZvyN6cvYxdOTaW8RKSRackSZIkSdKxUrY6nHUH3LwQzrkHytWG3Rvgo7uiO7V/9BfYtTHolIE4oV5F6lYqxb7sXD5esinoOEWi8lXRTYl2f/IJWatX/+z5oVCIfi36AdFNiQoy5V3/YdEpSZIkSZJ0rKWVg1NuhJvmw0XDoGozyNwJUx6Ch9vA2Jtg64qgUx5ToVCI3u0Sa/p6WuNGlO3WDSIRMl54sUBjLmxyIaWTS7Nqxyqmr58e44SJxaJTkiRJkiQpKMmp0P5K+M0XcMUrUO8kyM2E2cPhsY7w6lWwbnbQKY+Zg7uvf7J0M7v2ZwecpmhUvjp6V+f2N98kd/v2nz2/bGpZLmoanfL+ypJXYhkt4RRJ0blmzRoWLVpEnruFSZIkSZIkFV44DC3Og2s/hGvGQ7NzgQgsehue6Q7DL4CvP4IEn8rcslY5mlQrQ1ZOHhMWJcYU/tInn0xa8+ZE9u1j22uvFWjMFS2uAGDyt5NZt3tdLOMllEIVnc899xwPPvjgIceuv/56GjduTJs2bWjdujVr164t0oCSJEmSJEklSoMu0H803DAN2vWDcDKs/gxe+gU8fRp89Trkxv9mPYeTiNPXQ6EQlQdF7+rc9tLLRLJ//k7VxhUac3Ktk8mL5DF66ehYR0wYhSo6//Wvf1GpUqX8n8ePH8/zzz/PCy+8wMyZM6lYsSJ//etfizykJEmSJElSiVOjFfR5Cn43D07+DaSUgQ1fwRvXwmMdYMYzkLU36JRF7oID09c/W76FbXuyAk5TNMpfcD5JVauSs3EjO8d/UKAx/Vv0B2DM8jHsz9kfy3gJo1BF5/Lly+nUqVP+z2+//TYXXXQRAwYMoEOHDtx7771MnDixyENKkiRJkiSVWBXrwbn3wc0L4MzboXQV2L4G3hsa3bho8j9gb0bQKYtM0+plaVWrPDl5EcYv3BB0nCIRTk2lUv/obuoZI0YUaDf10+ueTu0ytdmRuYP3V70f64gJoVBF5759+yhfvnz+z59//jmnn356/s+NGzdmw4bEuAAlSZIkSZKKldKVodutMGQBnPcAVKwPe7fAJ/fAQ61h/B9hx7dBpywSiTZ9HaDSFVcQSk1l/4IF7Jsz52fPTwoncXmLy4HopkQFKUdLukIVnQ0aNGD27OhOX1u2bGHhwoV07do1//kNGzZQoUKFok0oSZIkSZKk/0gtDZ2vg9/OhV88CzXaQPYemP4EPNIO3vw1bFocdMqjckHbWgBMW7mVTTsTY9p2cuXKVLjoQgAyhg8v0JhLml5CWlIaizMWM3/z/BimSwyFKjoHDRrE4MGD+dvf/kbfvn1p0aIFHTt2zH/+888/p3Xr1kUeUpIkSZIkSf8lKRnaXAq//gyufAMangZ5OTD/FXjiZBh5OayZFnTKI1Kvcmna169IJALvfbU+6DhFpvJVVwGw66OJZBVgQ++K6RU5r9F5AIxcMjKm2RJBoYrOW2+9leuuu44xY8aQnp7Oa6+9dsjzU6dOpV+/fkUaUJIkSZIkST8hFIKmPeDqcfCrj6HlhUAIlo2H58+FZ8+BJe9BXl7QSQvl4KZEY79MnKIz7bjjKHPqqRCJkPHiiwUac0WLKwCYsHoCm/dujmW8uFeoojMcDnP33Xczd+5c3n//fVq2bHnI86+99hrXXnttkQaUJEmSJElSAdXtCJe/CDfOgg6DICkV1n4Bo/rBk11g7suQEx87mZ/fphahEMxes4112/cFHafIVB40CIAdr79B7q5dP3t+qyqtOKHaCeREcnh92euxjhfXClV0AowePZoBAwbQt29fnnrqqVhkkiRJkiRJ0tGo2hQufBSGfAVdh0Baedi8BN7+DTx6Anz+OGT+fMkWpJoV0uncsDIA736ZOJsSlTm1K6lNm5C3dy/bXytYcdm/ZX/qlq1LzTI1Y5wuvhWq6HzyySfp168fs2bNYvny5QwePJjf//73scomSZIkSZKko1GuJpz9V7h5AfT4K5StCTvXwYe3w0PHw8S/we7iOx36P7uvJ8709VAolH9XZ8ZLLxLJyfnZMec0OIdxfcbR57g+sY4X1wpVdD7++OPcddddLF26lHnz5jFixAieeOKJWGWTJEmSJElSUUivAKcOgSFfQu9HoUpT2L8DPnsAHm4N426BjFVBp/yBXq1rkhQO8dW6HazasifoOEWmQu/eJFWqRM5369k1YcLPnp8UTiIpnHQMksW3QhWdK1euZNCBxhmgf//+5OTksH594rTqkiRJkiRJCSs5DToOgsEz4LIXoU5HyNkPs56FxzrAa9fA+vlBp8xXpWwaXZtWBWDc/MSZvh5OT6fSgQ29M4aPCDhN4ihU0ZmZmUmZMmX+MzgcJjU1lX37EmdBWEmSJEmSpIQXToJWF8KvJsKgcdFd2yN5sHAMPH06vHAxrJwEkUjQSendthYAYxNonU6ASv37EUpJYd/8+exbsDDoOAkhubAD7rjjDkqXLp3/c1ZWFvfccw8VKlTIP/bggw8WTTpJkiRJkiTFTigEjU6Lfm34CqY+AgvGwMpPol+1TohOeW95YbQcDcA5x9fk9jcXsGzjbpZu2EXzmuUCyVHUkqtWpfrvf09a0yakH98q6DgJoVBF5+mnn87SpUsPOXbKKaewcuXK/J9DoVDRJJMkSZIkSdKxU7MN/OLf0P0OmPY4zHkR1s+D166Gyo3hlN9Cu/6Qkn5MY1UolUK35tWYsGgjY+d/R/OazY/p74+lylcNDDpCQilU0Tlp0qRDft6yZQupqamUL1++KDNJkiRJkiQpKJUawHn/B91ugxn/in5lrIRxN8Mn98HJv4ZO10KpiscsUu92taNF55ff8T/nNPNGOx1WodboBNi+fTuDBw+matWq1KhRg0qVKlGzZk3++Mc/snfv3lhklCRJkiRJ0rFWpiqc+ScYsgDOvR/K14U9m2Di3fBQa/jwz7Dz2KybeVaL6qSnhFmzdS9frdtxTH6n4k+h7ujMyMigS5curFu3jgEDBtCyZUsAFi1axGOPPcaECROYMmUKX375JdOnT+d3v/tdTEJLkiRJkiTpGEkrCyffACf+Cha8EV3Hc9Mi+PwxmP4UtLscTrkJqjWLWYQyacmc1bIG7365nrHzv6Nt3Yox+12KX4W6o/Puu+8mNTWVFStW8PTTTzNkyBCGDBnCv/71L77++muysrIYOHAgZ5999iGbE0mSJEmSJCnOJaVAuyvghs+h/6tQ/xTIy4a5L8GwzjBqAKydGbNf37ttbQDGfbmevLzgd4NX8VOoovOtt97igQceoEaNGj94rmbNmvzjH//gjTfe4JZbbmHQoEFFFlKSJEmSJEnFRCgEzXrCL9+HaydA8/OBCCwZB8/2gOfPg2UfQqRoy8gzmlejbFoy63fsZ84324r0tZUYClV0rl+/nuOPP/5Hn2/dujXhcJi77rrrqINJkiRJkiSpmKvXGfqNhMEz4IQrIZwCa6bCyL7wZFeYPxpys4vkV6WnJHHO8dGb78bOPzZrgyq+FKrorFq1KqtXr/7R51etWkX16tWPNpMkSZIkSZLiSbXmcPEwuGk+dLkRUsvCpoXw5vXwaPvoWp5Ze4761/RuF52+/u5X68nJzTvq11NiKVTR2bNnT26//XaysrJ+8FxmZiZ33HEH5557bpGFkyRJkiRJUhypUAd63gM3L4Dud0CZarBjLYy/LbpT+yf3wZ6tR/zypzatSsXSKWzZncUXqzKKMLgSQaF2Xb/77rvp1KkTxx13HIMHD6ZFixZEIhEWL17ME088QWZmJi+88EKsskqSJEmSJCkelKoEpw+FLoNh3sjoDu3bVsHk+6O7tne4KvpcpQaFetmUpDC9WtfilRnfMHb+d3RtWjVGb0DxqFB3dNatW5dp06bRqlUr/vjHP3LxxRfTp08fbr/9dlq1asXUqVOpX79+rLJKkiRJkiQpnqSUghOvhd/Ohkufh1rtIGcfzHg6OqX9jV/BhgWFesne7WoB8P6CDWTlOH1d/1GoOzoBGjVqxPvvv8+2bdtYvnw5AE2bNqVy5cpFHk6SJEmSJEkJIJwErS+B4/vAykkw9eHo969ei3417QFdh0DDU6O7uv+EkxpVoVq5NDbvymTK15vp3qLGMXgDigeFuqPz+ypVqkTnzp3p3LmzJackSZIkSZJ+XigETc6Eq96G6ydFi89QGL7+CEZcAP8+CxaPhbwfv1MzKRzi/DbRuzrHzl9/jIIrHhxx0SlJkiRJkiQdsdrtoe/w6LT2Tr+EpDRYNxtGXwnDOsOcFyAn87BDD05f/3DhBvZn5x7D0CrOLDolSZIkSZIUnMqN4YKHoju1nzYU0ivA1uXwzm/h4bYw5WHYv/OQIe3rVaJOxVLsycrlkyWbgsmtYseiU5IkSZIkScErWx3OugNuXgjn3APlasPuDfDRXfDQ8TDhLti1AYBwOMQFbaN3dY770unrirLolCRJkiRJUvGRVg5OuRFumg8XDYOqzSBzZ3QDo4fbwDu/g60r6N2uNgATl2xkd2ZOsJlVLFh0SpIkSZIkqfhJToX2V8JvvoArXoF6J0FuFswZAY915PgpN3Jupe/Yn53HxMUbg06rYsCiU5IkSZIkScVXOAwtzoNrP4RrxkOzc4EIocXv8NS+oYxM+Tsrp70NkUjQSRUwi05JkiRJkiTFhwZdoP9ouGEatOtHJJzMKUmLuHnjH8l98jT46nXIdRp7SWXRKUmSJEmSpPhSoxX0eYrQ7+YxJu1C9kTSSNr0FbxxLTzWAWY8A1l7g06pY8yiU5IkSZIkSfGpYj3Wn3wXXTMf5fXyV0HpKrB9Dbw3FB5pC5sWB51Qx5BFpyRJkiRJkuLWBW1rsZ1y3Lr5XLZcNxvOewAq1IM9m2H6E0HH0zFk0SlJkiRJkqS41aBKGdrWrUBeBN5fsgM6XwfnPxh9cuXkYMPpmLLolCRJkiRJUlzr3bY2AGPnr48eaHAKhJOj09gzVgWYTMeSRackSZIkSZLi2vltawEwY3UG63fsg7SyUPfE6JOrvKuzpLDolCRJkiRJUlyrXbEUJzasBMC7Xx64q7NRt+j3lZOCCaVjzqJTkiRJkiRJca93uwPT1w8WnY3PiH5f9Snk5QUTSseURackSZIkSZLiXq/WtQiHYP7a7XyzdS/U6QgpZWDvVti4IOh4OgYsOiVJkiRJkhT3qpVL45QmVQEY++V3kJwKDbtGn3SdzhLBolOSJEmSJEkJoXe76KZEY+d/Fz3gOp0likWnJEmSJEmSEkLP42uSkhRiyYZdLN+46z/rdK75HHKyAs2m2LPolCRJkiRJUkKoWDqV04+rBhzYlKh6KyhdFbL3wrczA06nWLPolCRJkiRJUsK44MD09XHzvyMSCkHjA9PXXacz4Vl0SpIkSZIkKWH0aFmDtOQwK7fsYeF3O7+3TqdFZ6Kz6JQkSZIkSVLCKJeeQvcW1YEDu68fXKdz3SzI3BVcMMWcRackSZIkSZISSu92tQEYN389kYr1oVJDyMuJbkqkhGXRKUmSJEmSpIRyZvPqlElNYt32fcxdu/0/d3WunBRgKsWaRackSZIkSZISSqnUJM5uVQOAsfO/c53OEsKiU5IkSZIkSQnn4PT1d79cT27D06MHNy2E3ZsCTKVYsuiUJEmSJElSwjntuGqUT09m065MZmwMQc020SdWfRpsMMWMRackSZIkSZISTmpymF6tawEHdl/Pn77+SYCpFEsWnZIkSZIkSUpIB6evv//VenIafm+dzkgkwFSKFYtOSZIkSZIkJaSTG1emSplUtu3N5vOcZhBOgR1rIWNl0NEUAxadkiRJkiRJSkjJSWHOaxOdvv72wh1Qr3P0iVXuvp6ILDolSZIkSZKUsA5OX/9w4QayG5wWPbhyUnCBFDMWnZIkSZIkSUpYnRpUomb5dHZl5jAnqV304KpPIS8v2GAqchadkiRJkiRJSljhcIgL2kanr49cVw1Sy8K+bbDhy4CTqahZdEqSJEmSJCmh5U9fX7yV3PpdowddpzPhWHRKkiRJkiQpobWtW4H6lUuzLzuXxaXaRw+6TmfCseiUJEmSJElSQguFQvRuF52+/vq2ptGDa6ZBTmaAqVTULDolSZIkSZKU8A5OXx+5qgx5ZapDzj74dmbAqVSULDolSZIkSZKU8JrXKMdx1cuSlRvh24onRg86fT2hWHRKkiRJkiQp4YVCIS5oe2BTon0togdXuiFRIrHolCRJkiRJUolwwYF1OodvaBg9sG427N8ZXCAVKYtOSZIkSZIklQhNqpXl+Nrl+TavCjtL14dILqyZGnQsFRGLTkmSJEmSJJUYBzcl+oI20QOu05kwLDolSZIkSZJUYpzfJjp9/c0dTaMHXKczYVh0SpIkSZIkqcSoV7k0HepX5PPcVkQIwebFsGtD0LFUBCw6JUmSJEmSVKL0bleb7ZRjZXKT6IFVnwYbSEXColOSJEmSJEklyvltahEKwYT9LaIHXKczIVh0SpIkSZIkqUSpXj6dkxtVYWpe6+iBlZMhEgk2lI6aRackSZIkSZJKnN7tajMzrznZJMPOb2HriqAj6ShZdEqSJEmSJKnEObd1TbLD6czKbRY9sGpSoHl09Cw6JUmSJEmSVOJULpPKqU2rMjXv+OgB1+mMexadkiRJkiRJKpF6t6udv05nZNVnkJcbcCIdDYtOSZIkSZIklUjnHF+DJeGm7IyUIrR/O6yfH3QkHQWLTkmSJEmSJJVI5dNTOK15Tb7IaxU9sGpysIF0VCw6JUmSJEmSVGJFp69H1+mMrLTojGcWnZIkSZIkSSqxzmpZnVnhdgBE1nwO2fsDTqQjZdEpSZIkSZKkEqt0ajKNWnZgY6Qi4dxM+HZG0JF0hCw6JUmSJEmSVKIdsvv6iknBhtERs+iUJEmSJElSidateTXmJLUFYM+SiQGn0ZGy6JQkSZIkSVKJlpacRFqzMwEoveVL2Lc92EA6IhadkiRJkiRJKvFO63gCK/JqESaP3FVTgo6jI2DRKUmSJEmSpBKva9OqzA63AWDjvPEBp9GRsOiUJEmSJElSiZeSFCa7QTcAktZ8GnAaHQmLTkmSJEmSJAlodlIv8iIhamSuISvj26DjqJAsOiVJkiRJkiSgQ4vGLAk3BmD5F+8GnEaFZdEpSZIkSZIkAUnhEBnVuwCwZ/HEgNOosCw6JUmSJEmSpANqnHAuAPV3zGRfZk7AaVQYFp2SJEmSJEnSAU07nkUmKdQMZTBj1hdBx1EhWHRKkiRJkiRJB4RSS7O+fDsA1s8dH3AaFYZFpyRJkiRJkvQ9pZp3B6DKpmns2p8dcBoVlEWnJEmSJEmS9D3V2/UEoHNoIR8t+i7gNCooi05JkiRJkiTpe0K127M/qSwVQnv5auanQcdRAVl0SpIkSZIkSd8XTiKnflcASq/9jO17swIOpIKw6JQkSZIkSZL+S9kWPQA4ObSA8Qs2BJxGBWHRKUmSJEmSJP23xmcAcGLSMnKz9gabRQVi0SlJkiRJkiT9t6rHESlbizSyGVDbOzrjgUWnJEmSJEmS9N9CIUJNzog+XjkpyCQqIItOSZIkSZIk6XAadYt+Xzk52BwqEItOSZIkSZIk6XAaHyg6v5sL+7YFm0U/y6JTkiRJkiRJOpzytaFqMyACq6cEnUY/w6JTkiRJkiRJ+jH509cnBRpDP8+iU5IkSZIkSfoxjc+IfnedzmLPolOSJEmSJEn6MQ1PhVAYti6HHeuCTqOfYNEpSZIkSZIk/ZhSFaF2++jjVd7VWZxZdEqSJEmSJEk/xXU644JFpyRJkiRJkvRTvr9OZyQSaBT9OItOSZIkSZIk6afUOwmS02H3Bti8NOg0+hEWnZIkSZIkSdJPSUmH+idHH7tOZ7Fl0SlJkiRJkiT9nPx1Oi06iyuLTkmSJEmSJOnnHFync/VnkJsTaBQdnkWnJEmSJEmS9HNqtYP0CpC5E9bPCzqNDsOiU5IkSZIkSfo54SRodHr08cpPgs2iw7LolCRJkiRJkgrCdTqLNYtOSZIkSZIkqSAanxn9vvYLyNobbBb9gEWnJEmSJEmSVBBVmkD5OpCbBWunB51G/8WiU5IkSZIkSSqIUOh709cnBRpFP2TRKUmSJEmSJBVU4zOi312ns9ix6JQkSZIkSZIK6uDO6+vnw96MYLPoEBadkiRJkiRJUkGVrwXVWgARWP1Z0Gn0PRadkiRJkiRJUmG4TmexZNEpSZIkSZIkFYbrdBZLFp2SJEmSJElSYTTsCqEwZKyA7WuDTqMDLDolSZIkSZKkwkivAHU6Rh+v8q7O4sKiU5IkSZIkSSos1+ksdiw6JUmSJEmSpML6/jqdkUigURRl0SlJkiRJkiQVVr3OkFwK9myCTYuDTiMsOiVJkiRJkqTCS06DBl2ij12ns1iw6JQkSZIkSZKORP46nRadxYFFpyRJkiRJknQkDq7TuXoK5OYEGkUWnZIkSZIkSdKRqdkWSlWCrF3w3Zyg05R4Fp2SJEmSJEnSkQiHodHp0ccrJwUaRRadkiRJkiRJ0pFznc5iw6JTkiRJkiRJOlIH1+lc+wVk7Qk0Skln0SlJkiRJkiQdqcqNoUI9yMuGb6YFnaZEs+iUJEmSJEmSjlQo9L3p65MCjVLSWXRKkiRJkiRJR+Pg9HXX6QyURackSZIkSZJ0NA7uvL7hS9izNdgsJZhFpyRJkiRJknQ0ytWA6q2ij1d/GmyWEsyiU5IkSZIkSTpartMZOItOSZIkSZIk6Wi5TmfgLDolSZIkSZKko9XgFAglwbZVsG1N0GlKJItOSZIkSZIk6Will4e6naKPV3lXZxAsOiVJkiRJkqSi4DqdgbLolCRJkiRJkorCwXU6V30KkUigUUoii05JkiRJkiSpKNQ9EVJKw57NsGlR0GlKHItOSZIkSZIkqSgkp0Y3JQKnrwfAolOSJEmSJEkqKvnrdLoh0bFm0SlJkiRJkiQVlYPrdK6ZCrnZgUYpaSw6JUmSJEmSpKJSozWUrgJZu2Hd7KDTlCgWnZIkSZIkSVJRCYeh4WnRx67TeUxZdEqSJEmSJElF6eD0ddfpPKYsOiVJkiRJkqSi1PjAhkTfzoDM3cFmKUEsOiVJkiRJkqSiVKkRVKwPeTnwzbSg05QYFp2SJEmSJElSUQqFoNGBuzpdp/OYseiUJEmSJEmSiprrdB5zFp2SJEmSJElSUTt4R+fGr2D35mCzlBAWnZIkSZIkSVJRK1sNarSOPl79abBZSgiLTkmSJEmSJCkWXKfzmLLolCRJkiRJkmLBdTqPKYtOSZIkSZIkKRYanALhZNi+BjJWBZ0m4Vl0SpIkSZIkSbGQVhbqnhh9vMq7OmPNolOSJEmSJEmKlfx1Oi06Y82iU5IkSZIkSYqVg+t0rpoMeXmBRkl0Fp2SJEmSJElSrNTpCCllYO9W2LQw6DQJzaJTkiRJkiRJipXkVGjYNfp45aRAoyQ6i05JkiRJkiQpllyn85iw6JQkSZIkSZJiqfGBonPNVMjJCjZLArPolCRJkiRJkmKp+vFQuipk74V1s4JOk7AsOiVJkiRJkqRYCoeh0enRx67TGTMWnZIkSZIkSVKsNT4j+t11OmPGolOSJEmSJEmKtYPrdK6bBZm7gs2SoCw6JUmSJEmSpFir1DD6lZcDaz4POk1CsuiUJEmSJEmSjoVGB+7qdJ3OmLDoLKS9e/fSoEEDhg4dGnQUSZIkSZIkxRPX6Ywpi85Cuueeezj55JODjiFJkiRJkqR4c3Dn9U0LYfemYLMkIIvOQli+fDlLliyhV69eQUeRJEmSJElSvClTFWq2iT5e9WmwWRJQsSo677//fkKhEEOGDCnS1/3000/p3bs3tWvXJhQK8dZbbx32vGHDhtGwYUPS09M56aSTmDFjxiHPDx06lPvuu69Is0mSJEmSJKkEyV+n85NgcySgYlN0zpw5k6effpq2bdv+5HlTp04lOzv7B8cXLVrExo0bDztmz549tGvXjmHDhv3o644ePZpbbrmFu+66izlz5tCuXTt69uzJpk3R24jffvttmjVrRrNmzQrxriRJkiRJkqTvaXxm9PvKTyESCTZLgikWRefu3bsZMGAAzzzzDJUqVfrR8/Ly8hg8eDD9+/cnNzc3//jSpUvp3r07I0aMOOy4Xr168fe//50+ffr86Gs/+OCDXHfddVxzzTW0atWKp556itKlS/Pcc88BMH36dEaNGkXDhg0ZOnQozzzzDHffffcRvmNJkiRJkiSVSA26QDgFdnwD21YFnSahFIuic/DgwZx//vn06NHjJ88Lh8O89957zJ07l6uuuoq8vDxWrFhB9+7dufjii7n11luP6PdnZWUxe/bsQ35/OBymR48eTJs2DYD77ruPtWvXsnr1ah544AGuu+467rzzzsO+3rBhw2jVqhUnnnjiEeWRJEmSJElSgkotA/U6Rx+vnBRolEQTeNE5atQo5syZU+C1L2vXrs3HH3/MlClT6N+/P927d6dHjx48+eSTR5xhy5Yt5ObmUqNGjUOO16hRgw0bNhT69QYPHsyiRYuYOXPmEWeSJEmSJElSgspfp3NysDkSTHKQv3zt2rXcdNNNTJgwgfT09AKPq1+/Pi+++CLdunWjcePGPPvss4RCoRgmPdTVV199zH6XJEmSJEmSEkzjM2DSvdGd1/PyIBz4vYgJIdD/irNnz2bTpk106NCB5ORkkpOTmTx5Mo8++ijJycmHrMP5fRs3buT666+nd+/e7N27l5tvvvmoclStWpWkpKQfbGa0ceNGataseVSvLUmSJEmSJB2iTgdILQv7MmDjV0GnSRiBFp1nnXUWX331FfPmzcv/6tSpEwMGDGDevHkkJSX9YMyWLVs466yzaNmyJWPGjGHixImMHj2aoUOHHnGO1NRUOnbsyMSJE/OP5eXlMXHiRLp06XLErytJkiRJkiT9QFIKNOgafew6nUUm0Knr5cqVo3Xr1occK1OmDFWqVPnBcYiWj7169aJBgwaMHj2a5ORkWrVqxYQJE+jevTt16tQ57N2du3fv5uuvv87/edWqVcybN4/KlStTv359AG655RYGDRpEp06d6Ny5Mw8//DB79uzhmmuuKeJ3LUmSJEmSpBKv8Rmw/IPoOp1dbwo6TUIItOgsrHA4zL333stpp51Gampq/vF27drx0UcfUa1atcOOmzVrFmeeeWb+z7fccgsAgwYNYvjw4QBcfvnlbN68mTvvvJMNGzZwwgknMH78+B9sUCRJkiRJkiQdtcYHNiRa8znkZEJyWrB5EkCxKzonTZr0k8+fffbZhz3evn37Hx1zxhlnEIlEfvZ333jjjdx4440/e54kSZIkSZJ0VKq3gjLVYM9m+HYmNDw16ERxzy2dJEmSJEmSpGMtFIJGB+7qdJ3OImHRKUmSJEmSJAWh8RnR7ysnBxojUVh0SpIkSZIkSUE4uE7nutmwf2ewWRKARackSZIkSZIUhIr1oXJjiOTCmqlBp4l7Fp2SJEmSJElSUFyns8hYdEqSJEmSJElBcZ3OImPRKUmSJEmSJAWl0elACDYvhl0bgk4T1yw6JUmSJEmSpKCUrgy12kYfr/o02CxxzqJTkiRJkiRJClL+Op1OXz8aFp2SJEmSJElSkPLX6ZwEkUiQSeKaRackSZIkSZIUpPpdICkVdn4LGSuDThO3LDolSZIkSZKkIKWWhnonRR+v/CTYLHHMolOSJEmSJEkKmut0HjWLTkmSJEmSJClojQ8Unas+hbzcYLPEKYtOSZIkSZIkKWi1O0BqOdi/HTZ8GXSauGTRKUmSJEmSJAUtKRkanhp9vHJSoFHilUWnJEmSJEmSVBw0PiP63XU6j4hFpyRJkiRJklQcHFyn85tpkL0/2CxxyKJTkiRJkiRJKg6qtYCyNSBnP3w7I+g0cceiU5IkSZIkSSoOQiFodOCuTtfpLDSLTkmSJEmSJKm4cJ3OI2bRKUmSJEmSJBUXB9fp/G4O7NseaJR4Y9EpSZIkSZIkFRcV6kKVphDJgzVTg04TVyw6JUmSJEmSpOLEdTqPiEWnJEmSJEmSVJy4TucRseiUJEmSJEmSipOGpwIh2LIUdq4POk3csOiUJEmSJEmSipPSlaH2CdHHq7yrs6AsOiVJkiRJkqTiJn+dTovOgrLolCRJkiRJkoqb/HU6J0EkEmSSuGHRKUmSJEmSJBU39U+GpDTY9R1s/TroNHHBolOSJEmSJEkqblJKQb3O0ccrJwUaJV5YdEqSJEmSJEnF0fenr+tnWXRKkiRJkiRJxdHBonP1Z5CXG2iUeGDRKUmSJEmSJBVHtU6AtAqwfwesnxd0mmLPolOSJEmSJEkqjpKSoeGp0cdOX/9ZFp2SJEmSJElScZW/TufkQGPEA4tOSZIkSZIkqbhq3C36/ZvpkL0v2CzFnEWnJEmSJEmSVFxVbQblakFuJqz9Iug0xZpFpyRJkiRJklRchULQ6MBdna7T+ZMsOiVJkiRJkqTizHU6C8SiU5IkSZIkSSrODq7T+d1c2Lct2CzFmEWnJEmSJEmSVJyVrx1dq5MIrJ4SdJpiy6JTkiRJkiRJKu7y1+l0+vqPseiUJEmSJEmSirv8dTonBZmiWLPolCRJkiRJkoq7hqdCKAxbl8OOdUGnKZYsOiVJkiRJkqTirlRFqN0++niV09cPx6JTkiRJkiRJigeu0/mTLDolSZIkSZKkeND4YNE5CSKRQKMURxadkiRJkiRJUjyodzIkp8PuDbBlWdBpih2LTkmSJEmSJCkepKRDvZOij919/QcsOiVJkiRJkqR40fiM6HfX6fwBi05JkiRJkiQpXhxcp3P1Z5CbE2yWYsaiU5IkSZIkSYoXtU6A9AqQuRPWzws6TbFi0SlJkiRJkiTFi3ASNDwt+njlJ8FmKWYsOiVJkiRJkqR44jqdh2XRKUmSJEmSJMWTg0Xn2i8ga2+gUYoTi05JkiRJkiQpnlRpCuXrQG4WrJ0edJpiw6JTkiRJkiRJiiehEDQ6sPv6ykmBRilOLDolSZIkSZKkeOM6nT9g0SlJkiRJkiTFm0anR7+vnw97M4LNUkxYdEqSJEmSJEnxpnwtqNYCiMDqz4JOUyxYdEqSJEmSJEnxKH+dTqevg0WnJEmSJEmSFJ/y1+mcFGSKYsOiU5IkSZIkSYpHDbtCKAx7t7pOJ5AcdABJkiRJkiRJRyC9AvxmOlRpCuGkoNMEzqJTkiRJkiRJilfVmgedoNhw6rokSZIkSZKkuGfRKUmSJEmSJCnuWXRKkiRJkiRJinsWnZIkSZIkSZLinkWnJEmSJEmSpLhn0SlJkiRJkiQp7ll0SpIkSZIkSYp7Fp2SJEmSJEmS4p5FpyRJkiRJkqS4Z9EpSZIkSZIkKe5ZdEqSJEmSJEmKexadkiRJkiRJkuKeRackSZIkSZKkuGfRKUmSJEmSJCnuWXRKkiRJkiRJinsWnZIkSZIkSZLinkWnJEmSJEmSpLhn0SlJkiRJkiQp7ll0SpIkSZIkSYp7Fp2SJEmSJEmS4p5FpyRJkiRJkqS4lxx0gEQWiUQA2LlzZ8BJdKxkZ2ezd+9edu7cSUpKStBxpCPmtaxE4HWsROG1rEThtaxE4bWsRBEv1/LBXu1gz/ZTLDpjaNeuXQDUq1cv4CSSJEmSJElS/Nq1axcVKlT4yXNCkYLUoToieXl5fPfdd5QrV45QKBR0HB0DO3fupF69eqxdu5by5csHHUc6Yl7LSgRex0oUXstKFF7LShRey0oU8XItRyIRdu3aRe3atQmHf3oVTu/ojKFwOEzdunWDjqEAlC9fvlj/kZAKymtZicDrWInCa1mJwmtZicJrWYkiHq7ln7uT8yA3I5IkSZIkSZIU9yw6JUmSJEmSJMU9i06pCKWlpXHXXXeRlpYWdBTpqHgtKxF4HStReC0rUXgtK1F4LStRJOK17GZEkiRJkiRJkuKed3RKkiRJkiRJinsWnZIkSZIkSZLinkWnJEmSJEmSpLhn0SlJkiRJkiQp7ll0Sj9h2LBhNGzYkPT0dE466SRmzJjxk+c//PDDNG/enFKlSlGvXj1uvvlm9u/fn/98bm4ud9xxB40aNaJUqVI0adKEv/3tb7gnmGKtMNdydnY2d999N02aNCE9PZ127doxfvz4o3pNqagU9bV83333ceKJJ1KuXDmqV6/OxRdfzNKlS2P9NqSY/F0+6P777ycUCjFkyJAYJJcOFYtred26dVx55ZVUqVKFUqVK0aZNG2bNmhXLt6ESrqivYz/3KQiffvopvXv3pnbt2oRCId56662fHTNp0iQ6dOhAWloaTZs2Zfjw4T84J+4+90UkHdaoUaMiqampkeeeey6ycOHCyHXXXRepWLFiZOPGjYc9/+WXX46kpaVFXn755ciqVasiH3zwQaRWrVqRm2++Of+ce+65J1KlSpXIuHHjIqtWrYq89tprkbJly0YeeeSRY/W2VAIV9lq+9dZbI7Vr1468++67kRUrVkSeeOKJSHp6emTOnDlH/JpSUYjFtdyzZ8/I888/H1mwYEFk3rx5kfPOOy9Sv379yO7du4/V21IJFItr+aAZM2ZEGjZsGGnbtm3kpptuivE7UUkXi2s5IyMj0qBBg8jVV18d+eKLLyIrV66MfPDBB5Gvv/76WL0tlTCxuI793KcgvPfee5Hbb789MmbMmAgQefPNN3/y/JUrV0ZKly4dueWWWyKLFi2KPPbYY5GkpKTI+PHj88+Jx899Fp3Sj+jcuXNk8ODB+T/n5uZGateuHbnvvvsOe/7gwYMj3bt3P+TYLbfcEunatWv+z+eff37kl7/85SHnXHLJJZEBAwYUYXLpUIW9lmvVqhV5/PHHDzn239dpYV9TKgqxuJb/26ZNmyJAZPLkyUUTWjqMWF3Lu3btihx33HGRCRMmRLp162bRqZiLxbV82223RU499dTYBJYOIxbXsZ/7FLSCFJ233npr5Pjjjz/k2OWXXx7p2bNn/s/x+LnPqevSYWRlZTF79mx69OiRfywcDtOjRw+mTZt22DGnnHIKs2fPzr+Ne+XKlbz33nucd955h5wzceJEli1bBsD8+fOZMmUKvXr1iuG7UUl2JNdyZmYm6enphxwrVaoUU6ZMOeLXlI5WLK7lw9mxYwcAlStXLoLU0g/F8loePHgw559//iGvLcVKrK7ld955h06dOtG3b1+qV69O+/bteeaZZ2LzJlTixeo69nOf4sG0adN+8G+Gnj175l/78fq5LznoAFJxtGXLFnJzc6lRo8Yhx2vUqMGSJUsOO6Z///5s2bKFU089lUgkQk5ODr/+9a/505/+lH/OH/7wB3bu3EmLFi1ISkoiNzeXe+65hwEDBsT0/ajkOpJruWfPnjz44IOcfvrpNGnShIkTJzJmzBhyc3OP+DWloxWLa/m/5eXlMWTIELp27Urr1q2L/D1IELtredSoUcyZM4eZM2fGNL90UKyu5ZUrV/Lkk09yyy238Kc//YmZM2fyu9/9jtTUVAYNGhTT96SSJ1bXsZ/7FA82bNhw2Gt/586d7Nu3j23btsXl5z7v6JSKyKRJk7j33nt54oknmDNnDmPGjOHdd9/lb3/7W/45r776Ki+//DIjR45kzpw5jBgxggceeIARI0YEmFw61COPPMJxxx1HixYtSE1N5cYbb+Saa64hHPZ/MhRfCnstDx48mAULFjBq1KhjnFT6aT93La9du5abbrqJl19++Qd3GUnFSUH+Lufl5dGhQwfuvfde2rdvz/XXX891113HU089FWBy6T8Kch37uU8Kjp9apcOoWrUqSUlJbNy48ZDjGzdupGbNmocdc8cddzBw4EB+9atf0aZNG/r06cO9997LfffdR15eHgC///3v+cMf/sAVV1xBmzZtGDhwIDfffDP33XdfzN+TSqYjuZarVavGW2+9xZ49e1izZg1LliyhbNmyNG7c+IhfUzpasbiWv+/GG29k3LhxfPLJJ9StWzcm70GC2FzLs2fPZtOmTXTo0IHk5GSSk5OZPHkyjz76KMnJyT96F7N0NGL1d7lWrVq0atXqkHEtW7bkm2++Kfo3oRIvVtexn/sUD2rWrHnYa798+fKUKlUqbj/3WXRKh5GamkrHjh2ZOHFi/rG8vDwmTpxIly5dDjtm7969P7hLKCkpCYBIJPKT5xwsQqWidiTX8kHp6enUqVOHnJwc3njjDS666KKjfk3pSMXiWobo3+cbb7yRN998k48//phGjRrF7D1IEJtr+ayzzuKrr75i3rx5+V+dOnViwIABzJs3L//fI1JRitXf5a5du7J06dJDzl+2bBkNGjQo2jcgEbvr2M99igddunQ55NoHmDBhQv61H7ef+wLeDEkqtkaNGhVJS0uLDB8+PLJo0aLI9ddfH6lYsWJkw4YNkUgkEhk4cGDkD3/4Q/75d911V6RcuXKRV155JbJy5crIhx9+GGnSpEnksssuyz9n0KBBkTp16kTGjRsXWbVqVWTMmDGRqlWrRm699dZj/v5UchT2Wp4+fXrkjTfeiKxYsSLy6aefRrp37x5p1KhRZNu2bQV+TSkWYnEt33DDDZEKFSpEJk2aFFm/fn3+1969e4/121MJEotr+b+567qOhVhcyzNmzIgkJydH7rnnnsjy5csjL7/8cqR06dKRl1566Vi/PZUQsbiO/dynIOzatSsyd+7cyNy5cyNA5MEHH4zMnTs3smbNmkgkEon84Q9/iAwcODD//JUrV0ZKly4d+f3vfx9ZvHhxZNiwYZGkpKTI+PHj88+Jx899Fp3ST3jsscci9evXj6SmpkY6d+4cmT59ev5z3bp1iwwaNCj/5+zs7Mhf/vKXSJMmTSLp6emRevXqRX7zm98c8j94O3fujNx0002R+vXrR9LT0yONGzeO3H777ZHMzMxj+K5UEhXmWp40aVKkZcuWkbS0tEiVKlUiAwcOjKxbt65QrynFSlFfy8Bhv55//vlj9I5UUsXi7/L3WXTqWInFtTx27NhI69atI2lpaZEWLVpE/vWvfx2Lt6ISrKivYz/3KQiffPLJYf9de/D6HTRoUKRbt24/GHPCCSdEUlNTI40bNz7sv4Hj7XNfKBI5MKdWkiRJkiRJkuKUa3RKkiRJkiRJinsWnZIkSZIkSZLinkWnJEmSJEmSpLhn0SlJkiRJkiQp7ll0SpIkSZIkSYp7Fp2SJEmSJEmS4p5FpyRJkiRJkqS4Z9EpSZIkFcJf/vIXTjjhhPyfr776ai6++OLA8kiSJCnKolOSJEmSJElS3LPolCRJUsLIysoKOoIkSZICYtEpSZKkuHXGGWdw4403MmTIEKpWrUrPnj1ZsGABvXr1omzZstSoUYOBAweyZcuW/DF5eXn84x//oGnTpqSlpVG/fn3uueee/Odvu+02mjVrRunSpWncuDF33HEH2dnZQbw9SZIkFYJFpyRJkuLaiBEjSE1NZerUqdx///10796d9u3bM2vWLMaPH8/GjRu57LLL8s//4x//yP33388dd9zBokWLGDlyJDVq1Mh/vly5cgwfPpxFixbxyCOP8Mwzz/DQQw8F8dYkSZJUCKFIJBIJOoQkSZJ0JM444wx27tzJnDlzAPj73//OZ599xgcffJB/zrfffku9evVYunQptWrVolq1ajz++OP86le/KtDveOCBBxg1ahSzZs0CopsRvfXWW8ybNw+Ibka0fft23nrrrSJ9b5IkSSqc5KADSJIkSUejY8eO+Y/nz5/PJ598QtmyZX9w3ooVK9i+fTuZmZmcddZZP/p6o0eP5tFHH2XFihXs3r2bnJwcypcvH5PskiRJKjoWnZIkSYprZcqUyX+8e/duevfuzf/+7//+4LxatWqxcuXKn3ytadOmMWDAAP7617/Ss2dPKlSowKhRo/jnP/9Z5LklSZJUtCw6JUmSlDA6dOjAG2+8QcOGDUlO/uE/dY877jhKlSrFxIkTDzt1/fPPP6dBgwbcfvvt+cfWrFkT08ySJEkqGm5GJEmSpIQxePBgMjIy6NevHzNnzmTFihV88MEHXHPNNeTm5pKens5tt93GrbfeygsvvMCKFSuYPn06zz77LBAtQr/55htGjRrFihUrePTRR3nzzTcDfleSJEkqCItOSZIkJYzatWszdepUcnNzOeecc2jTpg1DhgyhYsWKhMPRf/recccd/M///A933nknLVu25PLLL2fTpk0AXHjhhdx8883ceOONnHDCCXz++efccccdQb4lSZIkFZC7rkuSJEmSJEmKe97RKUmSJEmSJCnuWXRKkiRJkiRJinsWnZIkSZIkSZLinkWnJEmSJEmSpLhn0SlJkiRJkiQp7ll0SpIkSZIkSYp7Fp2SJEmSJEmS4p5FpyRJkiRJkqS4Z9EpSZIkSZIkKe5ZdEqSJEmSJEmKexadkiRJkiRJkuKeRackSZIkSZKkuPf/Ac/+u+pc7UL0AAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots(len(search_config_names), 1, figsize=(16, len(search_config_names)*8))\n", - "fig.suptitle(\n", - " f'Effects of index parameters on QPS/recall trade-off ({DATASET_FILENAME})\\n' + \\\n", - " f'k = {k}, n_lists = {n_lists}')\n", - "\n", - "for j, search_label in enumerate(search_config_names):\n", - " labels = []\n", - " for i, index_label in enumerate(build_configs.keys()):\n", - " ax[j].plot(bench_recall_ip[i, j, :], bench_qps_ip[i, j, :])\n", - " labels.append(index_label)\n", - "\n", - " ax[j].set_title(f\"search: {search_label}\")\n", - " ax[j].legend(labels)\n", - " ax[j].set_xlabel('recall')\n", - " ax[j].set_ylabel('QPS')\n", - " ax[j].set_yscale('log')\n", - " ax[j].grid()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Looks like `pq_dim = 128`, `pq_bits = 6` is the best parameter set for the `SIFT-128` dataset." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.16" - }, - "vscode": { - "interpreter": { - "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/notebooks/utils.py b/notebooks/utils.py deleted file mode 100644 index 311efc98bc..0000000000 --- a/notebooks/utils.py +++ /dev/null @@ -1,102 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import cupy as cp -import h5py -import os -import tempfile -import time -import urllib - -## Check the quality of the prediction (recall) -def calc_recall(found_indices, ground_truth): - found_indices = cp.asarray(found_indices) - bs, k = found_indices.shape - if bs != ground_truth.shape[0]: - raise RuntimeError( - "Batch sizes do not match {} vs {}".format( - bs, ground_truth.shape[0] - ) - ) - if k > ground_truth.shape[1]: - raise RuntimeError( - "Not enough indices in the ground truth ({} > {})".format( - k, ground_truth.shape[1] - ) - ) - n = 0 - # Go over the batch - for i in range(bs): - # Note, ivf-pq does not guarantee the ordered input, hence the use of intersect1d - n += cp.intersect1d(found_indices[i, :k], ground_truth[i, :k]).size - recall = n / found_indices.size - return recall - - -class BenchmarkTimer: - """Provides a context manager that runs a code block `reps` times - and records results to the instance variable `timings`. Use like: - .. code-block:: python - timer = BenchmarkTimer(rep=5) - for _ in timer.benchmark_runs(): - ... do something ... - print(np.min(timer.timings)) - - This class is borrowed from the rapids/cuml benchmark suite - """ - - def __init__(self, reps=1, warmup=0): - self.warmup = warmup - self.reps = reps - self.timings = [] - - def benchmark_runs(self): - for r in range(self.reps + self.warmup): - t0 = time.time() - yield r - t1 = time.time() - self.timings.append(t1 - t0) - if r >= self.warmup: - self.timings.append(t1 - t0) - - -def load_dataset(dataset_url="http://ann-benchmarks.com/sift-128-euclidean.hdf5", work_folder=None): - """Download dataset from url. It is expected that the dataset contains a hdf5 file in ann-benchmarks format - - Parameters - ---------- - dataset_url address of hdf5 file - work_folder name of the local folder to store the dataset - - """ - dataset_filename = dataset_url.split("/")[-1] - - # We'll need to load store some data in this tutorial - if work_folder is None: - work_folder = os.path.join(tempfile.gettempdir(), "raft_example") - - if not os.path.exists(work_folder): - os.makedirs(work_folder) - print("The index and data will be saved in", work_folder) - - ## download the dataset - dataset_path = os.path.join(work_folder, dataset_filename) - if not os.path.exists(dataset_path): - urllib.request.urlretrieve(dataset_url, dataset_path) - - f = h5py.File(dataset_path, "r") - - return f diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index c286d3debf..758c1e4711 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -53,7 +53,6 @@ if(NOT raft_FOUND) set(BUILD_TESTS OFF) set(BUILD_PRIMS_BENCH OFF) - set(BUILD_ANN_BENCH OFF) set(RAFT_COMPILE_LIBRARY ON) set(CUDA_STATIC_RUNTIME ON) set(CUDA_STATIC_MATH_LIBRARIES ON) @@ -66,12 +65,14 @@ if(NOT raft_FOUND) add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) - set_property(TARGET raft_lib PROPERTY INSTALL_RPATH - "$ORIGIN/../nvidia/cublas/lib" - "$ORIGIN/../nvidia/curand/lib" - "$ORIGIN/../nvidia/cusolver/lib" - "$ORIGIN/../nvidia/cusparse/lib" - "$ORIGIN/../nvidia/nvjitlink/lib" + set_property( + TARGET raft_lib + PROPERTY INSTALL_RPATH + "$ORIGIN/../nvidia/cublas/lib" + "$ORIGIN/../nvidia/curand/lib" + "$ORIGIN/../nvidia/cusolver/lib" + "$ORIGIN/../nvidia/cusparse/lib" + "$ORIGIN/../nvidia/nvjitlink/lib" ) endif() @@ -85,11 +86,8 @@ endif() rapids_cython_init() add_subdirectory(pylibraft/common) -add_subdirectory(pylibraft/distance) -add_subdirectory(pylibraft/matrix) -add_subdirectory(pylibraft/neighbors) add_subdirectory(pylibraft/random) -add_subdirectory(pylibraft/cluster) +add_subdirectory(pylibraft/sparse) if(DEFINED cython_lib_dir) rapids_cython_add_rpath_entries(TARGET raft PATHS "${cython_lib_dir}") diff --git a/python/pylibraft/pylibraft/cluster/CMakeLists.txt b/python/pylibraft/pylibraft/cluster/CMakeLists.txt deleted file mode 100644 index 562cff5098..0000000000 --- a/python/pylibraft/pylibraft/cluster/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Set the list of Cython files to build -set(cython_sources kmeans.pyx) -set(linked_libraries raft::compiled) - -# Build all of the Cython targets -rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX cluster_ -) diff --git a/python/pylibraft/pylibraft/cluster/__init__.pxd b/python/pylibraft/pylibraft/cluster/__init__.pxd deleted file mode 100644 index 273b4497cc..0000000000 --- a/python/pylibraft/pylibraft/cluster/__init__.pxd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/cluster/__init__.py b/python/pylibraft/pylibraft/cluster/__init__.py deleted file mode 100644 index 00b12eab9b..0000000000 --- a/python/pylibraft/pylibraft/cluster/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .kmeans import ( - KMeansParams, - cluster_cost, - compute_new_centroids, - fit, - init_plus_plus, -) - -__all__ = [ - "KMeansParams", - "cluster_cost", - "compute_new_centroids", - "fit", - "init_plus_plus", -] diff --git a/python/pylibraft/pylibraft/cluster/cpp/kmeans.pxd b/python/pylibraft/pylibraft/cluster/cpp/kmeans.pxd deleted file mode 100644 index 4a5a47de68..0000000000 --- a/python/pylibraft/pylibraft/cluster/cpp/kmeans.pxd +++ /dev/null @@ -1,106 +0,0 @@ -# -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport uintptr_t -from libcpp cimport bool, nullptr - -from pylibraft.cluster.cpp.kmeans_types cimport KMeansParams -from pylibraft.common.cpp.mdspan cimport * -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources - - -cdef extern from "raft_runtime/cluster/kmeans.hpp" \ - namespace "raft::runtime::cluster::kmeans" nogil: - - cdef void update_centroids( - const device_resources& handle, - const double *X, - int n_samples, - int n_features, - int n_clusters, - const double *sample_weights, - const double *centroids, - const int* labels, - double *new_centroids, - double *weight_per_cluster) except + - - cdef void update_centroids( - const device_resources& handle, - const float *X, - int n_samples, - int n_features, - int n_clusters, - const float *sample_weights, - const float *centroids, - const int* labels, - float *new_centroids, - float *weight_per_cluster) except + - - cdef void cluster_cost( - const device_resources& handle, - const float* X, - int n_samples, - int n_features, - int n_clusters, - const float * centroids, - float * cost) except + - - cdef void cluster_cost( - const device_resources& handle, - const double* X, - int n_samples, - int n_features, - int n_clusters, - const double * centroids, - double * cost) except + - - cdef void init_plus_plus( - const device_resources & handle, - const KMeansParams& params, - device_matrix_view[float, int, row_major] X, - device_matrix_view[float, int, row_major] centroids) except + - - cdef void init_plus_plus( - const device_resources & handle, - const KMeansParams& params, - device_matrix_view[double, int, row_major] X, - device_matrix_view[double, int, row_major] centroids) except + - - cdef void fit( - const device_resources & handle, - const KMeansParams& params, - device_matrix_view[float, int, row_major] X, - optional[device_vector_view[float, int]] sample_weight, - device_matrix_view[float, int, row_major] inertia, - host_scalar_view[float, int] inertia, - host_scalar_view[int, int] n_iter) except + - - cdef void fit( - const device_resources & handle, - const KMeansParams& params, - device_matrix_view[double, int, row_major] X, - optional[device_vector_view[double, int]] sample_weight, - device_matrix_view[double, int, row_major] inertia, - host_scalar_view[double, int] inertia, - host_scalar_view[int, int] n_iter) except + diff --git a/python/pylibraft/pylibraft/cluster/cpp/kmeans_types.pxd b/python/pylibraft/pylibraft/cluster/cpp/kmeans_types.pxd deleted file mode 100644 index 12cecd4336..0000000000 --- a/python/pylibraft/pylibraft/cluster/cpp/kmeans_types.pxd +++ /dev/null @@ -1,44 +0,0 @@ -# -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from libcpp cimport bool - -from pylibraft.distance.distance_type cimport DistanceType -from pylibraft.random.cpp.rng_state cimport RngState - - -cdef extern from "raft/cluster/kmeans_types.hpp" \ - namespace "raft::cluster::kmeans": - - ctypedef enum InitMethod 'raft::cluster::KMeansParams::InitMethod': - KMeansPlusPlus 'raft::cluster::kmeans::KMeansParams::InitMethod::KMeansPlusPlus' # noqa - Random 'raft::cluster::kmeans::KMeansParams::InitMethod::Random' - Array 'raft::cluster::kmeans::KMeansParams::InitMethod::Array' - - cdef cppclass KMeansParams: - KMeansParams() except + - int n_clusters - InitMethod init - int max_iter - double tol - int verbosity - RngState rng_state - DistanceType metric - int n_init - double oversampling_factor - int batch_samples - int batch_centroids - bool inertia_check diff --git a/python/pylibraft/pylibraft/cluster/kmeans.pyx b/python/pylibraft/pylibraft/cluster/kmeans.pyx deleted file mode 100644 index f4af519dc1..0000000000 --- a/python/pylibraft/pylibraft/cluster/kmeans.pyx +++ /dev/null @@ -1,589 +0,0 @@ -# -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport uintptr_t -from libcpp cimport nullptr - -from collections import namedtuple -from enum import IntEnum - -from pylibraft.common import Handle, cai_wrapper, device_ndarray -from pylibraft.common.handle import auto_sync_handle - -from pylibraft.common.handle cimport device_resources -from pylibraft.random.cpp.rng_state cimport RngState - -from pylibraft.common.input_validation import * -from pylibraft.distance import DISTANCE_TYPES - -from pylibraft.cluster.cpp cimport kmeans as cpp_kmeans, kmeans_types -from pylibraft.cluster.cpp.kmeans cimport ( - cluster_cost as cpp_cluster_cost, - init_plus_plus as cpp_init_plus_plus, - update_centroids, -) -from pylibraft.common.cpp.mdspan cimport * -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources - -from pylibraft.common import auto_convert_output - - -@auto_sync_handle -@auto_convert_output -def compute_new_centroids(X, - centroids, - labels, - new_centroids, - sample_weights=None, - weight_per_cluster=None, - handle=None): - """ - Compute new centroids given an input matrix and existing centroids - - Parameters - ---------- - - X : Input CUDA array interface compliant matrix shape (m, k) - centroids : Input CUDA array interface compliant matrix shape - (n_clusters, k) - labels : Input CUDA array interface compliant matrix shape - (m, 1) - new_centroids : Writable CUDA array interface compliant matrix shape - (n_clusters, k) - sample_weights : Optional input CUDA array interface compliant matrix shape - (n_clusters, 1) default: None - weight_per_cluster : Optional writable CUDA array interface compliant - matrix shape (n_clusters, 1) default: None - batch_samples : Optional integer specifying the batch size for X to compute - distances in batches. default: m - batch_centroids : Optional integer specifying the batch size for centroids - to compute distances in batches. default: n_clusters - {handle_docstring} - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.common import Handle - >>> from pylibraft.cluster.kmeans import compute_new_centroids - >>> # A single RAFT handle can optionally be reused across - >>> # pylibraft functions. - >>> handle = Handle() - >>> n_samples = 5000 - >>> n_features = 50 - >>> n_clusters = 3 - >>> X = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> centroids = cp.random.random_sample((n_clusters, n_features), - ... dtype=cp.float32) - ... - >>> labels = cp.random.randint(0, high=n_clusters, size=n_samples, - ... dtype=cp.int32) - >>> new_centroids = cp.empty((n_clusters, n_features), - ... dtype=cp.float32) - >>> compute_new_centroids( - ... X, centroids, labels, new_centroids, handle=handle - ... ) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - - x_cai = X.__cuda_array_interface__ - centroids_cai = centroids.__cuda_array_interface__ - new_centroids_cai = new_centroids.__cuda_array_interface__ - labels_cai = labels.__cuda_array_interface__ - - m = x_cai["shape"][0] - x_k = x_cai["shape"][1] - n_clusters = centroids_cai["shape"][0] - - centroids_k = centroids_cai["shape"][1] - new_centroids_k = centroids_cai["shape"][1] - - x_dt = np.dtype(x_cai["typestr"]) - centroids_dt = np.dtype(centroids_cai["typestr"]) - new_centroids_dt = np.dtype(new_centroids_cai["typestr"]) - labels_dt = np.dtype(labels_cai["typestr"]) - - if not do_cols_match(X, centroids): - raise ValueError("X and centroids must have same number of columns.") - - if not do_rows_match(X, labels): - raise ValueError("X and labels must have same number of rows") - - x_ptr = x_cai["data"][0] - centroids_ptr = centroids_cai["data"][0] - new_centroids_ptr = new_centroids_cai["data"][0] - labels_ptr = labels_cai["data"][0] - - if sample_weights is not None: - sample_weights_cai = sample_weights.__cuda_array_interface__ - sample_weights_ptr = sample_weights_cai["data"][0] - sample_weights_dt = np.dtype(sample_weights_cai["typestr"]) - else: - sample_weights_ptr = nullptr - - if weight_per_cluster is not None: - weight_per_cluster_cai = weight_per_cluster.__cuda_array_interface__ - weight_per_cluster_ptr = weight_per_cluster_cai["data"][0] - weight_per_cluster_dt = np.dtype(weight_per_cluster_cai["typestr"]) - else: - weight_per_cluster_ptr = nullptr - - handle = handle if handle is not None else Handle() - cdef device_resources *h = handle.getHandle() - - x_c_contiguous = is_c_contiguous(x_cai) - centroids_c_contiguous = is_c_contiguous(centroids_cai) - new_centroids_c_contiguous = is_c_contiguous(new_centroids_cai) - - if not x_c_contiguous or not centroids_c_contiguous \ - or not new_centroids_c_contiguous: - raise ValueError("Inputs must all be c contiguous") - - if not do_dtypes_match(X, centroids, new_centroids): - raise ValueError("Inputs must all have the same dtypes " - "(float32 or float64)") - - if x_dt == np.float32: - update_centroids(deref(h), - x_ptr, - m, - x_k, - n_clusters, - sample_weights_ptr, - centroids_ptr, - labels_ptr, - new_centroids_ptr, - weight_per_cluster_ptr) - elif x_dt == np.float64: - update_centroids(deref(h), - x_ptr, - m, - x_k, - n_clusters, - sample_weights_ptr, - centroids_ptr, - labels_ptr, - new_centroids_ptr, - weight_per_cluster_ptr) - else: - raise ValueError("dtype %s not supported" % x_dt) - - -@auto_sync_handle -@auto_convert_output -def init_plus_plus(X, n_clusters=None, seed=None, handle=None, centroids=None): - """ - Compute initial centroids using the "kmeans++" algorithm. - - Parameters - ---------- - - X : Input CUDA array interface compliant matrix shape (m, k) - n_clusters : Number of clusters to select - seed : Controls the random sampling of centroids - centroids : Optional writable CUDA array interface compliant matrix shape - (n_clusters, k). Use instead of passing `n_clusters`. - {handle_docstring} - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.cluster.kmeans import init_plus_plus - >>> n_samples = 5000 - >>> n_features = 50 - >>> n_clusters = 3 - >>> X = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - - >>> centroids = init_plus_plus(X, n_clusters) - """ - if (n_clusters is not None and - centroids is not None and n_clusters != centroids.shape[0]): - msg = ("Parameters 'n_clusters' and 'centroids' " - "are exclusive. Only pass one at a time.") - raise RuntimeError(msg) - - cdef device_resources *h = handle.getHandle() - - X_cai = cai_wrapper(X) - X_cai.validate_shape_dtype(expected_dims=2) - dtype = X_cai.dtype - - if centroids is not None: - n_clusters = centroids.shape[0] - else: - centroids_shape = (n_clusters, X_cai.shape[1]) - centroids = device_ndarray.empty(centroids_shape, dtype=dtype) - - centroids_cai = cai_wrapper(centroids) - - # Can't set attributes of KMeansParameters after creating it, so taking - # a detour via a dict to collect the possible constructor arguments - params_ = dict(n_clusters=n_clusters) - if seed is not None: - params_["seed"] = seed - params = KMeansParams(**params_) - - if dtype == np.float64: - cpp_init_plus_plus( - deref(h), params.c_obj, - make_device_matrix_view[double, int, row_major]( - X_cai.data, - X_cai.shape[0], X_cai.shape[1]), - make_device_matrix_view[double, int, row_major]( - centroids_cai.data, - centroids_cai.shape[0], centroids_cai.shape[1]), - ) - elif dtype == np.float32: - cpp_init_plus_plus( - deref(h), params.c_obj, - make_device_matrix_view[float, int, row_major]( - X_cai.data, - X_cai.shape[0], X_cai.shape[1]), - make_device_matrix_view[float, int, row_major]( - centroids_cai.data, - centroids_cai.shape[0], centroids_cai.shape[1]), - ) - else: - raise ValueError(f"Unhandled dtype ({dtype}) for X.") - - return centroids - - -@auto_sync_handle -@auto_convert_output -def cluster_cost(X, centroids, handle=None): - """ - Compute cluster cost given an input matrix and existing centroids - - Parameters - ---------- - X : Input CUDA array interface compliant matrix shape (m, k) - centroids : Input CUDA array interface compliant matrix shape - (n_clusters, k) - {handle_docstring} - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.cluster.kmeans import cluster_cost - >>> n_samples = 5000 - >>> n_features = 50 - >>> n_clusters = 3 - >>> X = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> centroids = cp.random.random_sample((n_clusters, n_features), - ... dtype=cp.float32) - >>> inertia = cluster_cost(X, centroids) - """ - x_cai = X.__cuda_array_interface__ - centroids_cai = centroids.__cuda_array_interface__ - - m = x_cai["shape"][0] - x_k = x_cai["shape"][1] - n_clusters = centroids_cai["shape"][0] - - centroids_k = centroids_cai["shape"][1] - - x_dt = np.dtype(x_cai["typestr"]) - centroids_dt = np.dtype(centroids_cai["typestr"]) - - if not do_cols_match(X, centroids): - raise ValueError("X and centroids must have same number of columns.") - - x_ptr = x_cai["data"][0] - centroids_ptr = centroids_cai["data"][0] - - handle = handle if handle is not None else Handle() - cdef device_resources *h = handle.getHandle() - - x_c_contiguous = is_c_contiguous(x_cai) - centroids_c_contiguous = is_c_contiguous(centroids_cai) - - if not x_c_contiguous or not centroids_c_contiguous: - raise ValueError("Inputs must all be c contiguous") - - if not do_dtypes_match(X, centroids): - raise ValueError("Inputs must all have the same dtypes " - "(float32 or float64)") - - cdef float f_cost = 0 - cdef double d_cost = 0 - - if x_dt == np.float32: - cpp_cluster_cost(deref(h), - x_ptr, - m, - x_k, - n_clusters, - centroids_ptr, - &f_cost) - return f_cost - elif x_dt == np.float64: - cpp_cluster_cost(deref(h), - x_ptr, - m, - x_k, - n_clusters, - centroids_ptr, - &d_cost) - return d_cost - else: - raise ValueError("dtype %s not supported" % x_dt) - - -class InitMethod(IntEnum): - """ Method for initializing kmeans """ - KMeansPlusPlus = kmeans_types.InitMethod.KMeansPlusPlus - Random = kmeans_types.InitMethod.Random - Array = kmeans_types.InitMethod.Array - - -cdef class KMeansParams: - """ Specifies hyper-parameters for the kmeans algorithm. - - Parameters - ---------- - n_clusters : int, optional - The number of clusters to form as well as the number of centroids - to generate - max_iter : int, optional - Maximum number of iterations of the k-means algorithm for a single run - tol : float, optional - Relative tolerance with regards to inertia to declare convergence - verbosity : int, optional - seed: int, optional - Seed to the random number generator. - metric : str, optional - Metric names to use for distance computation, see - :func:`pylibraft.distance.pairwise_distance` for valid values. - init : InitMethod, optional - n_init : int, optional - Number of instance k-means algorithm will be run with different seeds. - oversampling_factor : float, optional - Oversampling factor for use in the k-means algorithm - """ - cdef kmeans_types.KMeansParams c_obj - - def __init__(self, - n_clusters: Optional[int] = None, - max_iter: Optional[int] = None, - tol: Optional[float] = None, - verbosity: Optional[int] = None, - seed: Optional[int] = None, - metric: Optional[str] = None, - init: Optional[InitMethod] = None, - n_init: Optional[int] = None, - oversampling_factor: Optional[float] = None, - batch_samples: Optional[int] = None, - batch_centroids: Optional[int] = None, - inertia_check: Optional[bool] = None): - if n_clusters is not None: - self.c_obj.n_clusters = n_clusters - if max_iter is not None: - self.c_obj.max_iter = max_iter - if tol is not None: - self.c_obj.tol = tol - if verbosity is not None: - self.c_obj.verbosity = verbosity - if seed is not None: - self.c_obj.rng_state.seed = seed - if metric is not None: - distance = DISTANCE_TYPES.get(metric) - if distance is None: - valid_metrics = list(DISTANCE_TYPES.keys()) - raise ValueError(f"Unknown metric '{metric}'. Valid values " - f"are: {valid_metrics}") - self.c_obj.metric = distance - if init is not None: - self.c_obj.init = init - if n_init is not None: - self.c_obj.n_init = n_init - if oversampling_factor is not None: - self.c_obj.oversampling_factor = oversampling_factor - if batch_samples is not None: - self.c_obj.batch_samples = batch_samples - if batch_centroids is not None: - self.c_obj.batch_centroids = batch_centroids - if inertia_check is not None: - self.c_obj.inertia_check = inertia_check - - @property - def n_clusters(self): - return self.c_obj.n_clusters - - @property - def max_iter(self): - return self.c_obj.max_iter - - @property - def tol(self): - return self.c_obj.tol - - @property - def verbosity(self): - return self.c_obj.verbosity - - @property - def seed(self): - return self.c_obj.rng_state.seed - - @property - def init(self): - return InitMethod(self.c_obj.init) - - @property - def oversampling_factor(self): - return self.c_obj.oversampling_factor - - @property - def batch_samples(self): - return self.c_obj.batch_samples - - @property - def batch_centroids(self): - return self.c_obj.batch_centroids - - @property - def inertia_check(self): - return self.c_obj.inertia_check - -FitOutput = namedtuple("FitOutput", "centroids inertia n_iter") - - -@auto_sync_handle -@auto_convert_output -def fit( - KMeansParams params, X, centroids=None, sample_weights=None, handle=None -): - """ - Find clusters with the k-means algorithm - - Parameters - ---------- - - params : KMeansParams - Parameters to use to fit KMeans model - X : Input CUDA array interface compliant matrix shape (m, k) - centroids : Optional writable CUDA array interface compliant matrix - shape (n_clusters, k) - sample_weights : Optional input CUDA array interface compliant matrix shape - (n_clusters, 1) default: None - {handle_docstring} - - Returns - ------- - centroids : raft.device_ndarray - The computed centroids for each cluster - inertia : float - Sum of squared distances of samples to their closest cluster center - n_iter : int - The number of iterations used to fit the model - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.cluster.kmeans import fit, KMeansParams - >>> n_samples = 5000 - >>> n_features = 50 - >>> n_clusters = 3 - >>> X = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - - >>> params = KMeansParams(n_clusters=n_clusters) - >>> centroids, inertia, n_iter = fit(params, X) - """ - cdef device_resources *h = handle.getHandle() - - cdef float f_inertia = 0.0 - cdef double d_inertia = 0.0 - cdef int n_iter = 0 - - cdef optional[device_vector_view[const double, int]] d_sample_weights - cdef optional[device_vector_view[const float, int]] f_sample_weights - - X_cai = cai_wrapper(X) - dtype = X_cai.dtype - - if centroids is None: - centroids_shape = (params.n_clusters, X_cai.shape[1]) - centroids = device_ndarray.empty(centroids_shape, dtype=dtype) - centroids_cai = cai_wrapper(centroids) - - # validate inputs have are all c-contiguous, and have a consistent dtype - # and expected shape - X_cai.validate_shape_dtype(2) - centroids_cai.validate_shape_dtype(2, dtype) - if sample_weights is not None: - sample_weights_cai = cai_wrapper(sample_weights) - sample_weights_cai.validate_shape_dtype(1, dtype) - - if dtype == np.float64: - if sample_weights is not None: - d_sample_weights = make_device_vector_view( - sample_weights_cai.data, - sample_weights_cai.shape[0]) - - cpp_kmeans.fit( - deref(h), - params.c_obj, - make_device_matrix_view[double, int, row_major]( - X_cai.data, - X_cai.shape[0], X_cai.shape[1]), - d_sample_weights, - make_device_matrix_view[double, int, row_major]( - centroids_cai.data, - centroids_cai.shape[0], centroids_cai.shape[1]), - make_host_scalar_view[double, int](&d_inertia), - make_host_scalar_view[int, int](&n_iter)) - return FitOutput(centroids, d_inertia, n_iter) - - elif dtype == np.float32: - if sample_weights is not None: - f_sample_weights = make_device_vector_view( - sample_weights_cai.data, - sample_weights_cai.shape[0]) - - cpp_kmeans.fit( - deref(h), - params.c_obj, - make_device_matrix_view[float, int, row_major]( - X_cai.data, - X_cai.shape[0], X_cai.shape[1]), - f_sample_weights, - make_device_matrix_view[float, int, row_major]( - centroids_cai.data, - centroids_cai.shape[0], centroids_cai.shape[1]), - make_host_scalar_view[float, int](&f_inertia), - make_host_scalar_view[int, int](&n_iter)) - return FitOutput(centroids, f_inertia, n_iter) - - else: - raise ValueError(f"unhandled dtype {dtype}") diff --git a/python/pylibraft/pylibraft/common/handle.pxd b/python/pylibraft/pylibraft/common/handle.pxd index c090663547..78a07ac8bf 100644 --- a/python/pylibraft/pylibraft/common/handle.pxd +++ b/python/pylibraft/pylibraft/common/handle.pxd @@ -1,5 +1,5 @@ # -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,8 +22,8 @@ from libcpp.memory cimport shared_ptr, unique_ptr -from rmm._lib.cuda_stream_pool cimport cuda_stream_pool -from rmm._lib.cuda_stream_view cimport cuda_stream_view +from rmm.librmm.cuda_stream_pool cimport cuda_stream_pool +from rmm.librmm.cuda_stream_view cimport cuda_stream_view # Keeping `handle_t` around for backwards compatibility at the diff --git a/python/pylibraft/pylibraft/common/handle.pyx b/python/pylibraft/pylibraft/common/handle.pyx index 7e3dc289e0..d256e671bf 100644 --- a/python/pylibraft/pylibraft/common/handle.pyx +++ b/python/pylibraft/pylibraft/common/handle.pyx @@ -1,5 +1,5 @@ # -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,7 +24,10 @@ import functools from cuda.ccudart cimport cudaStream_t from libc.stdint cimport uintptr_t -from rmm._lib.cuda_stream_view cimport cuda_stream_per_thread, cuda_stream_view +from rmm.librmm.cuda_stream_view cimport ( + cuda_stream_per_thread, + cuda_stream_view, +) from .cuda cimport Stream diff --git a/python/pylibraft/pylibraft/common/interruptible.pxd b/python/pylibraft/pylibraft/common/interruptible.pxd index aaccf8aeab..27259571ea 100644 --- a/python/pylibraft/pylibraft/common/interruptible.pxd +++ b/python/pylibraft/pylibraft/common/interruptible.pxd @@ -1,5 +1,5 @@ # -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ from libcpp.memory cimport shared_ptr -from rmm._lib.cuda_stream_view cimport cuda_stream_view +from rmm.librmm.cuda_stream_view cimport cuda_stream_view cdef extern from "raft/core/interruptible.hpp" namespace "raft" nogil: diff --git a/python/pylibraft/pylibraft/common/interruptible.pyx b/python/pylibraft/pylibraft/common/interruptible.pyx index bb5415428f..c489f2ee20 100644 --- a/python/pylibraft/pylibraft/common/interruptible.pyx +++ b/python/pylibraft/pylibraft/common/interruptible.pyx @@ -1,5 +1,5 @@ # -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ import signal from cuda.ccudart cimport cudaStream_t from cython.operator cimport dereference -from rmm._lib.cuda_stream_view cimport cuda_stream_view +from rmm.librmm.cuda_stream_view cimport cuda_stream_view from .cuda cimport Stream diff --git a/python/pylibraft/pylibraft/distance/CMakeLists.txt b/python/pylibraft/pylibraft/distance/CMakeLists.txt deleted file mode 100644 index 2530e07a98..0000000000 --- a/python/pylibraft/pylibraft/distance/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Set the list of Cython files to build -set(cython_sources pairwise_distance.pyx fused_l2_nn.pyx fused_distance_nn.pyx) -set(linked_libraries raft::raft raft::compiled) - -# Build all of the Cython targets -rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX distance_ -) diff --git a/python/pylibraft/pylibraft/distance/__init__.pxd b/python/pylibraft/pylibraft/distance/__init__.pxd deleted file mode 100644 index 273b4497cc..0000000000 --- a/python/pylibraft/pylibraft/distance/__init__.pxd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/distance/__init__.py b/python/pylibraft/pylibraft/distance/__init__.py deleted file mode 100644 index d16ab30b2f..0000000000 --- a/python/pylibraft/pylibraft/distance/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .fused_distance_nn import fused_distance_nn_argmin -from .fused_l2_nn import fused_l2_nn_argmin -from .pairwise_distance import DISTANCE_TYPES, distance as pairwise_distance - -__all__ = [ - "fused_distance_nn_argmin", - "fused_l2_nn_argmin", - "pairwise_distance", -] diff --git a/python/pylibraft/pylibraft/distance/distance_type.pxd b/python/pylibraft/pylibraft/distance/distance_type.pxd deleted file mode 100644 index e058238d45..0000000000 --- a/python/pylibraft/pylibraft/distance/distance_type.pxd +++ /dev/null @@ -1,44 +0,0 @@ -# -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -cdef extern from "raft/distance/distance_types.hpp" namespace "raft::distance": - - ctypedef enum DistanceType: - L2Expanded "raft::distance::DistanceType::L2Expanded" - L2SqrtExpanded "raft::distance::DistanceType::L2SqrtExpanded" - CosineExpanded "raft::distance::DistanceType::CosineExpanded" - L1 "raft::distance::DistanceType::L1" - L2Unexpanded "raft::distance::DistanceType::L2Unexpanded" - L2SqrtUnexpanded "raft::distance::DistanceType::L2SqrtUnexpanded" - InnerProduct "raft::distance::DistanceType::InnerProduct" - Linf "raft::distance::DistanceType::Linf" - Canberra "raft::distance::DistanceType::Canberra" - LpUnexpanded "raft::distance::DistanceType::LpUnexpanded" - CorrelationExpanded "raft::distance::DistanceType::CorrelationExpanded" - JaccardExpanded "raft::distance::DistanceType::JaccardExpanded" - HellingerExpanded "raft::distance::DistanceType::HellingerExpanded" - Haversine "raft::distance::DistanceType::Haversine" - BrayCurtis "raft::distance::DistanceType::BrayCurtis" - JensenShannon "raft::distance::DistanceType::JensenShannon" - HammingUnexpanded "raft::distance::DistanceType::HammingUnexpanded" - KLDivergence "raft::distance::DistanceType::KLDivergence" - RusselRaoExpanded "raft::distance::DistanceType::RusselRaoExpanded" - DiceExpanded "raft::distance::DistanceType::DiceExpanded" - Precomputed "raft::distance::DistanceType::Precomputed" diff --git a/python/pylibraft/pylibraft/distance/fused_distance_nn.pyx b/python/pylibraft/pylibraft/distance/fused_distance_nn.pyx deleted file mode 100644 index 0e9fa4b366..0000000000 --- a/python/pylibraft/pylibraft/distance/fused_distance_nn.pyx +++ /dev/null @@ -1,200 +0,0 @@ -# -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport uintptr_t -from libcpp cimport bool - -from .distance_type cimport DistanceType - -from pylibraft.common import ( - Handle, - auto_convert_output, - cai_wrapper, - device_ndarray, -) -from pylibraft.common.handle import auto_sync_handle - -from pylibraft.common.handle cimport device_resources - - -cdef extern from "raft_runtime/distance/fused_distance_nn.hpp" \ - namespace "raft::runtime::distance" nogil: - - void fused_distance_nn_min_arg( - const device_resources &handle, - int* min, - const float* x, - const float* y, - int m, - int n, - int k, - bool sqrt, - DistanceType metric, - bool isRowMajor, - float metric_arg) except + - - -from pylibraft.distance.pairwise_distance import DISTANCE_TYPES - -SUPPORTED_DISTANCES = ["euclidean", "l2", "cosine", "sqeuclidean"] - - -@auto_sync_handle -@auto_convert_output -def fused_distance_nn_argmin(X, Y, out=None, sqrt=True, metric="euclidean", - handle=None): - """ - Compute the 1-nearest neighbors between X and Y using the distance metrics - - Valid values for metric: - ["euclidean", "l2", "cosine", "sqeuclidean"] - - Parameters - ---------- - - X : CUDA array interface compliant matrix shape (m, k) - Y : CUDA array interface compliant matrix shape (n, k) - out : Writable CUDA array interface matrix shape (m, 1) - metric : string denoting the metric type (default="euclidean") - - {handle_docstring} - - Examples - -------- - To compute the 1-nearest neighbors argmin: - - >>> import cupy as cp - >>> from pylibraft.common import Handle - >>> from pylibraft.distance import fused_distance_nn_argmin - >>> n_samples = 5000 - >>> n_clusters = 5 - >>> n_features = 50 - >>> in1 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> in2 = cp.random.random_sample((n_clusters, n_features), - ... dtype=cp.float32) - >>> # A single RAFT handle can optionally be reused across - >>> # pylibraft functions. - >>> handle = Handle() - - >>> output = fused_distance_nn_argmin(in1, in2, handle=handle) - - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - - The output can also be computed in-place on a preallocated - array: - - >>> import cupy as cp - >>> from pylibraft.common import Handle - >>> from pylibraft.distance import fused_distance_nn_argmin - >>> n_samples = 5000 - >>> n_clusters = 5 - >>> n_features = 50 - >>> in1 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> in2 = cp.random.random_sample((n_clusters, n_features), - ... dtype=cp.float32) - >>> output = cp.empty((n_samples, 1), dtype=cp.int32) - >>> # A single RAFT handle can optionally be reused across - >>> # pylibraft functions. - >>> handle = Handle() - - >>> fused_distance_nn_argmin(in1, in2, out=output, handle=handle) - array(...) - - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - - x_cai = cai_wrapper(X) - y_cai = cai_wrapper(Y) - - x_dt = x_cai.dtype - y_dt = y_cai.dtype - - m = x_cai.shape[0] - n = y_cai.shape[0] - - if out is None: - output = device_ndarray.empty((m,), dtype="int32") - else: - output = out - - output_cai = cai_wrapper(output) - - x_k = x_cai.shape[1] - y_k = y_cai.shape[1] - - if x_k != y_k: - raise ValueError("Inputs must have same number of columns. " - "a=%s, b=%s" % (x_k, y_k)) - - if metric not in SUPPORTED_DISTANCES: - raise ValueError("metric %s is not supported" % metric) - - cdef DistanceType distance_type = DISTANCE_TYPES[metric] - - x_ptr = x_cai.data - y_ptr = y_cai.data - - d_ptr = output_cai.data - - handle = handle if handle is not None else Handle() - cdef device_resources *h = handle.getHandle() - - d_dt = output_cai.dtype - - x_c_contiguous = x_cai.c_contiguous - y_c_contiguous = y_cai.c_contiguous - - if x_c_contiguous != y_c_contiguous: - raise ValueError("Inputs must have matching strides") - - if not x_c_contiguous: - raise ValueError("Inputs must be C contiguous") - - if x_dt != y_dt: - raise ValueError("Inputs must have the same dtypes") - if d_dt != np.int32: - raise ValueError("Output array must be int32") - # unused arg for now. - metric_arg = 0.0 - if x_dt == np.float32: - fused_distance_nn_min_arg(deref(h), - d_ptr, - x_ptr, - y_ptr, - m, - n, - x_k, - sqrt, - distance_type, - x_c_contiguous, - metric_arg) - else: - raise ValueError("dtype %s not supported" % x_dt) - - return output diff --git a/python/pylibraft/pylibraft/distance/fused_l2_nn.pyx b/python/pylibraft/pylibraft/distance/fused_l2_nn.pyx deleted file mode 100644 index c8e7101ee0..0000000000 --- a/python/pylibraft/pylibraft/distance/fused_l2_nn.pyx +++ /dev/null @@ -1,193 +0,0 @@ -# -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport uintptr_t -from libcpp cimport bool - -from .distance_type cimport DistanceType - -from pylibraft.common import ( - Handle, - auto_convert_output, - cai_wrapper, - device_ndarray, -) -from pylibraft.common.handle import auto_sync_handle - -from pylibraft.common.handle cimport device_resources - - -cdef extern from "raft_runtime/distance/fused_l2_nn.hpp" \ - namespace "raft::runtime::distance" nogil: - - void fused_l2_nn_min_arg( - const device_resources &handle, - int* min, - const float* x, - const float* y, - int m, - int n, - int k, - bool sqrt) except + - - void fused_l2_nn_min_arg( - const device_resources &handle, - int* min, - const double* x, - const double* y, - int m, - int n, - int k, - bool sqrt) except + - - -@auto_sync_handle -@auto_convert_output -def fused_l2_nn_argmin(X, Y, out=None, sqrt=True, handle=None): - """ - Compute the 1-nearest neighbors between X and Y using the L2 distance - - Parameters - ---------- - - X : CUDA array interface compliant matrix shape (m, k) - Y : CUDA array interface compliant matrix shape (n, k) - output : Writable CUDA array interface matrix shape (m, 1) - {handle_docstring} - - Examples - -------- - To compute the 1-nearest neighbors argmin: - - >>> import cupy as cp - >>> from pylibraft.common import Handle - >>> from pylibraft.distance import fused_l2_nn_argmin - >>> n_samples = 5000 - >>> n_clusters = 5 - >>> n_features = 50 - >>> in1 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> in2 = cp.random.random_sample((n_clusters, n_features), - ... dtype=cp.float32) - >>> # A single RAFT handle can optionally be reused across - >>> # pylibraft functions. - >>> handle = Handle() - - >>> output = fused_l2_nn_argmin(in1, in2, handle=handle) - - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - - The output can also be computed in-place on a preallocated - array: - - >>> import cupy as cp - >>> from pylibraft.common import Handle - >>> from pylibraft.distance import fused_l2_nn_argmin - >>> n_samples = 5000 - >>> n_clusters = 5 - >>> n_features = 50 - >>> in1 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> in2 = cp.random.random_sample((n_clusters, n_features), - ... dtype=cp.float32) - >>> output = cp.empty((n_samples, 1), dtype=cp.int32) - >>> # A single RAFT handle can optionally be reused across - >>> # pylibraft functions. - >>> handle = Handle() - - >>> fused_l2_nn_argmin(in1, in2, out=output, handle=handle) - array(...) - - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - - x_cai = cai_wrapper(X) - y_cai = cai_wrapper(Y) - - x_dt = x_cai.dtype - y_dt = y_cai.dtype - - m = x_cai.shape[0] - n = y_cai.shape[0] - - if out is None: - output = device_ndarray.empty((m,), dtype="int32") - else: - output = out - - output_cai = cai_wrapper(output) - - x_k = x_cai.shape[1] - y_k = y_cai.shape[1] - - if x_k != y_k: - raise ValueError("Inputs must have same number of columns. " - "a=%s, b=%s" % (x_k, y_k)) - - x_ptr = x_cai.data - y_ptr = y_cai.data - - d_ptr = output_cai.data - - handle = handle if handle is not None else Handle() - cdef device_resources *h = handle.getHandle() - - d_dt = output_cai.dtype - - x_c_contiguous = x_cai.c_contiguous - y_c_contiguous = y_cai.c_contiguous - - if x_c_contiguous != y_c_contiguous: - raise ValueError("Inputs must have matching strides") - - if x_dt != y_dt: - raise ValueError("Inputs must have the same dtypes") - if d_dt != np.int32: - raise ValueError("Output array must be int32") - - if x_dt == np.float32: - fused_l2_nn_min_arg(deref(h), - d_ptr, - x_ptr, - y_ptr, - m, - n, - x_k, - sqrt) - elif x_dt == np.float64: - fused_l2_nn_min_arg(deref(h), - d_ptr, - x_ptr, - y_ptr, - m, - n, - x_k, - sqrt) - else: - raise ValueError("dtype %s not supported" % x_dt) - - return output diff --git a/python/pylibraft/pylibraft/distance/pairwise_distance.pyx b/python/pylibraft/pylibraft/distance/pairwise_distance.pyx deleted file mode 100644 index 20dadf0275..0000000000 --- a/python/pylibraft/pylibraft/distance/pairwise_distance.pyx +++ /dev/null @@ -1,242 +0,0 @@ -# -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport uintptr_t -from libcpp cimport bool - -from .distance_type cimport DistanceType - -from pylibraft.common import Handle -from pylibraft.common.handle import auto_sync_handle - -from pylibraft.common.handle cimport device_resources - -from pylibraft.common import auto_convert_output, cai_wrapper, device_ndarray - - -cdef extern from "raft_runtime/distance/pairwise_distance.hpp" \ - namespace "raft::runtime::distance" nogil: - - cdef void pairwise_distance(const device_resources &handle, - float *x, - float *y, - float *dists, - int m, - int n, - int k, - DistanceType metric, - bool isRowMajor, - float metric_arg) except + - - cdef void pairwise_distance(const device_resources &handle, - double *x, - double *y, - double *dists, - int m, - int n, - int k, - DistanceType metric, - bool isRowMajor, - float metric_arg) except + - -DISTANCE_TYPES = { - "l2": DistanceType.L2SqrtExpanded, - "sqeuclidean": DistanceType.L2Expanded, - "euclidean": DistanceType.L2SqrtExpanded, - "l1": DistanceType.L1, - "cityblock": DistanceType.L1, - "inner_product": DistanceType.InnerProduct, - "chebyshev": DistanceType.Linf, - "canberra": DistanceType.Canberra, - "cosine": DistanceType.CosineExpanded, - "lp": DistanceType.LpUnexpanded, - "correlation": DistanceType.CorrelationExpanded, - "jaccard": DistanceType.JaccardExpanded, - "hellinger": DistanceType.HellingerExpanded, - "braycurtis": DistanceType.BrayCurtis, - "jensenshannon": DistanceType.JensenShannon, - "hamming": DistanceType.HammingUnexpanded, - "kl_divergence": DistanceType.KLDivergence, - "minkowski": DistanceType.LpUnexpanded, - "russellrao": DistanceType.RusselRaoExpanded, - "dice": DistanceType.DiceExpanded, -} - -SUPPORTED_DISTANCES = ["euclidean", "l1", "cityblock", "l2", "inner_product", - "chebyshev", "minkowski", "canberra", "kl_divergence", - "correlation", "russellrao", "hellinger", "lp", - "hamming", "jensenshannon", "cosine", "sqeuclidean"] - - -@auto_sync_handle -@auto_convert_output -def distance(X, Y, out=None, metric="euclidean", p=2.0, handle=None): - """ - Compute pairwise distances between X and Y - - Valid values for metric: - ["euclidean", "l2", "l1", "cityblock", "inner_product", - "chebyshev", "canberra", "lp", "hellinger", "jensenshannon", - "kl_divergence", "russellrao", "minkowski", "correlation", - "cosine"] - - Parameters - ---------- - - X : CUDA array interface compliant matrix shape (m, k) - Y : CUDA array interface compliant matrix shape (n, k) - out : Optional writable CUDA array interface matrix shape (m, n) - metric : string denoting the metric type (default="euclidean") - p : metric parameter (currently used only for "minkowski") - {handle_docstring} - - Returns - ------- - - raft.device_ndarray containing pairwise distances - - Examples - -------- - To compute pairwise distances on cupy arrays: - - >>> import cupy as cp - >>> from pylibraft.common import Handle - >>> from pylibraft.distance import pairwise_distance - >>> n_samples = 5000 - >>> n_features = 50 - >>> in1 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> in2 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - - A single RAFT handle can optionally be reused across - pylibraft functions. - - >>> handle = Handle() - >>> output = pairwise_distance(in1, in2, metric="euclidean", handle=handle) - - pylibraft functions are often asynchronous so the - handle needs to be explicitly synchronized - - >>> handle.sync() - - It's also possible to write to a pre-allocated output array: - - >>> import cupy as cp - >>> from pylibraft.common import Handle - >>> from pylibraft.distance import pairwise_distance - >>> n_samples = 5000 - >>> n_features = 50 - >>> in1 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> in2 = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> output = cp.empty((n_samples, n_samples), dtype=cp.float32) - - A single RAFT handle can optionally be reused across - pylibraft functions. - - >>> - >>> handle = Handle() - >>> pairwise_distance(in1, in2, out=output, - ... metric="euclidean", handle=handle) - array(...) - - pylibraft functions are often asynchronous so the - handle needs to be explicitly synchronized - - >>> handle.sync() - """ - - x_cai = cai_wrapper(X) - y_cai = cai_wrapper(Y) - - m = x_cai.shape[0] - n = y_cai.shape[0] - - x_dt = x_cai.dtype - y_dt = y_cai.dtype - - if out is None: - dists = device_ndarray.empty((m, n), dtype=y_dt) - else: - dists = out - - x_k = x_cai.shape[1] - y_k = y_cai.shape[1] - - dists_cai = cai_wrapper(dists) - - if x_k != y_k: - raise ValueError("Inputs must have same number of columns. " - "a=%s, b=%s" % (x_k, y_k)) - - x_ptr = x_cai.data - y_ptr = y_cai.data - d_ptr = dists_cai.data - - handle = handle if handle is not None else Handle() - cdef device_resources *h = handle.getHandle() - - d_dt = dists_cai.dtype - - x_c_contiguous = x_cai.c_contiguous - y_c_contiguous = y_cai.c_contiguous - - if x_c_contiguous != y_c_contiguous: - raise ValueError("Inputs must have matching strides") - - if metric not in SUPPORTED_DISTANCES: - raise ValueError("metric %s is not supported" % metric) - - cdef DistanceType distance_type = DISTANCE_TYPES[metric] - - if x_dt != y_dt or x_dt != d_dt: - raise ValueError("Inputs must have the same dtypes") - - if x_dt == np.float32: - pairwise_distance(deref(h), - x_ptr, - y_ptr, - d_ptr, - m, - n, - x_k, - distance_type, - x_c_contiguous, - p) - elif x_dt == np.float64: - pairwise_distance(deref(h), - x_ptr, - y_ptr, - d_ptr, - m, - n, - x_k, - distance_type, - x_c_contiguous, - p) - else: - raise ValueError("dtype %s not supported" % x_dt) - - return dists diff --git a/python/pylibraft/pylibraft/matrix/CMakeLists.txt b/python/pylibraft/pylibraft/matrix/CMakeLists.txt deleted file mode 100644 index 5b7803db00..0000000000 --- a/python/pylibraft/pylibraft/matrix/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Set the list of Cython files to build -set(cython_sources select_k.pyx) -set(linked_libraries raft::raft raft::compiled) - -# Build all of the Cython targets -rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX matrix_ -) diff --git a/python/pylibraft/pylibraft/matrix/__init__.pxd b/python/pylibraft/pylibraft/matrix/__init__.pxd deleted file mode 100644 index a7e7b75096..0000000000 --- a/python/pylibraft/pylibraft/matrix/__init__.pxd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/matrix/__init__.py b/python/pylibraft/pylibraft/matrix/__init__.py deleted file mode 100644 index 5eb35795ed..0000000000 --- a/python/pylibraft/pylibraft/matrix/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .select_k import select_k - -__all__ = ["select_k"] diff --git a/python/pylibraft/pylibraft/matrix/cpp/select_k.pxd b/python/pylibraft/pylibraft/matrix/cpp/select_k.pxd deleted file mode 100644 index ab466fdce6..0000000000 --- a/python/pylibraft/pylibraft/matrix/cpp/select_k.pxd +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from libc.stdint cimport int64_t -from libcpp cimport bool - -from pylibraft.common.cpp.mdspan cimport device_matrix_view, row_major -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources - - -cdef extern from "raft_runtime/matrix/select_k.hpp" \ - namespace "raft::runtime::matrix" nogil: - - cdef void select_k(const device_resources & handle, - device_matrix_view[float, int64_t, row_major], - optional[device_matrix_view[int64_t, - int64_t, - row_major]], - device_matrix_view[float, int64_t, row_major], - device_matrix_view[int64_t, int64_t, row_major], - bool) except + diff --git a/python/pylibraft/pylibraft/matrix/select_k.pyx b/python/pylibraft/pylibraft/matrix/select_k.pyx deleted file mode 100644 index fbb1e2e5d3..0000000000 --- a/python/pylibraft/pylibraft/matrix/select_k.pyx +++ /dev/null @@ -1,133 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from cython.operator cimport dereference as deref -from libc.stdint cimport int64_t -from libcpp cimport bool - -import numpy as np - -from pylibraft.common import auto_convert_output, cai_wrapper, device_ndarray -from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.input_validation import is_c_contiguous - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - host_matrix_view, - make_device_matrix_view, - make_host_matrix_view, - row_major, -) -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources -from pylibraft.common.mdspan cimport get_dmv_float, get_dmv_int64 -from pylibraft.matrix.cpp.select_k cimport select_k as c_select_k - - -@auto_sync_handle -@auto_convert_output -def select_k(dataset, k=None, distances=None, indices=None, select_min=True, - handle=None): - """ - Selects the top k items from each row in a matrix - - - Parameters - ---------- - dataset : array interface compliant matrix, row-major layout, - shape (n_rows, dim). Supported dtype [float] - k : int - Number of items to return for each row. Optional if indices or - distances arrays are given (in which case their second dimension - is k). - distances : Optional array interface compliant matrix shape - (n_rows, k), dtype float. If supplied, - distances will be written here in-place. (default None) - indices : Optional array interface compliant matrix shape - (n_rows, k), dtype int64_t. If supplied, neighbor - indices will be written here in-place. (default None) - select_min: : bool - Whether to select the minimum or maximum K items - - {handle_docstring} - - Returns - ------- - distances: array interface compliant object containing resulting distances - shape (n_rows, k) - - indices: array interface compliant object containing resulting indices - shape (n_rows, k) - - Examples - -------- - - >>> import cupy as cp - - >>> from pylibraft.matrix import select_k - - >>> n_features = 50 - >>> n_rows = 1000 - - >>> queries = cp.random.random_sample((n_rows, n_features), - ... dtype=cp.float32) - >>> k = 40 - >>> distances, ids = select_k(queries, k) - >>> distances = cp.asarray(distances) - >>> ids = cp.asarray(ids) - """ - - dataset_cai = cai_wrapper(dataset) - - if k is None: - if indices is not None: - k = cai_wrapper(indices).shape[1] - elif distances is not None: - k = cai_wrapper(distances).shape[1] - else: - raise ValueError("Argument k must be specified if both indices " - "and distances arg is None") - - n_rows = dataset.shape[0] - if indices is None: - indices = device_ndarray.empty((n_rows, k), dtype='int64') - - if distances is None: - distances = device_ndarray.empty((n_rows, k), dtype='float32') - - distances_cai = cai_wrapper(distances) - indices_cai = cai_wrapper(indices) - - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef optional[device_matrix_view[int64_t, int64_t, row_major]] in_idx - - if dataset_cai.dtype == np.float32: - c_select_k(deref(handle_), - get_dmv_float(dataset_cai, check_shape=True), - in_idx, - get_dmv_float(distances_cai, check_shape=True), - get_dmv_int64(indices_cai, check_shape=True), - select_min) - else: - raise TypeError("dtype %s not supported" % dataset_cai.dtype) - - return distances, indices diff --git a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt deleted file mode 100644 index 069038a0e8..0000000000 --- a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Set the list of Cython files to build -set(cython_sources common.pyx refine.pyx brute_force.pyx hnsw.pyx rbc.pyx) -set(linked_libraries raft::raft raft::compiled) - -# Build all of the Cython targets -rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ -) - -add_subdirectory(cagra) -add_subdirectory(ivf_flat) -add_subdirectory(ivf_pq) diff --git a/python/pylibraft/pylibraft/neighbors/__init__.pxd b/python/pylibraft/pylibraft/neighbors/__init__.pxd deleted file mode 100644 index 273b4497cc..0000000000 --- a/python/pylibraft/pylibraft/neighbors/__init__.pxd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/neighbors/__init__.py b/python/pylibraft/pylibraft/neighbors/__init__.py deleted file mode 100644 index 86612b2fbb..0000000000 --- a/python/pylibraft/pylibraft/neighbors/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from pylibraft.neighbors import brute_force # type: ignore -from pylibraft.neighbors import hnsw # type: ignore -from pylibraft.neighbors import rbc # type: ignore -from pylibraft.neighbors import cagra, ivf_flat, ivf_pq - -from .refine import refine - -__all__ = [ - "common", - "refine", - "brute_force", - "ivf_flat", - "ivf_pq", - "cagra", - "hnsw", - "rbc", -] diff --git a/python/pylibraft/pylibraft/neighbors/brute_force.pyx b/python/pylibraft/pylibraft/neighbors/brute_force.pyx deleted file mode 100644 index 19d20fb75d..0000000000 --- a/python/pylibraft/pylibraft/neighbors/brute_force.pyx +++ /dev/null @@ -1,269 +0,0 @@ -# -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libcpp cimport bool, nullptr -from libcpp.vector cimport vector - -from pylibraft.distance.distance_type cimport DistanceType - -from pylibraft.common import ( - DeviceResources, - auto_convert_output, - cai_wrapper, - device_ndarray, -) - -from libc.stdint cimport int64_t, uintptr_t - -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources -from pylibraft.common.mdspan cimport get_dmv_bool, get_dmv_float, get_dmv_int64 - -from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.interruptible import cuda_interruptible - -from pylibraft.distance.distance_type cimport DistanceType - -# TODO: Centralize this - -from pylibraft.distance.pairwise_distance import DISTANCE_TYPES -from pylibraft.neighbors.common import _check_input_array - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - host_matrix_view, - make_device_matrix_view, - make_device_vector_view, - make_host_matrix_view, - row_major, -) -from pylibraft.neighbors.cpp.brute_force cimport ( - eps_neighbors as c_eps_neighbors, - knn as c_knn, -) - - -def _get_array_params(array_interface, check_dtype=None): - dtype = np.dtype(array_interface["typestr"]) - if check_dtype is None and dtype != check_dtype: - raise TypeError("dtype %s not supported" % dtype) - shape = array_interface["shape"] - if len(shape) != 2: - raise ValueError("Expected a 2D array, got %d D" % len(shape)) - data = array_interface["data"][0] - return (shape, dtype, data) - - -@auto_sync_handle -@auto_convert_output -def knn(dataset, queries, k=None, indices=None, distances=None, - metric="sqeuclidean", metric_arg=2.0, - global_id_offset=0, handle=None): - """ - Perform a brute-force nearest neighbors search. - - Parameters - ---------- - dataset : array interface compliant matrix, row-major layout, - shape (n_samples, dim). Supported dtype [float] - queries : array interface compliant matrix, row-major layout, - shape (n_queries, dim) Supported dtype [float] - k : int - Number of neighbors to search (k <= 2048). Optional if indices or - distances arrays are given (in which case their second dimension - is k). - indices : Optional array interface compliant matrix shape - (n_queries, k), dtype int64_t. If supplied, neighbor - indices will be written here in-place. (default None) - Supported dtype uint64 - distances : Optional array interface compliant matrix shape - (n_queries, k), dtype float. If supplied, neighbor - indices will be written here in-place. (default None) - {handle_docstring} - - Returns - ------- - indices: array interface compliant object containing resulting indices - shape (n_queries, k) - - distances: array interface compliant object containing resulting distances - shape (n_queries, k) - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors.brute_force import knn - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 40 - >>> distances, neighbors = knn(dataset, queries, k) - >>> distances = cp.asarray(distances) - >>> neighbors = cp.asarray(neighbors) - """ - - if handle is None: - handle = DeviceResources() - - dataset_cai = cai_wrapper(dataset) - queries_cai = cai_wrapper(queries) - - if k is None: - if indices is not None: - k = cai_wrapper(indices).shape[1] - elif distances is not None: - k = cai_wrapper(distances).shape[1] - else: - raise ValueError("Argument k must be specified if both indices " - "and distances arg is None") - - # we require c-contiguous (rowmajor) inputs here - _check_input_array(dataset_cai, [np.dtype("float32")]) - _check_input_array(queries_cai, [np.dtype("float32")], - exp_cols=dataset_cai.shape[1]) - - n_queries = queries_cai.shape[0] - - if indices is None: - indices = device_ndarray.empty((n_queries, k), dtype='int64') - - if distances is None: - distances = device_ndarray.empty((n_queries, k), dtype='float32') - - cdef DistanceType c_metric = DISTANCE_TYPES[metric] - - distances_cai = cai_wrapper(distances) - indices_cai = cai_wrapper(indices) - - cdef optional[float] c_metric_arg = metric_arg - cdef optional[int64_t] c_global_offset = global_id_offset - - cdef device_resources* handle_ = \ - handle.getHandle() - - if dataset_cai.dtype == np.float32: - with cuda_interruptible(): - c_knn(deref(handle_), - get_dmv_float(dataset_cai, check_shape=True), - get_dmv_float(queries_cai, check_shape=True), - get_dmv_int64(indices_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True), - c_metric, - c_metric_arg, - c_global_offset) - else: - raise TypeError("dtype %s not supported" % dataset_cai.dtype) - - return (distances, indices) - - -@auto_sync_handle -@auto_convert_output -def eps_neighbors(dataset, queries, eps, handle=None): - """ - Perform an epsilon neighborhood search using the L2-norm. - - Parameters - ---------- - dataset : array interface compliant matrix, row-major layout, - shape (n_samples, dim). Supported dtype [float] - queries : array interface compliant matrix, row-major layout, - shape (n_queries, dim) Supported dtype [float] - eps : threshold - {handle_docstring} - - Returns - ------- - adj: array interface compliant object containing bool adjacency mask - shape (n_queries, n_samples) - - vd: array interface compliant object containing row sums of adj - shape (n_queries + 1). vd[n_queries] contains the total sum - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors.brute_force import eps_neighbors - >>> handle = DeviceResources() - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> eps = 0.1 - >>> adj, vd = eps_neighbors(dataset, queries, eps, handle=handle) - >>> adj = cp.asarray(adj) - >>> vd = cp.asarray(vd) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - - if handle is None: - handle = DeviceResources() - - dataset_cai = cai_wrapper(dataset) - queries_cai = cai_wrapper(queries) - - # we require c-contiguous (rowmajor) inputs here - _check_input_array(dataset_cai, [np.dtype("float32")]) - _check_input_array(queries_cai, [np.dtype("float32")], - exp_cols=dataset_cai.shape[1]) - - n_queries = queries_cai.shape[0] - n_samples = dataset_cai.shape[0] - - adj = device_ndarray.empty((n_queries, n_samples), dtype='bool') - vd = device_ndarray.empty((n_queries + 1, ), dtype='int64') - adj_cai = cai_wrapper(adj) - vd_cai = cai_wrapper(vd) - - cdef device_resources* handle_ = \ - handle.getHandle() - - vd_vector_view = make_device_vector_view( - vd_cai.data, vd_cai.shape[0]) - - if dataset_cai.dtype == np.float32: - with cuda_interruptible(): - c_eps_neighbors( - deref(handle_), - get_dmv_float(dataset_cai, check_shape=True), - get_dmv_float(queries_cai, check_shape=True), - get_dmv_bool(adj_cai, check_shape=True), - vd_vector_view, - eps) - else: - raise TypeError("dtype %s not supported" % dataset_cai.dtype) - - return (adj, vd) diff --git a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt deleted file mode 100644 index 0939d7c5b3..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Set the list of Cython files to build -set(cython_sources cagra.pyx) -set(linked_libraries raft::raft raft::compiled) - -# Build all of the Cython targets -rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_cagra_ -) diff --git a/python/pylibraft/pylibraft/neighbors/cagra/__init__.pxd b/python/pylibraft/pylibraft/neighbors/cagra/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/neighbors/cagra/__init__.py b/python/pylibraft/pylibraft/neighbors/cagra/__init__.py deleted file mode 100644 index b2a872fc89..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cagra/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from .cagra import Index, IndexParams, SearchParams, build, load, save, search - -__all__ = [ - "Index", - "IndexParams", - "SearchParams", - "build", - "load", - "save", - "search", -] diff --git a/python/pylibraft/pylibraft/neighbors/cagra/cagra.pxd b/python/pylibraft/pylibraft/neighbors/cagra/cagra.pxd deleted file mode 100644 index 98537f8357..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cagra/cagra.pxd +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from libc.stdint cimport int8_t, uint8_t, uint32_t -from libcpp cimport bool -from libcpp.string cimport string - -cimport pylibraft.neighbors.cagra.cpp.c_cagra as c_cagra - - -cdef class Index: - cdef readonly bool trained - cdef str active_index_type - -cdef class IndexFloat(Index): - cdef c_cagra.index[float, uint32_t] * index - -cdef class IndexInt8(Index): - cdef c_cagra.index[int8_t, uint32_t] * index - -cdef class IndexUint8(Index): - cdef c_cagra.index[uint8_t, uint32_t] * index diff --git a/python/pylibraft/pylibraft/neighbors/cagra/cagra.pyx b/python/pylibraft/pylibraft/neighbors/cagra/cagra.pyx deleted file mode 100644 index 0e488a51ca..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cagra/cagra.pyx +++ /dev/null @@ -1,902 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import warnings - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport ( - int8_t, - int32_t, - int64_t, - uint8_t, - uint32_t, - uint64_t, - uintptr_t, -) -from libcpp cimport bool, nullptr -from libcpp.string cimport string - -from pylibraft.distance.distance_type cimport DistanceType - -from pylibraft.common import ( - DeviceResources, - ai_wrapper, - auto_convert_output, - cai_wrapper, - device_ndarray, -) -from pylibraft.common.cai_wrapper import wrap_array -from pylibraft.common.interruptible import cuda_interruptible - -from pylibraft.common.handle cimport device_resources - -from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.input_validation import is_c_contiguous - -from rmm._lib.memory_resource cimport ( - DeviceMemoryResource, - device_memory_resource, -) - -cimport pylibraft.neighbors.cagra.cpp.c_cagra as c_cagra -from pylibraft.common.optional cimport make_optional, optional - -from pylibraft.neighbors.common import _check_input_array, _get_metric - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - make_device_vector_view, - row_major, -) -from pylibraft.common.mdspan cimport ( - get_const_dmv_float, - get_const_dmv_int8, - get_const_dmv_uint8, - get_const_hmv_float, - get_const_hmv_int8, - get_const_hmv_uint8, - get_dmv_float, - get_dmv_int8, - get_dmv_int64, - get_dmv_uint8, - get_dmv_uint32, - get_hmv_float, - get_hmv_int8, - get_hmv_int64, - get_hmv_uint8, - get_hmv_uint32, - make_optional_view_int64, -) -from pylibraft.neighbors.common cimport _get_metric_string - - -cdef class IndexParams: - """ - Parameters to build index for CAGRA nearest neighbor search - - Parameters - ---------- - metric : string denoting the metric type, default="sqeuclidean" - Valid values for metric: ["sqeuclidean", "inner_product"], where - - sqeuclidean is the euclidean distance without the square root - operation, i.e.: distance(a,b) = \\sum_i (a_i - b_i)^2 - - inner_product is the dot product between two vectors i.e.: - distance(a, b) = \\sum_i (a_i * b_i) - intermediate_graph_degree : int, default = 128 - - graph_degree : int, default = 64 - - build_algo: string denoting the graph building algorithm to use, \ - default = "ivf_pq" - Valid values for algo: ["ivf_pq", "nn_descent"], where - - ivf_pq will use the IVF-PQ algorithm for building the knn graph - - nn_descent (experimental) will use the NN-Descent algorithm for - building the knn graph. It is expected to be generally - faster than ivf_pq. - """ - cdef c_cagra.index_params params - - def __init__(self, *, - metric="sqeuclidean", - intermediate_graph_degree=128, - graph_degree=64, - build_algo="ivf_pq"): - self.params.metric = _get_metric(metric) - self.params.metric_arg = 0 - self.params.intermediate_graph_degree = intermediate_graph_degree - self.params.graph_degree = graph_degree - if build_algo == "ivf_pq": - self.params.build_algo = c_cagra.graph_build_algo.IVF_PQ - elif build_algo == "nn_descent": - self.params.build_algo = c_cagra.graph_build_algo.NN_DESCENT - - @property - def metric(self): - return self.params.metric - - @property - def intermediate_graph_degree(self): - return self.params.intermediate_graph_degree - - @property - def graph_degree(self): - return self.params.graph_degree - - -cdef class Index: - - def __cinit__(self): - self.trained = False - self.active_index_type = None - - -cdef class IndexFloat(Index): - - def __cinit__(self, handle=None): - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - self.index = new c_cagra.index[float, uint32_t]( - deref(handle_)) - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.index.metric()) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["metric", "dim", "graph_degree"]] - attr_str = [m_str] + attr_str - return "Index(type=CAGRA, " + (", ".join(attr_str)) + ")" - - @auto_sync_handle - def update_dataset(self, dataset, handle=None): - """ Replace the dataset with a new dataset. - - Parameters - ---------- - dataset : array interface compliant matrix shape (n_samples, dim) - {handle_docstring} - """ - cdef device_resources* handle_ = \ - handle.getHandle() - - dataset_ai = wrap_array(dataset) - dataset_dt = dataset_ai.dtype - _check_input_array(dataset_ai, [np.dtype("float32")]) - - if dataset_ai.from_cai: - self.index[0].update_dataset(deref(handle_), - get_const_dmv_float(dataset_ai, - check_shape=True)) - else: - self.index[0].update_dataset(deref(handle_), - get_const_hmv_float(dataset_ai, - check_shape=True)) - - @property - def metric(self): - return self.index[0].metric() - - @property - def size(self): - return self.index[0].size() - - @property - def dim(self): - return self.index[0].dim() - - @property - def graph_degree(self): - return self.index[0].graph_degree() - - def __dealloc__(self): - if self.index is not NULL: - del self.index - - -cdef class IndexInt8(Index): - - def __cinit__(self, handle=None): - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - self.index = new c_cagra.index[int8_t, uint32_t]( - deref(handle_)) - - @auto_sync_handle - def update_dataset(self, dataset, handle=None): - """ Replace the dataset with a new dataset. - - Parameters - ---------- - dataset : array interface compliant matrix shape (n_samples, dim) - {handle_docstring} - """ - cdef device_resources* handle_ = \ - handle.getHandle() - - dataset_ai = wrap_array(dataset) - dataset_dt = dataset_ai.dtype - _check_input_array(dataset_ai, [np.dtype("byte")]) - - if dataset_ai.from_cai: - self.index[0].update_dataset(deref(handle_), - get_const_dmv_int8(dataset_ai, - check_shape=True)) - else: - self.index[0].update_dataset(deref(handle_), - get_const_hmv_int8(dataset_ai, - check_shape=True)) - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.index.metric()) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["metric", "dim", "graph_degree"]] - attr_str = [m_str] + attr_str - return "Index(type=CAGRA, " + (", ".join(attr_str)) + ")" - - @property - def metric(self): - return self.index[0].metric() - - @property - def size(self): - return self.index[0].size() - - @property - def dim(self): - return self.index[0].dim() - - @property - def graph_degree(self): - return self.index[0].graph_degree() - - def __dealloc__(self): - if self.index is not NULL: - del self.index - - -cdef class IndexUint8(Index): - - def __cinit__(self, handle=None): - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - self.index = new c_cagra.index[uint8_t, uint32_t]( - deref(handle_)) - - @auto_sync_handle - def update_dataset(self, dataset, handle=None): - """ Replace the dataset with a new dataset. - - Parameters - ---------- - dataset : array interface compliant matrix shape (n_samples, dim) - {handle_docstring} - """ - cdef device_resources* handle_ = \ - handle.getHandle() - - dataset_ai = wrap_array(dataset) - dataset_dt = dataset_ai.dtype - _check_input_array(dataset_ai, [np.dtype("ubyte")]) - - if dataset_ai.from_cai: - self.index[0].update_dataset(deref(handle_), - get_const_dmv_uint8(dataset_ai, - check_shape=True)) - else: - self.index[0].update_dataset(deref(handle_), - get_const_hmv_uint8(dataset_ai, - check_shape=True)) - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.index.metric()) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["metric", "dim", "graph_degree"]] - attr_str = [m_str] + attr_str - return "Index(type=CAGRA, " + (", ".join(attr_str)) + ")" - - @property - def metric(self): - return self.index[0].metric() - - @property - def size(self): - return self.index[0].size() - - @property - def dim(self): - return self.index[0].dim() - - @property - def graph_degree(self): - return self.index[0].graph_degree() - - def __dealloc__(self): - if self.index is not NULL: - del self.index - - -@auto_sync_handle -@auto_convert_output -def build(IndexParams index_params, dataset, handle=None): - """ - Build the CAGRA index from the dataset for efficient search. - - The build performs two different steps- first an intermediate knn-graph is - constructed, then it's optimized it to create the final graph. The - index_params object controls the node degree of these graphs. - - It is required that both the dataset and the optimized graph fit the - GPU memory. - - The following distance metrics are supported: - - L2 - - inner_product - - Parameters - ---------- - index_params : IndexParams object - dataset : CUDA array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - {handle_docstring} - - Returns - ------- - index: cagra.Index - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import cagra - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> k = 10 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> build_params = cagra.IndexParams(metric="sqeuclidean") - >>> index = cagra.build(build_params, dataset, handle=handle) - >>> distances, neighbors = cagra.search(cagra.SearchParams(), - ... index, dataset, - ... k, handle=handle) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - >>> distances = cp.asarray(distances) - >>> neighbors = cp.asarray(neighbors) - """ - dataset_ai = wrap_array(dataset) - dataset_dt = dataset_ai.dtype - _check_input_array(dataset_ai, [np.dtype('float32'), np.dtype('byte'), - np.dtype('ubyte')]) - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - if dataset_ai.from_cai: - if dataset_dt == np.float32: - idx_float = IndexFloat(handle) - idx_float.active_index_type = "float32" - with cuda_interruptible(): - c_cagra.build_device( - deref(handle_), - index_params.params, - get_dmv_float(dataset_ai, check_shape=True), - deref(idx_float.index)) - idx_float.trained = True - return idx_float - elif dataset_dt == np.byte: - idx_int8 = IndexInt8(handle) - idx_int8.active_index_type = "byte" - with cuda_interruptible(): - c_cagra.build_device( - deref(handle_), - index_params.params, - get_dmv_int8(dataset_ai, check_shape=True), - deref(idx_int8.index)) - idx_int8.trained = True - return idx_int8 - elif dataset_dt == np.ubyte: - idx_uint8 = IndexUint8(handle) - idx_uint8.active_index_type = "ubyte" - with cuda_interruptible(): - c_cagra.build_device( - deref(handle_), - index_params.params, - get_dmv_uint8(dataset_ai, check_shape=True), - deref(idx_uint8.index)) - idx_uint8.trained = True - return idx_uint8 - else: - raise TypeError("dtype %s not supported" % dataset_dt) - else: - if dataset_dt == np.float32: - idx_float = IndexFloat(handle) - idx_float.active_index_type = "float32" - with cuda_interruptible(): - c_cagra.build_host( - deref(handle_), - index_params.params, - get_hmv_float(dataset_ai, check_shape=True), - deref(idx_float.index)) - idx_float.trained = True - return idx_float - elif dataset_dt == np.byte: - idx_int8 = IndexInt8(handle) - idx_int8.active_index_type = "byte" - with cuda_interruptible(): - c_cagra.build_host( - deref(handle_), - index_params.params, - get_hmv_int8(dataset_ai, check_shape=True), - deref(idx_int8.index)) - idx_int8.trained = True - return idx_int8 - elif dataset_dt == np.ubyte: - idx_uint8 = IndexUint8(handle) - idx_uint8.active_index_type = "ubyte" - with cuda_interruptible(): - c_cagra.build_host( - deref(handle_), - index_params.params, - get_hmv_uint8(dataset_ai, check_shape=True), - deref(idx_uint8.index)) - idx_uint8.trained = True - return idx_uint8 - else: - raise TypeError("dtype %s not supported" % dataset_dt) - - -cdef class SearchParams: - """ - CAGRA search parameters - - Parameters - ---------- - max_queries: int, default = 0 - Maximum number of queries to search at the same time (batch size). - Auto select when 0. - itopk_size: int, default = 64 - Number of intermediate search results retained during the search. - This is the main knob to adjust trade off between accuracy and - search speed. Higher values improve the search accuracy. - max_iterations: int, default = 0 - Upper limit of search iterations. Auto select when 0. - algo: string denoting the search algorithm to use, default = "auto" - Valid values for algo: ["auto", "single_cta", "multi_cta"], where - - auto will automatically select the best value based on query size - - single_cta is better when query contains larger number of - vectors (e.g >10) - - multi_cta is better when query contains only a few vectors - team_size: int, default = 0 - Number of threads used to calculate a single distance. 4, 8, 16, - or 32. - search_width: int, default = 1 - Number of graph nodes to select as the starting point for the - search in each iteration. - min_iterations: int, default = 0 - Lower limit of search iterations. - thread_block_size: int, default = 0 - Thread block size. 0, 64, 128, 256, 512, 1024. - Auto selection when 0. - hashmap_mode: string denoting the type of hash map to use. - It's usually better to allow the algorithm to select this value, - default = "auto". - Valid values for hashmap_mode: ["auto", "small", "hash"], where - - auto will automatically select the best value based on algo - - small will use the small shared memory hash table with resetting. - - hash will use a single hash table in global memory. - hashmap_min_bitlen: int, default = 0 - Upper limit of hashmap fill rate. More than 0.1, less than 0.9. - hashmap_max_fill_rate: float, default = 0.5 - Upper limit of hashmap fill rate. More than 0.1, less than 0.9. - num_random_samplings: int, default = 1 - Number of iterations of initial random seed node selection. 1 or - more. - rand_xor_mask: int, default = 0x128394 - Bit mask used for initial random seed node selection. - """ - cdef c_cagra.search_params params - - def __init__(self, *, - max_queries=0, - itopk_size=64, - max_iterations=0, - algo="auto", - team_size=0, - search_width=1, - min_iterations=0, - thread_block_size=0, - hashmap_mode="auto", - hashmap_min_bitlen=0, - hashmap_max_fill_rate=0.5, - num_random_samplings=1, - rand_xor_mask=0x128394): - self.params.max_queries = max_queries - self.params.itopk_size = itopk_size - self.params.max_iterations = max_iterations - if algo == "single_cta": - self.params.algo = c_cagra.search_algo.SINGLE_CTA - elif algo == "multi_cta": - self.params.algo = c_cagra.search_algo.MULTI_CTA - elif algo == "multi_kernel": - self.params.algo = c_cagra.search_algo.MULTI_KERNEL - elif algo == "auto": - self.params.algo = c_cagra.search_algo.AUTO - else: - raise ValueError("`algo` value not supported.") - - self.params.team_size = team_size - self.params.search_width = search_width - self.params.min_iterations = min_iterations - self.params.thread_block_size = thread_block_size - if hashmap_mode == "hash": - self.params.hashmap_mode = c_cagra.hash_mode.HASH - elif hashmap_mode == "small": - self.params.hashmap_mode = c_cagra.hash_mode.SMALL - elif hashmap_mode == "auto": - self.params.hashmap_mode = c_cagra.hash_mode.AUTO - else: - raise ValueError("`hashmap_mode` value not supported.") - - self.params.hashmap_min_bitlen = hashmap_min_bitlen - self.params.hashmap_max_fill_rate = hashmap_max_fill_rate - self.params.num_random_samplings = num_random_samplings - self.params.rand_xor_mask = rand_xor_mask - - def __repr__(self): - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in [ - "max_queries", "itopk_size", "max_iterations", "algo", - "team_size", "search_width", "min_iterations", - "thread_block_size", "hashmap_mode", - "hashmap_min_bitlen", "hashmap_max_fill_rate", - "num_random_samplings", "rand_xor_mask"]] - return "SearchParams(type=CAGRA, " + (", ".join(attr_str)) + ")" - - @property - def max_queries(self): - return self.params.max_queries - - @property - def itopk_size(self): - return self.params.itopk_size - - @property - def max_iterations(self): - return self.params.max_iterations - - @property - def algo(self): - return self.params.algo - - @property - def team_size(self): - return self.params.team_size - - @property - def search_width(self): - return self.params.search_width - - @property - def min_iterations(self): - return self.params.min_iterations - - @property - def thread_block_size(self): - return self.params.thread_block_size - - @property - def hashmap_mode(self): - return self.params.hashmap_mode - - @property - def hashmap_min_bitlen(self): - return self.params.hashmap_min_bitlen - - @property - def hashmap_max_fill_rate(self): - return self.params.hashmap_max_fill_rate - - @property - def num_random_samplings(self): - return self.params.num_random_samplings - - @property - def rand_xor_mask(self): - return self.params.rand_xor_mask - - -@auto_sync_handle -@auto_convert_output -def search(SearchParams search_params, - Index index, - queries, - k, - neighbors=None, - distances=None, - handle=None): - """ - Find the k nearest neighbors for each query. - - Parameters - ---------- - search_params : SearchParams - index : Index - Trained CAGRA index. - queries : CUDA array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - k : int - The number of neighbors. - neighbors : Optional CUDA array interface compliant matrix shape - (n_queries, k), dtype int64_t. If supplied, neighbor - indices will be written here in-place. (default None) - distances : Optional CUDA array interface compliant matrix shape - (n_queries, k) If supplied, the distances to the - neighbors will be written here in-place. (default None) - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import cagra - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = cagra.build(cagra.IndexParams(), dataset, handle=handle) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 10 - >>> search_params = cagra.SearchParams( - ... max_queries=100, - ... itopk_size=64 - ... ) - >>> # Using a pooling allocator reduces overhead of temporary array - >>> # creation during search. This is useful if multiple searches - >>> # are performad with same query size. - >>> distances, neighbors = cagra.search(search_params, index, queries, - ... k, handle=handle) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - >>> neighbors = cp.asarray(neighbors) - >>> distances = cp.asarray(distances) - """ - - if not index.trained: - raise ValueError("Index need to be built before calling search.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - queries_cai = cai_wrapper(queries) - queries_dt = queries_cai.dtype - cdef uint32_t n_queries = queries_cai.shape[0] - - _check_input_array(queries_cai, [np.dtype('float32'), np.dtype('byte'), - np.dtype('ubyte')], - exp_cols=index.dim) - - if neighbors is None: - neighbors = device_ndarray.empty((n_queries, k), dtype='uint32') - - neighbors_cai = cai_wrapper(neighbors) - _check_input_array(neighbors_cai, [np.dtype('uint32')], - exp_rows=n_queries, exp_cols=k) - - if distances is None: - distances = device_ndarray.empty((n_queries, k), dtype='float32') - - distances_cai = cai_wrapper(distances) - _check_input_array(distances_cai, [np.dtype('float32')], - exp_rows=n_queries, exp_cols=k) - - cdef c_cagra.search_params params = search_params.params - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - if queries_dt == np.float32: - idx_float = index - with cuda_interruptible(): - c_cagra.search(deref(handle_), - params, - deref(idx_float.index), - get_dmv_float(queries_cai, check_shape=True), - get_dmv_uint32(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - elif queries_dt == np.byte: - idx_int8 = index - with cuda_interruptible(): - c_cagra.search(deref(handle_), - params, - deref(idx_int8.index), - get_dmv_int8(queries_cai, check_shape=True), - get_dmv_uint32(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - elif queries_dt == np.ubyte: - idx_uint8 = index - with cuda_interruptible(): - c_cagra.search(deref(handle_), - params, - deref(idx_uint8.index), - get_dmv_uint8(queries_cai, check_shape=True), - get_dmv_uint32(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - else: - raise ValueError("query dtype %s not supported" % queries_dt) - - return (distances, neighbors) - - -@auto_sync_handle -def save(filename, Index index, bool include_dataset=True, handle=None): - """ - Saves the index to a file. - - Saving / loading the index is experimental. The serialization format is - subject to change. - - Parameters - ---------- - filename : string - Name of the file. - index : Index - Trained CAGRA index. - include_dataset : bool - Whether or not to write out the dataset along with the index. Including - the dataset in the serialized index will use extra disk space, and - might not be desired if you already have a copy of the dataset on - disk. If this option is set to false, you will have to call - `index.update_dataset(dataset)` after loading the index. - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import cagra - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = cagra.build(cagra.IndexParams(), dataset, handle=handle) - >>> # Serialize and deserialize the cagra index built - >>> cagra.save("my_index.bin", index, handle=handle) - >>> index_loaded = cagra.load("my_index.bin", handle=handle) - """ - if not index.trained: - raise ValueError("Index need to be built before saving it.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - if index.active_index_type == "float32": - idx_float = index - c_cagra.serialize_file( - deref(handle_), c_filename, deref(idx_float.index), - include_dataset) - elif index.active_index_type == "byte": - idx_int8 = index - c_cagra.serialize_file( - deref(handle_), c_filename, deref(idx_int8.index), include_dataset) - elif index.active_index_type == "ubyte": - idx_uint8 = index - c_cagra.serialize_file( - deref(handle_), c_filename, deref(idx_uint8.index), - include_dataset) - else: - raise ValueError( - "Index dtype %s not supported" % index.active_index_type) - - -@auto_sync_handle -def load(filename, handle=None): - """ - Loads index from file. - - Saving / loading the index is experimental. The serialization format is - subject to change, therefore loading an index saved with a previous - version of raft is not guaranteed to work. - - Parameters - ---------- - filename : string - Name of the file. - {handle_docstring} - - Returns - ------- - index : Index - - """ - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - with open(filename, "rb") as f: - type_str = f.read(3).decode("utf8") - dataset_dt = np.dtype(type_str) - - if dataset_dt == np.float32: - idx_float = IndexFloat(handle) - c_cagra.deserialize_file( - deref(handle_), c_filename, idx_float.index) - idx_float.trained = True - idx_float.active_index_type = 'float32' - return idx_float - elif dataset_dt == np.byte: - idx_int8 = IndexInt8(handle) - c_cagra.deserialize_file( - deref(handle_), c_filename, idx_int8.index) - idx_int8.trained = True - idx_int8.active_index_type = 'byte' - return idx_int8 - elif dataset_dt == np.ubyte: - idx_uint8 = IndexUint8(handle) - c_cagra.deserialize_file( - deref(handle_), c_filename, idx_uint8.index) - idx_uint8.trained = True - idx_uint8.active_index_type = 'ubyte' - return idx_uint8 - else: - raise ValueError("Dataset dtype %s not supported" % dataset_dt) diff --git a/python/pylibraft/pylibraft/neighbors/cagra/cpp/__init__.pxd b/python/pylibraft/pylibraft/neighbors/cagra/cpp/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/neighbors/cagra/cpp/__init__.py b/python/pylibraft/pylibraft/neighbors/cagra/cpp/__init__.py deleted file mode 100644 index 8f2cc34855..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cagra/cpp/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/neighbors/cagra/cpp/c_cagra.pxd b/python/pylibraft/pylibraft/neighbors/cagra/cpp/c_cagra.pxd deleted file mode 100644 index 1dffd40186..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cagra/cpp/c_cagra.pxd +++ /dev/null @@ -1,255 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -import pylibraft.common.handle - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, int64_t, uint8_t, uint32_t, uint64_t -from libcpp cimport bool, nullptr -from libcpp.string cimport string - -from rmm._lib.memory_resource cimport device_memory_resource - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - host_matrix_view, - row_major, -) -from pylibraft.common.handle cimport device_resources -from pylibraft.common.mdspan cimport const_float, const_int8_t, const_uint8_t -from pylibraft.common.optional cimport optional -from pylibraft.distance.distance_type cimport DistanceType -from pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq cimport ( - ann_index, - ann_index_params, - ann_search_params, - index_params as ivfpq_ip, - search_params as ivfpq_sp, -) - - -cdef extern from "raft/neighbors/cagra_types.hpp" \ - namespace "raft::neighbors::cagra" nogil: - - ctypedef enum graph_build_algo: - IVF_PQ "raft::neighbors::cagra::graph_build_algo::IVF_PQ", - NN_DESCENT "raft::neighbors::cagra::graph_build_algo::NN_DESCENT" - - cpdef cppclass index_params(ann_index_params): - size_t intermediate_graph_degree - size_t graph_degree - graph_build_algo build_algo - - ctypedef enum search_algo: - SINGLE_CTA "raft::neighbors::cagra::search_algo::SINGLE_CTA", - MULTI_CTA "raft::neighbors::cagra::search_algo::MULTI_CTA", - MULTI_KERNEL "raft::neighbors::cagra::search_algo::MULTI_KERNEL", - AUTO "raft::neighbors::cagra::search_algo::AUTO" - - ctypedef enum hash_mode: - HASH "raft::neighbors::cagra::hash_mode::HASH", - SMALL "raft::neighbors::cagra::hash_mode::SMALL", - AUTO "raft::neighbors::cagra::hash_mode::AUTO" - - cpdef cppclass search_params(ann_search_params): - size_t max_queries - size_t itopk_size - size_t max_iterations - search_algo algo - size_t team_size - size_t search_width - size_t min_iterations - size_t thread_block_size - hash_mode hashmap_mode - size_t hashmap_min_bitlen - float hashmap_max_fill_rate - uint32_t num_random_samplings - uint64_t rand_xor_mask - - cdef cppclass index[T, IdxT](ann_index): - index(const device_resources&) - - DistanceType metric() - IdxT size() - uint32_t dim() - uint32_t graph_degree() - device_matrix_view[T, IdxT, row_major] dataset() - device_matrix_view[T, IdxT, row_major] graph() - - # hack: can't use the T template param here because of issues handling - # const w/ cython. introduce a new template param to get around this - void update_dataset[ValueT](const device_resources & handle, - host_matrix_view[ValueT, - int64_t, - row_major] dataset) - void update_dataset[ValueT](const device_resources & handle, - device_matrix_view[ValueT, - int64_t, - row_major] dataset) - -cdef extern from "raft_runtime/neighbors/cagra.hpp" \ - namespace "raft::runtime::neighbors::cagra" nogil: - - cdef void build_device( - const device_resources& handle, - const index_params& params, - device_matrix_view[float, int64_t, row_major] dataset, - index[float, uint32_t]& index) except + - - cdef void build_device( - const device_resources& handle, - const index_params& params, - device_matrix_view[int8_t, int64_t, row_major] dataset, - index[int8_t, uint32_t]& index) except + - - cdef void build_device( - const device_resources& handle, - const index_params& params, - device_matrix_view[uint8_t, int64_t, row_major] dataset, - index[uint8_t, uint32_t]& index) except + - - cdef void build_host( - const device_resources& handle, - const index_params& params, - host_matrix_view[float, int64_t, row_major] dataset, - index[float, uint32_t]& index) except + - - cdef void build_host( - const device_resources& handle, - const index_params& params, - host_matrix_view[int8_t, int64_t, row_major] dataset, - index[int8_t, uint32_t]& index) except + - - cdef void build_host( - const device_resources& handle, - const index_params& params, - host_matrix_view[uint8_t, int64_t, row_major] dataset, - index[uint8_t, uint32_t]& index) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[float, uint32_t]& index, - device_matrix_view[float, int64_t, row_major] queries, - device_matrix_view[uint32_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[int8_t, uint32_t]& index, - device_matrix_view[int8_t, int64_t, row_major] queries, - device_matrix_view[uint32_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[uint8_t, uint32_t]& index, - device_matrix_view[uint8_t, int64_t, row_major] queries, - device_matrix_view[uint32_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void serialize(const device_resources& handle, - string& str, - const index[float, uint32_t]& index, - bool include_dataset) except + - - cdef void deserialize(const device_resources& handle, - const string& str, - index[float, uint32_t]* index) except + - - cdef void serialize(const device_resources& handle, - string& str, - const index[uint8_t, uint32_t]& index, - bool include_dataset) except + - - cdef void deserialize(const device_resources& handle, - const string& str, - index[uint8_t, uint32_t]* index) except + - - cdef void serialize(const device_resources& handle, - string& str, - const index[int8_t, uint32_t]& index, - bool include_dataset) except + - - cdef void deserialize(const device_resources& handle, - const string& str, - index[int8_t, uint32_t]* index) except + - - cdef void serialize_file(const device_resources& handle, - const string& filename, - const index[float, uint32_t]& index, - bool include_dataset) except + - - cdef void deserialize_file(const device_resources& handle, - const string& filename, - index[float, uint32_t]* index) except + - - cdef void serialize_file(const device_resources& handle, - const string& filename, - const index[uint8_t, uint32_t]& index, - bool include_dataset) except + - - cdef void serialize_to_hnswlib( - const device_resources& handle, - string& str, - const index[float, uint32_t]& index) except + - - cdef void serialize_to_hnswlib( - const device_resources& handle, - string& str, - const index[uint8_t, uint32_t]& index) except + - - cdef void serialize_to_hnswlib( - const device_resources& handle, - string& str, - const index[int8_t, uint32_t]& index) except + - - cdef void serialize_to_hnswlib_file( - const device_resources& handle, - const string& filename, - const index[float, uint32_t]& index) except + - - cdef void serialize_to_hnswlib_file( - const device_resources& handle, - const string& filename, - const index[uint8_t, uint32_t]& index) except + - - cdef void serialize_to_hnswlib_file( - const device_resources& handle, - const string& filename, - const index[int8_t, uint32_t]& index) except + - - cdef void deserialize_file(const device_resources& handle, - const string& filename, - index[uint8_t, uint32_t]* index) except + - - cdef void serialize_file(const device_resources& handle, - const string& filename, - const index[int8_t, uint32_t]& index, - bool include_dataset) except + - - cdef void deserialize_file(const device_resources& handle, - const string& filename, - index[int8_t, uint32_t]* index) except + diff --git a/python/pylibraft/pylibraft/neighbors/common.pxd b/python/pylibraft/pylibraft/neighbors/common.pxd deleted file mode 100644 index b11ef3176e..0000000000 --- a/python/pylibraft/pylibraft/neighbors/common.pxd +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from pylibraft.distance.distance_type cimport DistanceType - - -cdef _get_metric_string(DistanceType metric) diff --git a/python/pylibraft/pylibraft/neighbors/common.pyx b/python/pylibraft/pylibraft/neighbors/common.pyx deleted file mode 100644 index 24c1abcf18..0000000000 --- a/python/pylibraft/pylibraft/neighbors/common.pyx +++ /dev/null @@ -1,63 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import warnings - -from pylibraft.distance.distance_type cimport DistanceType - -SUPPORTED_DISTANCES = { - "sqeuclidean": DistanceType.L2Expanded, - "euclidean": DistanceType.L2SqrtExpanded, - "inner_product": DistanceType.InnerProduct, - -} - - -def _get_metric(metric): - if metric not in SUPPORTED_DISTANCES: - if metric == "l2_expanded": - warnings.warn("Using l2_expanded as a metric name is deprecated," - " use sqeuclidean instead", FutureWarning) - return DistanceType.L2Expanded - - raise ValueError("metric %s is not supported" % metric) - return SUPPORTED_DISTANCES[metric] - - -cdef _get_metric_string(DistanceType metric): - return {DistanceType.L2Expanded : "sqeuclidean", - DistanceType.InnerProduct: "inner_product", - DistanceType.L2SqrtExpanded: "euclidean"}[metric] - - -def _check_input_array(cai, exp_dt, exp_rows=None, exp_cols=None): - if cai.dtype not in exp_dt: - raise TypeError("dtype %s not supported" % cai.dtype) - - if not cai.c_contiguous: - raise ValueError("Row major input is expected") - - if exp_cols is not None and cai.shape[1] != exp_cols: - raise ValueError("Incorrect number of columns, expected {} got {}" - .format(exp_cols, cai.shape[1])) - - if exp_rows is not None and cai.shape[0] != exp_rows: - raise ValueError("Incorrect number of rows, expected {} , got {}" - .format(exp_rows, cai.shape[0])) diff --git a/python/pylibraft/pylibraft/neighbors/cpp/__init__.pxd b/python/pylibraft/pylibraft/neighbors/cpp/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/neighbors/cpp/__init__.py b/python/pylibraft/pylibraft/neighbors/cpp/__init__.py deleted file mode 100644 index a7e7b75096..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cpp/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/neighbors/cpp/brute_force.pxd b/python/pylibraft/pylibraft/neighbors/cpp/brute_force.pxd deleted file mode 100644 index 5f6a83a9dc..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cpp/brute_force.pxd +++ /dev/null @@ -1,68 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -import pylibraft.common.handle - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, int64_t, uint8_t, uint64_t, uintptr_t -from libcpp cimport bool, nullptr -from libcpp.string cimport string -from libcpp.vector cimport vector - -from rmm._lib.memory_resource cimport device_memory_resource - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - host_matrix_view, - make_device_matrix_view, - make_device_vector_view, - make_host_matrix_view, - row_major, -) -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources -from pylibraft.distance.distance_type cimport DistanceType - - -cdef extern from "raft_runtime/neighbors/brute_force.hpp" \ - namespace "raft::runtime::neighbors::brute_force" nogil: - - cdef void knn(const device_resources & handle, - device_matrix_view[float, int64_t, row_major] index, - device_matrix_view[float, int64_t, row_major] search, - device_matrix_view[int64_t, int64_t, row_major] indices, - device_matrix_view[float, int64_t, row_major] distances, - DistanceType metric, - optional[float] metric_arg, - optional[int64_t] global_id_offset) except + - -cdef extern from "raft_runtime/neighbors/eps_neighborhood.hpp" \ - namespace "raft::runtime::neighbors::epsilon_neighborhood" nogil: - - cdef void eps_neighbors( - const device_resources & handle, - device_matrix_view[float, int64_t, row_major] index, - device_matrix_view[float, int64_t, row_major] search, - device_matrix_view[bool, int64_t, row_major] adj, - device_vector_view[int64_t, int64_t] vd, - float eps) except + diff --git a/python/pylibraft/pylibraft/neighbors/cpp/hnsw.pxd b/python/pylibraft/pylibraft/neighbors/cpp/hnsw.pxd deleted file mode 100644 index 7b2cf59c81..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cpp/hnsw.pxd +++ /dev/null @@ -1,82 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from libc.stdint cimport int8_t, int64_t, uint8_t, uint32_t, uint64_t -from libcpp.memory cimport unique_ptr -from libcpp.string cimport string - -from pylibraft.common.cpp.mdspan cimport ( - device_vector_view, - host_matrix_view, - row_major, -) -from pylibraft.common.handle cimport device_resources -from pylibraft.distance.distance_type cimport DistanceType -from pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq cimport ( - ann_index, - ann_search_params, -) - - -cdef extern from "raft/neighbors/hnsw.hpp" \ - namespace "raft::neighbors::hnsw" nogil: - - cpdef cppclass search_params(ann_search_params): - int ef - int num_threads - - cdef cppclass index[T](ann_index): - index(int dim, DistanceType metric) - - int dim() - DistanceType metric() - - -cdef extern from "raft_runtime/neighbors/hnsw.hpp" \ - namespace "raft::runtime::neighbors::hnsw" nogil: - cdef void search( - const device_resources& handle, - const search_params& params, - const index[float]& index, - host_matrix_view[float, int64_t, row_major] queries, - host_matrix_view[uint64_t, int64_t, row_major] neighbors, - host_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[int8_t]& index, - host_matrix_view[int8_t, int64_t, row_major] queries, - host_matrix_view[uint64_t, int64_t, row_major] neighbors, - host_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[uint8_t]& index, - host_matrix_view[uint8_t, int64_t, row_major] queries, - host_matrix_view[uint64_t, int64_t, row_major] neighbors, - host_matrix_view[float, int64_t, row_major] distances) except + - - cdef unique_ptr[index[T]] deserialize_file[T]( - const device_resources& handle, - const string& filename, - int dim, - DistanceType metric) except + diff --git a/python/pylibraft/pylibraft/neighbors/cpp/rbc.pxd b/python/pylibraft/pylibraft/neighbors/cpp/rbc.pxd deleted file mode 100644 index 531c0dc2c1..0000000000 --- a/python/pylibraft/pylibraft/neighbors/cpp/rbc.pxd +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -import pylibraft.common.handle - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, int64_t, uint8_t, uint64_t, uintptr_t -from libcpp cimport bool, nullptr -from libcpp.string cimport string -from libcpp.vector cimport vector - -from rmm._lib.memory_resource cimport device_memory_resource - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - host_matrix_view, - make_device_matrix_view, - make_host_matrix_view, - row_major, -) -from pylibraft.common.handle cimport device_resources -from pylibraft.distance.distance_type cimport DistanceType - - -cdef extern from "raft/neighbors/ball_cover_types.hpp" \ - namespace "raft::neighbors::ball_cover" nogil: - - cdef cppclass BallCoverIndex[IdxT, T, IntT, MatIdxT]: - BallCoverIndex(const device_resources& handle, - device_matrix_view[T, MatIdxT, row_major] dataset, - DistanceType metric) - - -cdef extern from "raft_runtime/neighbors/eps_neighborhood.hpp" \ - namespace "raft::runtime::neighbors::epsilon_neighborhood" nogil: - - cdef void eps_neighbors_rbc( - const device_resources & handle, - device_matrix_view[float, int64_t, row_major] index, - device_matrix_view[float, int64_t, row_major] search, - device_matrix_view[bool, int64_t, row_major] adj, - device_vector_view[int64_t, int64_t] vd, - float eps) except + - - cdef void build_rbc_index( - const device_resources & handle, - BallCoverIndex[int64_t, float, int64_t, int64_t] rbc_index) except + - - cdef void eps_neighbors_rbc_pass1( - const device_resources & handle, - BallCoverIndex[int64_t, float, int64_t, int64_t] rbc_index, - device_matrix_view[float, int64_t, row_major] search, - device_vector_view[int64_t, int64_t] adj_ia, - device_vector_view[int64_t, int64_t] vd, - float eps) except + - - cdef void eps_neighbors_rbc_pass2( - const device_resources & handle, - BallCoverIndex[int64_t, float, int64_t, int64_t] rbc_index, - device_matrix_view[float, int64_t, row_major] search, - device_vector_view[int64_t, int64_t] adj_ia, - device_vector_view[int64_t, int64_t] adj_ja, - device_vector_view[int64_t, int64_t] vd, - float eps) except + diff --git a/python/pylibraft/pylibraft/neighbors/hnsw.pyx b/python/pylibraft/pylibraft/neighbors/hnsw.pyx deleted file mode 100644 index e6f2d69eb8..0000000000 --- a/python/pylibraft/pylibraft/neighbors/hnsw.pyx +++ /dev/null @@ -1,490 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, uint8_t, uint32_t -from libcpp cimport bool -from libcpp.memory cimport unique_ptr -from libcpp.string cimport string - -cimport pylibraft.neighbors.cagra.cpp.c_cagra as c_cagra -from pylibraft.distance.distance_type cimport DistanceType -from pylibraft.neighbors.cagra.cagra cimport ( - Index, - IndexFloat, - IndexInt8, - IndexUint8, -) - -from pylibraft.common.handle import auto_sync_handle - -from pylibraft.common.handle cimport device_resources - -from pylibraft.common import DeviceResources, ai_wrapper, auto_convert_output - -cimport pylibraft.neighbors.cpp.hnsw as c_hnsw - -from pylibraft.neighbors.common import _check_input_array, _get_metric - -from pylibraft.common.mdspan cimport ( - get_hmv_float, - get_hmv_int8, - get_hmv_uint8, - get_hmv_uint64, -) -from pylibraft.neighbors.common cimport _get_metric_string - -import os -import uuid - -import numpy as np - - -cdef class HnswIndex: - cdef readonly bool trained - cdef str active_index_type - - def __cinit__(self): - self.trained = False - self.active_index_type = None - -cdef class HnswIndexFloat(HnswIndex): - cdef unique_ptr[c_hnsw.index[float]] index - - def __cinit__(self): - pass - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.metric) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["dim"]] - attr_str = [m_str] + attr_str - return "Index(type=hnsw, " + (", ".join(attr_str)) + ")" - - @property - def dim(self): - return self.index.get()[0].dim() - - @property - def metric(self): - return self.index.get()[0].metric() - -cdef class HnswIndexInt8(HnswIndex): - cdef unique_ptr[c_hnsw.index[int8_t]] index - - def __cinit__(self): - pass - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.metric) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["dim"]] - attr_str = [m_str] + attr_str - return "Index(type=hnsw, " + (", ".join(attr_str)) + ")" - - @property - def dim(self): - return self.index.get()[0].dim() - - @property - def metric(self): - return self.index.get()[0].metric() - -cdef class HnswIndexUint8(HnswIndex): - cdef unique_ptr[c_hnsw.index[uint8_t]] index - - def __cinit__(self): - pass - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.metric) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["dim"]] - attr_str = [m_str] + attr_str - return "Index(type=hnsw, " + (", ".join(attr_str)) + ")" - - @property - def dim(self): - return self.index.get()[0].dim() - - @property - def metric(self): - return self.index.get()[0].metric() - - -@auto_sync_handle -def save(filename, Index index, handle=None): - """ - Saves the CAGRA index as an hnswlib base-layer-only index to a file. - - Saving / loading the index is experimental. The serialization format is - subject to change. - - Parameters - ---------- - filename : string - Name of the file. - index : Index - Trained CAGRA index. - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import cagra - >>> from pylibraft.neighbors import hnsw - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = cagra.build(cagra.IndexParams(), dataset, handle=handle) - >>> # Serialize the CAGRA index to hnswlib base layer only index format - >>> hnsw.save("my_index.bin", index, handle=handle) - """ - if not index.trained: - raise ValueError("Index need to be built before saving it.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - cdef c_cagra.index[float, uint32_t] * c_index_float - cdef c_cagra.index[int8_t, uint32_t] * c_index_int8 - cdef c_cagra.index[uint8_t, uint32_t] * c_index_uint8 - - if index.active_index_type == "float32": - idx_float = index - c_index_float = \ - idx_float.index - c_cagra.serialize_to_hnswlib_file( - deref(handle_), c_filename, deref(c_index_float)) - elif index.active_index_type == "byte": - idx_int8 = index - c_index_int8 = \ - idx_int8.index - c_cagra.serialize_to_hnswlib_file( - deref(handle_), c_filename, deref(c_index_int8)) - elif index.active_index_type == "ubyte": - idx_uint8 = index - c_index_uint8 = \ - idx_uint8.index - c_cagra.serialize_to_hnswlib_file( - deref(handle_), c_filename, deref(c_index_uint8)) - else: - raise ValueError( - "Index dtype %s not supported" % index.active_index_type) - - -@auto_sync_handle -def load(filename, dim, dtype, metric="sqeuclidean", handle=None): - """ - Loads base-layer-only hnswlib index from file, which was originally - saved as a built CAGRA index. - - Saving / loading the index is experimental. The serialization format is - subject to change, therefore loading an index saved with a previous - version of raft is not guaranteed to work. - - Parameters - ---------- - filename : string - Name of the file. - dim : int - Dimensions of the training dataest - dtype : np.dtype of the saved index - Valid values for dtype: [np.float32, np.byte, np.ubyte] - metric : string denoting the metric type, default="sqeuclidean" - Valid values for metric: ["sqeuclidean", "inner_product"], where - - sqeuclidean is the euclidean distance without the square root - operation, i.e.: distance(a,b) = \\sum_i (a_i - b_i)^2, - - inner product distance is defined as - distance(a, b) = \\sum_i a_i * b_i. - {handle_docstring} - - Returns - ------- - index : HnswIndex - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import cagra - >>> from pylibraft.neighbors import hnsw - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = cagra.build(cagra.IndexParams(), dataset, handle=handle) - >>> # Serialize the CAGRA index to hnswlib base layer only index format - >>> hnsw.save("my_index.bin", index, handle=handle) - >>> index = hnsw.load("my_index.bin", n_features, np.float32, - ... "sqeuclidean") - """ - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - cdef HnswIndexFloat idx_float - cdef HnswIndexInt8 idx_int8 - cdef HnswIndexUint8 idx_uint8 - - cdef DistanceType c_metric = _get_metric(metric) - - if dtype == np.float32: - idx_float = HnswIndexFloat() - idx_float.index = c_hnsw.deserialize_file[float]( - deref(handle_), c_filename, dim, c_metric) - idx_float.trained = True - idx_float.active_index_type = 'float32' - return idx_float - elif dtype == np.byte: - idx_int8 = HnswIndexInt8(dim, metric) - idx_int8.index = c_hnsw.deserialize_file[int8_t]( - deref(handle_), c_filename, dim, c_metric) - idx_int8.trained = True - idx_int8.active_index_type = 'byte' - return idx_int8 - elif dtype == np.ubyte: - idx_uint8 = HnswIndexUint8(dim, metric) - idx_uint8.index = c_hnsw.deserialize_file[uint8_t]( - deref(handle_), c_filename, dim, c_metric) - idx_uint8.trained = True - idx_uint8.active_index_type = 'ubyte' - return idx_uint8 - else: - raise ValueError("Dataset dtype %s not supported" % dtype) - - -@auto_sync_handle -def from_cagra(Index index, handle=None): - """ - Returns an hnswlib base-layer-only index from a CAGRA index. - - NOTE: This method uses the filesystem to write the CAGRA index in - `/tmp/.bin` before reading it as an hnswlib index, - then deleting the temporary file. - - Saving / loading the index is experimental. The serialization format is - subject to change. - - Parameters - ---------- - index : Index - Trained CAGRA index. - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import cagra - >>> from pylibraft.neighbors import hnsw - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = cagra.build(cagra.IndexParams(), dataset, handle=handle) - >>> # Serialize the CAGRA index to hnswlib base layer only index format - >>> hnsw_index = hnsw.from_cagra(index, handle=handle) - """ - uuid_num = uuid.uuid4() - filename = f"/tmp/{uuid_num}.bin" - save(filename, index, handle=handle) - hnsw_index = load(filename, index.dim, np.dtype(index.active_index_type), - _get_metric_string(index.metric), handle=handle) - os.remove(filename) - return hnsw_index - - -cdef class SearchParams: - """ - Hnswlib search parameters - - Parameters - ---------- - ef: int, default=200 - Size of list from which final neighbors k will be selected. - ef should be greater than or equal to k. - num_threads: int, default=1 - Number of host threads to use to search the hnswlib index - and increase concurrency - """ - cdef c_hnsw.search_params params - - def __init__(self, ef=200, num_threads=1): - self.params.ef = ef - self.params.num_threads = num_threads - - def __repr__(self): - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in [ - "ef", "num_threads"]] - return "SearchParams(type=hnsw, " + ( - ", ".join(attr_str)) + ")" - - @property - def ef(self): - return self.params.ef - - @property - def num_threads(self): - return self.params.num_threads - - -@auto_sync_handle -@auto_convert_output -def search(SearchParams search_params, - HnswIndex index, - queries, - k, - neighbors=None, - distances=None, - handle=None): - """ - Find the k nearest neighbors for each query. - - Parameters - ---------- - search_params : SearchParams - index : HnswIndex - Trained CAGRA index saved as base-layer-only hnswlib index. - queries : array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - k : int - The number of neighbors. - neighbors : Optional array interface compliant matrix shape - (n_queries, k), dtype int64_t. If supplied, neighbor - indices will be written here in-place. (default None) - distances : Optional array interface compliant matrix shape - (n_queries, k) If supplied, the distances to the - neighbors will be written here in-place. (default None) - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> import numpy as np - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import cagra - >>> from pylibraft.neighbors import hnsw - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = cagra.build(cagra.IndexParams(), dataset, handle=handle) - >>> - >>> # Load saved base-layer-only hnswlib index from CAGRA index - >>> hnsw_index = hnsw.from_cagra(index, handle=handle) - >>> - >>> # Search hnswlib using the loaded index - >>> queries = np.random.random_sample((n_queries, n_features)).astype( - ... np.float32) - >>> k = 10 - >>> search_params = hnsw.SearchParams( - ... ef=20, - ... num_threads=5 - ... ) - >>> distances, neighbors = hnsw.search(search_params, hnsw_index, - ... queries, k, handle=handle) - """ - - if not index.trained: - raise ValueError("Index need to be built before calling search.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - queries_ai = ai_wrapper(queries) - queries_dt = queries_ai.dtype - cdef uint32_t n_queries = queries_ai.shape[0] - - _check_input_array(queries_ai, [np.dtype('float32'), np.dtype('byte'), - np.dtype('ubyte')], - exp_cols=index.dim) - - if neighbors is None: - neighbors = np.empty((n_queries, k), dtype='uint64') - - neighbors_ai = ai_wrapper(neighbors) - _check_input_array(neighbors_ai, [np.dtype('uint64')], - exp_rows=n_queries, exp_cols=k) - - if distances is None: - distances = np.empty((n_queries, k), dtype='float32') - - distances_ai = ai_wrapper(distances) - _check_input_array(distances_ai, [np.dtype('float32')], - exp_rows=n_queries, exp_cols=k) - - cdef c_hnsw.search_params params = search_params.params - cdef HnswIndexFloat idx_float - cdef HnswIndexInt8 idx_int8 - cdef HnswIndexUint8 idx_uint8 - - if queries_dt == np.float32: - idx_float = index - c_hnsw.search(deref(handle_), - params, - deref(idx_float.index), - get_hmv_float(queries_ai, check_shape=True), - get_hmv_uint64(neighbors_ai, check_shape=True), - get_hmv_float(distances_ai, check_shape=True)) - elif queries_dt == np.byte: - idx_int8 = index - c_hnsw.search(deref(handle_), - params, - deref(idx_int8.index), - get_hmv_int8(queries_ai, check_shape=True), - get_hmv_uint64(neighbors_ai, check_shape=True), - get_hmv_float(distances_ai, check_shape=True)) - elif queries_dt == np.ubyte: - idx_uint8 = index - c_hnsw.search(deref(handle_), - params, - deref(idx_uint8.index), - get_hmv_uint8(queries_ai, check_shape=True), - get_hmv_uint64(neighbors_ai, check_shape=True), - get_hmv_float(distances_ai, check_shape=True)) - else: - raise ValueError("query dtype %s not supported" % queries_dt) - - return (distances, neighbors) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/__init__.pxd b/python/pylibraft/pylibraft/neighbors/ivf_flat/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/__init__.py b/python/pylibraft/pylibraft/neighbors/ivf_flat/__init__.py deleted file mode 100644 index 057cb98f17..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .ivf_flat import ( - Index, - IndexParams, - SearchParams, - build, - extend, - load, - save, - search, -) - -__all__ = [ - "Index", - "IndexParams", - "SearchParams", - "build", - "extend", - "search", - "save", - "load", -] diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/__init__.pxd b/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/__init__.py b/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/__init__.py deleted file mode 100644 index 8f2cc34855..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/c_ivf_flat.pxd b/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/c_ivf_flat.pxd deleted file mode 100644 index a281d33310..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/cpp/c_ivf_flat.pxd +++ /dev/null @@ -1,183 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -import pylibraft.common.handle - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, int64_t, uint8_t, uint32_t, uintptr_t -from libcpp cimport bool, nullptr -from libcpp.string cimport string - -from rmm._lib.memory_resource cimport device_memory_resource - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - host_matrix_view, - make_device_matrix_view, - make_host_matrix_view, - row_major, -) -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources -from pylibraft.distance.distance_type cimport DistanceType -from pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq cimport ( - ann_index, - ann_index_params, - ann_search_params, -) - - -cdef extern from "raft/neighbors/ivf_flat_types.hpp" \ - namespace "raft::neighbors::ivf_flat" nogil: - - cpdef cppclass index_params(ann_index_params): - uint32_t n_lists - uint32_t kmeans_n_iters - double kmeans_trainset_fraction - bool adaptive_centers - bool conservative_memory_allocation - - cdef cppclass index[T, IdxT](ann_index): - index(const device_resources& handle, - DistanceType metric, - uint32_t n_lists, - bool adaptive_centers, - bool conservative_memory_allocation, - uint32_t dim) - IdxT size() - uint32_t dim() - DistanceType metric() - uint32_t n_lists() - bool adaptive_centers() - - cpdef cppclass search_params(ann_search_params): - uint32_t n_probes - - -cdef extern from "raft_runtime/neighbors/ivf_flat.hpp" \ - namespace "raft::runtime::neighbors::ivf_flat" nogil: - - cdef void build(const device_resources&, - const index_params& params, - device_matrix_view[float, int64_t, row_major] dataset, - index[float, int64_t]& index) except + - - cdef void build(const device_resources& handle, - const index_params& params, - device_matrix_view[int8_t, int64_t, row_major] dataset, - index[int8_t, int64_t]& index) except + - - cdef void build(const device_resources& handle, - const index_params& params, - device_matrix_view[uint8_t, int64_t, row_major] dataset, - index[uint8_t, int64_t]& index) except + - - cdef void extend( - const device_resources& handle, - device_matrix_view[float, int64_t, row_major] new_vectors, - optional[device_vector_view[int64_t, int64_t]] new_indices, - index[float, int64_t]* index) except + - - cdef void extend( - const device_resources& handle, - device_matrix_view[int8_t, int64_t, row_major] new_vectors, - optional[device_vector_view[int64_t, int64_t]] new_indices, - index[int8_t, int64_t]* index) except + - - cdef void extend( - const device_resources& handle, - device_matrix_view[uint8_t, int64_t, row_major] new_vectors, - optional[device_vector_view[int64_t, int64_t]] new_indices, - index[uint8_t, int64_t]* index) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[float, int64_t]& index, - device_matrix_view[float, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[int8_t, int64_t]& index, - device_matrix_view[int8_t, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[uint8_t, int64_t]& index, - device_matrix_view[uint8_t, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void serialize(const device_resources& handle, - string& str, - const index[float, int64_t]& index) except + - - cdef void deserialize(const device_resources& handle, - const string& str, - index[float, int64_t]* index) except + - - cdef void serialize(const device_resources& handle, - string& str, - const index[uint8_t, int64_t]& index) except + - - cdef void deserialize(const device_resources& handle, - const string& str, - index[uint8_t, int64_t]* index) except + - - cdef void serialize(const device_resources& handle, - string& str, - const index[int8_t, int64_t]& index) except + - - cdef void deserialize(const device_resources& handle, - const string& str, - index[int8_t, int64_t]* index) except + - - cdef void serialize_file(const device_resources& handle, - const string& filename, - const index[float, int64_t]& index) except + - - cdef void deserialize_file(const device_resources& handle, - const string& filename, - index[float, int64_t]* index) except + - - cdef void serialize_file(const device_resources& handle, - const string& filename, - const index[uint8_t, int64_t]& index) except + - - cdef void deserialize_file(const device_resources& handle, - const string& filename, - index[uint8_t, int64_t]* index) except + - - cdef void serialize_file(const device_resources& handle, - const string& filename, - const index[int8_t, int64_t]& index) except + - - cdef void deserialize_file(const device_resources& handle, - const string& filename, - index[int8_t, int64_t]* index) except + diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/ivf_flat.pyx b/python/pylibraft/pylibraft/neighbors/ivf_flat/ivf_flat.pyx deleted file mode 100644 index d8fbdc74da..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/ivf_flat.pyx +++ /dev/null @@ -1,823 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import warnings - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, int64_t, uint8_t, uint32_t, uintptr_t -from libcpp cimport bool, nullptr -from libcpp.string cimport string - -from pylibraft.distance.distance_type cimport DistanceType - -from pylibraft.common import ( - DeviceResources, - ai_wrapper, - auto_convert_output, - device_ndarray, -) -from pylibraft.common.cai_wrapper import cai_wrapper - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - make_device_vector_view, - row_major, -) - -from pylibraft.common.interruptible import cuda_interruptible - -from pylibraft.common.handle cimport device_resources - -from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.input_validation import is_c_contiguous - -from rmm._lib.memory_resource cimport ( - DeviceMemoryResource, - device_memory_resource, -) - -cimport pylibraft.neighbors.ivf_flat.cpp.c_ivf_flat as c_ivf_flat -from pylibraft.common.cpp.optional cimport optional - -from pylibraft.neighbors.common import _check_input_array, _get_metric - -from pylibraft.common.mdspan cimport ( - get_dmv_float, - get_dmv_int8, - get_dmv_int64, - get_dmv_uint8, -) -from pylibraft.neighbors.common cimport _get_metric_string -from pylibraft.neighbors.ivf_flat.cpp.c_ivf_flat cimport ( - index_params, - search_params, -) - - -cdef class IndexParams: - """ - Parameters to build index for IVF-FLAT nearest neighbor search - - Parameters - ---------- - n_list : int, default = 1024 - The number of clusters used in the coarse quantizer. - metric : string denoting the metric type, default="sqeuclidean" - Valid values for metric: ["sqeuclidean", "inner_product", - "euclidean"], where - - sqeuclidean is the euclidean distance without the square root - operation, i.e.: distance(a,b) = \\sum_i (a_i - b_i)^2, - - euclidean is the euclidean distance - - inner product distance is defined as - distance(a, b) = \\sum_i a_i * b_i. - kmeans_n_iters : int, default = 20 - The number of iterations searching for kmeans centers during index - building. - kmeans_trainset_fraction : int, default = 0.5 - If kmeans_trainset_fraction is less than 1, then the dataset is - subsampled, and only n_samples * kmeans_trainset_fraction rows - are used for training. - add_data_on_build : bool, default = True - After training the coarse and fine quantizers, we will populate - the index with the dataset if add_data_on_build == True, otherwise - the index is left empty, and the extend method can be used - to add new vectors to the index. - adaptive_centers : bool, default = False - By default (adaptive_centers = False), the cluster centers are - trained in `ivf_flat::build`, and and never modified in - `ivf_flat::extend`. The alternative behavior (adaptive_centers - = true) is to update the cluster centers for new data when it is - added. In this case, `index.centers()` are always exactly the - centroids of the data in the corresponding clusters. The drawback - of this behavior is that the centroids depend on the order of - adding new data (through the classification of the added data); - that is, `index.centers()` "drift" together with the changing - distribution of the newly added data. - """ - cdef c_ivf_flat.index_params params - - def __init__(self, *, - n_lists=1024, - metric="sqeuclidean", - kmeans_n_iters=20, - kmeans_trainset_fraction=0.5, - add_data_on_build=True, - bool adaptive_centers=False): - self.params.n_lists = n_lists - self.params.metric = _get_metric(metric) - self.params.metric_arg = 0 - self.params.kmeans_n_iters = kmeans_n_iters - self.params.kmeans_trainset_fraction = kmeans_trainset_fraction - self.params.add_data_on_build = add_data_on_build - self.params.adaptive_centers = adaptive_centers - - @property - def n_lists(self): - return self.params.n_lists - - @property - def metric(self): - return self.params.metric - - @property - def kmeans_n_iters(self): - return self.params.kmeans_n_iters - - @property - def kmeans_trainset_fraction(self): - return self.params.kmeans_trainset_fraction - - @property - def add_data_on_build(self): - return self.params.add_data_on_build - - @property - def adaptive_centers(self): - return self.params.adaptive_centers - - -cdef class Index: - cdef readonly bool trained - cdef str active_index_type - - def __cinit__(self): - self.trained = False - self.active_index_type = None - - -cdef class IndexFloat(Index): - cdef c_ivf_flat.index[float, int64_t] * index - - def __cinit__(self, handle=None): - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - # this is to keep track of which index type is being used - # We create a placeholder object. The actual parameter values do - # not matter, it will be replaced with a built index object later. - self.index = new c_ivf_flat.index[float, int64_t]( - deref(handle_), _get_metric("sqeuclidean"), - 1, - False, - False, - 4) - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.index.metric()) - attr_str = [ - attr + "=" + str(getattr(self, attr)) - for attr in ["size", "dim", "n_lists", "adaptive_centers"] - ] - attr_str = [m_str] + attr_str - return "Index(type=IVF-FLAT, " + (", ".join(attr_str)) + ")" - - @property - def dim(self): - return self.index[0].dim() - - @property - def size(self): - return self.index[0].size() - - @property - def metric(self): - return self.index[0].metric() - - @property - def n_lists(self): - return self.index[0].n_lists() - - @property - def adaptive_centers(self): - return self.index[0].adaptive_centers() - - -cdef class IndexInt8(Index): - cdef c_ivf_flat.index[int8_t, int64_t] * index - - def __cinit__(self, handle=None): - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - # this is to keep track of which index type is being used - # We create a placeholder object. The actual parameter values do - # not matter, it will be replaced with a built index object later. - self.index = new c_ivf_flat.index[int8_t, int64_t]( - deref(handle_), _get_metric("sqeuclidean"), - 1, - False, - False, - 4) - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.index.metric()) - attr_str = [ - attr + "=" + str(getattr(self, attr)) - for attr in ["size", "dim", "n_lists", "adaptive_centers"] - ] - attr_str = [m_str] + attr_str - return "Index(type=IVF-FLAT, " + (", ".join(attr_str)) + ")" - - @property - def dim(self): - return self.index[0].dim() - - @property - def size(self): - return self.index[0].size() - - @property - def metric(self): - return self.index[0].metric() - - @property - def n_lists(self): - return self.index[0].n_lists() - - @property - def adaptive_centers(self): - return self.index[0].adaptive_centers() - - -cdef class IndexUint8(Index): - cdef c_ivf_flat.index[uint8_t, int64_t] * index - - def __cinit__(self, handle=None): - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - # this is to keep track of which index type is being used - # We create a placeholder object. The actual parameter values do - # not matter, it will be replaced with a built index object later. - self.index = new c_ivf_flat.index[uint8_t, int64_t]( - deref(handle_), _get_metric("sqeuclidean"), - 1, - False, - False, - 4) - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.index.metric()) - attr_str = [ - attr + "=" + str(getattr(self, attr)) - for attr in ["size", "dim", "n_lists", "adaptive_centers"] - ] - attr_str = [m_str] + attr_str - return "Index(type=IVF-FLAT, " + (", ".join(attr_str)) + ")" - - @property - def dim(self): - return self.index[0].dim() - - @property - def size(self): - return self.index[0].size() - - @property - def metric(self): - return self.index[0].metric() - - @property - def n_lists(self): - return self.index[0].n_lists() - - @property - def adaptive_centers(self): - return self.index[0].adaptive_centers() - - -@auto_sync_handle -@auto_convert_output -def build(IndexParams index_params, dataset, handle=None): - """ - Builds an IVF-FLAT index that can be used for nearest neighbor search. - - Parameters - ---------- - index_params : IndexParams object - dataset : CUDA array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - {handle_docstring} - - Returns - ------- - index: ivf_flat.Index - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_flat - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> index_params = ivf_flat.IndexParams( - ... n_lists=1024, - ... metric="sqeuclidean") - >>> index = ivf_flat.build(index_params, dataset, handle=handle) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 10 - >>> distances, neighbors = ivf_flat.search(ivf_flat.SearchParams(), - ... index, queries, k, - ... handle=handle) - >>> distances = cp.asarray(distances) - >>> neighbors = cp.asarray(neighbors) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - dataset_cai = cai_wrapper(dataset) - dataset_dt = dataset_cai.dtype - _check_input_array(dataset_cai, [np.dtype('float32'), np.dtype('byte'), - np.dtype('ubyte')]) - - cdef int64_t n_rows = dataset_cai.shape[0] - cdef uint32_t dim = dataset_cai.shape[1] - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - if dataset_dt == np.float32: - idx_float = IndexFloat(handle) - idx_float.active_index_type = "float32" - with cuda_interruptible(): - c_ivf_flat.build(deref(handle_), - index_params.params, - get_dmv_float(dataset_cai, check_shape=True), - deref(idx_float.index)) - idx_float.trained = True - return idx_float - elif dataset_dt == np.byte: - idx_int8 = IndexInt8(handle) - idx_int8.active_index_type = "byte" - with cuda_interruptible(): - c_ivf_flat.build(deref(handle_), - index_params.params, - get_dmv_int8(dataset_cai, check_shape=True), - deref(idx_int8.index)) - idx_int8.trained = True - return idx_int8 - elif dataset_dt == np.ubyte: - idx_uint8 = IndexUint8(handle) - idx_uint8.active_index_type = "ubyte" - with cuda_interruptible(): - c_ivf_flat.build(deref(handle_), - index_params.params, - get_dmv_uint8(dataset_cai, check_shape=True), - deref(idx_uint8.index)) - idx_uint8.trained = True - return idx_uint8 - else: - raise TypeError("dtype %s not supported" % dataset_dt) - - -@auto_sync_handle -@auto_convert_output -def extend(Index index, new_vectors, new_indices, handle=None): - """ - Extend an existing index with new vectors. - - Parameters - ---------- - index : ivf_flat.Index - Trained ivf_flat object. - new_vectors : CUDA array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - new_indices : CUDA array interface compliant vector shape (n_samples) - Supported dtype [int64] - {handle_docstring} - - Returns - ------- - index: ivf_flat.Index - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_flat - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> index = ivf_flat.build(ivf_flat.IndexParams(), dataset, - ... handle=handle) - >>> n_rows = 100 - >>> more_data = cp.random.random_sample((n_rows, n_features), - ... dtype=cp.float32) - >>> indices = index.size + cp.arange(n_rows, dtype=cp.int64) - >>> index = ivf_flat.extend(index, more_data, indices) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 10 - >>> distances, neighbors = ivf_flat.search(ivf_flat.SearchParams(), - ... index, queries, - ... k, handle=handle) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - - >>> distances = cp.asarray(distances) - >>> neighbors = cp.asarray(neighbors) - """ - if not index.trained: - raise ValueError("Index need to be built before calling extend.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - vecs_cai = cai_wrapper(new_vectors) - vecs_dt = vecs_cai.dtype - cdef int64_t n_rows = vecs_cai.shape[0] - cdef uint32_t dim = vecs_cai.shape[1] - - _check_input_array(vecs_cai, [np.dtype(index.active_index_type)], - exp_cols=index.dim) - - idx_cai = cai_wrapper(new_indices) - _check_input_array(idx_cai, [np.dtype('int64')], exp_rows=n_rows) - if len(idx_cai.shape)!=1: - raise ValueError("Indices array is expected to be 1D") - - cdef optional[device_vector_view[int64_t, int64_t]] new_indices_opt - - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - if vecs_dt == np.float32: - idx_float = index - if idx_float.index.size() > 0: - new_indices_opt = make_device_vector_view( - idx_cai.data, - idx_cai.shape[0]) - with cuda_interruptible(): - c_ivf_flat.extend(deref(handle_), - get_dmv_float(vecs_cai, check_shape=True), - new_indices_opt, - idx_float.index) - elif vecs_dt == np.int8: - idx_int8 = index - if idx_int8.index[0].size() > 0: - new_indices_opt = make_device_vector_view( - idx_cai.data, - idx_cai.shape[0]) - with cuda_interruptible(): - c_ivf_flat.extend(deref(handle_), - get_dmv_int8(vecs_cai, check_shape=True), - new_indices_opt, - idx_int8.index) - elif vecs_dt == np.uint8: - idx_uint8 = index - if idx_uint8.index[0].size() > 0: - new_indices_opt = make_device_vector_view( - idx_cai.data, - idx_cai.shape[0]) - with cuda_interruptible(): - c_ivf_flat.extend(deref(handle_), - get_dmv_uint8(vecs_cai, check_shape=True), - new_indices_opt, - idx_uint8.index) - else: - raise TypeError("query dtype %s not supported" % vecs_dt) - - return index - - -cdef class SearchParams: - """ - IVF-FLAT search parameters - - Parameters - ---------- - n_probes: int, default = 1024 - The number of course clusters to select for the fine search. - """ - cdef c_ivf_flat.search_params params - - def __init__(self, *, n_probes=20): - self.params.n_probes = n_probes - - def __repr__(self): - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["n_probes"]] - return "SearchParams(type=IVF-FLAT, " + (", ".join(attr_str)) + ")" - - @property - def n_probes(self): - return self.params.n_probes - - -@auto_sync_handle -@auto_convert_output -def search(SearchParams search_params, - Index index, - queries, - k, - neighbors=None, - distances=None, - handle=None): - """ - Find the k nearest neighbors for each query. - - Parameters - ---------- - search_params : SearchParams - index : Index - Trained IVF-FLAT index. - queries : CUDA array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - k : int - The number of neighbors. - neighbors : Optional CUDA array interface compliant matrix shape - (n_queries, k), dtype int64_t. If supplied, neighbor - indices will be written here in-place. (default None) - distances : Optional CUDA array interface compliant matrix shape - (n_queries, k) If supplied, the distances to the - neighbors will be written here in-place. (default None) - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_flat - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = ivf_flat.build(ivf_flat.IndexParams(), dataset, - ... handle=handle) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 10 - >>> search_params = ivf_flat.SearchParams( - ... n_probes=20 - ... ) - >>> distances, neighbors = ivf_flat.search(search_params, index, - ... queries, k, handle=handle) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - >>> neighbors = cp.asarray(neighbors) - >>> distances = cp.asarray(distances) - """ - - if not index.trained: - raise ValueError("Index need to be built before calling search.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - queries_cai = cai_wrapper(queries) - queries_dt = queries_cai.dtype - cdef uint32_t n_queries = queries_cai.shape[0] - - _check_input_array(queries_cai, [np.dtype(index.active_index_type)], - exp_cols=index.dim) - - if neighbors is None: - neighbors = device_ndarray.empty((n_queries, k), dtype='int64') - - neighbors_cai = cai_wrapper(neighbors) - _check_input_array(neighbors_cai, [np.dtype('int64')], - exp_rows=n_queries, exp_cols=k) - - if distances is None: - distances = device_ndarray.empty((n_queries, k), dtype='float32') - - distances_cai = cai_wrapper(distances) - _check_input_array(distances_cai, [np.dtype('float32')], - exp_rows=n_queries, exp_cols=k) - - cdef c_ivf_flat.search_params params = search_params.params - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - if queries_dt == np.float32: - idx_float = index - with cuda_interruptible(): - c_ivf_flat.search(deref(handle_), - params, - deref(idx_float.index), - get_dmv_float(queries_cai, check_shape=True), - get_dmv_int64(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - elif queries_dt == np.byte: - idx_int8 = index - with cuda_interruptible(): - c_ivf_flat.search(deref(handle_), - params, - deref(idx_int8.index), - get_dmv_int8(queries_cai, check_shape=True), - get_dmv_int64(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - elif queries_dt == np.ubyte: - idx_uint8 = index - with cuda_interruptible(): - c_ivf_flat.search(deref(handle_), - params, - deref(idx_uint8.index), - get_dmv_uint8(queries_cai, check_shape=True), - get_dmv_int64(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - else: - raise ValueError("query dtype %s not supported" % queries_dt) - - return (distances, neighbors) - - -@auto_sync_handle -def save(filename, Index index, handle=None): - """ - Saves the index to a file. - - Saving / loading the index is experimental. The serialization format is - subject to change. - - Parameters - ---------- - filename : string - Name of the file. - index : Index - Trained IVF-Flat index. - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_flat - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = ivf_flat.build(ivf_flat.IndexParams(), dataset, - ... handle=handle) - >>> ivf_flat.save("my_index.bin", index, handle=handle) - """ - if not index.trained: - raise ValueError("Index need to be built before saving it.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - if index.active_index_type == "float32": - idx_float = index - c_ivf_flat.serialize_file( - deref(handle_), c_filename, deref(idx_float.index)) - elif index.active_index_type == "byte": - idx_int8 = index - c_ivf_flat.serialize_file( - deref(handle_), c_filename, deref(idx_int8.index)) - elif index.active_index_type == "ubyte": - idx_uint8 = index - c_ivf_flat.serialize_file( - deref(handle_), c_filename, deref(idx_uint8.index)) - else: - raise ValueError( - "Index dtype %s not supported" % index.active_index_type) - - -@auto_sync_handle -def load(filename, handle=None): - """ - Loads index from a file. - - Saving / loading the index is experimental. The serialization format is - subject to change, therefore loading an index saved with a previous - version of raft is not guaranteed to work. - - Parameters - ---------- - filename : string - Name of the file. - {handle_docstring} - - Returns - ------- - index : Index - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_flat - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build and save index - >>> handle = DeviceResources() - >>> index = ivf_flat.build(ivf_flat.IndexParams(), dataset, - ... handle=handle) - >>> ivf_flat.save("my_index.bin", index, handle=handle) - >>> del index - >>> n_queries = 100 - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> index = ivf_flat.load("my_index.bin", handle=handle) - >>> distances, neighbors = ivf_flat.search(ivf_flat.SearchParams(), - ... index, queries, k=10, - ... handle=handle) - """ - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - cdef IndexFloat idx_float - cdef IndexInt8 idx_int8 - cdef IndexUint8 idx_uint8 - - with open(filename, 'rb') as f: - type_str = f.read(3).decode('utf-8') - - dataset_dt = np.dtype(type_str) - - if dataset_dt == np.float32: - idx_float = IndexFloat(handle) - c_ivf_flat.deserialize_file( - deref(handle_), c_filename, idx_float.index) - idx_float.trained = True - idx_float.active_index_type = 'float32' - return idx_float - elif dataset_dt == np.byte: - idx_int8 = IndexInt8(handle) - c_ivf_flat.deserialize_file( - deref(handle_), c_filename, idx_int8.index) - idx_int8.trained = True - idx_int8.active_index_type = 'byte' - return idx_int8 - elif dataset_dt == np.ubyte: - idx_uint8 = IndexUint8(handle) - c_ivf_flat.deserialize_file( - deref(handle_), c_filename, idx_uint8.index) - idx_uint8.trained = True - idx_uint8.active_index_type = 'ubyte' - return idx_uint8 - else: - raise ValueError("Index dtype %s not supported" % dataset_dt) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt deleted file mode 100644 index af431adb16..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Set the list of Cython files to build -set(cython_sources ivf_pq.pyx) -set(linked_libraries raft::raft raft::compiled) - -# Build all of the Cython targets -rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfpq_ -) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/__init__.pxd b/python/pylibraft/pylibraft/neighbors/ivf_pq/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/__init__.py b/python/pylibraft/pylibraft/neighbors/ivf_pq/__init__.py deleted file mode 100644 index 3d604f829d..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .ivf_pq import ( - Index, - IndexParams, - SearchParams, - build, - extend, - load, - save, - search, -) - -__all__ = [ - "Index", - "IndexParams", - "SearchParams", - "build", - "extend", - "load", - "save", - "search", -] diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/__init__.pxd b/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/__init__.pxd deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/__init__.py b/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/__init__.py deleted file mode 100644 index 273b4497cc..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/c_ivf_pq.pxd b/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/c_ivf_pq.pxd deleted file mode 100644 index 895abbadca..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/cpp/c_ivf_pq.pxd +++ /dev/null @@ -1,178 +0,0 @@ -# -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -import pylibraft.common.handle - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, int64_t, uint8_t, uint32_t, uintptr_t -from libcpp cimport bool, nullptr -from libcpp.string cimport string - -from rmm._lib.memory_resource cimport device_memory_resource - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - row_major, -) -from pylibraft.common.handle cimport device_resources -from pylibraft.common.optional cimport optional -from pylibraft.distance.distance_type cimport DistanceType - - -cdef extern from "library_types.h": - ctypedef enum cudaDataType_t: - CUDA_R_32F "CUDA_R_32F" # float - CUDA_R_16F "CUDA_R_16F" # half - - # uint8 - used to refer to IVF-PQ's fp8 storage type - CUDA_R_8U "CUDA_R_8U" - -cdef extern from "raft/neighbors/ann_types.hpp" \ - namespace "raft::neighbors::ann" nogil: - - cdef cppclass ann_index "raft::neighbors::index": - pass - - cdef cppclass ann_index_params "raft::spatial::knn::index_params": - DistanceType metric - float metric_arg - bool add_data_on_build - - cdef cppclass ann_search_params "raft::spatial::knn::search_params": - pass - - -cdef extern from "raft/neighbors/ivf_pq_types.hpp" \ - namespace "raft::neighbors::ivf_pq" nogil: - - ctypedef enum codebook_gen: - PER_SUBSPACE "raft::neighbors::ivf_pq::codebook_gen::PER_SUBSPACE", - PER_CLUSTER "raft::neighbors::ivf_pq::codebook_gen::PER_CLUSTER" - - cpdef cppclass index_params(ann_index_params): - uint32_t n_lists - uint32_t kmeans_n_iters - double kmeans_trainset_fraction - uint32_t pq_bits - uint32_t pq_dim - codebook_gen codebook_kind - bool force_random_rotation - bool conservative_memory_allocation - - cdef cppclass index[IdxT](ann_index): - index(const device_resources& handle, - DistanceType metric, - codebook_gen codebook_kind, - uint32_t n_lists, - uint32_t dim, - uint32_t pq_bits, - uint32_t pq_dim, - bool conservative_memory_allocation) - - IdxT size() - uint32_t dim() - uint32_t pq_dim() - uint32_t pq_len() - uint32_t pq_bits() - DistanceType metric() - uint32_t n_lists() - uint32_t rot_dim() - codebook_gen codebook_kind() - bool conservative_memory_allocation() - - cpdef cppclass search_params(ann_search_params): - uint32_t n_probes - cudaDataType_t lut_dtype - cudaDataType_t internal_distance_dtype - - -cdef extern from "raft_runtime/neighbors/ivf_pq.hpp" \ - namespace "raft::runtime::neighbors::ivf_pq" nogil: - - cdef void build( - const device_resources& handle, - const index_params& params, - device_matrix_view[float, int64_t, row_major] dataset, - index[int64_t]* index) except + - - cdef void build( - const device_resources& handle, - const index_params& params, - device_matrix_view[int8_t, int64_t, row_major] dataset, - index[int64_t]* index) except + - - cdef void build( - const device_resources& handle, - const index_params& params, - device_matrix_view[uint8_t, int64_t, row_major] dataset, - index[int64_t]* index) except + - - cdef void extend( - const device_resources& handle, - device_matrix_view[float, int64_t, row_major] new_vectors, - optional[device_vector_view[int64_t, int64_t]] new_indices, - index[int64_t]* index) except + - - cdef void extend( - const device_resources& handle, - device_matrix_view[int8_t, int64_t, row_major] new_vectors, - optional[device_vector_view[int64_t, int64_t]] new_indices, - index[int64_t]* index) except + - - cdef void extend( - const device_resources& handle, - device_matrix_view[uint8_t, int64_t, row_major] new_vectors, - optional[device_vector_view[int64_t, int64_t]] new_indices, - index[int64_t]* index) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[int64_t]& index, - device_matrix_view[float, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[int64_t]& index, - device_matrix_view[int8_t, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void search( - const device_resources& handle, - const search_params& params, - const index[int64_t]& index, - device_matrix_view[uint8_t, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] neighbors, - device_matrix_view[float, int64_t, row_major] distances) except + - - cdef void serialize(const device_resources& handle, - const string& filename, - const index[int64_t]& index) except + - - cdef void deserialize(const device_resources& handle, - const string& filename, - index[int64_t]* index) except + diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pxd b/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pxd deleted file mode 100644 index 1b99da1fd7..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pxd +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# distutils: language = c++ - -cimport pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq as c_ivf_pq - - -cdef class IndexParams: - cdef c_ivf_pq.index_params params - -cdef class SearchParams: - cdef c_ivf_pq.search_params params diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pyx b/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pyx deleted file mode 100644 index 5b89f0d9a5..0000000000 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/ivf_pq.pyx +++ /dev/null @@ -1,799 +0,0 @@ -# -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import warnings - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport int32_t, int64_t, uint32_t, uintptr_t -from libcpp cimport bool, nullptr -from libcpp.string cimport string - -from pylibraft.distance.distance_type cimport DistanceType - -from pylibraft.common import ( - DeviceResources, - ai_wrapper, - auto_convert_output, - cai_wrapper, - device_ndarray, -) -from pylibraft.common.cai_wrapper import wrap_array -from pylibraft.common.interruptible import cuda_interruptible - -from pylibraft.common.handle cimport device_resources - -from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.input_validation import is_c_contiguous - -from rmm._lib.memory_resource cimport ( - DeviceMemoryResource, - device_memory_resource, -) - -cimport pylibraft.neighbors.ivf_flat.cpp.c_ivf_flat as c_ivf_flat -cimport pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq as c_ivf_pq -from pylibraft.common.optional cimport make_optional, optional - -from pylibraft.neighbors.common import _check_input_array, _get_metric - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - make_device_vector_view, - row_major, -) -from pylibraft.common.mdspan cimport ( - get_dmv_float, - get_dmv_int8, - get_dmv_int64, - get_dmv_uint8, - make_optional_view_int64, -) -from pylibraft.neighbors.common cimport _get_metric_string -from pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq cimport ( - index_params, - search_params, -) - - -cdef _get_codebook_string(c_ivf_pq.codebook_gen codebook): - return {c_ivf_pq.codebook_gen.PER_SUBSPACE: "subspace", - c_ivf_pq.codebook_gen.PER_CLUSTER: "cluster"}[codebook] - - -cdef _map_dtype_np_to_cuda(dtype, supported_dtypes=None): - if supported_dtypes is not None and dtype not in supported_dtypes: - raise TypeError("Type %s is not supported" % str(dtype)) - return {np.float32: c_ivf_pq.cudaDataType_t.CUDA_R_32F, - np.float16: c_ivf_pq.cudaDataType_t.CUDA_R_16F, - np.uint8: c_ivf_pq.cudaDataType_t.CUDA_R_8U}[dtype] - - -cdef _get_dtype_string(dtype): - return str({c_ivf_pq.cudaDataType_t.CUDA_R_32F: np.float32, - c_ivf_pq.cudaDataType_t.CUDA_R_16F: np.float16, - c_ivf_pq.cudaDataType_t.CUDA_R_8U: np.uint8}[dtype]) - - -cdef class IndexParams: - """ - Parameters to build index for IVF-PQ nearest neighbor search - - Parameters - ---------- - n_list : int, default = 1024 - The number of clusters used in the coarse quantizer. - metric : string denoting the metric type, default="sqeuclidean" - Valid values for metric: ["sqeuclidean", "inner_product", - "euclidean"], where - - sqeuclidean is the euclidean distance without the square root - operation, i.e.: distance(a,b) = \\sum_i (a_i - b_i)^2, - - euclidean is the euclidean distance - - inner product distance is defined as - distance(a, b) = \\sum_i a_i * b_i. - kmeans_n_iters : int, default = 20 - The number of iterations searching for kmeans centers during index - building. - kmeans_trainset_fraction : int, default = 0.5 - If kmeans_trainset_fraction is less than 1, then the dataset is - subsampled, and only n_samples * kmeans_trainset_fraction rows - are used for training. - pq_bits : int, default = 8 - The bit length of the vector element after quantization. - pq_dim : int, default = 0 - The dimensionality of a the vector after product quantization. - When zero, an optimal value is selected using a heuristic. Note - pq_dim * pq_bits must be a multiple of 8. Hint: a smaller 'pq_dim' - results in a smaller index size and better search performance, but - lower recall. If 'pq_bits' is 8, 'pq_dim' can be set to any number, - but multiple of 8 are desirable for good performance. If 'pq_bits' - is not 8, 'pq_dim' should be a multiple of 8. For good performance, - it is desirable that 'pq_dim' is a multiple of 32. Ideally, - 'pq_dim' should be also a divisor of the dataset dim. - codebook_kind : string, default = "subspace" - Valid values ["subspace", "cluster"] - force_random_rotation : bool, default = False - Apply a random rotation matrix on the input data and queries even - if `dim % pq_dim == 0`. Note: if `dim` is not multiple of `pq_dim`, - a random rotation is always applied to the input data and queries - to transform the working space from `dim` to `rot_dim`, which may - be slightly larger than the original space and and is a multiple - of `pq_dim` (`rot_dim % pq_dim == 0`). However, this transform is - not necessary when `dim` is multiple of `pq_dim` (`dim == rot_dim`, - hence no need in adding "extra" data columns / features). By - default, if `dim == rot_dim`, the rotation transform is - initialized with the identity matrix. When - `force_random_rotation == True`, a random orthogonal transform - matrix is generated regardless of the values of `dim` and `pq_dim`. - add_data_on_build : bool, default = True - After training the coarse and fine quantizers, we will populate - the index with the dataset if add_data_on_build == True, otherwise - the index is left empty, and the extend method can be used - to add new vectors to the index. - conservative_memory_allocation : bool, default = True - By default, the algorithm allocates more space than necessary for - individual clusters (`list_data`). This allows to amortize the cost - of memory allocation and reduce the number of data copies during - repeated calls to `extend` (extending the database). - To disable this behavior and use as little GPU memory for the - database as possible, set this flat to `True`. - """ - def __init__(self, *, - n_lists=1024, - metric="sqeuclidean", - kmeans_n_iters=20, - kmeans_trainset_fraction=0.5, - pq_bits=8, - pq_dim=0, - codebook_kind="subspace", - force_random_rotation=False, - add_data_on_build=True, - conservative_memory_allocation=False): - self.params.n_lists = n_lists - self.params.metric = _get_metric(metric) - self.params.metric_arg = 0 - self.params.kmeans_n_iters = kmeans_n_iters - self.params.kmeans_trainset_fraction = kmeans_trainset_fraction - self.params.pq_bits = pq_bits - self.params.pq_dim = pq_dim - if codebook_kind == "subspace": - self.params.codebook_kind = c_ivf_pq.codebook_gen.PER_SUBSPACE - elif codebook_kind == "cluster": - self.params.codebook_kind = c_ivf_pq.codebook_gen.PER_CLUSTER - else: - raise ValueError("Incorrect codebook kind %s" % codebook_kind) - self.params.force_random_rotation = force_random_rotation - self.params.add_data_on_build = add_data_on_build - self.params.conservative_memory_allocation = \ - conservative_memory_allocation - - @property - def n_lists(self): - return self.params.n_lists - - @property - def metric(self): - return self.params.metric - - @property - def kmeans_n_iters(self): - return self.params.kmeans_n_iters - - @property - def kmeans_trainset_fraction(self): - return self.params.kmeans_trainset_fraction - - @property - def pq_bits(self): - return self.params.pq_bits - - @property - def pq_dim(self): - return self.params.pq_dim - - @property - def codebook_kind(self): - return self.params.codebook_kind - - @property - def force_random_rotation(self): - return self.params.force_random_rotation - - @property - def add_data_on_build(self): - return self.params.add_data_on_build - - @property - def conservative_memory_allocation(self): - return self.params.conservative_memory_allocation - - -cdef class Index: - # We store a pointer to the index because it dose not have a trivial - # constructor. - cdef c_ivf_pq.index[int64_t] * index - cdef readonly bool trained - - def __cinit__(self, handle=None): - self.trained = False - self.index = NULL - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - # We create a placeholder object. The actual parameter values do - # not matter, it will be replaced with a built index object later. - self.index = new c_ivf_pq.index[int64_t]( - deref(handle_), _get_metric("sqeuclidean"), - c_ivf_pq.codebook_gen.PER_SUBSPACE, - 1, - 4, - 8, - 0, - False) - - def __dealloc__(self): - if self.index is not NULL: - del self.index - - def __repr__(self): - m_str = "metric=" + _get_metric_string(self.index.metric()) - code_str = "codebook=" + _get_codebook_string( - self.index.codebook_kind()) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["size", "dim", "pq_dim", "pq_bits", - "n_lists", "rot_dim"]] - attr_str = [m_str, code_str] + attr_str - return "Index(type=IVF-PQ, " + (", ".join(attr_str)) + ")" - - @property - def dim(self): - return self.index[0].dim() - - @property - def size(self): - return self.index[0].size() - - @property - def pq_dim(self): - return self.index[0].pq_dim() - - @property - def pq_len(self): - return self.index[0].pq_len() - - @property - def pq_bits(self): - return self.index[0].pq_bits() - - @property - def metric(self): - return self.index[0].metric() - - @property - def n_lists(self): - return self.index[0].n_lists() - - @property - def rot_dim(self): - return self.index[0].rot_dim() - - @property - def codebook_kind(self): - return self.index[0].codebook_kind() - - @property - def conservative_memory_allocation(self): - return self.index[0].conservative_memory_allocation() - - -@auto_sync_handle -@auto_convert_output -def build(IndexParams index_params, dataset, handle=None): - """ - Builds an IVF-PQ index that can be later used for nearest neighbor search. - - The input array can be either CUDA array interface compliant matrix or - array interface compliant matrix in host memory. - - Parameters - ---------- - index_params : IndexParams object - dataset : array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - {handle_docstring} - - Returns - ------- - index: ivf_pq.Index - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_pq - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> index_params = ivf_pq.IndexParams( - ... n_lists=1024, - ... metric="sqeuclidean", - ... pq_dim=10) - >>> index = ivf_pq.build(index_params, dataset, handle=handle) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 10 - >>> distances, neighbors = ivf_pq.search(ivf_pq.SearchParams(), index, - ... queries, k, handle=handle) - >>> distances = cp.asarray(distances) - >>> neighbors = cp.asarray(neighbors) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - dataset_cai = wrap_array(dataset) - dataset_dt = dataset_cai.dtype - _check_input_array(dataset_cai, [np.dtype('float32'), np.dtype('byte'), - np.dtype('ubyte')]) - - cdef int64_t n_rows = dataset_cai.shape[0] - cdef uint32_t dim = dataset_cai.shape[1] - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - idx = Index() - - if dataset_dt == np.float32: - with cuda_interruptible(): - c_ivf_pq.build(deref(handle_), - index_params.params, - get_dmv_float(dataset_cai, check_shape=True), - idx.index) - idx.trained = True - elif dataset_dt == np.byte: - with cuda_interruptible(): - c_ivf_pq.build(deref(handle_), - index_params.params, - get_dmv_int8(dataset_cai, check_shape=True), - idx.index) - idx.trained = True - elif dataset_dt == np.ubyte: - with cuda_interruptible(): - c_ivf_pq.build(deref(handle_), - index_params.params, - get_dmv_uint8(dataset_cai, check_shape=True), - idx.index) - idx.trained = True - else: - raise TypeError("dtype %s not supported" % dataset_dt) - - return idx - - -@auto_sync_handle -@auto_convert_output -def extend(Index index, new_vectors, new_indices, handle=None): - """ - Extend an existing index with new vectors. - - The input array can be either CUDA array interface compliant matrix or - array interface compliant matrix in host memory. - - Parameters - ---------- - index : ivf_pq.Index - Trained ivf_pq object. - new_vectors : array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - new_indices : array interface compliant vector shape (n_samples) - Supported dtype [int64] - {handle_docstring} - - Returns - ------- - index: ivf_pq.Index - - Examples - -------- - - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_pq - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> index = ivf_pq.build(ivf_pq.IndexParams(), dataset, handle=handle) - >>> n_rows = 100 - >>> more_data = cp.random.random_sample((n_rows, n_features), - ... dtype=cp.float32) - >>> indices = index.size + cp.arange(n_rows, dtype=cp.int64) - >>> index = ivf_pq.extend(index, more_data, indices) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 10 - >>> distances, neighbors = ivf_pq.search(ivf_pq.SearchParams(), - ... index, queries, - ... k, handle=handle) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - >>> distances = cp.asarray(distances) - >>> neighbors = cp.asarray(neighbors) - """ - if not index.trained: - raise ValueError("Index need to be built before calling extend.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - vecs_cai = wrap_array(new_vectors) - vecs_dt = vecs_cai.dtype - cdef optional[device_vector_view[int64_t, int64_t]] new_indices_opt - cdef int64_t n_rows = vecs_cai.shape[0] - cdef uint32_t dim = vecs_cai.shape[1] - - _check_input_array(vecs_cai, [np.dtype('float32'), np.dtype('byte'), - np.dtype('ubyte')], - exp_cols=index.dim) - - idx_cai = wrap_array(new_indices) - _check_input_array(idx_cai, [np.dtype('int64')], exp_rows=n_rows) - if len(idx_cai.shape)!=1: - raise ValueError("Indices array is expected to be 1D") - - if index.index.size() > 0: - new_indices_opt = make_device_vector_view( - idx_cai.data, - idx_cai.shape[0]) - - if vecs_dt == np.float32: - with cuda_interruptible(): - c_ivf_pq.extend(deref(handle_), - get_dmv_float(vecs_cai, check_shape=True), - new_indices_opt, - index.index) - elif vecs_dt == np.int8: - with cuda_interruptible(): - c_ivf_pq.extend(deref(handle_), - get_dmv_int8(vecs_cai, check_shape=True), - new_indices_opt, - index.index) - elif vecs_dt == np.uint8: - with cuda_interruptible(): - c_ivf_pq.extend(deref(handle_), - get_dmv_uint8(vecs_cai, check_shape=True), - new_indices_opt, - index.index) - else: - raise TypeError("query dtype %s not supported" % vecs_dt) - - return index - - -cdef class SearchParams: - """ - IVF-PQ search parameters - - Parameters - ---------- - n_probes: int, default = 1024 - The number of course clusters to select for the fine search. - lut_dtype: default = np.float32 - Data type of look up table to be created dynamically at search - time. The use of low-precision types reduces the amount of shared - memory required at search time, so fast shared memory kernels can - be used even for datasets with large dimansionality. Note that - the recall is slightly degraded when low-precision type is - selected. Possible values [np.float32, np.float16, np.uint8] - internal_distance_dtype: default = np.float32 - Storage data type for distance/similarity computation. - Possible values [np.float32, np.float16] - """ - def __init__(self, *, n_probes=20, - lut_dtype=np.float32, - internal_distance_dtype=np.float32): - self.params.n_probes = n_probes - self.params.lut_dtype = _map_dtype_np_to_cuda(lut_dtype) - self.params.internal_distance_dtype = \ - _map_dtype_np_to_cuda(internal_distance_dtype) - # TODO(tfeher): enable if #926 adds this - # self.params.shmem_carveout = self.shmem_carveout - - def __repr__(self): - lut_str = "lut_dtype=" + _get_dtype_string(self.params.lut_dtype) - idt_str = "internal_distance_dtype=" + \ - _get_dtype_string(self.params.internal_distance_dtype) - attr_str = [attr + "=" + str(getattr(self, attr)) - for attr in ["n_probes"]] - # TODO (tfeher) add "shmem_carveout" - attr_str = attr_str + [lut_str, idt_str] - return "SearchParams(type=IVF-PQ, " + (", ".join(attr_str)) + ")" - - @property - def n_probes(self): - return self.params.n_probes - - @property - def lut_dtype(self): - return self.params.lut_dtype - - @property - def internal_distance_dtype(self): - return self.params.internal_distance_dtype - - -@auto_sync_handle -@auto_convert_output -def search(SearchParams search_params, - Index index, - queries, - k, - neighbors=None, - distances=None, - DeviceMemoryResource memory_resource=None, - handle=None): - """ - Find the k nearest neighbors for each query. - - Parameters - ---------- - search_params : SearchParams - index : Index - Trained IVF-PQ index. - queries : CUDA array interface compliant matrix shape (n_samples, dim) - Supported dtype [float, int8, uint8] - k : int - The number of neighbors. - neighbors : Optional CUDA array interface compliant matrix shape - (n_queries, k), dtype int64_t. If supplied, neighbor - indices will be written here in-place. (default None) - distances : Optional CUDA array interface compliant matrix shape - (n_queries, k) If supplied, the distances to the - neighbors will be written here in-place. (default None) - memory_resource : RMM DeviceMemoryResource object, optional - This can be used to explicitly manage the temporary memory - allocation during search. Passing a pooling allocator can reduce - memory allocation overhead. If not specified, then the memory - resource from the raft handle is used. - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_pq - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = ivf_pq.build(ivf_pq.IndexParams(), dataset, handle=handle) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 10 - >>> search_params = ivf_pq.SearchParams( - ... n_probes=20, - ... lut_dtype=cp.float16, - ... internal_distance_dtype=cp.float32 - ... ) - >>> # Using a pooling allocator reduces overhead of temporary array - >>> # creation during search. This is useful if multiple searches - >>> # are performad with same query size. - >>> import rmm - >>> mr = rmm.mr.PoolMemoryResource( - ... rmm.mr.CudaMemoryResource(), - ... initial_pool_size=2**29, - ... maximum_pool_size=2**31 - ... ) - >>> distances, neighbors = ivf_pq.search(search_params, index, queries, - ... k, memory_resource=mr, - ... handle=handle) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - >>> neighbors = cp.asarray(neighbors) - >>> distances = cp.asarray(distances) - """ - - if not index.trained: - raise ValueError("Index need to be built before calling search.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - queries_cai = cai_wrapper(queries) - queries_dt = queries_cai.dtype - cdef uint32_t n_queries = queries_cai.shape[0] - - _check_input_array(queries_cai, [np.dtype('float32'), np.dtype('byte'), - np.dtype('ubyte')], - exp_cols=index.dim) - - if neighbors is None: - neighbors = device_ndarray.empty((n_queries, k), dtype='int64') - - neighbors_cai = cai_wrapper(neighbors) - _check_input_array(neighbors_cai, [np.dtype('int64')], - exp_rows=n_queries, exp_cols=k) - - if distances is None: - distances = device_ndarray.empty((n_queries, k), dtype='float32') - - distances_cai = cai_wrapper(distances) - _check_input_array(distances_cai, [np.dtype('float32')], - exp_rows=n_queries, exp_cols=k) - - cdef c_ivf_pq.search_params params = search_params.params - - cdef uintptr_t neighbors_ptr = neighbors_cai.data - cdef uintptr_t distances_ptr = distances_cai.data - # TODO(tfeher) pass mr_ptr arg - cdef device_memory_resource* mr_ptr = nullptr - if memory_resource is not None: - mr_ptr = memory_resource.get_mr() - - if queries_dt == np.float32: - with cuda_interruptible(): - c_ivf_pq.search(deref(handle_), - params, - deref(index.index), - get_dmv_float(queries_cai, check_shape=True), - get_dmv_int64(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - elif queries_dt == np.byte: - with cuda_interruptible(): - c_ivf_pq.search(deref(handle_), - params, - deref(index.index), - get_dmv_int8(queries_cai, check_shape=True), - get_dmv_int64(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - elif queries_dt == np.ubyte: - with cuda_interruptible(): - c_ivf_pq.search(deref(handle_), - params, - deref(index.index), - get_dmv_uint8(queries_cai, check_shape=True), - get_dmv_int64(neighbors_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True)) - else: - raise ValueError("query dtype %s not supported" % queries_dt) - - return (distances, neighbors) - - -@auto_sync_handle -def save(filename, Index index, handle=None): - """ - Saves the index to a file. - - Saving / loading the index is experimental. The serialization format is - subject to change. - - Parameters - ---------- - filename : string - Name of the file. - index : Index - Trained IVF-PQ index. - {handle_docstring} - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_pq - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build index - >>> handle = DeviceResources() - >>> index = ivf_pq.build(ivf_pq.IndexParams(), dataset, handle=handle) - >>> ivf_pq.save("my_index.bin", index, handle=handle) - """ - if not index.trained: - raise ValueError("Index need to be built before saving it.") - - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - - c_ivf_pq.serialize(deref(handle_), c_filename, deref(index.index)) - - -@auto_sync_handle -def load(filename, handle=None): - """ - Loads index from a file. - - Saving / loading the index is experimental. The serialization format is - subject to change, therefore loading an index saved with a previous - version of raft is not guaranteed to work. - - Parameters - ---------- - filename : string - Name of the file. - {handle_docstring} - - Returns - ------- - index : Index - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_pq - >>> n_samples = 50000 - >>> n_features = 50 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> # Build and save index - >>> handle = DeviceResources() - >>> index = ivf_pq.build(ivf_pq.IndexParams(), dataset, handle=handle) - >>> ivf_pq.save("my_index.bin", index, handle=handle) - >>> del index - >>> n_queries = 100 - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> index = ivf_pq.load("my_index.bin", handle=handle) - >>> distances, neighbors = ivf_pq.search(ivf_pq.SearchParams(), index, - ... queries, k=10, handle=handle) - """ - if handle is None: - handle = DeviceResources() - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef string c_filename = filename.encode('utf-8') - index = Index() - - c_ivf_pq.deserialize(deref(handle_), c_filename, index.index) - index.trained = True - - return index diff --git a/python/pylibraft/pylibraft/neighbors/rbc.pyx b/python/pylibraft/pylibraft/neighbors/rbc.pyx deleted file mode 100644 index a703dc1745..0000000000 --- a/python/pylibraft/pylibraft/neighbors/rbc.pyx +++ /dev/null @@ -1,241 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libcpp cimport bool, nullptr -from libcpp.vector cimport vector - -from pylibraft.common import ( - DeviceResources, - auto_convert_output, - cai_wrapper, - device_ndarray, -) - -from libc.stdint cimport int64_t, uintptr_t - -from pylibraft.common.cpp.optional cimport optional -from pylibraft.common.handle cimport device_resources -from pylibraft.common.mdspan cimport get_dmv_bool, get_dmv_float, get_dmv_int64 - -from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.interruptible import cuda_interruptible -from pylibraft.neighbors.common import _check_input_array, _get_metric - -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - device_vector_view, - host_matrix_view, - make_device_matrix_view, - make_device_vector_view, - make_host_matrix_view, - row_major, -) -from pylibraft.neighbors.cpp.rbc cimport ( - BallCoverIndex as c_BallCoverIndex, - build_rbc_index as c_build_rbc_index, - eps_neighbors_rbc as c_eps_neighbors_rbc, - eps_neighbors_rbc_pass1 as c_eps_neighbors_rbc_pass1, - eps_neighbors_rbc_pass2 as c_eps_neighbors_rbc_pass2, -) - - -cdef class RbcIndex: - cdef readonly bool trained - cdef str data_type - - def __cinit__(self): - self.trained = False - self.data_type = None - - -cdef class RbcIndexFloat(RbcIndex): - cdef c_BallCoverIndex[int64_t, float, int64_t, int64_t]* index - - def __cinit__(self, dataset, handle): - cdef device_resources* handle_ = \ - handle.getHandle() - self.index = new c_BallCoverIndex[int64_t, float, int64_t, int64_t]( - deref(handle_), - get_dmv_float(dataset, check_shape=True), - _get_metric("euclidean")) - - -@auto_sync_handle -@auto_convert_output -def build_rbc_index(dataset, handle=None): - """ - Builds a random ball cover index from dataset using the L2-norm. - - Parameters - ---------- - dataset : array interface compliant matrix, row-major layout, - shape (n_samples, dim). Supported dtype [float] - {handle_docstring} - - Returns - ------- - index : Index - - Examples - -------- - see 'eps_neighbors_sparse' - - """ - if handle is None: - handle = DeviceResources() - - dataset_cai = cai_wrapper(dataset) - - # we require c-contiguous (rowmajor) inputs here - _check_input_array(dataset_cai, [np.dtype("float32")]) - - cdef device_resources* handle_ = \ - handle.getHandle() - - cdef RbcIndexFloat rbc_index_float - - if dataset_cai.dtype == np.float32: - rbc_index_float = RbcIndexFloat(dataset=dataset_cai, handle=handle) - rbc_index_float.data_type = "float32" - with cuda_interruptible(): - c_build_rbc_index( - deref(handle_), - deref(rbc_index_float.index)) - rbc_index_float.trained = True - return rbc_index_float - else: - raise TypeError("dtype %s not supported" % dataset_cai.dtype) - - -@auto_sync_handle -@auto_convert_output -def eps_neighbors(RbcIndex rbc_index, queries, eps, handle=None): - """ - Perform an epsilon neighborhood search with random ball cover (rbc) - using the L2-norm. - - Parameters - ---------- - rbc_index : RbcIndex created via 'build_rbc_index'. - Supported dtype [float] - queries : array interface compliant matrix, row-major layout, - shape (n_queries, dim) Supported dtype [float] - eps : threshold - {handle_docstring} - - Returns - ------- - adj_ia: array interface compliant object containing row indices for - adj_ja - - adj_ja: array interface compliant object containing adjacency mask - column indices - - vd: array interface compliant object containing row sums of adj - shape (n_queries + 1). vd[n_queries] contains the total sum - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors.rbc import eps_neighbors - >>> from pylibraft.neighbors.rbc import build_rbc_index - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> eps = 0.1 - >>> handle = DeviceResources() - >>> rbc_index = build_rbc_index(dataset) - >>> adj_ia, adj_ja, vd = eps_neighbors(rbc_index, queries, eps) - >>> adj_ia = cp.asarray(adj_ia) - >>> adj_ja = cp.asarray(adj_ja) - >>> vd = cp.asarray(vd) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - if not rbc_index.trained: - raise ValueError("Index need to be built before calling extend.") - - if handle is None: - handle = DeviceResources() - - queries_cai = cai_wrapper(queries) - - _check_input_array(queries_cai, [np.dtype(rbc_index.data_type)]) - - n_queries = queries_cai.shape[0] - - adj_ia = device_ndarray.empty((n_queries + 1, ), dtype='int64') - vd = device_ndarray.empty((n_queries + 1, ), dtype='int64') - adj_ia_cai = cai_wrapper(adj_ia) - vd_cai = cai_wrapper(vd) - - cdef device_resources* handle_ = \ - handle.getHandle() - - vd_vector_view = make_device_vector_view( - vd_cai.data, vd_cai.shape[0]) - adj_ia_vector_view = make_device_vector_view( - adj_ia_cai.data, adj_ia_cai.shape[0]) - - cdef RbcIndexFloat rbc_index_float - - if queries_cai.dtype == np.float32: - rbc_index_float = rbc_index - with cuda_interruptible(): - c_eps_neighbors_rbc_pass1( - deref(handle_), - deref(rbc_index_float.index), - get_dmv_float(queries_cai, check_shape=True), - adj_ia_vector_view, - vd_vector_view, - eps) - else: - raise TypeError("dtype %s not supported" % queries_cai.dtype) - - handle.sync() - n_nnz = adj_ia.copy_to_host()[n_queries] - adj_ja = device_ndarray.empty((n_nnz, ), dtype='int64') - adj_ja_cai = cai_wrapper(adj_ja) - adj_ja_vector_view = make_device_vector_view( - adj_ja_cai.data, adj_ja_cai.shape[0]) - - if queries_cai.dtype == np.float32: - with cuda_interruptible(): - c_eps_neighbors_rbc_pass2( - deref(handle_), - deref(rbc_index_float.index), - get_dmv_float(queries_cai, check_shape=True), - adj_ia_vector_view, - adj_ja_vector_view, - vd_vector_view, - eps) - else: - raise TypeError("dtype %s not supported" % queries_cai.dtype) - - return (adj_ia, adj_ja, vd) diff --git a/python/pylibraft/pylibraft/neighbors/refine.pyx b/python/pylibraft/pylibraft/neighbors/refine.pyx deleted file mode 100644 index a9bf811c9f..0000000000 --- a/python/pylibraft/pylibraft/neighbors/refine.pyx +++ /dev/null @@ -1,375 +0,0 @@ -# -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -import numpy as np - -from cython.operator cimport dereference as deref -from libc.stdint cimport int8_t, int64_t, uint8_t, uintptr_t -from libcpp cimport bool, nullptr - -from pylibraft.distance.distance_type cimport DistanceType - -from pylibraft.common import ( - DeviceResources, - auto_convert_output, - cai_wrapper, - device_ndarray, -) - -from pylibraft.common.handle cimport device_resources - -from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.input_validation import is_c_contiguous -from pylibraft.common.interruptible import cuda_interruptible - -from pylibraft.distance.distance_type cimport DistanceType - -import pylibraft.neighbors.ivf_pq as ivf_pq -from pylibraft.neighbors.common import _get_metric - -cimport pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq as c_ivf_pq -from pylibraft.common.cpp.mdspan cimport ( - device_matrix_view, - host_matrix_view, - make_host_matrix_view, - row_major, -) -from pylibraft.common.mdspan cimport ( - get_dmv_float, - get_dmv_int8, - get_dmv_int64, - get_dmv_uint8, -) -from pylibraft.neighbors.common cimport _get_metric_string -from pylibraft.neighbors.ivf_pq.cpp.c_ivf_pq cimport ( - index_params, - search_params, -) - - -# We omit the const qualifiers in the interface for refine, because cython -# has an issue parsing it (https://github.com/cython/cython/issues/4180). -cdef extern from "raft_runtime/neighbors/refine.hpp" \ - namespace "raft::runtime::neighbors" nogil: - - cdef void c_refine "raft::runtime::neighbors::refine" ( - const device_resources& handle, - device_matrix_view[float, int64_t, row_major] dataset, - device_matrix_view[float, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] candidates, - device_matrix_view[int64_t, int64_t, row_major] indices, - device_matrix_view[float, int64_t, row_major] distances, - DistanceType metric) except + - - cdef void c_refine "raft::runtime::neighbors::refine" ( - const device_resources& handle, - device_matrix_view[uint8_t, int64_t, row_major] dataset, - device_matrix_view[uint8_t, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] candidates, - device_matrix_view[int64_t, int64_t, row_major] indices, - device_matrix_view[float, int64_t, row_major] distances, - DistanceType metric) except + - - cdef void c_refine "raft::runtime::neighbors::refine" ( - const device_resources& handle, - device_matrix_view[int8_t, int64_t, row_major] dataset, - device_matrix_view[int8_t, int64_t, row_major] queries, - device_matrix_view[int64_t, int64_t, row_major] candidates, - device_matrix_view[int64_t, int64_t, row_major] indices, - device_matrix_view[float, int64_t, row_major] distances, - DistanceType metric) except + - - cdef void c_refine "raft::runtime::neighbors::refine" ( - const device_resources& handle, - host_matrix_view[float, int64_t, row_major] dataset, - host_matrix_view[float, int64_t, row_major] queries, - host_matrix_view[int64_t, int64_t, row_major] candidates, - host_matrix_view[int64_t, int64_t, row_major] indices, - host_matrix_view[float, int64_t, row_major] distances, - DistanceType metric) except + - - cdef void c_refine "raft::runtime::neighbors::refine" ( - const device_resources& handle, - host_matrix_view[uint8_t, int64_t, row_major] dataset, - host_matrix_view[uint8_t, int64_t, row_major] queries, - host_matrix_view[int64_t, int64_t, row_major] candidates, - host_matrix_view[int64_t, int64_t, row_major] indices, - host_matrix_view[float, int64_t, row_major] distances, - DistanceType metric) except + - - cdef void c_refine "raft::runtime::neighbors::refine" ( - const device_resources& handle, - host_matrix_view[int8_t, int64_t, row_major] dataset, - host_matrix_view[int8_t, int64_t, row_major] queries, - host_matrix_view[int64_t, int64_t, row_major] candidates, - host_matrix_view[int64_t, int64_t, row_major] indices, - host_matrix_view[float, int64_t, row_major] distances, - DistanceType metric) except + - - -def _get_array_params(array_interface, check_dtype=None): - dtype = np.dtype(array_interface["typestr"]) - if check_dtype is None and dtype != check_dtype: - raise TypeError("dtype %s not supported" % dtype) - shape = array_interface["shape"] - if len(shape) != 2: - raise ValueError("Expected a 2D array, got %d D" % len(shape)) - data = array_interface["data"][0] - return (shape, dtype, data) - - -cdef host_matrix_view[float, int64_t, row_major] \ - get_host_matrix_view_float(array) except *: - shape, dtype, data = _get_array_params( - array.__array_interface__, check_dtype=np.float32) - return make_host_matrix_view[float, int64_t, row_major]( - data, shape[0], shape[1]) - - -cdef host_matrix_view[int64_t, int64_t, row_major] \ - get_host_matrix_view_int64_t(array) except *: - shape, dtype, data = _get_array_params( - array.__array_interface__, check_dtype=np.int64) - return make_host_matrix_view[int64_t, int64_t, row_major]( - data, shape[0], shape[1]) - - -cdef host_matrix_view[uint8_t, int64_t, row_major] \ - get_host_matrix_view_uint8(array) except *: - shape, dtype, data = _get_array_params( - array.__array_interface__, check_dtype=np.uint8) - return make_host_matrix_view[uint8_t, int64_t, row_major]( - data, shape[0], shape[1]) - - -cdef host_matrix_view[int8_t, int64_t, row_major] \ - get_host_matrix_view_int8(array) except *: - shape, dtype, data = _get_array_params( - array.__array_interface__, check_dtype=np.int8) - return make_host_matrix_view[int8_t, int64_t, row_major]( - data, shape[0], shape[1]) - - -@auto_sync_handle -@auto_convert_output -def refine(dataset, queries, candidates, k=None, indices=None, distances=None, - metric="sqeuclidean", handle=None): - """ - Refine nearest neighbor search. - - Refinement is an operation that follows an approximate NN search. The - approximate search has already selected n_candidates neighbor candidates - for each query. We narrow it down to k neighbors. For each query, we - calculate the exact distance between the query and its n_candidates - neighbor candidate, and select the k nearest ones. - - Input arrays can be either CUDA array interface compliant matrices or - array interface compliant matrices in host memory. All array must be in - the same memory space. - - Parameters - ---------- - index_params : IndexParams object - dataset : array interface compliant matrix, shape (n_samples, dim) - Supported dtype [float, int8, uint8] - queries : array interface compliant matrix, shape (n_queries, dim) - Supported dtype [float, int8, uint8] - candidates : array interface compliant matrix, shape (n_queries, k0) - Supported dtype int64 - k : int - Number of neighbors to search (k <= k0). Optional if indices or - distances arrays are given (in which case their second dimension - is k). - indices : Optional array interface compliant matrix shape \ - (n_queries, k). - If supplied, neighbor indices will be written here in-place. - (default None). Supported dtype int64. - distances : Optional array interface compliant matrix shape \ - (n_queries, k). - If supplied, neighbor indices will be written here in-place. - (default None) Supported dtype float. - {handle_docstring} - - Returns - ------- - index: ivf_pq.Index - - Examples - -------- - >>> import cupy as cp - >>> from pylibraft.common import DeviceResources - >>> from pylibraft.neighbors import ivf_pq, refine - >>> n_samples = 50000 - >>> n_features = 50 - >>> n_queries = 1000 - >>> dataset = cp.random.random_sample((n_samples, n_features), - ... dtype=cp.float32) - >>> handle = DeviceResources() - >>> index_params = ivf_pq.IndexParams(n_lists=1024, - ... metric="sqeuclidean", - ... pq_dim=10) - >>> index = ivf_pq.build(index_params, dataset, handle=handle) - >>> # Search using the built index - >>> queries = cp.random.random_sample((n_queries, n_features), - ... dtype=cp.float32) - >>> k = 40 - >>> _, candidates = ivf_pq.search(ivf_pq.SearchParams(), index, - ... queries, k, handle=handle) - >>> k = 10 - >>> distances, neighbors = refine(dataset, queries, candidates, k, - ... handle=handle) - >>> distances = cp.asarray(distances) - >>> neighbors = cp.asarray(neighbors) - >>> # pylibraft functions are often asynchronous so the - >>> # handle needs to be explicitly synchronized - >>> handle.sync() - """ - - if handle is None: - handle = DeviceResources() - - if hasattr(dataset, "__cuda_array_interface__"): - return _refine_device(dataset, queries, candidates, k, indices, - distances, metric, handle) - else: - return _refine_host(dataset, queries, candidates, k, indices, - distances, metric, handle) - - -def _refine_device(dataset, queries, candidates, k, indices, distances, - metric, handle): - cdef device_resources* handle_ = \ - handle.getHandle() - - if k is None: - if indices is not None: - k = cai_wrapper(indices).shape[1] - elif distances is not None: - k = cai_wrapper(distances).shape[1] - else: - raise ValueError("Argument k must be specified if both indices " - "and distances arg is None") - - queries_cai = cai_wrapper(queries) - dataset_cai = cai_wrapper(dataset) - candidates_cai = cai_wrapper(candidates) - n_queries = cai_wrapper(queries).shape[0] - - if indices is None: - indices = device_ndarray.empty((n_queries, k), dtype='int64') - - if distances is None: - distances = device_ndarray.empty((n_queries, k), dtype='float32') - - indices_cai = cai_wrapper(indices) - distances_cai = cai_wrapper(distances) - - cdef DistanceType c_metric = _get_metric(metric) - - if dataset_cai.dtype == np.float32: - with cuda_interruptible(): - c_refine(deref(handle_), - get_dmv_float(dataset_cai, check_shape=True), - get_dmv_float(queries_cai, check_shape=True), - get_dmv_int64(candidates_cai, check_shape=True), - get_dmv_int64(indices_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True), - c_metric) - elif dataset_cai.dtype == np.int8: - with cuda_interruptible(): - c_refine(deref(handle_), - get_dmv_int8(dataset_cai, check_shape=True), - get_dmv_int8(queries_cai, check_shape=True), - get_dmv_int64(candidates_cai, check_shape=True), - get_dmv_int64(indices_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True), - c_metric) - elif dataset_cai.dtype == np.uint8: - with cuda_interruptible(): - c_refine(deref(handle_), - get_dmv_uint8(dataset_cai, check_shape=True), - get_dmv_uint8(queries_cai, check_shape=True), - get_dmv_int64(candidates_cai, check_shape=True), - get_dmv_int64(indices_cai, check_shape=True), - get_dmv_float(distances_cai, check_shape=True), - c_metric) - else: - raise TypeError("dtype %s not supported" % dataset_cai.dtype) - - return (distances, indices) - - -def _refine_host(dataset, queries, candidates, k, indices, distances, - metric, handle): - cdef device_resources* handle_ = \ - handle.getHandle() - - if k is None: - if indices is not None: - k = indices.__array_interface__["shape"][1] - elif distances is not None: - k = distances.__array_interface__["shape"][1] - else: - raise ValueError("Argument k must be specified if both indices " - "and distances arg is None") - - n_queries = queries.__array_interface__["shape"][0] - - if indices is None: - indices = np.empty((n_queries, k), dtype='int64') - - if distances is None: - distances = np.empty((n_queries, k), dtype='float32') - - cdef DistanceType c_metric = _get_metric(metric) - - dtype = np.dtype(dataset.__array_interface__["typestr"]) - - if dtype == np.float32: - with cuda_interruptible(): - c_refine(deref(handle_), - get_host_matrix_view_float(dataset), - get_host_matrix_view_float(queries), - get_host_matrix_view_int64_t(candidates), - get_host_matrix_view_int64_t(indices), - get_host_matrix_view_float(distances), - c_metric) - elif dtype == np.int8: - with cuda_interruptible(): - c_refine(deref(handle_), - get_host_matrix_view_int8(dataset), - get_host_matrix_view_int8(queries), - get_host_matrix_view_int64_t(candidates), - get_host_matrix_view_int64_t(indices), - get_host_matrix_view_float(distances), - c_metric) - elif dtype == np.uint8: - with cuda_interruptible(): - c_refine(deref(handle_), - get_host_matrix_view_uint8(dataset), - get_host_matrix_view_uint8(queries), - get_host_matrix_view_int64_t(candidates), - get_host_matrix_view_int64_t(indices), - get_host_matrix_view_float(distances), - c_metric) - else: - raise TypeError("dtype %s not supported" % dtype) - - return (distances, indices) diff --git a/cpp/cmake/thirdparty/get_fmt.cmake b/python/pylibraft/pylibraft/sparse/CMakeLists.txt similarity index 71% rename from cpp/cmake/thirdparty/get_fmt.cmake rename to python/pylibraft/pylibraft/sparse/CMakeLists.txt index c06f8a78bb..3779fd2715 100644 --- a/cpp/cmake/thirdparty/get_fmt.cmake +++ b/python/pylibraft/pylibraft/sparse/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -12,11 +12,4 @@ # the License. # ============================================================================= -# Use CPM to find or clone fmt -function(find_and_configure_fmt) - - include(${rapids-cmake-dir}/cpm/fmt.cmake) - rapids_cpm_fmt(INSTALL_EXPORT_SET rmm-exports BUILD_EXPORT_SET rmm-exports) -endfunction() - -find_and_configure_fmt() \ No newline at end of file +add_subdirectory(linalg) diff --git a/python/raft-ann-bench/src/raft_ann_bench/__init__.py b/python/pylibraft/pylibraft/sparse/__init__.py similarity index 84% rename from python/raft-ann-bench/src/raft_ann_bench/__init__.py rename to python/pylibraft/pylibraft/sparse/__init__.py index 80a3b3f284..c77def5bb0 100644 --- a/python/raft-ann-bench/src/raft_ann_bench/__init__.py +++ b/python/pylibraft/pylibraft/sparse/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,4 +13,6 @@ # limitations under the License. # -from ._version import __git_commit__, __version__ +from pylibraft.sparse import linalg + +__all__ = ["linalg"] diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt b/python/pylibraft/pylibraft/sparse/linalg/CMakeLists.txt similarity index 82% rename from python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt rename to python/pylibraft/pylibraft/sparse/linalg/CMakeLists.txt index 37c57c45db..ef16981644 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt +++ b/python/pylibraft/pylibraft/sparse/linalg/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -13,12 +13,15 @@ # ============================================================================= # Set the list of Cython files to build -set(cython_sources ivf_flat.pyx) +set(cython_sources lanczos.pyx) + +# TODO: should finally be replaced with 'compiled' library to be more generic, when that is +# available set(linked_libraries raft::raft raft::compiled) # Build all of the Cython targets rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfflat_ + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX sparse_ ) diff --git a/python/pylibraft/pylibraft/cluster/cpp/__init__.pxd b/python/pylibraft/pylibraft/sparse/linalg/__init__.pxd similarity index 100% rename from python/pylibraft/pylibraft/cluster/cpp/__init__.pxd rename to python/pylibraft/pylibraft/sparse/linalg/__init__.pxd diff --git a/python/pylibraft/pylibraft/matrix/cpp/__init__.py b/python/pylibraft/pylibraft/sparse/linalg/__init__.py similarity index 85% rename from python/pylibraft/pylibraft/matrix/cpp/__init__.py rename to python/pylibraft/pylibraft/sparse/linalg/__init__.py index 8f2cc34855..04a8106496 100644 --- a/python/pylibraft/pylibraft/matrix/cpp/__init__.py +++ b/python/pylibraft/pylibraft/sparse/linalg/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,3 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +from .lanczos import eigsh + +__all__ = ["eigsh"] diff --git a/python/pylibraft/pylibraft/matrix/cpp/__init__.pxd b/python/pylibraft/pylibraft/sparse/linalg/cpp/__init__.pxd similarity index 100% rename from python/pylibraft/pylibraft/matrix/cpp/__init__.pxd rename to python/pylibraft/pylibraft/sparse/linalg/cpp/__init__.pxd diff --git a/python/pylibraft/pylibraft/cluster/cpp/__init__.py b/python/pylibraft/pylibraft/sparse/linalg/cpp/__init__.py similarity index 100% rename from python/pylibraft/pylibraft/cluster/cpp/__init__.py rename to python/pylibraft/pylibraft/sparse/linalg/cpp/__init__.py diff --git a/python/pylibraft/pylibraft/sparse/linalg/lanczos.pyx b/python/pylibraft/pylibraft/sparse/linalg/lanczos.pyx new file mode 100644 index 0000000000..dc2a84b428 --- /dev/null +++ b/python/pylibraft/pylibraft/sparse/linalg/lanczos.pyx @@ -0,0 +1,277 @@ +# +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# cython: profile=False +# distutils: language = c++ +# cython: embedsignature = True +# cython: language_level = 3 + +import cupy as cp +import numpy as np + +from cython.operator cimport dereference as deref +from libc.stdint cimport int64_t, uint32_t, uint64_t, uintptr_t + +from pylibraft.common import Handle, cai_wrapper, device_ndarray +from pylibraft.common.handle import auto_sync_handle + +from libcpp cimport bool + +from pylibraft.common.cpp.mdspan cimport ( + col_major, + device_matrix_view, + device_vector_view, + make_device_matrix_view, + make_device_vector_view, + row_major, +) +from pylibraft.common.cpp.optional cimport optional +from pylibraft.common.handle cimport device_resources +from pylibraft.random.cpp.rng_state cimport RngState + + +cdef extern from "raft/sparse/solver/lanczos_types.hpp" \ + namespace "raft::sparse::solver" nogil: + + cdef cppclass lanczos_solver_config[ValueTypeT]: + int n_components + int max_iterations + int ncv + ValueTypeT tolerance + uint64_t seed + +cdef lanczos_solver_config[float] config_float +cdef lanczos_solver_config[double] config_double + +cdef extern from "raft_runtime/solver/lanczos.hpp" \ + namespace "raft::runtime::solver" nogil: + + cdef void lanczos_solver( + const device_resources &handle, + lanczos_solver_config[double] config, + device_vector_view[int64_t, uint32_t] rows, + device_vector_view[int64_t, uint32_t] cols, + device_vector_view[double, uint32_t] vals, + optional[device_vector_view[double, uint32_t]] v0, + device_vector_view[double, uint32_t] eigenvalues, + device_matrix_view[double, uint32_t, col_major] eigenvectors) except + + + cdef void lanczos_solver( + const device_resources &handle, + lanczos_solver_config[float] config, + device_vector_view[int64_t, uint32_t] rows, + device_vector_view[int64_t, uint32_t] cols, + device_vector_view[float, uint32_t] vals, + optional[device_vector_view[float, uint32_t]] v0, + device_vector_view[float, uint32_t] eigenvalues, + device_matrix_view[float, uint32_t, col_major] eigenvectors) except + + + cdef void lanczos_solver( + const device_resources &handle, + lanczos_solver_config[double] config, + device_vector_view[int, uint32_t] rows, + device_vector_view[int, uint32_t] cols, + device_vector_view[double, uint32_t] vals, + optional[device_vector_view[double, uint32_t]] v0, + device_vector_view[double, uint32_t] eigenvalues, + device_matrix_view[double, uint32_t, col_major] eigenvectors) except + + + cdef void lanczos_solver( + const device_resources &handle, + lanczos_solver_config[float] config, + device_vector_view[int, uint32_t] rows, + device_vector_view[int, uint32_t] cols, + device_vector_view[float, uint32_t] vals, + optional[device_vector_view[float, uint32_t]] v0, + device_vector_view[float, uint32_t] eigenvalues, + device_matrix_view[float, uint32_t, col_major] eigenvectors) except + + + +@auto_sync_handle +def eigsh(A, k=6, v0=None, ncv=None, maxiter=None, + tol=0, seed=None, handle=None): + """ + Find ``k`` eigenvalues and eigenvectors of the real symmetric square + matrix or complex Hermitian matrix ``A``. + + Solves ``Ax = wx``, the standard eigenvalue problem for ``w`` eigenvalues + with corresponding eigenvectors ``x``. + + Args: + a (spmatrix): A symmetric square sparse CSR matrix with + dimension ``(n, n)``. ``a`` must be of type + :class:`cupyx.scipy.sparse._csr.csr_matrix` + k (int): The number of eigenvalues and eigenvectors to compute. Must be + ``1 <= k < n``. + v0 (ndarray): Starting vector for iteration. If ``None``, a random + unit vector is used. + ncv (int): The number of Lanczos vectors generated. Must be + ``k + 1 < ncv < n``. If ``None``, default value is used. + maxiter (int): Maximum number of Lanczos update iterations. + If ``None``, default value is used. + tol (float): Tolerance for residuals ``||Ax - wx||``. If ``0``, machine + precision is used. + + Returns: + tuple: + It returns ``w`` and ``x`` + where ``w`` is eigenvalues and ``x`` is eigenvectors. + + .. seealso:: + :func:`scipy.sparse.linalg.eigsh` + :func:`cupyx.scipy.sparse.linalg.eigsh` + + .. note:: + This function uses the thick-restart Lanczos methods + (https://sdm.lbl.gov/~kewu/ps/trlan.html). + + """ + + if A is None: + raise Exception("'A' cannot be None!") + + rows = A.indptr + cols = A.indices + vals = A.data + + rows = cai_wrapper(rows) + cols = cai_wrapper(cols) + vals = cai_wrapper(vals) + + IndexType = rows.dtype + ValueType = vals.dtype + + N = A.shape[0] + n = N + nnz = A.nnz + + rows_ptr = rows.data + cols_ptr = cols.data + vals_ptr = vals.data + cdef optional[device_vector_view[double, uint32_t]] d_v0 + cdef optional[device_vector_view[float, uint32_t]] f_v0 + + if ncv is None: + ncv = min(n, max(2*k + 1, 20)) + else: + ncv = min(max(ncv, k + 2), n - 1) + + seed = seed if seed is not None else 42 + if maxiter is None: + maxiter = 10 * n + if tol == 0: + tol = np.finfo(ValueType).eps + + eigenvectors = device_ndarray.empty((N, k), dtype=ValueType, order='F') + eigenvalues = device_ndarray.empty((k), dtype=ValueType, order='F') + + eigenvectors_cai = cai_wrapper(eigenvectors) + eigenvalues_cai = cai_wrapper(eigenvalues) + + eigenvectors_ptr = eigenvectors_cai.data + eigenvalues_ptr = eigenvalues_cai.data + + handle = handle if handle is not None else Handle() + cdef device_resources *h = handle.getHandle() + + if IndexType == np.int32 and ValueType == np.float32: + config_float.n_components = k + config_float.max_iterations = maxiter + config_float.ncv = ncv + config_float.tolerance = tol + config_float.seed = seed + if v0 is not None: + v0 = cai_wrapper(v0) + v0_ptr = v0.data + f_v0 = make_device_vector_view(v0_ptr, N) + lanczos_solver( + deref(h), + config_float, + make_device_vector_view(rows_ptr, (N + 1)), + make_device_vector_view(cols_ptr, nnz), + make_device_vector_view(vals_ptr, nnz), + f_v0, + make_device_vector_view(eigenvalues_ptr, k), + make_device_matrix_view[float, uint32_t, col_major]( + eigenvectors_ptr, N, k), + ) + elif IndexType == np.int64 and ValueType == np.float32: + config_float.n_components = k + config_float.max_iterations = maxiter + config_float.ncv = ncv + config_float.tolerance = tol + config_float.seed = seed + if v0 is not None: + v0 = cai_wrapper(v0) + v0_ptr = v0.data + f_v0 = make_device_vector_view(v0_ptr, N) + lanczos_solver( + deref(h), + config_float, + make_device_vector_view(rows_ptr, (N + 1)), + make_device_vector_view(cols_ptr, nnz), + make_device_vector_view(vals_ptr, nnz), + f_v0, + make_device_vector_view(eigenvalues_ptr, k), + make_device_matrix_view[float, uint32_t, col_major]( + eigenvectors_ptr, N, k), + ) + elif IndexType == np.int32 and ValueType == np.float64: + config_double.n_components = k + config_double.max_iterations = maxiter + config_double.ncv = ncv + config_double.tolerance = tol + config_double.seed = seed + if v0 is not None: + v0 = cai_wrapper(v0) + v0_ptr = v0.data + d_v0 = make_device_vector_view(v0_ptr, N) + lanczos_solver( + deref(h), + config_double, + make_device_vector_view(rows_ptr, (N + 1)), + make_device_vector_view(cols_ptr, nnz), + make_device_vector_view(vals_ptr, nnz), + d_v0, + make_device_vector_view(eigenvalues_ptr, k), + make_device_matrix_view[double, uint32_t, col_major]( + eigenvectors_ptr, N, k), + ) + elif IndexType == np.int64 and ValueType == np.float64: + config_double.n_components = k + config_double.max_iterations = maxiter + config_double.ncv = ncv + config_double.tolerance = tol + config_double.seed = seed + if v0 is not None: + v0 = cai_wrapper(v0) + v0_ptr = v0.data + d_v0 = make_device_vector_view(v0_ptr, N) + lanczos_solver( + deref(h), + config_double, + make_device_vector_view(rows_ptr, (N + 1)), + make_device_vector_view(cols_ptr, nnz), + make_device_vector_view(vals_ptr, nnz), + d_v0, + make_device_vector_view(eigenvalues_ptr, k), + make_device_matrix_view[double, uint32_t, col_major]( + eigenvectors_ptr, N, k), + ) + else: + raise ValueError("dtype IndexType=%s and ValueType=%s not supported" % + (IndexType, ValueType)) + + return (cp.asarray(eigenvalues), cp.asarray(eigenvectors)) diff --git a/python/pylibraft/pylibraft/test/ann_utils.py b/python/pylibraft/pylibraft/test/ann_utils.py deleted file mode 100644 index 60db7f3273..0000000000 --- a/python/pylibraft/pylibraft/test/ann_utils.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# h ttp://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import numpy as np - - -def generate_data(shape, dtype): - if dtype == np.byte: - x = np.random.randint(-127, 128, size=shape, dtype=np.byte) - elif dtype == np.ubyte: - x = np.random.randint(0, 255, size=shape, dtype=np.ubyte) - else: - x = np.random.random_sample(shape).astype(dtype) - - return x - - -def calc_recall(ann_idx, true_nn_idx): - assert ann_idx.shape == true_nn_idx.shape - n = 0 - for i in range(ann_idx.shape[0]): - n += np.intersect1d(ann_idx[i, :], true_nn_idx[i, :]).size - recall = n / ann_idx.size - return recall diff --git a/python/pylibraft/pylibraft/test/test_brute_force.py b/python/pylibraft/pylibraft/test/test_brute_force.py deleted file mode 100644 index 42095c3b9f..0000000000 --- a/python/pylibraft/pylibraft/test/test_brute_force.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from scipy.spatial.distance import cdist - -from pylibraft.common import DeviceResources, Stream, device_ndarray -from pylibraft.neighbors.brute_force import knn - - -@pytest.mark.parametrize("n_index_rows", [32, 100]) -@pytest.mark.parametrize("n_query_rows", [32, 100]) -@pytest.mark.parametrize("n_cols", [40, 100]) -@pytest.mark.parametrize("k", [1, 5, 32]) -@pytest.mark.parametrize( - "metric", - [ - "euclidean", - "cityblock", - "chebyshev", - "canberra", - "correlation", - "russellrao", - "cosine", - "sqeuclidean", - # "inner_product", - ], -) -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("dtype", [np.float32]) -def test_knn(n_index_rows, n_query_rows, n_cols, k, inplace, metric, dtype): - index = np.random.random_sample((n_index_rows, n_cols)).astype(dtype) - queries = np.random.random_sample((n_query_rows, n_cols)).astype(dtype) - - # RussellRao expects boolean arrays - if metric == "russellrao": - index[index < 0.5] = 0.0 - index[index >= 0.5] = 1.0 - queries[queries < 0.5] = 0.0 - queries[queries >= 0.5] = 1.0 - - indices = np.zeros((n_query_rows, k), dtype="int64") - distances = np.zeros((n_query_rows, k), dtype=dtype) - - index_device = device_ndarray(index) - - queries_device = device_ndarray(queries) - indices_device = device_ndarray(indices) - distances_device = device_ndarray(distances) - - s2 = Stream() - handle = DeviceResources(stream=s2) - ret_distances, ret_indices = knn( - index_device, - queries_device, - k, - indices=indices_device, - distances=distances_device, - metric=metric, - handle=handle, - ) - handle.sync() - - pw_dists = cdist(queries, index, metric=metric) - - distances_device = ret_distances if not inplace else distances_device - - actual_distances = distances_device.copy_to_host() - - actual_distances[actual_distances <= 1e-5] = 0.0 - argsort = np.argsort(pw_dists, axis=1) - - for i in range(pw_dists.shape[0]): - expected_indices = argsort[i] - gpu_dists = actual_distances[i] - - cpu_ordered = pw_dists[i, expected_indices] - np.testing.assert_allclose( - cpu_ordered[:k], gpu_dists, atol=1e-3, rtol=1e-3 - ) - - -def test_knn_check_col_major_inputs(): - # make sure that we get an exception if passed col-major inputs, - # instead of returning incorrect results - cp = pytest.importorskip("cupy") - n_index_rows, n_query_rows, n_cols = 128, 16, 32 - index = cp.random.random_sample((n_index_rows, n_cols), dtype="float32") - queries = cp.random.random_sample((n_query_rows, n_cols), dtype="float32") - - with pytest.raises(ValueError): - knn(cp.asarray(index, order="F"), queries, k=4) - - with pytest.raises(ValueError): - knn(index, cp.asarray(queries, order="F"), k=4) - - # shouldn't throw an exception with c-contiguous inputs - knn(index, queries, k=4) diff --git a/python/pylibraft/pylibraft/test/test_cagra.py b/python/pylibraft/pylibraft/test/test_cagra.py deleted file mode 100644 index ef8e54917a..0000000000 --- a/python/pylibraft/pylibraft/test/test_cagra.py +++ /dev/null @@ -1,292 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# h ttp://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from sklearn.neighbors import NearestNeighbors -from sklearn.preprocessing import normalize - -from pylibraft.common import device_ndarray -from pylibraft.neighbors import cagra -from pylibraft.test.ann_utils import calc_recall, generate_data - - -def run_cagra_build_search_test( - n_rows=10000, - n_cols=10, - n_queries=100, - k=10, - dtype=np.float32, - metric="sqeuclidean", - intermediate_graph_degree=128, - graph_degree=64, - build_algo="ivf_pq", - array_type="device", - compare=True, - inplace=True, - add_data_on_build=True, - search_params={}, -): - dataset = generate_data((n_rows, n_cols), dtype) - if metric == "inner_product": - dataset = normalize(dataset, norm="l2", axis=1) - dataset_device = device_ndarray(dataset) - - build_params = cagra.IndexParams( - metric=metric, - intermediate_graph_degree=intermediate_graph_degree, - graph_degree=graph_degree, - build_algo=build_algo, - ) - - if array_type == "device": - index = cagra.build(build_params, dataset_device) - else: - index = cagra.build(build_params, dataset) - - assert index.trained - - if not add_data_on_build: - dataset_1 = dataset[: n_rows // 2, :] - dataset_2 = dataset[n_rows // 2 :, :] - indices_1 = np.arange(n_rows // 2, dtype=np.uint32) - indices_2 = np.arange(n_rows // 2, n_rows, dtype=np.uint32) - if array_type == "device": - dataset_1_device = device_ndarray(dataset_1) - dataset_2_device = device_ndarray(dataset_2) - indices_1_device = device_ndarray(indices_1) - indices_2_device = device_ndarray(indices_2) - index = cagra.extend(index, dataset_1_device, indices_1_device) - index = cagra.extend(index, dataset_2_device, indices_2_device) - else: - index = cagra.extend(index, dataset_1, indices_1) - index = cagra.extend(index, dataset_2, indices_2) - - queries = generate_data((n_queries, n_cols), dtype) - out_idx = np.zeros((n_queries, k), dtype=np.uint32) - out_dist = np.zeros((n_queries, k), dtype=np.float32) - - queries_device = device_ndarray(queries) - out_idx_device = device_ndarray(out_idx) if inplace else None - out_dist_device = device_ndarray(out_dist) if inplace else None - - search_params = cagra.SearchParams(**search_params) - - ret_output = cagra.search( - search_params, - index, - queries_device, - k, - neighbors=out_idx_device, - distances=out_dist_device, - ) - - if not inplace: - out_dist_device, out_idx_device = ret_output - - if not compare: - return - - out_idx = out_idx_device.copy_to_host() - out_dist = out_dist_device.copy_to_host() - - # Calculate reference values with sklearn - skl_metric = { - "sqeuclidean": "sqeuclidean", - "inner_product": "cosine", - "euclidean": "euclidean", - }[metric] - nn_skl = NearestNeighbors( - n_neighbors=k, algorithm="brute", metric=skl_metric - ) - nn_skl.fit(dataset) - skl_idx = nn_skl.kneighbors(queries, return_distance=False) - - recall = calc_recall(out_idx, skl_idx) - assert recall > 0.7 - - -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.uint8]) -@pytest.mark.parametrize("array_type", ["device", "host"]) -@pytest.mark.parametrize("build_algo", ["ivf_pq", "nn_descent"]) -def test_cagra_dataset_dtype_host_device( - dtype, array_type, inplace, build_algo -): - # Note that inner_product tests use normalized input which we cannot - # represent in int8, therefore we test only sqeuclidean metric here. - run_cagra_build_search_test( - dtype=dtype, - inplace=inplace, - array_type=array_type, - build_algo=build_algo, - ) - - -@pytest.mark.parametrize( - "params", - [ - { - "intermediate_graph_degree": 64, - "graph_degree": 32, - "add_data_on_build": True, - "k": 1, - "metric": "sqeuclidean", - "build_algo": "ivf_pq", - }, - { - "intermediate_graph_degree": 32, - "graph_degree": 16, - "add_data_on_build": False, - "k": 5, - "metric": "sqeuclidean", - "build_algo": "ivf_pq", - }, - { - "intermediate_graph_degree": 128, - "graph_degree": 32, - "add_data_on_build": True, - "k": 10, - "metric": "sqeuclidean", - "build_algo": "nn_descent", - }, - ], -) -def test_cagra_index_params(params): - # Note that inner_product tests use normalized input which we cannot - # represent in int8, therefore we test only sqeuclidean metric here. - run_cagra_build_search_test( - k=params["k"], - metric=params["metric"], - graph_degree=params["graph_degree"], - intermediate_graph_degree=params["intermediate_graph_degree"], - compare=False, - build_algo=params["build_algo"], - ) - - -@pytest.mark.parametrize( - "params", - [ - { - "max_queries": 100, - "itopk_size": 32, - "max_iterations": 100, - "algo": "single_cta", - "team_size": 0, - "search_width": 1, - "min_iterations": 1, - "thread_block_size": 64, - "hashmap_mode": "hash", - "hashmap_min_bitlen": 0.2, - "hashmap_max_fill_rate": 0.5, - "num_random_samplings": 1, - }, - { - "max_queries": 10, - "itopk_size": 128, - "max_iterations": 0, - "algo": "multi_cta", - "team_size": 8, - "search_width": 2, - "min_iterations": 10, - "thread_block_size": 0, - "hashmap_mode": "auto", - "hashmap_min_bitlen": 0.9, - "hashmap_max_fill_rate": 0.5, - "num_random_samplings": 10, - }, - { - "max_queries": 0, - "itopk_size": 64, - "max_iterations": 0, - "algo": "multi_kernel", - "team_size": 16, - "search_width": 1, - "min_iterations": 0, - "thread_block_size": 0, - "hashmap_mode": "auto", - "hashmap_min_bitlen": 0, - "hashmap_max_fill_rate": 0.5, - "num_random_samplings": 1, - }, - { - "max_queries": 0, - "itopk_size": 64, - "max_iterations": 0, - "algo": "auto", - "team_size": 32, - "search_width": 4, - "min_iterations": 0, - "thread_block_size": 0, - "hashmap_mode": "auto", - "hashmap_min_bitlen": 0, - "hashmap_max_fill_rate": 0.5, - "num_random_samplings": 1, - }, - ], -) -def test_cagra_search_params(params): - # Note that inner_product tests use normalized input which we cannot - # represent in int8, therefore we test only sqeuclidean metric here. - run_cagra_build_search_test(search_params=params) - - -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.ubyte]) -@pytest.mark.parametrize("include_dataset", [True, False]) -def test_save_load(dtype, include_dataset): - n_rows = 10000 - n_cols = 50 - n_queries = 1000 - - dataset = generate_data((n_rows, n_cols), dtype) - dataset_device = device_ndarray(dataset) - - build_params = cagra.IndexParams() - index = cagra.build(build_params, dataset_device) - - assert index.trained - filename = "my_index.bin" - cagra.save(filename, index, include_dataset=include_dataset) - loaded_index = cagra.load(filename) - - # if we didn't save the dataset with the index, we need to update the - # index with an already loaded copy - if not include_dataset: - loaded_index.update_dataset(dataset) - - queries = generate_data((n_queries, n_cols), dtype) - - queries_device = device_ndarray(queries) - search_params = cagra.SearchParams() - k = 10 - - distance_dev, neighbors_dev = cagra.search( - search_params, index, queries_device, k - ) - - neighbors = neighbors_dev.copy_to_host() - dist = distance_dev.copy_to_host() - del index - - distance_dev, neighbors_dev = cagra.search( - search_params, loaded_index, queries_device, k - ) - - neighbors2 = neighbors_dev.copy_to_host() - dist2 = distance_dev.copy_to_host() - - assert np.all(neighbors == neighbors2) - assert np.allclose(dist, dist2, rtol=1e-6) diff --git a/python/pylibraft/pylibraft/test/test_distance.py b/python/pylibraft/pylibraft/test/test_distance.py deleted file mode 100644 index 34ed86db01..0000000000 --- a/python/pylibraft/pylibraft/test/test_distance.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from scipy.spatial.distance import cdist - -from pylibraft.common import DeviceResources, Stream, device_ndarray -from pylibraft.distance import pairwise_distance - - -@pytest.mark.parametrize("n_rows", [50, 100]) -@pytest.mark.parametrize("n_cols", [10, 50]) -@pytest.mark.parametrize( - "metric", - [ - "euclidean", - "cityblock", - "chebyshev", - "canberra", - "correlation", - "hamming", - "jensenshannon", - "russellrao", - "cosine", - "sqeuclidean", - "inner_product", - ], -) -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("order", ["F", "C"]) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_distance(n_rows, n_cols, inplace, metric, order, dtype): - input1 = np.random.random_sample((n_rows, n_cols)) - input1 = np.asarray(input1, order=order).astype(dtype) - - # RussellRao expects boolean arrays - if metric == "russellrao": - input1[input1 < 0.5] = 0 - input1[input1 >= 0.5] = 1 - - # JensenShannon expects probability arrays - elif metric == "jensenshannon": - norm = np.sum(input1, axis=1) - input1 = (input1.T / norm).T - - output = np.zeros((n_rows, n_rows), dtype=dtype) - - if metric == "inner_product": - expected = np.matmul(input1, input1.T) - else: - expected = cdist(input1, input1, metric) - - input1_device = device_ndarray(input1) - output_device = device_ndarray(output) if inplace else None - - s2 = Stream() - handle = DeviceResources(stream=s2) - ret_output = pairwise_distance( - input1_device, input1_device, output_device, metric, handle=handle - ) - handle.sync() - - output_device = ret_output if not inplace else output_device - - actual = output_device.copy_to_host() - - assert np.allclose(expected, actual, atol=1e-3, rtol=1e-3) diff --git a/python/pylibraft/pylibraft/test/test_doctests.py b/python/pylibraft/pylibraft/test/test_doctests.py index c75f565236..b1de79dfcd 100644 --- a/python/pylibraft/pylibraft/test/test_doctests.py +++ b/python/pylibraft/pylibraft/test/test_doctests.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,10 +20,6 @@ import pytest -import pylibraft.cluster -import pylibraft.distance -import pylibraft.matrix -import pylibraft.neighbors import pylibraft.random # Code adapted from https://github.com/rapidsai/cudf/blob/branch-23.02/python/cudf/cudf/tests/test_doctests.py # noqa @@ -92,16 +88,7 @@ def _find_doctests_in_obj(obj, finder=None, criteria=None): # since the root pylibraft module doesn't import submodules (or define an # __all__) we are explicitly adding all the submodules we want to run # doctests for here -DOC_STRINGS = list(_find_doctests_in_obj(pylibraft.cluster)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.common)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.distance)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.matrix.select_k)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.neighbors)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.neighbors.brute_force)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.neighbors.cagra)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.neighbors.ivf_flat)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.neighbors.ivf_pq)) -DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.neighbors.refine)) +DOC_STRINGS = list(_find_doctests_in_obj(pylibraft.common)) DOC_STRINGS.extend(_find_doctests_in_obj(pylibraft.random)) diff --git a/python/pylibraft/pylibraft/test/test_eps_neighborhood.py b/python/pylibraft/pylibraft/test/test_eps_neighborhood.py deleted file mode 100644 index f2643de904..0000000000 --- a/python/pylibraft/pylibraft/test/test_eps_neighborhood.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from scipy.sparse import csr_array - -from pylibraft.common import DeviceResources, Stream -from pylibraft.neighbors.brute_force import eps_neighbors as eps_neighbors_bf -from pylibraft.neighbors.rbc import ( - build_rbc_index, - eps_neighbors as eps_neighbors_rbc, -) - - -def test_bf_eps_neighbors_check_col_major_inputs(): - # make sure that we get an exception if passed col-major inputs, - # instead of returning incorrect results - cp = pytest.importorskip("cupy") - n_index_rows, n_query_rows, n_cols = 128, 16, 32 - eps = 0.02 - index = cp.random.random_sample((n_index_rows, n_cols), dtype="float32") - queries = cp.random.random_sample((n_query_rows, n_cols), dtype="float32") - - with pytest.raises(ValueError): - eps_neighbors_bf(cp.asarray(index, order="F"), queries, eps) - - with pytest.raises(ValueError): - eps_neighbors_bf(index, cp.asarray(queries, order="F"), eps) - - # shouldn't throw an exception with c-contiguous inputs - eps_neighbors_bf(index, queries, eps) - - -def test_rbc_eps_neighbors_check_col_major_inputs(): - # make sure that we get an exception if passed col-major inputs, - # instead of returning incorrect results - cp = pytest.importorskip("cupy") - n_index_rows, n_query_rows, n_cols = 128, 16, 32 - eps = 0.02 - index = cp.random.random_sample((n_index_rows, n_cols), dtype="float32") - queries = cp.random.random_sample((n_query_rows, n_cols), dtype="float32") - - with pytest.raises(ValueError): - build_rbc_index(cp.asarray(index, order="F")) - - rbc_index = build_rbc_index(index) - - with pytest.raises(ValueError): - eps_neighbors_rbc(rbc_index, cp.asarray(queries, order="F"), eps) - - eps_neighbors_rbc(rbc_index, queries, eps) - - -@pytest.mark.parametrize("n_index_rows", [32, 100, 1000]) -@pytest.mark.parametrize("n_query_rows", [32, 100, 1000]) -@pytest.mark.parametrize("n_cols", [2, 3, 40, 100]) -def test_eps_neighbors(n_index_rows, n_query_rows, n_cols): - s2 = Stream() - handle = DeviceResources(stream=s2) - - cp = pytest.importorskip("cupy") - eps = 0.02 - index = cp.random.random_sample((n_index_rows, n_cols), dtype="float32") - queries = cp.random.random_sample((n_query_rows, n_cols), dtype="float32") - - # brute force - adj_bf, vd_bf = eps_neighbors_bf(index, queries, eps, handle=handle) - adj_bf = cp.asarray(adj_bf) - vd_bf = cp.asarray(vd_bf) - - rbc_index = build_rbc_index(index, handle=handle) - adj_rbc_ia, adj_rbc_ja, vd_rbc = eps_neighbors_rbc( - rbc_index, queries, eps, handle=handle - ) - adj_rbc_ia = cp.asarray(adj_rbc_ia) - adj_rbc_ja = cp.asarray(adj_rbc_ja) - vd_rbc = cp.asarray(vd_rbc) - - np.testing.assert_array_equal(vd_bf.get(), vd_rbc.get()) - - adj_rbc = csr_array( - ( - np.ones(adj_rbc_ia.get()[n_query_rows]), - adj_rbc_ja.get(), - adj_rbc_ia.get(), - ), - shape=(n_query_rows, n_index_rows), - ).toarray() - np.testing.assert_array_equal(adj_bf.get(), adj_rbc) diff --git a/python/pylibraft/pylibraft/test/test_fused_distance_argmin.py b/python/pylibraft/pylibraft/test/test_fused_distance_argmin.py deleted file mode 100755 index 6736128242..0000000000 --- a/python/pylibraft/pylibraft/test/test_fused_distance_argmin.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from scipy.spatial.distance import cdist - -from pylibraft.common import DeviceResources, device_ndarray -from pylibraft.distance import fused_distance_nn_argmin - - -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("n_rows", [10, 100]) -@pytest.mark.parametrize("n_clusters", [50, 100]) -@pytest.mark.parametrize("n_cols", [128, 31]) -@pytest.mark.parametrize("dtype", [np.float32]) -@pytest.mark.parametrize( - "metric", - [ - "euclidean", - "cosine", - "sqeuclidean", - ], -) -def test_fused_distance_nn_minarg( - n_rows, n_cols, n_clusters, dtype, inplace, metric -): - input1 = np.random.random_sample((n_rows, n_cols)) - input1 = np.asarray(input1, order="C").astype(dtype) - - input2 = np.random.random_sample((n_clusters, n_cols)) - input2 = np.asarray(input2, order="C").astype(dtype) - - output = np.zeros((n_rows), dtype="int32") - expected = cdist(input1, input2, metric) - - expected = expected.argmin(axis=1) - - input1_device = device_ndarray(input1) - input2_device = device_ndarray(input2) - output_device = device_ndarray(output) if inplace else None - - is_sqrt = True if metric == "sqeuclidean" else False - handle = DeviceResources() - ret_output = fused_distance_nn_argmin( - input1_device, - input2_device, - output_device, - is_sqrt, - metric, - handle=handle, - ) - handle.sync() - output_device = ret_output if not inplace else output_device - actual = output_device.copy_to_host() - - assert np.allclose(expected, actual, rtol=1e-4) diff --git a/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py b/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py deleted file mode 100644 index 086bb26f17..0000000000 --- a/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from scipy.spatial.distance import cdist - -from pylibraft.common import DeviceResources, device_ndarray -from pylibraft.distance import fused_l2_nn_argmin - - -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("n_rows", [10, 100]) -@pytest.mark.parametrize("n_clusters", [5, 10]) -@pytest.mark.parametrize("n_cols", [3, 5]) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_fused_l2_nn_minarg(n_rows, n_cols, n_clusters, dtype, inplace): - input1 = np.random.random_sample((n_rows, n_cols)) - input1 = np.asarray(input1, order="C").astype(dtype) - - input2 = np.random.random_sample((n_clusters, n_cols)) - input2 = np.asarray(input2, order="C").astype(dtype) - - output = np.zeros((n_rows), dtype="int32") - expected = cdist(input1, input2, metric="euclidean") - - expected = expected.argmin(axis=1) - - input1_device = device_ndarray(input1) - input2_device = device_ndarray(input2) - output_device = device_ndarray(output) if inplace else None - - handle = DeviceResources() - ret_output = fused_l2_nn_argmin( - input1_device, input2_device, output_device, True, handle=handle - ) - handle.sync() - output_device = ret_output if not inplace else output_device - actual = output_device.copy_to_host() - - assert np.allclose(expected, actual, rtol=1e-4) diff --git a/python/pylibraft/pylibraft/test/test_handle.py b/python/pylibraft/pylibraft/test/test_handle.py index bb07df1000..d77a6820f1 100644 --- a/python/pylibraft/pylibraft/test/test_handle.py +++ b/python/pylibraft/pylibraft/test/test_handle.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,27 +17,37 @@ import pytest from pylibraft.common import DeviceResources, Stream, device_ndarray -from pylibraft.distance import pairwise_distance +from pylibraft.random import rmat cupy = pytest.importorskip("cupy") -@pytest.mark.parametrize("stream", [cupy.cuda.Stream().ptr, Stream()]) -def test_handle_external_stream(stream): +def generate_theta(r_scale, c_scale): + max_scale = max(r_scale, c_scale) + theta = np.random.random_sample(max_scale * 4) + for i in range(max_scale): + a = theta[4 * i] + b = theta[4 * i + 1] + c = theta[4 * i + 2] + d = theta[4 * i + 3] + total = a + b + c + d + theta[4 * i] = a / total + theta[4 * i + 1] = b / total + theta[4 * i + 2] = c / total + theta[4 * i + 3] = d / total + theta_device = device_ndarray(theta) + return theta, theta_device - input1 = np.random.random_sample((50, 3)) - input1 = np.asarray(input1, order="F").astype("float") - output = np.zeros((50, 50), dtype="float") +@pytest.mark.parametrize("stream", [cupy.cuda.Stream().ptr, Stream()]) +def test_handle_external_stream(stream): - input1_device = device_ndarray(input1) - output_device = device_ndarray(output) + theta, theta_device = generate_theta(16, 16) + out_buff = np.empty((1000, 2), dtype=np.int32) + output_device = device_ndarray(out_buff) - # We are just testing that this doesn't segfault - handle = DeviceResources(stream) - pairwise_distance( - input1_device, input1_device, output_device, "euclidean", handle=handle - ) + handle = DeviceResources() + rmat(output_device, theta_device, 16, 16, 12345, handle=handle) handle.sync() with pytest.raises(ValueError): diff --git a/python/pylibraft/pylibraft/test/test_hnsw.py b/python/pylibraft/pylibraft/test/test_hnsw.py deleted file mode 100644 index 8cdf8c904f..0000000000 --- a/python/pylibraft/pylibraft/test/test_hnsw.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# h ttp://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from sklearn.neighbors import NearestNeighbors -from sklearn.preprocessing import normalize - -from pylibraft.neighbors import cagra, hnsw -from pylibraft.test.ann_utils import calc_recall, generate_data - - -def run_hnsw_build_search_test( - n_rows=10000, - n_cols=10, - n_queries=100, - k=10, - dtype=np.float32, - metric="sqeuclidean", - build_algo="ivf_pq", - intermediate_graph_degree=128, - graph_degree=64, - search_params={}, -): - dataset = generate_data((n_rows, n_cols), dtype) - if metric == "inner_product": - dataset = normalize(dataset, norm="l2", axis=1) - if dtype in [np.int8, np.uint8]: - pytest.skip( - "inner_product metric is not supported for int8/uint8 data" - ) - if build_algo == "nn_descent": - pytest.skip("inner_product metric is not supported for nn_descent") - - build_params = cagra.IndexParams( - metric=metric, - intermediate_graph_degree=intermediate_graph_degree, - graph_degree=graph_degree, - build_algo=build_algo, - ) - - index = cagra.build(build_params, dataset) - - assert index.trained - - hnsw_index = hnsw.from_cagra(index) - - queries = generate_data((n_queries, n_cols), dtype) - out_idx = np.zeros((n_queries, k), dtype=np.uint32) - - search_params = hnsw.SearchParams(**search_params) - - out_dist, out_idx = hnsw.search(search_params, hnsw_index, queries, k) - - # Calculate reference values with sklearn - skl_metric = { - "sqeuclidean": "sqeuclidean", - "inner_product": "cosine", - "euclidean": "euclidean", - }[metric] - nn_skl = NearestNeighbors( - n_neighbors=k, algorithm="brute", metric=skl_metric - ) - nn_skl.fit(dataset) - skl_idx = nn_skl.kneighbors(queries, return_distance=False) - - recall = calc_recall(out_idx, skl_idx) - assert recall > 0.95 - - -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.uint8]) -@pytest.mark.parametrize("k", [10, 20]) -@pytest.mark.parametrize("ef", [30, 40]) -@pytest.mark.parametrize("num_threads", [2, 4]) -@pytest.mark.parametrize("metric", ["sqeuclidean", "inner_product"]) -@pytest.mark.parametrize("build_algo", ["ivf_pq", "nn_descent"]) -def test_hnsw(dtype, k, ef, num_threads, metric, build_algo): - # Note that inner_product tests use normalized input which we cannot - # represent in int8, therefore we test only sqeuclidean metric here. - run_hnsw_build_search_test( - dtype=dtype, - k=k, - metric=metric, - build_algo=build_algo, - search_params={"ef": ef, "num_threads": num_threads}, - ) diff --git a/python/pylibraft/pylibraft/test/test_ivf_flat.py b/python/pylibraft/pylibraft/test/test_ivf_flat.py deleted file mode 100644 index 2e38dab7bc..0000000000 --- a/python/pylibraft/pylibraft/test/test_ivf_flat.py +++ /dev/null @@ -1,518 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# h ttp://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from sklearn.metrics import pairwise_distances -from sklearn.neighbors import NearestNeighbors -from sklearn.preprocessing import normalize - -from pylibraft.common import device_ndarray -from pylibraft.neighbors import ivf_flat - - -def generate_data(shape, dtype): - if dtype == np.byte: - x = np.random.randint(-127, 128, size=shape, dtype=np.byte) - elif dtype == np.ubyte: - x = np.random.randint(0, 255, size=shape, dtype=np.ubyte) - else: - x = np.random.random_sample(shape).astype(dtype) - - return x - - -def calc_recall(ann_idx, true_nn_idx): - assert ann_idx.shape == true_nn_idx.shape - n = 0 - for i in range(ann_idx.shape[0]): - n += np.intersect1d(ann_idx[i, :], true_nn_idx[i, :]).size - recall = n / ann_idx.size - return recall - - -def check_distances(dataset, queries, metric, out_idx, out_dist, eps=None): - """ - Calculate the real distance between queries and dataset[out_idx], - and compare it to out_dist. - """ - if eps is None: - # Quantization leads to errors in the distance calculation. - # The aim of this test is not to test precision, but to catch obvious - # errors. - eps = 0.1 - - dist = np.empty(out_dist.shape, out_dist.dtype) - for i in range(queries.shape[0]): - X = queries[np.newaxis, i, :] - Y = dataset[out_idx[i, :], :] - if metric == "sqeuclidean": - dist[i, :] = pairwise_distances(X, Y, "sqeuclidean") - elif metric == "euclidean": - dist[i, :] = pairwise_distances(X, Y, "euclidean") - elif metric == "inner_product": - dist[i, :] = np.matmul(X, Y.T) - else: - raise ValueError("Invalid metric") - - dist_eps = abs(dist) - dist_eps[dist < 1e-3] = 1e-3 - diff = abs(out_dist - dist) / dist_eps - - assert np.mean(diff) < eps - - -def run_ivf_flat_build_search_test( - n_rows, - n_cols, - n_queries, - k, - n_lists, - metric, - dtype, - add_data_on_build=True, - n_probes=100, - kmeans_trainset_fraction=1, - kmeans_n_iters=20, - compare=True, - inplace=True, - array_type="device", -): - dataset = generate_data((n_rows, n_cols), dtype) - if metric == "inner_product": - dataset = normalize(dataset, norm="l2", axis=1) - dataset_device = device_ndarray(dataset) - - build_params = ivf_flat.IndexParams( - n_lists=n_lists, - metric=metric, - kmeans_n_iters=kmeans_n_iters, - kmeans_trainset_fraction=kmeans_trainset_fraction, - add_data_on_build=add_data_on_build, - ) - - if array_type == "device": - index = ivf_flat.build(build_params, dataset_device) - else: - index = ivf_flat.build(build_params, dataset) - - assert index.trained - - assert index.metric == build_params.metric - assert index.n_lists == build_params.n_lists - - if not add_data_on_build: - dataset_1 = dataset[: n_rows // 2, :] - dataset_2 = dataset[n_rows // 2 :, :] - indices_1 = np.arange(n_rows // 2, dtype=np.int64) - indices_2 = np.arange(n_rows // 2, n_rows, dtype=np.int64) - if array_type == "device": - dataset_1_device = device_ndarray(dataset_1) - dataset_2_device = device_ndarray(dataset_2) - indices_1_device = device_ndarray(indices_1) - indices_2_device = device_ndarray(indices_2) - index = ivf_flat.extend(index, dataset_1_device, indices_1_device) - index = ivf_flat.extend(index, dataset_2_device, indices_2_device) - else: - index = ivf_flat.extend(index, dataset_1, indices_1) - index = ivf_flat.extend(index, dataset_2, indices_2) - - assert index.size >= n_rows - - queries = generate_data((n_queries, n_cols), dtype) - out_idx = np.zeros((n_queries, k), dtype=np.int64) - out_dist = np.zeros((n_queries, k), dtype=np.float32) - - queries_device = device_ndarray(queries) - out_idx_device = device_ndarray(out_idx) if inplace else None - out_dist_device = device_ndarray(out_dist) if inplace else None - - search_params = ivf_flat.SearchParams(n_probes=n_probes) - - ret_output = ivf_flat.search( - search_params, - index, - queries_device, - k, - neighbors=out_idx_device, - distances=out_dist_device, - ) - - if not inplace: - out_dist_device, out_idx_device = ret_output - - if not compare: - return - - out_idx = out_idx_device.copy_to_host() - out_dist = out_dist_device.copy_to_host() - - # Calculate reference values with sklearn - skl_metric = { - "sqeuclidean": "sqeuclidean", - "inner_product": "cosine", - "euclidean": "euclidean", - }[metric] - nn_skl = NearestNeighbors( - n_neighbors=k, algorithm="brute", metric=skl_metric - ) - nn_skl.fit(dataset) - skl_idx = nn_skl.kneighbors(queries, return_distance=False) - - recall = calc_recall(out_idx, skl_idx) - assert recall > 0.7 - - check_distances(dataset, queries, metric, out_idx, out_dist) - - -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("n_rows", [10000]) -@pytest.mark.parametrize("n_cols", [10]) -@pytest.mark.parametrize("n_queries", [100]) -@pytest.mark.parametrize("n_lists", [100]) -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.uint8]) -@pytest.mark.parametrize("array_type", ["device"]) -def test_ivf_pq_dtypes( - n_rows, n_cols, n_queries, n_lists, dtype, inplace, array_type -): - # Note that inner_product tests use normalized input which we cannot - # represent in int8, therefore we test only sqeuclidean metric here. - run_ivf_flat_build_search_test( - n_rows=n_rows, - n_cols=n_cols, - n_queries=n_queries, - k=10, - n_lists=n_lists, - metric="sqeuclidean", - dtype=dtype, - inplace=inplace, - array_type=array_type, - ) - - -@pytest.mark.parametrize( - "params", - [ - pytest.param( - { - "n_rows": 0, - "n_cols": 10, - "n_queries": 10, - "k": 1, - "n_lists": 10, - }, - marks=pytest.mark.xfail(reason="empty dataset"), - ), - {"n_rows": 1, "n_cols": 10, "n_queries": 10, "k": 1, "n_lists": 1}, - {"n_rows": 10, "n_cols": 1, "n_queries": 10, "k": 10, "n_lists": 10}, - # {"n_rows": 999, "n_cols": 42, "n_queries": 453, "k": 137, - # "n_lists": 53}, - ], -) -def test_ivf_flat_n(params): - # We do not test recall, just confirm that we can handle edge cases for - # certain parameters - run_ivf_flat_build_search_test( - n_rows=params["n_rows"], - n_cols=params["n_cols"], - n_queries=params["n_queries"], - k=params["k"], - n_lists=params["n_lists"], - metric="sqeuclidean", - dtype=np.float32, - compare=False, - ) - - -@pytest.mark.parametrize( - "metric", ["sqeuclidean", "inner_product", "euclidean"] -) -@pytest.mark.parametrize("dtype", [np.float32]) -def test_ivf_flat_build_params(metric, dtype): - run_ivf_flat_build_search_test( - n_rows=10000, - n_cols=10, - n_queries=1000, - k=10, - n_lists=100, - metric=metric, - dtype=dtype, - add_data_on_build=True, - n_probes=100, - ) - - -@pytest.mark.parametrize( - "params", - [ - { - "n_lists": 100, - "trainset_fraction": 0.9, - "n_iters": 30, - }, - ], -) -def test_ivf_flat_params(params): - run_ivf_flat_build_search_test( - n_rows=10000, - n_cols=16, - n_queries=1000, - k=10, - n_lists=params["n_lists"], - metric="sqeuclidean", - dtype=np.float32, - kmeans_trainset_fraction=params.get("trainset_fraction", 1.0), - kmeans_n_iters=params.get("n_iters", 20), - ) - - -@pytest.mark.parametrize( - "params", - [ - { - "k": 10, - "n_probes": 100, - }, - { - "k": 10, - "n_probes": 99, - }, - { - "k": 10, - "n_probes": 100, - }, - { - "k": 129, - "n_probes": 100, - }, - { - "k": 257, - "n_probes": 100, - }, - { - "k": 4096, - "n_probes": 100, - }, - ], -) -def test_ivf_pq_search_params(params): - run_ivf_flat_build_search_test( - n_rows=10000, - n_cols=16, - n_queries=1000, - k=params["k"], - n_lists=100, - n_probes=params["n_probes"], - metric="sqeuclidean", - dtype=np.float32, - ) - - -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.uint8]) -@pytest.mark.parametrize("array_type", ["device"]) -def test_extend(dtype, array_type): - run_ivf_flat_build_search_test( - n_rows=10000, - n_cols=10, - n_queries=100, - k=10, - n_lists=100, - metric="sqeuclidean", - dtype=dtype, - add_data_on_build=False, - array_type=array_type, - ) - - -def test_build_assertions(): - with pytest.raises(TypeError): - run_ivf_flat_build_search_test( - n_rows=1000, - n_cols=10, - n_queries=100, - k=10, - n_lists=100, - metric="sqeuclidean", - dtype=np.float64, - ) - - n_rows = 1000 - n_cols = 100 - n_queries = 212 - k = 10 - dataset = generate_data((n_rows, n_cols), np.float32) - dataset_device = device_ndarray(dataset) - - index_params = ivf_flat.IndexParams( - n_lists=50, - metric="sqeuclidean", - kmeans_n_iters=20, - kmeans_trainset_fraction=1, - add_data_on_build=False, - ) - - index = ivf_flat.Index() - - queries = generate_data((n_queries, n_cols), np.float32) - out_idx = np.zeros((n_queries, k), dtype=np.int64) - out_dist = np.zeros((n_queries, k), dtype=np.float32) - - queries_device = device_ndarray(queries) - out_idx_device = device_ndarray(out_idx) - out_dist_device = device_ndarray(out_dist) - - search_params = ivf_flat.SearchParams(n_probes=50) - - with pytest.raises(ValueError): - # Index must be built before search - ivf_flat.search( - search_params, - index, - queries_device, - k, - out_idx_device, - out_dist_device, - ) - - index = ivf_flat.build(index_params, dataset_device) - assert index.trained - - indices = np.arange(n_rows + 1, dtype=np.int64) - indices_device = device_ndarray(indices) - - with pytest.raises(ValueError): - # Dataset dimension mismatch - ivf_flat.extend(index, queries_device, indices_device) - - with pytest.raises(ValueError): - # indices dimension mismatch - ivf_flat.extend(index, dataset_device, indices_device) - - -@pytest.mark.parametrize( - "params", - [ - {"q_dt": np.float64}, - {"q_order": "F"}, - {"q_cols": 101}, - {"idx_dt": np.uint32}, - {"idx_order": "F"}, - {"idx_rows": 42}, - {"idx_cols": 137}, - {"dist_dt": np.float64}, - {"dist_order": "F"}, - {"dist_rows": 42}, - {"dist_cols": 137}, - ], -) -def test_search_inputs(params): - """Test with invalid input dtype, order, or dimension.""" - n_rows = 1000 - n_cols = 100 - n_queries = 256 - k = 10 - dtype = np.float32 - - q_dt = params.get("q_dt", np.float32) - q_order = params.get("q_order", "C") - queries = generate_data( - (n_queries, params.get("q_cols", n_cols)), q_dt - ).astype(q_dt, order=q_order) - queries_device = device_ndarray(queries) - - idx_dt = params.get("idx_dt", np.int64) - idx_order = params.get("idx_order", "C") - out_idx = np.zeros( - (params.get("idx_rows", n_queries), params.get("idx_cols", k)), - dtype=idx_dt, - order=idx_order, - ) - out_idx_device = device_ndarray(out_idx) - - dist_dt = params.get("dist_dt", np.float32) - dist_order = params.get("dist_order", "C") - out_dist = np.zeros( - (params.get("dist_rows", n_queries), params.get("dist_cols", k)), - dtype=dist_dt, - order=dist_order, - ) - out_dist_device = device_ndarray(out_dist) - - index_params = ivf_flat.IndexParams( - n_lists=50, metric="sqeuclidean", add_data_on_build=True - ) - - dataset = generate_data((n_rows, n_cols), dtype) - dataset_device = device_ndarray(dataset) - index = ivf_flat.build(index_params, dataset_device) - assert index.trained - - with pytest.raises(Exception): - search_params = ivf_flat.SearchParams(n_probes=50) - ivf_flat.search( - search_params, - index, - queries_device, - k, - out_idx_device, - out_dist_device, - ) - - -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.ubyte]) -def test_save_load(dtype): - n_rows = 10000 - n_cols = 50 - n_queries = 1000 - - dataset = generate_data((n_rows, n_cols), dtype) - dataset_device = device_ndarray(dataset) - - build_params = ivf_flat.IndexParams(n_lists=100, metric="sqeuclidean") - index = ivf_flat.build(build_params, dataset_device) - - assert index.trained - filename = "my_index.bin" - ivf_flat.save(filename, index) - loaded_index = ivf_flat.load(filename) - - assert index.metric == loaded_index.metric - assert index.n_lists == loaded_index.n_lists - assert index.dim == loaded_index.dim - assert index.adaptive_centers == loaded_index.adaptive_centers - - queries = generate_data((n_queries, n_cols), dtype) - - queries_device = device_ndarray(queries) - search_params = ivf_flat.SearchParams(n_probes=100) - k = 10 - - distance_dev, neighbors_dev = ivf_flat.search( - search_params, index, queries_device, k - ) - - neighbors = neighbors_dev.copy_to_host() - dist = distance_dev.copy_to_host() - del index - - distance_dev, neighbors_dev = ivf_flat.search( - search_params, loaded_index, queries_device, k - ) - - neighbors2 = neighbors_dev.copy_to_host() - dist2 = distance_dev.copy_to_host() - - assert np.all(neighbors == neighbors2) - assert np.allclose(dist, dist2, rtol=1e-6) diff --git a/python/pylibraft/pylibraft/test/test_ivf_pq.py b/python/pylibraft/pylibraft/test/test_ivf_pq.py deleted file mode 100644 index aa58e2a8fc..0000000000 --- a/python/pylibraft/pylibraft/test/test_ivf_pq.py +++ /dev/null @@ -1,550 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# h ttp://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from sklearn.metrics import pairwise_distances -from sklearn.neighbors import NearestNeighbors -from sklearn.preprocessing import normalize - -from pylibraft.common import device_ndarray -from pylibraft.neighbors import ivf_pq - - -def generate_data(shape, dtype): - if dtype == np.byte: - x = np.random.randint(-127, 128, size=shape, dtype=np.byte) - elif dtype == np.ubyte: - x = np.random.randint(0, 255, size=shape, dtype=np.ubyte) - else: - x = np.random.random_sample(shape).astype(dtype) - - return x - - -def calc_recall(ann_idx, true_nn_idx): - assert ann_idx.shape == true_nn_idx.shape - n = 0 - for i in range(ann_idx.shape[0]): - n += np.intersect1d(ann_idx[i, :], true_nn_idx[i, :]).size - recall = n / ann_idx.size - return recall - - -def check_distances(dataset, queries, metric, out_idx, out_dist, eps=None): - """ - Calculate the real distance between queries and dataset[out_idx], - and compare it to out_dist. - """ - if eps is None: - # Quantization leads to errors in the distance calculation. - # The aim of this test is not to test precision, but to catch obvious - # errors. - eps = 0.1 - - dist = np.empty(out_dist.shape, out_dist.dtype) - for i in range(queries.shape[0]): - X = queries[np.newaxis, i, :] - Y = dataset[out_idx[i, :], :] - if metric == "sqeuclidean": - dist[i, :] = pairwise_distances(X, Y, "sqeuclidean") - elif metric == "euclidean": - dist[i, :] = pairwise_distances(X, Y, "euclidean") - elif metric == "inner_product": - dist[i, :] = np.matmul(X, Y.T) - else: - raise ValueError("Invalid metric") - - dist_eps = abs(dist) - dist_eps[dist < 1e-3] = 1e-3 - diff = abs(out_dist - dist) / dist_eps - - assert np.mean(diff) < eps - - -def run_ivf_pq_build_search_test( - n_rows, - n_cols, - n_queries, - k, - n_lists, - metric, - dtype, - pq_bits=8, - pq_dim=0, - codebook_kind="subspace", - add_data_on_build="True", - n_probes=100, - lut_dtype=np.float32, - internal_distance_dtype=np.float32, - force_random_rotation=False, - kmeans_trainset_fraction=1, - kmeans_n_iters=20, - compare=True, - inplace=True, - array_type="device", -): - dataset = generate_data((n_rows, n_cols), dtype) - if metric == "inner_product": - dataset = normalize(dataset, norm="l2", axis=1) - dataset_device = device_ndarray(dataset) - - build_params = ivf_pq.IndexParams( - n_lists=n_lists, - metric=metric, - kmeans_n_iters=kmeans_n_iters, - kmeans_trainset_fraction=kmeans_trainset_fraction, - pq_bits=pq_bits, - pq_dim=pq_dim, - codebook_kind=codebook_kind, - force_random_rotation=force_random_rotation, - add_data_on_build=add_data_on_build, - ) - - if array_type == "device": - index = ivf_pq.build(build_params, dataset_device) - else: - index = ivf_pq.build(build_params, dataset) - - assert index.trained - if pq_dim != 0: - assert index.pq_dim == build_params.pq_dim - assert index.pq_bits == build_params.pq_bits - assert index.metric == build_params.metric - assert index.n_lists == build_params.n_lists - - if not add_data_on_build: - dataset_1 = dataset[: n_rows // 2, :] - dataset_2 = dataset[n_rows // 2 :, :] - indices_1 = np.arange(n_rows // 2, dtype=np.int64) - indices_2 = np.arange(n_rows // 2, n_rows, dtype=np.int64) - if array_type == "device": - dataset_1_device = device_ndarray(dataset_1) - dataset_2_device = device_ndarray(dataset_2) - indices_1_device = device_ndarray(indices_1) - indices_2_device = device_ndarray(indices_2) - index = ivf_pq.extend(index, dataset_1_device, indices_1_device) - index = ivf_pq.extend(index, dataset_2_device, indices_2_device) - else: - index = ivf_pq.extend(index, dataset_1, indices_1) - index = ivf_pq.extend(index, dataset_2, indices_2) - - assert index.size >= n_rows - - queries = generate_data((n_queries, n_cols), dtype) - out_idx = np.zeros((n_queries, k), dtype=np.int64) - out_dist = np.zeros((n_queries, k), dtype=np.float32) - - queries_device = device_ndarray(queries) - out_idx_device = device_ndarray(out_idx) if inplace else None - out_dist_device = device_ndarray(out_dist) if inplace else None - - search_params = ivf_pq.SearchParams( - n_probes=n_probes, - lut_dtype=lut_dtype, - internal_distance_dtype=internal_distance_dtype, - ) - - ret_output = ivf_pq.search( - search_params, - index, - queries_device, - k, - neighbors=out_idx_device, - distances=out_dist_device, - ) - - if not inplace: - out_dist_device, out_idx_device = ret_output - - if not compare: - return - - out_idx = out_idx_device.copy_to_host() - out_dist = out_dist_device.copy_to_host() - - # Calculate reference values with sklearn - skl_metric = { - "sqeuclidean": "sqeuclidean", - "inner_product": "cosine", - "euclidean": "euclidean", - }[metric] - nn_skl = NearestNeighbors( - n_neighbors=k, algorithm="brute", metric=skl_metric - ) - nn_skl.fit(dataset) - skl_idx = nn_skl.kneighbors(queries, return_distance=False) - - recall = calc_recall(out_idx, skl_idx) - assert recall > 0.7 - - check_distances(dataset, queries, metric, out_idx, out_dist) - - -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("n_rows", [10000]) -@pytest.mark.parametrize("n_cols", [10]) -@pytest.mark.parametrize("n_queries", [100]) -@pytest.mark.parametrize("n_lists", [100]) -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.uint8]) -@pytest.mark.parametrize("array_type", ["host", "device"]) -def test_ivf_pq_dtypes( - n_rows, n_cols, n_queries, n_lists, dtype, inplace, array_type -): - # Note that inner_product tests use normalized input which we cannot - # represent in int8, therefore we test only sqeuclidean metric here. - run_ivf_pq_build_search_test( - n_rows=n_rows, - n_cols=n_cols, - n_queries=n_queries, - k=10, - n_lists=n_lists, - metric="sqeuclidean", - dtype=dtype, - inplace=inplace, - array_type=array_type, - ) - - -@pytest.mark.parametrize( - "params", - [ - pytest.param( - { - "n_rows": 0, - "n_cols": 10, - "n_queries": 10, - "k": 1, - "n_lists": 10, - }, - marks=pytest.mark.xfail(reason="empty dataset"), - ), - {"n_rows": 1, "n_cols": 10, "n_queries": 10, "k": 1, "n_lists": 1}, - {"n_rows": 10, "n_cols": 1, "n_queries": 10, "k": 10, "n_lists": 10}, - # {"n_rows": 999, "n_cols": 42, "n_queries": 453, "k": 137, - # "n_lists": 53}, - ], -) -def test_ivf_pq_n(params): - # We do not test recall, just confirm that we can handle edge cases for - # certain parameters - run_ivf_pq_build_search_test( - n_rows=params["n_rows"], - n_cols=params["n_cols"], - n_queries=params["n_queries"], - k=params["k"], - n_lists=params["n_lists"], - metric="sqeuclidean", - dtype=np.float32, - compare=False, - ) - - -@pytest.mark.parametrize( - "metric", ["sqeuclidean", "inner_product", "euclidean"] -) -@pytest.mark.parametrize("dtype", [np.float32]) -@pytest.mark.parametrize("codebook_kind", ["subspace", "cluster"]) -@pytest.mark.parametrize("rotation", [True, False]) -def test_ivf_pq_build_params(metric, dtype, codebook_kind, rotation): - run_ivf_pq_build_search_test( - n_rows=10000, - n_cols=10, - n_queries=1000, - k=10, - n_lists=100, - metric=metric, - dtype=dtype, - pq_bits=8, - pq_dim=0, - codebook_kind=codebook_kind, - add_data_on_build=True, - n_probes=100, - force_random_rotation=rotation, - ) - - -@pytest.mark.parametrize( - "params", - [ - {"pq_dims": 10, "pq_bits": 8, "n_lists": 100}, - {"pq_dims": 16, "pq_bits": 7, "n_lists": 100}, - {"pq_dims": 0, "pq_bits": 8, "n_lists": 90}, - { - "pq_dims": 0, - "pq_bits": 8, - "n_lists": 100, - "trainset_fraction": 0.9, - "n_iters": 30, - }, - ], -) -def test_ivf_pq_params(params): - run_ivf_pq_build_search_test( - n_rows=10000, - n_cols=16, - n_queries=1000, - k=10, - n_lists=params["n_lists"], - metric="sqeuclidean", - dtype=np.float32, - pq_bits=params["pq_bits"], - pq_dim=params["pq_dims"], - kmeans_trainset_fraction=params.get("trainset_fraction", 1.0), - kmeans_n_iters=params.get("n_iters", 20), - ) - - -@pytest.mark.parametrize( - "params", - [ - { - "k": 10, - "n_probes": 100, - "lut": np.float16, - "idd": np.float32, - }, - { - "k": 10, - "n_probes": 99, - "lut": np.uint8, - "idd": np.float32, - }, - { - "k": 10, - "n_probes": 100, - "lut": np.float16, - "idd": np.float16, - }, - { - "k": 129, - "n_probes": 100, - "lut": np.float32, - "idd": np.float32, - }, - ], -) -def test_ivf_pq_search_params(params): - run_ivf_pq_build_search_test( - n_rows=10000, - n_cols=16, - n_queries=1000, - k=params["k"], - n_lists=100, - n_probes=params["n_probes"], - metric="sqeuclidean", - dtype=np.float32, - lut_dtype=params["lut"], - internal_distance_dtype=params["idd"], - ) - - -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.uint8]) -@pytest.mark.parametrize("array_type", ["host", "device"]) -def test_extend(dtype, array_type): - run_ivf_pq_build_search_test( - n_rows=10000, - n_cols=10, - n_queries=100, - k=10, - n_lists=100, - metric="sqeuclidean", - dtype=dtype, - add_data_on_build=False, - array_type=array_type, - ) - - -def test_build_assertions(): - with pytest.raises(TypeError): - run_ivf_pq_build_search_test( - n_rows=1000, - n_cols=10, - n_queries=100, - k=10, - n_lists=100, - metric="sqeuclidean", - dtype=np.float64, - ) - - n_rows = 1000 - n_cols = 100 - n_queries = 212 - k = 10 - dataset = generate_data((n_rows, n_cols), np.float32) - dataset_device = device_ndarray(dataset) - - index_params = ivf_pq.IndexParams( - n_lists=50, - metric="sqeuclidean", - kmeans_n_iters=20, - kmeans_trainset_fraction=1, - add_data_on_build=False, - ) - - index = ivf_pq.Index() - - queries = generate_data((n_queries, n_cols), np.float32) - out_idx = np.zeros((n_queries, k), dtype=np.int64) - out_dist = np.zeros((n_queries, k), dtype=np.float32) - - queries_device = device_ndarray(queries) - out_idx_device = device_ndarray(out_idx) - out_dist_device = device_ndarray(out_dist) - - search_params = ivf_pq.SearchParams(n_probes=50) - - with pytest.raises(ValueError): - # Index must be built before search - ivf_pq.search( - search_params, - index, - queries_device, - k, - out_idx_device, - out_dist_device, - ) - - index = ivf_pq.build(index_params, dataset_device) - assert index.trained - - indices = np.arange(n_rows + 1, dtype=np.int64) - indices_device = device_ndarray(indices) - - with pytest.raises(ValueError): - # Dataset dimension mismatch - ivf_pq.extend(index, queries_device, indices_device) - - with pytest.raises(ValueError): - # indices dimension mismatch - ivf_pq.extend(index, dataset_device, indices_device) - - -@pytest.mark.parametrize( - "params", - [ - {"q_dt": np.float64}, - {"q_order": "F"}, - {"q_cols": 101}, - {"idx_dt": np.uint32}, - {"idx_order": "F"}, - {"idx_rows": 42}, - {"idx_cols": 137}, - {"dist_dt": np.float64}, - {"dist_order": "F"}, - {"dist_rows": 42}, - {"dist_cols": 137}, - ], -) -def test_search_inputs(params): - """Test with invalid input dtype, order, or dimension.""" - n_rows = 1000 - n_cols = 100 - n_queries = 256 - k = 10 - dtype = np.float32 - - q_dt = params.get("q_dt", np.float32) - q_order = params.get("q_order", "C") - queries = generate_data( - (n_queries, params.get("q_cols", n_cols)), q_dt - ).astype(q_dt, order=q_order) - queries_device = device_ndarray(queries) - - idx_dt = params.get("idx_dt", np.int64) - idx_order = params.get("idx_order", "C") - out_idx = np.zeros( - (params.get("idx_rows", n_queries), params.get("idx_cols", k)), - dtype=idx_dt, - order=idx_order, - ) - out_idx_device = device_ndarray(out_idx) - - dist_dt = params.get("dist_dt", np.float32) - dist_order = params.get("dist_order", "C") - out_dist = np.zeros( - (params.get("dist_rows", n_queries), params.get("dist_cols", k)), - dtype=dist_dt, - order=dist_order, - ) - out_dist_device = device_ndarray(out_dist) - - index_params = ivf_pq.IndexParams( - n_lists=50, metric="sqeuclidean", add_data_on_build=True - ) - - dataset = generate_data((n_rows, n_cols), dtype) - dataset_device = device_ndarray(dataset) - index = ivf_pq.build(index_params, dataset_device) - assert index.trained - - with pytest.raises(Exception): - search_params = ivf_pq.SearchParams(n_probes=50) - ivf_pq.search( - search_params, - index, - queries_device, - k, - out_idx_device, - out_dist_device, - ) - - -def test_save_load(): - n_rows = 10000 - n_cols = 50 - n_queries = 1000 - dtype = np.float32 - - dataset = generate_data((n_rows, n_cols), dtype) - dataset_device = device_ndarray(dataset) - - build_params = ivf_pq.IndexParams(n_lists=100, metric="sqeuclidean") - index = ivf_pq.build(build_params, dataset_device) - - assert index.trained - filename = "my_index.bin" - ivf_pq.save(filename, index) - loaded_index = ivf_pq.load(filename) - - assert index.pq_dim == loaded_index.pq_dim - assert index.pq_bits == loaded_index.pq_bits - assert index.metric == loaded_index.metric - assert index.n_lists == loaded_index.n_lists - assert index.size == loaded_index.size - - queries = generate_data((n_queries, n_cols), dtype) - - queries_device = device_ndarray(queries) - search_params = ivf_pq.SearchParams(n_probes=100) - k = 10 - - distance_dev, neighbors_dev = ivf_pq.search( - search_params, index, queries_device, k - ) - - neighbors = neighbors_dev.copy_to_host() - dist = distance_dev.copy_to_host() - del index - - distance_dev, neighbors_dev = ivf_pq.search( - search_params, loaded_index, queries_device, k - ) - - neighbors2 = neighbors_dev.copy_to_host() - dist2 = distance_dev.copy_to_host() - - assert np.all(neighbors == neighbors2) - assert np.allclose(dist, dist2, rtol=1e-6) diff --git a/python/pylibraft/pylibraft/test/test_kmeans.py b/python/pylibraft/pylibraft/test/test_kmeans.py deleted file mode 100644 index 8736c6ee7a..0000000000 --- a/python/pylibraft/pylibraft/test/test_kmeans.py +++ /dev/null @@ -1,206 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest - -from pylibraft.cluster.kmeans import ( - KMeansParams, - cluster_cost, - compute_new_centroids, - fit, - init_plus_plus, -) -from pylibraft.common import DeviceResources, device_ndarray -from pylibraft.distance import pairwise_distance - - -@pytest.mark.parametrize("n_rows", [100]) -@pytest.mark.parametrize("n_cols", [5, 25]) -@pytest.mark.parametrize("n_clusters", [5, 15]) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_kmeans_fit(n_rows, n_cols, n_clusters, dtype): - # generate some random input points / centroids - X_host = np.random.random_sample((n_rows, n_cols)).astype(dtype) - centroids = device_ndarray(X_host[:n_clusters]) - X = device_ndarray(X_host) - - # compute the inertia, before fitting centroids - original_inertia = cluster_cost(X, centroids) - - params = KMeansParams(n_clusters=n_clusters, seed=42) - - # fit the centroids, make sure inertia has gone down - # TODO: once we have make_blobs exposed to python - # (https://github.com/rapidsai/raft/issues/1059) - # we should use that to test out the kmeans fit, like the C++ - # tests do right now - centroids, inertia, n_iter = fit(params, X, centroids) - assert inertia < original_inertia - assert n_iter >= 1 - assert np.allclose(cluster_cost(X, centroids), inertia, rtol=1e-6) - - -@pytest.mark.parametrize("n_rows", [100]) -@pytest.mark.parametrize("n_cols", [5, 25]) -@pytest.mark.parametrize("n_clusters", [5, 15]) -@pytest.mark.parametrize("metric", ["euclidean", "sqeuclidean"]) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -@pytest.mark.parametrize("additional_args", [True, False]) -def test_compute_new_centroids( - n_rows, n_cols, metric, n_clusters, dtype, additional_args -): - - # A single RAFT handle can optionally be reused across - # pylibraft functions. - handle = DeviceResources() - - X = np.random.random_sample((n_rows, n_cols)).astype(dtype) - X_device = device_ndarray(X) - - centroids = X[:n_clusters] - centroids_device = device_ndarray(centroids) - - weight_per_cluster = np.zeros((n_clusters,), dtype=dtype) - weight_per_cluster_device = ( - device_ndarray(weight_per_cluster) if additional_args else None - ) - - new_centroids = np.zeros((n_clusters, n_cols), dtype=dtype) - new_centroids_device = device_ndarray(new_centroids) - - sample_weights = np.ones((n_rows,)).astype(dtype) / n_rows - sample_weights_device = ( - device_ndarray(sample_weights) if additional_args else None - ) - - # Compute new centroids naively - dists = np.zeros((n_rows, n_clusters), dtype=dtype) - dists_device = device_ndarray(dists) - pairwise_distance(X_device, centroids_device, dists_device, metric=metric) - handle.sync() - - labels = np.argmin(dists_device.copy_to_host(), axis=1).astype(np.int32) - labels_device = device_ndarray(labels) - - expected_centers = np.empty((n_clusters, n_cols), dtype=dtype) - expected_wX = X * sample_weights.reshape((-1, 1)) - for i in range(n_clusters): - j = expected_wX[labels == i] - j = j.sum(axis=0) - g = sample_weights[labels == i].sum() - expected_centers[i, :] = j / g - - compute_new_centroids( - X_device, - centroids_device, - labels_device, - new_centroids_device, - sample_weights=sample_weights_device, - weight_per_cluster=weight_per_cluster_device, - handle=handle, - ) - - # pylibraft functions are often asynchronous so the - # handle needs to be explicitly synchronized - handle.sync() - - actual_centers = new_centroids_device.copy_to_host() - - assert np.allclose(expected_centers, actual_centers, rtol=1e-6) - - -@pytest.mark.parametrize("n_rows", [100]) -@pytest.mark.parametrize("n_cols", [5, 25]) -@pytest.mark.parametrize("n_clusters", [4, 15]) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_cluster_cost(n_rows, n_cols, n_clusters, dtype): - X = np.random.random_sample((n_rows, n_cols)).astype(dtype) - X_device = device_ndarray(X) - - centroids = X[:n_clusters] - centroids_device = device_ndarray(centroids) - - inertia = cluster_cost(X_device, centroids_device) - - # compute the nearest centroid to each sample - distances = pairwise_distance( - X_device, centroids_device, metric="sqeuclidean" - ).copy_to_host() - cluster_ids = np.argmin(distances, axis=1) - - cluster_distances = np.take_along_axis( - distances, cluster_ids[:, None], axis=1 - ) - - # need reduced tolerance for float32 - tol = 1e-3 if dtype == np.float32 else 1e-6 - assert np.allclose(inertia, sum(cluster_distances), rtol=tol, atol=tol) - - -@pytest.mark.parametrize("n_rows", [100]) -@pytest.mark.parametrize("n_cols", [5, 25]) -@pytest.mark.parametrize("n_clusters", [4, 15]) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_init_plus_plus(n_rows, n_cols, n_clusters, dtype): - X = np.random.random_sample((n_rows, n_cols)).astype(dtype) - X_device = device_ndarray(X) - - centroids = init_plus_plus(X_device, n_clusters, seed=1) - centroids_ = centroids.copy_to_host() - - assert centroids_.shape == (n_clusters, X.shape[1]) - - # Centroids are selected from the existing points - for centroid in centroids_: - assert (centroid == X).all(axis=1).any() - - -@pytest.mark.parametrize("n_rows", [100]) -@pytest.mark.parametrize("n_cols", [5, 25]) -@pytest.mark.parametrize("n_clusters", [4, 15]) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_init_plus_plus_preallocated_output(n_rows, n_cols, n_clusters, dtype): - X = np.random.random_sample((n_rows, n_cols)).astype(dtype) - X_device = device_ndarray(X) - - centroids = device_ndarray.empty((n_clusters, n_cols), dtype=dtype) - - new_centroids = init_plus_plus(X_device, centroids=centroids, seed=1) - new_centroids_ = new_centroids.copy_to_host() - - # The shape should not have changed - assert new_centroids_.shape == centroids.shape - - # Centroids are selected from the existing points - for centroid in new_centroids_: - assert (centroid == X).all(axis=1).any() - - -def test_init_plus_plus_exclusive_arguments(): - # Check an exception is raised when n_clusters and centroids shape - # are inconsistent. - X = np.random.random_sample((10, 5)).astype(np.float64) - X = device_ndarray(X) - - n_clusters = 3 - - centroids = np.random.random_sample((n_clusters + 1, 5)).astype(np.float64) - centroids = device_ndarray(centroids) - - with pytest.raises( - RuntimeError, match="Parameters 'n_clusters' and 'centroids'" - ): - init_plus_plus(X, n_clusters, centroids=centroids) diff --git a/python/pylibraft/pylibraft/test/test_refine.py b/python/pylibraft/pylibraft/test/test_refine.py deleted file mode 100644 index 397ea70ec7..0000000000 --- a/python/pylibraft/pylibraft/test/test_refine.py +++ /dev/null @@ -1,233 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# h ttp://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from sklearn.neighbors import NearestNeighbors -from sklearn.preprocessing import normalize -from test_ivf_pq import calc_recall, check_distances, generate_data - -from pylibraft.common import device_ndarray -from pylibraft.neighbors import refine - - -def run_refine( - n_rows=500, - n_cols=50, - n_queries=100, - metric="sqeuclidean", - k0=40, - k=10, - inplace=False, - dtype=np.float32, - memory_type="device", -): - - dataset = generate_data((n_rows, n_cols), dtype) - queries = generate_data((n_queries, n_cols), dtype) - - if metric == "inner_product": - if dtype != np.float32: - pytest.skip("Normalized input cannot be represented in int8") - return - dataset = normalize(dataset, norm="l2", axis=1) - queries = normalize(queries, norm="l2", axis=1) - - dataset_device = device_ndarray(dataset) - queries_device = device_ndarray(queries) - - # Calculate reference values with sklearn - skl_metric = {"sqeuclidean": "euclidean", "inner_product": "cosine"}[ - metric - ] - nn_skl = NearestNeighbors( - n_neighbors=k0, algorithm="brute", metric=skl_metric - ) - nn_skl.fit(dataset) - skl_dist, candidates = nn_skl.kneighbors(queries) - candidates = candidates.astype(np.int64) - candidates_device = device_ndarray(candidates) - - out_idx = np.zeros((n_queries, k), dtype=np.int64) - out_dist = np.zeros((n_queries, k), dtype=np.float32) - out_idx_device = device_ndarray(out_idx) if inplace else None - out_dist_device = device_ndarray(out_dist) if inplace else None - - if memory_type == "device": - if inplace: - refine( - dataset_device, - queries_device, - candidates_device, - indices=out_idx_device, - distances=out_dist_device, - metric=metric, - ) - else: - out_dist_device, out_idx_device = refine( - dataset_device, - queries_device, - candidates_device, - k=k, - metric=metric, - ) - out_idx = out_idx_device.copy_to_host() - out_dist = out_dist_device.copy_to_host() - elif memory_type == "host": - if inplace: - refine( - dataset, - queries, - candidates, - indices=out_idx, - distances=out_dist, - metric=metric, - ) - else: - out_dist, out_idx = refine( - dataset, queries, candidates, k=k, metric=metric - ) - - skl_idx = candidates[:, :k] - - recall = calc_recall(out_idx, skl_idx) - if recall <= 0.999: - # We did not find the same neighbor indices. - # We could have found other neighbor with same distance. - if metric == "sqeuclidean": - skl_dist = np.power(skl_dist[:, :k], 2) - elif metric == "inner_product": - skl_dist = 1 - skl_dist[:, :k] - else: - raise ValueError("Invalid metric") - mask = out_idx != skl_idx - assert np.all(out_dist[mask] <= skl_dist[mask] + 1.0e-6) - - check_distances(dataset, queries, metric, out_idx, out_dist, 0.001) - - -@pytest.mark.parametrize("n_queries", [100, 1024, 37]) -@pytest.mark.parametrize("inplace", [True, False]) -@pytest.mark.parametrize("metric", ["sqeuclidean", "inner_product"]) -@pytest.mark.parametrize("dtype", [np.float32, np.int8, np.uint8]) -@pytest.mark.parametrize("memory_type", ["device", "host"]) -def test_refine_dtypes(n_queries, dtype, inplace, metric, memory_type): - run_refine( - n_rows=2000, - n_queries=n_queries, - n_cols=50, - k0=40, - k=10, - dtype=dtype, - inplace=inplace, - metric=metric, - memory_type=memory_type, - ) - - -@pytest.mark.parametrize( - "params", - [ - pytest.param( - { - "n_rows": 0, - "n_cols": 10, - "n_queries": 10, - "k0": 10, - "k": 1, - }, - marks=pytest.mark.xfail(reason="empty dataset"), - ), - {"n_rows": 1, "n_cols": 10, "n_queries": 10, "k": 1, "k0": 1}, - {"n_rows": 10, "n_cols": 1, "n_queries": 10, "k": 10, "k0": 10}, - {"n_rows": 999, "n_cols": 42, "n_queries": 453, "k0": 137, "k": 53}, - ], -) -@pytest.mark.parametrize("memory_type", ["device", "host"]) -def test_refine_row_col(params, memory_type): - run_refine( - n_rows=params["n_rows"], - n_queries=params["n_queries"], - n_cols=params["n_cols"], - k0=params["k0"], - k=params["k"], - memory_type=memory_type, - ) - - -@pytest.mark.parametrize("memory_type", ["device", "host"]) -def test_input_dtype(memory_type): - with pytest.raises(Exception): - run_refine(dtype=np.float64, memory_type=memory_type) - - -@pytest.mark.parametrize( - "params", - [ - {"idx_shape": None, "dist_shape": None, "k": None}, - {"idx_shape": [100, 9], "dist_shape": None, "k": 10}, - {"idx_shape": [101, 10], "dist_shape": None, "k": None}, - {"idx_shape": None, "dist_shape": [100, 11], "k": 10}, - {"idx_shape": None, "dist_shape": [99, 10], "k": None}, - ], -) -@pytest.mark.parametrize("memory_type", ["device", "host"]) -def test_input_assertions(params, memory_type): - n_cols = 5 - n_queries = 100 - k0 = 40 - dtype = np.float32 - dataset = generate_data((500, n_cols), dtype) - dataset_device = device_ndarray(dataset) - - queries = generate_data((n_queries, n_cols), dtype) - queries_device = device_ndarray(queries) - - candidates = np.random.randint( - 0, 500, size=(n_queries, k0), dtype=np.int64 - ) - candidates_device = device_ndarray(candidates) - - if params["idx_shape"] is not None: - out_idx = np.zeros(params["idx_shape"], dtype=np.int64) - out_idx_device = device_ndarray(out_idx) - else: - out_idx_device = None - if params["dist_shape"] is not None: - out_dist = np.zeros(params["dist_shape"], dtype=np.float32) - out_dist_device = device_ndarray(out_dist) - else: - out_dist_device = None - - if memory_type == "device": - with pytest.raises(Exception): - distances, indices = refine( - dataset_device, - queries_device, - candidates_device, - k=params["k"], - indices=out_idx_device, - distances=out_dist_device, - ) - else: - with pytest.raises(Exception): - distances, indices = refine( - dataset, - queries, - candidates, - k=params["k"], - indices=out_idx, - distances=out_dist, - ) diff --git a/python/pylibraft/pylibraft/test/test_select_k.py b/python/pylibraft/pylibraft/test/test_select_k.py deleted file mode 100644 index 203e735b9c..0000000000 --- a/python/pylibraft/pylibraft/test/test_select_k.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest - -from pylibraft.common import device_ndarray -from pylibraft.matrix import select_k - - -@pytest.mark.parametrize("n_rows", [32, 100]) -@pytest.mark.parametrize("n_cols", [40, 100]) -@pytest.mark.parametrize("k", [1, 5, 16, 35]) -@pytest.mark.parametrize("inplace", [True, False]) -def test_select_k(n_rows, n_cols, k, inplace): - dataset = np.random.random_sample((n_rows, n_cols)).astype("float32") - dataset_device = device_ndarray(dataset) - - indices = np.zeros((n_rows, k), dtype="int64") - distances = np.zeros((n_rows, k), dtype="float32") - indices_device = device_ndarray(indices) - distances_device = device_ndarray(distances) - - ret_distances, ret_indices = select_k( - dataset_device, - k=k, - distances=distances_device, - indices=indices_device, - ) - - distances_device = ret_distances if not inplace else distances_device - actual_distances = distances_device.copy_to_host() - argsort = np.argsort(dataset, axis=1) - - for i in range(dataset.shape[0]): - expected_indices = argsort[i] - gpu_dists = actual_distances[i] - - cpu_ordered = dataset[i, expected_indices] - np.testing.assert_allclose( - cpu_ordered[:k], gpu_dists, atol=1e-4, rtol=1e-4 - ) diff --git a/python/pylibraft/pylibraft/test/test_sparse.py b/python/pylibraft/pylibraft/test/test_sparse.py new file mode 100644 index 0000000000..10b261d322 --- /dev/null +++ b/python/pylibraft/pylibraft/test/test_sparse.py @@ -0,0 +1,142 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import cupy +import cupyx.scipy.sparse.linalg # NOQA +import numpy +import pytest +from cupyx.scipy import sparse + +from pylibraft.sparse.linalg import eigsh + + +def shaped_random( + shape, xp=cupy, dtype=numpy.float32, scale=10, seed=0, order="C" +): + """ + Returns an array filled with random values. + + Args + ---- + shape(tuple): Shape of returned ndarray. + xp(numpy or cupy): Array module to use. + dtype(dtype): Dtype of returned ndarray. + scale(float): Scaling factor of elements. + seed(int): Random seed. + + Returns + ------- + numpy.ndarray or cupy.ndarray: The array with + given shape, array module, + + If ``dtype`` is ``numpy.bool_``, the elements are + independently drawn from ``True`` and ``False`` + with same probabilities. + Otherwise, the array is filled with samples + independently and identically drawn + from uniform distribution over :math:`[0, scale)` + with specified dtype. + """ + numpy.random.seed(seed) + dtype = numpy.dtype(dtype) + if dtype == "?": + a = numpy.random.randint(2, size=shape) + elif dtype.kind == "c": + a = numpy.random.rand(*shape) + 1j * numpy.random.rand(*shape) + a *= scale + else: + a = numpy.random.rand(*shape) * scale + return xp.asarray(a, dtype=dtype, order=order) + + +class TestEigsh: + n = 30 + density = 0.33 + tol = {numpy.float32: 1e-5, numpy.complex64: 1e-5, "default": 1e-12} + res_tol = {"f": 1e-5, "d": 1e-12} + return_eigenvectors = True + + def _make_matrix(self, dtype, xp): + shape = (self.n, self.n) + a = shaped_random(shape, xp, dtype=dtype) + mask = shaped_random(shape, xp, dtype="f", scale=1) + a[mask > self.density] = 0 + a = a * a.conj().T + return a + + def _test_eigsh(self, a, k, xp, sp): + expected_ret = sp.linalg.eigsh( + a, k=k, return_eigenvectors=self.return_eigenvectors + ) + actual_ret = eigsh(a, k=k) + if self.return_eigenvectors: + w, x = actual_ret + exp_w, _ = expected_ret + # Check the residuals to see if eigenvectors are correct. + ax_xw = a @ x - xp.multiply(x, w.reshape(1, k)) + res = xp.linalg.norm(ax_xw) / xp.linalg.norm(w) + tol = self.res_tol[numpy.dtype(a.dtype).char.lower()] + assert res < tol + else: + w = actual_ret + exp_w = expected_ret + w = xp.sort(w) + cupy.allclose(w, exp_w, rtol=tol, atol=tol) + + @pytest.mark.parametrize("format", ["csr"]) # , 'csc', 'coo']) + @pytest.mark.parametrize("k", [3, 6, 12]) + @pytest.mark.parametrize("dtype", ["f", "d"]) + def test_sparse(self, format, k, dtype, xp=cupy, sp=sparse): + if format == "csc": + pytest.xfail("may be buggy") # trans=True + + a = self._make_matrix(dtype, xp) + a = sp.coo_matrix(a).asformat(format) + return self._test_eigsh(a, k, xp, sp) + + def test_invalid(self): + xp, sp = cupy, sparse + a = xp.diag(xp.ones((self.n,), dtype="f")) + with pytest.raises(ValueError): + sp.linalg.eigsh(xp.ones((2, 1), dtype="f")) + with pytest.raises(ValueError): + sp.linalg.eigsh(a, k=0) + a = xp.diag(xp.ones((self.n,), dtype="f")) + with pytest.raises(ValueError): + sp.linalg.eigsh(xp.ones((1,), dtype="f")) + with pytest.raises(TypeError): + sp.linalg.eigsh(xp.ones((2, 2), dtype="i")) + with pytest.raises(ValueError): + sp.linalg.eigsh(a, k=self.n) + + def test_starting_vector(self): + # Make symmetric matrix + aux = self._make_matrix("f", cupy) + aux = sparse.coo_matrix(aux).asformat("csr") + matrix = (aux + aux.T) / 2.0 + + # Find reference eigenvector + ew, ev = eigsh(matrix, k=1) + v = ev[:, 0] + + # Obtain non-converged eigenvector from random initial guess. + ew_aux, ev_aux = eigsh(matrix, k=1, ncv=1, maxiter=0) + v_aux = cupy.copysign(ev_aux[:, 0], v) + + # Obtain eigenvector using known eigenvector as initial guess. + ew_v0, ev_v0 = eigsh(matrix, k=1, v0=v.copy(), ncv=1, maxiter=0) + v_v0 = cupy.copysign(ev_v0[:, 0], v) + + assert cupy.linalg.norm(v - v_v0) < cupy.linalg.norm(v - v_aux) diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index a540915585..bb01602b33 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "nvidia-curand", "nvidia-cusolver", "nvidia-cusparse", - "rmm==24.10.*,>=0.0.0a0", + "rmm==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -123,14 +123,23 @@ build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4,!=3.30.0", "cuda-python", - "cython>=3.0.0", + "cython>=3.0.0,<3.1.0a0", "ninja", - "rmm==24.10.*,>=0.0.0a0", + "rmm==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" +[tool.pydistcheck] +select = [ + "distro-too-large-compressed", +] + +# detect when package size grows significantly +max_allowed_size_compressed = '825M' + [tool.pytest.ini_options] filterwarnings = [ "error", + "ignore:.*cuda..* module is deprecated.*:DeprecationWarning", ] diff --git a/python/raft-ann-bench/LICENSE b/python/raft-ann-bench/LICENSE deleted file mode 120000 index 30cff7403d..0000000000 --- a/python/raft-ann-bench/LICENSE +++ /dev/null @@ -1 +0,0 @@ -../../LICENSE \ No newline at end of file diff --git a/python/raft-ann-bench/pyproject.toml b/python/raft-ann-bench/pyproject.toml deleted file mode 100644 index 0e4fda1f00..0000000000 --- a/python/raft-ann-bench/pyproject.toml +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. - -[build-system] -build-backend = "rapids_build_backend.build" -requires = [ - "rapids-build-backend>=0.3.0,<0.4.0.dev0", - "setuptools", - "wheel", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. - -[project] -name = "raft-ann-bench" -dynamic = ["version"] -description = "RAFT ANN benchmarks" -authors = [ - { name = "NVIDIA Corporation" }, -] -license = { text = "Apache 2.0" } -requires-python = ">=3.10" -dependencies = [ -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -classifiers = [ - "Intended Audience :: Developers", - "Topic :: Database", - "Topic :: Scientific/Engineering", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", -] - -[project.urls] -Homepage = "https://github.com/rapidsai/raft" - -[tool.setuptools.packages.find] -where = ["src"] - -[tool.setuptools.package-data] -"*" = ["*.*", "VERSION"] - -[tool.isort] -line_length = 79 -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -combine_as_imports = true -order_by_type = true -skip = [ - "thirdparty", - ".eggs", - ".git", - ".hg", - ".mypy_cache", - ".tox", - ".venv", - "_build", - "buck-out", - "build", - "dist", -] - -[tool.setuptools.dynamic] -version = { file = "raft_ann_bench/VERSION" } - -[tool.rapids-build-backend] -build-backend = "setuptools.build_meta" -requires = [] -dependencies-file = "../../dependencies.yaml" -commit-files = ["src/raft_ann_bench/GIT_COMMIT"] -matrix-entry = "cuda_suffixed=true" diff --git a/python/raft-ann-bench/src/raft_ann_bench/VERSION b/python/raft-ann-bench/src/raft_ann_bench/VERSION deleted file mode 120000 index a4e948506b..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/VERSION +++ /dev/null @@ -1 +0,0 @@ -../../../../VERSION \ No newline at end of file diff --git a/python/raft-ann-bench/src/raft_ann_bench/_version.py b/python/raft-ann-bench/src/raft_ann_bench/_version.py deleted file mode 100644 index 0fa0ba80bc..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/_version.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import importlib.resources - -__version__ = ( - importlib.resources.files(__package__) - .joinpath("VERSION") - .read_text() - .strip() -) -try: - __git_commit__ = ( - importlib.resources.files(__package__) - .joinpath("GIT_COMMIT") - .read_text() - .strip() - ) -except FileNotFoundError: - __git_commit__ = "" - -__all__ = ["__version__", "__git_commit__"] diff --git a/python/raft-ann-bench/src/raft_ann_bench/constraints/__init__.py b/python/raft-ann-bench/src/raft_ann_bench/constraints/__init__.py deleted file mode 100644 index e94ee56c92..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/constraints/__init__.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -DTYPE_SIZES = {"float": 4, "half": 2, "fp8": 1} - - -def raft_cagra_build_constraints(params, dims): - if "graph_degree" in params and "intermediate_graph_degree" in params: - return params["graph_degree"] <= params["intermediate_graph_degree"] - return True - - -def raft_ivf_pq_build_constraints(params, dims): - if "pq_dim" in params: - return params["pq_dim"] <= dims - return True - - -def raft_ivf_pq_search_constraints(params, build_params, k, batch_size): - ret = True - if "internalDistanceDtype" in params and "smemLutDtype" in params: - ret = ( - DTYPE_SIZES[params["smemLutDtype"]] - <= DTYPE_SIZES[params["internalDistanceDtype"]] - ) - - if "nlist" in build_params and "nprobe" in params: - ret = ret and build_params["nlist"] >= params["nprobe"] - return ret - - -def raft_cagra_search_constraints(params, build_params, k, batch_size): - ret = True - if "itopk" in params: - ret = ret and params["itopk"] >= k - return ret - - -def hnswlib_search_constraints(params, build_params, k, batch_size): - if "ef" in params: - return params["ef"] >= k - - -def faiss_gpu_ivf_pq_build_constraints(params, dims): - ret = True - # M must be defined - ret = params["M"] <= dims and dims % params["M"] == 0 - if "use_raft" in params and params["use_raft"]: - return ret - pq_bits = 8 - if "bitsPerCode" in params: - pq_bits = params["bitsPerCode"] - lookup_table_size = 4 - if "useFloat16" in params and params["useFloat16"]: - lookup_table_size = 2 - # FAISS constraint to check if lookup table fits in shared memory - # for now hard code maximum shared memory per block to 49 kB (the value for A100 and V100) - return ret and lookup_table_size * params["M"] * (2**pq_bits) <= 49152 - - -def faiss_gpu_ivf_pq_search_constraints(params, build_params, k, batch_size): - ret = True - if "nlist" in build_params and "nprobe" in params: - ret = ret and build_params["nlist"] >= params["nprobe"] - return ret diff --git a/python/raft-ann-bench/src/raft_ann_bench/data_export/__main__.py b/python/raft-ann-bench/src/raft_ann_bench/data_export/__main__.py deleted file mode 100644 index c8a6375577..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/data_export/__main__.py +++ /dev/null @@ -1,257 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import argparse -import json -import os -import sys -import traceback -import warnings - -import pandas as pd - -skip_build_cols = set( - [ - "algo_name", - "index_name", - "time", - "name", - "family_index", - "per_family_instance_index", - "run_name", - "run_type", - "repetitions", - "repetition_index", - "iterations", - "real_time", - "time_unit", - "index_size", - ] -) - -skip_search_cols = ( - set(["recall", "qps", "latency", "items_per_second", "Recall", "Latency"]) - | skip_build_cols -) - -metrics = { - "k-nn": { - "description": "Recall", - "worst": float("-inf"), - "lim": [0.0, 1.03], - }, - "throughput": { - "description": "Queries per second (1/s)", - "worst": float("-inf"), - }, - "latency": { - "description": "Search Latency (s)", - "worst": float("inf"), - }, -} - - -def read_file(dataset, dataset_path, method): - dir = os.path.join(dataset_path, dataset, "result", method) - for file in os.listdir(dir): - if file.endswith(".json"): - with open( - os.path.join(dir, file), "r", encoding="ISO-8859-1" - ) as f: - try: - data = json.load(f) - df = pd.DataFrame(data["benchmarks"]) - filename_split = file.split(",") - algo_name = (filename_split[0], filename_split[1]) - yield os.path.join(dir, file), algo_name, df - except Exception as e: - print( - "An error occurred processing file %s (%s). " - "Skipping..." % (file, e) - ) - - -def convert_json_to_csv_build(dataset, dataset_path): - for file, algo_name, df in read_file(dataset, dataset_path, "build"): - try: - if "base" in algo_name[1]: - algo_name = algo_name[0] - else: - algo_name = "_".join(algo_name) - df["name"] = df["name"].str.split("/").str[0] - write = pd.DataFrame( - { - "algo_name": [algo_name] * len(df), - "index_name": df["name"], - "time": df["real_time"], - } - ) - for name in df: - if name not in skip_build_cols: - write[name] = df[name] - write.to_csv(file.replace(".json", ".csv"), index=False) - except Exception as e: - print( - "An error occurred processing file %s (%s). Skipping..." - % (file, e) - ) - traceback.print_exc() - - -def create_pointset(data, xn, yn): - xm, ym = (metrics[xn], metrics[yn]) - rev_y = -1 if ym["worst"] < 0 else 1 - rev_x = -1 if xm["worst"] < 0 else 1 - - y_idx = 3 if yn == "throughput" else 4 - data.sort(key=lambda t: (rev_y * t[y_idx], rev_x * t[2])) - - lines = [] - last_x = xm["worst"] - comparator = ( - (lambda xv, lx: xv > lx) if last_x < 0 else (lambda xv, lx: xv < lx) - ) - for d in data: - if comparator(d[2], last_x): - last_x = d[2] - lines.append(d) - return lines - - -def get_frontier(df, metric): - lines = create_pointset(df.values.tolist(), "k-nn", metric) - return pd.DataFrame(lines, columns=df.columns) - - -def convert_json_to_csv_search(dataset, dataset_path): - for file, algo_name, df in read_file(dataset, dataset_path, "search"): - try: - build_file = os.path.join( - dataset_path, - dataset, - "result", - "build", - f"{','.join(algo_name)}.csv", - ) - print(build_file) - if "base" in algo_name[1]: - algo_name = algo_name[0] - else: - algo_name = "_".join(algo_name) - df["name"] = df["name"].str.split("/").str[0] - try: - write = pd.DataFrame( - { - "algo_name": [algo_name] * len(df), - "index_name": df["name"], - "recall": df["Recall"], - "throughput": df["items_per_second"], - "latency": df["Latency"], - } - ) - except Exception as e: - print( - "Search file %s (%s) missing a key. Skipping..." - % (file, e) - ) - for name in df: - if name not in skip_search_cols: - write[name] = df[name] - - if os.path.exists(build_file): - build_df = pd.read_csv(build_file) - write_ncols = len(write.columns) - write["build time"] = None - write["build threads"] = None - write["build cpu_time"] = None - write["build GPU"] = None - - try: - for col_idx in range(6, len(build_df.columns)): - col_name = build_df.columns[col_idx] - write[col_name] = None - - for s_index, search_row in write.iterrows(): - for b_index, build_row in build_df.iterrows(): - if ( - search_row["index_name"] - == build_row["index_name"] - ): - write.iloc[ - s_index, write_ncols - ] = build_df.iloc[b_index, 2] - write.iloc[ - s_index, write_ncols + 1 : - ] = build_df.iloc[b_index, 3:] - break - except Exception as e: - print( - "Build file %s (%s) missing a key. Skipping..." - % (build_file, e) - ) - else: - warnings.warn( - f"Build CSV not found for {algo_name}, " - f"build params won't be " - "appended in the Search CSV" - ) - - write.to_csv(file.replace(".json", ",raw.csv"), index=False) - throughput = get_frontier(write, "throughput") - throughput.to_csv( - file.replace(".json", ",throughput.csv"), index=False - ) - latency = get_frontier(write, "latency") - latency.to_csv(file.replace(".json", ",latency.csv"), index=False) - except Exception as e: - print( - "An error occurred processing file %s (%s). Skipping..." - % (file, e) - ) - traceback.print_exc() - - -def main(): - - call_path = os.getcwd() - if "RAPIDS_DATASET_ROOT_DIR" in os.environ: - default_dataset_path = os.getenv("RAPIDS_DATASET_ROOT_DIR") - else: - default_dataset_path = os.path.join(call_path, "datasets/") - - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - "--dataset", help="dataset to download", default="glove-100-inner" - ) - parser.add_argument( - "--dataset-path", - help="path to dataset folder", - default=default_dataset_path, - ) - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - args = parser.parse_args() - - convert_json_to_csv_build(args.dataset, args.dataset_path) - convert_json_to_csv_search(args.dataset, args.dataset_path) - - -if __name__ == "__main__": - main() diff --git a/python/raft-ann-bench/src/raft_ann_bench/generate_groundtruth/__main__.py b/python/raft-ann-bench/src/raft_ann_bench/generate_groundtruth/__main__.py deleted file mode 100644 index e6f7aaf99c..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/generate_groundtruth/__main__.py +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import argparse -import os -import sys - -import cupy as cp -import numpy as np -import rmm -from pylibraft.common import DeviceResources -from pylibraft.neighbors.brute_force import knn -from rmm.allocators.cupy import rmm_cupy_allocator - -from .utils import memmap_bin_file, suffix_from_dtype, write_bin - - -def generate_random_queries(n_queries, n_features, dtype=np.float32): - print("Generating random queries") - if np.issubdtype(dtype, np.integer): - queries = cp.random.randint( - 0, 255, size=(n_queries, n_features), dtype=dtype - ) - else: - queries = cp.random.uniform(size=(n_queries, n_features)).astype(dtype) - return queries - - -def choose_random_queries(dataset, n_queries): - print("Choosing random vector from dataset as query vectors") - query_idx = np.random.choice( - dataset.shape[0], size=(n_queries,), replace=False - ) - return dataset[query_idx, :] - - -def calc_truth(dataset, queries, k, metric="sqeuclidean"): - handle = DeviceResources() - n_samples = dataset.shape[0] - n = 500000 # batch size for processing neighbors - i = 0 - indices = None - distances = None - queries = cp.asarray(queries, dtype=cp.float32) - - while i < n_samples: - print("Step {0}/{1}:".format(i // n, n_samples // n)) - n_batch = n if i + n <= n_samples else n_samples - i - - X = cp.asarray(dataset[i : i + n_batch, :], cp.float32) - - D, Ind = knn(X, queries, k, metric=metric, handle=handle) - handle.sync() - - D, Ind = cp.asarray(D), cp.asarray(Ind) - Ind += i # shift neighbor index by offset i - - if distances is None: - distances = D - indices = Ind - else: - distances = cp.concatenate([distances, D], axis=1) - indices = cp.concatenate([indices, Ind], axis=1) - idx = cp.argsort(distances, axis=1)[:, :k] - distances = cp.take_along_axis(distances, idx, axis=1) - indices = cp.take_along_axis(indices, idx, axis=1) - - i += n_batch - - return distances, indices - - -def main(): - pool = rmm.mr.PoolMemoryResource( - rmm.mr.CudaMemoryResource(), initial_pool_size=2**30 - ) - rmm.mr.set_current_device_resource(pool) - cp.cuda.set_allocator(rmm_cupy_allocator) - - parser = argparse.ArgumentParser( - prog="generate_groundtruth", - description="Generate true neighbors using exact NN search. " - "The input and output files are in big-ann-benchmark's binary format.", - epilog="""Example usage - # With existing query file - python -m raft_ann_bench.generate_groundtruth --dataset /dataset/base.\ -fbin --output=groundtruth_dir --queries=/dataset/query.public.10K.fbin - - # With randomly generated queries - python -m raft_ann_bench.generate_groundtruth --dataset /dataset/base.\ -fbin --output=groundtruth_dir --queries=random --n_queries=10000 - - # Using only a subset of the dataset. Define queries by randomly - # selecting vectors from the (subset of the) dataset. - python -m raft_ann_bench.generate_groundtruth --dataset /dataset/base.\ -fbin --nrows=2000000 --cols=128 --output=groundtruth_dir \ ---queries=random-choice --n_queries=10000 - """, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - - parser.add_argument("dataset", type=str, help="input dataset file name") - parser.add_argument( - "--queries", - type=str, - default="random", - help="Queries file name, or one of 'random-choice' or 'random' " - "(default). 'random-choice': select n_queries vectors from the input " - "dataset. 'random': generate n_queries as uniform random numbers.", - ) - parser.add_argument( - "--output", - type=str, - default="", - help="output directory name (default current dir)", - ) - - parser.add_argument( - "--n_queries", - type=int, - default=10000, - help="Number of quries to generate (if no query file is given). " - "Default: 10000.", - ) - - parser.add_argument( - "-N", - "--rows", - default=None, - type=int, - help="use only first N rows from dataset, by default the whole " - "dataset is used", - ) - parser.add_argument( - "-D", - "--cols", - default=None, - type=int, - help="number of features (dataset columns). " - "Default: read from dataset file.", - ) - parser.add_argument( - "--dtype", - type=str, - help="Dataset dtype. When not specified, then derived from extension." - " Supported types: 'float32', 'float16', 'uint8', 'int8'", - ) - - parser.add_argument( - "-k", - type=int, - default=100, - help="Number of neighbors (per query) to calculate", - ) - parser.add_argument( - "--metric", - type=str, - default="sqeuclidean", - help="Metric to use while calculating distances. Valid metrics are " - "those that are accepted by pylibraft.neighbors.brute_force.knn. Most" - " commonly used with RAFT ANN are 'sqeuclidean' and 'inner_product'", - ) - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - args = parser.parse_args() - - if args.rows is not None: - print("Reading subset of the data, nrows=", args.rows) - else: - print("Reading whole dataset") - - # Load input data - dataset = memmap_bin_file( - args.dataset, args.dtype, shape=(args.rows, args.cols) - ) - n_features = dataset.shape[1] - dtype = dataset.dtype - - print( - "Dataset size {:6.1f} GB, shape {}, dtype {}".format( - dataset.size * dataset.dtype.itemsize / 1e9, - dataset.shape, - np.dtype(dtype), - ) - ) - - if len(args.output) > 0: - os.makedirs(args.output, exist_ok=True) - - if args.queries == "random" or args.queries == "random-choice": - if args.n_queries is None: - raise RuntimeError( - "n_queries must be given to generate random queries" - ) - if args.queries == "random": - queries = generate_random_queries( - args.n_queries, n_features, dtype - ) - elif args.queries == "random-choice": - queries = choose_random_queries(dataset, args.n_queries) - - queries_filename = os.path.join( - args.output, "queries" + suffix_from_dtype(dtype) - ) - print("Writing queries file", queries_filename) - write_bin(queries_filename, queries) - else: - print("Reading queries from file", args.queries) - queries = memmap_bin_file(args.queries, dtype) - - print("Calculating true nearest neighbors") - distances, indices = calc_truth(dataset, queries, args.k, args.metric) - - write_bin( - os.path.join(args.output, "groundtruth.neighbors.ibin"), - indices.astype(np.uint32), - ) - write_bin( - os.path.join(args.output, "groundtruth.distances.fbin"), - distances.astype(np.float32), - ) - - -if __name__ == "__main__": - main() diff --git a/python/raft-ann-bench/src/raft_ann_bench/generate_groundtruth/utils.py b/python/raft-ann-bench/src/raft_ann_bench/generate_groundtruth/utils.py deleted file mode 100644 index 3f2dd11a16..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/generate_groundtruth/utils.py +++ /dev/null @@ -1,103 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os - -import numpy as np - - -def dtype_from_filename(filename): - ext = os.path.splitext(filename)[1] - if ext == ".fbin": - return np.float32 - if ext == ".hbin": - return np.float16 - elif ext == ".ibin": - return np.int32 - elif ext == ".u8bin": - return np.ubyte - elif ext == ".i8bin": - return np.byte - else: - raise RuntimeError("Not supported file extension" + ext) - - -def suffix_from_dtype(dtype): - if dtype == np.float32: - return ".fbin" - if dtype == np.float16: - return ".hbin" - elif dtype == np.int32: - return ".ibin" - elif dtype == np.ubyte: - return ".u8bin" - elif dtype == np.byte: - return ".i8bin" - else: - raise RuntimeError("Not supported dtype extension" + dtype) - - -def memmap_bin_file( - bin_file, dtype, shape=None, mode="r", size_dtype=np.uint32 -): - extent_itemsize = np.dtype(size_dtype).itemsize - offset = int(extent_itemsize) * 2 - if bin_file is None: - return None - if dtype is None: - dtype = dtype_from_filename(bin_file) - - if mode[0] == "r": - a = np.memmap(bin_file, mode=mode, dtype=size_dtype, shape=(2,)) - if shape is None: - shape = (a[0], a[1]) - else: - shape = tuple( - [ - aval if sval is None else sval - for aval, sval in zip(a, shape) - ] - ) - - return np.memmap( - bin_file, mode=mode, dtype=dtype, offset=offset, shape=shape - ) - elif mode[0] == "w": - if shape is None: - raise ValueError("Need to specify shape to map file in write mode") - - print("creating file", bin_file) - dirname = os.path.dirname(bin_file) - if len(dirname) > 0: - os.makedirs(dirname, exist_ok=True) - a = np.memmap(bin_file, mode=mode, dtype=size_dtype, shape=(2,)) - a[0] = shape[0] - a[1] = shape[1] - a.flush() - del a - fp = np.memmap( - bin_file, mode="r+", dtype=dtype, offset=offset, shape=shape - ) - return fp - - # print('# {}: shape: {}, dtype: {}'.format(bin_file, shape, dtype)) - - -def write_bin(fname, data): - print("writing", fname, data.shape, data.dtype, "...") - with open(fname, "wb") as f: - np.asarray(data.shape, dtype=np.uint32).tofile(f) - data.tofile(f) diff --git a/python/raft-ann-bench/src/raft_ann_bench/get_dataset/__main__.py b/python/raft-ann-bench/src/raft_ann_bench/get_dataset/__main__.py deleted file mode 100644 index 0a6c37aabc..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/get_dataset/__main__.py +++ /dev/null @@ -1,115 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -import subprocess -import sys -from urllib.request import urlretrieve - - -def get_dataset_path(name, ann_bench_data_path): - if not os.path.exists(ann_bench_data_path): - os.mkdir(ann_bench_data_path) - return os.path.join(ann_bench_data_path, f"{name}.hdf5") - - -def download_dataset(url, path): - if not os.path.exists(path): - print(f"downloading {url} -> {path}...") - urlretrieve(url, path) - - -def convert_hdf5_to_fbin(path, normalize): - scripts_path = os.path.dirname(os.path.realpath(__file__)) - ann_bench_scripts_path = os.path.join(scripts_path, "hdf5_to_fbin.py") - print(f"calling script {ann_bench_scripts_path}") - if normalize and "angular" in path: - subprocess.run( - ["python", ann_bench_scripts_path, "-n", "%s" % path], check=True - ) - else: - subprocess.run( - ["python", ann_bench_scripts_path, "%s" % path], check=True - ) - - -def move(name, ann_bench_data_path): - if "angular" in name: - new_name = name.replace("angular", "inner") - else: - new_name = name - new_path = os.path.join(ann_bench_data_path, new_name) - if not os.path.exists(new_path): - os.mkdir(new_path) - for bin_name in [ - "base.fbin", - "query.fbin", - "groundtruth.neighbors.ibin", - "groundtruth.distances.fbin", - ]: - os.rename( - f"{ann_bench_data_path}/{name}.{bin_name}", - f"{new_path}/{bin_name}", - ) - - -def download(name, normalize, ann_bench_data_path): - path = get_dataset_path(name, ann_bench_data_path) - try: - url = f"http://ann-benchmarks.com/{name}.hdf5" - download_dataset(url, path) - - convert_hdf5_to_fbin(path, normalize) - - move(name, ann_bench_data_path) - except Exception: - print(f"Cannot download {url}") - raise - - -def main(): - call_path = os.getcwd() - if "RAPIDS_DATASET_ROOT_DIR" in os.environ: - default_dataset_path = os.getenv("RAPIDS_DATASET_ROOT_DIR") - else: - default_dataset_path = os.path.join(call_path, "datasets/") - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - "--dataset", help="dataset to download", default="glove-100-angular" - ) - parser.add_argument( - "--dataset-path", - help="path to download dataset", - default=default_dataset_path, - ) - parser.add_argument( - "--normalize", - help="normalize cosine distance to inner product", - action="store_true", - ) - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - args = parser.parse_args() - - download(args.dataset, args.normalize, args.dataset_path) - - -if __name__ == "__main__": - main() diff --git a/python/raft-ann-bench/src/raft_ann_bench/get_dataset/fbin_to_f16bin.py b/python/raft-ann-bench/src/raft_ann_bench/get_dataset/fbin_to_f16bin.py deleted file mode 100755 index ee7410e0cc..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/get_dataset/fbin_to_f16bin.py +++ /dev/null @@ -1,49 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from __future__ import absolute_import, division, print_function - -import sys - -import numpy as np - - -def read_fbin(fname): - shape = np.fromfile(fname, dtype=np.uint32, count=2) - if float(shape[0]) * shape[1] * 4 > 2000000000: - data = np.memmap(fname, dtype=np.float32, offset=8, mode="r").reshape( - shape - ) - else: - data = np.fromfile(fname, dtype=np.float32, offset=8).reshape(shape) - return data - - -def write_bin(fname, data): - with open(fname, "wb") as f: - np.asarray(data.shape, dtype=np.uint32).tofile(f) - data.tofile(f) - - -if len(sys.argv) != 3: - print( - "usage: %s input.fbin output.f16bin" % (sys.argv[0]), - file=sys.stderr, - ) - sys.exit(-1) - -data = read_fbin(sys.argv[1]).astype(np.float16) -write_bin(sys.argv[2], data) diff --git a/python/raft-ann-bench/src/raft_ann_bench/get_dataset/hdf5_to_fbin.py b/python/raft-ann-bench/src/raft_ann_bench/get_dataset/hdf5_to_fbin.py deleted file mode 100755 index ba853c63f5..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/get_dataset/hdf5_to_fbin.py +++ /dev/null @@ -1,90 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import sys - -import h5py -import numpy as np - - -def normalize(x): - norm = np.linalg.norm(x, axis=1) - return (x.T / norm).T - - -def write_bin(fname, data): - with open(fname, "wb") as f: - np.asarray(data.shape, dtype=np.uint32).tofile(f) - data.tofile(f) - - -if __name__ == "__main__": - if len(sys.argv) != 2 and len(sys.argv) != 3: - print( - "usage: %s [-n] .hdf5\n" % (sys.argv[0]), - " -n: normalize base/query set\n", - "outputs: .base.fbin\n", - " .query.fbin\n", - " .groundtruth.neighbors.ibin\n", - " .groundtruth.distances.fbin", - file=sys.stderr, - ) - sys.exit(-1) - - need_normalize = False - if len(sys.argv) == 3: - assert sys.argv[1] == "-n" - need_normalize = True - fname_prefix = sys.argv[-1] - assert fname_prefix.endswith(".hdf5") - fname_prefix = fname_prefix[:-5] - - hdf5 = h5py.File(sys.argv[-1], "r") - assert ( - hdf5.attrs["distance"] == "angular" - or hdf5.attrs["distance"] == "euclidean" - ) - assert hdf5["train"].dtype == np.float32 - assert hdf5["test"].dtype == np.float32 - assert hdf5["neighbors"].dtype == np.int32 - assert hdf5["distances"].dtype == np.float32 - - base = hdf5["train"][:] - query = hdf5["test"][:] - if need_normalize: - base = normalize(base) - query = normalize(query) - elif hdf5.attrs["distance"] == "angular": - print( - "warning: input has angular distance, ", - "specify -n to normalize base/query set!\n", - ) - - output_fname = fname_prefix + ".base.fbin" - print("writing", output_fname, "...") - write_bin(output_fname, base) - - output_fname = fname_prefix + ".query.fbin" - print("writing", output_fname, "...") - write_bin(output_fname, query) - - output_fname = fname_prefix + ".groundtruth.neighbors.ibin" - print("writing", output_fname, "...") - write_bin(output_fname, hdf5["neighbors"][:]) - - output_fname = fname_prefix + ".groundtruth.distances.fbin" - print("writing", output_fname, "...") - write_bin(output_fname, hdf5["distances"][:]) diff --git a/python/raft-ann-bench/src/raft_ann_bench/plot/__main__.py b/python/raft-ann-bench/src/raft_ann_bench/plot/__main__.py deleted file mode 100644 index 86fd527f5f..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/plot/__main__.py +++ /dev/null @@ -1,623 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is inspired by -# 1: https://github.com/erikbern/ann-benchmarks/blob/main/plot.py -# 2: https://github.com/erikbern/ann-benchmarks/blob/main/ann_benchmarks/plotting/utils.py # noqa: E501 -# 3: https://github.com/erikbern/ann-benchmarks/blob/main/ann_benchmarks/plotting/metrics.py # noqa: E501 -# Licence: https://github.com/erikbern/ann-benchmarks/blob/main/LICENSE - -import argparse -import itertools -import os -import sys -from collections import OrderedDict - -import matplotlib as mpl -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd - -mpl.use("Agg") - -metrics = { - "k-nn": { - "description": "Recall", - "worst": float("-inf"), - "lim": [0.0, 1.03], - }, - "throughput": { - "description": "Queries per second (1/s)", - "worst": float("-inf"), - }, - "latency": { - "description": "Search Latency (s)", - "worst": float("inf"), - }, -} - - -def positive_int(input_str: str) -> int: - try: - i = int(input_str) - if i < 1: - raise ValueError - except ValueError: - raise argparse.ArgumentTypeError( - f"{input_str} is not a positive integer" - ) - - return i - - -def positive_float(input_str: str) -> float: - try: - i = float(input_str) - if i < 0.0: - raise ValueError - except ValueError: - raise argparse.ArgumentTypeError( - f"{input_str} is not a positive float" - ) - - return i - - -def generate_n_colors(n): - vs = np.linspace(0.3, 0.9, 7) - colors = [(0.9, 0.4, 0.4, 1.0)] - - def euclidean(a, b): - return sum((x - y) ** 2 for x, y in zip(a, b)) - - while len(colors) < n: - new_color = max( - itertools.product(vs, vs, vs), - key=lambda a: min(euclidean(a, b) for b in colors), - ) - colors.append(new_color + (1.0,)) - return colors - - -def create_linestyles(unique_algorithms): - colors = dict( - zip(unique_algorithms, generate_n_colors(len(unique_algorithms))) - ) - linestyles = dict( - (algo, ["--", "-.", "-", ":"][i % 4]) - for i, algo in enumerate(unique_algorithms) - ) - markerstyles = dict( - (algo, ["+", "<", "o", "*", "x"][i % 5]) - for i, algo in enumerate(unique_algorithms) - ) - faded = dict( - (algo, (r, g, b, 0.3)) for algo, (r, g, b, a) in colors.items() - ) - return dict( - ( - algo, - (colors[algo], faded[algo], linestyles[algo], markerstyles[algo]), - ) - for algo in unique_algorithms - ) - - -def create_plot_search( - all_data, - x_scale, - y_scale, - fn_out, - linestyles, - dataset, - k, - batch_size, - mode, - time_unit, - x_start, -): - xn = "k-nn" - xm, ym = (metrics[xn], metrics[mode]) - xm["lim"][0] = x_start - # Now generate each plot - handles = [] - labels = [] - plt.figure(figsize=(12, 9)) - - # Sorting by mean y-value helps aligning plots with labels - def mean_y(algo): - points = np.array(all_data[algo], dtype=object) - return -np.log(np.array(points[:, 3], dtype=np.float32)).mean() - - # Find range for logit x-scale - min_x, max_x = 1, 0 - for algo in sorted(all_data.keys(), key=mean_y): - points = np.array(all_data[algo], dtype=object) - xs = points[:, 2] - ys = points[:, 3] - min_x = min([min_x] + [x for x in xs if x > 0]) - max_x = max([max_x] + [x for x in xs if x < 1]) - color, faded, linestyle, marker = linestyles[algo] - (handle,) = plt.plot( - xs, - ys, - "-", - label=algo, - color=color, - ms=7, - mew=3, - lw=3, - marker=marker, - ) - handles.append(handle) - - labels.append(algo) - - ax = plt.gca() - y_description = ym["description"] - if mode == "latency": - y_description = y_description.replace("(s)", f"({time_unit})") - ax.set_ylabel(y_description) - ax.set_xlabel("Recall") - # Custom scales of the type --x-scale a3 - if x_scale[0] == "a": - alpha = float(x_scale[1:]) - - def fun(x): - return 1 - (1 - x) ** (1 / alpha) - - def inv_fun(x): - return 1 - (1 - x) ** alpha - - ax.set_xscale("function", functions=(fun, inv_fun)) - if alpha <= 3: - ticks = [inv_fun(x) for x in np.arange(0, 1.2, 0.2)] - plt.xticks(ticks) - if alpha > 3: - from matplotlib import ticker - - ax.xaxis.set_major_formatter(ticker.LogitFormatter()) - # plt.xticks(ticker.LogitLocator().tick_values(min_x, max_x)) - plt.xticks([0, 1 / 2, 1 - 1e-1, 1 - 1e-2, 1 - 1e-3, 1 - 1e-4, 1]) - # Other x-scales - else: - ax.set_xscale(x_scale) - ax.set_yscale(y_scale) - ax.set_title(f"{dataset} k={k} batch_size={batch_size}") - plt.gca().get_position() - # plt.gca().set_position([box.x0, box.y0, box.width * 0.8, box.height]) - ax.legend( - handles, - labels, - loc="center left", - bbox_to_anchor=(1, 0.5), - prop={"size": 9}, - ) - plt.grid(visible=True, which="major", color="0.65", linestyle="-") - plt.setp(ax.get_xminorticklabels(), visible=True) - - # Logit scale has to be a subset of (0,1) - if "lim" in xm and x_scale != "logit": - x0, x1 = xm["lim"] - plt.xlim(max(x0, 0), min(x1, 1)) - elif x_scale == "logit": - plt.xlim(min_x, max_x) - if "lim" in ym: - plt.ylim(ym["lim"]) - - # Workaround for bug https://github.com/matplotlib/matplotlib/issues/6789 - ax.spines["bottom"]._adjust_location() - - print(f"writing search output to {fn_out}") - plt.savefig(fn_out, bbox_inches="tight") - plt.close() - - -def create_plot_build( - build_results, search_results, linestyles, fn_out, dataset, k, batch_size -): - bt_80 = [0] * len(linestyles) - - bt_90 = [0] * len(linestyles) - - bt_95 = [0] * len(linestyles) - - bt_99 = [0] * len(linestyles) - - data = OrderedDict() - colors = OrderedDict() - - # Sorting by mean y-value helps aligning plots with labels - - def mean_y(algo): - points = np.array(search_results[algo], dtype=object) - return -np.log(np.array(points[:, 3], dtype=np.float32)).mean() - - for pos, algo in enumerate(sorted(search_results.keys(), key=mean_y)): - points = np.array(search_results[algo], dtype=object) - # x is recall, ls is algo_name, idxs is index_name - xs = points[:, 2] - ls = points[:, 0] - idxs = points[:, 1] - - len_80, len_90, len_95, len_99 = 0, 0, 0, 0 - for i in range(len(xs)): - if xs[i] >= 0.80 and xs[i] < 0.90: - bt_80[pos] = bt_80[pos] + build_results[(ls[i], idxs[i])][0][2] - len_80 = len_80 + 1 - elif xs[i] >= 0.9 and xs[i] < 0.95: - bt_90[pos] = bt_90[pos] + build_results[(ls[i], idxs[i])][0][2] - len_90 = len_90 + 1 - elif xs[i] >= 0.95 and xs[i] < 0.99: - bt_95[pos] = bt_95[pos] + build_results[(ls[i], idxs[i])][0][2] - len_95 = len_95 + 1 - elif xs[i] >= 0.99: - bt_99[pos] = bt_99[pos] + build_results[(ls[i], idxs[i])][0][2] - len_99 = len_99 + 1 - if len_80 > 0: - bt_80[pos] = bt_80[pos] / len_80 - if len_90 > 0: - bt_90[pos] = bt_90[pos] / len_90 - if len_95 > 0: - bt_95[pos] = bt_95[pos] / len_95 - if len_99 > 0: - bt_99[pos] = bt_99[pos] / len_99 - data[algo] = [ - bt_80[pos], - bt_90[pos], - bt_95[pos], - bt_99[pos], - ] - colors[algo] = linestyles[algo][0] - - index = [ - "@80% Recall", - "@90% Recall", - "@95% Recall", - "@99% Recall", - ] - - df = pd.DataFrame(data, index=index) - df.replace(0.0, np.nan, inplace=True) - df = df.dropna(how="all") - plt.figure(figsize=(12, 9)) - ax = df.plot.bar(rot=0, color=colors) - fig = ax.get_figure() - print(f"writing build output to {fn_out}") - plt.title( - "Average Build Time within Recall Range " - f"for k={k} batch_size={batch_size}" - ) - plt.suptitle(f"{dataset}") - plt.ylabel("Build Time (s)") - fig.savefig(fn_out) - - -def load_lines(results_path, result_files, method, index_key, mode, time_unit): - results = dict() - - for result_filename in result_files: - try: - with open(os.path.join(results_path, result_filename), "r") as f: - lines = f.readlines() - lines = lines[:-1] if lines[-1] == "\n" else lines - - if method == "build": - key_idx = [2] - elif method == "search": - y_idx = 3 if mode == "throughput" else 4 - key_idx = [2, y_idx] - - for line in lines[1:]: - split_lines = line.split(",") - - algo_name = split_lines[0] - index_name = split_lines[1] - - if index_key == "algo": - dict_key = algo_name - elif index_key == "index": - dict_key = (algo_name, index_name) - if dict_key not in results: - results[dict_key] = [] - to_add = [algo_name, index_name] - for key_i in key_idx: - to_add.append(float(split_lines[key_i])) - if ( - mode == "latency" - and time_unit != "s" - and method == "search" - ): - to_add[-1] = ( - to_add[-1] * (10**3) - if time_unit == "ms" - else to_add[-1] * (10**6) - ) - results[dict_key].append(to_add) - except Exception: - print( - f"An error occurred processing file {result_filename}. " - "Skipping..." - ) - - return results - - -def load_all_results( - dataset_path, - algorithms, - groups, - algo_groups, - k, - batch_size, - method, - index_key, - raw, - mode, - time_unit, -): - results_path = os.path.join(dataset_path, "result", method) - result_files = os.listdir(results_path) - if method == "build": - result_files = [ - result_file - for result_file in result_files - if ".csv" in result_file - ] - elif method == "search": - if raw: - suffix = ",raw" - else: - suffix = f",{mode}" - result_files = [ - result_file - for result_file in result_files - if f"{suffix}.csv" in result_file - ] - if len(result_files) == 0: - raise FileNotFoundError(f"No CSV result files found in {results_path}") - - if method == "search": - filter_k_bs = [] - for result_filename in result_files: - filename_split = result_filename.split(",") - if ( - int(filename_split[-3][1:]) == k - and int(filename_split[-2][2:]) == batch_size - ): - filter_k_bs.append(result_filename) - result_files = filter_k_bs - - algo_group_files = [ - result_filename.replace(".csv", "").split(",")[:2] - for result_filename in result_files - ] - algo_group_files = list(zip(*algo_group_files)) - - if len(algorithms) > 0: - final_results = [ - result_files[i] - for i in range(len(result_files)) - if (algo_group_files[0][i] in algorithms) - and (algo_group_files[1][i] in groups) - ] - else: - final_results = [ - result_files[i] - for i in range(len(result_files)) - if (algo_group_files[1][i] in groups) - ] - - if len(algo_groups) > 0: - split_algo_groups = [ - algo_group.split(".") for algo_group in algo_groups - ] - split_algo_groups = list(zip(*split_algo_groups)) - final_algo_groups = [ - result_files[i] - for i in range(len(result_files)) - if (algo_group_files[0][i] in split_algo_groups[0]) - and (algo_group_files[1][i] in split_algo_groups[1]) - ] - final_results = final_results + final_algo_groups - final_results = set(final_results) - - results = load_lines( - results_path, final_results, method, index_key, mode, time_unit - ) - - return results - - -def main(): - call_path = os.getcwd() - if "RAPIDS_DATASET_ROOT_DIR" in os.environ: - default_dataset_path = os.getenv("RAPIDS_DATASET_ROOT_DIR") - else: - default_dataset_path = os.path.join(call_path, "datasets/") - - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - "--dataset", help="dataset to plot", default="glove-100-inner" - ) - parser.add_argument( - "--dataset-path", - help="path to dataset folder", - default=default_dataset_path, - ) - parser.add_argument( - "--output-filepath", - help="directory for PNG to be saved", - default=os.getcwd(), - ) - parser.add_argument( - "--algorithms", - help="plot only comma separated list of named \ - algorithms. If parameters `groups` and `algo-groups \ - are both undefined, then group `base` is plot by default", - default=None, - ) - parser.add_argument( - "--groups", - help="plot only comma separated groups of parameters", - default="base", - ) - parser.add_argument( - "--algo-groups", - "--algo-groups", - help='add comma separated . to plot. \ - Example usage: "--algo-groups=raft_cagra.large,hnswlib.large"', - ) - parser.add_argument( - "-k", - "--count", - default=10, - type=positive_int, - help="the number of nearest neighbors to search for", - ) - parser.add_argument( - "-bs", - "--batch-size", - default=10000, - type=positive_int, - help="number of query vectors to use in each query trial", - ) - parser.add_argument("--build", action="store_true") - parser.add_argument("--search", action="store_true") - parser.add_argument( - "--x-scale", - help="Scale to use when drawing the X-axis. \ - Typically linear, logit or a2", - default="linear", - ) - parser.add_argument( - "--y-scale", - help="Scale to use when drawing the Y-axis", - choices=["linear", "log", "symlog", "logit"], - default="linear", - ) - parser.add_argument( - "--x-start", - help="Recall values to start the x-axis from", - default=0.8, - type=positive_float, - ) - parser.add_argument( - "--mode", - help="search mode whose Pareto frontier is used on the y-axis", - choices=["throughput", "latency"], - default="throughput", - ) - parser.add_argument( - "--time-unit", - help="time unit to plot when mode is latency", - choices=["s", "ms", "us"], - default="ms", - ) - parser.add_argument( - "--raw", - help="Show raw results (not just Pareto frontier) of mode arg", - action="store_true", - ) - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - args = parser.parse_args() - - if args.algorithms: - algorithms = args.algorithms.split(",") - else: - algorithms = [] - groups = args.groups.split(",") - if args.algo_groups: - algo_groups = args.algo_groups.split(",") - else: - algo_groups = [] - k = args.count - batch_size = args.batch_size - if not args.build and not args.search: - build = True - search = True - else: - build = args.build - search = args.search - - search_output_filepath = os.path.join( - args.output_filepath, - f"search-{args.dataset}-k{k}-batch_size{batch_size}.png", - ) - build_output_filepath = os.path.join( - args.output_filepath, - f"build-{args.dataset}-k{k}-batch_size{batch_size}.png", - ) - - search_results = load_all_results( - os.path.join(args.dataset_path, args.dataset), - algorithms, - groups, - algo_groups, - k, - batch_size, - "search", - "algo", - args.raw, - args.mode, - args.time_unit, - ) - linestyles = create_linestyles(sorted(search_results.keys())) - if search: - create_plot_search( - search_results, - args.x_scale, - args.y_scale, - search_output_filepath, - linestyles, - args.dataset, - k, - batch_size, - args.mode, - args.time_unit, - args.x_start, - ) - if build: - build_results = load_all_results( - os.path.join(args.dataset_path, args.dataset), - algorithms, - groups, - algo_groups, - k, - batch_size, - "build", - "index", - args.raw, - args.mode, - args.time_unit, - ) - create_plot_build( - build_results, - search_results, - linestyles, - build_output_filepath, - args.dataset, - k, - batch_size, - ) - - -if __name__ == "__main__": - main() diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/__main__.py b/python/raft-ann-bench/src/raft_ann_bench/run/__main__.py deleted file mode 100644 index c34377d733..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/__main__.py +++ /dev/null @@ -1,614 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import itertools -import json -import os -import subprocess -import sys -import uuid -import warnings -from importlib import import_module - -import yaml - -log_levels = { - "off": 0, - "error": 1, - "warn": 2, - "info": 3, - "debug": 4, - "trace": 5, -} - - -def parse_log_level(level_str): - if level_str not in log_levels: - raise ValueError("Invalid log level: %s" % level_str) - return log_levels[level_str.lower()] - - -def positive_int(input_str: str) -> int: - try: - i = int(input_str) - if i < 1: - raise ValueError - except ValueError: - raise argparse.ArgumentTypeError( - f"{input_str} is not a positive integer" - ) - - return i - - -def merge_build_files(build_dir, build_file, temp_build_file): - - build_dict = {} - - # If build file exists, read it - build_json_path = os.path.join(build_dir, build_file) - tmp_build_json_path = os.path.join(build_dir, temp_build_file) - if os.path.isfile(build_json_path): - try: - with open(build_json_path, "r") as f: - build_dict = json.load(f) - except Exception as e: - print( - "Error loading existing build file: %s (%s)" - % (build_json_path, e) - ) - - temp_build_dict = {} - if os.path.isfile(tmp_build_json_path): - with open(tmp_build_json_path, "r") as f: - temp_build_dict = json.load(f) - else: - raise ValueError("Temp build file not found: %s" % tmp_build_json_path) - - tmp_benchmarks = ( - temp_build_dict["benchmarks"] - if "benchmarks" in temp_build_dict - else {} - ) - benchmarks = build_dict["benchmarks"] if "benchmarks" in build_dict else {} - - # If the build time is absolute 0 then an error occurred - final_bench_dict = {} - for b in benchmarks: - if b["real_time"] > 0: - final_bench_dict[b["name"]] = b - - for tmp_bench in tmp_benchmarks: - if tmp_bench["real_time"] > 0: - final_bench_dict[tmp_bench["name"]] = tmp_bench - - temp_build_dict["benchmarks"] = [v for k, v in final_bench_dict.items()] - with open(build_json_path, "w") as f: - json_str = json.dumps(temp_build_dict, indent=2) - f.write(json_str) - - -def validate_algorithm(algos_conf, algo, gpu_present): - algos_conf_keys = set(algos_conf.keys()) - if gpu_present: - return algo in algos_conf_keys - else: - return ( - algo in algos_conf_keys - and algos_conf[algo]["requires_gpu"] is False - ) - - -def find_executable(algos_conf, algo, group, k, batch_size): - executable = algos_conf[algo]["executable"] - - file_name = (f"{algo},{group}", f"{algo},{group},k{k},bs{batch_size}") - - build_path = os.getenv("RAFT_HOME") - if build_path is not None: - build_path = os.path.join( - build_path, "cpp", "build", "release", executable - ) - if os.path.exists(build_path): - print(f"-- Using RAFT bench from repository in {build_path}. ") - return (executable, build_path, file_name) - - # if there is no build folder present, we look in the conda environment - conda_path = os.getenv("CONDA_PREFIX") - if conda_path is not None: - conda_path = os.path.join(conda_path, "bin", "ann", executable) - if os.path.exists(conda_path): - print("-- Using RAFT bench found in conda environment. ") - return (executable, conda_path, file_name) - - else: - raise FileNotFoundError(executable) - - -def run_build_and_search( - conf_file, - conf_filename, - conf_filedir, - executables_to_run, - dataset_path, - force, - build, - search, - dry_run, - k, - batch_size, - search_threads, - mode="throughput", - raft_log_level="info", -): - for ( - executable, - ann_executable_path, - output_filename, - ) in executables_to_run.keys(): - # Need to write temporary configuration - temp_conf_filename = ( - f"{conf_filename}_{output_filename[1]}_{uuid.uuid1()}.json" - ) - with open(temp_conf_filename, "w") as f: - temp_conf = dict() - temp_conf["dataset"] = conf_file["dataset"] - temp_conf["search_basic_param"] = conf_file["search_basic_param"] - temp_conf["index"] = executables_to_run[ - (executable, ann_executable_path, output_filename) - ]["index"] - json_str = json.dumps(temp_conf, indent=2) - f.write(json_str) - - legacy_result_folder = os.path.join( - dataset_path, conf_file["dataset"]["name"], "result" - ) - os.makedirs(legacy_result_folder, exist_ok=True) - if build: - build_folder = os.path.join(legacy_result_folder, "build") - os.makedirs(build_folder, exist_ok=True) - build_file = f"{output_filename[0]}.json" - temp_build_file = f"{build_file}.lock" - cmd = [ - ann_executable_path, - "--build", - "--data_prefix=" + dataset_path, - "--benchmark_out_format=json", - "--benchmark_counters_tabular=true", - "--benchmark_out=" - + f"{os.path.join(build_folder, temp_build_file)}", - "--raft_log_level=" + f"{parse_log_level(raft_log_level)}", - ] - if force: - cmd = cmd + ["--force"] - cmd = cmd + [temp_conf_filename] - - if dry_run: - print( - "Benchmark command for %s:\n%s\n" - % (output_filename[0], " ".join(cmd)) - ) - else: - try: - subprocess.run(cmd, check=True) - merge_build_files( - build_folder, build_file, temp_build_file - ) - except Exception as e: - print("Error occurred running benchmark: %s" % e) - finally: - os.remove(os.path.join(build_folder, temp_build_file)) - if not search: - os.remove(temp_conf_filename) - - if search: - search_folder = os.path.join(legacy_result_folder, "search") - os.makedirs(search_folder, exist_ok=True) - search_file = f"{output_filename[1]}.json" - cmd = [ - ann_executable_path, - "--search", - "--data_prefix=" + dataset_path, - "--benchmark_counters_tabular=true", - "--override_kv=k:%s" % k, - "--override_kv=n_queries:%s" % batch_size, - "--benchmark_min_warmup_time=1", - "--benchmark_out_format=json", - "--mode=%s" % mode, - "--benchmark_out=" - + f"{os.path.join(search_folder, search_file)}", - "--raft_log_level=" + f"{parse_log_level(raft_log_level)}", - ] - if force: - cmd = cmd + ["--force"] - - if search_threads: - cmd = cmd + ["--threads=%s" % search_threads] - - cmd = cmd + [temp_conf_filename] - if dry_run: - print( - "Benchmark command for %s:\n%s\n" - % (output_filename[1], " ".join(cmd)) - ) - else: - try: - subprocess.run(cmd, check=True) - except Exception as e: - print("Error occurred running benchmark: %s" % e) - finally: - os.remove(temp_conf_filename) - - -def main(): - scripts_path = os.path.dirname(os.path.realpath(__file__)) - call_path = os.getcwd() - - # Read list of allowed algorithms - try: - import rmm # noqa: F401 - - gpu_present = True - except ImportError: - gpu_present = False - - with open(f"{scripts_path}/algos.yaml", "r") as f: - algos_yaml = yaml.safe_load(f) - - if "RAPIDS_DATASET_ROOT_DIR" in os.environ: - default_dataset_path = os.getenv("RAPIDS_DATASET_ROOT_DIR") - else: - default_dataset_path = os.path.join(call_path, "datasets/") - - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - - parser.add_argument( - "--subset-size", - type=positive_int, - help="the number of subset rows of the dataset to build the index", - ) - parser.add_argument( - "-k", - "--count", - default=10, - type=positive_int, - help="the number of nearest neighbors to search for", - ) - parser.add_argument( - "-bs", - "--batch-size", - default=10000, - type=positive_int, - help="number of query vectors to use in each query trial", - ) - parser.add_argument( - "--dataset-configuration", - help="path to YAML configuration file for datasets", - ) - parser.add_argument( - "--configuration", - help="path to YAML configuration file or directory for algorithms\ - Any run groups found in the specified file/directory will \ - automatically override groups of the same name present in the \ - default configurations, including `base`", - ) - parser.add_argument( - "--dataset", - help="name of dataset", - default="glove-100-inner", - ) - parser.add_argument( - "--dataset-path", - help="path to dataset folder, by default will look in " - "RAPIDS_DATASET_ROOT_DIR if defined, otherwise a datasets " - "subdirectory from the calling directory", - default=default_dataset_path, - ) - parser.add_argument("--build", action="store_true") - parser.add_argument("--search", action="store_true") - parser.add_argument( - "--algorithms", - help="run only comma separated list of named \ - algorithms. If parameters `groups` and `algo-groups \ - are both undefined, then group `base` is run by default", - default=None, - ) - parser.add_argument( - "--groups", - help="run only comma separated groups of parameters", - default="base", - ) - parser.add_argument( - "--algo-groups", - help='add comma separated . to run. \ - Example usage: "--algo-groups=raft_cagra.large,hnswlib.large"', - ) - parser.add_argument( - "-f", - "--force", - help="re-run algorithms even if their results \ - already exist", - action="store_true", - ) - - parser.add_argument( - "-m", - "--search-mode", - help="run search in 'latency' (measure individual batches) or " - "'throughput' (pipeline batches and measure end-to-end) mode", - default="latency", - ) - - parser.add_argument( - "-t", - "--search-threads", - help="specify the number threads to use for throughput benchmark." - " Single value or a pair of min and max separated by ':'. " - "Example: --search-threads=1:4. Power of 2 values between 'min' " - "and 'max' will be used. If only 'min' is specified, then a " - "single test is run with 'min' threads. By default min=1, " - "max=.", - default=None, - ) - - parser.add_argument( - "-r", - "--dry-run", - help="dry-run mode will convert the yaml config for the specified " - "algorithms and datasets to the json format that's consumed " - "by the lower-level c++ binaries and then print the command " - "to run execute the benchmarks but will not actually execute " - "the command.", - action="store_true", - ) - parser.add_argument( - "--raft-log-level", - help="Log level, possible values are " - "[off, error, warn, info, debug, trace]. " - "Default: 'info'. Note that 'debug' or more detailed " - "logging level requires that the library is compiled with " - "-DRAFT_ACTIVE_LEVEL= where >= ", - default="info", - ) - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - args = parser.parse_args() - - # If both build and search are not provided, - # run both - if not args.build and not args.search: - build = True - search = True - else: - build = args.build - search = args.search - - dry_run = args.dry_run - - mode = args.search_mode - k = args.count - batch_size = args.batch_size - - # Read configuration file associated to datasets - if args.dataset_configuration: - dataset_conf_f = args.dataset_configuration - else: - dataset_conf_f = os.path.join(scripts_path, "conf", "datasets.yaml") - with open(dataset_conf_f, "r") as f: - dataset_conf_all = yaml.safe_load(f) - - dataset_conf = None - for dataset in dataset_conf_all: - if args.dataset == dataset["name"]: - dataset_conf = dataset - break - if not dataset_conf: - raise ValueError("Could not find a dataset configuration") - - conf_file = dict() - conf_file["dataset"] = dataset_conf - if args.subset_size: - conf_file["dataset"]["subset_size"] = args.subset_size - - conf_file["search_basic_param"] = {} - conf_file["search_basic_param"]["k"] = k - conf_file["search_basic_param"]["batch_size"] = batch_size - - algos_conf_fs = os.listdir(os.path.join(scripts_path, "conf", "algos")) - algos_conf_fs = [ - os.path.join(scripts_path, "conf", "algos", f) - for f in algos_conf_fs - if ".json" not in f - ] - conf_filedir = os.path.join(scripts_path, "conf", "algos") - if args.configuration: - if os.path.isdir(args.configuration): - conf_filedir = args.configuration - algos_conf_fs = algos_conf_fs + [ - os.path.join(args.configuration, f) - for f in os.listdir(args.configuration) - if ".json" not in f - ] - elif os.path.isfile(args.configuration): - conf_filedir = os.path.normpath(args.configuration).split(os.sep) - conf_filedir = os.path.join(*conf_filedir[:-1]) - algos_conf_fs = algos_conf_fs + [args.configuration] - - filter_algos = True if args.algorithms else False - if filter_algos: - allowed_algos = args.algorithms.split(",") - named_groups = args.groups.split(",") - filter_algo_groups = True if args.algo_groups else False - allowed_algo_groups = None - if filter_algo_groups: - allowed_algo_groups = [ - algo_group.split(".") for algo_group in args.algo_groups.split(",") - ] - allowed_algo_groups = list(zip(*allowed_algo_groups)) - algos_conf = dict() - for algo_f in algos_conf_fs: - with open(algo_f, "r") as f: - try: - algo = yaml.safe_load(f) - except Exception as e: - warnings.warn( - f"Could not load YAML config {algo_f} due to " - + e.with_traceback() - ) - continue - insert_algo = True - insert_algo_group = False - if filter_algos: - if algo["name"] not in allowed_algos: - insert_algo = False - if filter_algo_groups: - if algo["name"] in allowed_algo_groups[0]: - insert_algo_group = True - - def add_algo_group(group_list): - if algo["name"] not in algos_conf: - algos_conf[algo["name"]] = {"groups": {}} - for group in algo["groups"].keys(): - if group in group_list: - algos_conf[algo["name"]]["groups"][group] = algo[ - "groups" - ][group] - if "constraints" in algo: - algos_conf[algo["name"]]["constraints"] = algo[ - "constraints" - ] - - if insert_algo: - add_algo_group(named_groups) - if insert_algo_group: - add_algo_group(allowed_algo_groups[1]) - - executables_to_run = dict() - for algo in algos_conf.keys(): - validate_algorithm(algos_yaml, algo, gpu_present) - for group in algos_conf[algo]["groups"].keys(): - executable = find_executable( - algos_yaml, algo, group, k, batch_size - ) - if executable not in executables_to_run: - executables_to_run[executable] = {"index": []} - build_params = algos_conf[algo]["groups"][group]["build"] or {} - search_params = algos_conf[algo]["groups"][group]["search"] or {} - - param_names = [] - param_lists = [] - for param in build_params.keys(): - param_names.append(param) - param_lists.append(build_params[param]) - - all_build_params = itertools.product(*param_lists) - - search_param_names = [] - search_param_lists = [] - for search_param in search_params.keys(): - search_param_names.append(search_param) - search_param_lists.append(search_params[search_param]) - - for params in all_build_params: - index = {"algo": algo, "build_param": {}} - if group != "base": - index_name = f"{algo}_{group}" - else: - index_name = f"{algo}" - for i in range(len(params)): - index["build_param"][param_names[i]] = params[i] - index_name += "." + f"{param_names[i]}{params[i]}" - - if "constraints" in algos_conf[algo]: - if "build" in algos_conf[algo]["constraints"]: - importable = algos_conf[algo]["constraints"]["build"] - importable = importable.split(".") - module = ".".join(importable[:-1]) - func = importable[-1] - validator = import_module(module) - build_constraints = getattr(validator, func) - if "dims" not in conf_file["dataset"]: - raise ValueError( - "`dims` needed for build constraints but not " - "specified in datasets.yaml" - ) - if not build_constraints( - index["build_param"], conf_file["dataset"]["dims"] - ): - continue - index_filename = ( - index_name - if len(index_name) < 128 - else str(hash(index_name)) - ) - index["name"] = index_name - index["file"] = os.path.join( - args.dataset_path, args.dataset, "index", index_filename - ) - index["search_params"] = [] - all_search_params = itertools.product(*search_param_lists) - for search_params in all_search_params: - search_dict = dict() - for i in range(len(search_params)): - search_dict[search_param_names[i]] = search_params[i] - if "constraints" in algos_conf[algo]: - if "search" in algos_conf[algo]["constraints"]: - importable = algos_conf[algo]["constraints"][ - "search" - ] - importable = importable.split(".") - module = ".".join(importable[:-1]) - func = importable[-1] - validator = import_module(module) - search_constraints = getattr(validator, func) - if search_constraints( - search_dict, - index["build_param"], - k, - batch_size, - ): - index["search_params"].append(search_dict) - else: - index["search_params"].append(search_dict) - executables_to_run[executable]["index"].append(index) - - if len(index["search_params"]) == 0: - print("No search parameters were added to configuration") - - run_build_and_search( - conf_file, - f"{args.dataset}", - conf_filedir, - executables_to_run, - args.dataset_path, - args.force, - build, - search, - dry_run, - k, - batch_size, - args.search_threads, - mode, - args.raft_log_level, - ) - - -if __name__ == "__main__": - main() diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/algos.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/algos.yaml deleted file mode 100644 index e382bdcba6..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/algos.yaml +++ /dev/null @@ -1,42 +0,0 @@ -faiss_gpu_flat: - executable: FAISS_GPU_FLAT_ANN_BENCH - requires_gpu: true -faiss_gpu_ivf_flat: - executable: FAISS_GPU_IVF_FLAT_ANN_BENCH - requires_gpu: true -faiss_gpu_ivf_pq: - executable: FAISS_GPU_IVF_PQ_ANN_BENCH - requires_gpu: true -faiss_gpu_ivf_sq: - executable: FAISS_GPU_IVF_PQ_ANN_BENCH - requires_gpu: true -faiss_cpu_flat: - executable: FAISS_CPU_FLAT_ANN_BENCH - requires_gpu: false -faiss_cpu_ivf_flat: - executable: FAISS_CPU_IVF_FLAT_ANN_BENCH - requires_gpu: false -faiss_cpu_ivf_pq: - executable: FAISS_CPU_IVF_PQ_ANN_BENCH - requires_gpu: false -raft_ivf_flat: - executable: RAFT_IVF_FLAT_ANN_BENCH - requires_gpu: true -raft_ivf_pq: - executable: RAFT_IVF_PQ_ANN_BENCH - requires_gpu: true -raft_cagra: - executable: RAFT_CAGRA_ANN_BENCH - requires_gpu: true -raft_brute_force: - executable: RAFT_BRUTE_FORCE_ANN_BENCH - requires_gpu: true -ggnn: - executable: GGNN_ANN_BENCH - requires_gpu: true -hnswlib: - executable: HNSWLIB_ANN_BENCH - requires_gpu: false -raft_cagra_hnswlib: - executable: RAFT_CAGRA_HNSWLIB_ANN_BENCH - requires_gpu: true diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_flat.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_flat.yaml deleted file mode 100644 index 25eaf03d40..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_flat.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: faiss_cpu_flat -groups: - base: - build: - search: diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_ivf_flat.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_ivf_flat.yaml deleted file mode 100644 index 29c145f86d..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_ivf_flat.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: faiss_cpu_ivf_flat -groups: - base: - build: - nlist: [2048] - ratio: [10] - useFloat16: [False] - search: - nprobe: [1, 5, 10, 50, 100, 200] - refine_ratio: [1] \ No newline at end of file diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_ivf_pq.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_ivf_pq.yaml deleted file mode 100644 index a531ec8294..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_cpu_ivf_pq.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: faiss_cpu_ivf_pq -groups: - base: - build: - nlist: [1024, 2048, 4096, 8192] - M: [48, 32, 16] - ratio: [10] - bitsPerCode: [8, 6, 5, 4] - search: - nprobe: [1, 5, 10, 50, 100, 200] - large: - build: - nlist: [8192, 16384, 32768, 65536] - M: [48, 32, 16] - ratio: [10] - bitsPerCode: [8, 6, 5, 4] - search: - nprobe: [20, 30, 40, 50, 100, 200, 500, 1000] diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_flat.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_flat.yaml deleted file mode 100644 index a722e1b91c..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_flat.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: faiss_gpu_flat -groups: - base: - build: - search: diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_ivf_flat.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_ivf_flat.yaml deleted file mode 100644 index e4abc35f5c..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_ivf_flat.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: faiss_gpu_ivf_flat -groups: - base: - build: - nlist: [2048] - ratio: [10] - useFloat16: [False, True] - use_raft: [False] - search: - nprobe: [1, 5, 10, 50, 100, 200] - refine_ratio: [1] -groups: - baseraft: - build: - nlist: [2048] - ratio: [10] - useFloat16: [False, True] - use_raft: [True] - search: - nprobe: [1, 5, 10, 50, 100, 200] - refine_ratio: [1] \ No newline at end of file diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_ivf_pq.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_ivf_pq.yaml deleted file mode 100644 index 7560ceaa9c..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/faiss_gpu_ivf_pq.yaml +++ /dev/null @@ -1,77 +0,0 @@ -name: faiss_gpu_ivf_pq -constraints: - build: raft-ann-bench.constraints.faiss_gpu_ivf_pq_build_constraints - search: raft-ann-bench.constraints.faiss_gpu_ivf_pq_search_constraints -groups: - base: - build: - nlist: [1024, 2048, 4096, 8192] - M: [64, 32, 16] - ratio: [10] - usePrecomputed: [False, True] - useFloat16: [False, True] - use_raft: [False] - bitsPerCode: [8] - search: - nprobe: [1, 5, 10, 50, 100, 200] - refine_ratio: [1, 2, 4] - baseraft: - build: - nlist: [1024, 2048, 4096, 8192] - M: [64, 32, 16] - ratio: [10] - usePrecomputed: [False] - useFloat16: [False, True] - use_raft: [True] - bitsPerCode: [8, 6, 5, 4] - search: - nprobe: [1, 5, 10, 50, 100, 200] - refine_ratio: [1, 2, 4] - large: - build: - nlist: [8192, 16384, 32768, 65536] - M: [48, 32, 16] - ratio: [4] - usePrecomputed: [False, True] - useFloat16: [False, True] - use_raft: [False] - bitsPerCode: [8] - search: - nprobe: [20, 30, 40, 50, 100, 200, 500, 1000] - refine_ratio: [1, 2, 4] - largeraft: - build: - nlist: [8192, 16384, 32768, 65536] - M: [48, 32, 16] - ratio: [4] - usePrecomputed: [False] - useFloat16: [False, True] - use_raft: [True] - bitsPerCode: [8, 6, 5, 4] - search: - nprobe: [20, 30, 40, 50, 100, 200, 500, 1000] - refine_ratio: [1, 2, 4] - 100M: - build: - nlist: [50000] - M: [48] - ratio: [10] - usePrecomputed: [False, True] - useFloat16: [False, True] - use_raft: [False] - bitsPerCode: [8] - search: - nprobe: [20, 30, 40, 50, 100, 200, 500, 1000] - refine_ratio: [1] - 100Mraft: - build: - nlist: [50000] - M: [48] - ratio: [10] - usePrecomputed: [False, True] - useFloat16: [False, True] - use_raft: [True] - bitsPerCode: [8, 6, 5, 4] - search: - nprobe: [20, 30, 40, 50, 100, 200, 500, 1000] - refine_ratio: [1] \ No newline at end of file diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/hnswlib.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/hnswlib.yaml deleted file mode 100644 index e7a4e6b506..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/hnswlib.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: hnswlib -constraints: - search: raft_ann_bench.constraints.hnswlib_search_constraints -groups: - base: - build: - M: [12, 16, 24, 36] - efConstruction: [64, 128, 256, 512] - search: - ef: [10, 20, 40, 60, 80, 120, 200, 400, 600, 800] diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_brute_force.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_brute_force.yaml deleted file mode 100644 index da99841f9b..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_brute_force.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: raft_brute_force -groups: - base: - build: - search: diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_cagra.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_cagra.yaml deleted file mode 100644 index bb66b4b232..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_cagra.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: raft_cagra -constraints: - build: raft_ann_bench.constraints.raft_cagra_build_constraints - search: raft_ann_bench.constraints.raft_cagra_search_constraints -groups: - base: - build: - graph_degree: [32, 64, 128, 256] - intermediate_graph_degree: [32, 64, 96, 128] - graph_build_algo: ["NN_DESCENT"] - search: - itopk: [32, 64, 128, 256, 512] - search_width: [1, 2, 4, 8, 16, 32, 64] diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_cagra_hnswlib.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_cagra_hnswlib.yaml deleted file mode 100644 index 3ac2d16b68..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_cagra_hnswlib.yaml +++ /dev/null @@ -1,11 +0,0 @@ -name: raft_cagra_hnswlib -constraints: - search: raft_ann_bench.constraints.hnswlib_search_constraints -groups: - base: - build: - graph_degree: [32, 64, 128, 256] - intermediate_graph_degree: [32, 64, 96, 128] - graph_build_algo: ["NN_DESCENT"] - search: - ef: [10, 20, 40, 60, 80, 120, 200, 400, 600, 800] diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_ivf_flat.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_ivf_flat.yaml deleted file mode 100644 index c36a26514d..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_ivf_flat.yaml +++ /dev/null @@ -1,9 +0,0 @@ -name: raft_ivf_flat -groups: - base: - build: - nlist: [1024, 2048, 4096, 8192, 16384, 32000, 64000] - ratio: [1, 2, 4] - niter: [20, 25] - search: - nprobe: [1, 5, 10, 50, 100, 200, 500, 1000, 2000] \ No newline at end of file diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_ivf_pq.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_ivf_pq.yaml deleted file mode 100644 index bcdcde42a2..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/algos/raft_ivf_pq.yaml +++ /dev/null @@ -1,41 +0,0 @@ -name: raft_ivf_pq -constraints: - build: raft_ann_bench.constraints.raft_ivf_pq_build_constraints - search: raft_ann_bench.constraints.raft_ivf_pq_search_constraints -groups: - base: - build: - nlist: [1024, 2048, 4096, 8192] - pq_dim: [64, 32, 16] - pq_bits: [8, 6, 5, 4] - ratio: [10] - niter: [25] - search: - nprobe: [1, 5, 10, 50, 100, 200] - internalDistanceDtype: ["float"] - smemLutDtype: ["float", "fp8", "half"] - refine_ratio: [1, 2, 4] - large: - build: - nlist: [8192, 16384, 32768, 65536] - pq_dim: [48, 32, 16] - pq_bits: [8, 6, 5, 4] - ratio: [4] - niter: [20] - search: - nprobe: [20, 30, 40, 50, 100, 200, 500, 1000] - internalDistanceDtype: ["float"] - smemLutDtype: ["float", "fp8", "half"] - refine_ratio: [1, 2, 4] - 100M: - build: - nlist: [50000] - pq_dim: [48] - pq_bits: [8, 6, 5, 4] - ratio: [10] - niter: [10] - search: - nprobe: [20, 30, 40, 50, 100, 200, 500, 1000] - internalDistanceDtype: ["float"] - smemLutDtype: ["float", "fp8", "half"] - refine_ratio: [1] diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/bigann-100M.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/bigann-100M.json deleted file mode 100644 index 55abca25d2..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/bigann-100M.json +++ /dev/null @@ -1,192 +0,0 @@ -{ - "dataset": { - "name": "bigann-100M", - "base_file": "bigann-1B/base.1B.u8bin", - "subset_size": 100000000, - "query_file": "bigann-1B/query.public.10K.u8bin", - "groundtruth_neighbors_file": "bigann-100M/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - - "search_basic_param": { - "batch_size": 10000, - "k": 10 - }, - - "index": [ - { - "name": "raft_ivf_pq.dimpq64-cluster5K", - "algo": "raft_ivf_pq", - "build_param": {"niter": 25, "nlist": 5000, "pq_dim": 64, "ratio": 10}, - "file": "bigann-100M/raft_ivf_pq/dimpq64-cluster5K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "half" } - ] - }, - { - "name": "raft_ivf_pq.dimpq64-cluster10K", - "algo": "raft_ivf_pq", - "build_param": {"niter": 25, "nlist": 10000, "pq_dim": 64, "ratio": 10}, - "file": "bigann-100M/raft_ivf_pq/dimpq64-cluster5K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "half" } - ] - }, - { - "name": "hnswlib.M12", - "algo": "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file": "bigann-100M/hnswlib/M12", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M16", - "algo": "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file": "bigann-100M/hnswlib/M16", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M24", - "algo": "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file": "bigann-100M/hnswlib/M24", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M36", - "algo": "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file": "bigann-100M/hnswlib/M36", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "raft_ivf_flat.nlist100K", - "algo": "raft_ivf_flat", - "build_param": {"nlist": 100000, "niter": 25, "ratio": 5}, - "file": "bigann-100M/raft_ivf_flat/nlist100K", - "search_params": [ - {"max_batch":10000, "max_k":10, "nprobe":20}, - {"max_batch":10000, "max_k":10, "nprobe":30}, - {"max_batch":10000, "max_k":10, "nprobe":40}, - {"max_batch":10000, "max_k":10, "nprobe":50}, - {"max_batch":10000, "max_k":10, "nprobe":100}, - {"max_batch":10000, "max_k":10, "nprobe":200}, - {"max_batch":10000, "max_k":10, "nprobe":500}, - {"max_batch":10000, "max_k":10, "nprobe":1000} - ] - }, - { - "name": "raft_cagra.dim32", - "algo": "raft_cagra", - "build_param": {"graph_degree": 32}, - "file": "bigann-100M/raft_cagra/dim32", - "search_params": [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ] - }, - { - "name": "raft_cagra.dim64", - "algo": "raft_cagra", - "build_param": {"graph_degree": 64}, - "file": "bigann-100M/raft_cagra/dim64", - "search_params": [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ] - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/datasets.yaml b/python/raft-ann-bench/src/raft_ann_bench/run/conf/datasets.yaml deleted file mode 100644 index 188d24d20f..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/datasets.yaml +++ /dev/null @@ -1,127 +0,0 @@ -- name: bigann-1B - base_file: bigann-1B/base.1B.u8bin - subset_size: 100000000 - dims: 128 - query_file: bigann-1B/query.public.10K.u8bin - groundtruth_neighbors_file: bigann-1B/groundtruth.neighbors.ibin - distance: euclidean - -- name: deep-1B - base_file: deep-1B/base.1B.fbin - query_file: deep-1B/query.public.10K.fbin - dims: 96 - groundtruth_neighbors_file: deep-1B/groundtruth.neighbors.ibin - distance: inner_product - -- name: bigann-100M - base_file: bigann-100M/base.1B.u8bin - subset_size: 100000000 - dims: 128 - query_file: bigann-100M/query.public.10K.u8bin - groundtruth_neighbors_file: bigann-100M/groundtruth.neighbors.ibin - distance: euclidean - -- name: deep-image-96-inner - base_file: deep-image-96-inner/base.fbin - query_file: deep-image-96-inner/query.fbin - dims: 96 - groundtruth_neighbors_file: deep-image-96-inner/groundtruth.neighbors.ibin - distance: euclidean - -- name: fashion-mnist-784-euclidean - dims: 784 - base_file: fashion-mnist-784-euclidean/base.fbin - query_file: fashion-mnist-784-euclidean/query.fbin - groundtruth_neighbors_file: fashion-mnist-784-euclidean/groundtruth.neighbors.ibin - distance: euclidean - -- name: gist-960-euclidean - dims: 960 - base_file: gist-960-euclidean/base.fbin - query_file: gist-960-euclidean/query.fbin - groundtruth_neighbors_file: gist-960-euclidean/groundtruth.neighbors.ibin - distance: euclidean - -- name: glove-50-angular - dims: 50 - base_file: glove-50-angular/base.fbin - query_file: glove-50-angular/query.fbin - groundtruth_neighbors_file: glove-50-angular/groundtruth.neighbors.ibin - distance: euclidean - -- name: glove-50-inner - dims: 50 - base_file: glove-50-inner/base.fbin - query_file: glove-50-inner/query.fbin - groundtruth_neighbors_file: glove-50-inner/groundtruth.neighbors.ibin - distance: euclidean - -- name: glove-100-angular - dims: 100 - base_file: glove-100-angular/base.fbin - query_file: glove-100-angular/query.fbin - groundtruth_neighbors_file: glove-100-angular/groundtruth.neighbors.ibin - distance: euclidean - -- name: glove-100-inner - dims: 100 - base_file: glove-100-inner/base.fbin - query_file: glove-100-inner/query.fbin - groundtruth_neighbors_file: glove-100-inner/groundtruth.neighbors.ibin - distance: euclidean - -- name: lastfm-65-angular - dims: 65 - base_file: lastfm-65-angular/base.fbin - query_file: lastfm-65-angular/query.fbin - groundtruth_neighbors_file: lastfm-65-angular/groundtruth.neighbors.ibin - distance: euclidean - -- name: mnist-784-euclidean - dims: 784 - base_file: mnist-784-euclidean/base.fbin - query_file: mnist-784-euclidean/query.fbin - groundtruth_neighbors_file: mnist-784-euclidean/groundtruth.neighbors.ibin - distance: euclidean - -- name: nytimes-256-angular - dims: 256 - base_file: nytimes-256-angular/base.fbin - query_file: nytimes-256-angular/query.fbin - groundtruth_neighbors_file: nytimes-256-angular/groundtruth.neighbors.ibin - distance: euclidean - -- name: nytimes-256-inner - dims: 256 - base_file: nytimes-256-inner/base.fbin - query_file: nytimes-256-inner/query.fbin - groundtruth_neighbors_file: nytimes-256-inner/groundtruth.neighbors.ibin - distance: euclidean - -- name: sift-128-euclidean - dims: 128 - base_file: sift-128-euclidean/base.fbin - query_file: sift-128-euclidean/query.fbin - groundtruth_neighbors_file: sift-128-euclidean/groundtruth.neighbors.ibin - distance: euclidean - -- name: wiki_all_1M - dims: 768 - base_file: wiki_all_1M/base.1M.fbin - query_file: wiki_all_1M/queries.fbin - groundtruth_neighbors_file: wiki_all_1M/groundtruth.1M.neighbors.ibin - distance: euclidean - -- name: wiki_all_10M - dims: 768 - base_file: wiki_all_10M/base.10M.fbin - query_file: wiki_all_10M/queries.fbin - groundtruth_neighbors_file: wiki_all_10M/groundtruth.10M.neighbors.ibin - distance: euclidean - -- name: wiki_all_88M - dims: 768 - base_file: wiki_all_88M/base.88M.fbin - query_file: wiki_all_88M/queries.fbin - groundtruth_neighbors_file: wiki_all_88M/groundtruth.88M.neighbors.ibin - distance: euclidean diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-100M.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-100M.json deleted file mode 100644 index ea92a0de18..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-100M.json +++ /dev/null @@ -1,458 +0,0 @@ -{ - "dataset": { - "name": "deep-100M", - "base_file": "deep-100M/base.1B.fbin", - "subset_size": 100000000, - "query_file": "deep-100M/query.public.10K.fbin", - "groundtruth_neighbors_file": "deep-100M/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - - "search_basic_param": { - "batch_size": 10000, - "k": 10 - }, - - "index": [ - { - "name": "hnswlib.M12", - "algo": "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file": "deep-100M/hnswlib/M12", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M16", - "algo": "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file": "deep-100M/hnswlib/M16", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M24", - "algo": "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file": "deep-100M/hnswlib/M24", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M36", - "algo": "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file": "deep-100M/hnswlib/M36", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "faiss_gpu_ivf_flat.nlist50K", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist":50000}, - "file": "deep-100M/faiss_gpu_ivf_flat/nlist50K", - "search_params": [ - {"nprobe":20}, - {"nprobe":30}, - {"nprobe":40}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500}, - {"nprobe":1000} - ] - }, - { - "name": "faiss_gpu_ivf_flat.nlist100K", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist":100000}, - "file": "deep-100M/faiss_gpu_ivf_flat/nlist100K", - "search_params": [ - {"nprobe":20}, - {"nprobe":30}, - {"nprobe":40}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500}, - {"nprobe":1000} - ] - }, - { - "name": "faiss_gpu_ivf_flat.nlist200K", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist":200000}, - "file": "deep-100M/faiss_gpu_ivf_flat/nlist200K", - "search_params": [ - {"nprobe":20}, - {"nprobe":30}, - {"nprobe":40}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500}, - {"nprobe":1000} - ] - }, - { - "name": "faiss_gpu_ivf_pq.M48-nlist16K", - "algo": "faiss_gpu_ivf_pq", - "build_param": {"nlist":16384, "M":48}, - "file": "deep-100M/faiss_gpu_ivf_pq/M48-nlist16K", - "search_params": [ - {"nprobe":10}, - {"nprobe":20}, - {"nprobe":30}, - {"nprobe":40}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500} - ] - }, - { - "name": "faiss_gpu_ivf_pq.M48-nlist50K", - "algo": "faiss_gpu_ivf_pq", - "build_param": {"nlist":50000, "M":48}, - "file": "deep-100M/faiss_gpu_ivf_pq/M48-nlist50K", - "search_params": [ - {"nprobe":20}, - {"nprobe":30}, - {"nprobe":40}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500}, - {"nprobe":1000} - ] - }, - { - "name": "faiss_gpu_ivf_pq.M48-nlist100K", - "algo": "faiss_gpu_ivf_pq", - "build_param": {"nlist":100000, "M":48}, - "file": "deep-100M/faiss_gpu_ivf_pq/M48-nlist100K", - "search_params": [ - {"nprobe":20}, - {"nprobe":30}, - {"nprobe":40}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500}, - {"nprobe":1000} - ] - }, - { - "name": "faiss_gpu_ivf_pq.M48-nlist200K", - "algo": "faiss_gpu_ivf_pq", - "build_param": {"nlist":200000, "M":48}, - "file": "deep-100M/faiss_gpu_ivf_pq/M48-nlist200K", - "search_params": [ - {"nprobe":20}, - {"nprobe":30}, - {"nprobe":40}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500}, - {"nprobe":1000} - ] - }, - - - { - "name": "raft_ivf_flat.nlist50K", - "algo": "raft_ivf_flat", - "build_param": {"nlist": 50000, "niter": 25, "ratio": 5}, - "file": "deep-100M/raft_ivf_flat/nlist50K", - "search_params": [ - {"max_batch":10000, "max_k":10, "nprobe":20}, - {"max_batch":10000, "max_k":10, "nprobe":30}, - {"max_batch":10000, "max_k":10, "nprobe":40}, - {"max_batch":10000, "max_k":10, "nprobe":50}, - {"max_batch":10000, "max_k":10, "nprobe":100}, - {"max_batch":10000, "max_k":10, "nprobe":200}, - {"max_batch":10000, "max_k":10, "nprobe":500}, - {"max_batch":10000, "max_k":10, "nprobe":1000} - ] - }, - { - "name": "raft_ivf_flat.nlist100K", - "algo": "raft_ivf_flat", - "build_param": {"nlist": 100000, "niter": 25, "ratio": 5}, - "file": "deep-100M/raft_ivf_flat/nlist100K", - "search_params": [ - {"max_batch":10000, "max_k":10, "nprobe":20}, - {"max_batch":10000, "max_k":10, "nprobe":30}, - {"max_batch":10000, "max_k":10, "nprobe":40}, - {"max_batch":10000, "max_k":10, "nprobe":50}, - {"max_batch":10000, "max_k":10, "nprobe":100}, - {"max_batch":10000, "max_k":10, "nprobe":200}, - {"max_batch":10000, "max_k":10, "nprobe":500}, - {"max_batch":10000, "max_k":10, "nprobe":1000} - ] - }, - { - "name": "raft_ivf_flat.nlist200K", - "algo": "raft_ivf_flat", - "build_param": {"nlist": 200000, "niter": 25, "ratio": 5}, - "file": "deep-100M/raft_ivf_flat/nlist200K", - "search_params": [ - {"max_batch":10000, "max_k":10, "nprobe":20}, - {"max_batch":10000, "max_k":10, "nprobe":30}, - {"max_batch":10000, "max_k":10, "nprobe":40}, - {"max_batch":10000, "max_k":10, "nprobe":50}, - {"max_batch":10000, "max_k":10, "nprobe":100}, - {"max_batch":10000, "max_k":10, "nprobe":200}, - {"max_batch":10000, "max_k":10, "nprobe":500}, - {"max_batch":10000, "max_k":10, "nprobe":1000} - ] - }, -{ - "name": "raft_ivf_pq.d96b5n50K", - "algo": "raft_ivf_pq", - "build_param": {"nlist": 50000, "pq_dim": 96, "pq_bits": 5, "ratio": 10, "niter": 25}, - "file": "deep-100M/raft_ivf_pq/d96b5n50K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 2000, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 5000, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 2 }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 2000, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 5000, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 2000, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 5000, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 2000, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 5000, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 2 }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 2000, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 }, - { "nprobe": 5000, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 2 } - ] - }, - { - "name": "raft_ivf_pq.d64b5n50K", - "algo": "raft_ivf_pq", - "build_param": {"nlist": 50000, "pq_dim": 64, "pq_bits": 5, "ratio": 10, "niter": 25}, - "file": "deep-100M/raft_ivf_pq/d64b5n50K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 2000, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 5000, "internalDistanceDtype": "float", "smemLutDtype": "float", "refine_ratio": 4 }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 2000, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 5000, "internalDistanceDtype": "float", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 2000, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 5000, "internalDistanceDtype": "float", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 2000, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 5000, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 2000, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 }, - { "nprobe": 5000, "internalDistanceDtype": "half", "smemLutDtype": "fp8", "refine_ratio": 4 } - ] - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-angular/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "float"} - ], - "search_result_file": "result/deep-image-96-angular/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_cagra.dim32", - "algo": "raft_cagra", - "build_param": {"graph_degree": 32, "intermediate_graph_degree": 48}, - "file": "deep-100M/raft_cagra/dim32", - "search_params": [ - {"itopk": 32, "search_width": 1, "max_iterations": 0, "algo": "single_cta"}, - {"itopk": 32, "search_width": 1, "max_iterations": 32, "algo": "single_cta"}, - {"itopk": 64, "search_width": 4, "max_iterations": 16, "algo": "single_cta"}, - {"itopk": 64, "search_width": 1, "max_iterations": 64, "algo": "single_cta"}, - {"itopk": 96, "search_width": 2, "max_iterations": 48, "algo": "single_cta"}, - {"itopk": 128, "search_width": 8, "max_iterations": 16, "algo": "single_cta"}, - {"itopk": 128, "search_width": 2, "max_iterations": 64, "algo": "single_cta"}, - {"itopk": 192, "search_width": 8, "max_iterations": 24, "algo": "single_cta"}, - {"itopk": 192, "search_width": 2, "max_iterations": 96, "algo": "single_cta"}, - {"itopk": 256, "search_width": 8, "max_iterations": 32, "algo": "single_cta"}, - {"itopk": 384, "search_width": 8, "max_iterations": 48, "algo": "single_cta"}, - {"itopk": 512, "search_width": 8, "max_iterations": 64, "algo": "single_cta"} - ] - }, - { - "name": "raft_cagra.dim32.multi_cta", - "algo": "raft_cagra", - "build_param": {"graph_degree": 32, "intermediate_graph_degree": 48}, - "file": "deep-100M/raft_cagra/dim32", - "search_params": [ - {"itopk": 32, "search_width": 1, "max_iterations": 0, "algo": "multi_cta"}, - {"itopk": 32, "search_width": 1, "max_iterations": 32, "algo": "multi_cta"}, - {"itopk": 64, "search_width": 4, "max_iterations": 16, "algo": "multi_cta"}, - {"itopk": 64, "search_width": 1, "max_iterations": 64, "algo": "multi_cta"}, - {"itopk": 96, "search_width": 2, "max_iterations": 48, "algo": "multi_cta"}, - {"itopk": 128, "search_width": 8, "max_iterations": 16, "algo": "multi_cta"}, - {"itopk": 128, "search_width": 2, "max_iterations": 64, "algo": "multi_cta"}, - {"itopk": 192, "search_width": 8, "max_iterations": 24, "algo": "multi_cta"}, - {"itopk": 192, "search_width": 2, "max_iterations": 96, "algo": "multi_cta"}, - {"itopk": 256, "search_width": 8, "max_iterations": 32, "algo": "multi_cta"}, - {"itopk": 384, "search_width": 8, "max_iterations": 48, "algo": "multi_cta"}, - {"itopk": 512, "search_width": 8, "max_iterations": 64, "algo": "multi_cta"} - ] - }, - { - "name": "raft_cagra.dim32.multi_kernel", - "algo": "raft_cagra", - "build_param": {"graph_degree": 32, "intermediate_graph_degree": 48}, - "file": "deep-100M/raft_cagra/dim32", - "search_params": [ - {"itopk": 32, "search_width": 1, "max_iterations": 0, "algo": "multi_kernel"}, - {"itopk": 32, "search_width": 1, "max_iterations": 32, "algo": "multi_kernel"}, - {"itopk": 64, "search_width": 4, "max_iterations": 16, "algo": "multi_kernel"}, - {"itopk": 64, "search_width": 1, "max_iterations": 64, "algo": "multi_kernel"}, - {"itopk": 96, "search_width": 2, "max_iterations": 48, "algo": "multi_kernel"}, - {"itopk": 128, "search_width": 8, "max_iterations": 16, "algo": "multi_kernel"}, - {"itopk": 128, "search_width": 2, "max_iterations": 64, "algo": "multi_kernel"}, - {"itopk": 192, "search_width": 8, "max_iterations": 24, "algo": "multi_kernel"}, - {"itopk": 192, "search_width": 2, "max_iterations": 96, "algo": "multi_kernel"}, - {"itopk": 256, "search_width": 8, "max_iterations": 32, "algo": "multi_kernel"}, - {"itopk": 384, "search_width": 8, "max_iterations": 48, "algo": "multi_kernel"}, - {"itopk": 512, "search_width": 8, "max_iterations": 64, "algo": "multi_kernel"} - ] - }, - { - "name": "raft_cagra.dim64", - "algo": "raft_cagra", - "build_param": {"graph_degree": 64}, - "file": "deep-100M/raft_cagra/dim64", - "search_params": [ - {"itopk": 32, "search_width": 1, "max_iterations": 0}, - {"itopk": 32, "search_width": 1, "max_iterations": 32}, - {"itopk": 64, "search_width": 4, "max_iterations": 16}, - {"itopk": 64, "search_width": 1, "max_iterations": 64}, - {"itopk": 96, "search_width": 2, "max_iterations": 48}, - {"itopk": 128, "search_width": 8, "max_iterations": 16}, - {"itopk": 128, "search_width": 2, "max_iterations": 64}, - {"itopk": 192, "search_width": 8, "max_iterations": 24}, - {"itopk": 192, "search_width": 2, "max_iterations": 96}, - {"itopk": 256, "search_width": 8, "max_iterations": 32}, - {"itopk": 384, "search_width": 8, "max_iterations": 48}, - {"itopk": 512, "search_width": 8, "max_iterations": 64} - ] - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-1B.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-1B.json deleted file mode 100644 index e5190e073e..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-1B.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "dataset": { - "name": "deep-1B", - "base_file": "deep-1B/base.1B.fbin", - "query_file": "deep-1B/query.public.10K.fbin", - "groundtruth_neighbors_file": "deep-1B/groundtruth.neighbors.ibin", - "distance": "inner_product" - }, - - "search_basic_param": { - "batch_size": 10000, - "k": 10 - }, - - "index": [ - { - "name": "faiss_gpu_ivf_pq.M48-nlist50K", - "algo": "faiss_gpu_ivf_pq", - "build_param": {"nlist":50000, "M":48}, - "file": "deep-1B/faiss_gpu_ivf_pq/M48-nlist50K", - "search_params": [ - {"nprobe":1}, - {"nprobe":5}, - {"nprobe":10}, - {"nprobe":50}, - {"nprobe":100}, - {"nprobe":200}, - {"nprobe":500}, - {"nprobe":1000}, - {"nprobe":2000} - ] - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-image-96-inner.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-image-96-inner.json deleted file mode 100644 index 3d69e775a1..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/deep-image-96-inner.json +++ /dev/null @@ -1,1013 +0,0 @@ -{ - "dataset": { - "name": "deep-image-96-inner", - "base_file": "deep-image-96-inner/base.fbin", - "query_file": "deep-image-96-inner/query.fbin", - "groundtruth_neighbors_file": "deep-image-96-inner/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/deep-image-96-inner/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/deep-image-96-inner/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/deep-image-96-inner/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/deep-image-96-inner/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/deep-image-96-inner/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/deep-image-96-inner/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/deep-image-96-inner/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/deep-image-96-inner/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - - "build_param": {}, - "file": "index/deep-image-96-inner/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/deep-image-96-inner/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/deep-image-96-inner/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/deep-image-96-inner/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/deep-image-96-inner/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - - "build_param": {"nlist": 1024, "pq_dim": 128, "ratio": 1, "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "half", "smemLutDtype": "half"}, - {"nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half"}, - {"nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half"}, - {"nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half"}, - {"nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half"}, - {"nprobe": 1024, "internalDistanceDtype": "half", "smemLutDtype": "half"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - {"nprobe": 1, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 5, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "float"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "half"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "fp8"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "fp8"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "half"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "half"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "fp8"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "fp8"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "half", "smemLutDtype": "float"}, - {"nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "float"}, - {"nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "float"}, - {"nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "float"}, - {"nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "float"}, - {"nprobe": 1024, "internalDistanceDtype": "half", "smemLutDtype": "float"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - {"nprobe": 10, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "float"}, - {"nprobe": 1024, "internalDistanceDtype": "float", "smemLutDtype": "float"} - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/deep-image-96-inner/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/deep-image-96-inner/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/deep-image-96-inner/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/deep-image-96-inner/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/deep-image-96-inner/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/deep-image-96-inner/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/deep-image-96-inner/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/fashion-mnist-784-euclidean.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/fashion-mnist-784-euclidean.json deleted file mode 100644 index 2c86b0c4ee..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/fashion-mnist-784-euclidean.json +++ /dev/null @@ -1,1352 +0,0 @@ -{ - "dataset": { - "name": "fashion-mnist-784-euclidean", - "base_file": "fashion-mnist-784-euclidean/base.fbin", - "query_file": "fashion-mnist-784-euclidean/query.fbin", - "groundtruth_neighbors_file": "fashion-mnist-784-euclidean/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/fashion-mnist-784-euclidean/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/fashion-mnist-784-euclidean/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/fashion-mnist-784-euclidean/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/fashion-mnist-784-euclidean/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/fashion-mnist-784-euclidean/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/fashion-mnist-784-euclidean/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/fashion-mnist-784-euclidean/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/fashion-mnist-784-euclidean/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/fashion-mnist-784-euclidean/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/fashion-mnist-784-euclidean/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/fashion-mnist-784-euclidean/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/fashion-mnist-784-euclidean/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/fashion-mnist-784-euclidean/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/fashion-mnist-784-euclidean/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/fashion-mnist-784-euclidean/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/fashion-mnist-784-euclidean/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/fashion-mnist-784-euclidean/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/gist-960-euclidean.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/gist-960-euclidean.json deleted file mode 100644 index c5480900a7..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/gist-960-euclidean.json +++ /dev/null @@ -1,1351 +0,0 @@ -{ - "dataset": { - "name": "gist-960-euclidean", - "base_file": "gist-960-euclidean/base.fbin", - "query_file": "gist-960-euclidean/query.fbin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/gist-960-euclidean/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/gist-960-euclidean/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/gist-960-euclidean/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/gist-960-euclidean/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/gist-960-euclidean/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/gist-960-euclidean/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/gist-960-euclidean/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/gist-960-euclidean/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/gist-960-euclidean/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/gist-960-euclidean/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/gist-960-euclidean/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/gist-960-euclidean/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/gist-960-euclidean/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/gist-960-euclidean/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/gist-960-euclidean/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/gist-960-euclidean/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/gist-960-euclidean/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/gist-960-euclidean/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/gist-960-euclidean/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/gist-960-euclidean/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-angular.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-angular.json deleted file mode 100644 index 2074ef13a3..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-angular.json +++ /dev/null @@ -1,1351 +0,0 @@ -{ - "dataset": { - "name": "glove-100-angular", - "base_file": "glove-100-angular/base.fbin", - "query_file": "glove-100-angular/query.fbin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-angular/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-angular/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-angular/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-angular/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-angular/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-angular/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-angular/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-angular/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/glove-100-angular/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/glove-100-angular/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/glove-100-angular/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/glove-100-angular/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/glove-100-angular/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-angular/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/glove-100-angular/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-angular/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/glove-100-angular/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-100-angular/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/glove-100-angular/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-100-angular/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-inner.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-inner.json deleted file mode 100644 index 5da3fa18d3..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-100-inner.json +++ /dev/null @@ -1,1314 +0,0 @@ -{ - "dataset": { - "name": "glove-100-inner", - "base_file": "glove-100-inner/base.fbin", - "query_file": "glove-100-inner/query.fbin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-inner/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-inner/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-inner/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-inner/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-inner/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-inner/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-100-inner/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-100-inner/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/glove-100-inner/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/glove-100-inner/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist":1024}, - "file": "glove-100-inner/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist":2048}, - "file": "glove-100-inner/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist":4096}, - "file": "glove-100-inner/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist":8192}, - "file": "glove-100-inner/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/glove-100-inner/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/glove-100-inner/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/glove-100-inner/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/glove-100-inner/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":2048, "quantizer_type":"fp16"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":4096, "quantizer_type":"fp16"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":8192, "quantizer_type":"fp16"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":16384, "quantizer_type":"fp16"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":1024, "quantizer_type":"int8"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":2048, "quantizer_type":"int8"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":4096, "quantizer_type":"int8"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":8192, "quantizer_type":"int8"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist":16384, "quantizer_type":"int8"}, - "file": "glove-100-inner/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-inner/faiss_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "glove-100-inner/faiss_gpu_flat/flat", - "search_params": [{}], - "search_result_file": "result/glove-100-inner/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-100-inner/raft_gpu_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-100-inner/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/glove-100-inner/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-100-inner/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/glove-100-inner/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-100-inner/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/glove-100-inner/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-100-inner/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-50-angular.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-50-angular.json deleted file mode 100644 index 11fa07c5c9..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-50-angular.json +++ /dev/null @@ -1,1351 +0,0 @@ -{ - "dataset": { - "name": "glove-50-angular", - "base_file": "glove-50-angular/base.fbin", - "query_file": "glove-50-angular/query.fbin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-angular/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-angular/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-angular/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-angular/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-angular/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-angular/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-angular/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-angular/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/glove-50-angular/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/glove-50-angular/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/glove-50-angular/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/glove-50-angular/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/glove-50-angular/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-angular/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/glove-50-angular/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-angular/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/glove-50-angular/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-50-angular/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/glove-50-angular/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-50-angular/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-50-inner.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-50-inner.json deleted file mode 100644 index 32613b7c16..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/glove-50-inner.json +++ /dev/null @@ -1,1351 +0,0 @@ -{ - "dataset": { - "name": "glove-50-inner", - "base_file": "glove-50-inner/base.fbin", - "query_file": "glove-50-inner/query.fbin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-inner/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-inner/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-inner/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-inner/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-inner/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-inner/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/glove-50-inner/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/glove-50-inner/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/glove-50-inner/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/glove-50-inner/raft_bfknn/bfknn" - }, - { - "name": "faiss_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/glove-50-inner/faiss_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_flat/nlist1024" - }, - { - "name": "faiss_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/glove-50-inner/faiss_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_flat/nlist2048" - }, - { - "name": "faiss_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/glove-50-inner/faiss_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_flat/nlist4096" - }, - { - "name": "faiss_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/glove-50-inner/faiss_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_flat/nlist8192" - }, - { - "name": "faiss_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/glove-50-inner/faiss_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_flat/nlist16384" - }, - { - "name": "faiss_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/glove-50-inner/faiss_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/glove-50-inner/faiss_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/glove-50-inner/faiss_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-inner/faiss_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/glove-50-inner/faiss_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/glove-50-inner/faiss_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "nprobe": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "nprobe": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/glove-50-inner/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/glove-50-inner/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/glove-50-inner/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/glove-50-inner/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-50-inner/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/glove-50-inner/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/glove-50-inner/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/lastfm-65-angular.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/lastfm-65-angular.json deleted file mode 100644 index 943d09231a..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/lastfm-65-angular.json +++ /dev/null @@ -1,1351 +0,0 @@ -{ - "dataset": { - "name": "lastfm-65-angular", - "base_file": "lastfm-65-angular/base.fbin", - "query_file": "lastfm-65-angular/query.fbin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/lastfm-65-angular/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/lastfm-65-angular/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/lastfm-65-angular/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/lastfm-65-angular/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/lastfm-65-angular/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/lastfm-65-angular/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/lastfm-65-angular/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/lastfm-65-angular/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/lastfm-65-angular/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/lastfm-65-angular/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/lastfm-65-angular/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/lastfm-65-angular/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/lastfm-65-angular/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/lastfm-65-angular/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/lastfm-65-angular/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/lastfm-65-angular/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/lastfm-65-angular/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/lastfm-65-angular/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/lastfm-65-angular/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/lastfm-65-angular/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/mnist-784-euclidean.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/mnist-784-euclidean.json deleted file mode 100644 index 04e7ecb469..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/mnist-784-euclidean.json +++ /dev/null @@ -1,1352 +0,0 @@ -{ - "dataset": { - "name": "mnist-784-euclidean", - "base_file": "mnist-784-euclidean/base.fbin", - "query_file": "mnist-784-euclidean/query.fbin", - "groundtruth_neighbors_file": "mnist-784-euclidean/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/mnist-784-euclidean/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/mnist-784-euclidean/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/mnist-784-euclidean/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/mnist-784-euclidean/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/mnist-784-euclidean/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/mnist-784-euclidean/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/mnist-784-euclidean/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/mnist-784-euclidean/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/mnist-784-euclidean/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/mnist-784-euclidean/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/mnist-784-euclidean/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/mnist-784-euclidean/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/mnist-784-euclidean/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/mnist-784-euclidean/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/mnist-784-euclidean/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/mnist-784-euclidean/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/mnist-784-euclidean/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/mnist-784-euclidean/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/nytimes-256-angular.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/nytimes-256-angular.json deleted file mode 100644 index df2a16f1f8..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/nytimes-256-angular.json +++ /dev/null @@ -1,1352 +0,0 @@ -{ - "dataset": { - "name": "nytimes-256-angular", - "base_file": "nytimes-256-angular/base.fbin", - "query_file": "nytimes-256-angular/query.fbin", - "groundtruth_neighbors_file": "nytimes-256-angular/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-angular/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-angular/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-angular/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-angular/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-angular/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-angular/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-angular/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-angular/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/nytimes-256-angular/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/nytimes-256-angular/raft_bfknn/bfknn" - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_flat/nlist1024" - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_flat/nlist2048" - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_flat/nlist4096" - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_flat/nlist8192" - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_flat/nlist16384" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-angular/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/nytimes-256-angular/faiss_gpu_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/nytimes-256-angular/faiss_gpu_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-angular/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/nytimes-256-angular/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-angular/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/nytimes-256-angular/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/nytimes-256-angular/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/nytimes-256-angular/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/nytimes-256-angular/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/nytimes-256-inner.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/nytimes-256-inner.json deleted file mode 100644 index 18942a95c3..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/nytimes-256-inner.json +++ /dev/null @@ -1,1352 +0,0 @@ -{ - "dataset": { - "name": "nytimes-256-inner", - "base_file": "nytimes-256-inner/base.fbin", - "query_file": "nytimes-256-inner/query.fbin", - "groundtruth_neighbors_file": "nytimes-256-inner/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 5000, - "k": 10, - "run_count": 3 - }, - "index": [ - { - "name" : "hnswlib.M12", - "algo" : "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-inner/hnswlib/M12", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-inner/hnswlib/M12" - }, - { - "name" : "hnswlib.M16", - "algo" : "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-inner/hnswlib/M16", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-inner/hnswlib/M16" - }, - { - "name" : "hnswlib.M24", - "algo" : "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-inner/hnswlib/M24", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-inner/hnswlib/M24" - }, - { - "name" : "hnswlib.M36", - "algo" : "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file" : "index/nytimes-256-inner/hnswlib/M36", - "search_params" : [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ], - "search_result_file" : "result/nytimes-256-inner/hnswlib/M36" - }, - - - - - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "index/nytimes-256-inner/raft_bfknn/bfknn", - "search_params": [ - { - "probe": 1 - } - ], - "search_result_file": "result/nytimes-256-inner/raft_bfknn/bfknn" - }, - { - "name": "faiss_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 1024 - }, - "file": "index/nytimes-256-inner/faiss_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_flat/nlist1024" - }, - { - "name": "faiss_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 2048 - }, - "file": "index/nytimes-256-inner/faiss_ivf_flat/nlist2048", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_flat/nlist2048" - }, - { - "name": "faiss_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 4096 - }, - "file": "index/nytimes-256-inner/faiss_ivf_flat/nlist4096", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_flat/nlist4096" - }, - { - "name": "faiss_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 8192 - }, - "file": "index/nytimes-256-inner/faiss_ivf_flat/nlist8192", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_flat/nlist8192" - }, - { - "name": "faiss_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": { - "nlist": 16384 - }, - "file": "index/nytimes-256-inner/faiss_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_flat/nlist16384" - }, - { - "name": "faiss_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": true - }, - "file": "index/nytimes-256-inner/faiss_ivf_pq/M64-nlist1024", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "index/nytimes-256-inner/faiss_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_pq/M64-nlist1024" - }, - { - "name": "faiss_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist1024-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist1024-fp16" - }, - { - "name": "faiss_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist2048-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist2048-fp16" - }, - { - "name": "faiss_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist4096-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist4096-fp16" - }, - { - "name": "faiss_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist8192-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist8192-fp16" - }, - { - "name": "faiss_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "fp16" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist16384-fp16", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist16384-fp16" - }, - { - "name": "faiss_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 1024, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist1024-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist1024-int8" - }, - { - "name": "faiss_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 2048, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist2048-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist2048-int8" - }, - { - "name": "faiss_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 4096, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist4096-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist4096-int8" - }, - { - "name": "faiss_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 8192, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist8192-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist8192-int8" - }, - { - "name": "faiss_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": { - "nlist": 16384, - "quantizer_type": "int8" - }, - "file": "index/nytimes-256-inner/faiss_ivf_sq/nlist16384-int8", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-inner/faiss_ivf_sq/nlist16384-int8" - }, - { - "name": "faiss_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "index/nytimes-256-inner/faiss_flat/flat", - "search_params": [ - {} - ], - "search_result_file": "result/nytimes-256-inner/faiss_flat/flat" - }, - - { - "name": "raft_ivf_pq.dimpq128-cluster1024", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 5, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-float-float" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq64-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq64-cluster1024-float-half", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 64, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq64-cluster1024-float-half", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "half" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq64-cluster1024-float-half" - }, - { - "name": "raft_ivf_pq.dimpq32-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 32, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq32-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq16-cluster1024-float-fp8", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 16, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "fp8" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq16-cluster1024-float-fp8" - }, - { - "name": "raft_ivf_pq.dimpq128-cluster1024-half-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 128, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-half-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "half", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq128-cluster1024-half-float" - }, - { - "name": "raft_ivf_pq.dimpq512-cluster1024-float-float", - "algo": "raft_ivf_pq", - "build_param": { - "nlist": 1024, - "pq_dim": 512, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_pq/dimpq512-cluster1024-float-float", - "search_params": [ - { - "k": 10, - "numProbes": 10, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 50, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 100, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 200, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 500, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - }, - { - "k": 10, - "numProbes": 1024, - "internalDistanceDtype": "float", - "smemLutDtype": "float" - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_pq/dimpq512-cluster1024-float-float" - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 1024, - "ratio": 1, - "niter": 25 - }, - "file": "index/nytimes-256-inner/raft_ivf_flat/nlist1024", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_flat/nlist1024" - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": { - "nlist": 16384, - "ratio": 2, - "niter": 20 - }, - "file": "index/nytimes-256-inner/raft_ivf_flat/nlist16384", - "search_params": [ - { - "nprobe": 1 - }, - { - "nprobe": 5 - }, - { - "nprobe": 10 - }, - { - "nprobe": 50 - }, - { - "nprobe": 100 - }, - { - "nprobe": 200 - }, - { - "nprobe": 500 - }, - { - "nprobe": 1000 - }, - { - "nprobe": 2000 - } - ], - "search_result_file": "result/nytimes-256-inner/raft_ivf_flat/nlist16384" - }, - - { - "name" : "raft_cagra.dim32", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 32 - }, - "file" : "index/nytimes-256-inner/raft_cagra/dim32", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/nytimes-256-inner/raft_cagra/dim32" - }, - - { - "name" : "raft_cagra.dim64", - "algo" : "raft_cagra", - "build_param": { - "graph_degree" : 64 - }, - "file" : "index/nytimes-256-inner/raft_cagra/dim64", - "search_params" : [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ], - "search_result_file" : "result/nytimes-256-inner/raft_cagra/dim64" - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/sift-128-euclidean.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/sift-128-euclidean.json deleted file mode 100644 index 791261251a..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/sift-128-euclidean.json +++ /dev/null @@ -1,498 +0,0 @@ -{ - "dataset": { - "name": "sift-128-euclidean", - "base_file": "sift-128-euclidean/base.fbin", - "query_file": "sift-128-euclidean/query.fbin", - "groundtruth_neighbors_file": "sift-128-euclidean/groundtruth.neighbors.ibin", - "distance": "euclidean" - }, - - "search_basic_param": { - "batch_size": 5000, - "k": 10 - }, - - "index": [ - { - "name": "hnswlib.M12", - "algo": "hnswlib", - "build_param": {"M":12, "efConstruction":500, "numThreads":32}, - "file": "sift-128-euclidean/hnswlib/M12", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M16", - "algo": "hnswlib", - "build_param": {"M":16, "efConstruction":500, "numThreads":32}, - "file": "sift-128-euclidean/hnswlib/M16", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M24", - "algo": "hnswlib", - "build_param": {"M":24, "efConstruction":500, "numThreads":32}, - "file": "sift-128-euclidean/hnswlib/M24", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "hnswlib.M36", - "algo": "hnswlib", - "build_param": {"M":36, "efConstruction":500, "numThreads":32}, - "file": "sift-128-euclidean/hnswlib/M36", - "search_params": [ - {"ef":10}, - {"ef":20}, - {"ef":40}, - {"ef":60}, - {"ef":80}, - {"ef":120}, - {"ef":200}, - {"ef":400}, - {"ef":600}, - {"ef":800} - ] - }, - { - "name": "raft_bfknn", - "algo": "raft_bfknn", - "build_param": {}, - "file": "sift-128-euclidean/raft_bfknn/bfknn", - "search_params": [{"probe": 1}] - }, - { - "name": "faiss_gpu_ivf_flat.nlist1024", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist": 1024}, - "file": "sift-128-euclidean/faiss_gpu_ivf_flat/nlist1024", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_flat.nlist2048", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist": 2048}, - "file": "sift-128-euclidean/faiss_gpu_ivf_flat/nlist2048", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_flat.nlist4096", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist": 4096}, - "file": "sift-128-euclidean/faiss_gpu_ivf_flat/nlist4096", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_flat.nlist8192", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist": 8192}, - "file": "sift-128-euclidean/faiss_gpu_ivf_flat/nlist8192", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_flat.nlist16384", - "algo": "faiss_gpu_ivf_flat", - "build_param": {"nlist": 16384}, - "file": "sift-128-euclidean/faiss_gpu_ivf_flat/nlist16384", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000}, - {"nprobe": 2000} - ] - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024", - "algo": "faiss_gpu_ivf_pq", - "build_param": {"nlist": 1024, "M": 64, "useFloat16": true, "usePrecomputed": true}, - "file": "sift-128-euclidean/faiss_gpu_ivf_pq/M64-nlist1024", - "search_params": [ - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_pq.M64-nlist1024.noprecomp", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "nlist": 1024, - "M": 64, - "useFloat16": true, - "usePrecomputed": false - }, - "file": "sift-128-euclidean/faiss_gpu_ivf_pq/M64-nlist1024.noprecomp", - "search_params": [ - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 1024, "quantizer_type": "fp16"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist1024-fp16", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 2048, "quantizer_type": "fp16"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist2048-fp16", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 4096, "quantizer_type": "fp16"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist4096-fp16", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 8192, "quantizer_type": "fp16"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist8192-fp16", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-fp16", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 16384, "quantizer_type": "fp16"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist16384-fp16", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000}, - {"nprobe": 2000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist1024-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 1024, "quantizer_type": "int8"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist1024-int8", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist2048-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 2048,"quantizer_type": "int8"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist2048-int8", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist4096-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 4096, "quantizer_type": "int8"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist4096-int8", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist8192-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 8192, "quantizer_type": "int8"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist8192-int8", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "faiss_gpu_ivf_sq.nlist16384-int8", - "algo": "faiss_gpu_ivf_sq", - "build_param": {"nlist": 16384, "quantizer_type": "int8"}, - "file": "sift-128-euclidean/faiss_gpu_ivf_sq/nlist16384-int8", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000}, - {"nprobe": 2000} - ] - }, - { - "name": "faiss_gpu_flat", - "algo": "faiss_gpu_flat", - "build_param": {}, - "file": "sift-128-euclidean/faiss_gpu_flat/flat", - "search_params": [{}] - }, - { - "name": "raft_ivf_pq.dimpq64-bitpq8-cluster1K", - "algo": "raft_ivf_pq", - "build_param": {"niter": 25, "nlist": 1000, "pq_dim": 64, "pq_bits": 8, "ratio": 1}, - "file": "sift-128-euclidean/raft_ivf_pq/dimpq64-bitpq8-cluster1K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "half" } - ] - }, - { - "name": "raft_ivf_pq.dimpq128-bitpq6-cluster1K", - "algo": "raft_ivf_pq", - "build_param": {"niter": 25, "nlist": 1000, "pq_dim": 128, "pq_bits": 6, "ratio": 1}, - "file": "sift-128-euclidean/raft_ivf_pq/dimpq128-bitpq6-cluster1K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "float" }, - { "nprobe": 20, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 30, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 40, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 50, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 100, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 200, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 500, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 1000, "internalDistanceDtype": "float", "smemLutDtype": "fp8" }, - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half" }, - { "nprobe": 1000, "internalDistanceDtype": "half", "smemLutDtype": "half" } - ] - }, - { - "name": "raft_ivf_flat.nlist1024", - "algo": "raft_ivf_flat", - "build_param": {"nlist": 1024, "ratio": 1, "niter": 25}, - "file": "sift-128-euclidean/raft_ivf_flat/nlist1024", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000} - ] - }, - { - "name": "raft_ivf_flat.nlist16384", - "algo": "raft_ivf_flat", - "build_param": {"nlist": 16384, "ratio": 2, "niter": 20}, - "file": "sift-128-euclidean/raft_ivf_flat/nlist16384", - "search_params": [ - {"nprobe": 1}, - {"nprobe": 5}, - {"nprobe": 10}, - {"nprobe": 50}, - {"nprobe": 100}, - {"nprobe": 200}, - {"nprobe": 500}, - {"nprobe": 1000}, - {"nprobe": 2000} - ] - }, - { - "name": "raft_cagra.dim32", - "algo": "raft_cagra", - "build_param": {"graph_degree": 32}, - "file": "sift-128-euclidean/raft_cagra/dim32", - "search_params": [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ] - }, - { - "name": "raft_cagra.dim64", - "algo": "raft_cagra", - "build_param": {"graph_degree": 64}, - "file": "sift-128-euclidean/raft_cagra/dim64", - "search_params": [ - {"itopk": 32}, - {"itopk": 64}, - {"itopk": 128} - ] - } - ] -} diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_10M.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_10M.json deleted file mode 100644 index e5f77e7858..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_10M.json +++ /dev/null @@ -1,200 +0,0 @@ -{ - "dataset": { - "name": "wiki_all_10M", - "base_file": "wiki_all_10M/base.88M.fbin", - "query_file": "wiki_all_10M/queries.fbin", - "groundtruth_neighbors_file": "wiki_all_10M/groundtruth.88M.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 10000, - "k": 10 - }, - "index": [ - { - "name": "hnswlib.M16.ef50", - "algo": "hnswlib", - "build_param": { "M": 16, "efConstruction": 50, "numThreads": 56 }, - "file": "wiki_all_10M/hnswlib/M16.ef50", - "search_params": [ - { "ef": 10, "numThreads": 56 }, - { "ef": 20, "numThreads": 56 }, - { "ef": 40, "numThreads": 56 }, - { "ef": 60, "numThreads": 56 }, - { "ef": 80, "numThreads": 56 }, - { "ef": 120, "numThreads": 56 }, - { "ef": 200, "numThreads": 56 }, - { "ef": 400, "numThreads": 56 }, - { "ef": 600, "numThreads": 56 }, - { "ef": 800, "numThreads": 56 } - ] - }, - { - "name": "faiss_ivf_pq.M32-nlist16K", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "M": 32, - "nlist": 16384, - "ratio": 2 - }, - "file": "wiki_all_10M/faiss_ivf_pq/M32-nlist16K_ratio2", - "search_params": [ - { "nprobe": 10 }, - { "nprobe": 20 }, - { "nprobe": 30 }, - { "nprobe": 40 }, - { "nprobe": 50 }, - { "nprobe": 100 }, - { "nprobe": 200 }, - { "nprobe": 500 } - ] - }, - { - "name": "faiss_ivf_pq.M64-nlist16K", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "M": 64, - "nlist": 16384, - "ratio": 2 - }, - "file": "wiki_all_10M/faiss_ivf_pq/M64-nlist16K_ratio2", - "search_params": [ - { "nprobe": 10 }, - { "nprobe": 20 }, - { "nprobe": 30 }, - { "nprobe": 40 }, - { "nprobe": 50 }, - { "nprobe": 100 }, - { "nprobe": 200 }, - { "nprobe": 500 } - ] - }, - { - "name": "raft_ivf_pq.d128-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 128, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_10M/raft_ivf_pq/d128-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 } - ] - }, - { - "name": "raft_ivf_pq.d64-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 64, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_10M/raft_ivf_pq/d64-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 } - ] - }, - { - "name": "raft_ivf_pq.d32-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 32, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_10M/raft_ivf_pq/d32-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 } - ] - }, - { - "name": "raft_ivf_pq.d32X-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 32, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_10M/raft_ivf_pq/d32-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 } - - ] - }, - { - "name": "raft_cagra.dim32.multi_cta", - "algo": "raft_cagra", - "build_param": { "graph_degree": 32, "intermediate_graph_degree": 48 }, - "file": "wiki_all_10M/raft_cagra/dim32.ibin", - "search_params": [ - { "itopk": 32, "search_width": 1, "max_iterations": 0, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 36, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 40, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 44, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 24, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 26, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 64, "search_width": 4, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 64, "search_width": 1, "max_iterations": 64, "algo": "multi_cta" }, - { "itopk": 96, "search_width": 2, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 128, "search_width": 8, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 128, "search_width": 2, "max_iterations": 64, "algo": "multi_cta" }, - { "itopk": 192, "search_width": 8, "max_iterations": 24, "algo": "multi_cta" }, - { "itopk": 192, "search_width": 2, "max_iterations": 96, "algo": "multi_cta" }, - { "itopk": 256, "search_width": 8, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 384, "search_width": 8, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 512, "search_width": 8, "max_iterations": 64, "algo": "multi_cta" } - ] - } - - ] -} - diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_1M.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_1M.json deleted file mode 100644 index 2d1ec1e322..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_1M.json +++ /dev/null @@ -1,216 +0,0 @@ -{ - "dataset": { - "name": "wiki_all_1M", - "base_file": "wiki_all_1M/base.1M.fbin", - "subset_size": 1000000, - "query_file": "wiki_all_1M/queries.fbin", - "groundtruth_neighbors_file": "wiki_all_1M/groundtruth.1M.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 10000, - "k": 10 - }, - "index": [ - { - "name": "hnswlib.M16.ef50", - "algo": "hnswlib", - "build_param": { "M": 16, "efConstruction": 50, "numThreads": 56 }, - "file": "wiki_all_1M/hnswlib/M16.ef50", - "search_params": [ - { "ef": 10, "numThreads": 56 }, - { "ef": 20, "numThreads": 56 }, - { "ef": 40, "numThreads": 56 }, - { "ef": 60, "numThreads": 56 }, - { "ef": 80, "numThreads": 56 }, - { "ef": 120, "numThreads": 56 }, - { "ef": 200, "numThreads": 56 }, - { "ef": 400, "numThreads": 56 }, - { "ef": 600, "numThreads": 56 }, - { "ef": 800, "numThreads": 56 } - ] - }, - { - "name": "faiss_ivf_pq.M32-nlist16K", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "M": 32, - "nlist": 16384, - "ratio": 2 - }, - "file": "wiki_all_1M/faiss_ivf_pq/M32-nlist16K_ratio2", - "search_params": [ - { "nprobe": 10 }, - { "nprobe": 20 }, - { "nprobe": 30 }, - { "nprobe": 40 }, - { "nprobe": 50 }, - { "nprobe": 100 }, - { "nprobe": 200 }, - { "nprobe": 500 } - ] - }, - { - "name": "faiss_ivf_pq.M64-nlist16K", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "M": 64, - "nlist": 16384, - "ratio": 2 - }, - "file": "wiki_all_1M/faiss_ivf_pq/M64-nlist16K_ratio2", - "search_params": [ - { "nprobe": 10 }, - { "nprobe": 20 }, - { "nprobe": 30 }, - { "nprobe": 40 }, - { "nprobe": 50 }, - { "nprobe": 100 }, - { "nprobe": 200 }, - { "nprobe": 500 } - ] - }, - { - "name": "raft_ivf_pq.d128-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 128, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_1M/raft_ivf_pq/d128-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 } - ] - }, - { - "name": "raft_ivf_pq.d64-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 64, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_1M/raft_ivf_pq/d64-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 } - ] - }, - { - "name": "raft_ivf_pq.d32-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 32, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_1M/raft_ivf_pq/d32-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 } - ] - }, - { - "name": "raft_ivf_pq.d32X-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 32, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_1M/raft_ivf_pq/d32-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 } - - ] - }, - { - "name": "raft_cagra.dim32.multi_cta", - "algo": "raft_cagra", - "build_param": { "graph_degree": 32, - "intermediate_graph_degree": 48, - "graph_build_algo": "NN_DESCENT", - "ivf_pq_build_pq_dim": 32, - "ivf_pq_build_pq_bits": 8, - "ivf_pq_build_nlist": 16384, - "ivf_pq_build_niter": 10, - "ivf_pq_build_ratio": 10, - "ivf_pq_search_nprobe": 30, - "ivf_pq_search_internalDistanceDtype": "half", - "ivf_pq_search_smemLutDtype": "half", - "ivf_pq_search_refine_ratio": 8, - "nn_descent_max_iterations": 10, - "nn_descent_intermediate_graph_degree": 72, - "nn_descent_termination_threshold": 0.001 - }, - "file": "wiki_all_1M/raft_cagra/dim32.ibin", - "search_params": [ - { "itopk": 32, "search_width": 1, "max_iterations": 0, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 36, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 40, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 44, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 24, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 26, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 64, "search_width": 4, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 64, "search_width": 1, "max_iterations": 64, "algo": "multi_cta" }, - { "itopk": 96, "search_width": 2, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 128, "search_width": 8, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 128, "search_width": 2, "max_iterations": 64, "algo": "multi_cta" }, - { "itopk": 192, "search_width": 8, "max_iterations": 24, "algo": "multi_cta" }, - { "itopk": 192, "search_width": 2, "max_iterations": 96, "algo": "multi_cta" }, - { "itopk": 256, "search_width": 8, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 384, "search_width": 8, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 512, "search_width": 8, "max_iterations": 64, "algo": "multi_cta" } - ] - } - - ] -} - diff --git a/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_88M.json b/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_88M.json deleted file mode 100644 index e50b40f554..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/run/conf/wiki_all_88M.json +++ /dev/null @@ -1,200 +0,0 @@ -{ - "dataset": { - "name": "wiki_all_88M", - "base_file": "wiki_all_88M/base.88M.fbin", - "query_file": "wiki_all_88M/queries.fbin", - "groundtruth_neighbors_file": "wiki_all_88M/groundtruth.88M.neighbors.ibin", - "distance": "euclidean" - }, - "search_basic_param": { - "batch_size": 10000, - "k": 10 - }, - "index": [ - { - "name": "hnswlib.M16.ef50", - "algo": "hnswlib", - "build_param": { "M": 16, "efConstruction": 50, "numThreads": 56 }, - "file": "wiki_all_88M/hnswlib/M16.ef50", - "search_params": [ - { "ef": 10, "numThreads": 56 }, - { "ef": 20, "numThreads": 56 }, - { "ef": 40, "numThreads": 56 }, - { "ef": 60, "numThreads": 56 }, - { "ef": 80, "numThreads": 56 }, - { "ef": 120, "numThreads": 56 }, - { "ef": 200, "numThreads": 56 }, - { "ef": 400, "numThreads": 56 }, - { "ef": 600, "numThreads": 56 }, - { "ef": 800, "numThreads": 56 } - ] - }, - { - "name": "faiss_ivf_pq.M32-nlist16K", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "M": 32, - "nlist": 16384, - "ratio": 2 - }, - "file": "wiki_all_88M/faiss_ivf_pq/M32-nlist16K_ratio2", - "search_params": [ - { "nprobe": 10 }, - { "nprobe": 20 }, - { "nprobe": 30 }, - { "nprobe": 40 }, - { "nprobe": 50 }, - { "nprobe": 100 }, - { "nprobe": 200 }, - { "nprobe": 500 } - ] - }, - { - "name": "faiss_ivf_pq.M64-nlist16K", - "algo": "faiss_gpu_ivf_pq", - "build_param": { - "M": 64, - "nlist": 16384, - "ratio": 2 - }, - "file": "wiki_all_88M/faiss_ivf_pq/M64-nlist16K_ratio2", - "search_params": [ - { "nprobe": 10 }, - { "nprobe": 20 }, - { "nprobe": 30 }, - { "nprobe": 40 }, - { "nprobe": 50 }, - { "nprobe": 100 }, - { "nprobe": 200 }, - { "nprobe": 500 } - ] - }, - { - "name": "raft_ivf_pq.d128-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 128, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_88M/raft_ivf_pq/d128-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 1 } - ] - }, - { - "name": "raft_ivf_pq.d64-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 64, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_88M/raft_ivf_pq/d64-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 } - ] - }, - { - "name": "raft_ivf_pq.d32-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 32, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_88M/raft_ivf_pq/d32-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 32 } - ] - }, - { - "name": "raft_ivf_pq.d32X-nlist16K", - "algo": "raft_ivf_pq", - "build_param": { - "pq_dim": 32, - "pq_bits": 8, - "nlist": 16384, - "niter": 10, - "ratio": 10 - }, - "file": "wiki_all_88M/raft_ivf_pq/d32-nlist16K", - "search_params": [ - { "nprobe": 20, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 16 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 8 }, - { "nprobe": 30, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 40, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 50, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 100, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 200, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 }, - { "nprobe": 500, "internalDistanceDtype": "half", "smemLutDtype": "half", "refine_ratio": 4 } - - ] - }, - { - "name": "raft_cagra.dim32.multi_cta", - "algo": "raft_cagra", - "build_param": { "graph_degree": 32, "intermediate_graph_degree": 48 }, - "file": "wiki_all_88M/raft_cagra/dim32.ibin", - "search_params": [ - { "itopk": 32, "search_width": 1, "max_iterations": 0, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 36, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 40, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 44, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 1, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 24, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 26, "algo": "multi_cta" }, - { "itopk": 32, "search_width": 2, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 64, "search_width": 4, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 64, "search_width": 1, "max_iterations": 64, "algo": "multi_cta" }, - { "itopk": 96, "search_width": 2, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 128, "search_width": 8, "max_iterations": 16, "algo": "multi_cta" }, - { "itopk": 128, "search_width": 2, "max_iterations": 64, "algo": "multi_cta" }, - { "itopk": 192, "search_width": 8, "max_iterations": 24, "algo": "multi_cta" }, - { "itopk": 192, "search_width": 2, "max_iterations": 96, "algo": "multi_cta" }, - { "itopk": 256, "search_width": 8, "max_iterations": 32, "algo": "multi_cta" }, - { "itopk": 384, "search_width": 8, "max_iterations": 48, "algo": "multi_cta" }, - { "itopk": 512, "search_width": 8, "max_iterations": 64, "algo": "multi_cta" } - ] - } - - ] -} - diff --git a/python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/__main__.py b/python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/__main__.py deleted file mode 100644 index c65360ebb0..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/__main__.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -import subprocess -import sys - - -def split_groundtruth(groundtruth_filepath): - ann_bench_scripts_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "split_groundtruth.pl" - ) - pwd = os.getcwd() - path_to_groundtruth = os.path.normpath(groundtruth_filepath).split(os.sep) - if len(path_to_groundtruth) > 1: - os.chdir(os.path.join(*path_to_groundtruth[:-1])) - groundtruth_filename = path_to_groundtruth[-1] - subprocess.run( - [ann_bench_scripts_path, groundtruth_filename, "groundtruth"], - check=True, - ) - os.chdir(pwd) - - -def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - "--groundtruth", - help="Path to billion-scale dataset groundtruth file", - required=True, - ) - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - args = parser.parse_args() - - split_groundtruth(args.groundtruth) - - -if __name__ == "__main__": - main() diff --git a/python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/split_groundtruth.pl b/python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/split_groundtruth.pl deleted file mode 100755 index b0a59f806c..0000000000 --- a/python/raft-ann-bench/src/raft_ann_bench/split_groundtruth/split_groundtruth.pl +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/perl - -# ============================================================================= -# Copyright (c) 2020-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. - -use warnings; -use strict; -use autodie qw(open close); - - -@ARGV == 2 - or die "usage: $0 input output_prefix\n"; - -open my $fh, '<:raw', $ARGV[0]; - -my $raw; -read($fh, $raw, 8); -my ($nrows, $dim) = unpack('LL', $raw); - -my $expected_size = 8 + $nrows * $dim * (4 + 4); -my $size = (stat($fh))[7]; -$size == $expected_size - or die("error: expected size is $expected_size, but actual size is $size\n"); - - -open my $fh_out1, '>:raw', "$ARGV[1].neighbors.ibin"; -open my $fh_out2, '>:raw', "$ARGV[1].distances.fbin"; - -print {$fh_out1} $raw; -print {$fh_out2} $raw; - -read($fh, $raw, $nrows * $dim * 4); -print {$fh_out1} $raw; -read($fh, $raw, $nrows * $dim * 4); -print {$fh_out2} $raw; diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index 197ddae05f..9ebbaa5298 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -45,7 +45,6 @@ if(NOT raft_FOUND) # raft-dask doesn't actually use raft libraries, it just needs the headers, so we can turn off all # library compilation and we don't need to install anything here. set(BUILD_TESTS OFF) - set(BUILD_ANN_BENCH OFF) set(BUILD_PRIMS_BENCH OFF) set(RAFT_COMPILE_LIBRARIES OFF) set(RAFT_COMPILE_DIST_LIBRARY OFF) diff --git a/python/raft-dask/cmake/thirdparty/get_ucxx.cmake b/python/raft-dask/cmake/thirdparty/get_ucxx.cmake index db6039393a..db9b5c6b4d 100644 --- a/python/raft-dask/cmake/thirdparty/get_ucxx.cmake +++ b/python/raft-dask/cmake/thirdparty/get_ucxx.cmake @@ -47,9 +47,9 @@ endfunction() # Change pinned tag here to test a commit in CI # To use a different RAFT locally, set the CMake variable # CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_ucxx(VERSION 0.40 +find_and_configure_ucxx(VERSION 0.41 FORK rapidsai - PINNED_TAG branch-0.40 + PINNED_TAG branch-0.41 EXCLUDE_FROM_ALL YES UCXX_STATIC ${RAFT_DASK_UCXX_STATIC} ) diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index d1f577120f..a9f4de5dc3 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -31,13 +31,13 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ - "dask-cuda==24.10.*,>=0.0.0a0", - "distributed-ucxx==0.40.*,>=0.0.0a0", + "dask-cuda==24.12.*,>=0.0.0a0", + "distributed-ucxx==0.41.*,>=0.0.0a0", "joblib>=0.11", "numba>=0.57", - "pylibraft==24.10.*,>=0.0.0a0", - "rapids-dask-dependency==24.10.*,>=0.0.0a0", - "ucx-py==0.40.*,>=0.0.0a0", + "pylibraft==24.12.*,>=0.0.0a0", + "rapids-dask-dependency==24.12.*,>=0.0.0a0", + "ucx-py==0.41.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -120,9 +120,17 @@ regex = "(?P.*)" build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4,!=3.30.0", - "cython>=3.0.0", + "cython>=3.0.0,<3.1.0a0", "libucx==1.15.0", "ninja", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true" + +[tool.pydistcheck] +select = [ + "distro-too-large-compressed", +] + +# detect when package size grows significantly +max_allowed_size_compressed = '300M' diff --git a/setup.cfg b/setup.cfg index e64641d05b..94140d4d00 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ per-file-ignores = # unlike the match option above this match-dir will have no effect when # pydocstyle is invoked from pre-commit. Therefore this exclusion list must # also be maintained in the pre-commit config file. -match-dir = ^(?!(ci|cpp|conda|docs|java|notebooks)).*$ +match-dir = ^(?!(ci|cpp|conda|docs)).*$ # Allow missing docstrings for docutils ignore-decorators = .*(docutils|doc_apply|copy_docstring).* select =