From 7a811400873d0777b330f07a8759c340048ab6e8 Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Thu, 12 Oct 2023 10:25:44 -0700 Subject: [PATCH 1/2] Add check-json to pre-commit linters, and fix invalid ann-bench JSON files The ann-bench conf files had a couple of datasets with invalid JSON config files, causing them to fail to run. This adds a check-json pre-commit hook to detect invalid json, which would have failed on these files with: ``` python/raft-ann-bench/src/raft-ann-bench/run/conf/mnist-784-euclidean.json: Failed to json decode (Expecting value: line 1 column 1 (char 0)) python/raft-ann-bench/src/raft-ann-bench/run/conf/glove-100-inner.json: Failed to json decode (Expecting ',' delimiter: line 686 column 7 (char 15218)) ``` --- .pre-commit-config.yaml | 5 ++++- .../src/raft-ann-bench/run/conf/glove-100-inner.json | 2 +- .../src/raft-ann-bench/run/conf/mnist-784-euclidean.json | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 66862ada5e..46ff7f639d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -105,7 +105,10 @@ repos: hooks: - id: rapids-dependency-file-generator args: ["--clean"] - + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.4.0 + hooks: + - id: check-json default_language_version: python: python3 diff --git a/python/raft-ann-bench/src/raft-ann-bench/run/conf/glove-100-inner.json b/python/raft-ann-bench/src/raft-ann-bench/run/conf/glove-100-inner.json index ce54ce3f6b..b4c7b2ccc6 100644 --- a/python/raft-ann-bench/src/raft-ann-bench/run/conf/glove-100-inner.json +++ b/python/raft-ann-bench/src/raft-ann-bench/run/conf/glove-100-inner.json @@ -682,7 +682,7 @@ "algo": "faiss_gpu_flat", "build_param": {}, "file": "glove-100-inner/faiss_gpu_flat/flat", - "search_params": [{}] + "search_params": [{}], "search_result_file": "result/glove-100-inner/faiss_gpu_flat/flat" }, diff --git a/python/raft-ann-bench/src/raft-ann-bench/run/conf/mnist-784-euclidean.json b/python/raft-ann-bench/src/raft-ann-bench/run/conf/mnist-784-euclidean.json index 6e982873bb..fff3bca1d7 100644 --- a/python/raft-ann-bench/src/raft-ann-bench/run/conf/mnist-784-euclidean.json +++ b/python/raft-ann-bench/src/raft-ann-bench/run/conf/mnist-784-euclidean.json @@ -1,4 +1,4 @@ -c{ +{ "dataset": { "name": "mnist-784-euclidean", "base_file": "mnist-784-euclidean/base.fbin", From 44131b8edd978d146e0f56b168f7404e46e64b46 Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Thu, 12 Oct 2023 11:47:39 -0700 Subject: [PATCH 2/2] bump check-json version, fix duplicate 'file' key error --- .pre-commit-config.yaml | 2 +- .../src/raft-ann-bench/run/conf/glove-100-inner.json | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 46ff7f639d..80ad3614bc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -106,7 +106,7 @@ repos: - id: rapids-dependency-file-generator args: ["--clean"] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v4.5.0 hooks: - id: check-json diff --git a/python/raft-ann-bench/src/raft-ann-bench/run/conf/glove-100-inner.json b/python/raft-ann-bench/src/raft-ann-bench/run/conf/glove-100-inner.json index b4c7b2ccc6..06ce6e3ac4 100644 --- a/python/raft-ann-bench/src/raft-ann-bench/run/conf/glove-100-inner.json +++ b/python/raft-ann-bench/src/raft-ann-bench/run/conf/glove-100-inner.json @@ -207,7 +207,6 @@ "algo": "faiss_gpu_ivf_flat", "build_param": {"nlist":8192}, "file": "glove-100-inner/faiss_gpu_ivf_flat/nlist8192", - "file": "index/glove-100-inner/faiss_ivf_flat/nlist8192", "search_params": [ { "nprobe": 1