From 5af2e9323c50e35f096459b81a9de170488d894e Mon Sep 17 00:00:00 2001 From: Sijie Xiang Date: Wed, 17 Jan 2024 19:11:10 -0800 Subject: [PATCH] Upgrade to MongoDB 7.0.4 (#415) upgrade to Mongo 7.0.4 --- .github/workflows/test.yml | 2 +- build.xml | 5 +- src/us/kbase/auth2/service/AuthBuilder.java | 14 +- .../auth2/service/AuthenticationService.java | 2 +- src/us/kbase/test/auth2/MongoController.java | 150 +++++++++++++ .../test/auth2/MongoStorageTestManager.java | 10 +- .../MongoStorageDuplicateKeyCheckerTest.java | 5 +- .../mongo/MongoStorageStartUpTest.java | 202 +++++++----------- .../lib/storage/mongo/MongoStorageTester.java | 4 +- 9 files changed, 257 insertions(+), 137 deletions(-) create mode 100644 src/us/kbase/test/auth2/MongoController.java diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1aee87ec..eceabf35 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,7 @@ jobs: # doesn't touch mongo against multiple mongo versions include: - java: '8' - mongo: 'mongodb-linux-x86_64-3.6.23' + mongo: 'mongodb-linux-x86_64-ubuntu2204-7.0.4' wired_tiger: 'false' - java: '11' mongo: 'mongodb-linux-x86_64-3.6.23' diff --git a/build.xml b/build.xml index 659d03d2..0a18e8bf 100644 --- a/build.xml +++ b/build.xml @@ -30,7 +30,10 @@ - + + + + diff --git a/src/us/kbase/auth2/service/AuthBuilder.java b/src/us/kbase/auth2/service/AuthBuilder.java index 5fda63be..e80de19a 100644 --- a/src/us/kbase/auth2/service/AuthBuilder.java +++ b/src/us/kbase/auth2/service/AuthBuilder.java @@ -2,13 +2,15 @@ import static java.util.Objects.requireNonNull; +import java.util.Arrays; import java.util.HashSet; import java.util.Set; import org.slf4j.LoggerFactory; -import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.MongoClientSettings; import com.mongodb.MongoCredential; import com.mongodb.MongoException; import com.mongodb.ServerAddress; @@ -62,15 +64,17 @@ public AuthBuilder( private MongoClient buildMongo(final AuthStartupConfig c) throws StorageInitException { //TODO ZLATER MONGO handle shards & replica sets + final MongoClientSettings.Builder mongoBuilder = MongoClientSettings.builder().applyToClusterSettings( + builder -> builder.hosts(Arrays.asList(new ServerAddress(c.getMongoHost())))); + try { if (c.getMongoUser().isPresent()) { final MongoCredential creds = MongoCredential.createCredential( c.getMongoUser().get(), c.getMongoDatabase(), c.getMongoPwd().get()); // unclear if and when it's safe to clear the password - return new MongoClient(new ServerAddress(c.getMongoHost()), creds, - MongoClientOptions.builder().build()); + return MongoClients.create(mongoBuilder.credential(creds).build()); } else { - return new MongoClient(new ServerAddress(c.getMongoHost())); + return MongoClients.create(mongoBuilder.build()); } } catch (MongoException e) { LoggerFactory.getLogger(getClass()).error( diff --git a/src/us/kbase/auth2/service/AuthenticationService.java b/src/us/kbase/auth2/service/AuthenticationService.java index b51db4a2..a69f8706 100644 --- a/src/us/kbase/auth2/service/AuthenticationService.java +++ b/src/us/kbase/auth2/service/AuthenticationService.java @@ -8,7 +8,7 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; diff --git a/src/us/kbase/test/auth2/MongoController.java b/src/us/kbase/test/auth2/MongoController.java new file mode 100644 index 00000000..6d33266d --- /dev/null +++ b/src/us/kbase/test/auth2/MongoController.java @@ -0,0 +1,150 @@ +package us.kbase.test.auth2; + +import static us.kbase.common.test.controllers.ControllerCommon.checkExe; +import static us.kbase.common.test.controllers.ControllerCommon.findFreePort; +import static us.kbase.common.test.controllers.ControllerCommon.makeTempDirs; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Scanner; +import java.util.stream.Collectors; + +import com.github.zafarkhaja.semver.Version; +import org.apache.commons.io.FileUtils; + + +/** Q&D Utility to run a Mongo server for the purposes of testing from + * Java. + * @author gaprice@lbl.gov, sijiex@lbl.gov + * + */ +public class MongoController { + + private final static String DATA_DIR = "data"; + + private final static List tempDirectories = + new LinkedList(); + static { + tempDirectories.add(DATA_DIR); + } + + private final static Version MONGO_DB_6_1 = + Version.forIntegers(6,1); + + private final Path tempDir; + + private final Process mongo; + + private final int port; + + public MongoController( + final String mongoExe, + final Path rootTempDir) + throws Exception { + this(mongoExe, rootTempDir, false); + } + + public MongoController( + final String mongoExe, + final Path rootTempDir, + final boolean useWiredTiger) + throws Exception { + checkExe(mongoExe, "mongod server"); + tempDir = makeTempDirs(rootTempDir, "MongoController-", tempDirectories); + port = findFreePort(); + Version dbVer = getMongoDBVer(mongoExe); + List command = getMongoServerStartCommand(mongoExe, useWiredTiger, dbVer); + mongo = startProcess(command); + } + + public int getServerPort() { + return port; + } + + public Path getTempDir() { + return tempDir; + } + + public void destroy(boolean deleteTempFiles) throws IOException { + if (mongo != null) { + mongo.destroy(); + } + if (tempDir != null && deleteTempFiles) { + FileUtils.deleteDirectory(tempDir.toFile()); + } + } + + private static Version getMongoDBVer(final String mongoExe) throws IOException { + + // build MongoDB version check command + List command = new LinkedList(); + command.addAll(Arrays.asList(mongoExe, "--version")); + + // start MongoDB version check process + ProcessBuilder checkVerPb = new ProcessBuilder(command); + Process checkVerProcess = checkVerPb.start(); + + // parse mongod --version output string + String dbVer = new BufferedReader( + new InputStreamReader(checkVerProcess.getInputStream())) + .lines() + .collect(Collectors.joining(" ")) + .split(" ")[2].substring(1); + + System.out.println("MongoDB version: " + dbVer); + checkVerProcess.destroy(); + return Version.valueOf(dbVer); + } + + private List getMongoServerStartCommand(final String mongoExe, + final boolean useWiredTiger, + final Version dbVer) { + List command = new LinkedList(); + command.addAll(Arrays.asList(mongoExe, "--port", "" + port, + "--dbpath", tempDir.resolve(DATA_DIR).toString())); + + // Starting in MongoDB 6.1, journaling is always enabled. + // As a result, MongoDB removes the storage.journal.enabled option + // and the corresponding --journal and --nojournal command-line options. + // https://www.mongodb.com/docs/manual/release-notes/6.1/#changes-to-journaling + if (dbVer.lessThan(MONGO_DB_6_1)) { + command.addAll(Arrays.asList("--nojournal")); + } + if (useWiredTiger) { + command.addAll(Arrays.asList("--storageEngine", "wiredTiger")); + } + return command; + } + + private Process startProcess(List command) throws Exception { + ProcessBuilder servpb = new ProcessBuilder(command) + .redirectErrorStream(true) + .redirectOutput(getTempDir().resolve("mongo.log").toFile()); + + Process mongoProcess = servpb.start(); + Thread.sleep(1000); //wait for server to start up + return mongoProcess; + } + + public static void main(String[] args) throws Exception { + us.kbase.common.test.controllers.mongo.MongoController ac = new us.kbase.common.test.controllers.mongo.MongoController( + "/kb/runtime/bin/mongod", + Paths.get("workspacetesttemp")); + System.out.println(ac.getServerPort()); + System.out.println(ac.getTempDir()); + Scanner reader = new Scanner(System.in); + System.out.println("any char to shut down"); + //get user input for a + reader.next(); + ac.destroy(false); + reader.close(); + } + +} + diff --git a/src/us/kbase/test/auth2/MongoStorageTestManager.java b/src/us/kbase/test/auth2/MongoStorageTestManager.java index 45d79ab9..ebf1a2cb 100644 --- a/src/us/kbase/test/auth2/MongoStorageTestManager.java +++ b/src/us/kbase/test/auth2/MongoStorageTestManager.java @@ -15,12 +15,12 @@ import org.bson.Document; import com.github.zafarkhaja.semver.Version; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; import us.kbase.auth2.cryptutils.RandomDataGenerator; import us.kbase.auth2.lib.storage.mongo.MongoStorage; -import us.kbase.common.test.controllers.mongo.MongoController; public class MongoStorageTestManager { @@ -42,16 +42,16 @@ public MongoStorageTestManager(final String dbName) throws Exception { wiredTiger = useWiredTigerEngine(); System.out.println(String.format("Testing against mongo executable %s on port %s", getMongoExe(), mongo.getServerPort())); - mc = new MongoClient("localhost:" + mongo.getServerPort()); + mc = MongoClients.create("mongodb://localhost:" + mongo.getServerPort()); db = mc.getDatabase(dbName); - + final Document bi = db.runCommand(new Document("buildinfo", 1)); final String version = bi.getString("version"); mongoDBVer = Version.valueOf(version); indexVer = mongoDBVer.greaterThanOrEqualTo(Version.forIntegers(3, 4)) ? 2 : 1; reset(); } - + public void destroy() throws Exception { if (mc != null) { mc.close(); diff --git a/src/us/kbase/test/auth2/lib/storage/mongo/MongoStorageDuplicateKeyCheckerTest.java b/src/us/kbase/test/auth2/lib/storage/mongo/MongoStorageDuplicateKeyCheckerTest.java index 19fe314c..cf19b290 100644 --- a/src/us/kbase/test/auth2/lib/storage/mongo/MongoStorageDuplicateKeyCheckerTest.java +++ b/src/us/kbase/test/auth2/lib/storage/mongo/MongoStorageDuplicateKeyCheckerTest.java @@ -131,8 +131,9 @@ public void unparseable() throws Exception { fail("expected exception"); } catch (InvocationTargetException ex) { TestCommon.assertExceptionCorrect((Exception) ex.getTargetException(), - new AuthStorageException("Unable to parse duplicate key error: some ")); - + new AuthStorageException("Unable to parse duplicate key error: " + + "Write operation error on server 127.0.0.1:27017. " + + "Write error: WriteError{code=11000, message='some ")); } } diff --git a/src/us/kbase/test/auth2/lib/storage/mongo/MongoStorageStartUpTest.java b/src/us/kbase/test/auth2/lib/storage/mongo/MongoStorageStartUpTest.java index 14e12c3a..f6642907 100644 --- a/src/us/kbase/test/auth2/lib/storage/mongo/MongoStorageStartUpTest.java +++ b/src/us/kbase/test/auth2/lib/storage/mongo/MongoStorageStartUpTest.java @@ -1,9 +1,9 @@ package us.kbase.test.auth2.lib.storage.mongo; import static org.hamcrest.CoreMatchers.is; + import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; - import static us.kbase.test.auth2.TestCommon.set; import java.util.Arrays; @@ -36,7 +36,7 @@ public void nullConstructor() throws Exception { assertThat("incorrect exception message", e.getMessage(), is("db")); } } - + @Test public void startUpAndCheckConfigDoc() throws Exception { final MongoDatabase db = mc.getDatabase("startUpAndCheckConfigDoc"); @@ -45,35 +45,35 @@ public void startUpAndCheckConfigDoc() throws Exception { assertThat("Only one config doc", col.countDocuments(), is(1L)); final FindIterable c = col.find(); final Document d = c.first(); - + assertThat("correct config key & value", (String)d.get("schema"), is("schema")); assertThat("not in update", (Boolean)d.get("inupdate"), is(false)); assertThat("schema v1", (Integer)d.get("schemaver"), is(1)); - + //check startup works with the config object in place final MongoStorage ms = new MongoStorage(db); ms.setCustomRole(new CustomRole("foo", "bar")); assertThat("failed basic storage operation", ms.getCustomRoles(), is(set(new CustomRole("foo", "bar")))); } - + @Test public void startUpWith2ConfigDocs() throws Exception { final MongoDatabase db = mc.getDatabase("startUpWith2ConfigDocs"); - + final Document m = new Document("schema", "schema") .append("inupdate", false) .append("schemaver", 1); - + db.getCollection("config").insertMany(Arrays.asList(m, // need to create a new document to create a new mongo _id new Document(m))); - + final Pattern errorPattern = Pattern.compile( "Failed to create index: Write failed with error code 11000 and error message " + - "'(exception: )?E11000 duplicate key error (index|collection): " + + "'(exception: )?(.*)E11000 duplicate key error (index|collection): " + "startUpWith2ConfigDocs.config( index: |\\.\\$)schema_1\\s+dup key: " + - "\\{ : \"schema\" \\}'"); + "(\\{ schema: \"schema\" \\}'|\\{ : \"schema\" \\}')"); try { new MongoStorage(db); fail("started mongo with bad config"); @@ -82,36 +82,36 @@ public void startUpWith2ConfigDocs() throws Exception { assertThat("exception did not match: \n" + e.getMessage(), match.matches(), is(true)); } } - + @Test public void startUpWithBadSchemaVersion() throws Exception { final MongoDatabase db = mc.getDatabase("startUpWithBadSchemaVersion"); - + final Document m = new Document("schema", "schema") .append("inupdate", false) .append("schemaver", 4); - + db.getCollection("config").insertOne(m); - + failMongoStart(db, new StorageInitException( "Incompatible database schema. Server is v1, DB is v4")); } - + @Test public void startUpWithUpdateInProgress() throws Exception { final MongoDatabase db = mc.getDatabase("startUpWithUpdateInProgress"); - + final Document m = new Document("schema", "schema") .append("inupdate", true) .append("schemaver", 1); - + db.getCollection("config").insertOne(m); - + failMongoStart(db, new StorageInitException( "The database is in the middle of an update from v1 of the " + "schema. Aborting startup.")); } - + private void failMongoStart(final MongoDatabase db, final Exception exp) throws Exception { try { @@ -121,13 +121,13 @@ private void failMongoStart(final MongoDatabase db, final Exception exp) TestCommon.assertExceptionCorrect(e, exp); } } - + /* The following tests ensure that all indexes are created correctly. The collection names - * are tested so that if a new collection is added the test will fail without altering + * are tested so that if a new collection is added the test will fail without altering * said test, at which time the coder will hopefully read this notice and add index tests * for the new collection. */ - + @Test public void checkCollectionNames() throws Exception { final Set names = new HashSet<>(); @@ -151,284 +151,246 @@ public void checkCollectionNames() throws Exception { db.listCollectionNames().forEach((Consumer) names::add); assertThat("incorrect collection names", names, is(expected)); } - + @Test public void indexesConfig() { - final Set indexes = new HashSet<>(); - db.getCollection("config").listIndexes().forEach((Consumer) indexes::add); + final Set indexes = getAndNormalizeIndexes("config"); assertThat("incorrect indexes", indexes, is(set( new Document("v", indexVer) .append("unique", true) .append("key", new Document("schema", 1)) - .append("name", "schema_1") - .append("ns", "test_mongostorage.config"), + .append("name", "schema_1"), new Document("v", indexVer) .append("key", new Document("_id", 1)) .append("name", "_id_") - .append("ns", "test_mongostorage.config") ))); } - + @Test public void indexesConfigApp() { - final Set indexes = new HashSet<>(); - db.getCollection("config_app").listIndexes().forEach((Consumer) indexes::add); + final Set indexes = getAndNormalizeIndexes("config_app"); assertThat("incorrect indexes", indexes, is(set( new Document("v", indexVer) .append("unique", true) .append("key", new Document("key", 1)) - .append("name", "key_1") - .append("ns", "test_mongostorage.config_app"), + .append("name", "key_1"), new Document("v", indexVer) .append("key", new Document("_id", 1)) .append("name", "_id_") - .append("ns", "test_mongostorage.config_app") ))); } - + @Test public void indexesConfigExt() { - final Set indexes = new HashSet<>(); - db.getCollection("config_ext").listIndexes().forEach((Consumer) indexes::add); + final Set indexes = getAndNormalizeIndexes("config_ext"); assertThat("incorrect indexes", indexes, is(set( new Document("v", indexVer) .append("unique", true) .append("key", new Document("key", 1)) - .append("name", "key_1") - .append("ns", "test_mongostorage.config_ext"), + .append("name", "key_1"), new Document("v", indexVer) .append("key", new Document("_id", 1)) .append("name", "_id_") - .append("ns", "test_mongostorage.config_ext") ))); } - + @Test public void indexesConfigProv() { - final Set indexes = new HashSet<>(); - db.getCollection("config_prov").listIndexes().forEach((Consumer) indexes::add); + final Set indexes = getAndNormalizeIndexes("config_prov"); assertThat("incorrect indexes", indexes, is(set( new Document("v", indexVer) .append("unique", true) .append("key", new Document("prov", 1).append("key", 1)) - .append("name", "prov_1_key_1") - .append("ns", "test_mongostorage.config_prov"), + .append("name", "prov_1_key_1"), new Document("v", indexVer) .append("key", new Document("_id", 1)) .append("name", "_id_") - .append("ns", "test_mongostorage.config_prov") ))); } - + @Test public void indexesCustRoles() { - final Set indexes = new HashSet<>(); - db.getCollection("cust_roles").listIndexes().forEach((Consumer) indexes::add); + final Set indexes = getAndNormalizeIndexes("cust_roles"); assertThat("incorrect indexes", indexes, is(set( new Document("v", indexVer) .append("unique", true) .append("key", new Document("id", 1)) - .append("name", "id_1") - .append("ns", "test_mongostorage.cust_roles"), + .append("name", "id_1"), new Document("v", indexVer) .append("key", new Document("_id", 1)) .append("name", "_id_") - .append("ns", "test_mongostorage.cust_roles") ))); } - + @Test public void indexesTestCustRoles() { - final Set indexes = new HashSet<>(); - db.getCollection("test_cust_roles").listIndexes() - .forEach((Consumer) indexes::add); + final Set indexes = getAndNormalizeIndexes("test_cust_roles"); assertThat("incorrect indexes", indexes, is(set( new Document("v", indexVer) .append("unique", true) .append("key", new Document("id", 1)) - .append("name", "id_1") - .append("ns", "test_mongostorage.test_cust_roles"), + .append("name", "id_1"), new Document("v", indexVer) .append("key", new Document("_id", 1)) - .append("name", "_id_") - .append("ns", "test_mongostorage.test_cust_roles"), + .append("name", "_id_"), new Document("v", indexVer) .append("key", new Document("expires", 1)) .append("name", "expires_1") - .append("ns", "test_mongostorage.test_cust_roles") .append("expireAfterSeconds", 0L) ))); } - + @Test public void indexesTempData() { - final Set indexes = new HashSet<>(); - db.getCollection("tempdata").listIndexes().forEach((Consumer) indexes::add); + final Set indexes = getAndNormalizeIndexes("tempdata"); assertThat("incorrect indexes", indexes, is(set( new Document("v", indexVer) .append("unique", true) .append("key", new Document("token", 1)) - .append("name", "token_1") - .append("ns", "test_mongostorage.tempdata"), + .append("name", "token_1"), new Document("v", indexVer) .append("key", new Document("expires", 1)) .append("name", "expires_1") - .append("ns", "test_mongostorage.tempdata") .append("expireAfterSeconds", 0L), new Document("v", indexVer) .append("sparse", true) .append("key", new Document("user", 1)) - .append("name", "user_1") - .append("ns", "test_mongostorage.tempdata"), + .append("name", "user_1"), new Document("v", indexVer) .append("key", new Document("_id", 1)) - .append("name", "_id_") - .append("ns", "test_mongostorage.tempdata"), + .append("name", "_id_"), new Document("v", indexVer) .append("unique", true) .append("key", new Document("id", 1)) .append("name", "id_1") - .append("ns", "test_mongostorage.tempdata") ))); } - + @Test public void indexesTokens() { - final Set indexes = new HashSet<>(); - db.getCollection("tokens").listIndexes().forEach((Consumer) indexes::add); + final Set indexes = getAndNormalizeIndexes("tokens"); assertThat("incorrect indexes", indexes, is(set( new Document("v", indexVer) .append("unique", true) .append("key", new Document("token", 1)) - .append("name", "token_1") - .append("ns", "test_mongostorage.tokens"), + .append("name", "token_1"), new Document("v", indexVer) .append("key", new Document("expires", 1)) .append("name", "expires_1") - .append("ns", "test_mongostorage.tokens") .append("expireAfterSeconds", 0L), new Document("v", indexVer) .append("key", new Document("_id", 1)) - .append("name", "_id_") - .append("ns", "test_mongostorage.tokens"), + .append("name", "_id_"), new Document("v", indexVer) .append("key", new Document("user", 1)) - .append("name", "user_1") - .append("ns", "test_mongostorage.tokens"), + .append("name", "user_1"), new Document("v", indexVer) .append("unique", true) .append("key", new Document("id", 1)) .append("name", "id_1") - .append("ns", "test_mongostorage.tokens") ))); } - + @Test public void indexesTestTokens() { - final Set indexes = new HashSet<>(); - db.getCollection("test_tokens").listIndexes().forEach((Consumer) indexes::add); + final Set indexes = getAndNormalizeIndexes("test_tokens"); assertThat("incorrect indexes", indexes, is(set( new Document("v", indexVer) .append("unique", true) .append("key", new Document("token", 1)) - .append("name", "token_1") - .append("ns", "test_mongostorage.test_tokens"), + .append("name", "token_1"), new Document("v", indexVer) .append("key", new Document("expires", 1)) .append("name", "expires_1") - .append("ns", "test_mongostorage.test_tokens") .append("expireAfterSeconds", 0L), new Document("v", indexVer) .append("key", new Document("_id", 1)) - .append("name", "_id_") - .append("ns", "test_mongostorage.test_tokens"), + .append("name", "_id_"), new Document("v", indexVer) .append("key", new Document("user", 1)) - .append("name", "user_1") - .append("ns", "test_mongostorage.test_tokens"), + .append("name", "user_1"), new Document("v", indexVer) .append("unique", true) .append("key", new Document("id", 1)) .append("name", "id_1") - .append("ns", "test_mongostorage.test_tokens") ))); } - + @Test public void indexesUsers() { - final Set indexes = new HashSet<>(); - db.getCollection("users").listIndexes().forEach((Consumer) indexes::add); + final Set indexes = getAndNormalizeIndexes("users"); assertThat("incorrect indexes", indexes, is(set( new Document("v", indexVer) .append("key", new Document("custrls", 1)) .append("name", "custrls_1") - .append("ns", "test_mongostorage.users") .append("sparse", true), new Document("v", indexVer) .append("key", new Document("dispcan", 1)) - .append("name", "dispcan_1") - .append("ns", "test_mongostorage.users"), + .append("name", "dispcan_1"), new Document("v", indexVer) .append("key", new Document("_id", 1)) - .append("name", "_id_") - .append("ns", "test_mongostorage.users"), + .append("name", "_id_"), new Document("v", indexVer) - .append("unique", true) + .append("unique", true) .append("key", new Document("idents.id", 1)) .append("name", "idents.id_1") - .append("ns", "test_mongostorage.users") .append("sparse", true), new Document("v", indexVer) .append("key", new Document("roles", 1)) .append("name", "roles_1") - .append("ns", "test_mongostorage.users") .append("sparse", true), new Document("v", indexVer) .append("unique", true) .append("key", new Document("user", 1)) - .append("name", "user_1") - .append("ns", "test_mongostorage.users"), + .append("name", "user_1"), new Document("v", indexVer) .append("key", new Document("anonid", 1)) .append("name", "anonid_1") - .append("ns", "test_mongostorage.users") .append("sparse", true) ))); } - + @Test public void indexesTestUsers() { - final Set indexes = new HashSet<>(); - db.getCollection("test_users").listIndexes().forEach((Consumer) indexes::add); + final Set indexes = getAndNormalizeIndexes("test_users"); assertThat("incorrect indexes", indexes, is(set( new Document("v", indexVer) .append("key", new Document("custrls", 1)) .append("name", "custrls_1") - .append("ns", "test_mongostorage.test_users") .append("sparse", true), new Document("v", indexVer) .append("key", new Document("dispcan", 1)) - .append("name", "dispcan_1") - .append("ns", "test_mongostorage.test_users"), + .append("name", "dispcan_1"), new Document("v", indexVer) .append("key", new Document("_id", 1)) - .append("name", "_id_") - .append("ns", "test_mongostorage.test_users"), + .append("name", "_id_"), new Document("v", indexVer) .append("key", new Document("roles", 1)) .append("name", "roles_1") - .append("ns", "test_mongostorage.test_users") .append("sparse", true), new Document("v", indexVer) .append("unique", true) .append("key", new Document("user", 1)) - .append("name", "user_1") - .append("ns", "test_mongostorage.test_users"), + .append("name", "user_1"), new Document("v", indexVer) .append("key", new Document("expires", 1)) .append("name", "expires_1") - .append("ns", "test_mongostorage.test_users") .append("expireAfterSeconds", 0L) ))); } + + private Set getAndNormalizeIndexes(final String collectionName) { + final Set indexes = new HashSet<>(); + for (Document index: db.getCollection(collectionName).listIndexes()) { + // In MongoDB 4.4, the listIndexes and the mongo shell helper method db.collection.getIndexes() + // no longer returns the namespace ns field in the index specification documents. + index.remove("ns"); + // some versions of Mongo return ints, some longs. Convert all to longs. + if (index.containsKey("expireAfterSeconds")) { + index.put("expireAfterSeconds", ((Number) index.get("expireAfterSeconds")).longValue()); + } + indexes.add(index); + } + return indexes; + } } diff --git a/src/us/kbase/test/auth2/lib/storage/mongo/MongoStorageTester.java b/src/us/kbase/test/auth2/lib/storage/mongo/MongoStorageTester.java index 0e53ca84..4679e6c7 100644 --- a/src/us/kbase/test/auth2/lib/storage/mongo/MongoStorageTester.java +++ b/src/us/kbase/test/auth2/lib/storage/mongo/MongoStorageTester.java @@ -7,12 +7,12 @@ import org.junit.BeforeClass; import com.github.zafarkhaja.semver.Version; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import com.mongodb.client.MongoDatabase; import us.kbase.auth2.cryptutils.RandomDataGenerator; import us.kbase.auth2.lib.storage.mongo.MongoStorage; -import us.kbase.common.test.controllers.mongo.MongoController; +import us.kbase.test.auth2.MongoController; import us.kbase.test.auth2.MongoStorageTestManager; public class MongoStorageTester {