diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d3cc9ba..9a806ff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,9 @@ on: - master - develop +env: + AUTH2_JAR_NAME: kbase-auth2-test-shadow-all-0.7.0.jar + jobs: workspace_deluxe_tests: runs-on: ubuntu-latest @@ -26,11 +29,7 @@ jobs: minio: '2019-05-23T00-29-34Z' wired_tiger: 'false' - go: '1.12' - mongo: 'mongodb-linux-x86_64-3.6.20' - minio: '2019-05-23T00-29-34Z' - wired_tiger: 'false' - - go: '1.12' - mongo: 'mongodb-linux-x86_64-3.6.20' + mongo: 'mongodb-linux-x86_64-ubuntu2204-7.0.4' minio: '2019-05-23T00-29-34Z' wired_tiger: 'true' steps: @@ -50,10 +49,9 @@ jobs: # move to parent dir to install binaries etc cd .. - # set up jars - # TODO switch to the auth shadow jar - git clone https://github.com/kbase/jars - export JARSDIR=`pwd`/jars/lib/jars/ + # set up auth2 jar + wget -q https://github.com/kbase/jars/raw/master/lib/jars/kbase/auth2/$AUTH2_JAR_NAME + export AUTH2JAR=`pwd`/$AUTH2_JAR_NAME # set up mongo wget -q http://fastdl.mongodb.org/linux/${{matrix.mongo}}.tgz @@ -72,7 +70,7 @@ jobs: sed -i "s#^test.mongo.exe.*#test.mongo.exe=$MONGOD#" test.cfg sed -i "s#^test.minio.exe.*#test.minio.exe=$MINIO#" test.cfg sed -i "s#^test.mongo.wired_tiger.*#test.mongo.wired_tiger=${{matrix.wired_tiger}}#" test.cfg - sed -i "s#^test.jars.dir.*#test.jars.dir=$JARSDIR#" test.cfg + sed -i "s#^test.auth2jar.*#test.auth2jar=$AUTH2JAR#" test.cfg cat test.cfg - name: Run tests diff --git a/auth/kbase_provider_test.go b/auth/kbase_provider_test.go index d520618..7340088 100644 --- a/auth/kbase_provider_test.go +++ b/auth/kbase_provider_test.go @@ -53,7 +53,7 @@ func (t *TestSuite) SetupSuite() { t.mongo = mongoctl auth, err := kbaseauthcontroller.New(kbaseauthcontroller.Params{ - JarsDir: tcfg.JarsDir, + Auth2Jar: tcfg.Auth2JarPath, MongoHost: "localhost:" + strconv.Itoa(mongoctl.GetPort()), MongoDatabase: "test_kb_auth_provider_authdb", RootTempDir: tcfg.TempDir, diff --git a/nodestore/mongostore_test.go b/nodestore/mongostore_test.go index 2302a91..d42bad6 100644 --- a/nodestore/mongostore_test.go +++ b/nodestore/mongostore_test.go @@ -105,7 +105,8 @@ func (t *TestSuite) TestConstructFailAddConfigIndex() { t.Contains(err.Error(), "E11000 duplicate key error", "incorrect error") t.Contains(err.Error(), testDB+".config", "incorrect error") t.Contains(err.Error(), "schema_1", "incorrect error") - t.Contains(err.Error(), "dup key: { : \"schema\" }", "incorrect error") + t.Contains(err.Error(), "dup key: { ", "incorrect error") + t.Contains(err.Error(), ": \"schema\" }", "incorrect error") } func (t *TestSuite) TestConstructFailTwoConfigDocs() { @@ -773,7 +774,7 @@ func (t *TestSuite) TestConfigIndexes() { "_id_": false, "schema_1": true, } - t.checkIndexes("config", testDB+".config", expected) + t.checkIndexes("config", expected) } func (t *TestSuite) TestUserIndexes() { @@ -782,7 +783,7 @@ func (t *TestSuite) TestUserIndexes() { "user_1": true, "id_1": true, } - t.checkIndexes("users", testDB+".users", expected) + t.checkIndexes("users", expected) } func (t *TestSuite) TestNodeIndexes() { @@ -790,12 +791,11 @@ func (t *TestSuite) TestNodeIndexes() { "_id_": false, "id_1": true, } - t.checkIndexes("nodes", testDB+".nodes", expected) + t.checkIndexes("nodes", expected) } func (t *TestSuite) checkIndexes( collection string, - expectedNamespace string, expectedIndexes map[string]bool) { _, err := NewMongoNodeStore(t.client.Database(testDB)) if err != nil { @@ -815,7 +815,6 @@ func (t *TestSuite) checkIndexes( t.Fail(err.Error()) } m := elem.Map() - t.Equal(expectedNamespace, m["ns"], "incorrect name space") if un, ok := m["unique"]; ok { names[m["name"].(string)] = un.(bool) } else { diff --git a/service/integration_test.go b/service/integration_test.go index 18034b3..ae4a07b 100644 --- a/service/integration_test.go +++ b/service/integration_test.go @@ -191,7 +191,7 @@ func (t *TestSuite) addTestRole(username string, role string) { func (t *TestSuite) setupAuth(cfg *testhelpers.TestConfig, ) (*kbaseauthcontroller.Controller, url.URL) { auth, err := kbaseauthcontroller.New(kbaseauthcontroller.Params{ - JarsDir: cfg.JarsDir, + Auth2Jar: cfg.Auth2JarPath, MongoHost: "localhost:" + strconv.Itoa(t.mongo.GetPort()), MongoDatabase: "test_kb_auth_provider_authdb", RootTempDir: cfg.TempDir, diff --git a/test.cfg.example b/test.cfg.example index 6e8a729..b58f209 100644 --- a/test.cfg.example +++ b/test.cfg.example @@ -10,9 +10,8 @@ test.mongo.exe=mongod # no. test.mongo.wired_tiger=false -# The path to the jars dir inside the jars repo (https://github.com/kbase/jars), e.g. -# [path to jars repo]/lib/jars -test.jars.dir = +# The path to the kbase auth2 service shadow test jar +test.auth2jar= # Where to store temporary files generated during the test. test.temp.dir=temp_test_dir diff --git a/test/kbaseauthcontroller/authjars.go b/test/kbaseauthcontroller/authjars.go deleted file mode 100644 index af96fd9..0000000 --- a/test/kbaseauthcontroller/authjars.go +++ /dev/null @@ -1,76 +0,0 @@ -package kbaseauthcontroller - -// this file simply lists the jars that are required to run the KBase auth server in test mode. -const ( - // authTemplates is the zip file containing templates for the server - authTemplates = "kbase/auth2/kbase-auth2templates-0.2.4.zip" -) - -// jars contains the list of jars in the KBase jars repo (https://github.com/kbase/jars) -// required for the auth server. -// might want to make the paths non-unix specific... yagni -var jars = []string{ - "kbase/auth2/kbase-auth2-0.2.4.jar", - "kbase/auth2/kbase-auth2test-0.2.4.jar", - //lib - "apache_commons/commons-codec-1.8.jar", - "apache_commons/commons-validator-1.5.1.jar", - "google/guava-18.0.jar", - "ini4j/ini4j-0.5.2.jar", - "jcommander/jcommander-1.48.jar", - "mongo/mongo-java-driver-3.3.0.jar", - "mustache/compiler-0.9.3.jar", - "nulab-inc/zxcvbn-1.2.2.jar", - //logging - "kbase/common/kbase-common-0.0.22.jar", - "jna/jna-3.4.0.jar", - "logback/logback-core-1.1.2.jar", - "logback/logback-classic-1.1.2.jar", - "slf4j/slf4j-api-1.7.25.jar", - "syslog4j/syslog4j-0.9.46.jar", - //yauaa - "yauaa/yauaa-1.3.jar", - "apache_commons/commons-lang3-3.5.jar", - "apache_commons/commons-collections4-4.1.jar", - "apache_commons/commons-logging-1.2.jar", - "kohsuke/args4j-2.33.jar", - "snakeyaml/snakeyaml-1.18.jar", - //jackson - "jackson/jackson-annotations-2.5.4.jar", - "jackson/jackson-core-2.5.4.jar", - "jackson/jackson-databind-2.5.4.jar", - "jackson/jackson-jaxrs-base-2.5.4.jar", - "jackson/jackson-jaxrs-json-provider-2.5.4.jar", - "jackson/jackson-module-jaxb-annotations-2.5.4.jar", - //jersey - "jersey/entity-filtering/jersey-entity-filtering-2.23.2.jar", - "jersey/entity-filtering/jersey-media-json-jackson-2.23.2.jar", - "jersey/mvc/jersey-mvc-2.23.2.jar", - "jersey/mvc/jersey-mvc-mustache-2.23.2.jar", - "jersey/jersey-client-2.23.2.jar", - "jersey/jersey-common-2.23.2.jar", - "jersey/jersey-container-servlet-2.23.2.jar", - "jersey/jersey-container-servlet-core-2.23.2.jar", - "jersey/jersey-guava-2.23.2.jar", - "jersey/jersey-media-jaxb-2.23.2.jar", - "jersey/jersey-server-2.23.2.jar", - //jerseydeps - "annotation/javax.annotation-api-1.2.jar", - "asm/asm-debug-all-5.0.4.jar", - "inject/javax.inject-2.5.0-b05.jar", - "javassist/javassist-3.20.0-GA.jar", - "jaxb/jaxb-api-2.2.7.jar", - "jaxrs/javax.ws.rs-api-2.0.1.jar", - "osgi/org.osgi.core-4.2.0.jar", - "persistence/persistence-api-1.0.jar", - "servlet/javax.servlet-api-3.0.1.jar", - "validationapi/validation-api-1.1.0.Final.jar", - //jerseydep_hk2 - "hk2/aopalliance-repackaged-2.5.0-b05.jar", - "hk2/hk2-api-2.5.0-b05.jar", - "hk2/hk2-locator-2.5.0-b05.jar", - "hk2/hk2-utils-2.5.0-b05.jar", - "hk2/osgi-resource-locator-1.0.1.jar", - //test - "jetty/jetty-all-9.3.11.v20160721-uber.jar", -} diff --git a/test/kbaseauthcontroller/controller.go b/test/kbaseauthcontroller/controller.go index ccc9795..99689f5 100644 --- a/test/kbaseauthcontroller/controller.go +++ b/test/kbaseauthcontroller/controller.go @@ -29,8 +29,8 @@ const ( // Params are Parameters for creating a KBase Auth2 service (https://github.com/kbase/auth2) // controller. type Params struct { - // JarsDir is the path to the /lib/jars directory of the - JarsDir string + // Auth2Jar is the path to the kbase auth2 jar. + Auth2Jar string // MongoHost is the mongo host. MongoHost string // MongoDatabase is the database to use for auth data. @@ -48,7 +48,7 @@ type Controller struct { // New creates a new controller. func New(p Params) (*Controller, error) { - classPath, err := getClassPath(p.JarsDir) + authJarPath, err := checkAuthJarExists(p.Auth2Jar) if err != nil { return nil, err } @@ -58,7 +58,7 @@ func New(p Params) (*Controller, error) { if err != nil { return nil, err } - err = installTemplates(p.JarsDir, templateDir) + err = installTemplates(authJarPath, templateDir) if err != nil { return nil, err } @@ -72,7 +72,7 @@ func New(p Params) (*Controller, error) { } strport := strconv.Itoa(port) cmdargs := []string{ - "-classpath", classPath, + "-classpath", authJarPath, "-DAUTH2_TEST_MONGOHOST=" + p.MongoHost, "-DAUTH2_TEST_MONGODB=" + p.MongoDatabase, "-DAUTH2_TEST_TEMPLATE_DIR=" + templateDir, @@ -119,50 +119,51 @@ func waitForStartup(port string) error { return startupErr } -func getClassPath(jarsDir string) (string, error) { - jarsDir, err := filepath.Abs(jarsDir) +func checkAuthJarExists(auth2Jar string) (string, error) { + jpath, err := filepath.Abs(auth2Jar) if err != nil { return "", err } - cp := []string(nil) - for _, j := range jars { // global variable, yech - jpath := path.Join(jarsDir, j) - if _, err := os.Stat(jpath); os.IsNotExist(err) { - return "", fmt.Errorf("Jar %v does not exist", jpath) - } - cp = append(cp, jpath) + if _, err := os.Stat(jpath); os.IsNotExist(err) { + return "", fmt.Errorf("jar %v does not exist", jpath) } - return strings.Join(cp, ":"), nil + return jpath, nil } -func installTemplates(jarsDir string, templateDir string) error { - templateZip := path.Join(jarsDir, authTemplates) - arch, err := zip.OpenReader(templateZip) // global variable, yech +func installTemplates(authJarPath string, templateDir string) error { + jar, err := zip.OpenReader(authJarPath) if err != nil { return err } - for _, f := range arch.File { - name := f.FileHeader.Name - if !strings.HasSuffix(name, "/") { // not a directory + + for _, f := range jar.File { + name := f.Name + // not a directory + if !strings.HasSuffix(name, "/") && strings.HasPrefix(name, "kbase_auth2_templates") { name = path.Clean(name) - if path.IsAbs(name) || strings.HasPrefix(name, "..") { - return fmt.Errorf("Zip file %v contains files outside the zip directory - "+ - "this is a sign of a malicious zip file", templateZip) + if filepath.Dir(name) != "kbase_auth2_templates" { + return fmt.Errorf("jar file %v contains files outside the directory - "+ + "this is a sign of a malicious jar file", authJarPath) } - target, err := filepath.Abs(path.Join(templateDir, name)) + dst, err := filepath.Abs(path.Join(templateDir, filepath.Base(name))) if err != nil { return err } - os.MkdirAll(path.Dir(target), 0600) - r, err := f.Open() + os.MkdirAll(path.Dir(dst), 0600) + + source, err := f.Open() + if err != nil { + return err + } + defer source.Close() + + destination, err := os.Create(dst) if err != nil { return err } - f, err := os.Create(target) + defer destination.Close() - io.Copy(f, r) - r.Close() - f.Close() + io.Copy(destination, source) } } return nil diff --git a/test/mongocontroller/controller.go b/test/mongocontroller/controller.go index 96869f7..42b572e 100644 --- a/test/mongocontroller/controller.go +++ b/test/mongocontroller/controller.go @@ -2,10 +2,12 @@ package mongocontroller import ( "context" + "fmt" "os" "os/exec" "path/filepath" "strconv" + "strings" "time" "go.mongodb.org/mongo-driver/mongo" @@ -28,7 +30,7 @@ type Params struct { UseWiredTiger bool } -// Controller is a Minio controller. +// Controller is a Mongo controller. type Controller struct { port int tempDir string @@ -53,14 +55,29 @@ func New(p Params) (*Controller, error) { if err != nil { return nil, err } + cmdargs := []string{ "--port", strconv.Itoa(port), "--dbpath", ddir, - "--nojournal", + } + + // check mongodb version + ver, err := getMongoDBVer(p.ExecutablePath) + if err != nil { + return nil, err + } + + // Starting in MongoDB 6.1, journaling is always enabled. + // As a result, MongoDB removes the storage.journal.enabled option and + // the corresponding --journal and --nojournal command-line options. + // https://www.mongodb.com/docs/manual/release-notes/6.1/#changes-to-journaling + if ver.LessThan(*semver.New("6.1.0")) { + cmdargs = append(cmdargs, "--nojournal") } if p.UseWiredTiger { cmdargs = append(cmdargs, "--storageEngine", "wiredTiger") } + cmd := exec.Command(p.ExecutablePath, cmdargs...) cmd.Stdout = outfile cmd.Stderr = outfile @@ -80,7 +97,9 @@ func New(p Params) (*Controller, error) { if err != nil { return nil, err } - res := client.Database("foo").RunCommand(nil, map[string]int{"buildinfo": 1}) + + // test mongo connection + res := client.Database("foo").RunCommand(context.Background(), map[string]int{"buildinfo": 1}) if res.Err() != nil { return nil, res.Err() } @@ -89,9 +108,12 @@ func New(p Params) (*Controller, error) { if err != nil { return nil, err } + if ver.String() != doc["version"].(string) { + return nil, fmt.Errorf("the two mongo versions should be the same: %s != %s", + ver.String(), doc["version"].(string)) + } // wired tiger will also not include index names for 3.0, but we're not going to test // that so screw it - ver := semver.New(doc["version"].(string)) return &Controller{port, tdir, cmd, ver.LessThan(*semver.New("3.2.1000"))}, nil } @@ -121,3 +143,14 @@ func (c *Controller) Destroy(deleteTempDir bool) error { } return nil } + +func getMongoDBVer(executablePath string) (*semver.Version, error) { + cmd := exec.Command(executablePath, "--version") + stdout, err := cmd.Output() + if err != nil { + return nil, err + } + rep := strings.Replace(string(stdout), "\n", " ", -1) + ver := strings.Split(rep, " ")[2][1:] + return semver.New(ver), err +} diff --git a/test/testhelpers/config.go b/test/testhelpers/config.go index 42b045f..918ca58 100644 --- a/test/testhelpers/config.go +++ b/test/testhelpers/config.go @@ -21,8 +21,8 @@ const ( TestMongoExe = "test.mongo.exe" // TestUseWiredTiger denotes that the MongoDB WiredTiger storage engine should be used. TestUseWiredTiger = "test.mongo.wired_tiger" - // TestJarsDir is the key in the config file for the path to the KBase jars directory. - TestJarsDir = "test.jars.dir" + // TestAuth2Jar is the key in the config file for the path to the KBase auth2 shadow test jar. + TestAuth2Jar = "test.auth2jar" // TestTempDir is the key in the config file for the temporary directory. TestTempDir = "test.temp.dir" // TestDeleteTempDir is the key in the config file for whether the temporary directory @@ -36,7 +36,7 @@ type TestConfig struct { MinioExePath string MongoExePath string UseWiredTiger bool - JarsDir string + Auth2JarPath string TempDir string DeleteTempDir bool } @@ -71,7 +71,7 @@ func GetConfig() (*TestConfig, error) { if err != nil { return nil, err } - jarsdir, err := getValue(sec, TestJarsDir, configfile, true) + auth2Jar, err := getValue(sec, TestAuth2Jar, configfile, true) if err != nil { return nil, err } @@ -88,7 +88,7 @@ func GetConfig() (*TestConfig, error) { MinioExePath: minio, MongoExePath: mongo, UseWiredTiger: wiredTiger == "true", - JarsDir: jarsdir, + Auth2JarPath: auth2Jar, TempDir: tempDir, DeleteTempDir: del != "false", },