diff --git a/.golangci.yml b/.golangci.yml index 5d283ef..057104c 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -53,9 +53,15 @@ issues: - path: _test\.go linters: - funlen + - path: cert.go + linters: + - misspell - path: clog.go linters: - gochecknoinits + - path: config.go + linters: + - exhaustive max-issues-per-linter: 0 max-same-issues: 0 new-from-rev: master diff --git a/clog/clog.go b/clog/clog.go index a1caf8a..46ca4a4 100644 --- a/clog/clog.go +++ b/clog/clog.go @@ -1,7 +1,7 @@ /******************************************************************************* * Copyright (c) 2020 Genome Research Ltd. * - * Author: Sendu Bala + * Author: Sendu Bala , * Based on: https://blog.gopheracademy.com/advent-2016/context-logging/ * * Permission is hereby granted, free of charge, to any person obtaining @@ -30,6 +30,7 @@ package clog import ( "bytes" "context" + "os" log "github.com/inconshreveable/log15" "github.com/sb10/l15h" @@ -152,3 +153,16 @@ func Error(ctx context.Context, msg string, args ...interface{}) { func Crit(ctx context.Context, msg string, args ...interface{}) { logger(ctx).Crit(msg, args...) } + +// Fatal logs the given message with context and args to the global logger at +// the error level before exiting. 'fatal' is set true in stack trace. +func Fatal(ctx context.Context, msg string, args ...interface{}) { + args = append(args, "fatal", true) + logger(ctx).Crit(msg, args...) + + if os.Getenv("FATAL_EXIT_TEST") == "1" { + return + } + + os.Exit(1) +} diff --git a/clog/clog_test.go b/clog/clog_test.go index 7d66e3f..bc614df 100644 --- a/clog/clog_test.go +++ b/clog/clog_test.go @@ -1,7 +1,7 @@ /******************************************************************************* * Copyright (c) 2020 Genome Research Ltd. * - * Author: Sendu Bala + * Author: Sendu Bala , * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -27,11 +27,12 @@ package clog import ( "context" + "os" "testing" "github.com/inconshreveable/log15" . "github.com/smartystreets/goconvey/convey" - "github.com/wtsi-ssg/wr/internal" + "github.com/wtsi-ssg/wr/fs/filepath" ) func TestLogger(t *testing.T) { @@ -155,16 +156,28 @@ func TestLogger(t *testing.T) { So(lmsg, ShouldContainSubstring, "stack=") So(lmsg, ShouldContainSubstring, retryLogMsg) }) + + Convey("Fatal works and has a stack trace", func() { + os.Setenv("FATAL_EXIT_TEST", "1") + defer os.Unsetenv("FATAL_EXIT_TEST") + Fatal(ctx, "msg", "foo", 1) + lmsg := buff.String() + hasMsgAndFoo("crit", lmsg) + So(lmsg, ShouldContainSubstring, "fatal=true") + So(lmsg, ShouldNotContainSubstring, "caller=clog") + So(lmsg, ShouldContainSubstring, "stack=") + So(lmsg, ShouldContainSubstring, retryLogMsg) + }) }) Convey("You can log to a file", t, func() { - logPath := internal.FilePathInTempDir(t, "clog.log") + logPath := filepath.FilePathInTempDir(t, "clog.log") err := ToFileAtLevel(logPath, "debug") So(err, ShouldBeNil) Debug(background, "msg") - So(internal.FileAsString(logPath), ShouldContainSubstring, "msg=msg") + So(filepath.FileAsString(logPath), ShouldContainSubstring, "msg=msg") Convey("And append to a file", func() { err = ToFileAtLevel(logPath, "debug") @@ -172,7 +185,7 @@ func TestLogger(t *testing.T) { Debug(background, "foo") - logs := internal.FileAsString(logPath) + logs := filepath.FileAsString(logPath) So(logs, ShouldContainSubstring, "msg=msg") So(logs, ShouldContainSubstring, "msg=foo") }) diff --git a/fs/filepath/filepath.go b/fs/filepath/filepath.go index 66ff353..9d08e11 100644 --- a/fs/filepath/filepath.go +++ b/fs/filepath/filepath.go @@ -26,7 +26,9 @@ package filepath // this file implements utility routines for manipulating filename paths. -import "path/filepath" +import ( + "path/filepath" +) // RelToAbsPath returns the absolute path of a file given its relative path and // the directory name. diff --git a/internal/tests.go b/fs/filepath/tests.go similarity index 99% rename from internal/tests.go rename to fs/filepath/tests.go index db3ad61..b77fd84 100644 --- a/internal/tests.go +++ b/fs/filepath/tests.go @@ -23,7 +23,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ******************************************************************************/ -package internal +package filepath import ( "io/ioutil" diff --git a/internal/tests_test.go b/fs/filepath/tests_test.go similarity index 97% rename from internal/tests_test.go rename to fs/filepath/tests_test.go index bbb6dc8..3db139f 100644 --- a/internal/tests_test.go +++ b/fs/filepath/tests_test.go @@ -23,7 +23,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ******************************************************************************/ -package internal +package filepath import ( "fmt" @@ -40,7 +40,7 @@ func TestTestFuncs(t *testing.T) { basename := "foo" path := FilePathInTempDir(t, basename) fmt.Printf("got path %s\n", path) - So(path, ShouldStartWith, "/tmp") + So(path, ShouldStartWith, os.TempDir()) So(path, ShouldEndWith, basename) _, err := os.Open(filepath.Dir(path)) So(err, ShouldBeNil) diff --git a/go.mod b/go.mod index e72cedf..168fa31 100644 --- a/go.mod +++ b/go.mod @@ -3,20 +3,23 @@ module github.com/wtsi-ssg/wr go 1.15 require ( + github.com/Microsoft/go-winio v0.4.16 // indirect + github.com/creasty/defaults v1.5.1 github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/go-stack/stack v1.8.0 // indirect - github.com/gogo/protobuf v1.3.1 // indirect github.com/inconshreveable/log15 v0.0.0-20200109203555-b30bc20e4fd1 + github.com/jinzhu/configor v1.2.1 github.com/mattn/go-colorable v0.1.7 // indirect + github.com/olekukonko/tablewriter v0.0.5 github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.1 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/ricochet2200/go-disk-usage v0.0.0-20150921141558-f0d1b743428f github.com/rs/xid v1.2.1 github.com/sb10/l15h v0.0.0-20170510122137-64c488bf8e22 + github.com/shirou/gopsutil/v3 v3.21.3 github.com/smartystreets/goconvey v1.6.4 - golang.org/x/sys v0.0.0-20200918174421-af09f7315aff // indirect + github.com/stretchr/testify v1.7.0 // indirect + golang.org/x/sys v0.0.0-20210217105451-b926d437f341 ) diff --git a/go.sum b/go.sum index c89e732..6aaaaea 100644 --- a/go.sum +++ b/go.sum @@ -1,55 +1,87 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/creasty/defaults v1.5.1 h1:j8WexcS3d/t4ZmllX4GEkl4wIB/trOr035ajcLHCISM= +github.com/creasty/defaults v1.5.1/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.2+incompatible h1:vFgEHPqWBTp4pTjdLwjAA4bSo3gvIGOYwuJTlEjVBCw= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/inconshreveable/log15 v0.0.0-20200109203555-b30bc20e4fd1 h1:KUDFlmBg2buRWNzIcwLlKvfcnujcHQRQ1As1LoaCLAM= github.com/inconshreveable/log15 v0.0.0-20200109203555-b30bc20e4fd1/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= +github.com/jinzhu/configor v1.2.1 h1:OKk9dsR8i6HPOCZR8BcMtcEImAFjIhbJFZNyn5GCZko= +github.com/jinzhu/configor v1.2.1/go.mod h1:nX89/MOmDba7ZX7GCyU/VIaQ2Ar2aizBl2d3JLF/rDc= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/ricochet2200/go-disk-usage v0.0.0-20150921141558-f0d1b743428f h1:w4VLAgWDnrcBDFSi8Ppn/MrB/Z1A570+MV90CvMtVVA= github.com/ricochet2200/go-disk-usage v0.0.0-20150921141558-f0d1b743428f/go.mod h1:yhevTRDiduxPJHQDCtlqUn53ojFPkRh/mKhMUzQUCpc= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/sb10/l15h v0.0.0-20170510122137-64c488bf8e22 h1:1ECjRVBhG3NLRKTbvZ07fIQ5BiLnZFc3qLxqM6H6Rn8= github.com/sb10/l15h v0.0.0-20170510122137-64c488bf8e22/go.mod h1:s4RlXXC/L+BTwtp3zv5UREYJOftKFBWLsUCILdaMYeU= +github.com/shirou/gopsutil/v3 v3.21.3 h1:wgcdAHZS2H6qy4JFewVTtqfiYxFzCeEJod/mLztdPG8= +github.com/shirou/gopsutil/v3 v3.21.3/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= +github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200918174421-af09f7315aff h1:1CPUrky56AcgSpxz/KfgzQWzfG09u5YOL8MvPYBlrL8= -golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210217105451-b926d437f341 h1:2/QtM1mL37YmcsT8HaDNHDgTqqFVw+zr8UzMiBVLzYU= +golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -ithub.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/cert.go b/internal/cert.go new file mode 100644 index 0000000..83d608c --- /dev/null +++ b/internal/cert.go @@ -0,0 +1,345 @@ +/******************************************************************************* + * Copyright (c) 2020 Genome Research Ltd. + * + * Author: Sendu Bala , Ashwini Chhipa + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ + +package internal + +import ( + crand "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "io" + "io/ioutil" + "math/big" + "net" + "os" + "time" +) + +const ( + // validFor is time duration of certificate validity. + validFor = 365 * 24 * time.Hour + + // certFileFlags are flags for certificate files. + certFileFlags int = os.O_RDWR | os.O_CREATE | os.O_TRUNC + + // certMode is the file mode for certificate file. + certMode os.FileMode = 0666 + + // serverKeyFlags are flags for server key file. + serverKeyFlags int = os.O_WRONLY | os.O_CREATE | os.O_TRUNC + + // serverKeyMode is the file mode for server key file. + serverKeyMode os.FileMode = 0600 +) + +// CertificateErr is supplied to CertError to define the certain type of +// certificate related errors. +type CertificateErr string + +// ErrCert* are the reasons related to certificates. +const ( + ErrCertParse CertificateErr = "could not be parsed" + ErrCertCreate CertificateErr = "could not be created" + ErrCertExists CertificateErr = "already exists" + ErrCertEncode CertificateErr = "could not encode" + ErrCertNotFound CertificateErr = "cert could not be found" +) + +// CertError records a certificate-related error. +type CertError struct { + Type CertificateErr // one of our CertificateErr constants + Path string // path to the certificate file + Err error // In the case of ErrParseCert, the parsing error +} + +// Error returns an error with a reason related to certificate and its path. +func (ce *CertError) Error() string { + msg := ce.Path + " " + string(ce.Type) + if ce.Err != nil { + msg += " [" + ce.Err.Error() + "]" + } + + return msg +} + +// NumberError records a number related error. +type NumberError struct { + Err error +} + +// Error returns a number related error. +func (n *NumberError) Error() string { + return fmt.Sprintf("failed to generate serial number: %s", n.Err) +} + +// GenerateCerts creates a CA certificate which is used to sign a created server +// certificate which will have a corresponding key, all saved as PEM files. An +// error is generated if any of the files already exist. +// +// randReader := crand.Reader to be declared in calling function. +func GenerateCerts(caFile, serverPemFile, serverKeyFile, domain string, + bitsForRootRSAKey int, bitsForServerRSAKey int, randReader io.Reader, fileFlags int) error { + err := checkIfCertsExist([]string{caFile, serverPemFile, serverKeyFile}) + if err != nil { + return err + } + + // generate RSA keys for root + rootKey, err := rsa.GenerateKey(crand.Reader, bitsForRootRSAKey) + if err != nil { + return err + } + + // generate RSA keys for server + serverKey, err := rsa.GenerateKey(crand.Reader, bitsForServerRSAKey) + if err != nil { + return err + } + + // generate root CA + err = generateCertificates(caFile, domain, rootKey, serverKey, serverPemFile, randReader, fileFlags) + if err != nil { + return err + } + + // store the server's key + pemBlock := &pem.Block{ + Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(serverKey)} + err = encodeAndSavePEM(pemBlock, serverKeyFile, serverKeyFlags, serverKeyMode) + + return err +} + +// checkIfCertsExist checks if any of the certificate files exist, if yes then +// returns error. +func checkIfCertsExist(certFiles []string) error { + for _, cFile := range certFiles { + if _, err := os.Stat(cFile); err == nil { + return &CertError{Type: ErrCertExists, Path: cFile, Err: err} + } + } + + return nil +} + +// generateCertificates generates root and server certificates. +func generateCertificates(caFile, domain string, rootKey *rsa.PrivateKey, serverKey *rsa.PrivateKey, + serverPemFile string, randReader io.Reader, fileFlags int) error { + // create templates for root and server certificates. + // rootCertTemplate : rootServerTemplates[0] + // serverCertTemplate : rootServerTemplates[1] + rootServerTemplates := make([]*x509.Certificate, 2) + for i := 0; i < len(rootServerTemplates); i++ { + certTmplt, err := certTemplate(domain, randReader) + if err != nil { + return err + } + + rootServerTemplates[i] = certTmplt + } + + // generate server cert + rootServerTemplates[0].IsCA = true + rootServerTemplates[0].KeyUsage |= x509.KeyUsageCertSign + + rootCert, err := generateRootCert(caFile, rootServerTemplates[0], rootKey, randReader, fileFlags) + if err != nil { + return err + } + + // generate server cert, signed by root CA + err = generateServerCert(serverPemFile, rootCert, rootServerTemplates[1], rootKey, serverKey, + randReader, fileFlags) + + return err +} + +// certTemplate creates a certificate template with a random serial number, +// valid from now until validFor. It will be valid for supplied domain. +func certTemplate(domain string, randReader io.Reader) (*x509.Certificate, error) { + serialNumLimit := new(big.Int).Lsh(big.NewInt(1), 128) + + serialNumber, err := crand.Int(randReader, serialNumLimit) + if err != nil { + return nil, &NumberError{Err: err} + } + + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{Organization: []string{"wr manager"}}, + SignatureAlgorithm: x509.SHA256WithRSA, + NotBefore: time.Now(), + NotAfter: time.Now().Add(validFor), + IPAddresses: []net.IP{net.ParseIP("0.0.0.0"), net.ParseIP("127.0.0.1")}, + DNSNames: []string{domain}, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + return &template, nil +} + +// generateRootCert generates and returns root certificate. +func generateRootCert(caFile string, template *x509.Certificate, rootKey *rsa.PrivateKey, + randReader io.Reader, fileFlags int) (*x509.Certificate, error) { + // generate root certificate + rootCertByte, err := createCertFromTemplate(template, template, &rootKey.PublicKey, rootKey, randReader) + if err != nil { + return nil, err + } + + rootCert, err := parseCertAndSavePEM(rootCertByte, caFile, fileFlags) + if err != nil { + return nil, err + } + + return rootCert, err +} + +// createCertFromTemplate creates a certificate given a template, siginign it +// against its parent. Returned in DER encoding. +func createCertFromTemplate(template, parentCert *x509.Certificate, pubKey interface{}, + parentPvtKey interface{}, randReader io.Reader) ([]byte, error) { + certDER, err := x509.CreateCertificate(randReader, template, parentCert, pubKey, parentPvtKey) + if err != nil { + return nil, &CertError{Type: ErrCertCreate, Err: err} + } + + return certDER, nil +} + +// parseCertAndSavePEM parses the certificate to reuse it and saves it in PEM +// format to certPath. +func parseCertAndSavePEM(certByte []byte, certPath string, flags int) (*x509.Certificate, error) { + // parse the resulting certificate so we can use it again + cert, err := x509.ParseCertificate(certByte) + if err != nil { + return nil, &CertError{Type: ErrCertParse, Path: certPath, Err: err} + } + + block := &pem.Block{Type: "CERTIFICATE", Bytes: certByte} + + err = encodeAndSavePEM(block, certPath, flags, certMode) + if err != nil { + return nil, err + } + + return cert, nil +} + +// encodeAndSavePEM encodes the certificate and saves it in PEM format. +func encodeAndSavePEM(block *pem.Block, certPath string, flags int, mode os.FileMode) error { + certOut, err := os.OpenFile(certPath, flags, mode) + if err != nil { + return &CertError{Type: ErrCertCreate, Path: certPath, Err: err} + } + + err = pem.Encode(certOut, block) + if err != nil { + return &CertError{Type: ErrCertEncode, Path: certPath, Err: err} + } + + err = certOut.Close() + + return err +} + +// generateServerCert generates and returns server certificate signed by root +// CA. +func generateServerCert(serverPemFile string, rootCert *x509.Certificate, template *x509.Certificate, + rootKey *rsa.PrivateKey, serverKey *rsa.PrivateKey, randReader io.Reader, fileFlags int) error { + // generate server cert + servCertBtye, err := createCertFromTemplate(template, rootCert, &serverKey.PublicKey, rootKey, randReader) + if err != nil { + return err + } + + _, err = parseCertAndSavePEM(servCertBtye, serverPemFile, fileFlags) + if err != nil { + return err + } + + return err +} + +// CheckCerts checks if the given cert and key file are readable. If one or +// both of them are not, returns an error. +func CheckCerts(serverPemFile string, serverKeyFile string) error { + if _, err := os.Stat(serverPemFile); err != nil { + return err + } else if _, err := os.Stat(serverKeyFile); err != nil { + return err + } + + return nil +} + +// CertExpiry returns the time that the certificate given by the path to a pem +// file will expire. +func CertExpiry(certFile string) (time.Time, error) { + certPEMBlock, err := ioutil.ReadFile(certFile) + if err != nil { + return time.Now(), err + } + + cert := findPEMBlockAndReturnCert(certPEMBlock) + if len(cert.Certificate) == 0 { + return time.Now(), &CertError{Type: ErrCertNotFound, Path: certFile} + } + + // We don't need to parse the public key for TLS, but we so do anyway + // to check that it looks sane and matches the private key. + x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return time.Now(), &CertError{Type: ErrCertParse, Path: certFile, Err: err} + } + + return x509Cert.NotAfter, nil +} + +// findPEMBlockAndReturnCert finds the next PEM formatted block in the input +// and then returns a tls certificate. +func findPEMBlockAndReturnCert(certPEMBlock []byte) tls.Certificate { + var cert tls.Certificate + + for { + var certDERBlock *pem.Block + + certDERBlock, certPEMBlock = pem.Decode(certPEMBlock) + if certDERBlock == nil { + break + } + + if certDERBlock.Type == "CERTIFICATE" { + cert.Certificate = append(cert.Certificate, certDERBlock.Bytes) + } + } + + return cert +} diff --git a/internal/cert_test.go b/internal/cert_test.go new file mode 100644 index 0000000..27f8d5d --- /dev/null +++ b/internal/cert_test.go @@ -0,0 +1,272 @@ +/******************************************************************************* + * Copyright (c) 2020 Genome Research Ltd. + * + * Author: Ashwini Chhipa + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ + +package internal + +import ( + "bytes" + crand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "io/ioutil" + "log" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/smartystreets/goconvey/convey" +) + +const ( + bitsForRootRSAKey int = 2048 + blockFileWrite int = os.O_RDONLY | os.O_CREATE | os.O_TRUNC + fileMode os.FileMode = 0600 +) + +func TestCertFuncs(t *testing.T) { + Convey("Given the certificates key file paths", t, func() { + certtmpdir, err1 := ioutil.TempDir("/tmp", "wr_jobqueue_cert_dir_") + if err1 != nil { + log.Fatal(err1) + } + defer os.RemoveAll(certtmpdir) + + caFile := filepath.Join(certtmpdir, "ca.pem") + certFile := filepath.Join(certtmpdir, "cert.pem") + keyFile := filepath.Join(certtmpdir, "key.pem") + certDomain := "localhost" + + Convey("Check that they don't exist", func() { + err := checkIfCertsExist([]string{caFile, certFile, keyFile}) + So(err, ShouldBeNil) + }) + + Convey("Given an RSA key and a certificate template", func() { + rsaKey, err := rsa.GenerateKey(crand.Reader, bitsForRootRSAKey) + So(err, ShouldBeNil) + So(rsaKey, ShouldNotBeNil) + + r := bytes.NewReader([]byte{}) + errCertTmplt, err := certTemplate(certDomain, r) + So(err, ShouldNotBeNil) + So(errCertTmplt, ShouldBeNil) + + certTmplt, err := certTemplate(certDomain, crand.Reader) + So(err, ShouldBeNil) + So(certTmplt, ShouldNotBeNil) + + Convey("it can create a certificate from it", func() { + Convey("not when an empty template is used", func() { + testTmpl := x509.Certificate{} + certByte, err := createCertFromTemplate(&testTmpl, certTmplt, &rsaKey.PublicKey, rsaKey, crand.Reader) + So(err, ShouldNotBeNil) + So(certByte, ShouldBeNil) + }) + + Convey("when a non-empty template is used", func() { + certByte, err := createCertFromTemplate(certTmplt, certTmplt, &rsaKey.PublicKey, rsaKey, crand.Reader) + So(err, ShouldBeNil) + So(certByte, ShouldNotBeNil) + + Convey("and given a pemblock, it can encode and save pem file", func() { + pemBlock := &pem.Block{Type: "CERTIFICATE", Bytes: certByte} + Convey("when file can be written", func() { + err = encodeAndSavePEM(pemBlock, caFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileMode) + So(err, ShouldBeNil) + }) + + Convey("not when file cannot be created", func() { + err = encodeAndSavePEM(pemBlock, caFile, os.O_RDONLY, fileMode) + So(err, ShouldNotBeNil) + }) + + Convey("not when file cannot be written", func() { + err = encodeAndSavePEM(pemBlock, caFile, blockFileWrite, fileMode) + So(err, ShouldNotBeNil) + }) + }) + + Convey("and parse the Certificate", func() { + Convey("for a non-empty certificate template byte", func() { + cert, err := parseCertAndSavePEM(certByte, caFile, certFileFlags) + So(cert, ShouldNotBeNil) + So(err, ShouldBeNil) + }) + + Convey("but not for a empty certificate template byte", func() { + empByte := []byte{} + errCert, err := parseCertAndSavePEM(empByte, caFile, certFileFlags) + So(errCert, ShouldBeNil) + So(err, ShouldNotBeNil) + }) + + Convey("and not when file cannot be written", func() { + cert, err := parseCertAndSavePEM(certByte, caFile, blockFileWrite) + So(cert, ShouldBeNil) + So(err, ShouldNotBeNil) + }) + }) + }) + + Convey("generate a root certificate", func() { + rootCert, err := generateRootCert(caFile, certTmplt, rsaKey, crand.Reader, certFileFlags) + So(rootCert, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("not with an empty template", func() { + empRootCert, err := generateRootCert(caFile, &x509.Certificate{}, rsaKey, crand.Reader, certFileFlags) + So(empRootCert, ShouldBeNil) + So(err, ShouldNotBeNil) + }) + + Convey("and not when file cannot be written", func() { + empRootCert, err := generateRootCert(caFile, certTmplt, rsaKey, crand.Reader, blockFileWrite) + So(empRootCert, ShouldBeNil) + So(err, ShouldNotBeNil) + }) + + Convey("and then generate a server certificate", func() { + err := generateServerCert(certFile, rootCert, certTmplt, rsaKey, rsaKey, crand.Reader, certFileFlags) + So(err, ShouldBeNil) + + Convey("not with an empty template", func() { + err = generateServerCert(certFile, rootCert, &x509.Certificate{}, rsaKey, rsaKey, crand.Reader, certFileFlags) + So(err, ShouldNotBeNil) + }) + + Convey("and not when file cannot be written", func() { + err = generateServerCert(certFile, rootCert, certTmplt, rsaKey, rsaKey, crand.Reader, blockFileWrite) + So(err, ShouldNotBeNil) + }) + }) + }) + }) + }) + + Convey("and an RSA key, it can generate both root and server certificates", func() { + rsaKey, err := rsa.GenerateKey(crand.Reader, bitsForRootRSAKey) + So(err1, ShouldBeNil) + + err = generateCertificates(caFile, certDomain, rsaKey, rsaKey, certFile, crand.Reader, certFileFlags) + So(err, ShouldBeNil) + + Convey("not with an empty serial number in template", func() { + err = generateCertificates(caFile, certDomain, rsaKey, rsaKey, certFile, bytes.NewReader([]byte{}), certFileFlags) + So(err, ShouldNotBeNil) + }) + + Convey("and not when files cannot be written", func() { + err = generateCertificates(caFile, certDomain, rsaKey, rsaKey, certFile, crand.Reader, blockFileWrite) + So(err, ShouldNotBeNil) + }) + + Convey("and it can store the server's private key", func() { + pemBlock := &pem.Block{ + Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rsaKey)} + err = encodeAndSavePEM(pemBlock, keyFile, serverKeyFlags, serverKeyMode) + So(err, ShouldBeNil) + }) + }) + + Convey("it can generate the root and server certificate", func() { + Convey("not when zero bits for root rsa key is used", func() { + err := GenerateCerts(caFile, certFile, keyFile, certDomain, 0, bitsForRootRSAKey, crand.Reader, certFileFlags) + So(err, ShouldNotBeNil) + }) + + Convey("not when zero bits for server rsa key is used", func() { + err := GenerateCerts(caFile, certFile, keyFile, certDomain, bitsForRootRSAKey, 0, crand.Reader, certFileFlags) + So(err, ShouldNotBeNil) + }) + + Convey("not when files cannot be written", func() { + err := GenerateCerts(caFile, certFile, keyFile, certDomain, bitsForRootRSAKey, bitsForRootRSAKey, crand.Reader, + blockFileWrite) + So(err, ShouldNotBeNil) + }) + + Convey("when bits and file flags are correct", func() { + err := GenerateCerts(caFile, certFile, keyFile, certDomain, bitsForRootRSAKey, bitsForRootRSAKey, crand.Reader, + certFileFlags) + So(err, ShouldBeNil) + + Convey("check if cert files exists", func() { + err = checkIfCertsExist([]string{caFile, certFile, keyFile}) + So(err, ShouldNotBeNil) + }) + + Convey("trying to generate certificates again will fail", func() { + err = GenerateCerts(caFile, certFile, keyFile, certDomain, bitsForRootRSAKey, bitsForRootRSAKey, crand.Reader, + certFileFlags) + So(err, ShouldNotBeNil) + }) + + Convey("Check if certificate files are readable", func() { + err = CheckCerts(certFile, keyFile) + So(err, ShouldBeNil) + err = CheckCerts("/tmp/random.pem", keyFile) + So(err, ShouldNotBeNil) + err = CheckCerts(certFile, "/tmp/random.pem") + So(err, ShouldNotBeNil) + }) + + Convey("Find PEM Block in a file and Return Certifcate", func() { + certPEMBlock, err := ioutil.ReadFile(certFile) + So(err, ShouldBeNil) + + ccert := findPEMBlockAndReturnCert(certPEMBlock) + So(ccert, ShouldNotBeNil) + + ccert1 := findPEMBlockAndReturnCert([]byte{}) + So(len(ccert1.Certificate), ShouldEqual, 0) + }) + + Convey("Check that certificate expires in a year", func() { + expiry, err := CertExpiry(caFile) + So(err, ShouldBeNil) + So(expiry, ShouldHappenBetween, time.Now().Add(364*24*time.Hour), time.Now().Add(366*24*time.Hour)) + + _, err = CertExpiry("/tmp/exp.pem") + So(err, ShouldNotBeNil) + + empCertFile := filepath.Join(certtmpdir, "emp.pem") + err = ioutil.WriteFile(empCertFile, []byte{0}, fileMode) + So(err, ShouldBeNil) + + expiry, err = CertExpiry(empCertFile) + So(expiry, ShouldNotBeNil) + So(err, ShouldNotBeNil) + }) + + Convey("test check", func() { + expiry, err := CertExpiry("testdata/cert/wrongCert.pem") + So(expiry, ShouldNotBeNil) + So(err, ShouldNotBeNil) + }) + }) + }) + }) +} diff --git a/internal/config.go b/internal/config.go new file mode 100644 index 0000000..67137fe --- /dev/null +++ b/internal/config.go @@ -0,0 +1,545 @@ +/******************************************************************************* + * Copyright (c) 2020 Genome Research Ltd. + * + * Author: Sendu Bala , Ashwini Chhipa + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ + +package internal + +import ( + "context" + "fmt" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + + "github.com/creasty/defaults" + "github.com/jinzhu/configor" + "github.com/olekukonko/tablewriter" + "github.com/wtsi-ssg/wr/clog" +) + +const ( + // configCommonBasename is the basename of a wr config file. + configCommonBasename = ".wr_config.yml" + + // S3Prefix is the prefix used by S3 paths. + S3Prefix = "s3://" + + // Production is the name of the main deployment. + Production = "production" + + // Development is the name of the development deployment, used during + // testing. + Development = "development" + + // ConfigSourceEnvVar is a config value source. + ConfigSourceEnvVar = "env var" + + // ConfigSourceDefault is a config value source. + ConfigSourceDefault = "default" + + // sourcesProperty is a source property. + sourcesProperty = "sources" + + // maxPort is the maximum port available. + maxPort = 65535 + + // minport is the minimum port used. + minPort = 1021 + + // portsPerUser is the number of ports used for each user. + portsPerUser = 4 +) + +// Config holds the configuration options for jobqueue server and client. +type Config struct { + ManagerPort string `default:""` + ManagerWeb string `default:""` + ManagerHost string `default:"localhost"` + ManagerDir string `default:"~/.wr"` + ManagerPidFile string `default:"pid"` + ManagerLogFile string `default:"log"` + ManagerDBFile string `default:"db"` + ManagerDBBkFile string `default:"db_bk"` + ManagerTokenFile string `default:"client.token"` + ManagerUploadDir string `default:"uploads"` + ManagerUmask int `default:"007"` + ManagerScheduler string `default:"local"` + ManagerCAFile string `default:"ca.pem"` + ManagerCertFile string `default:"cert.pem"` + ManagerKeyFile string `default:"key.pem"` + ManagerCertDomain string `default:"localhost"` + ManagerSetDomainIP bool `default:"false"` + RunnerExecShell string `default:"bash"` + Deployment string `default:"production"` + CloudFlavor string `default:""` + CloudFlavorManager string `default:""` + CloudFlavorSets string `default:""` + CloudKeepAlive int `default:"120"` + CloudServers int `default:"-1"` + CloudCIDR string `default:"192.168.0.0/18"` + CloudGateway string `default:"192.168.0.1"` + CloudDNS string `default:"8.8.4.4,8.8.8.8"` + CloudOS string `default:"bionic-server"` + ContainerImage string `default:"ubuntu:latest"` + CloudUser string `default:"ubuntu"` + CloudRAM int `default:"2048"` + CloudDisk int `default:"1"` + CloudScript string `default:""` + CloudConfigFiles string `default:"~/.s3cfg,~/.aws/credentials,~/.aws/config"` + CloudSpawns int `default:"10"` + CloudAutoConfirmDead int `default:"30"` + DeploySuccessScript string `default:""` + sources map[string]string +} + +// FileExistsError records a file exist error. +type FileExistsError struct { + Path string + Err error +} + +// Error returns error when a file already exists. +func (f *FileExistsError) Error() string { + return fmt.Sprintf("file [%s] already exists: %s", f.Path, f.Err) +} + +// Source returns where the value of a Config field was defined. +func (c Config) Source(field string) string { + if c.sources == nil { + return ConfigSourceDefault + } + + source, set := c.sources[field] + if !set { + return ConfigSourceDefault + } + + return source +} + +// IsProduction tells you if we're in the production deployment. +func (c Config) IsProduction() bool { + return c.Deployment == Production +} + +// String retruns the string value of property. +func (c Config) String() string { + vals := reflect.ValueOf(c) + typeOfC := vals.Type() + + tableString := &strings.Builder{} + table := tablewriter.NewWriter(tableString) + table.SetHeader([]string{"Config", "Value", "Source"}) + table.SetAlignment(tablewriter.ALIGN_LEFT) + + for i := 0; i < vals.NumField(); i++ { + property := typeOfC.Field(i).Name + if property == sourcesProperty { + continue + } + + source := c.sources[property] + if source == "" { + source = ConfigSourceDefault + } + + table.Append([]string{property, fmt.Sprintf("%v", vals.Field(i).Interface()), source}) + } + + table.Render() + + return tableString.String() +} + +// ConfigLoadFromParentDir loads and returns the config from a parent directory. +func ConfigLoadFromParentDir(ctx context.Context, deployment string) *Config { + pwd := GetPWD(ctx) + pwd = filepath.Dir(pwd) + + uid := os.Getuid() + + return mergeAllConfigs(ctx, uid, deployment, pwd, true) +} + +// ConfigLoadFromParentDir loads and returns the config from a non-parent directory. +func ConfigLoadFromNonParentDir(ctx context.Context, deployment string) *Config { + pwd := GetPWD(ctx) + + uid := os.Getuid() + + return mergeAllConfigs(ctx, uid, deployment, pwd, false) +} + +// mergeAllConfigs function loads and merges all the configs and returns a final config. +func mergeAllConfigs(ctx context.Context, uid int, deployment string, pwd string, useparentdir bool) *Config { + // if deployment not set on the command line + if deployment != Development && deployment != Production { + deployment = DefaultDeployment(ctx) + } + + // load and merge default and env vars configs + configDef := mergeDefaultAndEnvVarsConfigs(ctx) + + // read all config files and merge them + configDef.mergeAllConfigFiles(ctx, uid, deployment, pwd, useparentdir) + + return configDef +} + +// DefaultDeployment works out the default deployment. +func DefaultDeployment(ctx context.Context) string { + pwd := GetPWD(ctx) + + // if we're in the git repository + var deployment string + + _, err := os.Stat(filepath.Join(pwd, "jobqueue", "server.go")) + if err == nil { + // force development + deployment = Development + } else { + // default to production + deployment = Production + } + + // and allow env var to override with development + deploymentEnv := os.Getenv("WR_DEPLOYMENT") + if deploymentEnv != "" { + if deploymentEnv == Development { + deployment = Development + } + } + + return deployment +} + +// mergeDefaultAndEnvVarsConfigs returns a merged Default and EnvVar config. +func mergeDefaultAndEnvVarsConfigs(ctx context.Context) *Config { + // load default config before setting env vars using configor + configDef := loadDefaultConfig(ctx) + + // set ManagerUmask + setenvManagerUmask() + + // load env var using configor + configEnv := getEnvVarsConfig(ctx) + + // merge default and env var config + configDef.merge(configEnv, ConfigSourceEnvVar) + + return configDef +} + +// getDefaultConfig loads and return the default configs. +func loadDefaultConfig(ctx context.Context) *Config { + // because we want to know the source of every value, we can't take + // advantage of configor.Load() being able to take all env vars and config + // files at once. We do it repeatedly and merge results instead + config := &Config{} + if cerr := defaults.Set(config); cerr != nil { + clog.Fatal(ctx, cerr.Error()) + } + + return config +} + +// setenvManagerUmask sets the WR_MANAGERUMASK env variable. +func setenvManagerUmask() { + // load env vars. ManagerUmask is likely to be zero prefixed by user, but + // that is not converted to int correctly, so fix first + umask := os.Getenv("WR_MANAGERUMASK") + if umask != "" && strings.HasPrefix(umask, "0") { + umask = strings.TrimLeft(umask, "0") + os.Setenv("WR_MANAGERUMASK", umask) + } +} + +// getEnvVarsConfig loads the env variables and returns the config. +func getEnvVarsConfig(ctx context.Context) *Config { + // we don't os.Setenv("CONFIGOR_ENV", deployment) to stop configor loading + // files before we want it to + err := os.Setenv("CONFIGOR_ENV_PREFIX", "WR") + if err != nil { + clog.Fatal(ctx, err.Error()) + } + + configEnv := &Config{} + + err = configor.Load(configEnv) + if err != nil { + clog.Fatal(ctx, err.Error()) + } + + return configEnv +} + +// merge compares existing to new Config values, and for each one that has +// changed, sets the given source on the changed property in our sources, +// and sets the new value on ourselves. +func (c *Config) merge(new *Config, source string) { + v := reflect.ValueOf(*c) + typeOfC := v.Type() + vNew := reflect.ValueOf(*new) + + if c.sources == nil { + c.sources = make(map[string]string) + } + + for i := 0; i < v.NumField(); i++ { + property := typeOfC.Field(i).Name + if property == sourcesProperty { + continue + } + + if vNew.Field(i).Interface() != v.Field(i).Interface() { + c.sources[property] = source + + adrField := reflect.ValueOf(c).Elem().Field(i) + setSourceOnChangeProp(typeOfC, adrField, vNew, i) + } + } +} + +// mergeAllConfigFiles merges all the config files and adjusts config properties. +func (c *Config) mergeAllConfigFiles(ctx context.Context, uid int, deployment string, pwd string, useparentdir bool) { + configDeploymentBasename := ".wr_config." + deployment + ".yml" + + if configDir := os.Getenv("WR_CONFIG_DIR"); configDir != "" { + c.configLoadFromFile(ctx, filepath.Join(configDir, configCommonBasename)) + c.configLoadFromFile(ctx, filepath.Join(configDir, configDeploymentBasename)) + } + + if useparentdir { + home, herr := os.UserHomeDir() + if herr != nil || home == "" { + errStr := "could not find home dir" + clog.Fatal(ctx, errStr) + } + + c.configLoadFromFile(ctx, filepath.Join(home, configCommonBasename)) + c.configLoadFromFile(ctx, filepath.Join(home, configDeploymentBasename)) + } + + c.configLoadFromFile(ctx, filepath.Join(pwd, configCommonBasename)) + c.configLoadFromFile(ctx, filepath.Join(pwd, configDeploymentBasename)) + + // adjust config properties and return + c.adjustConfigProperties(ctx, uid, deployment) +} + +// configLoadFromFile loads a config from a file and merges into the current config. +func (c *Config) configLoadFromFile(ctx context.Context, path string) { + _, err := os.Stat(path) + if err != nil { + return + } + + configFile := c.clone() + + err = configor.Load(configFile, path) + if err != nil { + clog.Fatal(ctx, err.Error()) + } + + c.merge(configFile, path) +} + +// clone makes a new Config with our values. +func (c *Config) clone() *Config { + new := &Config{} + + v := reflect.ValueOf(*c) + typeOfC := v.Type() + + for i := 0; i < v.NumField(); i++ { + property := typeOfC.Field(i).Name + if property == sourcesProperty { + continue + } + + adrField := reflect.ValueOf(new).Elem().Field(i) + setSourceOnChangeProp(typeOfC, adrField, v, i) + } + + new.sources = make(map[string]string) + for key, val := range c.sources { + new.sources[key] = val + } + + return new +} + +// setSourceOnChangeProp sets the source of a property, when its value is changed. +func setSourceOnChangeProp(typeOfC reflect.Type, adrField reflect.Value, newVal reflect.Value, idx int) { + switch typeOfC.Field(idx).Type.Kind() { + case reflect.String: + adrField.SetString(newVal.Field(idx).String()) + case reflect.Int: + adrField.SetInt(newVal.Field(idx).Int()) + case reflect.Bool: + adrField.SetBool(newVal.Field(idx).Bool()) + default: + return + } +} + +// adjustConfigProperties adjusts te config properties for pid, log file, upload dir paths; certs and db files. +func (c *Config) adjustConfigProperties(ctx context.Context, uid int, deployment string) { + c.Deployment = deployment + + // convert the possible ~/ in Manager_dir to abs path to user's home + c.ManagerDir = TildaToHome(c.ManagerDir) + c.ManagerDir += "_" + deployment + + c.convRelativeToAbsManagerPaths() + c.convRelativeToAbsManagerPathForCert() + c.convRelativeToAbsManagerPathForDBFiles() + c.setManagerPort(ctx, uid) +} + +// convRelativeToAbsManagerPath converts the possible relative paths of pid, logfile and upload dir to +// abs paths in ManagerDir. +func (c *Config) convRelativeToAbsManagerPaths() { + if !filepath.IsAbs(c.ManagerPidFile) { + c.ManagerPidFile = filepath.Join(c.ManagerDir, c.ManagerPidFile) + } + + if !filepath.IsAbs(c.ManagerLogFile) { + c.ManagerLogFile = filepath.Join(c.ManagerDir, c.ManagerLogFile) + } + + if !filepath.IsAbs(c.ManagerUploadDir) { + c.ManagerUploadDir = filepath.Join(c.ManagerDir, c.ManagerUploadDir) + } +} + +// convRelativeToAbsManagerPathForCert converts the possible relative paths in cert files to +// abs paths in ManagerDir. +func (c *Config) convRelativeToAbsManagerPathForCert() { + if !filepath.IsAbs(c.ManagerCAFile) { + c.ManagerCAFile = filepath.Join(c.ManagerDir, c.ManagerCAFile) + } + + if !filepath.IsAbs(c.ManagerCertFile) { + c.ManagerCertFile = filepath.Join(c.ManagerDir, c.ManagerCertFile) + } + + if !filepath.IsAbs(c.ManagerKeyFile) { + c.ManagerKeyFile = filepath.Join(c.ManagerDir, c.ManagerKeyFile) + } + + if !filepath.IsAbs(c.ManagerTokenFile) { + c.ManagerTokenFile = filepath.Join(c.ManagerDir, c.ManagerTokenFile) + } +} + +// convRelativeToAbsManagerPathForDBFiles converts the possible relative paths in db files to +// abs paths in ManagerDir. +func (c *Config) convRelativeToAbsManagerPathForDBFiles() { + if !filepath.IsAbs(c.ManagerDBFile) { + c.ManagerDBFile = filepath.Join(c.ManagerDir, c.ManagerDBFile) + } + + if !filepath.IsAbs(c.ManagerDBBkFile) { + if !IsRemote(c.ManagerDBBkFile) { + c.ManagerDBBkFile = filepath.Join(c.ManagerDir, c.ManagerDBBkFile) + } + } +} + +// setManagerPort sets the cli and web interface ports for manager. +func (c *Config) setManagerPort(ctx context.Context, uid int) { + // if not explicitly set, calculate ports that no one else would be + // assigned by us (and hope no other software is using it...) + if c.ManagerPort == "" { + c.ManagerPort = calculatePort(ctx, uid, c.Deployment, "cli") + } + + if c.ManagerWeb == "" { + c.ManagerWeb = calculatePort(ctx, uid, c.Deployment, "webi") + } +} + +// Calculate a port number that will be unique to this user, deployment and +// ptype ("cli" or "webi"). +func calculatePort(ctx context.Context, uid int, deployment string, ptype string) string { + // get the minimum port number + pn := getMinPort(uid) + + if pn+3 > maxPort { + errStr := "Could not calculate a suitable unique port number for you, since your user id is so large;" + errStr += "please manually set your manager_port and manager_web config options." + clog.Fatal(ctx, errStr) + } + + if deployment == Development { + pn += 2 + } + + if ptype == "webi" { + pn++ + } + + // it's easier for the things that use this port number if it's a string + // (because it's used as part of a connection string) + return strconv.Itoa(pn) +} + +// getMinPort calculates and returns the minimum port available. +func getMinPort(uid int) int { + // our port must be greater than 1024, and by basing on user id we can + // avoid conflicts with other users of wr on the same machine; we + // multiply by 4 because we have to reserve 4 ports for each user + return minPort + (uid * portsPerUser) +} + +// DefaultConfig works out the default config for when we need to be able to +// report the default before we know what deployment the user has actually +// chosen, ie. before we have a final config. +func DefaultConfig(ctx context.Context) *Config { + return ConfigLoadFromNonParentDir(ctx, DefaultDeployment(ctx)) +} + +// DefaultServer works out the default server (we need this to be able to report +// this default before we know what deployment the user has actually chosen, ie. +// before we have a final config). +func DefaultServer(ctx context.Context) string { + config := DefaultConfig(ctx) + + return config.ManagerHost + ":" + config.ManagerPort +} + +// InS3 tells you if a path is to a file in S3. +func InS3(path string) bool { + return strings.HasPrefix(path, S3Prefix) +} + +// IsRemote tells you if a path is to a remote file system or object store, +// based on its URI. +func IsRemote(path string) bool { + // (right now we only support S3, but IsRemote is to future-proof us and + // avoid calling InS3() directly) + return InS3(path) +} diff --git a/internal/config_test.go b/internal/config_test.go new file mode 100644 index 0000000..3f800bf --- /dev/null +++ b/internal/config_test.go @@ -0,0 +1,548 @@ +/******************************************************************************* + * Copyright (c) 2020 Genome Research Ltd. + * + * Author: Ashwini Chhipa + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ + +package internal + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "testing" + + . "github.com/smartystreets/goconvey/convey" + "github.com/wtsi-ssg/wr/clog" +) + +func fileTestSetup(dir, mport, mweb1, mweb2 string) (string, string, error) { + path := filepath.Join(dir, ".wr_config.yml") + + _, err := os.Stat(path) + if err == nil { + return path, "", &FileExistsError{Path: path, Err: nil} + } + + path2 := filepath.Join(dir, ".wr_config.development.yml") + + _, err = os.Stat(path2) + if err == nil { + return path, "", &FileExistsError{Path: path, Err: nil} + } + + file, err := os.Create(path) + if err != nil { + return path, path2, err + } + + file2, err := os.Create(path2) + if err != nil { + return path, path2, err + } + + _, err = file.WriteString(fmt.Sprintf("managerport: \"%s\"\n", mport)) + So(err, ShouldBeNil) + + _, err = file.WriteString(fmt.Sprintf("managerweb: \"%s\"\n", mweb1)) + So(err, ShouldBeNil) + file.Close() + + _, err = file2.WriteString(fmt.Sprintf("managerweb: \"%s\"\n", mweb2)) + So(err, ShouldBeNil) + file2.Close() + + return path, path2, nil +} + +func fileTestTeardown(path, path2 string) { + err := os.Remove(path) + if err != nil { + fmt.Printf("\nfailed to delete %s: %s\n", path, err) + } + + err = os.Remove(path2) + if err != nil { + fmt.Printf("\nfailed to delete %s: %s\n", path2, err) + } +} + +func TestConfig(t *testing.T) { + ctx := context.Background() + + Convey("Given a path it can check if", t, func() { + pathS3 := "s3://test1" + pathNotS3 := "/tmp/test2" + + Convey("it is a path to a file in S3 bucket", func() { + So(InS3(pathS3), ShouldEqual, true) + So(InS3(pathNotS3), ShouldEqual, false) + }) + + Convey("it is a remote file system path", func() { + So(IsRemote(pathS3), ShouldEqual, true) + So(IsRemote(pathNotS3), ShouldEqual, false) + }) + }) + + Convey("Given a user id", t, func() { + uid := 1000 + Convey("it can get the minimum port number for it", func() { + So(getMinPort(uid), ShouldEqual, 5021) + }) + + Convey("it can calculate a unique port for the user", func() { + Convey("for different deployments and port types", func() { + So(calculatePort(ctx, uid, "development", "webi"), ShouldEqual, "5024") + So(calculatePort(ctx, uid, "development", "cli"), ShouldEqual, "5023") + So(calculatePort(ctx, uid, "production", "webi"), ShouldEqual, "5022") + So(calculatePort(ctx, uid, "production", "cli"), ShouldEqual, "5021") + }) + + Convey("but not if uid is a big number", func() { + uid = 65534 + buff := clog.ToBufferAtLevel("crit") + defer clog.ToDefault() + os.Setenv("FATAL_EXIT_TEST", "1") + defer os.Unsetenv("FATAL_EXIT_TEST") + _ = calculatePort(ctx, uid, "development", "webi") + bufferStr := buff.String() + So(bufferStr, ShouldContainSubstring, "fatal=true") + So(bufferStr, ShouldNotContainSubstring, "caller=clog") + So(bufferStr, ShouldContainSubstring, "stack=") + So(bufferStr, ShouldContainSubstring, "user id is so large") + }) + }) + }) + + Convey("Set WR Manager umask", t, func() { + Convey("When env variable is not set", func() { + setenvManagerUmask() + So(os.Getenv("WR_MANAGERUMASK"), ShouldBeEmpty) + }) + + Convey("When env variable is set but umask doesn't have 0 prefix", func() { + os.Setenv("WR_MANAGERUMASK", "666") + defer func() { + os.Unsetenv("WR_MANAGERUMASK") + }() + setenvManagerUmask() + So(os.Getenv("WR_MANAGERUMASK"), ShouldEqual, "666") + }) + + Convey("When env variable is set but umask has 0 prefix", func() { + os.Setenv("WR_MANAGERUMASK", "0666") + defer func() { + os.Unsetenv("WR_MANAGERUMASK") + }() + setenvManagerUmask() + So(os.Getenv("WR_MANAGERUMASK"), ShouldEqual, "666") + }) + }) + + Convey("Given a default wr config", t, func() { + defConfig := loadDefaultConfig(ctx) + So(defConfig, ShouldNotBeNil) + So(defConfig.ManagerPort, ShouldBeEmpty) + So(defConfig.Source("ManagerPort"), ShouldEqual, "default") + So(defConfig.ManagerWeb, ShouldBeEmpty) + + Convey("it can check if deployment is production", func() { + So(defConfig.IsProduction(), ShouldBeTrue) + }) + + Convey("it can clone it", func() { + clonedConfig := defConfig.clone() + So(defConfig.ManagerHost, ShouldEqual, clonedConfig.ManagerHost) + So(defConfig.CloudCIDR, ShouldEqual, clonedConfig.CloudCIDR) + So(defConfig.CloudDNS, ShouldEqual, clonedConfig.CloudDNS) + So(defConfig.CloudRAM, ShouldEqual, clonedConfig.CloudRAM) + So(defConfig.CloudAutoConfirmDead, ShouldEqual, clonedConfig.CloudAutoConfirmDead) + }) + + Convey("and a user id, it can set the manager port", func() { + uid := 1000 + So(defConfig.ManagerPort, ShouldBeEmpty) + So(defConfig.ManagerWeb, ShouldBeEmpty) + + defConfig.setManagerPort(ctx, uid) + + So(defConfig.ManagerPort, ShouldEqual, "5021") + So(defConfig.ManagerWeb, ShouldEqual, "5022") + }) + + Convey("it can convert the relative to Abs path for DB files", func() { + So(defConfig.ManagerDir, ShouldEqual, "~/.wr") + So(defConfig.ManagerDBFile, ShouldEqual, "db") + So(defConfig.ManagerDBBkFile, ShouldEqual, "db_bk") + + defConfig.convRelativeToAbsManagerPathForDBFiles() + + So(defConfig.ManagerDBFile, ShouldEqual, "~/.wr/db") + So(defConfig.ManagerDBBkFile, ShouldEqual, "~/.wr/db_bk") + }) + + Convey("it can convert the relative to Abs path for certificate files", func() { + So(defConfig.ManagerDir, ShouldEqual, "~/.wr") + So(defConfig.ManagerCAFile, ShouldEqual, "ca.pem") + So(defConfig.ManagerCertFile, ShouldEqual, "cert.pem") + So(defConfig.ManagerKeyFile, ShouldEqual, "key.pem") + So(defConfig.ManagerTokenFile, ShouldEqual, "client.token") + + defConfig.convRelativeToAbsManagerPathForCert() + + So(defConfig.ManagerCAFile, ShouldEqual, "~/.wr/ca.pem") + So(defConfig.ManagerCertFile, ShouldEqual, "~/.wr/cert.pem") + So(defConfig.ManagerKeyFile, ShouldEqual, "~/.wr/key.pem") + So(defConfig.ManagerTokenFile, ShouldEqual, "~/.wr/client.token") + }) + + Convey("it can convert the relative to Abs path for other paths", func() { + So(defConfig.ManagerDir, ShouldEqual, "~/.wr") + So(defConfig.ManagerPidFile, ShouldEqual, "pid") + So(defConfig.ManagerLogFile, ShouldEqual, "log") + So(defConfig.ManagerUploadDir, ShouldEqual, "uploads") + + defConfig.convRelativeToAbsManagerPaths() + + So(defConfig.ManagerPidFile, ShouldEqual, "~/.wr/pid") + So(defConfig.ManagerLogFile, ShouldEqual, "~/.wr/log") + So(defConfig.ManagerUploadDir, ShouldEqual, "~/.wr/uploads") + }) + + Convey("user id and deployment type, it can adjust config properties", func() { + uid := 1000 + deployment := "development" + userHomeDir, err := os.UserHomeDir() + So(err, ShouldBeNil) + expectedManageDir := filepath.Join(userHomeDir, ".wr_"+deployment) + + So(defConfig.ManagerDir, ShouldEqual, "~/.wr") + + defConfig.adjustConfigProperties(ctx, uid, deployment) + + So(defConfig.ManagerDir, ShouldEqual, expectedManageDir) + So(defConfig.ManagerPidFile, ShouldEqual, filepath.Join(expectedManageDir, "pid")) + So(defConfig.ManagerCAFile, ShouldEqual, filepath.Join(expectedManageDir, "ca.pem")) + So(defConfig.ManagerDBFile, ShouldEqual, filepath.Join(expectedManageDir, "db")) + So(defConfig.ManagerPort, ShouldEqual, "5023") + }) + + Convey("it can also merge with another config", func() { + otherConfig := loadDefaultConfig(ctx) + otherConfig.ManagerPort = "2000" + + defConfig.merge(otherConfig, "default") + So(otherConfig.ManagerPort, ShouldEqual, "2000") + So(defConfig.ManagerPort, ShouldEqual, "2000") + }) + + Convey("It can be overridden with a config file given its path", func() { + dir, err := ioutil.TempDir("", "wr_conf_test") + So(err, ShouldBeNil) + defer os.RemoveAll(dir) + + mport := "1234" + mweb1 := "1235" + mweb2 := "1236" + path, path2, err := fileTestSetup(dir, mport, mweb1, mweb2) + defer fileTestTeardown(path, path2) + So(err, ShouldBeNil) + + defConfig.configLoadFromFile(ctx, path) + So(defConfig.ManagerPort, ShouldEqual, mport) + So(defConfig.Source("ManagerPort"), ShouldEqual, path) + + So(defConfig.ManagerWeb, ShouldEqual, mweb1) + So(defConfig.Source("ManagerWeb"), ShouldEqual, path) + + defConfig.configLoadFromFile(ctx, path2) + So(defConfig.ManagerPort, ShouldEqual, mport) + So(defConfig.Source("ManagerPort"), ShouldEqual, path) + + So(defConfig.ManagerWeb, ShouldEqual, mweb2) + So(defConfig.Source("ManagerWeb"), ShouldEqual, path2) + + _, _, err = fileTestSetup(dir, mport, mweb1, mweb2) + So(err, ShouldNotBeNil) + }) + + Convey("These can be overridden with config files in WR_CONFIG_DIR", func() { + uid := 1000 + dir, err := ioutil.TempDir("", "wr_conf_test") + So(err, ShouldBeNil) + defer os.RemoveAll(dir) + + mport := "1234" + mweb1 := "1235" + mweb2 := "1236" + path, path2, err := fileTestSetup(dir, mport, mweb1, mweb2) + defer fileTestTeardown(path, path2) + So(err, ShouldBeNil) + + os.Setenv("WR_CONFIG_DIR", dir) + defer func() { + os.Unsetenv("WR_CONFIG_DIR") + }() + + defConfig.mergeAllConfigFiles(ctx, uid, "production", "", false) + + So(defConfig.ManagerPort, ShouldEqual, mport) + So(defConfig.Source("ManagerPort"), ShouldEqual, path) + So(defConfig.ManagerWeb, ShouldEqual, mweb1) + So(defConfig.Source("ManagerWeb"), ShouldEqual, path) + + Convey("These can be overridden with config files in home dir", func() { + realHome, err := os.UserHomeDir() + So(err, ShouldBeNil) + newHome, err := ioutil.TempDir(dir, "home") + So(err, ShouldBeNil) + os.Setenv("HOME", newHome) + defer func() { + os.Setenv("HOME", realHome) + }() + home, err := os.UserHomeDir() + So(err, ShouldBeNil) + So(home, ShouldNotEqual, realHome) + + mport := "1334" + mweb1 := "1335" + mweb2 := "1336" + path3, path4, err := fileTestSetup(home, mport, mweb1, mweb2) + defer fileTestTeardown(path3, path4) + So(err, ShouldBeNil) + + defConfig.mergeAllConfigFiles(ctx, uid, "production", "", true) + So(defConfig.ManagerPort, ShouldEqual, mport) + So(defConfig.Source("ManagerPort"), ShouldEqual, path3) + So(defConfig.ManagerWeb, ShouldEqual, mweb1) + So(defConfig.Source("ManagerWeb"), ShouldEqual, path3) + + Convey("not if home directory is empty", func() { + os.Unsetenv("HOME") + buff := clog.ToBufferAtLevel("fatal") + defer clog.ToDefault() + os.Setenv("FATAL_EXIT_TEST", "1") + defer func() { + os.Setenv("HOME", realHome) + os.Unsetenv("FATAL_EXIT_TEST") + }() + defConfig.mergeAllConfigFiles(ctx, uid, "production", "", true) + + bufferStr := buff.String() + So(bufferStr, ShouldContainSubstring, "fatal=true") + So(bufferStr, ShouldNotContainSubstring, "caller=clog") + So(bufferStr, ShouldContainSubstring, "stack=") + }) + + Convey("These can be overridden with config files in current dir", func() { + pwd, err := os.Getwd() + So(err, ShouldBeNil) + mport = "1434" + mweb1 = "1435" + mweb2 = "1436" + path5, path6, err := fileTestSetup(pwd, mport, mweb1, mweb2) + defer fileTestTeardown(path5, path6) + So(err, ShouldBeNil) + + defConfig.mergeAllConfigFiles(ctx, uid, "production", pwd, true) + So(defConfig.ManagerPort, ShouldEqual, mport) + So(defConfig.Source("ManagerPort"), ShouldEqual, path5) + So(defConfig.ManagerWeb, ShouldEqual, mweb1) + So(defConfig.Source("ManagerWeb"), ShouldEqual, path5) + }) + }) + }) + }) + + Convey("Set source on the change of a config field property", t, func() { + type testConfig struct { + ManagerHost string `default:"localhost"` + ManagerUmask int `default:"7"` + RandomFloatValue float32 `default:"1.1"` + ManagerSetDomainIP bool `default:"false"` + } + + old := &testConfig{} + old.ManagerUmask = 4 + + new := &testConfig{} + + v := reflect.ValueOf(*old) + typeOfC := v.Type() + + adrFieldString := reflect.ValueOf(new).Elem().Field(0) + adrFieldBool := reflect.ValueOf(new).Elem().Field(1) + adrFieldInt := reflect.ValueOf(new).Elem().Field(2) + adrFieldFloat := reflect.ValueOf(new).Elem().Field(3) + + setSourceOnChangeProp(typeOfC, adrFieldString, v, 0) + setSourceOnChangeProp(typeOfC, adrFieldBool, v, 1) + setSourceOnChangeProp(typeOfC, adrFieldInt, v, 2) + setSourceOnChangeProp(typeOfC, adrFieldFloat, v, 3) + + So(new.ManagerUmask, ShouldEqual, 4) + }) + + Convey("It can get the default deployment", t, func() { + Convey("when it not running in server mode", func() { + defDeployment := DefaultDeployment(ctx) + So(defDeployment, ShouldEqual, "production") + + Convey("it can get overridden if WR_DEPLOYMENT env variable is set", func() { + os.Setenv("WR_DEPLOYMENT", "development") + defer func() { + os.Unsetenv("WR_DEPLOYMENT") + }() + + defDeployment := DefaultDeployment(ctx) + So(defDeployment, ShouldEqual, "development") + }) + }) + + Convey("when it is running in server mode", func() { + orgPWD, err := os.Getwd() + So(err, ShouldBeNil) + + dir, err := ioutil.TempDir("", "wr_conf_test") + So(err, ShouldBeNil) + path := dir + "/jobqueue/server.go" + err = os.MkdirAll(path, 0777) + So(err, ShouldBeNil) + defer func() { + defer os.RemoveAll(dir) + }() + + err = os.Chdir(dir) + So(err, ShouldBeNil) + defer func() { + err = os.Chdir(orgPWD) + }() + + defDeployment := DefaultDeployment(ctx) + So(defDeployment, ShouldEqual, "development") + }) + }) + + Convey("It can create a config with env vars", t, func() { + os.Setenv("WR_MANAGERPORT", "1234") + os.Setenv("WR_MANAGERUMASK", "77") + os.Setenv("WR_MANAGERSETDOMAINIP", "true") + defer func() { + os.Unsetenv("WR_MANAGERPORT") + os.Unsetenv("WR_MANAGERUMASK") + os.Unsetenv("WR_MANAGERSETDOMAINIP") + }() + + envVarConfig := getEnvVarsConfig(ctx) + + So(envVarConfig.ManagerPort, ShouldEqual, "1234") + So(envVarConfig.ManagerWeb, ShouldBeEmpty) + So(envVarConfig.ManagerUmask, ShouldEqual, 77) + So(envVarConfig.ManagerSetDomainIP, ShouldBeTrue) + }) + + Convey("It can merge Default Config and Env Var config", t, func() { + os.Setenv("WR_MANAGERPORT", "1234") + os.Setenv("WR_MANAGERUMASK", "077") + os.Setenv("WR_MANAGERSETDOMAINIP", "true") + defer func() { + os.Unsetenv("WR_MANAGERPORT") + os.Unsetenv("WR_MANAGERUMASK") + os.Unsetenv("WR_MANAGERSETDOMAINIP") + }() + + mergedConfig := mergeDefaultAndEnvVarsConfigs(ctx) + So(mergedConfig.ManagerPort, ShouldEqual, "1234") + So(mergedConfig.Source("ManagerPort"), ShouldEqual, ConfigSourceEnvVar) + So(mergedConfig.ManagerWeb, ShouldBeEmpty) + So(mergedConfig.Source("ManagerWeb"), ShouldEqual, ConfigSourceDefault) + So(mergedConfig.ManagerUmask, ShouldEqual, 77) + So(mergedConfig.Source("ManagerUmask"), ShouldEqual, ConfigSourceEnvVar) + So(mergedConfig.ManagerSetDomainIP, ShouldBeTrue) + So(mergedConfig.Source("ManagerSetDomainIP"), ShouldEqual, ConfigSourceEnvVar) + }) + + Convey("It can merge all the configs and return a final config", t, func() { + // env config + os.Setenv("WR_MANAGERUMASK", "077") + defer os.Unsetenv("WR_MANAGERUMASK") + + // config file in pwd + uid := 1000 + pwd, err := os.Getwd() + So(err, ShouldBeNil) + mport := "5434" + mweb1 := "5435" + mweb2 := "5436" + path7, path8, err := fileTestSetup(pwd, mport, mweb1, mweb2) + defer fileTestTeardown(path7, path8) + So(err, ShouldBeNil) + + finalConfig := mergeAllConfigs(ctx, uid, "production", pwd, true) + So(finalConfig.ManagerPort, ShouldEqual, mport) + So(finalConfig.Source("ManagerPort"), ShouldEqual, path7) + So(finalConfig.ManagerWeb, ShouldEqual, mweb1) + So(finalConfig.Source("ManagerWeb"), ShouldEqual, path7) + So(finalConfig.ManagerUmask, ShouldEqual, 77) + So(finalConfig.Source("ManagerUmask"), ShouldEqual, ConfigSourceEnvVar) + + Convey("it can override deployment to default deployment if it's not development or production", func() { + finalConfig := mergeAllConfigs(ctx, uid, "testDeployment", pwd, true) + So(finalConfig.IsProduction(), ShouldBeTrue) + }) + }) + + Convey("ConfigLoad* gives default values to start with", t, func() { + if os.Getuid() > 16128 { + fmt.Print(os.Getuid()) + fmt.Printf("Failed to calculate a suitable unique port number since your user id is so large.\n") + fmt.Printf("Skipping tests...\n") + t.Skip("skipping tests; user id is very large.") + } + + Convey("When loaded from parent directory", func() { + config := ConfigLoadFromParentDir(ctx, "development") + So(config, ShouldNotBeNil) + So(config.IsProduction(), ShouldBeFalse) + }) + + Convey("When loaded from non parent directory", func() { + config := ConfigLoadFromNonParentDir(ctx, "testing") + So(config, ShouldNotBeNil) + So(config.IsProduction(), ShouldBeTrue) + }) + + Convey("It can get the default config", func() { + config := DefaultConfig(ctx) + So(config, ShouldNotBeNil) + So(config.IsProduction(), ShouldBeTrue) + + Convey("It can get the Default server", func() { + server := DefaultServer(ctx) + So(server, ShouldEqual, config.ManagerHost+":"+config.ManagerPort) + }) + }) + }) +} diff --git a/internal/testdata/cert/wrongCert.pem b/internal/testdata/cert/wrongCert.pem new file mode 100644 index 0000000..826f518 --- /dev/null +++ b/internal/testdata/cert/wrongCert.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDLTCCAhWgAwIBAgIQTCJWjEAbgGLGxUyO22OGmzANBgkqhkiG9w0BAQsFADAV +MRMwEQYDVQQKEwp3ciBtYW5hZ2VyMB4XDTIxMDIwMzAxMDQ1OVoXDTIyMDIwMzAx +MDQ1OVowFTETMBEGA1UEChMKd3IgbWFuYWdlcjCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBAKqaIjJhNZ3zhhUd3vu6xoblNBpUeb30plT27g5x94cwWZoj +yZaK+3+EoEcPIG7zv6DnrZgycHjBV3Fvl9jfpAujP/GKBU6rciglZX54csCoZMAP +pOh92CNEtk+i0gUqy6+0IcnxdIen4pckhpFwoNs7n1tCRp2vn/t4oN3pDqAjSP6F +reec9LE6WOXyIlEH3QhkCLT5eOA8TMoZUtDwWfh70DsvROnXy6xlJt5frsRNflkz +m+vWt5wNxs7DJoJaiSDcGUIMdzKFTLyyLa+r8SquQYnlMkJe4Lu9h9RmazdQcKGA +1HjXYKGBPLWmE8sJ4dPiV/D4oME6UexSvwW+/YkCAwEAAaN5MHcwDgYDVR0PAQH/ +BAQDAgKkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYD +VR0OBBYEFM7ra0qFZHOOppFpc/wMmpFzHRsjMCAGA1UdEQQZMBeCCWxvY2FsaG9z +dIcEAAAAAIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAUQ4aPYWpLNpc90NA5cp0 +GJrQ+Pbo8FcbFOYJmUs/zzzEaf0v8Usg93UYPh3p+RHpg5erMm1uahVNmUKAn4Jq +KOmk25/6qy7PvUUQEZhBqvdGlD3Ewm+tJV+FbnDStJJbYT0aQM8jADK+YuKMolGk +okxiDaHK60lSv8eP7qQDCk8zUOU/72W3k7Z0qmzWK1mNLw9Ur8AjX6dazlEkOFY+ +ZlozyxIAtbbaD4DySCWE9dv5E8F7tWL5Nl66OYwx5Drhd/Tx0Ld0gS0+cH6TVR+n +uP5jBOAi8h56OlE/JVjio7xdJc7PlJiWpUEiludO1/2S050ywcQe3k29J+83+MF8 +jnnj +-----END CERTIFICATE----- \ No newline at end of file diff --git a/internal/testdata/linux/virtualmemory/issue1/proc/meminfo b/internal/testdata/linux/virtualmemory/issue1/proc/meminfo new file mode 100644 index 0000000..24eb61b --- /dev/null +++ b/internal/testdata/linux/virtualmemory/issue1/proc/meminfo @@ -0,0 +1,42 @@ + total: used: free: shared: buffers: cached: +Mem: 260579328 136073216 124506112 0 4915200 94064640 +Swap: 0 0 0 +MemTotal: 254472 kB +MemFree: 121588 kB +MemShared: 0 kB +Buffers: 4800 kB +Cached: 91860 kB +SwapCached: 0 kB +Active: 106236 kB +Inactive: 8380 kB +MemAvailable: -6 kB +Active(anon): 17956 kB +Inactive(anon): 0 kB +Active(file): 88280 kB +Inactive(file): 8380 kB +Unevictable: 0 kB +Mlocked: 0 kB +HighTotal: 131072 kB +HighFree: 66196 kB +LowTotal: 123400 kB +LowFree: 55392 kB +SwapTotal: 0 kB +SwapFree: 0 kB +Dirty: 0 kB +Writeback: 0 kB +AnonPages: 17992 kB +Mapped: 37884 kB +Shmem: 0 kB +Slab: 9076 kB +SReclaimable: 2700 kB +SUnreclaim: 6376 kB +KernelStack: 624 kB +PageTables: 396 kB +NFS_Unstable: 0 kB +Bounce: 0 kB +WritebackTmp: 0 kB +CommitLimit: 127236 kB +Committed_AS: 24968 kB +VmallocTotal: 1949696 kB +VmallocUsed: 0 kB +VmallocChunk: 0 kB diff --git a/internal/utils.go b/internal/utils.go new file mode 100644 index 0000000..31fc20e --- /dev/null +++ b/internal/utils.go @@ -0,0 +1,241 @@ +/******************************************************************************* + * Copyright (c) 2020 Genome Research Ltd. + * + * Author: Sendu Bala , Ashwini Chhipa + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ + +package internal + +// this file has general utility functions. + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/shirou/gopsutil/v3/mem" + "github.com/wtsi-ssg/wr/clog" + "github.com/wtsi-ssg/wr/math/convert" +) + +// keyvalueStruct is the struct to define a key-value pair. +type keyvalueStruct struct { + Key string + Value int +} + +// keyvalueStructs is the struct to define a list of keyvalueStruct. +type keyvalueStructs []keyvalueStruct + +// PathReadError records an path read error. +type PathReadError struct { + path string + Err error +} + +// Error returns an error related to path could not be read. +func (p *PathReadError) Error() string { + return fmt.Sprintf("path [%s] could not be read: %s", p.path, p.Err) +} + +// SortMapKeysByIntValue sorts the keys of a map[string]int by its values. +func SortMapKeysByIntValue(imap map[string]int) []string { + // create keyval + keyval := createKeyvalFromMap(imap) + + // sort the keyval + keyval.sliceSort() + + // sort the map by its values and return the sorted keys + return sortKeyvalstruct(len(imap), keyval) +} + +// sliceSort function sorts the slice. +func (k keyvalueStructs) sliceSort() { + sort.Slice(k, func(i, j int) bool { + return k[i].Value < k[j].Value + }) +} + +// createKeyvalFromMap function creates a keyvaluestruct from map[string]int. +func createKeyvalFromMap(imap map[string]int) keyvalueStructs { + keyval := keyvalueStructs{} + + for k, v := range imap { + keyval = append(keyval, keyvalueStruct{k, v}) + } + + return keyval +} + +// sortKeyvalstruct function sorts the keyvaluestruct by values and return the +// sorted keys. +func sortKeyvalstruct(maplen int, keyval []keyvalueStruct) []string { + sortedKeys := make([]string, 0, maplen) + for _, kv := range keyval { + sortedKeys = append(sortedKeys, kv.Key) + } + + return sortedKeys +} + +// ReverseSortMapKeysByIntValue reverse sorts the keys of a map[string]int by +// its values. +func ReverseSortMapKeysByIntValue(imap map[string]int) []string { + // create keyval + keyval := createKeyvalFromMap(imap) + + // sort the keyval in reverse order + keyval.sliceSortReverse() + + // sort the map by its values and return the sorted keys + return sortKeyvalstruct(len(imap), keyval) +} + +// sliceSortReverse function reverse sorts the slice. +func (k keyvalueStructs) sliceSortReverse() { + sort.Slice(k, func(i, j int) bool { + return k[i].Value > k[j].Value + }) +} + +// SortMapKeysByMapIntValue sorts the keys of a map[string]map[string]int by a +// the values found at a given sub value. +func SortMapKeysByMapIntValue(imap map[string]map[string]int, criterion string) []string { + // create keyval + keyval := createKeyvalFromMapOfMap(imap, criterion) + + // sort the keyval + keyval.sliceSort() + + // sort the map by its values and return the sorted keys + return sortKeyvalstruct(len(imap), keyval) +} + +// createKeyvalFromMapOfMap function creates a keyvaluestruct from +// map[string]map[string]int. +func createKeyvalFromMapOfMap(imap map[string]map[string]int, criterion string) keyvalueStructs { + keyval := keyvalueStructs{} + + for k, v := range imap { + keyval = append(keyval, keyvalueStruct{k, v[criterion]}) + } + + return keyval +} + +// ReverseSortMapKeysByMapIntValue reverse sorts the keys of a +// map[string]map[string]int by a the values found at a given sub value. +func ReverseSortMapKeysByMapIntValue(imap map[string]map[string]int, criterion string) []string { + // create keyval + keyval := createKeyvalFromMapOfMap(imap, criterion) + + // sort the keyval in reverse order + keyval.sliceSortReverse() + + // sort the map by its values and return the sorted keys + return sortKeyvalstruct(len(imap), keyval) +} + +// DedupSortStrings removes duplicates and then sorts the given strings, +// returning a new slice. +func DedupSortStrings(istrings []string) []string { + if len(istrings) == 0 { + return istrings + } + + elementmap := make(map[string]bool) + dedup := []string{} + + for _, entry := range istrings { + if _, value := elementmap[entry]; !value { + elementmap[entry] = true + + dedup = append(dedup, entry) + } + } + + sort.Strings(dedup) + + return dedup +} + +// ProcMeminfoMBs uses gopsutil (amd64 freebsd, linux, windows, darwin, openbds +// only!) to find the total number of MBs of memory physically installed on the +// current system. +func ProcMeminfoMBs() (int, error) { + v, err := mem.VirtualMemory() + if err != nil { + return 0, err + } + + // convert bytes to MB + return convert.BytesToMB(v.Total), err +} + +// PathToContent takes the path to a file and returns its contents as a string. +// If path begins with a tilda, TildaToHome() is used to first convert the path +// to an absolute path, in order to find the file. +func PathToContent(path string) (string, error) { + if path == "" { + return "", &PathReadError{"", nil} + } + + absPath := TildaToHome(path) + + contents, err := ioutil.ReadFile(absPath) + if err != nil { + return "", &PathReadError{absPath, err} + } + + return string(contents), nil +} + +// TildaToHome converts a path beginning with ~/ to the absolute path based in +// the current home directory. If that cannot be determined, path is returned +// unaltered. +func TildaToHome(path string) string { + if path == "" { + return "" + } + + home, herr := os.UserHomeDir() + if herr == nil && home != "" && strings.HasPrefix(path, "~/") { + path = strings.TrimLeft(path, "~/") + path = filepath.Join(home, path) + } + + return path +} + +// GetPWD returns the present working directory. +func GetPWD(ctx context.Context) string { + pwd, err := os.Getwd() + if err != nil { + clog.Fatal(ctx, err.Error()) + } + + return pwd +} diff --git a/internal/utils_test.go b/internal/utils_test.go new file mode 100644 index 0000000..3f9473b --- /dev/null +++ b/internal/utils_test.go @@ -0,0 +1,204 @@ +/******************************************************************************* + * Copyright (c) 2020 Genome Research Ltd. + * + * Author: Ashwini Chhipa + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ + +package internal + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + . "github.com/smartystreets/goconvey/convey" + "github.com/wtsi-ssg/wr/math/convert" + "golang.org/x/sys/unix" +) + +// memIssue1 is the directory containing wrong memory info. +// testdata/linux/virtualmemory/issue1/proc/meminfo . +const memIssue1 string = "issue1" + +func TestUtilsFuncs(t *testing.T) { + testmap := map[string]int{ + "k1": 10, + "k2": 30, + "k3": 5, + "k4": 15, + } + + testmapmap := map[string]map[string]int{ + "k1": {"ka1": 10, + "ka2": 20, + }, + "k2": {"ka1": 5, + "ka2": 30, + }, + "k3": {"ka1": 100, + "ka2": 200, + }, + "k4": {"ka1": 50, + "ka2": 2, + }, + } + + Convey("Given a map[string]int create Keyval struct from it", t, func() { + testmapNil := map[string]int{} + tnil := createKeyvalFromMap(testmapNil) + So(len(tnil), ShouldEqual, 0) + + t := createKeyvalFromMap(testmap) + So(len(t), ShouldEqual, len(testmap)) + }) + + Convey("Given a map[string]int create Keyval struct and sort the slice", t, func() { + t := createKeyvalFromMap(testmap) + t.sliceSort() + So(t[0].Value, ShouldBeLessThanOrEqualTo, t[1].Value) + So(t[1].Value, ShouldBeLessThanOrEqualTo, t[2].Value) + So(t[2].Value, ShouldBeLessThanOrEqualTo, t[3].Value) + }) + + Convey("Given a map[string]map[string]int create Keyval struct and reverse sort the slice", t, func() { + t := createKeyvalFromMapOfMap(testmapmap, "ka1") + t.sliceSortReverse() + So(t[3].Value, ShouldBeLessThanOrEqualTo, t[2].Value) + So(t[2].Value, ShouldBeLessThanOrEqualTo, t[1].Value) + So(t[1].Value, ShouldBeLessThanOrEqualTo, t[0].Value) + }) + + Convey("Given a map[string]int sort the Keyval struct", t, func() { + So(sortKeyvalstruct(0, []keyvalueStruct{}), ShouldBeEmpty) + + t := createKeyvalFromMap(testmap) + t.sliceSort() + So(sortKeyvalstruct(len(testmap), t), ShouldResemble, []string{"k3", "k1", "k4", "k2"}) + }) + + Convey("Given a map[string]int sort and reverse sort the map by value", t, func() { + So(SortMapKeysByIntValue(testmap), ShouldResemble, []string{"k3", "k1", "k4", "k2"}) + So(ReverseSortMapKeysByIntValue(testmap), ShouldResemble, []string{"k2", "k4", "k1", "k3"}) + }) + + Convey("Given a map[string]map[string]int sort and reverse sort it by value with a given criterion", t, func() { + criterion := "ka1" + So(SortMapKeysByMapIntValue(testmapmap, criterion), ShouldResemble, []string{"k2", "k1", "k4", "k3"}) + + criterion = "ka2" + So(ReverseSortMapKeysByMapIntValue(testmapmap, criterion), ShouldResemble, []string{"k3", "k2", "k1", "k4"}) + }) + + Convey("Given a slice remove the duplicates from it and then sort it", t, func() { + So(DedupSortStrings([]string{}), ShouldBeEmpty) + + testlist := []string{"k3", "k3", "k4", "k1"} + + So(DedupSortStrings(testlist), ShouldResemble, []string{"k1", "k3", "k4"}) + }) + + Convey("Given a path starting with ~/ check it's absolute path", t, func() { + So(TildaToHome(""), ShouldBeEmpty) + + home, herr := os.UserHomeDir() + So(herr, ShouldEqual, nil) + filepth := filepath.Join(home, "testing_absolute_path.text") + _, err := os.Create(filepth) + So(err, ShouldEqual, nil) + + So(TildaToHome("~/testing_absolute_path.text"), ShouldEqual, filepth) + defer os.Remove(filepth) + }) + + Convey("Given a path to a file check it's content", t, func() { + empContent, err := PathToContent("") + So(err, ShouldNotBeNil) + So(empContent, ShouldBeEmpty) + + home, herr := os.UserHomeDir() + So(herr, ShouldEqual, nil) + filepth := filepath.Join(home, "testing_pathtocontent.text") + + file, err := os.Create(filepth) + So(err, ShouldEqual, nil) + + wrtn, err := file.WriteString("hello") + So(err, ShouldEqual, nil) + fmt.Printf("wrote %d bytes\n", wrtn) + + content, err := PathToContent(filepth) + So(content, ShouldEqual, "hello") + So(err, ShouldEqual, nil) + + content, err = PathToContent("random.txt") + So(content, ShouldEqual, "") + So(err, ShouldNotBeNil) + + defer os.Remove(filepth) + }) + + Convey("It can get the virtual memory of the system in MB", t, func() { + memStat, err := ProcMeminfoMBs() + So(memStat, ShouldNotEqual, 0) + So(err, ShouldBeNil) + + if runtime.GOOS != "linux" { + t.Skip("skipping test; test coverage is only for linux machines.") + } + + memStat, err = ProcMeminfoMBs() + So(memStat, ShouldNotEqual, 0) + So(err, ShouldBeNil) + + totalSysMem, err := unix.SysctlUint64("hw.memsize") + So(err, ShouldBeNil) + + So(convert.BytesToMB(totalSysMem), ShouldEqual, memStat) + + Convey("not with the wrong test data in linux", func() { + origProc := os.Getenv("HOST_PROC") + defer os.Setenv("HOST_PROC", origProc) + + // set HOST_PROC to testdata for wrong meminfo + os.Setenv("HOST_PROC", filepath.Join("testdata/linux/virtualmemory/", memIssue1, "proc")) + memWStat, errw := ProcMeminfoMBs() + So(memWStat, ShouldEqual, 0) + So(errw, ShouldNotBeNil) + }) + }) + + Convey("It can get the present working directory", t, func() { + ctx := context.Background() + pWD := GetPWD(ctx) + So(pWD, ShouldNotBeEmpty) + + tempDir := os.TempDir() + err := os.Chdir(tempDir) + So(err, ShouldBeNil) + + pWD = GetPWD(ctx) + So(strings.TrimSuffix(pWD, "/"), ShouldEndWith, strings.TrimSuffix(tempDir, "/")) + }) +}