From 68aa0e3a36e0caae5c8e33dc205a25f60050fb75 Mon Sep 17 00:00:00 2001 From: blacktop Date: Mon, 3 Sep 2018 16:24:34 -0400 Subject: [PATCH] fix elasticsearch stuff --- Gopkg.lock | 24 +- Gopkg.toml | 8 +- commands/elk.go | 2 +- commands/lookup.go | 4 +- commands/scan.go | 12 +- config/bindata.go | 4 +- config/config.toml | 4 +- malice/database/database.go | 2 +- malice/docker/client/client.go | 2 +- malice/persist/file.go | 2 +- plugins/plugins.go | 2 +- plugins/templates/go/scan.go | 4 +- .../{go-plugin-utils => pkgs}/LICENSE | 2 +- .../database/database.go | 0 .../database/elasticsearch/elasticsearch.go | 38 +-- .../pkgs}/database/elasticsearch/mapping.go | 0 .../{go-plugin-utils => pkgs}/utils/utils.go | 0 .../maliceio/go-plugin-utils/LICENSE | 201 --------------- .../maliceio/go-plugin-utils/utils/utils.go | 244 ------------------ 19 files changed, 52 insertions(+), 503 deletions(-) rename vendor/github.com/malice-plugins/{go-plugin-utils => pkgs}/LICENSE (99%) rename vendor/github.com/malice-plugins/{go-plugin-utils => pkgs}/database/database.go (100%) rename vendor/github.com/{maliceio/go-plugin-utils => malice-plugins/pkgs}/database/elasticsearch/elasticsearch.go (92%) rename vendor/github.com/{maliceio/go-plugin-utils => malice-plugins/pkgs}/database/elasticsearch/mapping.go (100%) rename vendor/github.com/malice-plugins/{go-plugin-utils => pkgs}/utils/utils.go (100%) delete mode 100644 vendor/github.com/maliceio/go-plugin-utils/LICENSE delete mode 100644 vendor/github.com/maliceio/go-plugin-utils/utils/utils.go diff --git a/Gopkg.lock b/Gopkg.lock index 2fadb5a2..05a81859 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -322,26 +322,16 @@ revision = "60711f1a8329503b04e1c88535f419d0bb440bff" [[projects]] - digest = "1:f960ee99af09fb9d035da5a7bc99ab1b52c6b99f1e6f7bfb38f82f211b1ba5d4" - name = "github.com/malice-plugins/go-plugin-utils" + branch = "master" + digest = "1:1d495dace08d0462cac180cd80ffb8712bdedf7ce4a01898c7d332b5dcc27c35" + name = "github.com/malice-plugins/pkgs" packages = [ "database", - "utils", - ] - pruneopts = "UT" - revision = "b36f32401f48ad1629a2a2d0b30a9ee0dc237e2f" - version = "0.1.3" - -[[projects]] - digest = "1:a6efb4dfc4c2faca529bc93d6ae4cb511b4ef7f8b55d6e2289c6a6c17f87d3b5" - name = "github.com/maliceio/go-plugin-utils" - packages = [ "database/elasticsearch", "utils", ] pruneopts = "UT" - revision = "b36f32401f48ad1629a2a2d0b30a9ee0dc237e2f" - version = "0.1.3" + revision = "8b60ba5208b1f40281d09751d83dafd0e7db6e15" [[projects]] digest = "1:cdb899c199f907ac9fb50495ec71212c95cb5b0e0a8ee0800da0238036091033" @@ -543,7 +533,7 @@ "windows", ] pruneopts = "UT" - revision = "fa5fdf94c78965f1aa8423f0cc50b8b8d728b05a" + revision = "2b024373dcd9800f0cae693839fac6ede8d64a8c" [[projects]] digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18" @@ -689,8 +679,8 @@ "github.com/fsnotify/fsnotify", "github.com/gorilla/mux", "github.com/jordan-wright/email", - "github.com/maliceio/go-plugin-utils/database/elasticsearch", - "github.com/maliceio/go-plugin-utils/utils", + "github.com/malice-plugins/pkgs/database/elasticsearch", + "github.com/malice-plugins/pkgs/utils", "github.com/mattn/go-runewidth", "github.com/parnurzeal/gorequest", "github.com/pkg/errors", diff --git a/Gopkg.toml b/Gopkg.toml index e366b356..d5cfab98 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -49,17 +49,13 @@ version = "1.0.2" [[constraint]] - name = "gopkg.in/dancannon/gorethink.v2" - version = "2.2.2" + branch = "master" + name = "github.com/malice-plugins/pkgs" [[constraint]] name = "gopkg.in/natefinch/lumberjack.v2" version = "2.0.0" -[[override]] - name = "github.com/olivere/elastic" - version = "^6.0.0" - [[override]] name = "github.com/docker/distribution" revision = "b38e5838b7b2f2ad48e06ec4b500011976080621" diff --git a/commands/elk.go b/commands/elk.go index b63b891e..01dd7a08 100644 --- a/commands/elk.go +++ b/commands/elk.go @@ -2,7 +2,7 @@ package commands import ( log "github.com/Sirupsen/logrus" - "github.com/maliceio/go-plugin-utils/database/elasticsearch" + "github.com/malice-plugins/pkgs/database/elasticsearch" "github.com/maliceio/malice/config" "github.com/maliceio/malice/malice/database" "github.com/maliceio/malice/malice/docker/client" diff --git a/commands/lookup.go b/commands/lookup.go index bb4378a0..c7866b86 100644 --- a/commands/lookup.go +++ b/commands/lookup.go @@ -4,8 +4,8 @@ import ( "fmt" log "github.com/Sirupsen/logrus" - "github.com/maliceio/go-plugin-utils/database/elasticsearch" - "github.com/maliceio/go-plugin-utils/utils" + "github.com/malice-plugins/pkgs/database/elasticsearch" + "github.com/malice-plugins/pkgs/utils" "github.com/maliceio/malice/config" "github.com/maliceio/malice/malice/database" "github.com/maliceio/malice/malice/docker/client" diff --git a/commands/scan.go b/commands/scan.go index c6593732..e2d52a9c 100644 --- a/commands/scan.go +++ b/commands/scan.go @@ -7,8 +7,8 @@ import ( log "github.com/Sirupsen/logrus" "github.com/fatih/structs" - "github.com/maliceio/go-plugin-utils/database/elasticsearch" - "github.com/maliceio/go-plugin-utils/utils" + "github.com/malice-plugins/pkgs/database/elasticsearch" + "github.com/malice-plugins/pkgs/utils" "github.com/maliceio/malice/config" "github.com/maliceio/malice/malice/database" "github.com/maliceio/malice/malice/docker/client" @@ -22,9 +22,11 @@ import ( func cmdScan(path string, logs bool) error { es := elasticsearch.Database{ - URL: config.Conf.DB.URL, - Index: utils.Getopt("MALICE_ELASTICSEARCH_INDEX", "malice"), - Type: "samples", + URL: utils.Getopt("MALICE_ELASTICSEARCH_URL", config.Conf.DB.URL), + Index: utils.Getopt("MALICE_ELASTICSEARCH_INDEX", "malice"), + Type: utils.Getopt("MALICE_ELASTICSEARCH_TYPE", "samples"), + Username: utils.Getopt("MALICE_ELASTICSEARCH_USERNAME", config.Conf.DB.Username), + Password: utils.Getopt("MALICE_ELASTICSEARCH_PASSWORD", config.Conf.DB.Password), } if len(path) > 0 { diff --git a/config/bindata.go b/config/bindata.go index 5c963ffb..d168e0bd 100644 --- a/config/bindata.go +++ b/config/bindata.go @@ -68,7 +68,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var _configConfigToml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x53\x4b\x6f\xdb\x3c\x10\xbc\xf3\x57\x2c\xe4\xcb\xf7\x01\x8d\x2c\x4b\x4e\xe2\x0a\xf0\x21\x08\x72\x48\xd1\xb4\x40\x72\x34\x82\x62\x25\xad\x25\xc2\x7c\x08\x24\xe5\x3c\x7e\x7d\xb1\xb4\xe5\x44\x6d\x2e\x05\x22\x1f\x68\x72\xf6\x31\x33\x5c\xce\xe0\xda\xf6\x2f\x4e\xb6\x5d\x80\xff\xea\xff\x21\xcf\x16\x05\x9c\xf1\xb2\x82\x4a\x61\xbd\x0b\xb6\x87\x6f\xd6\x77\x03\xc2\x1d\x4a\x43\x5f\xe0\x4a\x29\xb8\xe7\x04\x0f\xf7\xe4\xc9\xed\xa9\x49\xc5\x0c\x1e\x88\xe0\xfb\xed\xf5\xcd\x8f\x87\x1b\xd8\x5a\x07\x4a\xd6\x64\x3c\x81\x34\x5b\xeb\x34\x06\x69\x4d\x2a\xc4\xec\x73\x3e\x31\x83\xbb\x2b\xee\x06\xd7\xd6\x6c\x65\x3b\xb8\xd8\x00\xfe\xbd\xce\x27\xf1\x11\x41\x06\x45\xb0\x86\xe4\x0e\x59\x39\xdc\x0f\x26\x48\x4d\x53\x7e\x89\xd8\x93\xf3\x4c\x74\x0d\xc9\x3e\x4b\x8b\x34\xcf\x12\x21\x36\x38\x84\xce\xba\x47\x01\x60\x50\xc7\x2a\xa3\xf7\x89\x00\xb0\xae\x45\x23\x5f\x0f\x0a\x4f\x1d\x6e\x7f\x72\xe6\x13\x55\x9c\x36\x38\xc5\x48\x96\xc6\x5f\xb9\xca\x38\x0f\x1b\x2d\xcd\xaf\x23\xb4\xc8\x2f\x23\xb8\x28\x8b\xa2\x28\x38\x95\x34\x4a\xc5\xc9\x9d\xf5\x81\x43\xbc\x0e\x7d\x4a\xcf\xa8\x7b\x45\x69\x6d\x35\xd7\xe8\xad\x63\x2c\x3f\xe7\x26\x9e\x1c\xc7\xf1\xca\x3c\x23\x8e\xde\xf3\x19\xaf\x4f\xd6\x35\x5c\xb8\xc1\x80\x15\x7a\x7a\xaf\x47\x47\xce\x67\xa4\xd0\x07\x59\x73\xa6\xd4\xd8\xbe\x83\xe6\x47\xc8\x13\xba\xba\x2b\x2f\xd2\x65\xf2\xa6\xab\x0b\xa1\x2f\xe7\x73\x65\x6b\x54\xcc\xb6\xfc\x9a\x67\x51\xe2\xec\x8f\x88\x69\x91\x31\x6a\x24\xcc\x81\x23\x69\x26\x7b\xda\x5b\x17\x58\xc5\x86\x13\x98\x35\xdf\x9c\x1d\xa2\xf0\x4c\x00\x90\xc1\x4a\x11\x87\x07\x37\x90\x10\x9b\x41\x7e\xa0\x6d\x27\x2b\x34\xf8\x91\xb4\x03\x32\x6a\x8a\x4f\x26\x1a\x79\xd2\x33\x21\xb1\x5c\x16\x8f\x1f\x35\x25\xb3\x97\xce\x1a\x4d\x26\x30\xee\x86\x38\x0c\x0d\xed\x49\xd9\x9e\x4f\xa3\xf7\xb6\xde\x51\x9c\x24\x8d\x75\x27\x0d\x9d\x4d\x59\x26\xb1\x72\xd3\x5b\x69\xe2\x9d\x87\x7a\x6a\x6c\x5e\x5c\x5e\x24\x13\x07\x16\xd1\x82\x4a\x9a\xc6\xbf\x95\x29\xe7\x1a\xd5\x13\x3a\x2a\x9d\xe5\x70\x25\xcd\xce\xff\x7d\xcf\xe5\xe4\x3e\x38\xb0\xee\x07\x58\xc3\x79\x76\xfc\x98\x27\x69\xeb\x5e\xf8\x30\x5f\xe6\xab\x15\x1f\x8a\x8d\xb2\x6d\x7b\x90\xb1\x95\x8a\xa6\x12\x52\x65\xdb\x24\x0a\x7c\xf6\xf2\x95\x81\x45\x76\xd8\x1e\x5c\x2f\x8e\xbb\x0a\xeb\xdd\xd0\x33\xab\x4b\x66\xc8\x12\xe3\x8b\x5c\xc3\x16\x95\x67\x47\x7b\x67\x9f\x5f\xde\xbc\x3e\x21\x00\x3c\x4e\xe3\x74\xf0\x7f\x7f\xd8\xfc\x0e\x00\x00\xff\xff\xcb\xa9\xce\xd5\x2e\x05\x00\x00") +var _configConfigToml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x53\xcb\x6e\xdb\x30\x10\xbc\xf3\x2b\x16\xf2\xa5\x05\x1a\x59\x96\x9c\xc4\x15\xe0\x43\x10\xe4\x90\xa2\x69\x81\xe4\x68\x04\xc5\x4a\x5a\x4b\x84\xf9\x10\x48\xca\x79\x7c\x7d\xb1\x94\xed\x44\x68\x2e\x05\x22\x1f\x68\x72\xf6\x31\x33\x5c\xce\xe0\xda\xf6\x2f\x4e\xb6\x5d\x80\x2f\xf5\x57\xc8\xb3\x45\x01\x67\xbc\xac\xa0\x52\x58\xef\x82\xed\xe1\x87\xf5\xdd\x80\x70\x87\xd2\xd0\x37\xb8\x52\x0a\xee\x39\xc1\xc3\x3d\x79\x72\x7b\x6a\x52\x31\x83\x07\x22\xf8\x79\x7b\x7d\xf3\xeb\xe1\x06\xb6\xd6\x81\x92\x35\x19\x4f\x20\xcd\xd6\x3a\x8d\x41\x5a\x93\x0a\x31\xfb\x9c\x4f\xcc\xe0\xee\x8a\xbb\xc1\xb5\x35\x5b\xd9\x0e\x2e\x36\x80\xff\xaf\xf3\x49\x7c\x44\x90\x41\x11\xac\x21\xb9\x43\x56\x0e\xf7\x83\x09\x52\xd3\x94\x5f\x22\xf6\xe4\x3c\x13\x5d\x43\xb2\xcf\xd2\x22\xcd\xb3\x44\x88\x0d\x0e\xa1\xb3\xee\x51\x00\x18\xd4\xb1\xca\xd1\xfb\x44\x00\x58\xd7\xa2\x91\xaf\xa3\xc2\x53\x87\xdb\xdf\x9c\xf9\x44\x15\xa7\x0d\x4e\x31\x92\xa5\xf1\x57\xae\x32\xce\xc3\x46\x4b\xf3\xe7\x00\x2d\xf2\xcb\x08\x2e\xca\xa2\x28\x0a\x4e\x25\x8d\x52\x71\x72\x67\x7d\xe0\x10\xaf\x43\x9f\xd2\x33\xea\x5e\x51\x5a\x5b\xcd\x35\x7a\xeb\x18\xcb\xcf\xb9\x89\x27\xc7\x71\xbc\x32\xcf\x88\xa3\xf7\x7c\xc6\xeb\x93\x75\x0d\x17\x6e\x30\x60\x85\x9e\xde\xeb\xd1\x91\xf3\x19\x29\xf4\x41\xd6\x9c\x29\x35\xb6\xef\xa0\xf9\x01\xf2\x84\xae\xee\xca\x8b\x74\xc9\x41\xb3\xa3\xb2\x2e\x84\xbe\x9c\xcf\x95\xad\x51\x31\xdf\xf2\x7b\x9e\x45\x91\x53\x7c\x5a\xe4\x14\x73\x20\xcc\x81\x47\xd2\x4c\xf6\xb4\xb7\x2e\xb0\x8a\x0d\x27\x30\x6b\xbe\x39\x3b\x44\xe1\x99\x00\x20\x83\x95\x22\x0e\x0f\x6e\x20\x21\x36\x83\xfc\x40\xdb\x4e\x56\x68\xf0\x23\x69\x23\x72\xd4\x14\x9f\x4c\x34\xf2\xa4\x66\x42\x62\xb9\x2c\x1e\x3f\x6a\x4a\x66\x2f\x9d\x35\x9a\x4c\x60\xdc\x0d\x71\x18\x1a\xda\x93\xb2\x3d\x9f\x46\xef\x6d\xbd\xa3\x38\x49\x1a\xeb\x4e\x1a\x3a\x9b\xb2\x4c\x62\xe5\xa6\xb7\xd2\xc4\x3b\x0f\xf5\xd4\xd6\xbc\xb8\xbc\x48\x26\x0e\x2c\xa2\x05\x95\x34\x8d\x7f\x2b\x53\xce\x35\xaa\x27\x74\x54\x3a\xcb\xe1\x4a\x9a\x9d\xff\xf7\x9e\xcb\xc9\x7d\x70\x60\xdd\x0f\xb0\x86\xf3\xec\xf0\x31\x4f\xd2\xd6\xbd\xf0\x61\xbe\xcc\x57\x2b\x3e\x14\x1b\x65\xdb\x76\x94\xb1\x95\x8a\xa6\x12\x52\x65\xdb\x24\x0a\x7c\xf6\xf2\x95\x81\x45\x36\x6e\x47\xd7\x8b\xc3\xae\xc2\x7a\x37\xf4\xcc\xea\x92\x19\xb2\xc4\xf8\x22\xd7\xb0\x45\xe5\xd9\xd1\xde\xd9\xe7\x97\x37\xaf\x4f\x08\x00\x8f\xd3\x71\x3a\xf8\xbf\x1f\x37\x7f\x03\x00\x00\xff\xff\x09\x22\xea\x9c\x2e\x05\x00\x00") func configConfigTomlBytes() ([]byte, error) { return bindataRead( @@ -83,7 +83,7 @@ func configConfigToml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "config/config.toml", size: 1326, mode: os.FileMode(420), modTime: time.Unix(1536001378, 0)} + info := bindataFileInfo{name: "config/config.toml", size: 1326, mode: os.FileMode(420), modTime: time.Unix(1536003338, 0)} a := &asset{bytes: bytes, info: info} return a, nil } diff --git a/config/config.toml b/config/config.toml index 9d6b3c9f..fa7dbce3 100644 --- a/config/config.toml +++ b/config/config.toml @@ -25,8 +25,8 @@ version = "v0.3.20" [database] name = "malice-elastic" image = "malice/elasticsearch:6.4" - url = "http://localhost:9200" - # url = "http://elasticsearch:9200" + # url = "http://localhost:9200" + url = "http://elasticsearch:9200" username = "" password = "" ports = [9200] diff --git a/malice/database/database.go b/malice/database/database.go index 52859243..cd542605 100644 --- a/malice/database/database.go +++ b/malice/database/database.go @@ -9,7 +9,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/go-connections/nat" "github.com/docker/go-units" - "github.com/maliceio/go-plugin-utils/database/elasticsearch" + "github.com/malice-plugins/pkgs/database/elasticsearch" "github.com/maliceio/malice/config" "github.com/maliceio/malice/malice/docker/client" "github.com/maliceio/malice/malice/docker/client/container" diff --git a/malice/docker/client/client.go b/malice/docker/client/client.go index d5c69e32..df28773a 100644 --- a/malice/docker/client/client.go +++ b/malice/docker/client/client.go @@ -7,7 +7,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/docker/client" - "github.com/maliceio/go-plugin-utils/utils" + "github.com/malice-plugins/pkgs/utils" "github.com/maliceio/malice/config" "golang.org/x/net/context" ) diff --git a/malice/persist/file.go b/malice/persist/file.go index 6a6dd45c..9a92950f 100644 --- a/malice/persist/file.go +++ b/malice/persist/file.go @@ -26,7 +26,7 @@ import ( "github.com/docker/docker/pkg/stdcopy" "github.com/docker/go-units" "github.com/dustin/go-jsonpointer" - "github.com/maliceio/go-plugin-utils/utils" + "github.com/malice-plugins/pkgs/utils" "github.com/maliceio/malice/config" "github.com/maliceio/malice/malice/docker/client" er "github.com/maliceio/malice/malice/errors" diff --git a/plugins/plugins.go b/plugins/plugins.go index d68934ac..6aa72a0f 100644 --- a/plugins/plugins.go +++ b/plugins/plugins.go @@ -15,7 +15,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types/strslice" runconfigopts "github.com/docker/docker/runconfig/opts" - "github.com/maliceio/go-plugin-utils/utils" + "github.com/malice-plugins/pkgs/utils" "github.com/maliceio/malice/config" "github.com/maliceio/malice/malice/docker/client" "github.com/maliceio/malice/malice/docker/client/container" diff --git a/plugins/templates/go/scan.go b/plugins/templates/go/scan.go index 3f1b051c..83a249dd 100644 --- a/plugins/templates/go/scan.go +++ b/plugins/templates/go/scan.go @@ -11,8 +11,8 @@ import ( log "github.com/Sirupsen/logrus" "github.com/fatih/structs" - "github.com/maliceio/go-plugin-utils/database/elasticsearch" - "github.com/maliceio/go-plugin-utils/utils" + "github.com/malice-plugins/pkgs/database/elasticsearch" + "github.com/malice-plugins/pkgs/utils" "github.com/urfave/cli" ) diff --git a/vendor/github.com/malice-plugins/go-plugin-utils/LICENSE b/vendor/github.com/malice-plugins/pkgs/LICENSE similarity index 99% rename from vendor/github.com/malice-plugins/go-plugin-utils/LICENSE rename to vendor/github.com/malice-plugins/pkgs/LICENSE index b161bd8c..39f1f078 100644 --- a/vendor/github.com/malice-plugins/go-plugin-utils/LICENSE +++ b/vendor/github.com/malice-plugins/pkgs/LICENSE @@ -178,7 +178,7 @@ APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a diff --git a/vendor/github.com/malice-plugins/go-plugin-utils/database/database.go b/vendor/github.com/malice-plugins/pkgs/database/database.go similarity index 100% rename from vendor/github.com/malice-plugins/go-plugin-utils/database/database.go rename to vendor/github.com/malice-plugins/pkgs/database/database.go diff --git a/vendor/github.com/maliceio/go-plugin-utils/database/elasticsearch/elasticsearch.go b/vendor/github.com/malice-plugins/pkgs/database/elasticsearch/elasticsearch.go similarity index 92% rename from vendor/github.com/maliceio/go-plugin-utils/database/elasticsearch/elasticsearch.go rename to vendor/github.com/malice-plugins/pkgs/database/elasticsearch/elasticsearch.go index 91bdc5e9..243761a6 100644 --- a/vendor/github.com/maliceio/go-plugin-utils/database/elasticsearch/elasticsearch.go +++ b/vendor/github.com/malice-plugins/pkgs/database/elasticsearch/elasticsearch.go @@ -8,8 +8,8 @@ import ( "time" log "github.com/Sirupsen/logrus" - "github.com/malice-plugins/go-plugin-utils/database" - "github.com/malice-plugins/go-plugin-utils/utils" + "github.com/malice-plugins/pkgs/database" + "github.com/malice-plugins/pkgs/utils" "github.com/olivere/elastic" "github.com/pkg/errors" ) @@ -27,15 +27,18 @@ type Database struct { } var ( - defaultHost string - defaultPort string - defaultURL string + defaultIndex string + defaultType string + defaultHost string + defaultPort string + defaultURL string ) func init() { + defaultIndex = utils.Getopt("MALICE_ELASTICSEARCH_INDEX", "malice") + defaultType = utils.Getopt("MALICE_ELASTICSEARCH_TYPE", "samples") defaultHost = utils.Getopt("MALICE_ELASTICSEARCH_HOST", "localhost") defaultPort = utils.Getopt("MALICE_ELASTICSEARCH_PORT", "9200") - defaultURL = utils.Getopt("MALICE_ELASTICSEARCH_URL", fmt.Sprintf("http://%s:%s", defaultHost, defaultPort)) } // getURL with the following order of precedence @@ -45,6 +48,12 @@ func init() { func (db *Database) getURL() { // If not set use defaults + if len(strings.TrimSpace(db.Index)) == 0 { + db.Index = defaultIndex + } + if len(strings.TrimSpace(db.Type)) == 0 { + db.Type = defaultType + } if len(strings.TrimSpace(db.Host)) == 0 { db.Host = defaultHost } @@ -54,18 +63,15 @@ func (db *Database) getURL() { // If user set URL param use it if len(strings.TrimSpace(db.URL)) == 0 { - db.URL = defaultURL - } + // If running in docker use `elasticsearch` + if _, exists := os.LookupEnv("MALICE_IN_DOCKER"); exists { + db.URL = utils.Getopt("MALICE_ELASTICSEARCH_URL", fmt.Sprintf("%s:%s", "elasticsearch", db.Port)) + log.WithField("elasticsearch_url", db.URL).Debug("running malice in docker") + return + } - // If running in docker use `elasticsearch` - if _, exists := os.LookupEnv("MALICE_IN_DOCKER"); exists { - log.WithField("elasticsearch", db.URL).Debug("running malice in docker") - // TODO: change MALICE_ELASTICSEARCH to MALICE_ELASTICSEARCH_HOST - db.URL = utils.Getopt("MALICE_ELASTICSEARCH_URL", fmt.Sprintf("http://%s:%s", "elasticsearch", db.Port)) - return + db.URL = utils.Getopt("MALICE_ELASTICSEARCH_URL", fmt.Sprintf("%s:%s", db.Host, db.Port)) } - - db.URL = fmt.Sprintf("http://%s:%s", db.Host, db.Port) } // Init initalizes ElasticSearch for use with malice diff --git a/vendor/github.com/maliceio/go-plugin-utils/database/elasticsearch/mapping.go b/vendor/github.com/malice-plugins/pkgs/database/elasticsearch/mapping.go similarity index 100% rename from vendor/github.com/maliceio/go-plugin-utils/database/elasticsearch/mapping.go rename to vendor/github.com/malice-plugins/pkgs/database/elasticsearch/mapping.go diff --git a/vendor/github.com/malice-plugins/go-plugin-utils/utils/utils.go b/vendor/github.com/malice-plugins/pkgs/utils/utils.go similarity index 100% rename from vendor/github.com/malice-plugins/go-plugin-utils/utils/utils.go rename to vendor/github.com/malice-plugins/pkgs/utils/utils.go diff --git a/vendor/github.com/maliceio/go-plugin-utils/LICENSE b/vendor/github.com/maliceio/go-plugin-utils/LICENSE deleted file mode 100644 index b161bd8c..00000000 --- a/vendor/github.com/maliceio/go-plugin-utils/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2016 blacktop - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/maliceio/go-plugin-utils/utils/utils.go b/vendor/github.com/maliceio/go-plugin-utils/utils/utils.go deleted file mode 100644 index b2aec438..00000000 --- a/vendor/github.com/maliceio/go-plugin-utils/utils/utils.go +++ /dev/null @@ -1,244 +0,0 @@ -package utils - -import ( - "archive/zip" - "bytes" - "context" - "crypto/sha256" - "errors" - "fmt" - "io" - "io/ioutil" - "log" - "os" - "os/exec" - "path/filepath" - "regexp" - "strings" - - "github.com/parnurzeal/gorequest" -) - -// AppHelpTemplate is a default malice plugin help template -var AppHelpTemplate = `Usage: {{.Name}} {{if .Flags}}[OPTIONS] {{end}}COMMAND [arg...] - -{{.Usage}} - -Version: {{.Version}}{{if or .Author .Email}} - -Author:{{if .Author}} - {{.Author}}{{if .Email}} - <{{.Email}}>{{end}}{{else}} - {{.Email}}{{end}}{{end}} -{{if .Flags}} -Options: - {{range .Flags}}{{.}} - {{end}}{{end}} -Commands: - {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}} - {{end}} -Run '{{.Name}} COMMAND --help' for more information on a command. -` - -var camelingRegex = regexp.MustCompile("[0-9A-Za-z]+") - -// CamelCase converts strings to their camel case equivalent -func CamelCase(src string) string { - byteSrc := []byte(src) - chunks := camelingRegex.FindAll(byteSrc, -1) - for idx, val := range chunks { - if idx > 0 { - chunks[idx] = bytes.Title(val) - } - } - return string(bytes.Join(chunks, nil)) -} - -// Getopt reads environment variables. -// If not found will return a supplied default value -func Getopt(name, dfault string) string { - value := os.Getenv(name) - if value == "" { - value = dfault - } - return value -} - -// Getopts reads from user input then environment variable and finally a sane default. -func Getopts(userInput, envVar, dfault string) string { - - if len(strings.TrimSpace(userInput)) > 0 { - return userInput - } - value := os.Getenv(envVar) - if value == "" { - value = dfault - } - return value -} - -// Assert asserts there was no error, else log.Fatal -func Assert(err error) { - if err != nil { - log.Fatal(err) - } -} - -// GetSHA256 calculates a file's sha256sum -func GetSHA256(name string) string { - - dat, err := ioutil.ReadFile(name) - Assert(err) - - h256 := sha256.New() - _, err = h256.Write(dat) - Assert(err) - - return fmt.Sprintf("%x", h256.Sum(nil)) -} - -// RunCommand runs cmd on file -func RunCommand(ctx context.Context, cmd string, args ...string) (string, error) { - - var c *exec.Cmd - - if ctx != nil { - c = exec.CommandContext(ctx, cmd, args...) - } else { - c = exec.Command(cmd, args...) - } - - output, err := c.Output() - if err != nil { - return string(output), err - } - - // check for exec context timeout - if ctx != nil { - if ctx.Err() == context.DeadlineExceeded { - return "", fmt.Errorf("command %s timed out", cmd) - } - } - - return string(output), nil -} - -func printStatus(resp gorequest.Response, body string, errs []error) { - fmt.Println(resp.Status) -} - -// RemoveDuplicates removes duplicate items from a list -func RemoveDuplicates(elements []string) []string { - // Use map to record duplicates as we find them. - encountered := map[string]bool{} - result := []string{} - - for v := range elements { - if encountered[elements[v]] == true { - // Do not add duplicate. - } else { - // Record this element as an encountered element. - encountered[elements[v]] = true - // Append to result slice. - result = append(result, elements[v]) - } - } - // Return the new slice. - return result -} - -// GetHashType returns the hash type (md5, sha1, sha256, sha512) -func GetHashType(hash string) (string, error) { - var validMD5 = regexp.MustCompile(`^[a-fA-F\d]{32}$`) - var validSHA1 = regexp.MustCompile(`^[a-fA-F\d]{40}$`) - var validSHA256 = regexp.MustCompile(`^[a-fA-F\d]{64}$`) - var validSHA512 = regexp.MustCompile(`^[a-fA-F\d]{128}$`) - - switch { - case validMD5.MatchString(hash): - return "md5", nil - case validSHA1.MatchString(hash): - return "sha1", nil - case validSHA256.MatchString(hash): - return "sha256", nil - case validSHA512.MatchString(hash): - return "sha512", nil - default: - return "", errors.New("this is not a valid hash") - } -} - -// AskForConfirmation prompts user for yes/no response -func AskForConfirmation() bool { - var response string - _, err := fmt.Scanln(&response) - if err != nil { - log.Fatal(err) - } - okayResponses := []string{"y", "yes"} - nokayResponses := []string{"n", "no"} - if StringInSlice(strings.ToLower(response), okayResponses) { - return true - } - if StringInSlice(strings.ToLower(response), nokayResponses) { - return false - } - fmt.Println("Please type yes or no and then press enter:") - return AskForConfirmation() -} - -// Unzip unzips archive to target location -func Unzip(archive, target string) error { - - // fmt.Println("Unzip archive ", target) - - reader, err := zip.OpenReader(archive) - if err != nil { - return err - } - defer reader.Close() - - for _, file := range reader.File { - filePath := filepath.Join(target, file.Name) - - if file.FileInfo().IsDir() { - os.MkdirAll(filePath, file.Mode()) - continue - } - fileReader, err := file.Open() - if err != nil { - return err - } - defer fileReader.Close() - - targetFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode()) - if err != nil { - return err - } - defer targetFile.Close() - - if _, err := io.Copy(targetFile, fileReader); err != nil { - return err - } - } - return nil -} - -// SliceContainsString returns if slice contains substring -func SliceContainsString(a string, list []string) bool { - for _, b := range list { - if strings.Contains(b, a) { - return true - } - } - return false -} - -// StringInSlice returns whether or not a string exists in a slice -func StringInSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -}