From 20c2fd0c5feb0ab754acc6366c40da6cc5114bf0 Mon Sep 17 00:00:00 2001 From: Dustin Willis Webber Date: Wed, 13 May 2015 13:41:48 -0400 Subject: [PATCH] version bump 0.4.1 - see changelog for more information --- CHANGELOG.md | 6 ++- Makefile | 14 +++--- envdb.go | 5 ++- envdb_test.go | 1 + helpers.go | 54 ++++++++++++++++++++++ http.go | 4 ++ node.go | 29 ++++++------ node_database.go | 60 ++++++++++++++++++------- query.go | 96 ++++++++++++++++++---------------------- server.go | 83 +++++++++++++++++++++++++--------- web/index.html | 2 +- web/public/css/envdb.css | 83 +++++++++++++++++++++++++++++----- web/public/js/envdb.js | 17 ++++--- 13 files changed, 325 insertions(+), 129 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51b01df..19c2707 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,14 @@ # Changelog -## 0.4.1 2015-04-18 +## 0.4.1 2015-05-13 + Add loading indicator for system information request + bug fixes + + store envdb version for nodes + + Allow nodes list to scroll if needed. + + Update MakeFile version parsing logic. + + Revert removing debug symbols from build ## 0.4.0 2015-04-18 diff --git a/Makefile b/Makefile index 4ec2155..da58f35 100755 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ NAME="envdb" DESCRIPTION="Ask your environment questions" WEBSITE="http://envdb.io" -VERSION=$(shell cat $(NAME).go | grep -oP "Version\s+?\=\s?\"\K.*?(?=\"$|$\)") +VERSION=$(shell cat $(NAME).go | grep "Version =" | sed 's/Version\ \=//' | sed 's/"//g' | tr -d '[[:space:]]') CWD=$(shell pwd) GITHUB_USER=mephux @@ -29,7 +29,7 @@ all: bindata @$(ECHO) "$(OK_COLOR)==> Building $(NAME) $(VERSION) $(NO_COLOR)" @godep go build -ldflags "-s" -o bin/$(NAME) @chmod +x bin/$(NAME) - @$(ECHO) "$(OK_COLOR)==> Done$(NO_COLOR)" + @$(ECHO) "$(OK_COLOR)==> Done building$(NO_COLOR)" build: bindata all @$(ECHO) "$(OK_COLOR)==> Installing dependencies$(NO_COLOR)" @@ -42,7 +42,7 @@ updatedeps: @godep update ... bindata: - @$(ECHO) "$(OK_COLOR)==> Embedding Assets$(NO_COLOR)" + @$(ECHO) "$(OK_COLOR)==> Embedding assets$(NO_COLOR)" @go-bindata web/... test: @@ -53,13 +53,13 @@ goxBuild: @CGO_ENABLED=1 gox -os="$(CCOS)" -arch="$(CCARCH)" -build-toolchain gox: - @$(ECHO) "$(OK_COLOR)==> Cross Compiling $(NAME)$(NO_COLOR)" + @$(ECHO) "$(OK_COLOR)==> Cross compiling $(NAME)$(NO_COLOR)" @mkdir -p Godeps/_workspace/src/github.com/$(GITHUB_USER)/$(NAME) @cp -R *.go Godeps/_workspace/src/github.com/$(GITHUB_USER)/$(NAME) @CGO_ENABLED=1 GOPATH=$(shell godep path) gox -ldflags "-s" -os="$(CCOS)" -arch="$(CCARCH)" -output=$(CCOUTPUT) @rm -rf Godeps/_workspace/src/github.com/$(GITHUB_USER)/$(NAME) -release: clean bindata test gox setup package +release: clean all test gox setup package @for os in $(CCOS); do \ for arch in $(CCARCH); do \ cd pkg/$$os-$$arch/; \ @@ -67,7 +67,7 @@ release: clean bindata test gox setup package cd ../../; \ done \ done - @$(ECHO) "$(OK_COLOR)==> Done Cross Compiling $(NAME)$(NO_COLOR)" + @$(ECHO) "$(OK_COLOR)==> Done cross compiling $(NAME)$(NO_COLOR)" clean: @$(ECHO) "$(OK_COLOR)==> Cleaning$(NO_COLOR)" @@ -80,7 +80,7 @@ clean: @rm -rf package/ setup: - @$(ECHO) "$(OK_COLOR)==> Building Packages $(NAME)$(NO_COLOR)" + @$(ECHO) "$(OK_COLOR)==> Building packages $(NAME)$(NO_COLOR)" @echo $(VERSION) > .Version @mkdir -p package/root/usr/bin @cp -R pkg/linux-amd64/$(NAME) package/root/usr/bin diff --git a/envdb.go b/envdb.go index 1e59978..11278e5 100644 --- a/envdb.go +++ b/envdb.go @@ -17,7 +17,7 @@ const ( Name = "envdb" // Version application version number - Version = "0.4.0" + Version = "0.4.1" // DefaultServerPort the default tcp server port DefaultServerPort = 3636 @@ -64,6 +64,9 @@ var ( // debug logging and serving assets from disk // is enabled. DevMode bool + + // TestMode + TestMode bool ) func initLogger() { diff --git a/envdb_test.go b/envdb_test.go index fa2ec5d..15f1de4 100644 --- a/envdb_test.go +++ b/envdb_test.go @@ -18,6 +18,7 @@ var ( ) func TestMain(m *testing.M) { + TestMode = true *quiet = true initLogger() diff --git a/helpers.go b/helpers.go index 6f0fdc3..6a7eb0d 100644 --- a/helpers.go +++ b/helpers.go @@ -5,6 +5,8 @@ import ( "os" "regexp" "runtime" + "strconv" + "strings" ) const ( @@ -48,3 +50,55 @@ func IsExist(path string) bool { _, err := os.Stat(path) return err == nil || os.IsExist(err) } + +// VersionCheck +func VersionCheck(base, version string) bool { + if version == base { + return true + } + + sv := strings.Split(version, ".") + cv := strings.Split(base, ".") + + if len(sv) != 3 { + return false + } + + svi, err := strconv.Atoi(sv[0]) + + cvi, err := strconv.Atoi(cv[0]) + + if err != nil { + return false + } + + if svi < cvi { + return false + } + + svi2, err := strconv.Atoi(sv[1]) + + cvi2, err := strconv.Atoi(cv[1]) + + if err != nil { + return false + } + + if svi2 < cvi2 { + return false + } + + svi3, err := strconv.Atoi(sv[1]) + + cvi3, err := strconv.Atoi(cv[1]) + + if err != nil { + return false + } + + if svi3 < cvi3 { + return false + } + + return true +} diff --git a/http.go b/http.go index bccb2e7..4bfb8c2 100644 --- a/http.go +++ b/http.go @@ -119,6 +119,10 @@ func NewWebServer(webPort int, server *Server) { return server.Disconnect(id) }) + gotalk.Handle("disconnect-dead", func(id string) error { + return server.DisconnectDead(id) + }) + gotalk.Handle("delete", func(id string) error { return server.Delete(id) }) diff --git a/node.go b/node.go index cafe711..fa700cb 100644 --- a/node.go +++ b/node.go @@ -51,6 +51,7 @@ func (node *Node) Handlers() { handlers.HandleBufferNotification("die", func(s *gotalk.Sock, name string, b []byte) { KillClient = true node.Socket.Close() + Log.Warn(string(b)) Connection <- true }) @@ -96,17 +97,17 @@ func (node *Node) Handlers() { node.Config.WriteCache() } - has, version := OsQueryInfo() + info := OsQueryInfo() - Log.Infof("osquery enabled: %t", has) + Log.Infof("osquery enabled: %t", info.Enabled) - if has { - Log.Infof("osquery version: %s", version) + if info.Enabled { + Log.Infof("osquery version: %s", info.Version) } - if !CheckOsQueryVersion(version) { + if !VersionCheck(MinOsQueryVersion, info.Version) { Log.Errorf("%s requires osqueryi version %s or later.", Name, MinOsQueryVersion) - has = false + info.Enabled = false } var hostname = "n/a" @@ -129,13 +130,15 @@ func (node *Node) Handlers() { rmsg := Message{ Error: err, Data: map[string]interface{}{ - "name": node.Name, - "id": node.Id, - "osquery": has, - "osquery-version": version, - "ip": ip, - "hostname": hostname, - "os": os, + "envdb-version": Version, + "name": node.Name, + "id": node.Id, + "osquery": info.Enabled, + "osquery-version": info.Version, + "osquery-config-path": info.ConfigPath, + "ip": ip, + "hostname": hostname, + "os": os, }, } diff --git a/node_database.go b/node_database.go index 526e80c..d370144 100644 --- a/node_database.go +++ b/node_database.go @@ -9,16 +9,18 @@ import ( type NodeDb struct { Id int64 - NodeId string - Name string - Ip string - Hostname string - Os string + NodeId string + EnvdbVersion string + Name string + Ip string + Hostname string + Os string Online bool - OsQuery bool - OsQueryVersion string + OsQuery bool + OsQueryVersion string + OsQueryConfigPath string PendingDelete bool @@ -26,6 +28,28 @@ type NodeDb struct { Updated time.Time `xorm:"UPDATED"` } +// NodeUpdateOnlineStatus will update a nodes connection +// state on server start to clean up nodes that didn't properly disconnect +// if the server is killed without running cleanup. +func NodeUpdateOnlineStatus() error { + nodes, err := AllNodes() + + if err != nil { + return err + } + + for _, node := range nodes { + if node.Online { + node.Online = false + if err := node.Update(); err != nil { + return err + } + } + } + + return nil +} + // AllNodes Return all nodes in the database func AllNodes() ([]*NodeDb, error) { var nodes []*NodeDb @@ -72,11 +96,13 @@ func NodeUpdateOrCreate(node *NodeData) (*NodeDb, error) { Log.Debug("Found existing node record.") find.Name = node.Name + find.EnvdbVersion = node.EnvdbVersion find.Ip = node.Ip find.Hostname = node.Hostname find.Os = node.Os find.OsQuery = node.OsQuery find.OsQueryVersion = node.OsQueryVersion + find.OsQueryConfigPath = node.OsQueryConfigPath find.Online = node.Online find.PendingDelete = node.PendingDelete @@ -99,15 +125,17 @@ func NodeUpdateOrCreate(node *NodeData) (*NodeDb, error) { Log.Debugf("Creating a new record.") a := &NodeDb{ - NodeId: node.Id, - Name: node.Name, - Ip: node.Ip, - Hostname: node.Hostname, - Os: node.Os, - Online: node.Online, - OsQuery: node.OsQuery, - OsQueryVersion: node.OsQueryVersion, - PendingDelete: false, + NodeId: node.Id, + Name: node.Name, + EnvdbVersion: node.EnvdbVersion, + Ip: node.Ip, + Hostname: node.Hostname, + Os: node.Os, + Online: node.Online, + OsQuery: node.OsQuery, + OsQueryVersion: node.OsQueryVersion, + OsQueryConfigPath: node.OsQueryConfigPath, + PendingDelete: false, } if _, err := sess.Insert(a); err != nil { diff --git a/query.go b/query.go index 904f3b3..8800c7e 100644 --- a/query.go +++ b/query.go @@ -1,9 +1,9 @@ package main import ( + "encoding/json" "errors" "os/exec" - "strconv" "strings" "github.com/nu7hatch/gouuid" @@ -11,7 +11,8 @@ import ( const ( // MinOsQueryVersion Supported osqueryi version - MinOsQueryVersion = "1.4.4" + MinOsQueryVersion = "1.4.4" + DefaultOsQueryConfigPath = "/etc/osquery/osquery.conf" ) // Query Holds the raw sql and @@ -43,6 +44,13 @@ type Response struct { Error error `json:"error"` } +// OsQueryInfo holds information about osquery +type OsQueryMetadata struct { + Enabled bool + Version string + ConfigPath string +} + // Initialize a new Response func NewResponse() *Response { var id string @@ -58,79 +66,61 @@ func NewResponse() *Response { } } -// Check that the node has a proper osqueryi version -func CheckOsQueryVersion(version string) bool { - if version == MinOsQueryVersion { - return true - } - - sv := strings.Split(version, ".") - cv := strings.Split(MinOsQueryVersion, ".") - - if len(sv) != 3 { - return false +// OsQueryInfo ather information about osqueryi from the node. +func OsQueryInfo() *OsQueryMetadata { + var info = &OsQueryMetadata{ + Enabled: false, + Version: "", + ConfigPath: "", } - svi, err := strconv.Atoi(sv[0]) - - cvi, err := strconv.Atoi(cv[0]) + var output []byte - if err != nil { - return false - } + binary, lookErr := exec.LookPath("osqueryi") - if svi < cvi { - return false + if lookErr != nil { + return info } - svi2, err := strconv.Atoi(sv[1]) + output, err := exec.Command(binary, "--version").CombinedOutput() - cvi2, err := strconv.Atoi(cv[1]) + data := string(output) if err != nil { - return false - } + info.Version = data - if svi2 < cvi2 { - return false + return info } - svi3, err := strconv.Atoi(sv[1]) - - cvi3, err := strconv.Atoi(cv[1]) + newData := strings.Trim(strings.Replace(data, "osqueryi version ", "", -1), "\n") - if err != nil { - return false - } + info.Enabled = true + info.Version = newData - if svi3 < cvi3 { - return false + // Return before query exec + if TestMode { + return info } - return true -} + items := []string{binary, "--json", "select * from osquery_info;"} + if output, err := exec.Command("/usr/bin/sudo", items...).CombinedOutput(); err != nil { + return info + } else { -// Gather information about osqueryi from the node. -func OsQueryInfo() (bool, string) { - var output []byte + var jdata []map[string]interface{} - binary, lookErr := exec.LookPath("osqueryi") + if err := json.Unmarshal(output, &jdata); err != nil { + return info + } - if lookErr != nil { - return false, string(output) - } + if data, ok := jdata[0]["config_path"]; ok { + info.ConfigPath = data.(string) + } - output, err := exec.Command(binary, "--version").CombinedOutput() - - data := string(output) - - if err != nil { - return false, data + return info } - newData := strings.Trim(strings.Replace(data, "osqueryi version ", "", -1), "\n") - - return true, newData + return info } // Run a query for the node and return its diff --git a/server.go b/server.go index b879494..3837886 100644 --- a/server.go +++ b/server.go @@ -16,16 +16,18 @@ import ( // Holds node metadata. This struct is used by the server // to find and send command to individual nodes. type NodeData struct { - Id string `json:"id"` - Name string `json:"name"` - Ip string `json:"ip"` - Hostname string `json:"hostname"` - Socket *gotalk.Sock `json:"-"` - OsQuery bool - OsQueryVersion string - Online bool `json:"online"` - PendingDelete bool `json:"-"` - Os string `json:"os"` + Id string `json:"id"` + EnvdbVersion string `json:"envdb-version"` + Name string `json:"name"` + Ip string `json:"ip"` + Hostname string `json:"hostname"` + Socket *gotalk.Sock `json:"-"` + OsQuery bool + OsQueryVersion string + OsQueryConfigPath string + Online bool `json:"online"` + PendingDelete bool `json:"-"` + Os string `json:"os"` } // Server holds the tcp server socket, connected nodes @@ -60,6 +62,10 @@ func NewServer(port int) (*Server, error) { Log.Fatalf("Error: %s", err) } + if err := NodeUpdateOnlineStatus(); err != nil { + Log.Fatalf("Error: %s", err) + } + sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, os.Interrupt) @@ -113,17 +119,29 @@ func (server *Server) onAccept(s *gotalk.Sock) { Log.Infof("New node connected. (%s / %s)", resp.Data["name"], resp.Data["id"]) + if _, ok := resp.Data["envdb-version"]; !ok { + s.BufferNotify("die", []byte("This version of Envdb is out of date. Please upgrade.")) + return + } + + if !VersionCheck(Version, resp.Data["envdb-version"].(string)) { + s.BufferNotify("die", []byte("Envdb version mismatch")) + return + } + node := &NodeData{ - Id: resp.Data["id"].(string), - Name: resp.Data["name"].(string), - Online: true, - Socket: s, - OsQuery: resp.Data["osquery"].(bool), - OsQueryVersion: resp.Data["osquery-version"].(string), - Ip: resp.Data["ip"].(string), - Hostname: resp.Data["hostname"].(string), - PendingDelete: false, - Os: resp.Data["os"].(string), + Id: resp.Data["id"].(string), + Name: resp.Data["name"].(string), + EnvdbVersion: resp.Data["envdb-version"].(string), + Online: true, + Socket: s, + OsQuery: resp.Data["osquery"].(bool), + OsQueryVersion: resp.Data["osquery-version"].(string), + OsQueryConfigPath: resp.Data["osquery-config-path"].(string), + Ip: resp.Data["ip"].(string), + Hostname: resp.Data["hostname"].(string), + PendingDelete: false, + Os: resp.Data["os"].(string), } server.Nodes[s] = node @@ -210,6 +228,29 @@ func (server *Server) Disconnect(id string) error { return nil } +// DisconnectDead with disconnect a dead agent. +// +// This is useful when the server is killed for some reason without +// cleaning up the node connection state. +func (server *Server) DisconnectDead(id string) error { + node, err := GetNodeByNodeId(id) + + if err != nil { + return err + } + + node.Online = false + + if err := node.Update(); err != nil { + Log.Error("unable to update node record") + Log.Error("Error: ", err) + + return err + } + + return nil +} + // Disconnect and delete a node from the database. // // * NOTE: This is hacky because xorm has an issue with @@ -396,7 +437,6 @@ func (server *Server) sendTo(id string, in interface{}) *Response { // Send wraps the sendTo and sendAll functions and returns the Response // struct from the requested send type. func (server *Server) Send(id string, in interface{}) *Response { - if id == "all" { return server.sendAll(in) } @@ -421,7 +461,6 @@ func (server *Server) Ask(id, question string) (map[string]interface{}, error) { // Fetch a node by id from the Server.Nodes map func (server *Server) GetNodeById(id string) (*NodeData, error) { - for _, node := range server.Nodes { if node.Id == id { return node, nil diff --git a/web/index.html b/web/index.html index 7793b5e..aac3976 100644 --- a/web/index.html +++ b/web/index.html @@ -25,7 +25,7 @@ -