From af610d8bbebd272b496f0dc25c53218ab3bb5ee3 Mon Sep 17 00:00:00 2001 From: Edward Viaene Date: Fri, 23 Aug 2024 15:56:29 -0500 Subject: [PATCH] stats endpoint --- latest | 2 +- pkg/configmanager/handlers.go | 2 +- pkg/configmanager/server.go | 2 + pkg/configmanager/start_darwin.go | 6 +- pkg/configmanager/start_linux.go | 10 +-- pkg/logging/log.go | 4 +- pkg/rest/router.go | 1 + pkg/rest/stats.go | 106 +++++++++++++++++++++++++++++ pkg/rest/stats_test.go | 64 +++++++++++++++++ pkg/rest/types.go | 20 ++++++ pkg/storage/local/write.go | 2 +- pkg/wireguard/stats_linux.go | 22 ++++-- provisioning/scripts/install_s3.sh | 5 +- 13 files changed, 230 insertions(+), 16 deletions(-) create mode 100644 pkg/rest/stats.go create mode 100644 pkg/rest/stats_test.go diff --git a/latest b/latest index 795460f..15e52dc 100644 --- a/latest +++ b/latest @@ -1 +1 @@ -v1.1.0 +v1.1.1beta1 diff --git a/pkg/configmanager/handlers.go b/pkg/configmanager/handlers.go index 2b8edba..f66f7dd 100644 --- a/pkg/configmanager/handlers.go +++ b/pkg/configmanager/handlers.go @@ -120,7 +120,7 @@ func (c *ConfigManager) version(w http.ResponseWriter, r *http.Request) { func (c *ConfigManager) restartVpn(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodPost: - err := stopVPN(c.Storage) + err := stopVPN() if err != nil { // don't exit, as the VPN might be down already. fmt.Println("========= Warning =========") fmt.Printf("Warning: vpn stop error: %s\n", err) diff --git a/pkg/configmanager/server.go b/pkg/configmanager/server.go index 6b5ebf9..4ea37f9 100644 --- a/pkg/configmanager/server.go +++ b/pkg/configmanager/server.go @@ -40,6 +40,8 @@ func StartServer(port int) { log.Fatalf("could not refresh all clients: %s", err) } + startStats(localStorage) // start gathering of wireguard stats + log.Printf("Starting localhost http server at port %d\n", port) log.Fatal(http.ListenAndServe(fmt.Sprintf("127.0.0.1:%d", port), c.getRouter())) } diff --git a/pkg/configmanager/start_darwin.go b/pkg/configmanager/start_darwin.go index 47864ee..2f7bb3e 100644 --- a/pkg/configmanager/start_darwin.go +++ b/pkg/configmanager/start_darwin.go @@ -14,7 +14,11 @@ func startVPN(storage storage.Iface) error { return nil } -func stopVPN(storage storage.Iface) error { +func stopVPN() error { fmt.Printf("Warning: startVPN is not implemented in darwin\n") return nil } + +func startStats(storage storage.Iface) { + fmt.Printf("Warning: startStats is not implemented in darwin\n") +} diff --git a/pkg/configmanager/start_linux.go b/pkg/configmanager/start_linux.go index d58a10a..334f6df 100644 --- a/pkg/configmanager/start_linux.go +++ b/pkg/configmanager/start_linux.go @@ -16,12 +16,14 @@ func startVPN(storage storage.Iface) error { log.Fatalf("WriteWireGuardServerConfig error: %s", err) } - // run statistics go routine - go wireguard.RunStats(storage) - return wireguard.StartVPN() } -func stopVPN(storage storage.Iface) error { +func stopVPN() error { return wireguard.StopVPN() } + +func startStats(storage storage.Iface) { + // run statistics go routine + go wireguard.RunStats(storage) +} diff --git a/pkg/logging/log.go b/pkg/logging/log.go index 4cb44fb..2b0f555 100644 --- a/pkg/logging/log.go +++ b/pkg/logging/log.go @@ -10,13 +10,13 @@ const LOG_DEBUG = 16 func DebugLog(err error) { if Loglevel&LOG_DEBUG == LOG_DEBUG { - fmt.Println(err) + fmt.Println("debug: " + err.Error()) } } func ErrorLog(err error) { if Loglevel&LOG_ERROR == LOG_ERROR { - fmt.Println("debug: " + err.Error()) + fmt.Println("error: " + err.Error()) } } diff --git a/pkg/rest/router.go b/pkg/rest/router.go index 9c11787..c01cae8 100644 --- a/pkg/rest/router.go +++ b/pkg/rest/router.go @@ -62,6 +62,7 @@ func (c *Context) getRouter(assets fs.FS, indexHtml []byte) *http.ServeMux { mux.Handle("/api/saml-setup/{id}", c.authMiddleware(c.injectUserMiddleware(c.isAdminMiddleware(http.HandlerFunc(c.samlSetupElementHandler))))) mux.Handle("/api/users", c.authMiddleware(c.injectUserMiddleware(c.isAdminMiddleware(http.HandlerFunc(c.usersHandler))))) mux.Handle("/api/user/{id}", c.authMiddleware(c.injectUserMiddleware(c.isAdminMiddleware(http.HandlerFunc(c.userHandler))))) + mux.Handle("/api/stats/user", c.authMiddleware(c.injectUserMiddleware(c.isAdminMiddleware(http.HandlerFunc(c.userStatsHandler))))) return mux } diff --git a/pkg/rest/stats.go b/pkg/rest/stats.go new file mode 100644 index 0000000..8fc7615 --- /dev/null +++ b/pkg/rest/stats.go @@ -0,0 +1,106 @@ +package rest + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "net/http" + "path" + "strconv" + "strings" + "time" + + "github.com/in4it/wireguard-server/pkg/wireguard" +) + +func (c *Context) userStatsHandler(w http.ResponseWriter, r *http.Request) { + var userStatsResponse UserStatsResponse + statsFile := c.Storage.Client.ConfigPath(path.Join(wireguard.VPN_STATS_DIR, "user-"+time.Now().Format("2006-01-02")) + ".log") + if !c.Storage.Client.FileExists(statsFile) { // file does not exist so just return empty response + out, err := json.Marshal(userStatsResponse) + if err != nil { + c.returnError(w, fmt.Errorf("user stats response marshal error: %s", err), http.StatusBadRequest) + return + } + c.write(w, out) + return + } + logData, err := c.Storage.Client.ReadFile(statsFile) + if err != nil { + c.returnError(w, fmt.Errorf("readfile error: %s", err), http.StatusBadRequest) + return + } + scanner := bufio.NewScanner(bytes.NewReader(logData)) + + receiveBytesLast := make(map[string]int64) + transmitBytesLast := make(map[string]int64) + receiveBytesData := make(map[string][]UserStatsDataPoint) + transmitBytesData := make(map[string][]UserStatsDataPoint) + for scanner.Scan() { // all other entries + inputSplit := strings.Split(scanner.Text(), ",") + userID := inputSplit[1] + if _, ok := receiveBytesLast[userID]; !ok { + val, err := strconv.ParseInt(inputSplit[3], 10, 64) + if err == nil { + receiveBytesLast[userID] = val + } else { + receiveBytesLast[userID] = 0 + } + } + if _, ok := transmitBytesLast[userID]; !ok { + val, err := strconv.ParseInt(inputSplit[4], 10, 64) + if err == nil { + transmitBytesLast[userID] = val + } else { + transmitBytesLast[userID] = 0 + } + } + receiveBytes, err := strconv.ParseInt(inputSplit[3], 10, 64) + if err == nil { + if _, ok := receiveBytesData[userID]; !ok { + receiveBytesData[userID] = []UserStatsDataPoint{} + } + receiveBytesData[userID] = append(receiveBytesData[userID], UserStatsDataPoint{X: inputSplit[0], Y: receiveBytes - receiveBytesLast[userID]}) + } + transmitBytes, err := strconv.ParseInt(inputSplit[4], 10, 64) + if err == nil { + if _, ok := transmitBytesData[userID]; !ok { + transmitBytesData[userID] = []UserStatsDataPoint{} + } + transmitBytesData[userID] = append(transmitBytesData[userID], UserStatsDataPoint{X: inputSplit[0], Y: transmitBytes - transmitBytesLast[userID]}) + } + receiveBytesLast[userID] = receiveBytes + transmitBytesLast[userID] = transmitBytes + } + + if err := scanner.Err(); err != nil { + c.returnError(w, fmt.Errorf("log file read (scanner) error: %s", err), http.StatusBadRequest) + return + } + userStatsResponse.ReceiveBytes = UserStatsData{ + Datasets: []UserStatsDataset{}, + } + userStatsResponse.TransmitBytes = UserStatsData{ + Datasets: []UserStatsDataset{}, + } + for userID, data := range receiveBytesData { + userStatsResponse.ReceiveBytes.Datasets = append(userStatsResponse.ReceiveBytes.Datasets, UserStatsDataset{ + Label: userID, + Data: data, + }) + } + for userID, data := range transmitBytesData { + userStatsResponse.TransmitBytes.Datasets = append(userStatsResponse.TransmitBytes.Datasets, UserStatsDataset{ + Label: userID, + Data: data, + }) + } + + out, err := json.Marshal(userStatsResponse) + if err != nil { + c.returnError(w, fmt.Errorf("user stats response marshal error: %s", err), http.StatusBadRequest) + return + } + c.write(w, out) +} diff --git a/pkg/rest/stats_test.go b/pkg/rest/stats_test.go new file mode 100644 index 0000000..ea5b1f3 --- /dev/null +++ b/pkg/rest/stats_test.go @@ -0,0 +1,64 @@ +package rest + +import ( + "encoding/json" + "net/http/httptest" + "path" + "testing" + "time" + + testingmocks "github.com/in4it/wireguard-server/pkg/testing/mocks" + "github.com/in4it/wireguard-server/pkg/wireguard" +) + +func TestUserStatsHandler(t *testing.T) { + + storage := &testingmocks.MockMemoryStorage{} + + c, err := newContext(storage, SERVER_TYPE_VPN) + if err != nil { + t.Fatalf("Cannot create context") + } + testData := `2024-08-23T19:29:03,3df97301-5f73-407a-a26b-91829f1e7f48,1,12729136,24348520,2024-08-23T18:30:42 +2024-08-23T19:34:03,3df97301-5f73-407a-a26b-91829f1e7f48,1,13391716,25162108,2024-08-23T19:33:38 +2024-08-23T19:39:03,3df97301-5f73-407a-a26b-91829f1e7f48,1,14419152,27496068,2024-08-23T19:37:39 +2024-08-23T19:44:03,3df97301-5f73-407a-a26b-91829f1e7f48,1,16003988,30865740,2024-08-23T19:42:51 +2024-08-23T19:49:03,3df97301-5f73-407a-a26b-91829f1e7f48,1,19777928,57367624,2024-08-23T19:48:51 +2024-08-23T19:54:03,3df97301-5f73-407a-a26b-91829f1e7f48,1,23772276,75895264,2024-08-23T19:52:51 +2024-08-23T19:59:03,3df97301-5f73-407a-a26b-91829f1e7f48,1,25443216,81496940,2024-08-23T19:58:52 +2024-08-23T20:04:03,3df97301-5f73-407a-a26b-91829f1e7f48,1,26574324,83886164,2024-08-23T20:02:53 +2024-08-23T20:09:03,3df97301-5f73-407a-a26b-91829f1e7f48,1,39928520,85171728,2024-08-23T20:08:54` + + statsFile := c.Storage.Client.ConfigPath(path.Join(wireguard.VPN_STATS_DIR, "user-"+time.Now().Format("2006-01-02")) + ".log") + err = c.Storage.Client.WriteFile(statsFile, []byte(testData)) + if err != nil { + t.Fatalf("Cannot write test file") + } + + req := httptest.NewRequest("GET", "http://example.com/stats/user", nil) + w := httptest.NewRecorder() + c.userStatsHandler(w, req) + + resp := w.Result() + + if resp.StatusCode != 200 { + t.Fatalf("status code is not 200: %d", resp.StatusCode) + } + + defer resp.Body.Close() + + var userStatsResponse UserStatsResponse + + err = json.NewDecoder(resp.Body).Decode(&userStatsResponse) + if err != nil { + t.Fatalf("Cannot decode response from create user: %s", err) + } + + if userStatsResponse.ReceiveBytes.Datasets[0].Data[1].Y != 662580 { + t.Fatalf("unexpected data: %d", userStatsResponse.ReceiveBytes.Datasets[0].Data[1].Y) + } + if userStatsResponse.TransmitBytes.Datasets[0].Data[1].Y != 813588 { + t.Fatalf("unexpected data: %d", userStatsResponse.TransmitBytes.Datasets[0].Data[1].Y) + } + +} diff --git a/pkg/rest/types.go b/pkg/rest/types.go index d4287ac..223b0ae 100644 --- a/pkg/rest/types.go +++ b/pkg/rest/types.go @@ -174,3 +174,23 @@ type SAMLSetup struct { MetadataURL string `json:"metadataURL,omitempty"` RegenerateCert bool `json:"regenerateCert,omitempty"` } + +type UserStatsResponse struct { + ReceiveBytes UserStatsData `json:"receivedBytes"` + TransmitBytes UserStatsData `json:"transmitBytes"` +} +type UserStatsData struct { + Datasets []UserStatsDataset `json:"datasets"` +} +type UserStatsDataset struct { + Label string `json:"label"` + Data []UserStatsDataPoint `json:"data"` + Fill bool `json:"fill"` + BorderColor string `json:"borderColor"` + Tension float64 `json:"tension"` +} + +type UserStatsDataPoint struct { + X string `json:"x"` + Y int64 `json:"y"` +} diff --git a/pkg/storage/local/write.go b/pkg/storage/local/write.go index 1ca0508..00e135c 100644 --- a/pkg/storage/local/write.go +++ b/pkg/storage/local/write.go @@ -10,7 +10,7 @@ func (l *LocalStorage) WriteFile(name string, data []byte) error { } func (l *LocalStorage) AppendFile(name string, data []byte) error { - f, err := os.OpenFile("text.log", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + f, err := os.OpenFile(path.Join(l.path, name), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) if err != nil { return err } diff --git a/pkg/wireguard/stats_linux.go b/pkg/wireguard/stats_linux.go index 5d6abb8..93d57de 100644 --- a/pkg/wireguard/stats_linux.go +++ b/pkg/wireguard/stats_linux.go @@ -25,6 +25,11 @@ func RunStats(storage storage.Iface) { logging.ErrorLog(fmt.Errorf("could not create stats path: %s. Stats disabled", err)) return } + err = storage.EnsureOwnership(storage.ConfigPath(VPN_STATS_DIR), "vpn") + if err != nil { + logging.ErrorLog(fmt.Errorf("could not ensure ownership of stats path: %s. Stats disabled", err)) + return + } for { err := runStats(storage) if err != nil { @@ -49,6 +54,7 @@ func runStats(storage storage.Iface) error { if stat.PublicKey == peerConfig.PublicKey { user, connectionID := splitUserAndConnectionID(peerConfig.ID) statsEntries = append(statsEntries, StatsEntry{ + Timestamp: stat.Timestamp, User: user, ConnectionID: connectionID, TransmitBytes: stat.TransmitBytes, @@ -59,12 +65,18 @@ func runStats(storage storage.Iface) error { } } - statsCsv := statsToCsv(statsEntries) + if len(statsEntries) > 0 { + statsCsv := statsToCsv(statsEntries) - peerConfigPath := storage.ConfigPath(path.Join(VPN_STATS_DIR, "user-"+time.Now().Format("2006-01-02"))) - err = storage.AppendFile(peerConfigPath, statsCsv) - if err != nil { - return fmt.Errorf("could not append stats to file (%s): %s", peerConfigPath, err) + statsPath := storage.ConfigPath(path.Join(VPN_STATS_DIR, "user-"+time.Now().Format("2006-01-02")) + ".log") + err = storage.AppendFile(statsPath, statsCsv) + if err != nil { + return fmt.Errorf("could not append stats to file (%s): %s", statsPath, err) + } + err = storage.EnsureOwnership(statsPath, "vpn") + if err != nil { + return fmt.Errorf("could not ensure ownership of stats file (%s): %s", statsPath, err) + } } return nil } diff --git a/provisioning/scripts/install_s3.sh b/provisioning/scripts/install_s3.sh index d52216f..90f75aa 100755 --- a/provisioning/scripts/install_s3.sh +++ b/provisioning/scripts/install_s3.sh @@ -6,4 +6,7 @@ aws s3 cp ../reset-admin-password-linux-amd64 s3://in4it-vpn-server/assets/binar aws s3 cp ../reset-admin-password-linux-amd64.sha256 s3://in4it-vpn-server/assets/binaries/${LATEST}/reset-admin-password-linux-amd64.sha256 aws s3 cp ../configmanager-linux-amd64 s3://in4it-vpn-server/assets/binaries/${LATEST}/configmanager-linux-amd64 aws s3 cp ../configmanager-linux-amd64.sha256 s3://in4it-vpn-server/assets/binaries/${LATEST}/configmanager-linux-amd64.sha256 -aws s3 cp ../latest s3://in4it-vpn-server/assets/binaries/latest +if [ "$1" == "--release" ] ; then + echo "=> $LATEST released." + #aws s3 cp ../latest s3://in4it-vpn-server/assets/binaries/latest +fi