From cec24d917fd0bfe6912b0b68bdb16f78e237acb4 Mon Sep 17 00:00:00 2001 From: Edward Viaene Date: Mon, 9 Sep 2024 11:17:04 -0500 Subject: [PATCH] packet log retention (default 7) --- pkg/rest/setup.go | 14 ++++++++++ pkg/rest/types.go | 1 + pkg/storage/memory/storage.go | 4 +++ pkg/wireguard/packetlogger.go | 42 +++++++++++++++++++++++----- pkg/wireguard/packetlogger_test.go | 42 ++++++++++++++++++++++++++++ pkg/wireguard/types.go | 1 + webapp/src/Routes/Setup/VPNSetup.tsx | 21 ++++++++++++-- 7 files changed, 116 insertions(+), 9 deletions(-) diff --git a/pkg/rest/setup.go b/pkg/rest/setup.go index 4354dbf..52f51d3 100644 --- a/pkg/rest/setup.go +++ b/pkg/rest/setup.go @@ -175,6 +175,9 @@ func (c *Context) vpnSetupHandler(w http.ResponseWriter, r *http.Request) { packetLogTypes = append(packetLogTypes, k) } } + if vpnConfig.PacketLogsRetention == 0 { + vpnConfig.PacketLogsRetention = 7 + } setupRequest := VPNSetupRequest{ Routes: strings.Join(vpnConfig.ClientRoutes, ", "), VPNEndpoint: vpnConfig.Endpoint, @@ -186,6 +189,7 @@ func (c *Context) vpnSetupHandler(w http.ResponseWriter, r *http.Request) { DisableNAT: vpnConfig.DisableNAT, EnablePacketLogs: vpnConfig.EnablePacketLogs, PacketLogsTypes: packetLogTypes, + PacketLogsRetention: strconv.Itoa(vpnConfig.PacketLogsRetention), } out, err := json.Marshal(setupRequest) if err != nil { @@ -272,6 +276,16 @@ func (c *Context) vpnSetupHandler(w http.ResponseWriter, r *http.Request) { vpnConfig.EnablePacketLogs = setupRequest.EnablePacketLogs writeVPNConfig = true } + packetLogsRention, err := strconv.Atoi(setupRequest.PacketLogsRetention) + if err != nil || packetLogsRention < 1 { + c.returnError(w, fmt.Errorf("incorrect packet log retention. Enter a number of days the logs must be kept (minimum 1)"), http.StatusBadRequest) + return + } + if packetLogsRention != vpnConfig.PacketLogsRetention { + vpnConfig.PacketLogsRetention = packetLogsRention + writeVPNConfig = true + } + // packetlogtypes packetLogTypes := []string{} for k, enabled := range vpnConfig.PacketLogsTypes { diff --git a/pkg/rest/types.go b/pkg/rest/types.go index fe72db0..dbbcd8e 100644 --- a/pkg/rest/types.go +++ b/pkg/rest/types.go @@ -112,6 +112,7 @@ type VPNSetupRequest struct { DisableNAT bool `json:"disableNAT"` EnablePacketLogs bool `json:"enablePacketLogs"` PacketLogsTypes []string `json:"packetLogsTypes"` + PacketLogsRetention string `json:"packetLogsRetention"` } type TemplateSetupRequest struct { diff --git a/pkg/storage/memory/storage.go b/pkg/storage/memory/storage.go index 2246e63..1bde56c 100644 --- a/pkg/storage/memory/storage.go +++ b/pkg/storage/memory/storage.go @@ -115,6 +115,10 @@ func (m *MockMemoryStorage) Remove(name string) error { if m.Data == nil { m.Data = make(map[string]*MockReadWriterData) } + _, ok := m.Data[name] + if !ok { + return fmt.Errorf("file does not exist") + } delete(m.Data, name) return nil } diff --git a/pkg/wireguard/packetlogger.go b/pkg/wireguard/packetlogger.go index 4d977fb..c69f98c 100644 --- a/pkg/wireguard/packetlogger.go +++ b/pkg/wireguard/packetlogger.go @@ -322,19 +322,38 @@ func packetLoggerLogRotation(storage storage.Iface) error { if err != nil { return fmt.Errorf("readDir error: %s", err) } + vpnConfig, err := GetVPNConfig(storage) + if err != nil { + return fmt.Errorf("cannot get vpn config: %s", err) + } + packetLogRetention := 7 // default packet log retention + if vpnConfig.PacketLogsRetention > 0 { + packetLogRetention = vpnConfig.PacketLogsRetention + } for _, filename := range files { - filenameSplit := strings.Split(strings.TrimSuffix(filename, ".log"), "-") + filenameWithoutSuffix := filename + filenameWithoutSuffix = strings.TrimSuffix(filenameWithoutSuffix, ".log.gz") + filenameWithoutSuffix = strings.TrimSuffix(filenameWithoutSuffix, ".log") + filenameSplit := strings.Split(filenameWithoutSuffix, "-") if len(filenameSplit) > 3 { dateParsed, err := time.Parse("2006-01-02", strings.Join(filenameSplit[len(filenameSplit)-3:], "-")) if err == nil { if !dateutils.DateEqual(dateParsed, time.Now()) { - err := packetLoggerCompressLog(storage, filename) - if err != nil { - return fmt.Errorf("rotate log error: %s", err) + if strings.HasSuffix(filename, ".log") { + err := packetLoggerCompressLog(storage, filename) + if err != nil { + return fmt.Errorf("rotate log error: %s", err) + } + err = packetLoggerRenameLog(storage, filename) + if err != nil { + return fmt.Errorf("rotate log error (rename): %s", err) + } } - err = packetLoggerRenameLog(storage, filename) - if err != nil { - return fmt.Errorf("rotate log error (rename): %s", err) + if strings.HasSuffix(filename, ".log.gz") { + err = removeLogsAfterRetentionPeriod(storage, filename, dateParsed, packetLogRetention) + if err != nil { + return fmt.Errorf("remove log error (tried to remove logs after retention period has lapsed): %s", err) + } } } @@ -381,3 +400,12 @@ func packetLoggerRenameLog(storage storage.Iface, filename string) error { } return nil } +func removeLogsAfterRetentionPeriod(storage storage.Iface, filename string, filenameDate time.Time, retentionDays int) error { + if time.Since(filenameDate) >= (time.Duration(retentionDays) * 24 * time.Hour) { + err := storage.Remove(path.Join(VPN_STATS_DIR, VPN_PACKETLOGGER_DIR, filename)) + if err != nil { + return fmt.Errorf("cannot remove %s: %s", filename, err) + } + } + return nil +} diff --git a/pkg/wireguard/packetlogger_test.go b/pkg/wireguard/packetlogger_test.go index edc00bb..0b0e2ef 100644 --- a/pkg/wireguard/packetlogger_test.go +++ b/pkg/wireguard/packetlogger_test.go @@ -282,3 +282,45 @@ func TestGetTimeUntilTomorrowStartOfDay(t *testing.T) { t.Fatalf("date is not tomorrow") } } + +func TestPacketLoggerLogRotationDeletion(t *testing.T) { + prefix := path.Join(VPN_STATS_DIR, VPN_PACKETLOGGER_DIR) + + storage := &memorystorage.MockMemoryStorage{ + Data: map[string]*memorystorage.MockReadWriterData{}, + } + for i := 0; i < 20; i++ { + timestamp := time.Now().AddDate(0, 0, -1*i) + suffix := ".log" + if i > 1 { + suffix = ".log.gz" + } + key1 := path.Join(prefix, fmt.Sprintf("1-2-3-4-%s%s", timestamp.Format("2006-01-02"), suffix)) + value1 := []byte(timestamp.Format(TIMESTAMP_FORMAT) + `,https,10.189.184.2,64.233.180.104,60496,443,www.google.com`) + err := storage.WriteFile(key1, value1) + if err != nil { + t.Fatalf("write file error: %s", err) + } + } + + before, err := storage.ReadDir(prefix) + if err != nil { + t.Fatalf("readdir error: %s", err) + } + + err = packetLoggerLogRotation(storage) + if err != nil { + t.Fatalf("packetLoggerRotation error: %s", err) + } + + after, err := storage.ReadDir(prefix) + if err != nil { + t.Fatalf("readdir error: %s", err) + } + if len(before) != 20 { + t.Fatalf("expected to have written 20 files. Got: %d", len(before)) + } + if len(after) != 7 { + t.Fatalf("only expected 7 days of retention. Got: %d", len(after)) + } +} diff --git a/pkg/wireguard/types.go b/pkg/wireguard/types.go index 7632da3..45c8083 100644 --- a/pkg/wireguard/types.go +++ b/pkg/wireguard/types.go @@ -44,6 +44,7 @@ type VPNConfig struct { ClientRoutes []string `json:"clientRoutes"` EnablePacketLogs bool `json:"enablePacketLogs"` PacketLogsTypes map[string]bool `json:"packetLogsTypes"` + PacketLogsRetention int `json:"packetLogsRetention"` } type PubKeyExchange struct { diff --git a/webapp/src/Routes/Setup/VPNSetup.tsx b/webapp/src/Routes/Setup/VPNSetup.tsx index f51d00c..ca26665 100644 --- a/webapp/src/Routes/Setup/VPNSetup.tsx +++ b/webapp/src/Routes/Setup/VPNSetup.tsx @@ -25,6 +25,7 @@ type VPNSetupRequest = { disableNAT: boolean, enablePacketLogs: boolean, packetLogsTypes: string[], + packetLogsRetention: string, }; export function VPNSetup() { const [saved, setSaved] = useState(false) @@ -57,7 +58,8 @@ export function VPNSetup() { nameservers: "", disableNAT: false, enablePacketLogs: false, - packetLogsTypes: [] + packetLogsTypes: [], + packetLogsRetention: "", }, }); const setupMutation = useMutation({ @@ -241,7 +243,8 @@ export function VPNSetup() { - {form.getValues().enablePacketLogs ? + {form.getValues().enablePacketLogs ? + <> + + + + : null}