compression, fixes, wording changes (#18)

* compression, fixes, wording changes

* cleanup tmp files

pkg/rest/setup.go

@@ -312,6 +312,11 @@ func (c *Context) vpnSetupHandler(w http.ResponseWriter, r *http.Request) {
 				c.returnError(w, fmt.Errorf("could write vpn config: %s", err), http.StatusBadRequest)
 				return
 			}
+			err = wireguard.ReloadVPNServerConfig()
+			if err != nil {
+				c.returnError(w, fmt.Errorf("unable to reload server config: %s", err), http.StatusBadRequest)
+				return
+			}
 		}
 		if rewriteClientConfigs {
 			// rewrite client configs

pkg/rest/stats.go

@@ -3,8 +3,10 @@ package rest
 import (
 	"bufio"
 	"bytes"
+	"compress/gzip"
 	"encoding/json"
 	"fmt"
+	"io"
 	"math"
 	"net/http"
 	"path"
@@ -282,13 +284,19 @@ func (c *Context) packetLogsHandler(w http.ResponseWriter, r *http.Request) {
 		Data: []LogRow{},
 	}
 	// logs
-	statsFiles := []string{
-		path.Join(wireguard.VPN_STATS_DIR, wireguard.VPN_PACKETLOGGER_DIR, userID+"-"+date.Format("2006-01-02")+".log"),
+	statsFiles := []string{}
+	if offset > 0 {
+		statsFiles = append(statsFiles, path.Join(wireguard.VPN_STATS_DIR, wireguard.VPN_PACKETLOGGER_DIR, userID+"-"+date.AddDate(0, 0, -1).Format("2006-01-02")+".log"))
 	}
+	statsFiles = append(statsFiles, path.Join(wireguard.VPN_STATS_DIR, wireguard.VPN_PACKETLOGGER_DIR, userID+"-"+date.Format("2006-01-02")+".log"))
 	if !dateutils.DateEqual(time.Now(), date) { // date is in local timezone, and we are UTC, so also read next file
 		statsFiles = append(statsFiles, path.Join(wireguard.VPN_STATS_DIR, wireguard.VPN_PACKETLOGGER_DIR, userID+"-"+date.AddDate(0, 0, 1).Format("2006-01-02")+".log"))
 	}
-	statsFiles = filterNonExistentFiles(c.Storage.Client, statsFiles)
+	statsFiles, err = getCompressedFilesAndRemoveNonExistent(c.Storage.Client, statsFiles)
+	if err != nil {
+		c.returnError(w, fmt.Errorf("unable to get files for reading: %s", err), http.StatusBadRequest)
+		return
+	}
 	fileReaders, err := c.Storage.Client.OpenFilesFromPos(statsFiles, pos)
 	if err != nil {
 		c.returnError(w, fmt.Errorf("error while reading files: %s", err), http.StatusBadRequest)
@@ -353,14 +361,42 @@ func (c *Context) packetLogsHandler(w http.ResponseWriter, r *http.Request) {
 	c.write(w, out)
 }
 
-func filterNonExistentFiles(storage storage.Iface, files []string) []string {
+func getCompressedFilesAndRemoveNonExistent(storage storage.Iface, files []string) ([]string, error) {
 	res := []string{}
 	for _, file := range files {
+		tmpFile := path.Join(wireguard.VPN_PACKETLOGGER_TMP_DIR, path.Base(file))
 		if storage.FileExists(file) {
 			res = append(res, file)
+		} else if storage.FileExists(tmpFile) { // temporary file exists
+			res = append(res, tmpFile)
+		} else if storage.FileExists(file + ".gz") { // uncompress log file for random access
+			err := storage.EnsurePath(wireguard.VPN_PACKETLOGGER_TMP_DIR)
+			if err != nil {
+				return res, fmt.Errorf("ensure path error: %s", err)
+			}
+			compressedFile, err := storage.OpenFile(file + ".gz")
+			if err != nil {
+				return res, fmt.Errorf("unable to open compress filed (%s): %s", file+".gz", err)
+			}
+			gzipReader, err := gzip.NewReader(compressedFile)
+			if err != nil {
+				return res, fmt.Errorf("unable to open gzip reader (%s): %s", file+".gz", err)
+			}
+			fileWriter, err := storage.OpenFileForWriting(tmpFile)
+			if err != nil {
+				return res, fmt.Errorf("unable to open tmp file writer (%s): %s", tmpFile, err)
+			}
+			_, err = io.Copy(fileWriter, gzipReader)
+			if err != nil {
+				return res, fmt.Errorf("unable to uncompress to file (%s): %s", tmpFile, err)
+			}
+			fileWriter.Close()
+			gzipReader.Close()
+			compressedFile.Close()
+			res = append(res, tmpFile)
 		}
 	}
-	return res
+	return res, nil
 }
 
 func getColor(i int) string {

pkg/rest/stats_test.go

@@ -1,7 +1,10 @@
 package rest
 
 import (
+	"bytes"
+	"compress/gzip"
 	"encoding/json"
+	"io"
 	"net/http/httptest"
 	"path"
 	"strings"
@@ -78,3 +81,43 @@ func TestFilterLogRecord(t *testing.T) {
 		}
 	}
 }
+
+func TestGetCompressedFilesAndRemoveNonExistent(t *testing.T) {
+	now := time.Now()
+	storage := &memorystorage.MockMemoryStorage{}
+	testData := now.Format(wireguard.TIMESTAMP_FORMAT) + ",3df97301-5f73-407a-a26b-91829f1e7f48,1,12729136,24348520,2024-08-23T18:30:42\n"
+	files := []string{
+		path.Join(wireguard.VPN_STATS_DIR, wireguard.VPN_PACKETLOGGER_DIR, "1-2-3-4-"+now.AddDate(0, 0, -1).Format("2006-01-02")+".log"),
+		path.Join(wireguard.VPN_STATS_DIR, wireguard.VPN_PACKETLOGGER_DIR, "1-2-3-4-"+now.Format("2006-01-02")+".log"),
+	}
+	for k, file := range files {
+		if k != len(files)-1 { // only the last file is not compressed
+			fileWriter, err := storage.OpenFileForWriting(file + ".gz")
+			if err != nil {
+				t.Fatalf("open file for wring error: %s", err)
+			}
+			writer := gzip.NewWriter(fileWriter)
+			io.Copy(writer, bytes.NewReader([]byte(testData)))
+			writer.Close()
+			fileWriter.Close()
+		} else {
+			storage.WriteFile(file, []byte(testData))
+		}
+	}
+	outFiles, err := getCompressedFilesAndRemoveNonExistent(storage, files)
+	if err != nil {
+		t.Fatalf("get files error: %s", err)
+	}
+	if len(outFiles) != 2 {
+		t.Fatalf("expected 2 files, got: %d", len(outFiles))
+	}
+	for _, file := range outFiles {
+		body, err := storage.ReadFile(file)
+		if err != nil {
+			t.Fatalf("readfile error: %s", err)
+		}
+		if string(body) != string(testData) {
+			t.Fatalf("mismatch: got: %s vs expected: %s\n", string(body), string(testData))
+		}
+	}
+}

pkg/storage/iface.go

@@ -14,6 +14,7 @@ type Iface interface {
 	Rename(oldName, newName string) error
 	AppendFile(name string, data []byte) error
 	EnsurePermissions(name string, mode fs.FileMode) error
+	FileInfo(name string) (fs.FileInfo, error)
 	ReadWriter
 	Seeker
 }

pkg/storage/local/path.go

@@ -90,3 +90,7 @@ func (l *LocalStorage) Rename(oldName, newName string) error {
 func (l *LocalStorage) EnsurePermissions(name string, mode fs.FileMode) error {
 	return os.Chmod(path.Join(l.path, name), mode)
 }
+
+func (l *LocalStorage) FileInfo(name string) (fs.FileInfo, error) {
+	return os.Stat(name)
+}

pkg/storage/memory/fileinfo.go

@@ -0,0 +1,34 @@
+package memorystorage
+
+import (
+	"io/fs"
+	"time"
+)
+
+type FileInfo struct {
+	NameOut    string      // base name of the file
+	SizeOut    int64       // length in bytes for regular files; system-dependent for others
+	ModeOut    fs.FileMode // file mode bits
+	ModTimeOut time.Time   // modification time
+	IsDirOut   bool        // abbreviation for Mode().IsDir()
+	SysOut     any         // underlying data source (can return nil)
+}
+
+func (f FileInfo) Name() string {
+	return f.NameOut
+}
+func (f FileInfo) Size() int64 {
+	return f.SizeOut
+}
+func (f FileInfo) Mode() fs.FileMode {
+	return f.ModeOut
+}
+func (f FileInfo) ModTime() time.Time {
+	return f.ModTimeOut
+}
+func (f FileInfo) IsDir() bool {
+	return f.IsDirOut
+}
+func (f FileInfo) Sys() any {
+	return nil
+}

pkg/storage/memory/storage.go

@@ -1,7 +1,6 @@
 package memorystorage
 
 import (
-	"bufio"
 	"bytes"
 	"fmt"
 	"io"
@@ -11,14 +10,6 @@ import (
 	"strings"
 )
 
-type MyWriteCloser struct {
-	*bufio.Writer
-}
-
-func (mwc *MyWriteCloser) Close() error {
-	return nil
-}
-
 type MockReadWriterData []byte
 
 func (m *MockReadWriterData) Close() error {
@@ -30,7 +21,8 @@ func (m *MockReadWriterData) Write(p []byte) (nn int, err error) {
 }
 
 type MockMemoryStorage struct {
-	Data map[string]*MockReadWriterData
+	FileInfoData map[string]*FileInfo
+	Data         map[string]*MockReadWriterData
 }
 
 func (m *MockMemoryStorage) ConfigPath(filename string) string {
@@ -160,3 +152,10 @@ func (m *MockMemoryStorage) OpenFileForAppending(name string) (io.WriteCloser, e
 func (m *MockMemoryStorage) EnsurePermissions(name string, mode fs.FileMode) error {
 	return nil
 }
+func (m *MockMemoryStorage) FileInfo(name string) (fs.FileInfo, error) {
+	val, ok := m.FileInfoData[name]
+	if !ok {
+		return FileInfo{}, fmt.Errorf("couldn't get file info for: %s", name)
+	}
+	return val, nil
+}

pkg/wireguard/constants.go

@@ -9,6 +9,7 @@ const IP_LIST_PATH = "config/iplist.json"
 const VPN_CLIENTS_DIR = "clients"
 const VPN_STATS_DIR = "stats"
 const VPN_PACKETLOGGER_DIR = "packetlogs"
+const VPN_PACKETLOGGER_TMP_DIR = "tmp"
 const VPN_SERVER_SECRETS_PATH = "secrets"
 const VPN_PRIVATE_KEY_FILENAME = "priv.key"
 const PRESHARED_KEY_FILENAME = "preshared.key"