start of pagination

pkg/rest/stats.go

@@ -16,6 +16,8 @@ import (
 	"github.com/in4it/wireguard-server/pkg/wireguard"
 )
 
+const MAX_LOG_OUTPUT_LINES = 5
+
 func (c *Context) userStatsHandler(w http.ResponseWriter, r *http.Request) {
 	if r.PathValue("date") == "" {
 		c.returnError(w, fmt.Errorf("no date supplied"), http.StatusBadRequest)
@@ -255,11 +257,29 @@ func (c *Context) packetLogsHandler(w http.ResponseWriter, r *http.Request) {
 	// get filter
 	logTypeFilterQueryString := r.URL.Query().Get("logtype")
 	logTypeFilter := strings.Split(logTypeFilterQueryString, ",")
+	// initialize response
+	logData := LogData{
+		Schema: LogSchema{
+			Columns: map[string]string{
+				"Protocol":         "string",
+				"Source IP":        "string",
+				"Destination IP":   "string",
+				"Source Port":      "string",
+				"Destination Port": "string",
+				"Destination":      "string",
+			},
+		},
+		Data: []LogRow{},
+	}
 	// logs
 	statsFiles := []string{
 		path.Join(wireguard.VPN_STATS_DIR, wireguard.VPN_PACKETLOGGER_DIR, userID+"-"+date.Format("2006-01-02")+".log"),
 	}
+	if !dateEqual(time.Now(), date) { // date is in local timezone, and we are UTC, so also read next file
+		statsFiles = append(statsFiles, path.Join(wireguard.VPN_STATS_DIR, wireguard.VPN_PACKETLOGGER_DIR, userID+"-"+date.AddDate(0, 0, 1).Format("2006-01-02")+".log"))
+	}
 	logInputData := bytes.NewBuffer([]byte{})
+	//OpenFilesFromPos(statsFiles, 0) ([]io.Reader, error)
 	for _, statsFile := range statsFiles {
 		if c.Storage.Client.FileExists(statsFile) {
 			fileLogData, err := c.Storage.Client.ReadFile(statsFile)
@@ -268,48 +288,47 @@ func (c *Context) packetLogsHandler(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			logInputData.Write(fileLogData)
-		} else {
-			fmt.Printf("File does not exist: %s", statsFile)
 		}
 	}
 
+	pos := int64(0)
 	scanner := bufio.NewScanner(logInputData)
+	scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
+		advance, token, err = bufio.ScanLines(data, atEOF)
+		pos += int64(advance)
+		return
+	})
 
-	logData := LogData{
-		Schema: LogSchema{
-			Columns: map[string]string{
-				"Protocol":         "string",
-				"Source IP":        "string",
-				"Destination IP":   "string",
-				"Source Port":      "string",
-				"Destination Port": "string",
-				"Destination":      "string",
-			},
-		},
-		Data: []LogRow{},
-	}
-
-	for scanner.Scan() {
+	for scanner.Scan() && len(logData.Data) < MAX_LOG_OUTPUT_LINES {
 		inputSplit := strings.Split(scanner.Text(), ",")
 		timestamp, err := time.Parse(wireguard.TIMESTAMP_FORMAT, inputSplit[0])
 		if err != nil {
 			continue // invalid record
 		}
-		if !filterLogRecord(logTypeFilter, inputSplit[1]) {
-			row := LogRow{
-				Timestamp: timestamp.Add(time.Duration(offset) * time.Minute),
-				Data:      inputSplit[1:],
+		timestamp = timestamp.Add(time.Duration(offset) * time.Minute)
+		if dateEqual(timestamp, date) {
+			if !filterLogRecord(logTypeFilter, inputSplit[1]) {
+				row := LogRow{
+					Timestamp: timestamp.Format("2006-01-02 15:04:05"),
+					Data:      inputSplit[1:],
+				}
+				logData.Data = append(logData.Data, row)
 			}
-			logData.Data = append(logData.Data, row)
-
 		}
 	}
 
 	if err := scanner.Err(); err != nil {
 		c.returnError(w, fmt.Errorf("log file read (scanner) error: %s", err), http.StatusBadRequest)
 		return
+	} else {
+		if len(logData.Data) < MAX_LOG_OUTPUT_LINES { // todo: and check if it is last file
+			pos = -1 // no more records
+		}
 	}
 
+	// set position
+	logData.NextPos = pos
+
 	// logtypes
 	packetLogTypes := []string{}
 	for k, enabled := range vpnConfig.PacketLogsTypes {

pkg/rest/types.go

@@ -216,13 +216,14 @@ type LogDataResponse struct {
 }
 
 type LogData struct {
-	Schema LogSchema `json:"schema"`
-	Data   []LogRow  `json:"rows"`
+	Schema  LogSchema `json:"schema"`
+	Data    []LogRow  `json:"rows"`
+	NextPos int64     `json:"nextPos"`
 }
 type LogSchema struct {
 	Columns map[string]string `json:"columns"`
 }
 type LogRow struct {
-	Timestamp time.Time `json:"t"`
-	Data      []string  `json:"d"`
+	Timestamp string   `json:"t"`
+	Data      []string `json:"d"`
 }

pkg/storage/iface.go

@@ -1,5 +1,7 @@
 package storage
 
+import "io"
+
 type Iface interface {
 	GetPath() string
 	EnsurePath(path string) error
@@ -8,6 +10,7 @@ type Iface interface {
 	Remove(name string) error
 	AppendFile(name string, data []byte) error
 	ReadWriter
+	Seeker
 }
 
 type ReadWriter interface {
@@ -16,3 +19,7 @@ type ReadWriter interface {
 	FileExists(filename string) bool
 	ConfigPath(filename string) string
 }
+
+type Seeker interface {
+	OpenFilesFromPos(names []string, pos int64) ([]io.Reader, error)
+}

pkg/storage/local/read.go

@@ -1,10 +1,41 @@
 package localstorage
 
 import (
+	"fmt"
+	"io"
 	"os"
 	"path"
 )
 
 func (l *LocalStorage) ReadFile(name string) ([]byte, error) {
 	return os.ReadFile(path.Join(l.path, name))
 }
+
+func (l *LocalStorage) OpenFilesFromPos(names []string, pos int64) ([]io.Reader, error) {
+	readers := []io.Reader{}
+	for _, name := range names {
+		file, err := os.Open(path.Join(l.path, name))
+		if err != nil {
+			return nil, fmt.Errorf("cannot open file (%s): %s", name, err)
+		}
+		defer file.Close()
+		stat, err := file.Stat()
+		if err != nil {
+			return nil, fmt.Errorf("cannot get file stat (%s): %s", name, err)
+		}
+		if stat.Size() <= pos {
+			pos -= stat.Size()
+		} else {
+			_, err := file.Seek(pos, 0)
+			if err != nil {
+				return nil, fmt.Errorf("could not seek to pos (file: %s): %s", name, err)
+			}
+			pos = 0
+			readers = append(readers, file)
+		}
+	}
+	if len(readers) == 0 {
+		return nil, fmt.Errorf("no file contents to read")
+	}
+	return readers, nil
+}

pkg/testing/mocks/storage.go

@@ -2,6 +2,7 @@ package testingmocks
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"path"
 	"strings"
@@ -113,3 +114,7 @@ func (m *MockMemoryStorage) Remove(name string) error {
 	delete(m.Data, name)
 	return nil
 }
+
+func (m *MockMemoryStorage) OpenFilesFromPos(names []string, pos int64) ([]io.Reader, error) {
+	return nil, fmt.Errorf("not implemented")
+}

webapp/src/Routes/PacketLogs/PacketLogs.tsx

@@ -1,6 +1,6 @@
 import { Card, Container, Text, Table, Title, Button, Grid, Select, MultiSelect, Popover} from "@mantine/core";
 import { AppSettings } from "../../Constants/Constants";
-import {  useQuery } from "@tanstack/react-query";
+import {  keepPreviousData, useQuery } from "@tanstack/react-query";
 import { useAuthContext } from "../../Auth/Auth";
 import { Link, useParams, useSearchParams } from "react-router-dom";
 import { TbSettings } from "react-icons/tb";
@@ -28,18 +28,27 @@ type UserMap = {
   [key: string]: string;
 }
 
+function getDate(date:Date) {
+  var dd = String(date.getDate()).padStart(2, '0');
+  var mm = String(date.getMonth() + 1).padStart(2, '0'); //January is 0!
+  var yyyy = date.getFullYear();  
+  return yyyy + "-" + mm + '-' + dd;
+}
+
 export function PacketLogs() {
     const {authInfo} = useAuthContext();
-    const [currentQueryParameters, setSearchParams] = useSearchParams();
+    const timezoneOffset = new Date().getTimezoneOffset() * -1
+    const [currentQueryParameters] = useSearchParams();
     const dateParam = currentQueryParameters.get("date")
     const userParam = currentQueryParameters.get("user")
     const [logType, setLogType] = useState<string[]>([])
     const [logsDate, setLogsDate] = useState<Date | null>(dateParam === null ? new Date() : new Date(dateParam));
     const [user, setUser] = useState<string>(userParam === null ? "all" : userParam)
+    const [page, setPage] = useState(1)
     const { isPending, error, data } = useQuery<LogsDataResponse>({
-      queryKey: ['packetlogs', user, logsDate, logType],
+      queryKey: ['packetlogs', user, logsDate, logType, page],
       queryFn: () =>
-        fetch(AppSettings.url + '/stats/packetlogs/'+(user === undefined ? "all" : user)+'/'+(logsDate == undefined ? new Date().toISOString().slice(0, 10) : logsDate.toISOString().slice(0, 10)) + "?logtype="+encodeURIComponent(logType.join(",")), {
+        fetch(AppSettings.url + '/stats/packetlogs/'+(user === undefined || user === "" ? "all" : user)+'/'+(logsDate == undefined ? getDate(new Date()) : getDate(logsDate)) + "?offset="+timezoneOffset+"&logtype="+encodeURIComponent(logType.join(",")), {
           headers: {
             "Content-Type": "application/json",
             "Authorization": "Bearer " + authInfo.token
@@ -48,6 +57,7 @@ export function PacketLogs() {
           return res.json()
           }
         ),
+        placeholderData: page == 1 ? undefined : keepPreviousData,
     })
 
     if(isPending) return "Loading..."
@@ -66,7 +76,6 @@ export function PacketLogs() {
               : 
                 data.logTypes.length == 0 ? "Packet logs are activated, but no packet logging types are selected. Select at least one packet log type." : null
               }
-
             </Text>
             <Card.Section inheritPadding mt="sm" pb="md">
               <Link to="/setup/vpn">
@@ -152,7 +161,13 @@ export function PacketLogs() {
                   <Table.Th>Destination</Table.Th>
                   </Table.Tr>
               </Table.Thead>
-              <Table.Tbody>{rows}</Table.Tbody>
+              <Table.Tbody>
+                {user === undefined || user === "" || user === "all" ? 
+                  <Table.Tr key="nouser"><Table.Td colSpan={7}>Select a user to see log data.</Table.Td></Table.Tr>
+                :
+                  rows
+                }
+              </Table.Tbody>
           </Table>
         </Container>