Merge pull request #4 from skycoin/fix/remve-old-logic-add-new-endpoi…

…nt-files

UT new logic

README.md

@@ -11,6 +11,29 @@ Run `make build` to build the service and `make install` to install into go bina
 
 Refer to the [`cmd`](cmd) subdirectories for setting up the individual service locally.
 
+### DB Setup
+`uptime-tracker` needs database for running that we use postgresql here as default database. For setting it up, you just need run pg (by docker or install binary or etc.), make a database with UTF-8 character-set, and pass two credential as flag and save three of them as env variable before running services.
+
+First of all, you needs run postgres and redis. You can run it by docker:
+```
+docker run --name skywire-ut-pg -e POSTGRES_PASSWORD=secret -e POSTGRES_USER=skywire-ut -p 5432:5432 -d postgres
+docker run --name my-redis -p 6379:6379 -d redis
+```
+
+then, if you want run `uptime-tracker` service, you should pass `--pg-host` and `--pg-port` as flag on running its binary, and also save `PG_USER`, `PG_PASSWORD` and `PG_DATABASE` as env variable.
+```
+export PG_USER=skywire-ut
+export PG_PASSWORD=secret
+export PG_DATABASE=skywire-ut
+```
+and run it by
+
+```
+./bin/uptime-tracker --pg-host localhost --pg-port 5432 --store-data-path skywire-ut/daily-data
+```
+
+All tables created automatically and no need handle manually.
+
 ## Deployments
 
 TPD

cmd/uptime-tracker/commands/root.go

@@ -9,10 +9,17 @@ import (
 	"os"
 	"strings"
 
+	"github.com/SkycoinPro/skywire-services/internal/pg"
+	"github.com/SkycoinPro/skywire-services/internal/utmetrics"
+	"github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/api"
+	"github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/store"
 	logrussyslog "github.com/sirupsen/logrus/hooks/syslog"
 	"github.com/skycoin/dmsg/pkg/direct"
 	"github.com/skycoin/dmsg/pkg/dmsg"
 	"github.com/skycoin/dmsg/pkg/dmsghttp"
+	"github.com/spf13/cobra"
+	"gorm.io/gorm"
+
 	"github.com/skycoin/skywire-utilities/pkg/buildinfo"
 	"github.com/skycoin/skywire-utilities/pkg/cipher"
 	"github.com/skycoin/skywire-utilities/pkg/cmdutil"
@@ -22,13 +29,6 @@ import (
 	"github.com/skycoin/skywire-utilities/pkg/metricsutil"
 	"github.com/skycoin/skywire-utilities/pkg/storeconfig"
 	"github.com/skycoin/skywire-utilities/pkg/tcpproxy"
-	"github.com/spf13/cobra"
-	"gorm.io/gorm"
-
-	"github.com/SkycoinPro/skywire-services/internal/pg"
-	"github.com/SkycoinPro/skywire-services/internal/utmetrics"
-	"github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/api"
-	"github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/store"
 )
 
 const (
@@ -45,6 +45,7 @@ var (
 	redisPoolSize     int
 	pgHost            string
 	pgPort            string
+	pgMaxOpenConn     int
 	logEnabled        bool
 	syslogAddr        string
 	tag               string
@@ -53,6 +54,8 @@ var (
 	testing           bool
 	dmsgDisc          string
 	sk                cipher.SecKey
+	storeDataCutoff   int
+	storeDataPath     string
 )
 
 func init() {
@@ -63,6 +66,9 @@ func init() {
 	rootCmd.Flags().IntVar(&redisPoolSize, "redis-pool-size", 10, "redis connection pool size")
 	rootCmd.Flags().StringVar(&pgHost, "pg-host", "localhost", "host of postgres")
 	rootCmd.Flags().StringVar(&pgPort, "pg-port", "5432", "port of postgres")
+	rootCmd.Flags().IntVar(&pgMaxOpenConn, "pg-max-open-conn", 60, "maximum open connection of db")
+	rootCmd.Flags().IntVar(&storeDataCutoff, "store-data-cutoff", 7, "number of days data store in db")
+	rootCmd.Flags().StringVar(&storeDataPath, "store-data-path", "/var/lib/skywire-ut/daily-data", "path of db daily data store")
 	rootCmd.Flags().BoolVarP(&logEnabled, "log", "l", true, "enable request logging")
 	rootCmd.Flags().StringVar(&syslogAddr, "syslog", "", "syslog server address. E.g. localhost:514")
 	rootCmd.Flags().StringVar(&tag, "tag", "uptime_tracker", "logging tag")
@@ -116,7 +122,7 @@ var rootCmd = &cobra.Command{
 				pgPassword,
 				pgDatabase)
 
-			gormDB, err = pg.Init(dsn)
+			gormDB, err = pg.Init(dsn, pgMaxOpenConn)
 			if err != nil {
 				logger.Fatalf("Failed to connect to database %v", err)
 			}
@@ -151,7 +157,7 @@ var rootCmd = &cobra.Command{
 		}
 
 		enableMetrics := metricsAddr != ""
-		utAPI := api.New(logger, s, nonceStore, locDetails, enableLoadTesting, enableMetrics, m)
+		utAPI := api.New(logger, s, nonceStore, locDetails, enableLoadTesting, enableMetrics, m, storeDataCutoff, storeDataPath)
 
 		utPAPI := api.NewPrivate(logger, s)
 

docker/docker_push.sh

@@ -6,7 +6,7 @@ tag="$1"
 registry="$REGISTRY"
 
 if [ -z "$registry" ]; then
-	registry="skycoin"
+	registry="skycoinpro"
 fi
 
 if [ -z "$tag" ]; then

internal/pg/lib.go

@@ -7,11 +7,12 @@ import (
 )
 
 // Init creates a connection to database
-func Init(dns string) (*gorm.DB, error) {
+func Init(dns string, pgMaxOpenConn int) (*gorm.DB, error) {
 	db, err := gorm.Open(postgres.Open(dns), &gorm.Config{})
 	if err != nil {
 		return db, err
 	}
-
+	dbConf, _ := db.DB() //nolint
+	dbConf.SetMaxOpenConns(pgMaxOpenConn)
 	return db, nil
 }

pkg/uptime-tracker/api/api.go

@@ -6,21 +6,24 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"math"
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/SkycoinPro/skywire-services/internal/utmetrics"
+	"github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/store"
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/go-chi/httprate"
 	"github.com/go-echarts/go-echarts/v2/charts"
 	"github.com/go-echarts/go-echarts/v2/opts"
 	"github.com/sirupsen/logrus"
+
 	"github.com/skycoin/skywire-utilities/pkg/buildinfo"
 	"github.com/skycoin/skywire-utilities/pkg/cipher"
 	"github.com/skycoin/skywire-utilities/pkg/geo"
@@ -29,9 +32,6 @@ import (
 	"github.com/skycoin/skywire-utilities/pkg/logging"
 	"github.com/skycoin/skywire-utilities/pkg/metricsutil"
 	"github.com/skycoin/skywire-utilities/pkg/netutil"
-
-	"github.com/SkycoinPro/skywire-services/internal/utmetrics"
-	"github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/store"
 )
 
 const (
@@ -55,6 +55,8 @@ type API struct {
 	visorsCacheMu          sync.RWMutex
 	dailyUptimeCache       map[string]map[string]string
 	dailyUptimeCacheMu     sync.RWMutex
+	storeUptimesCutoff     int
+	storeUptimesPath       string
 }
 
 // PrivateAPI register all the PrivateAPI endpoints.
@@ -73,7 +75,7 @@ type HealthCheckResponse struct {
 
 // New constructs a new API instance.
 func New(log logrus.FieldLogger, s store.Store, nonceStore httpauth.NonceStore, locDetails geo.LocationDetails,
-	enableLoadTesting, enableMetrics bool, m utmetrics.Metrics) *API {
+	enableLoadTesting, enableMetrics bool, m utmetrics.Metrics, storeDataCutoff int, storeDataPath string) *API {
 	if log == nil {
 		log = logging.MustGetLogger("uptime_tracker")
 	}
@@ -84,6 +86,8 @@ func New(log logrus.FieldLogger, s store.Store, nonceStore httpauth.NonceStore,
 		store:                       s,
 		locDetails:                  locDetails,
 		startedAt:                   time.Now(),
+		storeUptimesCutoff:          storeDataCutoff,
+		storeUptimesPath:            storeDataPath,
 	}
 
 	r := chi.NewRouter()
@@ -141,6 +145,7 @@ func (api *API) log(r *http.Request) logrus.FieldLogger {
 
 // RunBackgroundTasks is function which runs periodic background tasks of API.
 func (api *API) RunBackgroundTasks(ctx context.Context, logger logrus.FieldLogger) {
+	api.dailyRoutine(logger)
 	cacheTicker := time.NewTicker(time.Minute * 5)
 	defer cacheTicker.Stop()
 	ticker := time.NewTicker(time.Second * 10)
@@ -153,6 +158,7 @@ func (api *API) RunBackgroundTasks(ctx context.Context, logger logrus.FieldLogge
 			return
 		case <-cacheTicker.C:
 			api.updateInternalCaches(logger)
+			api.dailyRoutine(logger)
 		case <-ticker.C:
 			api.updateInternalState(ctx, logger)
 		}
@@ -174,6 +180,45 @@ func (api *API) updateInternalCaches(logger logrus.FieldLogger) {
 	}
 }
 
+func (api *API) dailyRoutine(logger logrus.FieldLogger) {
+	oldestEntry, err := api.store.GetOldestEntry()
+	if err != nil {
+		logger.WithError(err).Warn("unable to fetch oldest entry from db")
+		return
+	}
+
+	from := oldestEntry.CreatedAt
+	to := time.Now().AddDate(0, 0, -(api.storeUptimesCutoff))
+
+	for to.After(from) {
+		timeValue := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.Now().Location())
+		data, err := api.store.GetSpecificDayData(timeValue)
+		if err != nil {
+			logger.WithField("date", timeValue.Format("2006-01-02")).WithError(err).Warn("unable to fetch data specific date from db")
+			return
+		}
+		err = api.storeDailyData(data, timeValue)
+		if err != nil {
+			logger.WithError(err).Warn("unable to save data to json file")
+			return
+		}
+		err = api.store.DeleteEntries(data)
+		if err != nil {
+			logger.WithError(err).Warn("unable to delete old entries from db")
+		}
+		from = from.AddDate(0, 0, 1)
+	}
+}
+
+func (api *API) storeDailyData(data []store.DailyUptimeHistory, timeValue time.Time) error {
+	// check path, make its if not available
+	os.MkdirAll(api.storeUptimesPath, os.ModePerm) //nolint
+	// save to file
+	file, _ := json.MarshalIndent(data, "", " ") //nolint
+	fileName := fmt.Sprintf("%s/%s-uptime-data.json", api.storeUptimesPath, timeValue.Format("2006-01-02"))
+	return os.WriteFile(fileName, file, 0644) //nolint
+}
+
 func (api *API) updateVisorsCache() error {
 	visors, err := api.store.GetAllVisors(api.locDetails)
 	if err != nil {
@@ -409,11 +454,8 @@ func (api *API) handleUptimes(w http.ResponseWriter, r *http.Request) {
 		var uptimesV2 store.UptimeResponseV2
 		for _, uptime := range uptimes {
 			var uptimev2 store.UptimeDefV2
-			uptimev2.Downtime = uptime.Downtime
 			uptimev2.Key = uptime.Key
-			uptimev2.Uptime = uptime.Uptime
 			uptimev2.Online = uptime.Online
-			uptimev2.Percentage = math.Round(uptime.Percentage*100) / 100
 			uptimev2.DailyOnlineHistory = dailyUptimeHistory[uptime.Key]
 			uptimev2.Version = uptime.Version
 			uptimesV2 = append(uptimesV2, uptimev2)

pkg/uptime-tracker/api/api_test.go

@@ -9,17 +9,16 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"testing"
-	"time"
+
+	"github.com/SkycoinPro/skywire-services/internal/utmetrics"
+	"github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/store"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/skycoin/skywire-utilities/pkg/cipher"
 	"github.com/skycoin/skywire-utilities/pkg/geo"
 	"github.com/skycoin/skywire-utilities/pkg/httpauth"
 	"github.com/skycoin/skywire-utilities/pkg/storeconfig"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/SkycoinPro/skywire-services/internal/utmetrics"
-	"github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/store"
 )
 
 var testPubKey, testSec = cipher.GenerateKeyPair()
@@ -42,7 +41,7 @@ func TestHandleUptimes(t *testing.T) {
 	nonceMock, err := httpauth.NewNonceStore(ctx, storeconfig.Config{Type: storeconfig.Memory}, "")
 	require.NoError(t, err)
 	api := New(nil, mock, nonceMock, geoFunc, false, false,
-		utmetrics.NewEmpty())
+		utmetrics.NewEmpty(), 0, "")
 
 	pk, _ := cipher.GenerateKeyPair()
 
@@ -63,19 +62,9 @@ func TestHandleUptimes(t *testing.T) {
 	var resp store.UptimeResponse
 	require.NoError(t, json.NewDecoder(bytes.NewBuffer(w.Body.Bytes())).Decode(&resp))
 
-	now := time.Now()
-	monthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location())
-	monthEnd := monthStart.AddDate(0, 1, 0)
-
-	totalMonthSeconds := float64(int(monthEnd.Sub(monthStart).Seconds()))
-
 	require.Len(t, resp, 1)
 	assert.Equal(t, pk.String(), resp[0].Key)
 
-	iterationsCount := store.UptimeSeconds * float64(iterations)
-	assert.Equal(t, iterationsCount, resp[0].Uptime)
-	assert.Equal(t, totalMonthSeconds-iterationsCount, resp[0].Downtime)
-	assert.Equal(t, iterationsCount/totalMonthSeconds*100, resp[0].Percentage)
 	assert.True(t, resp[0].Online)
 }
 
@@ -86,7 +75,7 @@ func TestAPI_handleUpdate(t *testing.T) {
 	nonceMock, err := httpauth.NewNonceStore(ctx, storeconfig.Config{Type: storeconfig.Memory}, "")
 	require.NoError(t, err)
 	api := New(nil, mock, nonceMock, geoFunc, false, false,
-		utmetrics.NewEmpty())
+		utmetrics.NewEmpty(), 0, "")
 
 	t.Run("StatusOK", func(t *testing.T) {
 		w := httptest.NewRecorder()
@@ -117,7 +106,7 @@ func TestApi_UpdateRemovedMethod(t *testing.T) {
 	nonceMock, err := httpauth.NewNonceStore(ctx, storeconfig.Config{Type: storeconfig.Memory}, "")
 	require.NoError(t, err)
 	api := New(nil, mock, nonceMock, geoFunc, false, false,
-		utmetrics.NewEmpty())
+		utmetrics.NewEmpty(), 0, "")
 
 	t.Run("StatusGone", func(t *testing.T) {
 		w := httptest.NewRecorder()