diff --git a/README.md b/README.md index fe27ca6..ba2637e 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,29 @@ Run `make build` to build the service and `make install` to install into go bina Refer to the [`cmd`](cmd) subdirectories for setting up the individual service locally. +### DB Setup +`uptime-tracker` needs database for running that we use postgresql here as default database. For setting it up, you just need run pg (by docker or install binary or etc.), make a database with UTF-8 character-set, and pass two credential as flag and save three of them as env variable before running services. + +First of all, you needs run postgres and redis. You can run it by docker: +``` +docker run --name skywire-ut-pg -e POSTGRES_PASSWORD=secret -e POSTGRES_USER=skywire-ut -p 5432:5432 -d postgres +docker run --name my-redis -p 6379:6379 -d redis +``` + +then, if you want run `uptime-tracker` service, you should pass `--pg-host` and `--pg-port` as flag on running its binary, and also save `PG_USER`, `PG_PASSWORD` and `PG_DATABASE` as env variable. +``` +export PG_USER=skywire-ut +export PG_PASSWORD=secret +export PG_DATABASE=skywire-ut +``` +and run it by + +``` +./bin/uptime-tracker --pg-host localhost --pg-port 5432 --store-data-path skywire-ut/daily-data +``` + +All tables created automatically and no need handle manually. + ## Deployments TPD diff --git a/cmd/uptime-tracker/commands/root.go b/cmd/uptime-tracker/commands/root.go index 0d40b6a..2cffc26 100644 --- a/cmd/uptime-tracker/commands/root.go +++ b/cmd/uptime-tracker/commands/root.go @@ -9,10 +9,17 @@ import ( "os" "strings" + "github.com/SkycoinPro/skywire-services/internal/pg" + "github.com/SkycoinPro/skywire-services/internal/utmetrics" + "github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/api" + "github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/store" logrussyslog "github.com/sirupsen/logrus/hooks/syslog" "github.com/skycoin/dmsg/pkg/direct" "github.com/skycoin/dmsg/pkg/dmsg" "github.com/skycoin/dmsg/pkg/dmsghttp" + "github.com/spf13/cobra" + "gorm.io/gorm" + "github.com/skycoin/skywire-utilities/pkg/buildinfo" "github.com/skycoin/skywire-utilities/pkg/cipher" "github.com/skycoin/skywire-utilities/pkg/cmdutil" @@ -22,13 +29,6 @@ import ( "github.com/skycoin/skywire-utilities/pkg/metricsutil" "github.com/skycoin/skywire-utilities/pkg/storeconfig" "github.com/skycoin/skywire-utilities/pkg/tcpproxy" - "github.com/spf13/cobra" - "gorm.io/gorm" - - "github.com/SkycoinPro/skywire-services/internal/pg" - "github.com/SkycoinPro/skywire-services/internal/utmetrics" - "github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/api" - "github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/store" ) const ( @@ -45,6 +45,7 @@ var ( redisPoolSize int pgHost string pgPort string + pgMaxOpenConn int logEnabled bool syslogAddr string tag string @@ -53,6 +54,8 @@ var ( testing bool dmsgDisc string sk cipher.SecKey + storeDataCutoff int + storeDataPath string ) func init() { @@ -63,6 +66,9 @@ func init() { rootCmd.Flags().IntVar(&redisPoolSize, "redis-pool-size", 10, "redis connection pool size") rootCmd.Flags().StringVar(&pgHost, "pg-host", "localhost", "host of postgres") rootCmd.Flags().StringVar(&pgPort, "pg-port", "5432", "port of postgres") + rootCmd.Flags().IntVar(&pgMaxOpenConn, "pg-max-open-conn", 60, "maximum open connection of db") + rootCmd.Flags().IntVar(&storeDataCutoff, "store-data-cutoff", 7, "number of days data store in db") + rootCmd.Flags().StringVar(&storeDataPath, "store-data-path", "/var/lib/skywire-ut/daily-data", "path of db daily data store") rootCmd.Flags().BoolVarP(&logEnabled, "log", "l", true, "enable request logging") rootCmd.Flags().StringVar(&syslogAddr, "syslog", "", "syslog server address. E.g. localhost:514") rootCmd.Flags().StringVar(&tag, "tag", "uptime_tracker", "logging tag") @@ -116,7 +122,7 @@ var rootCmd = &cobra.Command{ pgPassword, pgDatabase) - gormDB, err = pg.Init(dsn) + gormDB, err = pg.Init(dsn, pgMaxOpenConn) if err != nil { logger.Fatalf("Failed to connect to database %v", err) } @@ -151,7 +157,7 @@ var rootCmd = &cobra.Command{ } enableMetrics := metricsAddr != "" - utAPI := api.New(logger, s, nonceStore, locDetails, enableLoadTesting, enableMetrics, m) + utAPI := api.New(logger, s, nonceStore, locDetails, enableLoadTesting, enableMetrics, m, storeDataCutoff, storeDataPath) utPAPI := api.NewPrivate(logger, s) diff --git a/docker/docker_push.sh b/docker/docker_push.sh index 548329a..1adc0e1 100755 --- a/docker/docker_push.sh +++ b/docker/docker_push.sh @@ -6,7 +6,7 @@ tag="$1" registry="$REGISTRY" if [ -z "$registry" ]; then - registry="skycoin" + registry="skycoinpro" fi if [ -z "$tag" ]; then diff --git a/internal/pg/lib.go b/internal/pg/lib.go index c3b037f..3cf5e96 100644 --- a/internal/pg/lib.go +++ b/internal/pg/lib.go @@ -7,11 +7,12 @@ import ( ) // Init creates a connection to database -func Init(dns string) (*gorm.DB, error) { +func Init(dns string, pgMaxOpenConn int) (*gorm.DB, error) { db, err := gorm.Open(postgres.Open(dns), &gorm.Config{}) if err != nil { return db, err } - + dbConf, _ := db.DB() //nolint + dbConf.SetMaxOpenConns(pgMaxOpenConn) return db, nil } diff --git a/pkg/uptime-tracker/api/api.go b/pkg/uptime-tracker/api/api.go index cb1dcca..3ef780f 100644 --- a/pkg/uptime-tracker/api/api.go +++ b/pkg/uptime-tracker/api/api.go @@ -6,21 +6,24 @@ import ( "encoding/json" "errors" "fmt" - "math" "net" "net/http" "net/url" + "os" "strconv" "strings" "sync" "time" + "github.com/SkycoinPro/skywire-services/internal/utmetrics" + "github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/store" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/httprate" "github.com/go-echarts/go-echarts/v2/charts" "github.com/go-echarts/go-echarts/v2/opts" "github.com/sirupsen/logrus" + "github.com/skycoin/skywire-utilities/pkg/buildinfo" "github.com/skycoin/skywire-utilities/pkg/cipher" "github.com/skycoin/skywire-utilities/pkg/geo" @@ -29,9 +32,6 @@ import ( "github.com/skycoin/skywire-utilities/pkg/logging" "github.com/skycoin/skywire-utilities/pkg/metricsutil" "github.com/skycoin/skywire-utilities/pkg/netutil" - - "github.com/SkycoinPro/skywire-services/internal/utmetrics" - "github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/store" ) const ( @@ -55,6 +55,8 @@ type API struct { visorsCacheMu sync.RWMutex dailyUptimeCache map[string]map[string]string dailyUptimeCacheMu sync.RWMutex + storeUptimesCutoff int + storeUptimesPath string } // PrivateAPI register all the PrivateAPI endpoints. @@ -73,7 +75,7 @@ type HealthCheckResponse struct { // New constructs a new API instance. func New(log logrus.FieldLogger, s store.Store, nonceStore httpauth.NonceStore, locDetails geo.LocationDetails, - enableLoadTesting, enableMetrics bool, m utmetrics.Metrics) *API { + enableLoadTesting, enableMetrics bool, m utmetrics.Metrics, storeDataCutoff int, storeDataPath string) *API { if log == nil { log = logging.MustGetLogger("uptime_tracker") } @@ -84,6 +86,8 @@ func New(log logrus.FieldLogger, s store.Store, nonceStore httpauth.NonceStore, store: s, locDetails: locDetails, startedAt: time.Now(), + storeUptimesCutoff: storeDataCutoff, + storeUptimesPath: storeDataPath, } r := chi.NewRouter() @@ -141,6 +145,7 @@ func (api *API) log(r *http.Request) logrus.FieldLogger { // RunBackgroundTasks is function which runs periodic background tasks of API. func (api *API) RunBackgroundTasks(ctx context.Context, logger logrus.FieldLogger) { + api.dailyRoutine(logger) cacheTicker := time.NewTicker(time.Minute * 5) defer cacheTicker.Stop() ticker := time.NewTicker(time.Second * 10) @@ -153,6 +158,7 @@ func (api *API) RunBackgroundTasks(ctx context.Context, logger logrus.FieldLogge return case <-cacheTicker.C: api.updateInternalCaches(logger) + api.dailyRoutine(logger) case <-ticker.C: api.updateInternalState(ctx, logger) } @@ -174,6 +180,45 @@ func (api *API) updateInternalCaches(logger logrus.FieldLogger) { } } +func (api *API) dailyRoutine(logger logrus.FieldLogger) { + oldestEntry, err := api.store.GetOldestEntry() + if err != nil { + logger.WithError(err).Warn("unable to fetch oldest entry from db") + return + } + + from := oldestEntry.CreatedAt + to := time.Now().AddDate(0, 0, -(api.storeUptimesCutoff)) + + for to.After(from) { + timeValue := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.Now().Location()) + data, err := api.store.GetSpecificDayData(timeValue) + if err != nil { + logger.WithField("date", timeValue.Format("2006-01-02")).WithError(err).Warn("unable to fetch data specific date from db") + return + } + err = api.storeDailyData(data, timeValue) + if err != nil { + logger.WithError(err).Warn("unable to save data to json file") + return + } + err = api.store.DeleteEntries(data) + if err != nil { + logger.WithError(err).Warn("unable to delete old entries from db") + } + from = from.AddDate(0, 0, 1) + } +} + +func (api *API) storeDailyData(data []store.DailyUptimeHistory, timeValue time.Time) error { + // check path, make its if not available + os.MkdirAll(api.storeUptimesPath, os.ModePerm) //nolint + // save to file + file, _ := json.MarshalIndent(data, "", " ") //nolint + fileName := fmt.Sprintf("%s/%s-uptime-data.json", api.storeUptimesPath, timeValue.Format("2006-01-02")) + return os.WriteFile(fileName, file, 0644) //nolint +} + func (api *API) updateVisorsCache() error { visors, err := api.store.GetAllVisors(api.locDetails) if err != nil { @@ -409,11 +454,8 @@ func (api *API) handleUptimes(w http.ResponseWriter, r *http.Request) { var uptimesV2 store.UptimeResponseV2 for _, uptime := range uptimes { var uptimev2 store.UptimeDefV2 - uptimev2.Downtime = uptime.Downtime uptimev2.Key = uptime.Key - uptimev2.Uptime = uptime.Uptime uptimev2.Online = uptime.Online - uptimev2.Percentage = math.Round(uptime.Percentage*100) / 100 uptimev2.DailyOnlineHistory = dailyUptimeHistory[uptime.Key] uptimev2.Version = uptime.Version uptimesV2 = append(uptimesV2, uptimev2) diff --git a/pkg/uptime-tracker/api/api_test.go b/pkg/uptime-tracker/api/api_test.go index 0a0ea4f..f5ebca4 100644 --- a/pkg/uptime-tracker/api/api_test.go +++ b/pkg/uptime-tracker/api/api_test.go @@ -9,17 +9,16 @@ import ( "net/http" "net/http/httptest" "testing" - "time" + + "github.com/SkycoinPro/skywire-services/internal/utmetrics" + "github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/store" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/skycoin/skywire-utilities/pkg/cipher" "github.com/skycoin/skywire-utilities/pkg/geo" "github.com/skycoin/skywire-utilities/pkg/httpauth" "github.com/skycoin/skywire-utilities/pkg/storeconfig" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/SkycoinPro/skywire-services/internal/utmetrics" - "github.com/SkycoinPro/skywire-services/pkg/uptime-tracker/store" ) var testPubKey, testSec = cipher.GenerateKeyPair() @@ -42,7 +41,7 @@ func TestHandleUptimes(t *testing.T) { nonceMock, err := httpauth.NewNonceStore(ctx, storeconfig.Config{Type: storeconfig.Memory}, "") require.NoError(t, err) api := New(nil, mock, nonceMock, geoFunc, false, false, - utmetrics.NewEmpty()) + utmetrics.NewEmpty(), 0, "") pk, _ := cipher.GenerateKeyPair() @@ -63,19 +62,9 @@ func TestHandleUptimes(t *testing.T) { var resp store.UptimeResponse require.NoError(t, json.NewDecoder(bytes.NewBuffer(w.Body.Bytes())).Decode(&resp)) - now := time.Now() - monthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) - monthEnd := monthStart.AddDate(0, 1, 0) - - totalMonthSeconds := float64(int(monthEnd.Sub(monthStart).Seconds())) - require.Len(t, resp, 1) assert.Equal(t, pk.String(), resp[0].Key) - iterationsCount := store.UptimeSeconds * float64(iterations) - assert.Equal(t, iterationsCount, resp[0].Uptime) - assert.Equal(t, totalMonthSeconds-iterationsCount, resp[0].Downtime) - assert.Equal(t, iterationsCount/totalMonthSeconds*100, resp[0].Percentage) assert.True(t, resp[0].Online) } @@ -86,7 +75,7 @@ func TestAPI_handleUpdate(t *testing.T) { nonceMock, err := httpauth.NewNonceStore(ctx, storeconfig.Config{Type: storeconfig.Memory}, "") require.NoError(t, err) api := New(nil, mock, nonceMock, geoFunc, false, false, - utmetrics.NewEmpty()) + utmetrics.NewEmpty(), 0, "") t.Run("StatusOK", func(t *testing.T) { w := httptest.NewRecorder() @@ -117,7 +106,7 @@ func TestApi_UpdateRemovedMethod(t *testing.T) { nonceMock, err := httpauth.NewNonceStore(ctx, storeconfig.Config{Type: storeconfig.Memory}, "") require.NoError(t, err) api := New(nil, mock, nonceMock, geoFunc, false, false, - utmetrics.NewEmpty()) + utmetrics.NewEmpty(), 0, "") t.Run("StatusGone", func(t *testing.T) { w := httptest.NewRecorder() diff --git a/pkg/uptime-tracker/store/memory_store.go b/pkg/uptime-tracker/store/memory_store.go index 34382f3..28a86a7 100644 --- a/pkg/uptime-tracker/store/memory_store.go +++ b/pkg/uptime-tracker/store/memory_store.go @@ -78,7 +78,7 @@ func (s *memStore) GetAllUptimes(startYear int, startMonth time.Month, endYear i startDate := time.Date(startYear, startMonth, 1, 0, 0, 0, 0, time.Now().Location()) endDate := time.Date(endYear, endMonth, 1, 0, 0, 0, 0, time.Now().Location()) - var uptimes []map[string]string + var keys []string for ; startDate.Before(endDate) || startDate.Equal(endDate); startDate = startDate.AddDate(0, 1, 0) { if _, ok := s.visors[startDate.Year()]; !ok { continue @@ -90,10 +90,12 @@ func (s *memStore) GetAllUptimes(startYear int, startMonth time.Month, endYear i continue } - uptimes = append(uptimes, monthUptimes) + for pk := range monthUptimes { + keys = append(keys, pk) + } } - return makeUptimeResponse(uptimes, s.lastTS, map[string]string{}, startYear, startMonth, endYear, endMonth, nil) + return makeUptimeResponse(keys, s.lastTS, map[string]string{}, nil) } func (s *memStore) GetUptimes(pubKeys []string, startYear int, startMonth time.Month, endYear int, endMonth time.Month) (UptimeResponse, error) { @@ -103,7 +105,7 @@ func (s *memStore) GetUptimes(pubKeys []string, startYear int, startMonth time.M startDate := time.Date(startYear, startMonth, 1, 0, 0, 0, 0, time.Now().Location()) endDate := time.Date(endYear, endMonth, 1, 0, 0, 0, 0, time.Now().Location()) - var uptimes []map[string]string + var keys []string for ; startDate.Before(endDate) || startDate.Equal(endDate); startDate = startDate.AddDate(0, 1, 0) { if _, ok := s.visors[startDate.Year()]; !ok { continue @@ -114,17 +116,15 @@ func (s *memStore) GetUptimes(pubKeys []string, startYear int, startMonth time.M continue } - monthUptimes := make(map[string]string, len(pubKeys)) for _, pk := range pubKeys { - uptime, ok := s.visors[startDate.Year()][startDate.Month()][pk] + _, ok := s.visors[startDate.Year()][startDate.Month()][pk] if !ok { continue } - monthUptimes[pk] = uptime + keys = append(keys, pk) } - uptimes = append(uptimes, monthUptimes) } lastTSMap := make(map[string]string) @@ -138,7 +138,7 @@ func (s *memStore) GetUptimes(pubKeys []string, startYear int, startMonth time.M lastTSMap[pk] = ts } - return makeUptimeResponse(uptimes, lastTSMap, map[string]string{}, startYear, startMonth, endYear, endMonth, nil) + return makeUptimeResponse(keys, lastTSMap, map[string]string{}, nil) } func (s *memStore) GetAllVisors(locDetails geo.LocationDetails) (VisorsResponse, error) { @@ -151,19 +151,11 @@ func (s *memStore) GetAllVisors(locDetails geo.LocationDetails) (VisorsResponse, startYear, startMonth := now.Year(), now.Month() startDate := time.Date(startYear, startMonth, 1, 0, 0, 0, 0, time.Now().Location()) - currentMonthKeys, ok := s.visors[startDate.Year()][startDate.Month()] - if !ok { - return VisorsResponse{}, fmt.Errorf("error getting current Month Keys") - } for pk, ip := range s.ips[fmt.Sprintf("%d:%d", startDate.Year(), startDate.Month())] { ips[pk] = ip - - if ts, ok := s.lastTS[pk]; ok { - currentMonthKeys[pk] = ts - } } - return makeVisorsResponse(currentMonthKeys, ips, locDetails) + return makeVisorsResponse(ips, locDetails) } func (s *memStore) GetVisorsIPs(month string) (map[string]visorIPsResponse, error) { @@ -248,3 +240,15 @@ func (s *memStore) Close() { func (s *memStore) GetDailyUpdateHistory() (map[string]map[string]string, error) { return map[string]map[string]string{}, nil } + +func (s *memStore) DeleteEntries([]DailyUptimeHistory) error { + return nil +} + +func (s *memStore) GetOldestEntry() (DailyUptimeHistory, error) { + return DailyUptimeHistory{}, nil +} + +func (s *memStore) GetSpecificDayData(time time.Time) ([]DailyUptimeHistory, error) { + return []DailyUptimeHistory{}, nil +} diff --git a/pkg/uptime-tracker/store/memory_store_test.go b/pkg/uptime-tracker/store/memory_store_test.go index b1244cb..7b384c0 100644 --- a/pkg/uptime-tracker/store/memory_store_test.go +++ b/pkg/uptime-tracker/store/memory_store_test.go @@ -9,9 +9,10 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/skycoin/skywire-utilities/pkg/cipher" "github.com/skycoin/skywire-utilities/pkg/geo" - "github.com/stretchr/testify/require" ) func TestMemory(t *testing.T) { @@ -29,17 +30,10 @@ func testUptime(t *testing.T, store Store) { } now := time.Now() - monthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) - monthEnd := monthStart.AddDate(0, 1, 0) - - totalMonthSeconds := float64(int(monthEnd.Sub(monthStart).Seconds())) wantUptime := UptimeDef{ - Key: pk.String(), - Uptime: iterations, - Downtime: totalMonthSeconds - iterations, - Percentage: iterations / totalMonthSeconds * 100, - Online: true, + Key: pk.String(), + Online: true, } wantVisor := VisorDef{ diff --git a/pkg/uptime-tracker/store/postgres_store.go b/pkg/uptime-tracker/store/postgres_store.go index c888443..1408f8c 100644 --- a/pkg/uptime-tracker/store/postgres_store.go +++ b/pkg/uptime-tracker/store/postgres_store.go @@ -8,9 +8,10 @@ import ( "sync" "time" + "gorm.io/gorm" + "github.com/skycoin/skywire-utilities/pkg/geo" "github.com/skycoin/skywire-utilities/pkg/logging" - "gorm.io/gorm" ) type postgresStore struct { @@ -24,9 +25,6 @@ type postgresStore struct { // NewPostgresStore creates new uptimes postgres store. func NewPostgresStore(logger *logging.Logger, cl *gorm.DB) (Store, error) { // automigrate - if err := cl.AutoMigrate(Uptime{}); err != nil { - logger.Warn("failed to complete automigrate process") - } if err := cl.AutoMigrate(DailyUptimeHistory{}); err != nil { logger.Warn("failed to complete automigrate process") } @@ -53,16 +51,6 @@ func (s *postgresStore) UpdateUptime(pk, ip, version string) error { return nil } - // get existing data - var uptimeRecord Uptime - startDate := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) - err := s.client. - Where("created_at >= ? AND pub_key = ?", startDate, pk). - First(&uptimeRecord).Error - if err != nil && err != gorm.ErrRecordNotFound { - return err - } - // get existing data of daily record var dailyUptimeRecord DailyUptimeHistory startDailyDate := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) @@ -73,31 +61,23 @@ func (s *postgresStore) UpdateUptime(pk, ip, version string) error { return dailyErr } - if uptimeRecord.PubKey == "" { - uptimeRecord.PubKey = pk - } if dailyUptimeRecord.PubKey == "" { dailyUptimeRecord.PubKey = pk } - uptimeRecord.Online += seconds + dailyUptimeRecord.DailyOnline += seconds if ip != "" { ips := []string{} - if len(uptimeRecord.IPs) > 0 { - ips = strings.Split(uptimeRecord.IPs, ",") + if len(dailyUptimeRecord.IPs) > 0 { + ips = strings.Split(dailyUptimeRecord.IPs, ",") } ips = append(ips, ip) - uptimeRecord.IPs = uniqueIPs(ips) - uptimeRecord.LastIP = ip - } - uptimeRecord.Version = version - if err := s.client.Save(&uptimeRecord).Error; err != nil { - return fmt.Errorf("failed to create/update uptime record: %w", err) + dailyUptimeRecord.IPs = uniqueIPs(ips) + dailyUptimeRecord.LastIP = ip } - - dailyUptimeRecord.DailyOnline += seconds + dailyUptimeRecord.Version = version if err := s.client.Save(&dailyUptimeRecord).Error; err != nil { - return fmt.Errorf("failed to create/update daily uptime record: %w", err) + return fmt.Errorf("failed to create/update uptime record: %w", err) } // update cache @@ -112,81 +92,81 @@ func (s *postgresStore) GetAllUptimes(startYear int, startMonth time.Month, endY startDate := time.Date(startYear, startMonth, 1, 0, 0, 0, 0, time.Now().Location()) endDate := time.Date(endYear, endMonth, 1, 0, 0, 0, 0, time.Now().Location()) - var uptimes []map[string]string + var keys []string lastTSs := make(map[string]string) versions := make(map[string]string) var murError error - var uptimesRecords []Uptime + var uptimesRecords []DailyUptimeHistory for ; startDate.Before(endDate) || startDate.Equal(endDate); startDate = startDate.AddDate(0, 1, 0) { - monthUptimes := make(map[string]string) - if err := s.client.Where("created_at BETWEEN ? AND ?", startDate, startDate.AddDate(0, 1, 0).Add(-1*time.Second)).Find(&uptimesRecords).Error; err != nil { + if err := s.client.Where("created_at BETWEEN ? AND ?", startDate, startDate.AddDate(0, 1, 0).Add(-1*time.Second)).Order("id DESC").Find(&uptimesRecords).Error; err != nil { murError = errors.New("failed on fetching data from pg store") break } for _, record := range uptimesRecords { - monthUptimes[record.PubKey] = fmt.Sprint(record.Online) - if lastTSs[record.PubKey] <= fmt.Sprint(record.UpdatedAt.Unix()) { - lastTSs[record.PubKey] = fmt.Sprint(record.UpdatedAt.Unix()) + if _, ok := lastTSs[record.PubKey]; !ok { + if lastTSs[record.PubKey] <= fmt.Sprint(record.UpdatedAt.Unix()) { + lastTSs[record.PubKey] = fmt.Sprint(record.UpdatedAt.Unix()) + } + versions[record.PubKey] = record.Version + keys = append(keys, record.PubKey) } - versions[record.PubKey] = record.Version } - uptimes = append(uptimes, monthUptimes) } - return makeUptimeResponse(uptimes, lastTSs, versions, startYear, startMonth, endYear, endMonth, murError) + return makeUptimeResponse(keys, lastTSs, versions, murError) } func (s *postgresStore) GetUptimes(pubKeys []string, startYear int, startMonth time.Month, endYear int, endMonth time.Month) (UptimeResponse, error) { startDate := time.Date(startYear, startMonth, 1, 0, 0, 0, 0, time.Now().Location()) endDate := time.Date(endYear, endMonth, 1, 0, 0, 0, 0, time.Now().Location()) - var uptimes []map[string]string + var keys []string versions := make(map[string]string) - var uptimesRecords []Uptime + var uptimesRecords []DailyUptimeHistory var murError error lastTSs := make(map[string]string) for ; startDate.Before(endDate) || startDate.Equal(endDate); startDate = startDate.AddDate(0, 1, 0) { - monthUptimes := make(map[string]string) - if err := s.client.Where("created_at BETWEEN ? AND ? AND pub_key = ?", startDate, startDate.AddDate(0, 1, 0).Add(-1*time.Second), pubKeys).Find(&uptimesRecords).Error; err != nil { + if err := s.client.Where("created_at BETWEEN ? AND ? AND pub_key = ?", startDate, startDate.AddDate(0, 1, 0).Add(-1*time.Second), pubKeys).Order("id DESC").Find(&uptimesRecords).Error; err != nil { murError = errors.New("failed on fetching data from pg store") } for _, record := range uptimesRecords { - monthUptimes[record.PubKey] = fmt.Sprint(record.Online) - if lastTSs[record.PubKey] <= fmt.Sprint(record.UpdatedAt.Unix()) { - lastTSs[record.PubKey] = fmt.Sprint(record.UpdatedAt.Unix()) + if _, ok := lastTSs[record.PubKey]; !ok { + if lastTSs[record.PubKey] <= fmt.Sprint(record.UpdatedAt.Unix()) { + lastTSs[record.PubKey] = fmt.Sprint(record.UpdatedAt.Unix()) + } + versions[record.PubKey] = record.Version + keys = append(keys, record.PubKey) } - versions[record.PubKey] = record.Version } - uptimes = append(uptimes, monthUptimes) } - return makeUptimeResponse(uptimes, lastTSs, versions, startYear, startMonth, endYear, endMonth, murError) + return makeUptimeResponse(keys, lastTSs, versions, murError) } func (s *postgresStore) GetAllVisors(locDetails geo.LocationDetails) (VisorsResponse, error) { ips := make(map[string]string) - currentMonthData := make(map[string]string) - var uptimesRecords []Uptime + var uptimesRecords []DailyUptimeHistory now := time.Now() startYear, startMonth := now.Year(), now.Month() startDate := time.Date(startYear, startMonth, 1, 0, 0, 0, 0, time.Now().Location()) - if err := s.client.Where("created_at >= ?", startDate).Find(&uptimesRecords).Error; err != nil { + if err := s.client.Where("created_at >= ?", startDate).Order("id DESC").Find(&uptimesRecords).Error; err != nil { return VisorsResponse{}, err } for _, record := range uptimesRecords { - ips[record.PubKey] = record.LastIP - currentMonthData[record.PubKey] = fmt.Sprint(record.Online) + if _, ok := ips[record.PubKey]; !ok { + ips[record.PubKey] = record.LastIP + } } - return makeVisorsResponse(currentMonthData, ips, locDetails) + return makeVisorsResponse(ips, locDetails) } func (s *postgresStore) GetDailyUpdateHistory() (map[string]map[string]string, error) { var uptimesRecords []DailyUptimeHistory now := time.Now() - startDate := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Now().Location()).AddDate(0, 0, -40) + startDate := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Now().Location()).AddDate(0, 0, -7) if err := s.client.Where("created_at >= ?", startDate).Find(&uptimesRecords).Error; err != nil { return map[string]map[string]string{}, err } @@ -248,21 +228,23 @@ func (s *postgresStore) Close() { } func (s *postgresStore) readAllUptimeIPMembers(timeValue time.Time) (map[string]string, error) { - var uptimesRecords []Uptime + var uptimesRecords []DailyUptimeHistory response := make(map[string]string) if timeValue.IsZero() { - if err := s.client.Find(&uptimesRecords).Error; err != nil { + if err := s.client.Order("id DESC").Find(&uptimesRecords).Error; err != nil { return response, err } } else { - if err := s.client.Where("created_at BETWEEN ? AND ?", timeValue, timeValue.AddDate(0, 1, 0).Add(-1*time.Second)).Find(&uptimesRecords).Error; err != nil { + if err := s.client.Where("created_at BETWEEN ? AND ?", timeValue, timeValue.AddDate(0, 1, 0).Add(-1*time.Second)).Order("id DESC").Find(&uptimesRecords).Error; err != nil { return response, err } } for _, record := range uptimesRecords { - response[record.PubKey] = record.LastIP + if _, ok := response[record.PubKey]; !ok { + response[record.PubKey] = record.LastIP + } } if len(response) == 0 { @@ -290,35 +272,49 @@ func (s *postgresStore) setCache(pk string, ts int64) { func (s *postgresStore) GetNumberOfUptimesInCurrentMonth() (int, error) { var counter int64 now := time.Date(time.Now().Year(), time.Now().Month(), 1, 0, 0, 0, 0, time.Now().Location()) - err := s.client.Model(&Uptime{}).Where("created_at BETWEEN ? AND ?", now, now.AddDate(0, 1, 0).Add(-1*time.Second)).Count(&counter).Error + err := s.client.Model(&DailyUptimeHistory{}).Where("created_at BETWEEN ? AND ?", now, now.AddDate(0, 1, 0).Add(-1*time.Second)).Group("pub_key").Count(&counter).Error return int(counter), err } func (s *postgresStore) GetNumberOfUptimesByYearAndMonth(year int, month time.Month) (int, error) { var counter int64 timeValue := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.Now().Location()) - err := s.client.Model(&Uptime{}).Where("created_at BETWEEN ? AND ?", timeValue, timeValue.AddDate(0, 1, 0).Add(-1*time.Second)).Count(&counter).Error + err := s.client.Model(&DailyUptimeHistory{}).Where("created_at BETWEEN ? AND ?", timeValue, timeValue.AddDate(0, 1, 0).Add(-1*time.Second)).Group("pub_key").Count(&counter).Error return int(counter), err } -// Uptime is gorm.Model for uptime table -type Uptime struct { - ID uint `gorm:"primarykey" json:"-"` - CreatedAt time.Time - UpdatedAt time.Time - PubKey string - Online int - Version string - IPs string - LastIP string +func (s *postgresStore) DeleteEntries(data []DailyUptimeHistory) error { + for _, entry := range data { + err := s.client.Delete(&DailyUptimeHistory{}, entry.ID).Error + if err != nil { + return err + } + } + return nil +} + +func (s *postgresStore) GetOldestEntry() (DailyUptimeHistory, error) { + var data DailyUptimeHistory + err := s.client.Limit(1).Order("created_at asc").Find(&data).Error + return data, err +} + +func (s *postgresStore) GetSpecificDayData(timeValue time.Time) ([]DailyUptimeHistory, error) { + var data []DailyUptimeHistory + err := s.client.Where("created_at BETWEEN ? AND ?", timeValue, timeValue.AddDate(0, 0, 1).Add(-1*time.Second)).Find(&data).Error + return data, err } // DailyUptimeHistory is gorm.Model for daily uptime history table type DailyUptimeHistory struct { ID uint `gorm:"primarykey" json:"-"` CreatedAt time.Time + UpdatedAt time.Time PubKey string DailyOnline int + Version string + IPs string + LastIP string } func uniqueIPs(ips []string) string { diff --git a/pkg/uptime-tracker/store/store.go b/pkg/uptime-tracker/store/store.go index bee6724..df251d6 100644 --- a/pkg/uptime-tracker/store/store.go +++ b/pkg/uptime-tracker/store/store.go @@ -3,9 +3,10 @@ package store import ( "time" + "gorm.io/gorm" + "github.com/skycoin/skywire-utilities/pkg/geo" "github.com/skycoin/skywire-utilities/pkg/logging" - "gorm.io/gorm" ) const ( @@ -23,6 +24,9 @@ type Store interface { GetNumberOfUptimesByYearAndMonth(year int, month time.Month) (int, error) UpdateUptime(pk, ip, version string) error GetDailyUpdateHistory() (map[string]map[string]string, error) + DeleteEntries([]DailyUptimeHistory) error + GetOldestEntry() (DailyUptimeHistory, error) + GetSpecificDayData(time time.Time) ([]DailyUptimeHistory, error) Close() } diff --git a/pkg/uptime-tracker/store/uptime_response.go b/pkg/uptime-tracker/store/uptime_response.go index ca7dc50..c1ff225 100644 --- a/pkg/uptime-tracker/store/uptime_response.go +++ b/pkg/uptime-tracker/store/uptime_response.go @@ -1,7 +1,6 @@ package store import ( - "fmt" "sort" "strconv" "time" @@ -15,12 +14,9 @@ type UptimeResponse []UptimeDef // UptimeDef is the item of `UptimeResponse`. type UptimeDef struct { - Key string `json:"key"` - Uptime float64 `json:"uptime"` - Downtime float64 `json:"downtime"` - Percentage float64 `json:"percentage"` - Online bool `json:"online"` - Version string `json:"-"` + Key string `json:"key"` + Online bool `json:"online"` + Version string `json:"-"` } // UptimeResponseV2 is the tracker API response format v2 for `/uptimes`. @@ -29,81 +25,22 @@ type UptimeResponseV2 []UptimeDefV2 // UptimeDefV2 is the item of `UptimeResponseV2`. type UptimeDefV2 struct { Key string `json:"pk"` - Uptime float64 `json:"up"` - Downtime float64 `json:"down"` - Percentage float64 `json:"pct"` Online bool `json:"on"` Version string `json:"version,omitempty"` DailyOnlineHistory map[string]string `json:"daily,omitempty"` } -func makeUptimeResponse(uptimes []map[string]string, lastTS map[string]string, versions map[string]string, startYear int, startMonth time.Month, endYear int, endMonth time.Month, callingErr error) (UptimeResponse, error) { +func makeUptimeResponse(keys []string, lastTS map[string]string, versions map[string]string, callingErr error) (UptimeResponse, error) { if callingErr != nil { return UptimeResponse{}, callingErr } - if len(uptimes) == 0 { + if len(keys) == 0 { return UptimeResponse{}, nil } - startDate := time.Date(startYear, startMonth, 1, 0, 0, 0, 0, time.Now().Location()) - endDate := time.Date(endYear, endMonth, 1, 0, 0, 0, 0, time.Now().Location()).AddDate(0, 1, 0) - - // take just complete seconds - totalPeriodSeconds := float64(int(endDate.Sub(startDate).Seconds())) - - now := time.Now() - currentYear := now.Year() - currentMonth := now.Month() - - var ensureLimit bool - var totalPeriodSecondsPassed float64 - if endYear == currentYear && endMonth == currentMonth { - // interval ends with the current month, we'll need to ensure limits. All intervals which - // ended before the current month are complete, but for the current month, we need to ensure - // that uptime of the node is not more than the passed part of the month, i.e. uptime can't be - // 16 days on the 15th of the month - ensureLimit = true - - currentMonthStart := time.Date(currentYear, currentMonth, 1, 0, 0, 0, 0, now.Location()) - currentMonthEnd := currentMonthStart.AddDate(0, 1, 0) - currentMonthTotalSeconds := float64(int(currentMonthEnd.Sub(currentMonthStart).Seconds())) - - // take just complete seconds - partOfMonthPassed := float64(int(now.Sub(currentMonthStart).Seconds())) - - // initially `totalPeriodSeconds` contains full time interval including full current month, - // so we need to subtract total month seconds and add time of the month which is actually - // passed. And this is the hard cap for our uptimes, we can't overcome this boundary - totalPeriodSecondsPassed = totalPeriodSeconds - currentMonthTotalSeconds + partOfMonthPassed - } - - totalUptimes := make(map[string]int64, len(uptimes[0])) - for _, monthUptimes := range uptimes { - for pk, uptimeStr := range monthUptimes { - uptime, err := strconv.ParseInt(uptimeStr, 10, 64) - if err != nil { - return nil, fmt.Errorf("error parsing uptime value for visor %s: %w", pk, err) - } - - totalUptimes[pk] += uptime - } - } - response := make(UptimeResponse, 0) - for pk, uptime := range totalUptimes { - uptime64 := float64(uptime) - if ensureLimit && uptime64 > totalPeriodSecondsPassed { - uptime64 = totalPeriodSecondsPassed - } - if !ensureLimit && uptime64 > totalPeriodSeconds { - uptime64 = totalPeriodSeconds - } - - percentage := uptime64 / totalPeriodSeconds * 100 - if percentage > 100 { - percentage = 100 - } + for _, pk := range keys { online := false ts, err := strconv.ParseInt(lastTS[pk], 10, 64) @@ -112,12 +49,9 @@ func makeUptimeResponse(uptimes []map[string]string, lastTS map[string]string, v } entry := UptimeDef{ - Key: pk, - Uptime: uptime64, - Downtime: totalPeriodSeconds - uptime64, - Percentage: percentage, - Online: online, - Version: versions[pk], + Key: pk, + Online: online, + Version: versions[pk], } response = append(response, entry) diff --git a/pkg/uptime-tracker/store/visors_response.go b/pkg/uptime-tracker/store/visors_response.go index 679575b..c162440 100644 --- a/pkg/uptime-tracker/store/visors_response.go +++ b/pkg/uptime-tracker/store/visors_response.go @@ -19,14 +19,9 @@ type VisorDef struct { Lon float64 `json:"lon"` } -func makeVisorsResponse(currentMonthKeys, ips map[string]string, locDetails geo.LocationDetails) (VisorsResponse, error) { +func makeVisorsResponse(ips map[string]string, locDetails geo.LocationDetails) (VisorsResponse, error) { response := VisorsResponse{} - for pk := range currentMonthKeys { - ip, ok := ips[pk] - if !ok { - continue - } - + for pk, ip := range ips { geo, err := locDetails(net.ParseIP(ip)) if err != nil { log.WithError(err).