Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed history created_at field truncation for clikhouse history insert #207

Merged
merged 3 commits into from
Jul 22, 2024
Merged

Fixed history created_at field truncation for clikhouse history insert #207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4b5a4c7
Fixed history created_at field truncation for clikhouse history inser…
ice-myles Jul 22, 2024
72b6c8d
TestLoadBalanceKYCUsersALotOfUsers now time fix.
ice-myles Jul 22, 2024
d3884f6
TestLoadBalanceKYCUsers test now time fix.
ice-myles Jul 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion bookkeeper/storage/contract.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ type (
Ping(ctx context.Context) error
Insert(ctx context.Context, columns *Columns, input InsertMetadata, usrs []*model.User) error
SelectBalanceHistory(ctx context.Context, id int64, createdAts []stdlibtime.Time) ([]*BalanceHistory, error)
SelectTotalCoins(ctx context.Context, createdAts []stdlibtime.Time) ([]*TotalCoins, error)
SelectTotalCoins(ctx context.Context, createdAtTime stdlibtime.Time, parentInverval stdlibtime.Duration) ([]*TotalCoins, error)
DeleteUserInfo(ctx context.Context, id int64) error
}
BalanceHistory struct {
Expand Down
24 changes: 14 additions & 10 deletions bookkeeper/storage/select_total_coins.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,21 @@
-- SPDX-License-Identifier: ice License 1.0
WITH
WITH req_dates AS (
SELECT req_date FROM VALUES('req_date DateTime',('%[4]v'))
),
active_users AS (
SELECT DISTINCT ON (id, created_at)
created_at, id, id_t0, id_tminus1, pre_staking_allocation, pre_staking_bonus, balance_solo, balance_solo_ethereum, balance_t0, balance_t0_ethereum, balance_for_t0, balance_t1_ethereum
FROM %[1]v
WHERE created_at IN [ '%[2]v' ]
AND kyc_step_passed >= %[3]v
AND (kyc_step_blocked = 0 OR kyc_step_blocked >= %[3]v + 1)
SELECT
req_date AS created_at,
id, id_t0, id_tminus1, pre_staking_allocation, pre_staking_bonus, balance_solo, balance_solo_ethereum, balance_t0, balance_t0_ethereum, balance_for_t0, balance_t1_ethereum
FROM (
SELECT DISTINCT ON (id, created_at)
created_at, id, id_t0, id_tminus1, pre_staking_allocation, pre_staking_bonus, balance_solo, balance_solo_ethereum, balance_t0, balance_t0_ethereum, balance_for_t0, balance_t1_ethereum
FROM %[1]v
WHERE created_at >= '%[2]v' AND created_at < '%[6]v'
AND kyc_step_passed >= %[3]v
AND (kyc_step_blocked = 0 OR kyc_step_blocked >= %[3]v + 1)
) t, req_dates
),
valid_users_stopped_processing AS (
WITH req_dates AS (
SELECT req_date FROM VALUES('req_date DateTime',('%[4]v'))
)
SELECT req_date AS created_at,
id, id_t0, id_tminus1, pre_staking_allocation, pre_staking_bonus, balance_solo, balance_solo_ethereum, balance_t0, balance_t0_ethereum, balance_for_t0, balance_t1_ethereum
FROM (SELECT DISTINCT ON (id, created_at)
Expand Down
48 changes: 12 additions & 36 deletions bookkeeper/storage/storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ func (db *db) Insert(ctx context.Context, columns *Columns, input InsertMetadata
now := time.Now()
truncateDuration := stdlibtime.Minute
if !db.cfg.Development {
truncateDuration = stdlibtime.Hour
truncateDuration = 24 * stdlibtime.Hour
}

for _, usr := range usrs {
Expand Down Expand Up @@ -570,20 +570,19 @@ func (db *db) SelectBalanceHistory(ctx context.Context, id int64, createdAts []s
return res, nil
}

func (db *db) SelectTotalCoins(ctx context.Context, createdAts []stdlibtime.Time) ([]*TotalCoins, error) {
func (db *db) SelectTotalCoins(ctx context.Context, createdAtTime stdlibtime.Time, parentInverval stdlibtime.Duration) ([]*TotalCoins, error) {
var (
createdAt = proto.ColDateTime{Data: make([]proto.DateTime, 0, len(createdAts)), Location: stdlibtime.UTC}
balanceTotalStandard = make(proto.ColFloat64, 0, len(createdAts))
balanceTotalPreStaking = make(proto.ColFloat64, 0, len(createdAts))
balanceTotalEthereum = make(proto.ColFloat64, 0, len(createdAts))
res = make([]*TotalCoins, 0, len(createdAts))
createdAt = proto.ColDateTime{Data: make([]proto.DateTime, 0, 1), Location: stdlibtime.UTC}
balanceTotalStandard = make(proto.ColFloat64, 0, 1)
balanceTotalPreStaking = make(proto.ColFloat64, 0, 1)
balanceTotalEthereum = make(proto.ColFloat64, 0, 1)
res = make([]*TotalCoins, 0, 1)
)
createdAtArray := make([]string, 0, len(createdAts))
for _, date := range createdAts {
format := date.UTC().Format(stdlibtime.RFC3339)
createdAtArray = append(createdAtArray, format[0:len(format)-1])
}
sql := fmt.Sprintf(selectTotalCoinsSQL, tableName, strings.Join(createdAtArray, "','"), users.NoneKYCStep, strings.Join(createdAtArray, "'), ('"), createdAtArray[0])
formatCreatedAt := createdAtTime.UTC().Format(stdlibtime.RFC3339)
createdAtDate := formatCreatedAt[0 : len(formatCreatedAt)-1]
formatNotAfter := createdAtTime.UTC().Add(parentInverval).Format(stdlibtime.RFC3339)
notAfterDate := formatNotAfter[0 : len(formatNotAfter)-1]
sql := fmt.Sprintf(selectTotalCoinsSQL, tableName, createdAtDate, users.NoneKYCStep, createdAtDate, createdAtDate, notAfterDate)
if err := db.pools[atomic.AddUint64(&db.currentIndex, 1)%uint64(len(db.pools))].Do(ctx, ch.Query{
Body: sql,
Result: append(make(proto.Results, 0, 4),
Expand Down Expand Up @@ -615,29 +614,6 @@ func (db *db) SelectTotalCoins(ctx context.Context, createdAts []stdlibtime.Time
}); err != nil {
return nil, err
}
dedupedRes := make([]*TotalCoins, 0, len(createdAts))

for _, cAt := range createdAts {
var found *TotalCoins = nil
for _, rowA := range res {
if rowA.CreatedAt.Equal(cAt) {
found = rowA
break
}
}
if found != nil {
dedupedRes = append(dedupedRes, found)
} else {
dedupedRes = append(dedupedRes, &TotalCoins{
CreatedAt: time.New(cAt),
BalanceTotalStandard: 0,
BalanceTotalPreStaking: 0,
BalanceTotalEthereum: 0,
BalanceTotal: 0,
})
}
}
res = dedupedRes

return res, nil
}
Expand Down
28 changes: 14 additions & 14 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,24 +10,24 @@ require (
github.com/ethereum/go-ethereum v1.14.7
github.com/goccy/go-json v0.10.3
github.com/hashicorp/go-multierror v1.1.1
github.com/ice-blockchain/eskimo v1.373.0
github.com/ice-blockchain/eskimo v1.375.0
github.com/ice-blockchain/go-tarantool-client v0.0.0-20230327200757-4fc71fa3f7bb
github.com/ice-blockchain/wintr v1.147.0
github.com/ice-blockchain/wintr v1.148.0
github.com/imroc/req/v3 v3.43.7
github.com/oklog/ulid/v2 v2.1.0
github.com/pkg/errors v0.9.1
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
github.com/redis/go-redis/v9 v9.5.4
github.com/redis/go-redis/v9 v9.6.0
github.com/stretchr/testify v1.9.0
github.com/swaggo/swag v1.16.3
github.com/testcontainers/testcontainers-go v0.32.0
go.uber.org/zap v1.27.0
golang.org/x/exp v0.0.0-20240716175740-e3f259677ff7
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
)

require (
cloud.google.com/go v0.115.0 // indirect
cloud.google.com/go/auth v0.7.1 // indirect
cloud.google.com/go/auth v0.7.2 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
cloud.google.com/go/compute/metadata v0.5.0 // indirect
cloud.google.com/go/firestore v1.15.0 // indirect
Expand Down Expand Up @@ -67,15 +67,15 @@ require (
github.com/containerd/continuity v0.4.3 // indirect
github.com/containerd/errdefs v0.1.0 // indirect
github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect
github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
github.com/crate-crypto/go-kzg-4844 v1.1.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/dmarkham/enumer v1.5.10 // indirect
github.com/docker/distribution v2.8.3+incompatible // indirect
github.com/docker/docker v27.0.3+incompatible // indirect
github.com/docker/docker v27.1.0+incompatible // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/ethereum/c-kzg-4844 v1.0.3 // indirect
Expand Down Expand Up @@ -108,11 +108,11 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect
github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da // indirect
github.com/google/pprof v0.0.0-20240722153945-304e4f0156b8 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.5 // indirect
github.com/googleapis/gax-go/v2 v2.13.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
Expand All @@ -137,7 +137,7 @@ require (
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mmcloughlin/addchain v0.4.0 // indirect
github.com/moby/sys/mount v0.3.4 // indirect
Expand Down Expand Up @@ -211,11 +211,11 @@ require (
golang.org/x/text v0.16.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.23.0 // indirect
google.golang.org/api v0.188.0 // indirect
google.golang.org/api v0.189.0 // indirect
google.golang.org/appengine/v2 v2.0.6 // indirect
google.golang.org/genproto v0.0.0-20240711142825-46eb208f015d // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d // indirect
google.golang.org/genproto v0.0.0-20240722135656-d784300faade // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade // indirect
google.golang.org/grpc v1.65.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
Expand Down
Loading
Loading