Skip to content

Commit

Permalink
Fixed history created_at field truncation for clikhouse history insert (
Browse files Browse the repository at this point in the history 
#207)
  • Loading branch information
@ice-myles
ice-myles authored Jul 22, 2024
1 parent 6e66985 commit 5a5cfd7
Show file tree
Hide file tree
Showing 7 changed files with 73 additions and 93 deletions.
2 changes: 1 addition & 1 deletion bookkeeper/storage/contract.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ type (
Ping(ctx context.Context) error
Insert(ctx context.Context, columns *Columns, input InsertMetadata, usrs []*model.User) error
SelectBalanceHistory(ctx context.Context, id int64, createdAts []stdlibtime.Time) ([]*BalanceHistory, error)
SelectTotalCoins(ctx context.Context, createdAts []stdlibtime.Time) ([]*TotalCoins, error)
SelectTotalCoins(ctx context.Context, createdAtTime stdlibtime.Time, parentInverval stdlibtime.Duration) ([]*TotalCoins, error)
DeleteUserInfo(ctx context.Context, id int64) error
}
BalanceHistory struct {
Expand Down
24 changes: 14 additions & 10 deletions bookkeeper/storage/select_total_coins.sql
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,21 @@
-- SPDX-License-Identifier: ice License 1.0
WITH
WITH req_dates AS (
SELECT req_date FROM VALUES('req_date DateTime',('%[4]v'))
),
active_users AS (
SELECT DISTINCT ON (id, created_at)
created_at, id, id_t0, id_tminus1, pre_staking_allocation, pre_staking_bonus, balance_solo, balance_solo_ethereum, balance_t0, balance_t0_ethereum, balance_for_t0, balance_t1_ethereum
FROM %[1]v
WHERE created_at IN [ '%[2]v' ]
AND kyc_step_passed >= %[3]v
AND (kyc_step_blocked = 0 OR kyc_step_blocked >= %[3]v + 1)
SELECT
req_date AS created_at,
id, id_t0, id_tminus1, pre_staking_allocation, pre_staking_bonus, balance_solo, balance_solo_ethereum, balance_t0, balance_t0_ethereum, balance_for_t0, balance_t1_ethereum
FROM (
SELECT DISTINCT ON (id, created_at)
created_at, id, id_t0, id_tminus1, pre_staking_allocation, pre_staking_bonus, balance_solo, balance_solo_ethereum, balance_t0, balance_t0_ethereum, balance_for_t0, balance_t1_ethereum
FROM %[1]v
WHERE created_at >= '%[2]v' AND created_at < '%[6]v'
AND kyc_step_passed >= %[3]v
AND (kyc_step_blocked = 0 OR kyc_step_blocked >= %[3]v + 1)
) t, req_dates
),
valid_users_stopped_processing AS (
WITH req_dates AS (
SELECT req_date FROM VALUES('req_date DateTime',('%[4]v'))
)
SELECT req_date AS created_at,
id, id_t0, id_tminus1, pre_staking_allocation, pre_staking_bonus, balance_solo, balance_solo_ethereum, balance_t0, balance_t0_ethereum, balance_for_t0, balance_t1_ethereum
FROM (SELECT DISTINCT ON (id, created_at)
Expand Down
48 changes: 12 additions & 36 deletions bookkeeper/storage/storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ func (db *db) Insert(ctx context.Context, columns *Columns, input InsertMetadata
now := time.Now()
truncateDuration := stdlibtime.Minute
if !db.cfg.Development {
truncateDuration = stdlibtime.Hour
truncateDuration = 24 * stdlibtime.Hour
}

for _, usr := range usrs {
Expand Down Expand Up @@ -570,20 +570,19 @@ func (db *db) SelectBalanceHistory(ctx context.Context, id int64, createdAts []s
return res, nil
}

func (db *db) SelectTotalCoins(ctx context.Context, createdAts []stdlibtime.Time) ([]*TotalCoins, error) {
func (db *db) SelectTotalCoins(ctx context.Context, createdAtTime stdlibtime.Time, parentInverval stdlibtime.Duration) ([]*TotalCoins, error) {
var (
createdAt = proto.ColDateTime{Data: make([]proto.DateTime, 0, len(createdAts)), Location: stdlibtime.UTC}
balanceTotalStandard = make(proto.ColFloat64, 0, len(createdAts))
balanceTotalPreStaking = make(proto.ColFloat64, 0, len(createdAts))
balanceTotalEthereum = make(proto.ColFloat64, 0, len(createdAts))
res = make([]*TotalCoins, 0, len(createdAts))
createdAt = proto.ColDateTime{Data: make([]proto.DateTime, 0, 1), Location: stdlibtime.UTC}
balanceTotalStandard = make(proto.ColFloat64, 0, 1)
balanceTotalPreStaking = make(proto.ColFloat64, 0, 1)
balanceTotalEthereum = make(proto.ColFloat64, 0, 1)
res = make([]*TotalCoins, 0, 1)
)
createdAtArray := make([]string, 0, len(createdAts))
for _, date := range createdAts {
format := date.UTC().Format(stdlibtime.RFC3339)
createdAtArray = append(createdAtArray, format[0:len(format)-1])
}
sql := fmt.Sprintf(selectTotalCoinsSQL, tableName, strings.Join(createdAtArray, "','"), users.NoneKYCStep, strings.Join(createdAtArray, "'), ('"), createdAtArray[0])
formatCreatedAt := createdAtTime.UTC().Format(stdlibtime.RFC3339)
createdAtDate := formatCreatedAt[0 : len(formatCreatedAt)-1]
formatNotAfter := createdAtTime.UTC().Add(parentInverval).Format(stdlibtime.RFC3339)
notAfterDate := formatNotAfter[0 : len(formatNotAfter)-1]
sql := fmt.Sprintf(selectTotalCoinsSQL, tableName, createdAtDate, users.NoneKYCStep, createdAtDate, createdAtDate, notAfterDate)
if err := db.pools[atomic.AddUint64(&db.currentIndex, 1)%uint64(len(db.pools))].Do(ctx, ch.Query{
Body: sql,
Result: append(make(proto.Results, 0, 4),
Expand Down Expand Up @@ -615,29 +614,6 @@ func (db *db) SelectTotalCoins(ctx context.Context, createdAts []stdlibtime.Time
}); err != nil {
return nil, err
}
dedupedRes := make([]*TotalCoins, 0, len(createdAts))

for _, cAt := range createdAts {
var found *TotalCoins = nil
for _, rowA := range res {
if rowA.CreatedAt.Equal(cAt) {
found = rowA
break
}
}
if found != nil {
dedupedRes = append(dedupedRes, found)
} else {
dedupedRes = append(dedupedRes, &TotalCoins{
CreatedAt: time.New(cAt),
BalanceTotalStandard: 0,
BalanceTotalPreStaking: 0,
BalanceTotalEthereum: 0,
BalanceTotal: 0,
})
}
}
res = dedupedRes

return res, nil
}
Expand Down
28 changes: 14 additions & 14 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,24 +10,24 @@ require (
github.com/ethereum/go-ethereum v1.14.7
github.com/goccy/go-json v0.10.3
github.com/hashicorp/go-multierror v1.1.1
github.com/ice-blockchain/eskimo v1.373.0
github.com/ice-blockchain/eskimo v1.375.0
github.com/ice-blockchain/go-tarantool-client v0.0.0-20230327200757-4fc71fa3f7bb
github.com/ice-blockchain/wintr v1.147.0
github.com/ice-blockchain/wintr v1.148.0
github.com/imroc/req/v3 v3.43.7
github.com/oklog/ulid/v2 v2.1.0
github.com/pkg/errors v0.9.1
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
github.com/redis/go-redis/v9 v9.5.4
github.com/redis/go-redis/v9 v9.6.0
github.com/stretchr/testify v1.9.0
github.com/swaggo/swag v1.16.3
github.com/testcontainers/testcontainers-go v0.32.0
go.uber.org/zap v1.27.0
golang.org/x/exp v0.0.0-20240716175740-e3f259677ff7
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
)

require (
cloud.google.com/go v0.115.0 // indirect
cloud.google.com/go/auth v0.7.1 // indirect
cloud.google.com/go/auth v0.7.2 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
cloud.google.com/go/compute/metadata v0.5.0 // indirect
cloud.google.com/go/firestore v1.15.0 // indirect
Expand Down Expand Up @@ -67,15 +67,15 @@ require (
github.com/containerd/continuity v0.4.3 // indirect
github.com/containerd/errdefs v0.1.0 // indirect
github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect
github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
github.com/crate-crypto/go-kzg-4844 v1.1.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/dmarkham/enumer v1.5.10 // indirect
github.com/docker/distribution v2.8.3+incompatible // indirect
github.com/docker/docker v27.0.3+incompatible // indirect
github.com/docker/docker v27.1.0+incompatible // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/ethereum/c-kzg-4844 v1.0.3 // indirect
Expand Down Expand Up @@ -108,11 +108,11 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect
github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da // indirect
github.com/google/pprof v0.0.0-20240722153945-304e4f0156b8 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.5 // indirect
github.com/googleapis/gax-go/v2 v2.13.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
Expand All @@ -137,7 +137,7 @@ require (
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mmcloughlin/addchain v0.4.0 // indirect
github.com/moby/sys/mount v0.3.4 // indirect
Expand Down Expand Up @@ -211,11 +211,11 @@ require (
golang.org/x/text v0.16.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.23.0 // indirect
google.golang.org/api v0.188.0 // indirect
google.golang.org/api v0.189.0 // indirect
google.golang.org/appengine/v2 v2.0.6 // indirect
google.golang.org/genproto v0.0.0-20240711142825-46eb208f015d // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d // indirect
google.golang.org/genproto v0.0.0-20240722135656-d784300faade // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade // indirect
google.golang.org/grpc v1.65.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
Expand Down
Loading

0 comments on commit 5a5cfd7

Please sign in to comment.