Skip to content

Commit

Permalink
Database Replication POC
Browse files Browse the repository at this point in the history
  • Loading branch information
ludydoo committed May 17, 2022
1 parent 0a0d365 commit 30c8e4a
Show file tree
Hide file tree
Showing 47 changed files with 4,616 additions and 18 deletions.
2 changes: 1 addition & 1 deletion .prow/push-core-backend-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ presubmits:
decorate: true
spec:
containers:
- image: golang:1.16
- image: golang:1.18
command: [ "bash", "-c" ]
args:
- >
Expand Down
37 changes: 37 additions & 0 deletions cmd/serve_db.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package cmd

import (
"context"

"github.com/nrc-no/core/pkg/server/data"
"github.com/spf13/cobra"
)

// serveDataCmd represents the data command
var serveDataCmd = &cobra.Command{
Use: "data",
Short: "starts the data server",
RunE: func(cmd *cobra.Command, args []string) error {
if err := serveDb(ctx,
data.Options{
ServerOptions: coreOptions.Serve.Login,
}); err != nil {
return err
}
<-doneSignal
return nil
},
}

func init() {
serveCmd.AddCommand(serveDataCmd)
}

func serveDb(ctx context.Context, options data.Options) error {
server, err := data.NewServer(options)
if err != nil {
return err
}
server.Start(ctx)
return nil
}
73 changes: 66 additions & 7 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
module github.com/nrc-no/core

go 1.16
go 1.18

require (
github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect
github.com/coreos/go-oidc/v3 v3.0.0
github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0
github.com/emicklei/go-restful-openapi/v2 v2.6.0
Expand All @@ -20,30 +19,90 @@ require (
github.com/gorilla/securecookie v1.1.1
github.com/gorilla/sessions v1.2.1
github.com/jackc/pgconn v1.10.1
github.com/jmoiron/sqlx v1.3.5
github.com/juju/fslock v0.0.0-20160525022230-4d5c94c67b4b
github.com/lib/pq v1.10.4
github.com/lithammer/shortuuid/v3 v3.0.7
github.com/looplab/fsm v0.3.0
github.com/manifoldco/promptui v0.9.0
github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2 // indirect
github.com/mattn/go-sqlite3 v1.14.9
github.com/mattn/go-sqlite3 v1.14.12
github.com/ory/hydra-client-go v1.10.6
github.com/rs/cors v1.8.0
github.com/satori/go.uuid v1.2.0
github.com/snabb/isoweek v1.0.1
github.com/spf13/cobra v1.2.1
github.com/spf13/viper v1.9.0
github.com/stretchr/testify v1.7.0
go.mongodb.org/mongo-driver v1.5.3 // indirect
go.uber.org/zap v1.19.1
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20211023085530-d6a326fbbf70 // indirect
golang.org/x/tools v0.1.8
gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
gorm.io/driver/postgres v1.2.1
gorm.io/driver/sqlite v1.2.3
gorm.io/gorm v1.22.4
)

require (
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-openapi/analysis v0.20.0 // indirect
github.com/go-openapi/errors v0.20.1 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.5 // indirect
github.com/go-openapi/loads v0.20.2 // indirect
github.com/go-openapi/spec v0.20.3 // indirect
github.com/go-openapi/swag v0.19.15 // indirect
github.com/go-openapi/validate v0.20.2 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/uuid v1.2.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.1.1 // indirect
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
github.com/jackc/pgtype v1.8.1 // indirect
github.com/jackc/pgx/v4 v4.13.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.3 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.11 // indirect
github.com/magiconair/properties v1.8.5 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2 // indirect
github.com/mitchellh/mapstructure v1.4.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/spf13/afero v1.6.0 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
go.mongodb.org/mongo-driver v1.5.3 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
golang.org/x/mod v0.5.1 // indirect
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
golang.org/x/sys v0.0.0-20211023085530-d6a326fbbf70 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/ini.v1 v1.63.2 // indirect
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
12 changes: 7 additions & 5 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,8 @@ github.com/go-openapi/validate v0.20.2/go.mod h1:e7OJoKNgd0twXZwIn0A43tHbvIcr/rZ
github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
Expand Down Expand Up @@ -390,7 +392,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
Expand All @@ -412,7 +413,6 @@ github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5W
github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A=
github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg=
Expand Down Expand Up @@ -445,6 +445,8 @@ github.com/jinzhu/now v1.1.3 h1:PlHq1bSCSZL9K0wUhbm2pGLoTWs2GwVhsP6emvGV/ZI=
github.com/jinzhu/now v1.1.3/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
Expand Down Expand Up @@ -513,8 +515,10 @@ github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2y
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ1Z0=
github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
Expand Down Expand Up @@ -637,7 +641,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
Expand Down Expand Up @@ -884,7 +887,6 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211023085530-d6a326fbbf70 h1:SeSEfdIxyvwGJliREIJhRPPXvW6sDlLT+UQ3B0hD0NA=
golang.org/x/sys v0.0.0-20211023085530-d6a326fbbf70/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
Expand Down
33 changes: 33 additions & 0 deletions pkg/server/data/api/changestream.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package api

import (
"encoding/json"
"fmt"
)

type Changes struct {
Items []ChangeItem `json:"items"`
}

func (c Changes) String() string {
jsonBytes, err := json.Marshal(c)
if err != nil {
return fmt.Sprintf("%v", err)
}
return string(jsonBytes)
}

type ChangeItem struct {
Sequence int64 `json:"sequence"`
TableName string `json:"table_name"`
RecordID string `json:"record_id"`
RecordRevision Revision `json:"record_revision"`
}

func (c ChangeItem) String() string {
jsonBytes, err := json.Marshal(c)
if err != nil {
return fmt.Sprintf("%v", err)
}
return string(jsonBytes)
}
13 changes: 13 additions & 0 deletions pkg/server/data/api/const.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package api

const (
KeyRecordID = "_id"
KeyRevision = "_rev"
KeyPrevision = "_prev"
KeyDeleted = "_deleted"
ChangeStreamTableName = "_changes"
KeyCSSequence = "seq"
KeyCSTableName = "table_name"
KeyCSRecordID = "record_id"
KeyCSRecordRevision = "record_rev"
)
102 changes: 102 additions & 0 deletions pkg/server/data/api/errors.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package api

type Error struct {
Message string
Code ErrorCode
}

type ErrorCode uint8

const (
ErrCodeInvalidRevision ErrorCode = iota
ErrCodeInvalidPrevision
ErrCodeInvalidRecordID
ErrCodeDuplicateField
ErrCodeMissingRevision
ErrCodeInvalidTable
ErrCodeInvalidColumnType
ErrCodeEmptyTableColumns
ErrCodeDuplicateColumnName
ErrCodeInvalidColumnName
ErrCodeRecordNotFound
ErrCodeFieldNotFound
ErrCodeInternalError
ErrCodeTableAlreadyExists
ErrCodeUnsupportedDialect
ErrCodeInvalidTimestamp
ErrCodeMissingId
)

func (e *Error) Error() string {
return e.Message
}

func (e *Error) ErrorCode() ErrorCode {
return e.Code
}

func (e *Error) Is(other error) bool {
if other == nil {
return false
}
if e == other {
return true
}
if o, ok := other.(*Error); ok {
return e.Code == o.Code
}
return false
}

func IsError(err error, code ErrorCode) bool {
if err == nil {
return false
}
if e, ok := err.(*Error); ok {
return e.Code == code
}
return false
}

func NewError(code ErrorCode, message string) *Error {
return &Error{
Message: message,
Code: code,
}
}

var (
ErrInvalidRevision = NewError(ErrCodeInvalidRevision, "invalid revision")
ErrInvalidPrevision = NewError(ErrCodeInvalidPrevision, "invalid previous revision")
ErrInvalidRecordID = NewError(ErrCodeInvalidRecordID, "invalid record id")
ErrDuplicateField = NewError(ErrCodeDuplicateField, "duplicate field")
ErrMissingRevision = NewError(ErrCodeMissingRevision, "missing revision")
ErrInvalidTableName = NewError(ErrCodeInvalidTable, "invalid table name")
ErrEmptyColumns = NewError(ErrCodeEmptyTableColumns, "empty columns")
ErrDuplicateColumnName = NewError(ErrCodeDuplicateColumnName, "duplicate column name")
ErrInvalidColumnName = NewError(ErrCodeInvalidColumnName, "invalid column name")
ErrRecordNotFound = NewError(ErrCodeRecordNotFound, "record not found")
ErrFieldNotFound = NewError(ErrCodeFieldNotFound, "field not found")
ErrInvalidColumnType = NewError(ErrCodeInvalidColumnType, "invalid column type")
ErrTableAlreadyExists = NewError(ErrCodeTableAlreadyExists, "table already exists")
ErrUnsupportedDialect = NewError(ErrCodeUnsupportedDialect, "unsupported dialect")
ErrInvalidValueType = NewError(ErrCodeInternalError, "invalid value type")
ErrInvalidTimestamp = NewError(ErrCodeInvalidTimestamp, "invalid timestamp")
ErrMissingId = NewError(ErrCodeMissingId, "missing id")
)

func NewDuplicateColumnNameErr(name string) *Error {
return NewError(ErrCodeDuplicateColumnName, "duplicate column name: "+name)
}

func NewInvalidColumnNameErr(name string) *Error {
return NewError(ErrCodeInvalidColumnName, "invalid column name: "+name)
}

func NewInvalidColumnTypeErr(name string) *Error {
return NewError(ErrCodeInvalidColumnType, "invalid column type: "+name)
}

func NewTableAlreadyExistsErr(name string) *Error {
return NewError(ErrCodeTableAlreadyExists, "table already exists: "+name)
}
13 changes: 13 additions & 0 deletions pkg/server/data/api/i.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package api

type PutRecordOptions struct {
IsNew bool
}

type PutRecordOption func(o *PutRecordOptions)

var IsNew = func(isNew bool) PutRecordOption {
return func(o *PutRecordOptions) {
o.IsNew = isNew
}
}
Loading

0 comments on commit 30c8e4a

Please sign in to comment.