diff --git a/e2e_cleanup/go.mod b/e2e_cleanup/go.mod index 29854ac42..4779d8c87 100644 --- a/e2e_cleanup/go.mod +++ b/e2e_cleanup/go.mod @@ -4,42 +4,42 @@ go 1.23.0 require ( cloud.google.com/go/bigquery v1.63.1 - cloud.google.com/go/pubsub v1.43.0 + cloud.google.com/go/pubsub v1.45.1 github.com/snowflakedb/gosnowflake v1.11.2 github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 - google.golang.org/api v0.199.0 + google.golang.org/api v0.203.0 ) require ( - cloud.google.com/go v0.115.1 // indirect - cloud.google.com/go/auth v0.9.7 // indirect + cloud.google.com/go v0.116.0 // indirect + cloud.google.com/go/auth v0.9.9 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect cloud.google.com/go/compute/metadata v0.5.2 // indirect - cloud.google.com/go/iam v1.2.1 // indirect + cloud.google.com/go/iam v1.2.2 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/apache/arrow/go/v15 v15.0.2 // indirect - github.com/aws/aws-sdk-go-v2 v1.32.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.32.2 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.39 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.66.1 // indirect github.com/aws/smithy-go v1.22.0 // indirect github.com/danieljoos/wincred v1.2.2 // indirect github.com/dvsekhvalnov/jose2go v1.7.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/gabriel-vasile/mimetype v1.4.5 // indirect + github.com/gabriel-vasile/mimetype v1.4.6 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.3 // indirect @@ -52,7 +52,7 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect - github.com/klauspost/compress v1.17.10 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/mtibben/percent v0.2.1 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect @@ -60,13 +60,13 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect - go.opentelemetry.io/otel v1.30.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect golang.org/x/crypto v0.28.0 // indirect - golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6 // indirect + golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect @@ -77,9 +77,9 @@ require ( golang.org/x/time v0.7.0 // indirect golang.org/x/tools v0.26.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/genproto v0.0.0-20240930140551-af27646dc61f // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240930140551-af27646dc61f // indirect + google.golang.org/genproto v0.0.0-20241021214115-324edc3d5d38 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241021214115-324edc3d5d38 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect google.golang.org/grpc v1.67.1 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.35.1 // indirect ) diff --git a/e2e_cleanup/go.sum b/e2e_cleanup/go.sum index 0293c672d..42b74df80 100644 --- a/e2e_cleanup/go.sum +++ b/e2e_cleanup/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.115.1 h1:Jo0SM9cQnSkYfp44+v+NQXHpcHqlnRJk2qxh6yvxxxQ= -cloud.google.com/go v0.115.1/go.mod h1:DuujITeaufu3gL68/lOFIirVNJwQeyf5UXyi+Wbgknc= -cloud.google.com/go/auth v0.9.7 h1:ha65jNwOfI48YmUzNfMaUDfqt5ykuYIUnSartpU1+BA= -cloud.google.com/go/auth v0.9.7/go.mod h1:Xo0n7n66eHyOWWCnitop6870Ilwo3PiZyodVkkH1xWM= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go/auth v0.9.9 h1:BmtbpNQozo8ZwW2t7QJjnrQtdganSdmqeIBxHxNkEZQ= +cloud.google.com/go/auth v0.9.9/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= cloud.google.com/go/bigquery v1.63.1 h1:/6syiWrSpardKNxdvldS5CUTRJX1iIkSPXCjLjiGL+g= @@ -11,22 +11,22 @@ cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixA cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= cloud.google.com/go/datacatalog v1.22.1 h1:i0DyKb/o7j+0vgaFtimcRFjYsD6wFw1jpnODYUyiYRs= cloud.google.com/go/datacatalog v1.22.1/go.mod h1:MscnJl9B2lpYlFoxRjicw19kFTwEke8ReKL5Y/6TWg8= -cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU= -cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g= +cloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA= +cloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY= cloud.google.com/go/kms v1.20.0 h1:uKUvjGqbBlI96xGE669hcVnEMw1Px/Mvfa62dhM5UrY= cloud.google.com/go/kms v1.20.0/go.mod h1:/dMbFF1tLLFnQV44AoI2GlotbjowyUfgVwezxW291fM= cloud.google.com/go/longrunning v0.6.1 h1:lOLTFxYpr8hcRtcwWir5ITh1PAKUD/sG2lKrTSYjyMc= cloud.google.com/go/longrunning v0.6.1/go.mod h1:nHISoOZpBcmlwbJmiVk5oDRz0qG/ZxPynEGs1iZ79s0= -cloud.google.com/go/pubsub v1.43.0 h1:s3Qx+F96J7Kwey/uVHdK3QxFLIlOvvw4SfMYw2jFjb4= -cloud.google.com/go/pubsub v1.43.0/go.mod h1:LNLfqItblovg7mHWgU5g84Vhza4J8kTxx0YqIeTzcXY= +cloud.google.com/go/pubsub v1.45.1 h1:ZC/UzYcrmK12THWn1P72z+Pnp2vu/zCZRXyhAfP1hJY= +cloud.google.com/go/pubsub v1.45.1/go.mod h1:3bn7fTmzZFwaUjllitv1WlsNMkqBgGUb3UdMhI54eCc= cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= github.com/99designs/keyring v1.2.2 h1:pZd3neh/EmUzWONb35LxQfvuY7kiSXAq3HQd97+XBn0= github.com/99designs/keyring v1.2.2/go.mod h1:wes/FrByc8j7lFOAGLGSNEg8f/PaI3cgTBqhFkHUrPk= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= @@ -42,42 +42,42 @@ github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvK github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE= github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= -github.com/aws/aws-sdk-go-v2 v1.32.0 h1:GuHp7GvMN74PXD5C97KT5D87UhIy4bQPkflQKbfkndg= -github.com/aws/aws-sdk-go-v2 v1.32.0/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= +github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI= +github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= -github.com/aws/aws-sdk-go-v2/config v1.27.41 h1:esG3WpmEuNJ6F4kVFLumN8nCfA5VBav1KKb3JPx83O4= -github.com/aws/aws-sdk-go-v2/config v1.27.41/go.mod h1:haUg09ebP+ClvPjU3EB/xe0HF9PguO19PD2fdjM2X14= -github.com/aws/aws-sdk-go-v2/credentials v1.17.39 h1:tmVexAhoGqJxNE2oc4/SJqL+Jz1x1iCPt5ts9XcqZCU= -github.com/aws/aws-sdk-go-v2/credentials v1.17.39/go.mod h1:zgOdbDI9epE608PdboJ87CYvPIejAgFevazeJW6iauQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.15 h1:kGjlNc2IXXcxPDcfMyCshNCjVgxUhC/vTJv7NvC9wKk= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.15/go.mod h1:rk/HmqPo+dX0Uv0Q1+4w3QKFdICEGSsTYz1hRWvH8UI= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.28 h1:yUPy1fwOKNZ9L52E9TCMomU+mKXNCgqi17dtYIdSolk= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.28/go.mod h1:bJJP1cGMO0fPBgCjqHAWbc0WRbKrxrWU4hQfc/0ciAA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.19 h1:Q/k5wCeJkSWs+62kDfOillkNIJ5NqmE3iOfm48g/W8c= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.19/go.mod h1:Wns1C66VvtA2Bv/cUBuKZKQKdjo7EVMhp90aAa+8oTI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.19 h1:AYLE0lUfKvN6icFTR/p+NmD1amYKTbqHQ1Nm+jwE6BM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.19/go.mod h1:1giLakj64GjuH1NBzF/DXqly5DWHtMTaOzRZ53nFX0I= +github.com/aws/aws-sdk-go-v2/config v1.28.0 h1:FosVYWcqEtWNxHn8gB/Vs6jOlNwSoyOCA/g/sxyySOQ= +github.com/aws/aws-sdk-go-v2/config v1.28.0/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.34 h1:os83HS/WfOwi1LsZWLCSHTyj+whvPGaxUsq/D1Ol2Q0= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.34/go.mod h1:tG0BaDCAweumHRsOHm72tuPgAfRLASQThgthWYeTyV8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19 h1:FKdiFzTxlTRO71p0C7VrLbkkdW8qfMKF5+ej6bTmkT0= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19/go.mod h1:abO3pCj7WLQPTllnSeYImqFfkGrmJV0JovWo/gqT5N0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 h1:7edmS3VOBDhK00b/MwGtGglCm7hhwNYnjJs/PgFdMQE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21/go.mod h1:Q9o5h4HoIWG8XfzxqiuK/CGUbepCJ8uTlaE3bAbxytQ= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0 h1:FQNWhRuSq8QwW74GtU0MrveNhZbqvHsA4dkA9w8fTDQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0/go.mod h1:j/zZ3zmWfGCK91K73YsfHP53BSTLSjL/y6YN39XbBLM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.0 h1:AdbiDUgQZmM28rDIZbiSwFxz8+3B94aOXxzs6oH+EA0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.0/go.mod h1:uV476Bd80tiDTX4X2redMtagQUg65aU/gzPojSJ4kSI= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0 h1:1NKXS8XfhMM0bg5wVYa/eOH8AM2f6JijugbKEyQFTIg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0/go.mod h1:ph931DUfVfgrhZR7py9olSvHCiRpvaGxNvlWBcXxFds= -github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0 h1:2dSm7frMrw2tdJ0QvyccQNJyPGaP24dyDgZ6h1QJMGU= -github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0/go.mod h1:4XSVpw66upN8wND3JZA29eXl2NOZvfFVq7DIP6xvfuQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.0 h1:71FvP6XFj53NK+YiAEGVzeiccLVeFnHOCvMig0zOHsE= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.0/go.mod h1:UVJqtKXSd9YppRKgdBIkyv7qgbSGv5DchM3yX0BN2mU= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.0 h1:Uco4o19bi3AmBapImNzuMk+rfzlui52BDyVK1UfJeRA= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.0/go.mod h1:+HLFhCpnG08hBee8bUdfd1mBK+rFKPt4O5igR9lXDfk= -github.com/aws/aws-sdk-go-v2/service/sts v1.32.0 h1:GiQUjZM2KUZX68o/LpZ1xqxYMuvoxpRrOwYARYog3vc= -github.com/aws/aws-sdk-go-v2/service/sts v1.32.0/go.mod h1:dKnu7M4MAS2SDlng1ytxd03H+y0LoUfEQ5E2VaaSw/4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 h1:4FMHqLfk0efmTqhXVRL5xYRqlEBNBiRI7N6w4jsEdd4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2/go.mod h1:LWoqeWlK9OZeJxsROW2RqrSPvQHKTpp69r/iDjwsSaw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt3E41huP+GvQZJD38WLsgVp4iOtAjg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2/go.mod h1:/niFCtmuQNxqx9v8WAPq5qh7EH25U4BF6tjoyq9bObM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.66.1 h1:MkQ4unegQEStiQYmfFj+Aq5uTp265ncSmm0XTQwDwi0= +github.com/aws/aws-sdk-go-v2/service/s3 v1.66.1/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo= github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -96,8 +96,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4= -github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4= +github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= +github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -148,8 +148,8 @@ github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDP github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= -github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0= -github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -197,25 +197,25 @@ go.einride.tech/aip v0.68.0 h1:4seM66oLzTpz50u4K1zlJyOXQ3tCzcJN7I22tKkjipw= go.einride.tech/aip v0.68.0/go.mod h1:7y9FF8VtPWqpxuAxl0KQWqaULxW4zFIesD6zF5RIHHg= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0 h1:hCq2hNMwsegUvPzI7sPOvtO9cqyy5GbWt/Ybp2xrx8Q= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0/go.mod h1:LqaApwGx/oUmzsbqxkzuBvyoPpkxk3JQWnqfVrJ3wCA= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 h1:ZIg3ZT/aQ7AfKqdwp7ECpOK6vHqquXXuyTjIO8ZdmPs= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0/go.mod h1:DQAwmETtZV00skUwgD6+0U89g80NKsJE3DCKeLLPQMI= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0 h1:yMkBS9yViCc7U7yeLzJPM2XizlfdVvBRSmsQDWu6qc0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0/go.mod h1:n8MR6/liuGB5EmTETUBeU5ZgqMOlqKRxUaqPQBOANZ8= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6 h1:1wqE9dj9NpSm04INVsJhhEUzhuDVjbcyKH91sVyPATw= -golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -266,19 +266,19 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= -google.golang.org/api v0.199.0 h1:aWUXClp+VFJmqE0JPvpZOK3LDQMyFKYIow4etYd9qxs= -google.golang.org/api v0.199.0/go.mod h1:ohG4qSztDJmZdjK/Ar6MhbAmb/Rpi4JHOqagsh90K28= +google.golang.org/api v0.203.0 h1:SrEeuwU3S11Wlscsn+LA1kb/Y5xT8uggJSkIhD08NAU= +google.golang.org/api v0.203.0/go.mod h1:BuOVyCSYEPwJb3npWvDnNmFI92f3GeRnHNkETneT3SI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240930140551-af27646dc61f h1:mCJ6SGikSxVlt9scCayUl2dMq0msUgmBArqRY6umieI= -google.golang.org/genproto v0.0.0-20240930140551-af27646dc61f/go.mod h1:xtVODtPkMQRUZ4kqOTgp6JrXQrPevvfCSdk4mJtHUbM= -google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f h1:jTm13A2itBi3La6yTGqn8bVSrc3ZZ1r8ENHlIXBfnRA= -google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f/go.mod h1:CLGoBuH1VHxAUXVPP8FfPwPEVJB6lz3URE5mY2SuayE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240930140551-af27646dc61f h1:cUMEy+8oS78BWIH9OWazBkzbr090Od9tWBNtZHkOhf0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240930140551-af27646dc61f/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto v0.0.0-20241021214115-324edc3d5d38 h1:Q3nlH8iSQSRUwOskjbcSMcF2jiYMNiQYZ0c2KEJLKKU= +google.golang.org/genproto v0.0.0-20241021214115-324edc3d5d38/go.mod h1:xBI+tzfqGGN2JBeSebfKXFSdBpWVQ7sLW40PTupVRm4= +google.golang.org/genproto/googleapis/api v0.0.0-20241021214115-324edc3d5d38 h1:2oV8dfuIkM1Ti7DwXc0BJfnwr9csz4TDXI9EmiI+Rbw= +google.golang.org/genproto/googleapis/api v0.0.0-20241021214115-324edc3d5d38/go.mod h1:vuAjtvlwkDKF6L1GQ0SokiRLCGFfeBUXWr/aFFkHACc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -295,8 +295,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/flow/activities/flowable.go b/flow/activities/flowable.go index df78cc3dc..cacc0ba88 100644 --- a/flow/activities/flowable.go +++ b/flow/activities/flowable.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "time" + "github.com/jackc/pgerrcode" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" lua "github.com/yuin/gopher-lua" @@ -627,10 +628,12 @@ func (a *FlowableActivity) DropFlowSource(ctx context.Context, req *protos.DropF } defer connectors.CloseConnector(ctx, srcConn) - err = srcConn.PullFlowCleanup(ctx, req.FlowJobName) - if err != nil { + if err := srcConn.PullFlowCleanup(ctx, req.FlowJobName); err != nil { pullCleanupErr := fmt.Errorf("[DropFlowSource] failed to clean up source: %w", err) - a.Alerter.LogFlowError(ctx, req.FlowJobName, pullCleanupErr) + if !shared.IsSQLStateError(err, pgerrcode.ObjectInUse) { + // don't alert when PID active + a.Alerter.LogFlowError(ctx, req.FlowJobName, pullCleanupErr) + } return pullCleanupErr } @@ -647,8 +650,7 @@ func (a *FlowableActivity) DropFlowDestination(ctx context.Context, req *protos. } defer connectors.CloseConnector(ctx, dstConn) - err = dstConn.SyncFlowCleanup(ctx, req.FlowJobName) - if err != nil { + if err := dstConn.SyncFlowCleanup(ctx, req.FlowJobName); err != nil { syncFlowCleanupErr := fmt.Errorf("[DropFlowDestination] failed to clean up destination: %w", err) a.Alerter.LogFlowError(ctx, req.FlowJobName, syncFlowCleanupErr) return syncFlowCleanupErr @@ -661,7 +663,7 @@ func (a *FlowableActivity) SendWALHeartbeat(ctx context.Context) error { logger := activity.GetLogger(ctx) walHeartbeatEnabled, err := peerdbenv.PeerDBEnableWALHeartbeat(ctx, nil) if err != nil { - logger.Warn("unable to fetch wal heartbeat config. Skipping walheartbeat send.", slog.Any("error", err)) + logger.Warn("unable to fetch wal heartbeat config, skipping wal heartbeat send", slog.Any("error", err)) return err } if !walHeartbeatEnabled { @@ -670,13 +672,13 @@ func (a *FlowableActivity) SendWALHeartbeat(ctx context.Context) error { } walHeartbeatStatement, err := peerdbenv.PeerDBWALHeartbeatQuery(ctx, nil) if err != nil { - logger.Warn("unable to fetch wal heartbeat config. Skipping walheartbeat send.", slog.Any("error", err)) + logger.Warn("unable to fetch wal heartbeat config, skipping wal heartbeat send", slog.Any("error", err)) return err } pgPeers, err := a.getPostgresPeerConfigs(ctx) if err != nil { - logger.Warn("unable to fetch peers. Skipping walheartbeat send.", slog.Any("error", err)) + logger.Warn("unable to fetch peers, skipping wal heartbeat send", slog.Any("error", err)) return err } @@ -691,15 +693,15 @@ func (a *FlowableActivity) SendWALHeartbeat(ctx context.Context) error { pgConfig := pgPeer.GetPostgresConfig() pgConn, peerErr := connpostgres.NewPostgresConnector(ctx, nil, pgConfig) if peerErr != nil { - logger.Error(fmt.Sprintf("error creating connector for postgres peer %s with host %s: %v", - pgPeer.Name, pgConfig.Host, peerErr)) + logger.Error("error creating connector for postgres peer", + slog.String("peer", pgPeer.Name), slog.String("host", pgConfig.Host), slog.Any("error", err)) return } defer pgConn.Close() if cmdErr := pgConn.ExecuteCommand(ctx, walHeartbeatStatement); cmdErr != nil { - logger.Warn(fmt.Sprintf("could not send walheartbeat to peer %s: %v", pgPeer.Name, cmdErr)) + logger.Warn(fmt.Sprintf("could not send wal heartbeat to peer %s: %v", pgPeer.Name, cmdErr)) } - logger.Info("sent walheartbeat", slog.String("peer", pgPeer.Name)) + logger.Info("sent wal heartbeat", slog.String("peer", pgPeer.Name)) }() } diff --git a/flow/activities/flowable_core.go b/flow/activities/flowable_core.go index aabf8776c..66040396c 100644 --- a/flow/activities/flowable_core.go +++ b/flow/activities/flowable_core.go @@ -244,14 +244,12 @@ func syncCore[TPull connectors.CDCPullConnectorCore, TSync connectors.CDCSyncCon } syncBatchID += 1 - err = monitoring.AddCDCBatchForFlow(errCtx, a.CatalogPool, flowName, - monitoring.CDCBatchInfo{ - BatchID: syncBatchID, - RowsInBatch: 0, - BatchEndlSN: 0, - StartTime: startTime, - }) - if err != nil { + if err := monitoring.AddCDCBatchForFlow(errCtx, a.CatalogPool, flowName, monitoring.CDCBatchInfo{ + BatchID: syncBatchID, + RowsInBatch: 0, + BatchEndlSN: 0, + StartTime: startTime, + }); err != nil { a.Alerter.LogFlowError(ctx, flowName, err) return err } @@ -296,35 +294,24 @@ func syncCore[TPull connectors.CDCPullConnectorCore, TSync connectors.CDCSyncCon lastCheckpoint := recordBatchSync.GetLastCheckpoint() srcConn.UpdateReplStateLastOffset(lastCheckpoint) - err = monitoring.UpdateNumRowsAndEndLSNForCDCBatch( - ctx, - a.CatalogPool, - flowName, - res.CurrentSyncBatchID, - uint32(numRecords), - lastCheckpoint, - ) - if err != nil { + if err := monitoring.UpdateNumRowsAndEndLSNForCDCBatch( + ctx, a.CatalogPool, flowName, res.CurrentSyncBatchID, uint32(numRecords), lastCheckpoint, + ); err != nil { a.Alerter.LogFlowError(ctx, flowName, err) return nil, err } - err = monitoring.UpdateLatestLSNAtTargetForCDCFlow(ctx, a.CatalogPool, flowName, lastCheckpoint) - if err != nil { + if err := monitoring.UpdateLatestLSNAtTargetForCDCFlow(ctx, a.CatalogPool, flowName, lastCheckpoint); err != nil { a.Alerter.LogFlowError(ctx, flowName, err) return nil, err } if res.TableNameRowsMapping != nil { - err = monitoring.AddCDCBatchTablesForFlow(ctx, a.CatalogPool, flowName, - res.CurrentSyncBatchID, res.TableNameRowsMapping) - if err != nil { + if err := monitoring.AddCDCBatchTablesForFlow( + ctx, a.CatalogPool, flowName, res.CurrentSyncBatchID, res.TableNameRowsMapping, + ); err != nil { return nil, err } } - if err != nil { - a.Alerter.LogFlowError(ctx, flowName, err) - return nil, err - } pushedRecordsWithCount := fmt.Sprintf("pushed %d records", numRecords) activity.RecordHeartbeat(ctx, pushedRecordsWithCount) @@ -412,8 +399,7 @@ func replicateQRepPartition[TRead any, TWrite any, TSync connectors.QRepSyncConn return nil } - err = monitoring.UpdateStartTimeForPartition(ctx, a.CatalogPool, runUUID, partition, time.Now()) - if err != nil { + if err := monitoring.UpdateStartTimeForPartition(ctx, a.CatalogPool, runUUID, partition, time.Now()); err != nil { a.Alerter.LogFlowError(ctx, config.FlowJobName, err) return fmt.Errorf("failed to update start time for partition: %w", err) } @@ -436,9 +422,9 @@ func replicateQRepPartition[TRead any, TWrite any, TSync connectors.QRepSyncConn return fmt.Errorf("failed to pull records: %w", err) } numRecords := int64(tmp) - err = monitoring.UpdatePullEndTimeAndRowsForPartition(errCtx, - a.CatalogPool, runUUID, partition, numRecords) - if err != nil { + if err := monitoring.UpdatePullEndTimeAndRowsForPartition( + errCtx, a.CatalogPool, runUUID, partition, numRecords, + ); err != nil { logger.Error(err.Error()) } return nil @@ -513,7 +499,7 @@ func replicateXminPartition[TRead any, TWrite any, TSync connectors.QRepSyncConn numRecords, currentSnapshotXmin, pullErr = pullRecords(srcConn, ctx, config, partition, stream) if pullErr != nil { a.Alerter.LogFlowError(ctx, config.FlowJobName, pullErr) - logger.Warn(fmt.Sprintf("[xmin] failed to pull recordS: %v", pullErr)) + logger.Warn(fmt.Sprintf("[xmin] failed to pull records: %v", pullErr)) return pullErr } diff --git a/flow/alerting/alerting.go b/flow/alerting/alerting.go index 8d4163b08..e9df410f9 100644 --- a/flow/alerting/alerting.go +++ b/flow/alerting/alerting.go @@ -5,17 +5,21 @@ import ( "encoding/json" "errors" "fmt" + "io" "log/slog" + "net" "slices" "strings" "time" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" + "go.temporal.io/sdk/log" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/peerdbenv" + "github.com/PeerDB-io/peer-flow/shared" "github.com/PeerDB-io/peer-flow/shared/telemetry" ) @@ -40,12 +44,7 @@ type AlertKeys struct { func (a *Alerter) registerSendersFromPool(ctx context.Context) ([]AlertSenderConfig, error) { rows, err := a.CatalogPool.Query(ctx, - `SELECT - id, - service_type, - service_config, - enc_key_id, - alert_for_mirrors + `SELECT id, service_type, service_config, enc_key_id, alert_for_mirrors FROM peerdb_stats.alerting_config`) if err != nil { return nil, fmt.Errorf("failed to read alerter config from catalog: %w", err) @@ -127,7 +126,7 @@ func NewAlerter(ctx context.Context, catalogPool *pgxpool.Pool) *Alerter { snsMessageSender, err = telemetry.NewSNSMessageSenderWithNewClient(ctx, &telemetry.SNSMessageSenderConfig{ Topic: snsTopic, }) - logger.LoggerFromCtx(ctx).Info("Successfully registered sns telemetry sender") + shared.LoggerFromCtx(ctx).Info("Successfully registered sns telemetry sender") if err != nil { panic(fmt.Sprintf("unable to setup telemetry is nil for Alerter %+v", err)) } @@ -142,7 +141,7 @@ func NewAlerter(ctx context.Context, catalogPool *pgxpool.Pool) *Alerter { URL: incidentIoURL, Token: incidentIoAuth, }) - logger.LoggerFromCtx(ctx).Info("Successfully registered incident.io telemetry sender") + shared.LoggerFromCtx(ctx).Info("Successfully registered incident.io telemetry sender") if err != nil { panic(fmt.Sprintf("unable to setup incident.io telemetry is nil for Alerter %+v", err)) } @@ -158,7 +157,7 @@ func NewAlerter(ctx context.Context, catalogPool *pgxpool.Pool) *Alerter { func (a *Alerter) AlertIfSlotLag(ctx context.Context, alertKeys *AlertKeys, slotInfo *protos.SlotInfo) { alertSenderConfigs, err := a.registerSendersFromPool(ctx) if err != nil { - logger.LoggerFromCtx(ctx).Warn("failed to set alert senders", slog.Any("error", err)) + shared.LoggerFromCtx(ctx).Warn("failed to set alert senders", slog.Any("error", err)) return } @@ -169,7 +168,7 @@ func (a *Alerter) AlertIfSlotLag(ctx context.Context, alertKeys *AlertKeys, slot defaultSlotLagMBAlertThreshold, err := peerdbenv.PeerDBSlotLagMBAlertThreshold(ctx, nil) if err != nil { - logger.LoggerFromCtx(ctx).Warn("failed to get slot lag alert threshold from catalog", slog.Any("error", err)) + shared.LoggerFromCtx(ctx).Warn("failed to get slot lag alert threshold from catalog", slog.Any("error", err)) return } @@ -222,7 +221,7 @@ func (a *Alerter) AlertIfOpenConnections(ctx context.Context, alertKeys *AlertKe ) { alertSenderConfigs, err := a.registerSendersFromPool(ctx) if err != nil { - logger.LoggerFromCtx(ctx).Warn("failed to set alert senders", slog.Any("error", err)) + shared.LoggerFromCtx(ctx).Warn("failed to set alert senders", slog.Any("error", err)) return } @@ -234,7 +233,7 @@ func (a *Alerter) AlertIfOpenConnections(ctx context.Context, alertKeys *AlertKe // same as with slot lag, use lowest threshold for catalog defaultOpenConnectionsThreshold, err := peerdbenv.PeerDBOpenConnectionsAlertThreshold(ctx, nil) if err != nil { - logger.LoggerFromCtx(ctx).Warn("failed to get open connections alert threshold from catalog", slog.Any("error", err)) + shared.LoggerFromCtx(ctx).Warn("failed to get open connections alert threshold from catalog", slog.Any("error", err)) return } lowestOpenConnectionsThreshold := defaultOpenConnectionsThreshold @@ -274,22 +273,20 @@ func (a *Alerter) AlertIfOpenConnections(ctx context.Context, alertKeys *AlertKe } } -func (a *Alerter) AlertIfTooLongSinceLastNormalize(ctx context.Context, alertKeys *AlertKeys, - intervalSinceLastNormalize time.Duration, -) { +func (a *Alerter) AlertIfTooLongSinceLastNormalize(ctx context.Context, alertKeys *AlertKeys, intervalSinceLastNormalize time.Duration) { intervalSinceLastNormalizeThreshold, err := peerdbenv.PeerDBIntervalSinceLastNormalizeThresholdMinutes(ctx, nil) if err != nil { - logger.LoggerFromCtx(ctx). + shared.LoggerFromCtx(ctx). Warn("failed to get interval since last normalize threshold from catalog", slog.Any("error", err)) } if intervalSinceLastNormalizeThreshold == 0 { - logger.LoggerFromCtx(ctx).Info("Alerting disabled via environment variable, returning") + shared.LoggerFromCtx(ctx).Info("Alerting disabled via environment variable, returning") return } alertSenderConfigs, err := a.registerSendersFromPool(ctx) if err != nil { - logger.LoggerFromCtx(ctx).Warn("failed to set alert senders", slog.Any("error", err)) + shared.LoggerFromCtx(ctx).Warn("failed to set alert senders", slog.Any("error", err)) return } @@ -318,10 +315,8 @@ func (a *Alerter) AlertIfTooLongSinceLastNormalize(ctx context.Context, alertKey } func (a *Alerter) alertToProvider(ctx context.Context, alertSenderConfig AlertSenderConfig, alertKey string, alertMessage string) { - err := alertSenderConfig.Sender.sendAlert(ctx, alertKey, alertMessage) - if err != nil { - logger.LoggerFromCtx(ctx).Warn("failed to send alert", slog.Any("error", err)) - return + if err := alertSenderConfig.Sender.sendAlert(ctx, alertKey, alertMessage); err != nil { + shared.LoggerFromCtx(ctx).Warn("failed to send alert", slog.Any("error", err)) } } @@ -329,13 +324,14 @@ func (a *Alerter) alertToProvider(ctx context.Context, alertSenderConfig AlertSe // in the past X minutes, where X is configurable and defaults to 15 minutes // returns true if alert added to catalog, so proceed with processing alerts to slack func (a *Alerter) checkAndAddAlertToCatalog(ctx context.Context, alertConfigId int64, alertKey string, alertMessage string) bool { + logger := shared.LoggerFromCtx(ctx) dur, err := peerdbenv.PeerDBAlertingGapMinutesAsDuration(ctx, nil) if err != nil { - logger.LoggerFromCtx(ctx).Warn("failed to get alerting gap duration from catalog", slog.Any("error", err)) + logger.Warn("failed to get alerting gap duration from catalog", slog.Any("error", err)) return false } if dur == 0 { - logger.LoggerFromCtx(ctx).Warn("Alerting disabled via environment variable, returning") + logger.Warn("Alerting disabled via environment variable, returning") return false } @@ -344,53 +340,56 @@ func (a *Alerter) checkAndAddAlertToCatalog(ctx context.Context, alertConfigId i ORDER BY created_timestamp DESC LIMIT 1`, alertKey, alertConfigId) var createdTimestamp time.Time - err = row.Scan(&createdTimestamp) - if err != nil && err != pgx.ErrNoRows { - logger.LoggerFromCtx(ctx).Warn("failed to send alert", slog.Any("err", err)) + if err := row.Scan(&createdTimestamp); err != nil && err != pgx.ErrNoRows { + shared.LoggerFromCtx(ctx).Warn("failed to send alert", slog.Any("err", err)) return false } if time.Since(createdTimestamp) >= dur { - _, err = a.CatalogPool.Exec(ctx, + if _, err := a.CatalogPool.Exec(ctx, "INSERT INTO peerdb_stats.alerts_v1(alert_key,alert_message,alert_config_id) VALUES($1,$2,$3)", - alertKey, alertMessage, alertConfigId) - if err != nil { - logger.LoggerFromCtx(ctx).Warn("failed to insert alert", slog.Any("error", err)) + alertKey, alertMessage, alertConfigId, + ); err != nil { + shared.LoggerFromCtx(ctx).Warn("failed to insert alert", slog.Any("error", err)) return false } return true } - logger.LoggerFromCtx(ctx).Info( - fmt.Sprintf("Skipped sending alerts: last alert was sent at %s, which was >=%s ago", - createdTimestamp.String(), dur.String())) + logger.Info(fmt.Sprintf("Skipped sending alerts: last alert was sent at %s, which was >=%s ago", createdTimestamp.String(), dur.String())) return false } -func (a *Alerter) sendTelemetryMessage(ctx context.Context, flowName string, more string, level telemetry.Level) { +func (a *Alerter) sendTelemetryMessage( + ctx context.Context, + logger log.Logger, + flowName string, + more string, + level telemetry.Level, + tags ...string, +) { details := fmt.Sprintf("[%s] %s", flowName, more) attributes := telemetry.Attributes{ Level: level, DeploymentUID: peerdbenv.PeerDBDeploymentUID(), - Tags: []string{flowName, peerdbenv.PeerDBDeploymentUID()}, + Tags: append([]string{flowName, peerdbenv.PeerDBDeploymentUID()}, tags...), Type: flowName, } if a.snsTelemetrySender != nil { - _, err := a.snsTelemetrySender.SendMessage(ctx, details, details, attributes) - if err != nil { - logger.LoggerFromCtx(ctx).Warn("failed to send message to snsTelemetrySender", slog.Any("error", err)) - return + if status, err := a.snsTelemetrySender.SendMessage(ctx, details, details, attributes); err != nil { + logger.Warn("failed to send message to snsTelemetrySender", slog.Any("error", err)) + } else { + logger.Info("received status from snsTelemetrySender", slog.String("status", status)) } } if a.incidentIoTelemetrySender != nil { - status, err := a.incidentIoTelemetrySender.SendMessage(ctx, details, details, attributes) - if err != nil { - logger.LoggerFromCtx(ctx).Warn("failed to send message to incidentIoTelemetrySender", slog.Any("error", err)) - return + if status, err := a.incidentIoTelemetrySender.SendMessage(ctx, details, details, attributes); err != nil { + logger.Warn("failed to send message to incidentIoTelemetrySender", slog.Any("error", err)) + } else { + logger.Info("received status from incident.io", slog.String("status", status)) } - logger.LoggerFromCtx(ctx).Info("received status from incident.io", slog.String("status", *status)) } } @@ -412,36 +411,50 @@ func (a *Alerter) LogNonFlowCritical(ctx context.Context, eventType telemetry.Ev } func (a *Alerter) LogNonFlowEvent(ctx context.Context, eventType telemetry.EventType, key string, message string, level telemetry.Level) { - a.sendTelemetryMessage(ctx, string(eventType)+":"+key, message, level) + logger := shared.LoggerFromCtx(ctx) + a.sendTelemetryMessage(ctx, logger, string(eventType)+":"+key, message, level) } func (a *Alerter) LogFlowError(ctx context.Context, flowName string, err error) { - logger := logger.LoggerFromCtx(ctx) errorWithStack := fmt.Sprintf("%+v", err) + logger := shared.LoggerFromCtx(ctx) logger.Error(err.Error(), slog.Any("stack", errorWithStack)) - _, err = a.CatalogPool.Exec(ctx, - "INSERT INTO peerdb_stats.flow_errors(flow_name,error_message,error_type) VALUES($1,$2,$3)", - flowName, errorWithStack, "error") - if err != nil { + if _, err := a.CatalogPool.Exec( + ctx, "INSERT INTO peerdb_stats.flow_errors(flow_name,error_message,error_type) VALUES($1,$2,$3)", + flowName, errorWithStack, "error", + ); err != nil { logger.Warn("failed to insert flow error", slog.Any("error", err)) return } - a.sendTelemetryMessage(ctx, flowName, errorWithStack, telemetry.ERROR) + var tags []string + if errors.Is(err, context.Canceled) { + tags = append(tags, "err:Canceled") + } + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + tags = append(tags, "err:EOF") + } + if errors.Is(err, net.ErrClosed) { + tags = append(tags, "err:Closed") + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + tags = append(tags, "pgcode:"+pgErr.Code) + } + a.sendTelemetryMessage(ctx, logger, flowName, errorWithStack, telemetry.ERROR, tags...) } func (a *Alerter) LogFlowEvent(ctx context.Context, flowName string, info string) { - logger.LoggerFromCtx(ctx).Info(info) - a.sendTelemetryMessage(ctx, flowName, info, telemetry.INFO) + logger := shared.LoggerFromCtx(ctx) + logger.Info(info) + a.sendTelemetryMessage(ctx, logger, flowName, info, telemetry.INFO) } func (a *Alerter) LogFlowInfo(ctx context.Context, flowName string, info string) { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) logger.Info(info) - _, err := a.CatalogPool.Exec(ctx, - "INSERT INTO peerdb_stats.flow_errors(flow_name,error_message,error_type) VALUES($1,$2,$3)", - flowName, info, "info") - if err != nil { + if _, err := a.CatalogPool.Exec( + ctx, "INSERT INTO peerdb_stats.flow_errors(flow_name,error_message,error_type) VALUES($1,$2,$3)", flowName, info, "info", + ); err != nil { logger.Warn("failed to insert flow info", slog.Any("error", err)) - return } } diff --git a/flow/alerting/email_alert_sender.go b/flow/alerting/email_alert_sender.go index a0b96e4bf..cc8acd717 100644 --- a/flow/alerting/email_alert_sender.go +++ b/flow/alerting/email_alert_sender.go @@ -8,8 +8,8 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ses" "github.com/aws/aws-sdk-go-v2/service/ses/types" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/peerdbenv" + "github.com/PeerDB-io/peer-flow/shared" "github.com/PeerDB-io/peer-flow/shared/aws_common" ) @@ -66,7 +66,7 @@ func (e *EmailAlertSender) sendAlert(ctx context.Context, alertTitle string, ale }, }) if err != nil { - logger.LoggerFromCtx(ctx).Warn(fmt.Sprintf( + shared.LoggerFromCtx(ctx).Warn(fmt.Sprintf( "Error sending email alert from %v to %s subject=[%s], body=[%s], configurationSet=%s, replyToAddresses=[%v]", e.sourceEmail, e.emailAddresses, alertTitle, alertMessage, e.configurationSetName, e.replyToAddresses)) return err diff --git a/flow/cmd/api.go b/flow/cmd/api.go index 04b7eb80d..58e6beac0 100644 --- a/flow/cmd/api.go +++ b/flow/cmd/api.go @@ -25,7 +25,6 @@ import ( "github.com/PeerDB-io/peer-flow/auth" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" peerflow "github.com/PeerDB-io/peer-flow/workflows" @@ -189,7 +188,7 @@ func APIMain(ctx context.Context, args *APIServerParams) error { clientOptions := client.Options{ HostPort: args.TemporalHostPort, Namespace: args.TemporalNamespace, - Logger: slog.New(logger.NewHandler(slog.NewJSONHandler(os.Stdout, nil))), + Logger: slog.New(shared.NewSlogHandler(slog.NewJSONHandler(os.Stdout, nil))), } if peerdbenv.PeerDBTemporalEnableCertAuth() { diff --git a/flow/cmd/peer_data.go b/flow/cmd/peer_data.go index 3c23fb5f1..9faf61c39 100644 --- a/flow/cmd/peer_data.go +++ b/flow/cmd/peer_data.go @@ -90,10 +90,12 @@ func (h *FlowRequestHandler) GetPeerInfo( var version string versionConnector, err := connectors.GetAs[connectors.GetVersionConnector](ctx, nil, peer) - if err != nil && !errors.Is(err, errors.ErrUnsupported) { - slog.Error("failed to get version connector", slog.Any("error", err)) - } - if versionConnector != nil { + if err != nil { + if !errors.Is(err, errors.ErrUnsupported) { + slog.Error("failed to get version connector", slog.Any("error", err)) + } + } else { + defer connectors.CloseConnector(ctx, versionConnector) version, err = versionConnector.GetVersion(ctx) if err != nil { slog.Error("failed to get version", slog.Any("error", err)) @@ -359,7 +361,8 @@ func (h *FlowRequestHandler) GetSlotLagHistory( ctx context.Context, req *protos.GetSlotLagHistoryRequest, ) (*protos.GetSlotLagHistoryResponse, error) { - rows, err := h.pool.Query(ctx, `select updated_at, slot_size + rows, err := h.pool.Query(ctx, `select updated_at, slot_size, + coalesce(redo_lsn,''), coalesce(restart_lsn,''), coalesce(confirmed_flush_lsn,'') from peerdb_stats.peer_slot_size where slot_size is not null and peer_name = $1 @@ -372,12 +375,18 @@ func (h *FlowRequestHandler) GetSlotLagHistory( points, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (*protos.SlotLagPoint, error) { var updatedAt time.Time var slotSize int64 - if err := row.Scan(&updatedAt, &slotSize); err != nil { + var redoLSN string + var restartLSN string + var confirmedFlushLSN string + if err := row.Scan(&updatedAt, &slotSize, &redoLSN, &restartLSN, &confirmedFlushLSN); err != nil { return nil, err } return &protos.SlotLagPoint{ - UpdatedAt: float64(updatedAt.UnixMilli()), - SlotSize: float64(slotSize) / 1000.0, + Time: float64(updatedAt.UnixMilli()), + Size: float64(slotSize) / 1000.0, + RedoLSN: redoLSN, + RestartLSN: restartLSN, + ConfirmedLSN: confirmedFlushLSN, }, nil }) if err != nil { diff --git a/flow/cmd/snapshot_worker.go b/flow/cmd/snapshot_worker.go index f32a39cda..5c274a9d4 100644 --- a/flow/cmd/snapshot_worker.go +++ b/flow/cmd/snapshot_worker.go @@ -12,7 +12,6 @@ import ( "github.com/PeerDB-io/peer-flow/activities" "github.com/PeerDB-io/peer-flow/alerting" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" peerflow "github.com/PeerDB-io/peer-flow/workflows" @@ -27,7 +26,7 @@ func SnapshotWorkerMain(opts *SnapshotWorkerOptions) (client.Client, worker.Work clientOptions := client.Options{ HostPort: opts.TemporalHostPort, Namespace: opts.TemporalNamespace, - Logger: slog.New(logger.NewHandler(slog.NewJSONHandler(os.Stdout, nil))), + Logger: slog.New(shared.NewSlogHandler(slog.NewJSONHandler(os.Stdout, nil))), } if peerdbenv.PeerDBTemporalEnableCertAuth() { diff --git a/flow/cmd/validate_peer.go b/flow/cmd/validate_peer.go index 2a88132f7..44b675ed9 100644 --- a/flow/cmd/validate_peer.go +++ b/flow/cmd/validate_peer.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "time" "github.com/PeerDB-io/peer-flow/connectors" connpostgres "github.com/PeerDB-io/peer-flow/connectors/postgres" @@ -16,6 +17,8 @@ func (h *FlowRequestHandler) ValidatePeer( ctx context.Context, req *protos.ValidatePeerRequest, ) (*protos.ValidatePeerResponse, error) { + ctx, cancelCtx := context.WithTimeout(ctx, time.Minute) + defer cancelCtx() if req.Peer == nil { return &protos.ValidatePeerResponse{ Status: protos.ValidatePeerStatus_INVALID, diff --git a/flow/cmd/worker.go b/flow/cmd/worker.go index 4521ba3ca..bf2809d10 100644 --- a/flow/cmd/worker.go +++ b/flow/cmd/worker.go @@ -15,7 +15,6 @@ import ( "github.com/PeerDB-io/peer-flow/activities" "github.com/PeerDB-io/peer-flow/alerting" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/otel_metrics" "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" @@ -87,7 +86,7 @@ func WorkerSetup(opts *WorkerSetupOptions) (*workerSetupResponse, error) { clientOptions := client.Options{ HostPort: opts.TemporalHostPort, Namespace: opts.TemporalNamespace, - Logger: slog.New(logger.NewHandler(slog.NewJSONHandler(os.Stdout, nil))), + Logger: slog.New(shared.NewSlogHandler(slog.NewJSONHandler(os.Stdout, nil))), } if peerdbenv.PeerDBTemporalEnableCertAuth() { diff --git a/flow/connectors/bigquery/bigquery.go b/flow/connectors/bigquery/bigquery.go index 3b113410c..f990b2f19 100644 --- a/flow/connectors/bigquery/bigquery.go +++ b/flow/connectors/bigquery/bigquery.go @@ -19,7 +19,6 @@ import ( metadataStore "github.com/PeerDB-io/peer-flow/connectors/external_metadata" "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" @@ -106,7 +105,7 @@ func (c *BigQueryConnector) ValidateCheck(ctx context.Context) error { } func NewBigQueryConnector(ctx context.Context, config *protos.BigqueryConfig) (*BigQueryConnector, error) { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) bqsa, err := NewBigQueryServiceAccount(config) if err != nil { diff --git a/flow/connectors/clickhouse/clickhouse.go b/flow/connectors/clickhouse/clickhouse.go index 6328708b7..1c7d110e3 100644 --- a/flow/connectors/clickhouse/clickhouse.go +++ b/flow/connectors/clickhouse/clickhouse.go @@ -23,7 +23,6 @@ import ( metadataStore "github.com/PeerDB-io/peer-flow/connectors/external_metadata" "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" ) @@ -129,7 +128,7 @@ func NewClickHouseConnector( env map[string]string, config *protos.ClickhouseConfig, ) (*ClickHouseConnector, error) { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) database, err := Connect(ctx, config) if err != nil { return nil, fmt.Errorf("failed to open connection to ClickHouse peer: %w", err) diff --git a/flow/connectors/clickhouse/normalize.go b/flow/connectors/clickhouse/normalize.go index 6e0bfb971..c971c3f69 100644 --- a/flow/connectors/clickhouse/normalize.go +++ b/flow/connectors/clickhouse/normalize.go @@ -55,6 +55,7 @@ func (c *ClickHouseConnector) SetupNormalizedTable( } normalizedTableCreateSQL, err := generateCreateTableSQLForNormalizedTable( + ctx, config, tableIdentifier, tableSchema, @@ -77,6 +78,7 @@ func getColName(overrides map[string]string, name string) string { } func generateCreateTableSQLForNormalizedTable( + ctx context.Context, config *protos.SetupNormalizedTableBatchInput, tableIdentifier string, tableSchema *protos.TableSchema, @@ -178,6 +180,12 @@ func generateCreateTableSQLForNormalizedTable( stmtBuilder.WriteString(") ") } + if nullable, err := peerdbenv.PeerDBNullable(ctx, config.Env); err != nil { + return "", err + } else if nullable { + stmtBuilder.WriteString(" SETTINGS allow_nullable_key = 1") + } + return stmtBuilder.String(), nil } @@ -332,26 +340,26 @@ func (c *ClickHouseConnector) NormalizeRecords( switch clickHouseType { case "Date32", "Nullable(Date32)": projection.WriteString(fmt.Sprintf( - "toDate32(parseDateTime64BestEffortOrNull(JSONExtractString(_peerdb_data, '%s'))) AS `%s`,", + "toDate32(parseDateTime64BestEffortOrNull(JSONExtractString(_peerdb_data, '%s'),6)) AS `%s`,", colName, dstColName, )) if enablePrimaryUpdate { projectionUpdate.WriteString(fmt.Sprintf( - "toDate32(parseDateTime64BestEffortOrNull(JSONExtractString(_peerdb_match_data, '%s'))) AS `%s`,", + "toDate32(parseDateTime64BestEffortOrNull(JSONExtractString(_peerdb_match_data, '%s'),6)) AS `%s`,", colName, dstColName, )) } case "DateTime64(6)", "Nullable(DateTime64(6))": projection.WriteString(fmt.Sprintf( - "parseDateTime64BestEffortOrNull(JSONExtractString(_peerdb_data, '%s')) AS `%s`,", + "parseDateTime64BestEffortOrNull(JSONExtractString(_peerdb_data, '%s'),6) AS `%s`,", colName, dstColName, )) if enablePrimaryUpdate { projectionUpdate.WriteString(fmt.Sprintf( - "parseDateTime64BestEffortOrNull(JSONExtractString(_peerdb_match_data, '%s')) AS `%s`,", + "parseDateTime64BestEffortOrNull(JSONExtractString(_peerdb_match_data, '%s'),6) AS `%s`,", colName, dstColName, )) @@ -410,8 +418,9 @@ func (c *ClickHouseConnector) NormalizeRecords( } insertIntoSelectQuery := strings.Builder{} - insertIntoSelectQuery.WriteString("INSERT INTO ") + insertIntoSelectQuery.WriteString("INSERT INTO `") insertIntoSelectQuery.WriteString(tbl) + insertIntoSelectQuery.WriteString("` ") insertIntoSelectQuery.WriteString(colSelector.String()) insertIntoSelectQuery.WriteString(selectQuery.String()) diff --git a/flow/connectors/clickhouse/qrep.go b/flow/connectors/clickhouse/qrep.go index 1d5b2c1d7..376d05e7a 100644 --- a/flow/connectors/clickhouse/qrep.go +++ b/flow/connectors/clickhouse/qrep.go @@ -45,7 +45,7 @@ func (c *ClickHouseConnector) SyncQRepRecords( } func (c *ClickHouseConnector) getTableSchema(ctx context.Context, tableName string) ([]driver.ColumnType, error) { - queryString := fmt.Sprintf(`SELECT * FROM %s LIMIT 0`, tableName) + queryString := fmt.Sprintf("SELECT * FROM `%s` LIMIT 0", tableName) rows, err := c.query(ctx, queryString) if err != nil { return nil, fmt.Errorf("failed to execute query: %w", err) diff --git a/flow/connectors/clickhouse/qrep_avro_sync.go b/flow/connectors/clickhouse/qrep_avro_sync.go index b507f448e..fa7f03cf8 100644 --- a/flow/connectors/clickhouse/qrep_avro_sync.go +++ b/flow/connectors/clickhouse/qrep_avro_sync.go @@ -50,7 +50,7 @@ func (s *ClickHouseAvroSyncMethod) CopyStageToDestination(ctx context.Context, a if creds.AWS.SessionToken != "" { sessionTokenPart = fmt.Sprintf(", '%s'", creds.AWS.SessionToken) } - query := fmt.Sprintf("INSERT INTO %s SELECT * FROM s3('%s','%s','%s'%s, 'Avro')", + query := fmt.Sprintf("INSERT INTO `%s` SELECT * FROM s3('%s','%s','%s'%s, 'Avro')", s.config.DestinationTableIdentifier, avroFileUrl, creds.AWS.AccessKeyID, creds.AWS.SecretAccessKey, sessionTokenPart) @@ -145,7 +145,7 @@ func (s *ClickHouseAvroSyncMethod) SyncQRepRecords( if creds.AWS.SessionToken != "" { sessionTokenPart = fmt.Sprintf(", '%s'", creds.AWS.SessionToken) } - query := fmt.Sprintf("INSERT INTO %s(%s) SELECT %s FROM s3('%s','%s','%s'%s, 'Avro')", + query := fmt.Sprintf("INSERT INTO `%s`(%s) SELECT %s FROM s3('%s','%s','%s'%s, 'Avro')", config.DestinationTableIdentifier, selectorStr, selectorStr, avroFileUrl, creds.AWS.AccessKeyID, creds.AWS.SecretAccessKey, sessionTokenPart) diff --git a/flow/connectors/core.go b/flow/connectors/core.go index f55385ff4..8a6bbbc0e 100644 --- a/flow/connectors/core.go +++ b/flow/connectors/core.go @@ -12,7 +12,7 @@ import ( "github.com/PeerDB-io/peer-flow/alerting" connbigquery "github.com/PeerDB-io/peer-flow/connectors/bigquery" connclickhouse "github.com/PeerDB-io/peer-flow/connectors/clickhouse" - connelasticsearch "github.com/PeerDB-io/peer-flow/connectors/connelasticsearch" + connelasticsearch "github.com/PeerDB-io/peer-flow/connectors/elasticsearch" conneventhub "github.com/PeerDB-io/peer-flow/connectors/eventhub" connkafka "github.com/PeerDB-io/peer-flow/connectors/kafka" connmysql "github.com/PeerDB-io/peer-flow/connectors/mysql" @@ -22,10 +22,10 @@ import ( connsnowflake "github.com/PeerDB-io/peer-flow/connectors/snowflake" connsqlserver "github.com/PeerDB-io/peer-flow/connectors/sqlserver" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/otel_metrics/peerdb_gauges" "github.com/PeerDB-io/peer-flow/peerdbenv" + "github.com/PeerDB-io/peer-flow/shared" ) type Connector interface { @@ -433,7 +433,7 @@ func GetByNameAs[T Connector](ctx context.Context, env map[string]string, catalo func CloseConnector(ctx context.Context, conn Connector) { if err := conn.Close(); err != nil { - logger.LoggerFromCtx(ctx).Error("error closing connector", slog.Any("error", err)) + shared.LoggerFromCtx(ctx).Error("error closing connector", slog.Any("error", err)) } } diff --git a/flow/connectors/connelasticsearch/elasticsearch.go b/flow/connectors/elasticsearch/elasticsearch.go similarity index 99% rename from flow/connectors/connelasticsearch/elasticsearch.go rename to flow/connectors/elasticsearch/elasticsearch.go index c1f6b997f..e67516805 100644 --- a/flow/connectors/connelasticsearch/elasticsearch.go +++ b/flow/connectors/elasticsearch/elasticsearch.go @@ -22,7 +22,6 @@ import ( metadataStore "github.com/PeerDB-io/peer-flow/connectors/external_metadata" "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/model/qvalue" "github.com/PeerDB-io/peer-flow/peerdbenv" @@ -71,7 +70,7 @@ func NewElasticsearchConnector(ctx context.Context, return &ElasticsearchConnector{ PostgresMetadata: pgMetadata, client: esClient, - logger: logger.LoggerFromCtx(ctx), + logger: shared.LoggerFromCtx(ctx), }, nil } diff --git a/flow/connectors/connelasticsearch/qrep.go b/flow/connectors/elasticsearch/qrep.go similarity index 100% rename from flow/connectors/connelasticsearch/qrep.go rename to flow/connectors/elasticsearch/qrep.go diff --git a/flow/connectors/eventhub/eventhub.go b/flow/connectors/eventhub/eventhub.go index 8d89cd7da..01982bf71 100644 --- a/flow/connectors/eventhub/eventhub.go +++ b/flow/connectors/eventhub/eventhub.go @@ -15,7 +15,6 @@ import ( metadataStore "github.com/PeerDB-io/peer-flow/connectors/external_metadata" "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/pua" @@ -35,7 +34,7 @@ func NewEventHubConnector( ctx context.Context, config *protos.EventHubGroupConfig, ) (*EventHubConnector, error) { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) defaultAzureCreds, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { logger.Error("failed to get default azure credentials", "error", err) diff --git a/flow/connectors/eventhub/hub_batches.go b/flow/connectors/eventhub/hub_batches.go index 1f70cd8b3..e6c3e0bfa 100644 --- a/flow/connectors/eventhub/hub_batches.go +++ b/flow/connectors/eventhub/hub_batches.go @@ -10,7 +10,6 @@ import ( azeventhubs "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/shared" ) @@ -106,7 +105,7 @@ func (h *HubBatches) sendBatch( return err } - logger.LoggerFromCtx(ctx).Info("sendBatch", + shared.LoggerFromCtx(ctx).Info("sendBatch", slog.Int("events sent", int(events.NumEvents())), slog.String("event hub topic", tblName.ToString())) return nil } @@ -115,7 +114,7 @@ func (h *HubBatches) flushAllBatches( ctx context.Context, flowName string, ) error { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) if h.Len() == 0 { logger.Info("no events to send", slog.String(string(shared.FlowNameKey), flowName)) return nil diff --git a/flow/connectors/eventhub/hubmanager.go b/flow/connectors/eventhub/hubmanager.go index 680aacd15..3e134968d 100644 --- a/flow/connectors/eventhub/hubmanager.go +++ b/flow/connectors/eventhub/hubmanager.go @@ -16,7 +16,7 @@ import ( "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/shared" ) type EventHubManager struct { @@ -65,7 +65,7 @@ func (m *EventHubManager) GetOrCreateHubClient(ctx context.Context, name ScopedE hubTmp := hub.(*azeventhubs.ProducerClient) _, err := hubTmp.GetEventHubProperties(ctx, nil) if err != nil { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) logger.Info( fmt.Sprintf("eventhub %s not reachable. Will re-establish connection and re-create it.", name), slog.Any("error", err)) @@ -111,7 +111,7 @@ func (m *EventHubManager) Close(ctx context.Context) error { hub := value.(*azeventhubs.ProducerClient) err := m.closeProducerClient(ctx, hub) if err != nil { - logger.LoggerFromCtx(ctx).Error(fmt.Sprintf("failed to close eventhub client for %v", name), slog.Any("error", err)) + shared.LoggerFromCtx(ctx).Error(fmt.Sprintf("failed to close eventhub client for %v", name), slog.Any("error", err)) allErrors = errors.Join(allErrors, err) } return true @@ -161,7 +161,7 @@ func (m *EventHubManager) EnsureEventHubExists(ctx context.Context, name ScopedE partitionCount := int64(cfg.PartitionCount) retention := int64(cfg.MessageRetentionInDays) - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) if err != nil { opts := armeventhub.Eventhub{ Properties: &armeventhub.Properties{ diff --git a/flow/connectors/external_metadata/store.go b/flow/connectors/external_metadata/store.go index b586ea305..515b622ee 100644 --- a/flow/connectors/external_metadata/store.go +++ b/flow/connectors/external_metadata/store.go @@ -14,8 +14,8 @@ import ( "github.com/PeerDB-io/peer-flow/connectors/utils/monitoring" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/peerdbenv" + "github.com/PeerDB-io/peer-flow/shared" ) const ( @@ -36,7 +36,7 @@ func NewPostgresMetadata(ctx context.Context) (*PostgresMetadata, error) { return &PostgresMetadata{ pool: pool, - logger: logger.LoggerFromCtx(ctx), + logger: shared.LoggerFromCtx(ctx), }, nil } @@ -171,7 +171,7 @@ func (p *PostgresMetadata) FinishBatch(ctx context.Context, jobName string, sync } func (p *PostgresMetadata) UpdateNormalizeBatchID(ctx context.Context, jobName string, batchID int64) error { - p.logger.Info("updating normalize batch id for job") + p.logger.Info("updating normalize batch id for job", slog.Int64("batchID", batchID)) _, err := p.pool.Exec(ctx, `UPDATE `+lastSyncStateTableName+ ` SET normalize_batch_id=$2 WHERE job_name=$1`, jobName, batchID) @@ -180,9 +180,8 @@ func (p *PostgresMetadata) UpdateNormalizeBatchID(ctx context.Context, jobName s return err } - err = monitoring.UpdateEndTimeForCDCBatch(ctx, p.pool, jobName, batchID) - if err != nil { - p.logger.Error(fmt.Sprintf("failed to update end time for cdc batch - %d", batchID), slog.Any("error", err)) + if err := monitoring.UpdateEndTimeForCDCBatch(ctx, p.pool, jobName, batchID); err != nil { + p.logger.Error("failed to update end time for cdc batch", slog.Int64("batchID", batchID), slog.Any("error", err)) return err } diff --git a/flow/connectors/kafka/kafka.go b/flow/connectors/kafka/kafka.go index 5fa0ae857..ea0805b84 100644 --- a/flow/connectors/kafka/kafka.go +++ b/flow/connectors/kafka/kafka.go @@ -21,7 +21,6 @@ import ( metadataStore "github.com/PeerDB-io/peer-flow/connectors/external_metadata" "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/pua" @@ -68,7 +67,7 @@ func NewKafkaConnector( env map[string]string, config *protos.KafkaConfig, ) (*KafkaConnector, error) { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) optionalOpts := append( make([]kgo.Opt, 0, 7), kgo.SeedBrokers(config.Servers...), diff --git a/flow/connectors/postgres/cdc.go b/flow/connectors/postgres/cdc.go index b21484ca1..91eaf3eba 100644 --- a/flow/connectors/postgres/cdc.go +++ b/flow/connectors/postgres/cdc.go @@ -20,7 +20,6 @@ import ( "github.com/PeerDB-io/peer-flow/connectors/utils" geo "github.com/PeerDB-io/peer-flow/datatypes" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/model/qvalue" "github.com/PeerDB-io/peer-flow/shared" @@ -292,7 +291,7 @@ func PullCdcRecords[Items model.Items]( processor replProcessor[Items], replLock *sync.Mutex, ) error { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) // use only with taking replLock conn := p.replConn.PgConn() sendStandbyAfterReplLock := func(updateType string) error { @@ -451,8 +450,7 @@ func PullCdcRecords[Items model.Items]( } if errMsg, ok := rawMsg.(*pgproto3.ErrorResponse); ok { - logger.Error(fmt.Sprintf("received Postgres WAL error: %+v", errMsg)) - return fmt.Errorf("received Postgres WAL error: %+v", errMsg) + return shared.LogError(logger, fmt.Errorf("received Postgres WAL error: %+v", errMsg)) } msg, ok := rawMsg.(*pgproto3.CopyData) @@ -629,7 +627,7 @@ func processMessage[Items model.Items]( currentClientXlogPos pglogrepl.LSN, processor replProcessor[Items], ) (model.Record[Items], error) { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) logicalMsg, err := pglogrepl.Parse(xld.WALData) if err != nil { return nil, fmt.Errorf("error parsing logical message: %w", err) diff --git a/flow/connectors/postgres/client.go b/flow/connectors/postgres/client.go index 0d163f173..2d480f780 100644 --- a/flow/connectors/postgres/client.go +++ b/flow/connectors/postgres/client.go @@ -346,8 +346,8 @@ func (c *PostgresConnector) CreatePublication( // Create the publication to help filter changes only for the given tables stmt := fmt.Sprintf("CREATE PUBLICATION %s FOR TABLE %s%s", publication, tableNameString, pubViaRootString) if _, err = c.execWithLogging(ctx, stmt); err != nil { - c.logger.Warn(fmt.Sprintf("Error creating publication '%s': %v", publication, err)) - return fmt.Errorf("error creating publication '%s' : %w", publication, err) + c.logger.Warn("error creating publication", slog.String("publication", publication), slog.Any("error", err)) + return fmt.Errorf("error creating publication %s: %w", publication, err) } return nil } diff --git a/flow/connectors/postgres/postgres.go b/flow/connectors/postgres/postgres.go index 45730a87d..ad7ca3951 100644 --- a/flow/connectors/postgres/postgres.go +++ b/flow/connectors/postgres/postgres.go @@ -24,7 +24,6 @@ import ( "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/connectors/utils/monitoring" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/model/qvalue" peerdb_gauges "github.com/PeerDB-io/peer-flow/otel_metrics/peerdb_gauges" @@ -55,7 +54,7 @@ type ReplState struct { } func NewPostgresConnector(ctx context.Context, env map[string]string, pgConfig *protos.PostgresConfig) (*PostgresConnector, error) { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) flowNameInApplicationName, err := peerdbenv.PeerDBApplicationNamePerMirrorName(ctx, nil) if err != nil { logger.Error("Failed to get flow name from application name", slog.Any("error", err)) @@ -132,7 +131,7 @@ func (c *PostgresConnector) CreateReplConn(ctx context.Context) (*pgx.Conn, erro conn, err := c.ssh.NewPostgresConnFromConfig(ctx, replConfig) if err != nil { - logger.LoggerFromCtx(ctx).Error("failed to create replication connection", "error", err) + shared.LoggerFromCtx(ctx).Error("failed to create replication connection", "error", err) return nil, fmt.Errorf("failed to create replication connection: %w", err) } return conn, nil @@ -993,7 +992,7 @@ func (c *PostgresConnector) EnsurePullability( } if !req.CheckConstraints { - logger.LoggerFromCtx(ctx).Info("[no-constraints] ensured pullability table " + tableName) + shared.LoggerFromCtx(ctx).Info("[no-constraints] ensured pullability table " + tableName) continue } @@ -1176,7 +1175,7 @@ func (c *PostgresConnector) HandleSlotInfo( alertKeys *alerting.AlertKeys, slotMetricGauges peerdb_gauges.SlotMetricGauges, ) error { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) slotInfo, err := getSlotInfo(ctx, c.conn, alertKeys.SlotName, c.config.Database) if err != nil { diff --git a/flow/connectors/postgres/qrep.go b/flow/connectors/postgres/qrep.go index 48864b0c3..2a65ec353 100644 --- a/flow/connectors/postgres/qrep.go +++ b/flow/connectors/postgres/qrep.go @@ -20,7 +20,6 @@ import ( "github.com/PeerDB-io/peer-flow/connectors/utils" partition_utils "github.com/PeerDB-io/peer-flow/connectors/utils/partition" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/shared" ) @@ -166,8 +165,7 @@ func (c *PostgresConnector) getNumRowsPartitions( rows, err = tx.Query(ctx, partitionsQuery) } if err != nil { - c.logger.Error(fmt.Sprintf("failed to query for partitions: %v", err)) - return nil, fmt.Errorf("failed to query for partitions: %w", err) + return nil, shared.LogError(c.logger, fmt.Errorf("failed to query for partitions: %w", err)) } defer rows.Close() @@ -240,7 +238,7 @@ func (c *PostgresConnector) getMinMaxValues( } else { minMaxQuery := fmt.Sprintf("SELECT MIN(%[1]s), MAX(%[1]s) FROM %[2]s", quotedWatermarkColumn, parsedWatermarkTable.String()) if err := tx.QueryRow(ctx, minMaxQuery).Scan(&minValue, &maxValue); err != nil { - c.logger.Error(fmt.Sprintf("failed to query [%s] for min value: %v", minMaxQuery, err)) + c.logger.Error("failed to query for min value", slog.String("query", minMaxQuery), slog.Any("error", err)) return nil, nil, fmt.Errorf("failed to query for min value: %w", err) } else if maxValue != nil { switch v := minValue.(type) { @@ -460,7 +458,7 @@ func syncQRepRecords( defer func() { if err := tx.Rollback(context.Background()); err != nil { if err != pgx.ErrTxClosed { - logger.LoggerFromCtx(ctx).Error("failed to rollback transaction tx2", slog.Any("error", err), syncLog) + shared.LoggerFromCtx(ctx).Error("failed to rollback transaction tx2", slog.Any("error", err), syncLog) } } }() @@ -706,7 +704,7 @@ func BuildQuery(logger log.Logger, query string, flowJobName string) (string, er } res := buf.String() - logger.Info("templated query: " + res) + logger.Info("[pg] templated query", slog.String("query", res)) return res, nil } diff --git a/flow/connectors/postgres/ssh_wrapped_pool.go b/flow/connectors/postgres/ssh_wrapped_pool.go index 4e5868ccd..ec90f4667 100644 --- a/flow/connectors/postgres/ssh_wrapped_pool.go +++ b/flow/connectors/postgres/ssh_wrapped_pool.go @@ -13,7 +13,6 @@ import ( "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" ) @@ -36,7 +35,7 @@ func NewSSHTunnel( var err error clientConfig, err = utils.GetSSHClientConfig(sshConfig) if err != nil { - logger.LoggerFromCtx(ctx).Error("Failed to get SSH client config", "error", err) + shared.LoggerFromCtx(ctx).Error("Failed to get SSH client config", "error", err) return nil, err } } @@ -46,7 +45,7 @@ func NewSSHTunnel( sshServer: sshServer, } - err := tunnel.setupSSH(logger.LoggerFromCtx(ctx)) + err := tunnel.setupSSH(shared.LoggerFromCtx(ctx)) if err != nil { return nil, err } @@ -82,7 +81,7 @@ func (tunnel *SSHTunnel) NewPostgresConnFromPostgresConfig( ) (*pgx.Conn, error) { flowNameInApplicationName, err := peerdbenv.PeerDBApplicationNamePerMirrorName(ctx, nil) if err != nil { - logger.LoggerFromCtx(ctx).Error("Failed to get flow name from application name", slog.Any("error", err)) + shared.LoggerFromCtx(ctx).Error("Failed to get flow name from application name", slog.Any("error", err)) } var flowName string @@ -118,7 +117,7 @@ func (tunnel *SSHTunnel) NewPostgresConnFromConfig( } } - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) conn, err := pgx.ConnectConfig(ctx, connConfig) if err != nil { logger.Error("Failed to create pool", slog.Any("error", err)) diff --git a/flow/connectors/postgres/validate.go b/flow/connectors/postgres/validate.go index 97adf80af..2b5729f67 100644 --- a/flow/connectors/postgres/validate.go +++ b/flow/connectors/postgres/validate.go @@ -23,41 +23,57 @@ func (c *PostgresConnector) CheckSourceTables(ctx context.Context, tableArr := make([]string, 0, len(tableNames)) for _, parsedTable := range tableNames { var row pgx.Row - tableArr = append(tableArr, fmt.Sprintf(`(%s::text, %s::text)`, + tableArr = append(tableArr, fmt.Sprintf(`(%s::text,%s::text)`, QuoteLiteral(parsedTable.Schema), QuoteLiteral(parsedTable.Table))) - err := c.conn.QueryRow(ctx, - fmt.Sprintf("SELECT * FROM %s.%s LIMIT 0;", - QuoteIdentifier(parsedTable.Schema), QuoteIdentifier(parsedTable.Table))).Scan(&row) - if err != nil && err != pgx.ErrNoRows { + if err := c.conn.QueryRow(ctx, + fmt.Sprintf("SELECT * FROM %s.%s LIMIT 0", QuoteIdentifier(parsedTable.Schema), QuoteIdentifier(parsedTable.Table)), + ).Scan(&row); err != nil && err != pgx.ErrNoRows { return err } } - tableStr := strings.Join(tableArr, ",") - if pubName != "" && !noCDC { // Check if publication exists - err := c.conn.QueryRow(ctx, "SELECT pubname FROM pg_publication WHERE pubname=$1", pubName).Scan(nil) - if err != nil { + var alltables bool + if err := c.conn.QueryRow(ctx, "SELECT puballtables FROM pg_publication WHERE pubname=$1", pubName).Scan(&alltables); err != nil { if err == pgx.ErrNoRows { return fmt.Errorf("publication does not exist: %s", pubName) } return fmt.Errorf("error while checking for publication existence: %w", err) } - // Check if tables belong to publication - var pubTableCount int - err = c.conn.QueryRow(ctx, fmt.Sprintf(` - with source_table_components (sname, tname) as (values %s) - select COUNT(DISTINCT(schemaname,tablename)) from pg_publication_tables - INNER JOIN source_table_components stc - ON schemaname=stc.sname and tablename=stc.tname where pubname=$1;`, tableStr), pubName).Scan(&pubTableCount) - if err != nil { - return err - } + if !alltables { + // Check if tables belong to publication + tableStr := strings.Join(tableArr, ",") + + rows, err := c.conn.Query( + ctx, + fmt.Sprintf(`select schemaname,tablename + from (values %s) as input(schemaname,tablename) + where not exists ( + select * from pg_publication_tables pub + where pubname=$1 and pub.schemaname=input.schemaname and pub.tablename=input.tablename + )`, tableStr), + pubName, + ) + if err != nil { + return err + } + missing, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (string, error) { + var schema string + var table string + if err := row.Scan(&schema, &table); err != nil { + return "", err + } + return fmt.Sprintf("%s.%s", QuoteIdentifier(schema), QuoteIdentifier(table)), nil + }) + if err != nil { + return err + } - if pubTableCount != len(tableNames) { - return errors.New("not all tables belong to publication") + if len(missing) != 0 { + return errors.New("some tables missing from publication: " + strings.Join(missing, ", ")) + } } } diff --git a/flow/connectors/pubsub/pubsub.go b/flow/connectors/pubsub/pubsub.go index 6860cf3fc..49aed379c 100644 --- a/flow/connectors/pubsub/pubsub.go +++ b/flow/connectors/pubsub/pubsub.go @@ -15,7 +15,6 @@ import ( metadataStore "github.com/PeerDB-io/peer-flow/connectors/external_metadata" "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/pua" @@ -47,7 +46,7 @@ func NewPubSubConnector( return &PubSubConnector{ client: client, PostgresMetadata: pgMetadata, - logger: logger.LoggerFromCtx(ctx), + logger: shared.LoggerFromCtx(ctx), }, nil } diff --git a/flow/connectors/s3/s3.go b/flow/connectors/s3/s3.go index 4289cbca9..eac37cd7c 100644 --- a/flow/connectors/s3/s3.go +++ b/flow/connectors/s3/s3.go @@ -12,8 +12,8 @@ import ( metadataStore "github.com/PeerDB-io/peer-flow/connectors/external_metadata" "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" + "github.com/PeerDB-io/peer-flow/shared" ) type S3Connector struct { @@ -28,7 +28,7 @@ func NewS3Connector( ctx context.Context, config *protos.S3Config, ) (*S3Connector, error) { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) provider, err := utils.GetAWSCredentialsProvider(ctx, "s3", utils.PeerAWSCredentials{ Credentials: aws.Credentials{ diff --git a/flow/connectors/snowflake/client.go b/flow/connectors/snowflake/client.go index 335c5f9fb..72415403e 100644 --- a/flow/connectors/snowflake/client.go +++ b/flow/connectors/snowflake/client.go @@ -13,7 +13,6 @@ import ( peersql "github.com/PeerDB-io/peer-flow/connectors/sql" "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model/qvalue" "github.com/PeerDB-io/peer-flow/shared" ) @@ -56,7 +55,7 @@ func NewSnowflakeClient(ctx context.Context, config *protos.SnowflakeConfig) (*S return nil, fmt.Errorf("failed to open connection to Snowflake peer: %w", err) } - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) genericExecutor := *peersql.NewGenericSQLQueryExecutor( logger, database, snowflakeTypeToQValueKindMap, qvalue.QValueKindToSnowflakeTypeMap) diff --git a/flow/connectors/snowflake/qrep_avro_consolidate.go b/flow/connectors/snowflake/qrep_avro_consolidate.go index fe8ef88f5..547aef27e 100644 --- a/flow/connectors/snowflake/qrep_avro_consolidate.go +++ b/flow/connectors/snowflake/qrep_avro_consolidate.go @@ -9,6 +9,7 @@ import ( "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" + "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" ) @@ -136,8 +137,12 @@ func (s *SnowflakeAvroConsolidateHandler) getCopyTransformation(copyDstTable str s.config.SyncedAtColName, s.config.SoftDeleteColName, ) - return fmt.Sprintf("COPY INTO %s(%s) FROM (SELECT %s FROM @%s) FILE_FORMAT=(TYPE=AVRO), PURGE=TRUE", - copyDstTable, columnsSQL, transformationSQL, s.stage) + onErrorStr := "" + if onError := peerdbenv.GetEnvString("PEERDB_SNOWFLAKE_ON_ERROR", ""); onError != "" { + onErrorStr = ", ON_ERROR=" + onError + } + return fmt.Sprintf("COPY INTO %s(%s) FROM (SELECT %s FROM @%s) FILE_FORMAT=(TYPE=AVRO), PURGE=TRUE%s", + copyDstTable, columnsSQL, transformationSQL, s.stage, onErrorStr) } func (s *SnowflakeAvroConsolidateHandler) handleAppendMode(ctx context.Context) error { diff --git a/flow/connectors/snowflake/snowflake.go b/flow/connectors/snowflake/snowflake.go index 100279943..124a5c65a 100644 --- a/flow/connectors/snowflake/snowflake.go +++ b/flow/connectors/snowflake/snowflake.go @@ -21,7 +21,6 @@ import ( "github.com/PeerDB-io/peer-flow/connectors/utils" numeric "github.com/PeerDB-io/peer-flow/datatypes" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/model/qvalue" "github.com/PeerDB-io/peer-flow/peerdbenv" @@ -126,7 +125,7 @@ func (c *SnowflakeConnector) ValidateCheck(ctx context.Context) error { defer func() { deferErr := tx.Rollback() if deferErr != sql.ErrTxDone && deferErr != nil { - logger.LoggerFromCtx(ctx).Error("error while rolling back transaction for table check", + shared.LoggerFromCtx(ctx).Error("error while rolling back transaction for table check", "error", deferErr) } }() @@ -170,7 +169,7 @@ func NewSnowflakeConnector( ctx context.Context, snowflakeProtoConfig *protos.SnowflakeConfig, ) (*SnowflakeConnector, error) { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) PrivateKeyRSA, err := shared.DecodePKCS8PrivateKey([]byte(snowflakeProtoConfig.PrivateKey), snowflakeProtoConfig.Password) if err != nil { diff --git a/flow/connectors/sqlserver/qrep.go b/flow/connectors/sqlserver/qrep.go index 2dcc7ac1c..a725fd010 100644 --- a/flow/connectors/sqlserver/qrep.go +++ b/flow/connectors/sqlserver/qrep.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "log/slog" "text/template" "github.com/google/uuid" @@ -221,13 +222,11 @@ func BuildQuery(logger log.Logger, query string) (string, error) { } buf := new(bytes.Buffer) - - err = tmpl.Execute(buf, data) - if err != nil { + if err := tmpl.Execute(buf, data); err != nil { return "", err } res := buf.String() - logger.Info("templated query: " + res) + logger.Info("[ss] templated query", slog.String("query", res)) return res, nil } diff --git a/flow/connectors/sqlserver/sqlserver.go b/flow/connectors/sqlserver/sqlserver.go index 505052e0c..7017c8150 100644 --- a/flow/connectors/sqlserver/sqlserver.go +++ b/flow/connectors/sqlserver/sqlserver.go @@ -10,7 +10,7 @@ import ( peersql "github.com/PeerDB-io/peer-flow/connectors/sql" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/shared" ) type SQLServerConnector struct { @@ -31,12 +31,11 @@ func NewSQLServerConnector(ctx context.Context, config *protos.SqlServerConfig) return nil, err } - err = db.PingContext(ctx) - if err != nil { + if err := db.PingContext(ctx); err != nil { return nil, err } - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) genericExecutor := *peersql.NewGenericSQLQueryExecutor( logger, db, sqlServerTypeToQValueKindMap, qValueKindToSQLServerTypeMap) diff --git a/flow/connectors/utils/avro/avro_writer.go b/flow/connectors/utils/avro/avro_writer.go index 04f2bf7e6..9ce4064ed 100644 --- a/flow/connectors/utils/avro/avro_writer.go +++ b/flow/connectors/utils/avro/avro_writer.go @@ -22,7 +22,6 @@ import ( "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/shared" ) @@ -128,7 +127,7 @@ func (p *peerDBOCFWriter) createOCFWriter(w io.Writer) (*goavro.OCFWriter, error } func (p *peerDBOCFWriter) writeRecordsToOCFWriter(ctx context.Context, ocfWriter *goavro.OCFWriter) (int64, error) { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) schema := p.stream.Schema() avroConverter := model.NewQRecordAvroConverter( @@ -190,7 +189,7 @@ func (p *peerDBOCFWriter) WriteOCF(ctx context.Context, w io.Writer) (int, error func (p *peerDBOCFWriter) WriteRecordsToS3( ctx context.Context, bucketName, key string, s3Creds utils.AWSCredentialsProvider, ) (*AvroFile, error) { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) s3svc, err := utils.CreateS3Client(ctx, s3Creds) if err != nil { logger.Error("failed to create S3 client", slog.Any("error", err)) @@ -246,7 +245,7 @@ func (p *peerDBOCFWriter) WriteRecordsToAvroFile(ctx context.Context, filePath s } defer file.Close() printFileStats := func(message string) { - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) stats, err := file.Stat() if err != nil { return diff --git a/flow/connectors/utils/aws.go b/flow/connectors/utils/aws.go index 510fe8641..89fcab7b2 100644 --- a/flow/connectors/utils/aws.go +++ b/flow/connectors/utils/aws.go @@ -19,7 +19,7 @@ import ( smithyendpoints "github.com/aws/smithy-go/endpoints" "github.com/google/uuid" - "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/shared" ) const ( @@ -177,7 +177,7 @@ func GetAWSCredentialsProvider(ctx context.Context, connectorName string, peerCr EndpointUrl: peerCredentials.EndpointUrl, }, peerCredentials.Region) if peerCredentials.RoleArn == nil || *peerCredentials.RoleArn == "" { - logger.LoggerFromCtx(ctx).Info("Received AWS credentials from peer for connector: " + connectorName) + shared.LoggerFromCtx(ctx).Info("Received AWS credentials from peer for connector: " + connectorName) return staticProvider, nil } awsConfig, err := config.LoadDefaultConfig(ctx, func(options *config.LoadOptions) error { @@ -189,12 +189,12 @@ func GetAWSCredentialsProvider(ctx context.Context, connectorName string, peerCr if err != nil { return nil, err } - logger.LoggerFromCtx(ctx).Info("Received AWS credentials with role from peer for connector: " + connectorName) + shared.LoggerFromCtx(ctx).Info("Received AWS credentials with role from peer for connector: " + connectorName) return NewConfigBasedAWSCredentialsProvider(awsConfig), nil } envCredentialsProvider := LoadPeerDBAWSEnvConfigProvider(connectorName) if envCredentialsProvider != nil { - logger.LoggerFromCtx(ctx).Info("Received AWS credentials from PeerDB Env for connector: " + connectorName) + shared.LoggerFromCtx(ctx).Info("Received AWS credentials from PeerDB Env for connector: " + connectorName) return envCredentialsProvider, nil } @@ -204,7 +204,7 @@ func GetAWSCredentialsProvider(ctx context.Context, connectorName string, peerCr if err != nil { return nil, err } - logger.LoggerFromCtx(ctx).Info("Received AWS credentials from SDK config for connector: " + connectorName) + shared.LoggerFromCtx(ctx).Info("Received AWS credentials from SDK config for connector: " + connectorName) return NewConfigBasedAWSCredentialsProvider(awsConfig), nil } diff --git a/flow/connectors/utils/monitoring/monitoring.go b/flow/connectors/utils/monitoring/monitoring.go index 7b689164b..9c7397004 100644 --- a/flow/connectors/utils/monitoring/monitoring.go +++ b/flow/connectors/utils/monitoring/monitoring.go @@ -12,7 +12,6 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/shared" ) @@ -115,7 +114,7 @@ func AddCDCBatchTablesForFlow(ctx context.Context, pool *pgxpool.Pool, flowJobNa defer func() { err = insertBatchTablesTx.Rollback(context.Background()) if err != pgx.ErrTxClosed && err != nil { - logger.LoggerFromCtx(ctx).Error("error during transaction rollback", + shared.LoggerFromCtx(ctx).Error("error during transaction rollback", slog.Any("error", err), slog.String(string(shared.FlowNameKey), flowJobName)) } @@ -221,7 +220,7 @@ func addPartitionToQRepRun(ctx context.Context, pool *pgxpool.Pool, flowJobName runUUID string, partition *protos.QRepPartition, parentMirrorName string, ) error { if partition.Range == nil && partition.FullTablePartition { - logger.LoggerFromCtx(ctx).Info("partition"+partition.PartitionId+ + shared.LoggerFromCtx(ctx).Info("partition"+partition.PartitionId+ " is a full table partition. Metrics logging is skipped.", slog.String(string(shared.FlowNameKey), flowJobName)) return nil diff --git a/flow/e2e/clickhouse/peer_flow_ch_test.go b/flow/e2e/clickhouse/peer_flow_ch_test.go index 1ef7605ed..4dcee7feb 100644 --- a/flow/e2e/clickhouse/peer_flow_ch_test.go +++ b/flow/e2e/clickhouse/peer_flow_ch_test.go @@ -456,3 +456,46 @@ func (s ClickHouseSuite) Test_Replident_Full_Unchanged_TOAST_Updates() { env.Cancel() e2e.RequireEnvCanceled(s.t, env) } + +// Replicate a table called "table" and a column with hyphen in it +func (s ClickHouseSuite) Test_Weird_Table_And_Column() { + srcTableName := "table" + srcFullName := s.attachSchemaSuffix("\"table\"") + dstTableName := "table" + + _, err := s.Conn().Exec(context.Background(), fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + id SERIAL PRIMARY KEY, + key TEXT NOT NULL + ); + `, srcFullName)) + require.NoError(s.t, err) + + _, err = s.Conn().Exec(context.Background(), fmt.Sprintf(` + INSERT INTO %s (key) VALUES ('init'); + `, srcFullName)) + require.NoError(s.t, err) + + connectionGen := e2e.FlowConnectionGenerationConfig{ + FlowJobName: s.attachSuffix("clickhouse_test_weird_table_and_column"), + TableNameMapping: map[string]string{s.attachSchemaSuffix("table"): dstTableName}, + Destination: s.Peer().Name, + } + flowConnConfig := connectionGen.GenerateFlowConnectionConfigs(s.t) + flowConnConfig.DoInitialSnapshot = true + tc := e2e.NewTemporalClient(s.t) + env := e2e.ExecutePeerflow(tc, peerflow.CDCFlowWorkflow, flowConnConfig, nil) + e2e.SetupCDCFlowStatusQuery(s.t, env, flowConnConfig) + + e2e.EnvWaitForEqualTablesWithNames(env, s, "waiting on initial", srcTableName, dstTableName, "id,key") + + _, err = s.Conn().Exec(context.Background(), fmt.Sprintf(` + INSERT INTO %s (key) VALUES ('cdc'); + `, srcFullName)) + require.NoError(s.t, err) + + e2e.EnvWaitForEqualTablesWithNames(env, s, "waiting on cdc", srcTableName, dstTableName, "id,key") + + env.Cancel() + e2e.RequireEnvCanceled(s.t, env) +} diff --git a/flow/e2e/test_utils.go b/flow/e2e/test_utils.go index 6c2526e20..2eca0520c 100644 --- a/flow/e2e/test_utils.go +++ b/flow/e2e/test_utils.go @@ -26,7 +26,6 @@ import ( connsnowflake "github.com/PeerDB-io/peer-flow/connectors/snowflake" "github.com/PeerDB-io/peer-flow/e2eshared" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/model/qvalue" "github.com/PeerDB-io/peer-flow/peerdbenv" @@ -543,7 +542,7 @@ func (tw *testWriter) Write(p []byte) (int, error) { func NewTemporalClient(t *testing.T) client.Client { t.Helper() - logger := slog.New(logger.NewHandler( + logger := slog.New(shared.NewSlogHandler( slog.NewJSONHandler( &testWriter{t}, &slog.HandlerOptions{Level: slog.LevelWarn}, diff --git a/flow/go.mod b/flow/go.mod index e71c937b3..e6c76516d 100644 --- a/flow/go.mod +++ b/flow/go.mod @@ -5,7 +5,7 @@ go 1.23.1 require ( cloud.google.com/go v0.116.0 cloud.google.com/go/bigquery v1.63.1 - cloud.google.com/go/pubsub v1.44.0 + cloud.google.com/go/pubsub v1.45.1 cloud.google.com/go/storage v1.45.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v1.2.3 @@ -20,9 +20,9 @@ require ( github.com/aws/aws-sdk-go-v2 v1.32.2 github.com/aws/aws-sdk-go-v2/config v1.28.0 github.com/aws/aws-sdk-go-v2/credentials v1.17.41 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.33 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.34 github.com/aws/aws-sdk-go-v2/service/kms v1.37.2 - github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.66.1 github.com/aws/aws-sdk-go-v2/service/ses v1.28.2 github.com/aws/aws-sdk-go-v2/service/sns v1.33.2 github.com/aws/smithy-go v1.22.0 @@ -37,7 +37,7 @@ require ( github.com/jmoiron/sqlx v1.4.0 github.com/joho/godotenv v1.5.1 github.com/klauspost/compress v1.17.11 - github.com/lestrrat-go/jwx/v2 v2.1.1 + github.com/lestrrat-go/jwx/v2 v2.1.2 github.com/lib/pq v1.10.9 github.com/linkedin/goavro/v2 v2.13.0 github.com/microsoft/go-mssqldb v1.7.2 @@ -59,25 +59,25 @@ require ( go.opentelemetry.io/otel/metric v1.31.0 go.opentelemetry.io/otel/sdk v1.31.0 go.opentelemetry.io/otel/sdk/metric v1.31.0 - go.temporal.io/api v1.39.0 + go.temporal.io/api v1.40.0 go.temporal.io/sdk v1.29.1 go.uber.org/automaxprocs v1.6.0 golang.org/x/crypto v0.28.0 golang.org/x/sync v0.8.0 - google.golang.org/api v0.201.0 - google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 + google.golang.org/api v0.203.0 + google.golang.org/genproto/googleapis/api v0.0.0-20241021214115-324edc3d5d38 google.golang.org/grpc v1.67.1 google.golang.org/protobuf v1.35.1 ) require ( - cel.dev/expr v0.16.2 // indirect - cloud.google.com/go/auth v0.9.8 // indirect + cel.dev/expr v0.18.0 // indirect + cloud.google.com/go/auth v0.9.9 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect - cloud.google.com/go/monitoring v1.21.1 // indirect + cloud.google.com/go/monitoring v1.21.2 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect - github.com/ClickHouse/ch-go v0.63.0 // indirect + github.com/ClickHouse/ch-go v0.63.1 // indirect github.com/DataDog/zstd v1.5.6 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.3 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.3 // indirect @@ -130,7 +130,7 @@ require ( github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/prometheus/client_golang v1.20.5 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.60.0 // indirect + github.com/prometheus/common v0.60.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/segmentio/asm v1.2.0 // indirect @@ -143,12 +143,12 @@ require ( go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/term v0.25.0 // indirect - google.golang.org/grpc/stats/opentelemetry v0.0.0-20241018153737-98959d9a4904 // indirect + google.golang.org/grpc/stats/opentelemetry v0.0.0-20241025232817-cb329375b14e // indirect ) require ( cloud.google.com/go/compute/metadata v0.5.2 // indirect - cloud.google.com/go/iam v1.2.1 // indirect + cloud.google.com/go/iam v1.2.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 // indirect @@ -204,7 +204,7 @@ require ( golang.org/x/time v0.7.0 // indirect golang.org/x/tools v0.26.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/genproto v0.0.0-20241021214115-324edc3d5d38 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/flow/go.sum b/flow/go.sum index 4d1ffcc0f..b7992ec67 100644 --- a/flow/go.sum +++ b/flow/go.sum @@ -1,10 +1,10 @@ -cel.dev/expr v0.16.2 h1:RwRhoH17VhAu9U5CMvMhH1PDVgf0tuz9FT+24AfMLfU= -cel.dev/expr v0.16.2/go.mod h1:gXngZQMkWJoSbE8mOzehJlXQyubn/Vg0vR9/F3W7iw8= +cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo= +cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= -cloud.google.com/go/auth v0.9.8 h1:+CSJ0Gw9iVeSENVCKJoLHhdUykDgXSc4Qn+gu2BRtR8= -cloud.google.com/go/auth v0.9.8/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= +cloud.google.com/go/auth v0.9.9 h1:BmtbpNQozo8ZwW2t7QJjnrQtdganSdmqeIBxHxNkEZQ= +cloud.google.com/go/auth v0.9.9/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= cloud.google.com/go/bigquery v1.63.1 h1:/6syiWrSpardKNxdvldS5CUTRJX1iIkSPXCjLjiGL+g= @@ -13,18 +13,18 @@ cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixA cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= cloud.google.com/go/datacatalog v1.22.1 h1:i0DyKb/o7j+0vgaFtimcRFjYsD6wFw1jpnODYUyiYRs= cloud.google.com/go/datacatalog v1.22.1/go.mod h1:MscnJl9B2lpYlFoxRjicw19kFTwEke8ReKL5Y/6TWg8= -cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU= -cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g= +cloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA= +cloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY= cloud.google.com/go/kms v1.20.0 h1:uKUvjGqbBlI96xGE669hcVnEMw1Px/Mvfa62dhM5UrY= cloud.google.com/go/kms v1.20.0/go.mod h1:/dMbFF1tLLFnQV44AoI2GlotbjowyUfgVwezxW291fM= -cloud.google.com/go/logging v1.11.0 h1:v3ktVzXMV7CwHq1MBF65wcqLMA7i+z3YxbUsoK7mOKs= -cloud.google.com/go/logging v1.11.0/go.mod h1:5LDiJC/RxTt+fHc1LAt20R9TKiUTReDg6RuuFOZ67+A= +cloud.google.com/go/logging v1.12.0 h1:ex1igYcGFd4S/RZWOCU51StlIEuey5bjqwH9ZYjHibk= +cloud.google.com/go/logging v1.12.0/go.mod h1:wwYBt5HlYP1InnrtYI0wtwttpVU1rifnMT7RejksUAM= cloud.google.com/go/longrunning v0.6.1 h1:lOLTFxYpr8hcRtcwWir5ITh1PAKUD/sG2lKrTSYjyMc= cloud.google.com/go/longrunning v0.6.1/go.mod h1:nHISoOZpBcmlwbJmiVk5oDRz0qG/ZxPynEGs1iZ79s0= -cloud.google.com/go/monitoring v1.21.1 h1:zWtbIoBMnU5LP9A/fz8LmWMGHpk4skdfeiaa66QdFGc= -cloud.google.com/go/monitoring v1.21.1/go.mod h1:Rj++LKrlht9uBi8+Eb530dIrzG/cU/lB8mt+lbeFK1c= -cloud.google.com/go/pubsub v1.44.0 h1:pLaMJVDTlnUDIKT5L0k53YyLszfBbGoUBo/IqDK/fEI= -cloud.google.com/go/pubsub v1.44.0/go.mod h1:BD4a/kmE8OePyHoa1qAHEw1rMzXX+Pc8Se54T/8mc3I= +cloud.google.com/go/monitoring v1.21.2 h1:FChwVtClH19E7pJ+e0xUhJPGksctZNVOk2UhMmblmdU= +cloud.google.com/go/monitoring v1.21.2/go.mod h1:hS3pXvaG8KgWTSz+dAdyzPrGUYmi2Q+WFX8g2hqVEZU= +cloud.google.com/go/pubsub v1.45.1 h1:ZC/UzYcrmK12THWn1P72z+Pnp2vu/zCZRXyhAfP1hJY= +cloud.google.com/go/pubsub v1.45.1/go.mod h1:3bn7fTmzZFwaUjllitv1WlsNMkqBgGUb3UdMhI54eCc= cloud.google.com/go/storage v1.45.0 h1:5av0QcIVj77t+44mV4gffFC/LscFRUhto6UBMB5SimM= cloud.google.com/go/storage v1.45.0/go.mod h1:wpPblkIuMP5jCB/E48Pz9zIo2S/zD8g+ITmxKkPCITE= cloud.google.com/go/trace v1.11.1 h1:UNqdP+HYYtnm6lb91aNA5JQ0X14GnxkABGlfz2PzPew= @@ -66,8 +66,8 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mo github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/ClickHouse/ch-go v0.63.0 h1:P038rAdzNo4BIwr/L2VOJ1Wr7nHu6XXvjExBwb1eDj0= -github.com/ClickHouse/ch-go v0.63.0/go.mod h1:I1kJJCL3WJcBMGe1m+HVK0+nREaG+JOYYBWjrDrF3R0= +github.com/ClickHouse/ch-go v0.63.1 h1:s2JyZvWLTCSAGdtjMBBmAgQQHMco6pawLJMOXi0FODM= +github.com/ClickHouse/ch-go v0.63.1/go.mod h1:I1kJJCL3WJcBMGe1m+HVK0+nREaG+JOYYBWjrDrF3R0= github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h2lrmGGk17dhFo= github.com/ClickHouse/clickhouse-go/v2 v2.30.0/go.mod h1:i9ZQAojcayW3RsdCb3YR+n+wC2h65eJsZCscZ1Z1wyo= github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= @@ -112,8 +112,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0 github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.33 h1:X+4YY5kZRI/cOoSMVMGTqFXHAMg1bvvay7IBcqHpybQ= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.33/go.mod h1:DPynzu+cn92k5UQ6tZhX+wfTB4ah6QDU/NgdHqatmvk= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.34 h1:os83HS/WfOwi1LsZWLCSHTyj+whvPGaxUsq/D1Ol2Q0= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.34/go.mod h1:tG0BaDCAweumHRsOHm72tuPgAfRLASQThgthWYeTyV8= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= @@ -132,8 +132,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2/go.mod h1:/niFCtmuQNxqx9v8WAPq5qh7EH25U4BF6tjoyq9bObM= github.com/aws/aws-sdk-go-v2/service/kms v1.37.2 h1:tfBABi5R6aSZlhgTWHxL+opYUDOnIGoNcJLwVYv0jLM= github.com/aws/aws-sdk-go-v2/service/kms v1.37.2/go.mod h1:dZYFcQwuoh+cLOlFnZItijZptmyDhRIkOKWFO1CfzV8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0 h1:xA6XhTF7PE89BCNHJbQi8VvPzcgMtmGC5dr8S8N7lHk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.66.1 h1:MkQ4unegQEStiQYmfFj+Aq5uTp265ncSmm0XTQwDwi0= +github.com/aws/aws-sdk-go-v2/service/s3 v1.66.1/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8= github.com/aws/aws-sdk-go-v2/service/ses v1.28.2 h1:FtmzF/j5v++pa0tuuE0wwvWckHzad+vl/Dy5as0Ateo= github.com/aws/aws-sdk-go-v2/service/ses v1.28.2/go.mod h1:bSPQlnLDUiQy7XxmKqTBsCVkYrLfnYJbEyAmm/gWcaI= github.com/aws/aws-sdk-go-v2/service/sns v1.33.2 h1:GeVRrB1aJsGdXxdPY6VOv0SWs+pfdeDlKgiBxi0+V6I= @@ -355,8 +355,8 @@ github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCG github.com/lestrrat-go/httprc v1.0.6/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= -github.com/lestrrat-go/jwx/v2 v2.1.1 h1:Y2ltVl8J6izLYFs54BVcpXLv5msSW4o8eXwnzZLI32E= -github.com/lestrrat-go/jwx/v2 v2.1.1/go.mod h1:4LvZg7oxu6Q5VJwn7Mk/UwooNRnTHUpXBj2C4j3HNx0= +github.com/lestrrat-go/jwx/v2 v2.1.2 h1:6poete4MPsO8+LAEVhpdrNI4Xp2xdiafgl2RD89moBc= +github.com/lestrrat-go/jwx/v2 v2.1.2/go.mod h1:pO+Gz9whn7MPdbsqSJzG8TlEpMZCwQDXnFJ+zsUVh8Y= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -404,8 +404,8 @@ github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/j github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.60.0 h1:+V9PAREWNvJMAuJ1x1BaWl9dewMW4YrHZQbx0sJNllA= -github.com/prometheus/common v0.60.0/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= +github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= +github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= @@ -500,8 +500,8 @@ go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HY go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= -go.temporal.io/api v1.39.0 h1:pbhcfvNDB7mllb8lIBqPcg+m6LMG/IhTpdiFxe+0mYk= -go.temporal.io/api v1.39.0/go.mod h1:1WwYUMo6lao8yl0371xWUm13paHExN5ATYT/B7QtFis= +go.temporal.io/api v1.40.0 h1:rH3HvUUCFr0oecQTBW5tI6DdDQsX2Xb6OFVgt/bvLto= +go.temporal.io/api v1.40.0/go.mod h1:1WwYUMo6lao8yl0371xWUm13paHExN5ATYT/B7QtFis= go.temporal.io/sdk v1.29.1 h1:y+sUMbUhTU9rj50mwIZAPmcXCtgUdOWS9xHDYRYSgZ0= go.temporal.io/sdk v1.29.1/go.mod h1:kp//DRvn3CqQVBCtjL51Oicp9wrZYB2s6row1UgzcKQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -599,20 +599,20 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= -google.golang.org/api v0.201.0 h1:+7AD9JNM3tREtawRMu8sOjSbb8VYcYXJG/2eEOmfDu0= -google.golang.org/api v0.201.0/go.mod h1:HVY0FCHVs89xIW9fzf/pBvOEm+OolHa86G/txFezyq4= +google.golang.org/api v0.203.0 h1:SrEeuwU3S11Wlscsn+LA1kb/Y5xT8uggJSkIhD08NAU= +google.golang.org/api v0.203.0/go.mod h1:BuOVyCSYEPwJb3npWvDnNmFI92f3GeRnHNkETneT3SI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 h1:Df6WuGvthPzc+JiQ/G+m+sNX24kc0aTBqoDN/0yyykE= -google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53/go.mod h1:fheguH3Am2dGp1LfXkrvwqC/KlFq8F0nLq3LryOMrrE= -google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= -google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/genproto v0.0.0-20241021214115-324edc3d5d38 h1:Q3nlH8iSQSRUwOskjbcSMcF2jiYMNiQYZ0c2KEJLKKU= +google.golang.org/genproto v0.0.0-20241021214115-324edc3d5d38/go.mod h1:xBI+tzfqGGN2JBeSebfKXFSdBpWVQ7sLW40PTupVRm4= +google.golang.org/genproto/googleapis/api v0.0.0-20241021214115-324edc3d5d38 h1:2oV8dfuIkM1Ti7DwXc0BJfnwr9csz4TDXI9EmiI+Rbw= +google.golang.org/genproto/googleapis/api v0.0.0-20241021214115-324edc3d5d38/go.mod h1:vuAjtvlwkDKF6L1GQ0SokiRLCGFfeBUXWr/aFFkHACc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -621,8 +621,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/grpc/stats/opentelemetry v0.0.0-20241018153737-98959d9a4904 h1:Lplo3VKrYtWeryBkDI4SZ4kJTFaWO4qUGs+xX7N2bFc= -google.golang.org/grpc/stats/opentelemetry v0.0.0-20241018153737-98959d9a4904/go.mod h1:jzYlkSMbKypzuu6xoAEijsNVo9ZeDF1u/zCfFgsx7jg= +google.golang.org/grpc/stats/opentelemetry v0.0.0-20241025232817-cb329375b14e h1:SoMI+r+Qsp379U9BlVzrHtqAqYP3NEv9vNhYqUaAWOg= +google.golang.org/grpc/stats/opentelemetry v0.0.0-20241025232817-cb329375b14e/go.mod h1:jzYlkSMbKypzuu6xoAEijsNVo9ZeDF1u/zCfFgsx7jg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/flow/logger/from_ctx.go b/flow/logger/from_ctx.go deleted file mode 100644 index 555111076..000000000 --- a/flow/logger/from_ctx.go +++ /dev/null @@ -1,26 +0,0 @@ -package logger - -import ( - "context" - "log/slog" - - "go.temporal.io/sdk/activity" - "go.temporal.io/sdk/log" - - "github.com/PeerDB-io/peer-flow/shared" -) - -func LoggerFromCtx(ctx context.Context) log.Logger { - flowName, hasName := ctx.Value(shared.FlowNameKey).(string) - if activity.IsActivity(ctx) { - if hasName { - return log.With(activity.GetLogger(ctx), string(shared.FlowNameKey), flowName) - } else { - return activity.GetLogger(ctx) - } - } else if hasName { - return slog.With(string(shared.FlowNameKey), flowName) - } else { - return slog.Default() - } -} diff --git a/flow/logger/handler.go b/flow/logger/handler.go deleted file mode 100644 index 335178722..000000000 --- a/flow/logger/handler.go +++ /dev/null @@ -1,45 +0,0 @@ -package logger - -import ( - "context" - "log/slog" - "os" - - "github.com/PeerDB-io/peer-flow/shared" -) - -var _ slog.Handler = Handler{} - -var fields = []shared.ContextKey{shared.FlowNameKey, shared.PartitionIDKey} - -type Handler struct { - handler slog.Handler -} - -func NewHandler(handler slog.Handler) slog.Handler { - return Handler{ - handler: handler, - } -} - -func (h Handler) Enabled(ctx context.Context, level slog.Level) bool { - return h.handler.Enabled(ctx, level) -} - -func (h Handler) Handle(ctx context.Context, record slog.Record) error { - for _, field := range fields { - if v, ok := ctx.Value(field).(string); ok { - record.AddAttrs(slog.String(string(field), v)) - } - } - record.AddAttrs(slog.String(string(shared.DeploymentUIDKey), os.Getenv("PEERDB_DEPLOYMENT_UID"))) - return h.handler.Handle(ctx, record) -} - -func (h Handler) WithAttrs(attrs []slog.Attr) slog.Handler { - return Handler{h.handler.WithAttrs(attrs)} -} - -func (h Handler) WithGroup(name string) slog.Handler { - return h.handler.WithGroup(name) -} diff --git a/flow/main.go b/flow/main.go index 937cdc12b..4001a8891 100644 --- a/flow/main.go +++ b/flow/main.go @@ -14,14 +14,14 @@ import ( _ "go.uber.org/automaxprocs" "github.com/PeerDB-io/peer-flow/cmd" - "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/shared" ) func main() { appCtx, appClose := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer appClose() - slog.SetDefault(slog.New(logger.NewHandler(slog.NewJSONHandler(os.Stdout, nil)))) + slog.SetDefault(slog.New(shared.NewSlogHandler(slog.NewJSONHandler(os.Stdout, nil)))) temporalHostPortFlag := &cli.StringFlag{ Name: "temporal-host-port", diff --git a/flow/model/cdc_stream.go b/flow/model/cdc_stream.go index 2d328cf53..a2257687a 100644 --- a/flow/model/cdc_stream.go +++ b/flow/model/cdc_stream.go @@ -7,7 +7,6 @@ import ( "time" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/shared" ) @@ -53,7 +52,7 @@ func (r *CDCStream[T]) AddRecord(ctx context.Context, record Record[T]) error { } } - logger := logger.LoggerFromCtx(ctx) + logger := shared.LoggerFromCtx(ctx) ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() diff --git a/flow/model/qvalue/avro_converter.go b/flow/model/qvalue/avro_converter.go index 648a6aa7a..c2382e5b0 100644 --- a/flow/model/qvalue/avro_converter.go +++ b/flow/model/qvalue/avro_converter.go @@ -1,6 +1,7 @@ package qvalue import ( + "encoding/base64" "errors" "fmt" "log/slog" @@ -15,7 +16,19 @@ import ( "github.com/PeerDB-io/peer-flow/generated/protos" ) +type AvroSchemaField struct { + Name string `json:"name"` + Type interface{} `json:"type"` + LogicalType string `json:"logicalType,omitempty"` +} + +type AvroSchemaLogical struct { + Type string `json:"type"` + LogicalType string `json:"logicalType,omitempty"` +} + // https://avro.apache.org/docs/1.11.0/spec.html +// please make this generic at some point type AvroSchemaArray struct { Type string `json:"type"` Items string `json:"items"` @@ -39,17 +52,6 @@ type AvroSchemaRecord struct { Fields []AvroSchemaField `json:"fields"` } -type AvroSchemaLogical struct { - Type string `json:"type"` - LogicalType string `json:"logicalType,omitempty"` -} - -type AvroSchemaField struct { - Name string `json:"name"` - Type interface{} `json:"type"` - LogicalType string `json:"logicalType,omitempty"` -} - func TruncateOrLogNumeric(num decimal.Decimal, precision int16, scale int16, targetDB protos.DBType) (decimal.Decimal, error) { if targetDB == protos.DBType_SNOWFLAKE || targetDB == protos.DBType_BIGQUERY { bidigi := datatypes.CountDigits(num.BigInt()) @@ -96,6 +98,9 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, targetDWH protos.DBType, preci case QValueKindBoolean: return "boolean", nil case QValueKindBytes: + if targetDWH == protos.DBType_CLICKHOUSE { + return "string", nil + } return "bytes", nil case QValueKindNumeric: avroNumericPrecision, avroNumericScale := DetermineNumericSettingForDWH(precision, scale, targetDWH) @@ -105,18 +110,20 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, targetDWH protos.DBType, preci Precision: avroNumericPrecision, Scale: avroNumericScale, }, nil - case QValueKindTime, QValueKindTimeTZ, QValueKindDate: + case QValueKindDate: if targetDWH == protos.DBType_CLICKHOUSE { - if kind == QValueKindTime { - return "string", nil - } - if kind == QValueKindDate { - return AvroSchemaLogical{ - Type: "int", - LogicalType: "date", - }, nil - } - return "long", nil + return AvroSchemaLogical{ + Type: "int", + LogicalType: "date", + }, nil + } + return "string", nil + case QValueKindTime, QValueKindTimeTZ: + if targetDWH == protos.DBType_CLICKHOUSE { + return AvroSchemaLogical{ + Type: "long", + LogicalType: "time-micros", + }, nil } return "string", nil case QValueKindTimestamp, QValueKindTimestampTZ: @@ -155,11 +162,29 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, targetDWH protos.DBType, preci Items: "boolean", }, nil case QValueKindArrayDate: + if targetDWH == protos.DBType_CLICKHOUSE { + return AvroSchemaComplexArray{ + Type: "array", + Items: AvroSchemaField{ + Type: "int", + LogicalType: "date", + }, + }, nil + } return AvroSchemaArray{ Type: "array", Items: "string", }, nil case QValueKindArrayTimestamp, QValueKindArrayTimestampTZ: + if targetDWH == protos.DBType_CLICKHOUSE { + return AvroSchemaComplexArray{ + Type: "array", + Items: AvroSchemaField{ + Type: "long", + LogicalType: "timestamp-micros", + }, + }, nil + } return AvroSchemaArray{ Type: "array", Items: "string", @@ -203,26 +228,18 @@ func QValueToAvro(value QValue, field *QField, targetDWH protos.DBType, logger l if t == nil { return nil, nil } - if c.TargetDWH == protos.DBType_SNOWFLAKE { if c.Nullable { return c.processNullableUnion("string", t.(string)) } else { return t.(string), nil } - } - - if c.TargetDWH == protos.DBType_CLICKHOUSE { + } else { if c.Nullable { - return c.processNullableUnion("string", t.(string)) - } else { - return t.(string), nil + return goavro.Union("long.time-micros", t.(int64)), nil } + return t.(int64), nil } - if c.Nullable { - return goavro.Union("long.time-micros", t.(int64)), nil - } - return t.(int64), nil case QValueTimeTZ: t := c.processGoTimeTZ(v.Val) if t == nil { @@ -234,19 +251,12 @@ func QValueToAvro(value QValue, field *QField, targetDWH protos.DBType, logger l } else { return t.(string), nil } - } - - if c.TargetDWH == protos.DBType_CLICKHOUSE { + } else { if c.Nullable { - return c.processNullableUnion("long", t.(int64)) - } else { - return t.(int64), nil + return goavro.Union("long.time-micros", t.(int64)), nil } + return t.(int64), nil } - if c.Nullable { - return goavro.Union("long.time-micros", t.(int64)), nil - } - return t.(int64), nil case QValueTimestamp: t := c.processGoTimestamp(v.Val) if t == nil { @@ -374,9 +384,6 @@ func (c *QValueAvroConverter) processGoTime(t time.Time) interface{} { if c.TargetDWH == protos.DBType_SNOWFLAKE { return t.Format("15:04:05.999999") } - if c.TargetDWH == protos.DBType_CLICKHOUSE { - return t.Format("15:04:05.999999") - } return t.UnixMicro() } @@ -455,6 +462,13 @@ func (c *QValueAvroConverter) processNumeric(num decimal.Decimal) interface{} { } func (c *QValueAvroConverter) processBytes(byteData []byte) interface{} { + if c.TargetDWH == protos.DBType_CLICKHOUSE { + encoded := base64.StdEncoding.EncodeToString(byteData) + if c.Nullable { + return goavro.Union("string", encoded) + } + return encoded + } if c.Nullable { return goavro.Union("bytes", byteData) } diff --git a/flow/model/qvalue/kind.go b/flow/model/qvalue/kind.go index 669105a13..79e8f89e4 100644 --- a/flow/model/qvalue/kind.go +++ b/flow/model/qvalue/kind.go @@ -109,23 +109,26 @@ var QValueKindToClickHouseTypeMap = map[QValueKind]string{ QValueKindJSON: "String", QValueKindTimestamp: "DateTime64(6)", QValueKindTimestampTZ: "DateTime64(6)", - QValueKindTime: "String", + QValueKindTime: "DateTime64(6)", + QValueKindTimeTZ: "DateTime64(6)", QValueKindDate: "Date32", QValueKindBytes: "String", QValueKindStruct: "String", QValueKindUUID: "UUID", - QValueKindTimeTZ: "String", QValueKindInvalid: "String", QValueKindHStore: "String", // array types will be mapped to VARIANT - QValueKindArrayFloat32: "Array(Float32)", - QValueKindArrayFloat64: "Array(Float64)", - QValueKindArrayInt32: "Array(Int32)", - QValueKindArrayInt64: "Array(Int64)", - QValueKindArrayString: "Array(String)", - QValueKindArrayBoolean: "Array(Bool)", - QValueKindArrayInt16: "Array(Int16)", + QValueKindArrayFloat32: "Array(Float32)", + QValueKindArrayFloat64: "Array(Float64)", + QValueKindArrayInt32: "Array(Int32)", + QValueKindArrayInt64: "Array(Int64)", + QValueKindArrayString: "Array(String)", + QValueKindArrayBoolean: "Array(Bool)", + QValueKindArrayInt16: "Array(Int16)", + QValueKindArrayDate: "Array(Date)", + QValueKindArrayTimestamp: "Array(DateTime64(6))", + QValueKindArrayTimestampTZ: "Array(DateTime64(6))", } func (kind QValueKind) ToDWHColumnType(dwhType protos.DBType) (string, error) { diff --git a/flow/peerdbenv/config.go b/flow/peerdbenv/config.go index ecd4b8427..ecae67037 100644 --- a/flow/peerdbenv/config.go +++ b/flow/peerdbenv/config.go @@ -67,7 +67,7 @@ func PeerDBCatalogUser() string { // PEERDB_CATALOG_PASSWORD func PeerDBCatalogPassword(ctx context.Context) string { - val, err := GetKMSDecryptedEnvString(ctx, "PEERDB_CATALOG_PASSWORD", "") + val, err := GetKmsDecryptedEnvString(ctx, "PEERDB_CATALOG_PASSWORD", "") if err != nil { slog.Error("failed to decrypt PEERDB_CATALOG_PASSWORD", "error", err) panic(err) @@ -108,7 +108,7 @@ func PeerDBCurrentEncKeyID() string { } func PeerDBEncKeys(ctx context.Context) shared.PeerDBEncKeys { - val, err := GetKMSDecryptedEnvString(ctx, "PEERDB_ENC_KEYS", "") + val, err := GetKmsDecryptedEnvString(ctx, "PEERDB_ENC_KEYS", "") if err != nil { slog.Error("failed to decrypt PEERDB_ENC_KEYS", "error", err) panic(err) @@ -142,11 +142,11 @@ func PeerDBTemporalEnableCertAuth() bool { } func PeerDBTemporalClientCert(ctx context.Context) ([]byte, error) { - return GetEnvBase64EncodedBytes(ctx, "TEMPORAL_CLIENT_CERT", nil) + return GetKmsDecryptedEnvBase64EncodedBytes(ctx, "TEMPORAL_CLIENT_CERT", nil) } func PeerDBTemporalClientKey(ctx context.Context) ([]byte, error) { - return GetEnvBase64EncodedBytes(ctx, "TEMPORAL_CLIENT_KEY", nil) + return GetKmsDecryptedEnvBase64EncodedBytes(ctx, "TEMPORAL_CLIENT_KEY", nil) } func PeerDBGetIncidentIoUrl() string { diff --git a/flow/peerdbenv/dynamicconf.go b/flow/peerdbenv/dynamicconf.go index 28b01af43..1e2f22590 100644 --- a/flow/peerdbenv/dynamicconf.go +++ b/flow/peerdbenv/dynamicconf.go @@ -13,7 +13,7 @@ import ( "golang.org/x/exp/constraints" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/shared" ) var DynamicSettings = [...]*protos.DynamicSetting{ @@ -197,14 +197,14 @@ func dynLookup(ctx context.Context, env map[string]string, key string) (string, conn, err := GetCatalogConnectionPoolFromEnv(ctx) if err != nil { - logger.LoggerFromCtx(ctx).Error("Failed to get catalog connection pool", slog.Any("error", err)) + shared.LoggerFromCtx(ctx).Error("Failed to get catalog connection pool", slog.Any("error", err)) return "", fmt.Errorf("failed to get catalog connection pool: %w", err) } var value pgtype.Text query := "SELECT config_value FROM dynamic_settings WHERE config_name=$1" if err := conn.QueryRow(ctx, query, key).Scan(&value); err != nil && err != pgx.ErrNoRows { - logger.LoggerFromCtx(ctx).Error("Failed to get key", slog.Any("error", err)) + shared.LoggerFromCtx(ctx).Error("Failed to get key", slog.Any("error", err)) return "", fmt.Errorf("failed to get key: %w", err) } if !value.Valid { @@ -232,7 +232,7 @@ func dynamicConfSigned[T constraints.Signed](ctx context.Context, env map[string return strconv.ParseInt(value, 10, 64) }) if err != nil { - logger.LoggerFromCtx(ctx).Error("Failed to parse as int64", slog.Any("error", err)) + shared.LoggerFromCtx(ctx).Error("Failed to parse as int64", slog.Any("error", err)) return 0, fmt.Errorf("failed to parse as int64: %w", err) } @@ -244,7 +244,7 @@ func dynamicConfUnsigned[T constraints.Unsigned](ctx context.Context, env map[st return strconv.ParseUint(value, 10, 64) }) if err != nil { - logger.LoggerFromCtx(ctx).Error("Failed to parse as uint64", slog.Any("error", err)) + shared.LoggerFromCtx(ctx).Error("Failed to parse as uint64", slog.Any("error", err)) return 0, fmt.Errorf("failed to parse as uint64: %w", err) } @@ -254,7 +254,7 @@ func dynamicConfUnsigned[T constraints.Unsigned](ctx context.Context, env map[st func dynamicConfBool(ctx context.Context, env map[string]string, key string) (bool, error) { value, err := dynLookupConvert(ctx, env, key, strconv.ParseBool) if err != nil { - logger.LoggerFromCtx(ctx).Error("Failed to parse bool", slog.Any("error", err)) + shared.LoggerFromCtx(ctx).Error("Failed to parse bool", slog.Any("error", err)) return false, fmt.Errorf("failed to parse bool: %w", err) } diff --git a/flow/peerdbenv/env.go b/flow/peerdbenv/env.go index f8814bc31..ced2adeab 100644 --- a/flow/peerdbenv/env.go +++ b/flow/peerdbenv/env.go @@ -8,6 +8,7 @@ import ( "reflect" "strconv" "strings" + "sync" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" @@ -16,7 +17,7 @@ import ( ) const ( - KMSKeyIDEnvVar = "PEERDB_KMS_KEY_ID" + KmsKeyIDEnvVar = "PEERDB_KMS_KEY_ID" ) // GetEnvInt returns the value of the environment variable with the given name @@ -63,8 +64,8 @@ func GetEnvString(name string, defaultValue string) string { return val } -func decryptWithKMS(ctx context.Context, data []byte) ([]byte, error) { - keyID, exists := os.LookupEnv(KMSKeyIDEnvVar) +func decryptWithKms(ctx context.Context, data []byte) ([]byte, error) { + keyID, exists := os.LookupEnv(KmsKeyIDEnvVar) if !exists { return data, nil } @@ -86,7 +87,15 @@ func decryptWithKMS(ctx context.Context, data []byte) ([]byte, error) { return decrypted.Plaintext, nil } -func GetEnvBase64EncodedBytes(ctx context.Context, name string, defaultValue []byte) ([]byte, error) { +var kmsCache sync.Map + +func GetKmsDecryptedEnvBase64EncodedBytes(ctx context.Context, name string, defaultValue []byte) ([]byte, error) { + if cacheVal, ok := kmsCache.Load(name); ok { + if finalVal, ok := cacheVal.([]byte); ok { + return finalVal, nil + } + } + val, ok := os.LookupEnv(name) if !ok { return defaultValue, nil @@ -98,21 +107,26 @@ func GetEnvBase64EncodedBytes(ctx context.Context, name string, defaultValue []b return nil, fmt.Errorf("failed to decode base64 value for %s: %w", name, err) } - return decryptWithKMS(ctx, decoded) + finalVal, err := decryptWithKms(ctx, decoded) + if err != nil { + return finalVal, err + } + kmsCache.Store(name, finalVal) + return finalVal, nil } -func GetKMSDecryptedEnvString(ctx context.Context, name string, defaultValue string) (string, error) { +func GetKmsDecryptedEnvString(ctx context.Context, name string, defaultValue string) (string, error) { val, ok := os.LookupEnv(name) if !ok { return defaultValue, nil } - _, exists := os.LookupEnv(KMSKeyIDEnvVar) + _, exists := os.LookupEnv(KmsKeyIDEnvVar) if !exists { return val, nil } - ret, err := GetEnvBase64EncodedBytes(ctx, name, []byte(defaultValue)) + ret, err := GetKmsDecryptedEnvBase64EncodedBytes(ctx, name, []byte(defaultValue)) if err != nil { return defaultValue, fmt.Errorf("failed to get base64 encoded bytes for %s: %w", name, err) } diff --git a/flow/shared/logger.go b/flow/shared/logger.go new file mode 100644 index 000000000..b30e978e0 --- /dev/null +++ b/flow/shared/logger.go @@ -0,0 +1,54 @@ +package shared + +import ( + "context" + "log/slog" + "os" + + "go.temporal.io/sdk/activity" + "go.temporal.io/sdk/log" +) + +func LoggerFromCtx(ctx context.Context) log.Logger { + flowName, hasName := ctx.Value(FlowNameKey).(string) + if activity.IsActivity(ctx) { + if hasName { + return log.With(activity.GetLogger(ctx), string(FlowNameKey), flowName) + } else { + return activity.GetLogger(ctx) + } + } else if hasName { + return slog.With(string(FlowNameKey), flowName) + } else { + return slog.Default() + } +} + +func LogError(logger log.Logger, err error) error { + logger.Error(err.Error()) + return err +} + +var _ slog.Handler = SlogHandler{} + +var fields = []ContextKey{FlowNameKey, PartitionIDKey} + +type SlogHandler struct { + slog.Handler +} + +func NewSlogHandler(handler slog.Handler) slog.Handler { + return SlogHandler{ + Handler: handler, + } +} + +func (h SlogHandler) Handle(ctx context.Context, record slog.Record) error { + for _, field := range fields { + if v, ok := ctx.Value(field).(string); ok { + record.AddAttrs(slog.String(string(field), v)) + } + } + record.AddAttrs(slog.String(string(DeploymentUIDKey), os.Getenv("PEERDB_DEPLOYMENT_UID"))) + return h.Handler.Handle(ctx, record) +} diff --git a/flow/shared/telemetry/incidentio_message_sender.go b/flow/shared/telemetry/incidentio_message_sender.go index 7af302ddc..53e05628f 100644 --- a/flow/shared/telemetry/incidentio_message_sender.go +++ b/flow/shared/telemetry/incidentio_message_sender.go @@ -48,7 +48,7 @@ func (i *IncidentIoMessageSenderImpl) SendMessage( subject string, body string, attributes Attributes, -) (*string, error) { +) (string, error) { activityInfo := activity.Info{} if activity.IsActivity(ctx) { activityInfo = activity.GetInfo(ctx) @@ -83,38 +83,38 @@ func (i *IncidentIoMessageSenderImpl) SendMessage( alertJSON, err := json.Marshal(alert) if err != nil { - return nil, fmt.Errorf("error serializing alert %w", err) + return "", fmt.Errorf("error serializing alert %w", err) } req, err := http.NewRequest("POST", i.config.URL, bytes.NewBuffer(alertJSON)) if err != nil { - return nil, err + return "", err } req.Header.Add("Content-Type", "application/json") req.Header.Add("Authorization", "Bearer "+i.config.Token) resp, err := i.http.Do(req) if err != nil { - return nil, fmt.Errorf("incident.io request failed: %w", err) + return "", fmt.Errorf("incident.io request failed: %w", err) } defer resp.Body.Close() respBody, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("reading incident.io response body failed: %w", err) + return "", fmt.Errorf("reading incident.io response body failed: %w", err) } if resp.StatusCode != http.StatusAccepted { - return nil, fmt.Errorf("unexpected response from incident.io. status: %d. body: %s", resp.StatusCode, respBody) + return "", fmt.Errorf("unexpected response from incident.io. status: %d. body: %s", resp.StatusCode, respBody) } var incidentResponse IncidentIoResponse err = json.Unmarshal(respBody, &incidentResponse) if err != nil { - return nil, fmt.Errorf("deserializing incident.io failed: %w", err) + return "", fmt.Errorf("deserializing incident.io failed: %w", err) } - return &incidentResponse.Status, nil + return incidentResponse.Status, nil } func NewIncidentIoMessageSender(_ context.Context, config IncidentIoMessageSenderConfig) (Sender, error) { diff --git a/flow/shared/telemetry/incidentio_message_sender_test.go b/flow/shared/telemetry/incidentio_message_sender_test.go index 238717b3c..ce629abcd 100644 --- a/flow/shared/telemetry/incidentio_message_sender_test.go +++ b/flow/shared/telemetry/incidentio_message_sender_test.go @@ -180,7 +180,7 @@ func TestIncidentIoMessageSenderImpl_SendMessage(t *testing.T) { require.Error(t, err) } else { require.NoError(t, err) - require.Equal(t, tt.serverResponse.Status, *status) + require.Equal(t, tt.serverResponse.Status, status) } }) } diff --git a/flow/shared/telemetry/interface.go b/flow/shared/telemetry/interface.go index fc186b0c7..1c2ed159a 100644 --- a/flow/shared/telemetry/interface.go +++ b/flow/shared/telemetry/interface.go @@ -5,7 +5,7 @@ import ( ) type Sender interface { - SendMessage(ctx context.Context, subject string, body string, attributes Attributes) (*string, error) + SendMessage(ctx context.Context, subject string, body string, attributes Attributes) (string, error) } type Attributes struct { diff --git a/flow/shared/telemetry/sns_message_sender.go b/flow/shared/telemetry/sns_message_sender.go index 35c4075c4..193f47c52 100644 --- a/flow/shared/telemetry/sns_message_sender.go +++ b/flow/shared/telemetry/sns_message_sender.go @@ -28,7 +28,7 @@ type SNSMessageSenderConfig struct { Topic string `json:"topic"` } -func (s *SNSMessageSenderImpl) SendMessage(ctx context.Context, subject string, body string, attributes Attributes) (*string, error) { +func (s *SNSMessageSenderImpl) SendMessage(ctx context.Context, subject string, body string, attributes Attributes) (string, error) { activityInfo := activity.Info{} if activity.IsActivity(ctx) { activityInfo = activity.GetInfo(ctx) @@ -87,9 +87,9 @@ func (s *SNSMessageSenderImpl) SendMessage(ctx context.Context, subject string, TopicArn: aws.String(s.topic), }) if err != nil { - return nil, err + return "", err } - return publish.MessageId, nil + return *publish.MessageId, nil } func NewSNSMessageSenderWithNewClient(ctx context.Context, config *SNSMessageSenderConfig) (SNSMessageSender, error) { diff --git a/nexus/Cargo.lock b/nexus/Cargo.lock index e55e5c117..8dd801e5e 100644 --- a/nexus/Cargo.lock +++ b/nexus/Cargo.lock @@ -95,9 +95,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.15" +version = "0.6.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +checksum = "23a1e53f0f5d86382dafe1cf314783b2044280f406e7e1506368220ad11b1338" dependencies = [ "anstyle", "anstyle-parse", @@ -110,43 +110,43 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" [[package]] name = "anstyle-parse" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.90" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95" +checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" [[package]] name = "ar" @@ -187,7 +187,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -209,7 +209,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -220,7 +220,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -237,9 +237,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.8" +version = "1.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7198e6f03240fdceba36656d8be440297b6b82270325908c7381f37d826a74f6" +checksum = "2d6448cfb224dd6a9b9ac734f58622dd0d4751f3589f3b777345745f46b2eb14" dependencies = [ "aws-credential-types", "aws-runtime", @@ -304,9 +304,9 @@ dependencies = [ [[package]] name = "aws-sdk-kms" -version = "1.47.0" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "564a597a3c71a957d60a2e4c62c93d78ee5a0d636531e15b760acad983a5c18e" +checksum = "2afbd208dabc6785946d4ef2444eb1f54fe0aaf0f62f2a4f9a9e9c303aeff0be" dependencies = [ "aws-credential-types", "aws-runtime", @@ -326,9 +326,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.46.0" +version = "1.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc2faec3205d496c7e57eff685dd944203df7ce16a4116d0281c44021788a7b" +checksum = "a8776850becacbd3a82a4737a9375ddb5c6832a51379f24443a98e61513f852c" dependencies = [ "aws-credential-types", "aws-runtime", @@ -348,9 +348,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.47.0" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c93c241f52bc5e0476e259c953234dab7e2a35ee207ee202e86c0095ec4951dc" +checksum = "0007b5b8004547133319b6c4e87193eee2a0bcb3e4c18c75d09febe9dab7b383" dependencies = [ "aws-credential-types", "aws-runtime", @@ -370,9 +370,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.46.0" +version = "1.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b259429be94a3459fa1b00c5684faee118d74f9577cc50aebadc36e507c63b5f" +checksum = "9fffaa356e7f1c725908b75136d53207fa714e348f365671df14e95a60530ad3" dependencies = [ "aws-credential-types", "aws-runtime", @@ -393,9 +393,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.4" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc8db6904450bafe7473c6ca9123f88cc11089e41a025408f992db4e22d3be68" +checksum = "5619742a0d8f253be760bfbb8e8e8368c69e3587e4637af5754e488a611499b1" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -466,9 +466,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a065c0fe6fdbdf9f11817eb68582b2ab4aff9e9c39e986ae48f7ec576c6322db" +checksum = "be28bd063fa91fd871d131fc8b68d7cd4c5fa0869bea68daca50dcb1cbd76be2" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -510,9 +510,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.7" +version = "1.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147100a7bea70fa20ef224a6bad700358305f5dc0f84649c53769761395b355b" +checksum = "07c9cdc179e6afbf5d391ab08c85eac817b51c87e1892a5edb5f7bbdc64314b4" dependencies = [ "base64-simd", "bytes", @@ -672,7 +672,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -731,7 +731,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", "syn_derive", ] @@ -780,9 +780,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "bytes-utils" @@ -987,7 +987,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -1007,9 +1007,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "const-oid" @@ -1186,7 +1186,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -1207,7 +1207,7 @@ checksum = "d150dea618e920167e5973d70ae6ece4385b7164e0d799fe7c122dd0a5d912ad" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -1248,9 +1248,9 @@ checksum = "4445909572dbd556c457c849c4ca58623d84b27c8fff1e74b0b4227d8b90d17b" [[package]] name = "encoding_rs" -version = "0.8.34" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", ] @@ -1434,7 +1434,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -1975,9 +1975,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "3bda4c6077b0b08da2c48b172195795498381a7c8988c9e6212a6c55c5b9bd70" [[package]] name = "libredox" @@ -2690,29 +2690,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -2864,12 +2864,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.24" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "910d41a655dac3b764f1ade94821093d3610248694320cd072303a8eedcf221d" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -2906,9 +2906,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.88" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3a7fc5db1e57d5a779a352c8cdb57b29aa4c40cc69c3a68a7fedc815fbf2f9" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -2942,7 +2942,7 @@ dependencies = [ "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", - "syn 2.0.82", + "syn 2.0.85", "tempfile", ] @@ -2956,7 +2956,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -3192,14 +3192,14 @@ dependencies = [ "quote", "refinery-core", "regex", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] name = "regex" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -3408,9 +3408,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.37" +version = "0.38.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" dependencies = [ "bitflags", "errno", @@ -3648,9 +3648,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.213" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "3ea7893ff5e2466df8d720bb615088341b295f849602c6956047f8f80f0e9bc1" dependencies = [ "serde_derive", ] @@ -3666,13 +3666,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.213" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "7e85ad2009c50b58e87caa8cd6dac16bdf511bbfb7af6c33df902396aa480fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -3863,7 +3863,7 @@ source = "git+https://github.com/peerdb-io/sqlparser-rs.git?branch=main#8c341b80 dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -3918,9 +3918,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.82" +version = "2.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83540f837a8afc019423a8edb95b52a8effe46957ee402287f4292fae35be021" +checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" dependencies = [ "proc-macro2", "quote", @@ -3936,7 +3936,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -3985,22 +3985,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -4079,14 +4079,14 @@ checksum = "8d9ef545650e79f30233c0003bcc2504d7efac6dad25fca40744de773fe2049c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] name = "tokio" -version = "1.40.0" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" dependencies = [ "backtrace", "bytes", @@ -4108,7 +4108,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -4286,7 +4286,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -4392,7 +4392,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -4653,7 +4653,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", "wasm-bindgen-shared", ] @@ -4687,7 +4687,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5065,7 +5065,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] @@ -5085,7 +5085,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.82", + "syn 2.0.85", ] [[package]] diff --git a/nexus/server/src/main.rs b/nexus/server/src/main.rs index f96d5df29..e1207066c 100644 --- a/nexus/server/src/main.rs +++ b/nexus/server/src/main.rs @@ -936,6 +936,10 @@ struct Args { #[clap(long, default_value = "false", env = "PEERDB_MIGRATIONS_ONLY")] migrations_only: bool, + /// If set to true, nexus will not run any migrations + #[clap(long, default_value = "false", env = "PEERDB_MIGRATIONS_DISABLED")] + migrations_disabled: bool, + /// KMS Key ID for decrypting the catalog password #[clap(long, env = "PEERDB_KMS_KEY_ID")] kms_key_id: Option, @@ -1100,7 +1104,13 @@ pub async fn main() -> anyhow::Result<()> { let _guard = setup_tracing(args.log_dir.as_ref().map(|s| &s[..])); let catalog_config = get_catalog_config(&args).await?; - run_migrations(&catalog_config).await?; + if args.migrations_disabled && args.migrations_only { + return Err(anyhow::anyhow!("Invalid configuration, migrations cannot be enabled and disabled at the same time")); + } + + if !args.migrations_disabled { + run_migrations(&catalog_config).await?; + } if args.migrations_only { return Ok(()); } diff --git a/protos/route.proto b/protos/route.proto index 008a1e398..9b85da6f4 100644 --- a/protos/route.proto +++ b/protos/route.proto @@ -263,8 +263,11 @@ message SlotInfo { } message SlotLagPoint { - double updated_at = 1; - double slot_size = 2; + double time = 1; + double size = 2; + string redo_lSN = 3; + string restart_lSN = 4; + string confirmed_lSN = 5; } message GetSlotLagHistoryRequest { string peer_name = 1; diff --git a/ui/app/mirrors/create/cdc/sortingkey.tsx b/ui/app/mirrors/create/cdc/sortingkey.tsx index 48e7be039..d88dc2a7c 100644 --- a/ui/app/mirrors/create/cdc/sortingkey.tsx +++ b/ui/app/mirrors/create/cdc/sortingkey.tsx @@ -18,7 +18,11 @@ import { Label } from '@/lib/Label'; import { RowWithCheckbox } from '@/lib/Layout'; import { ToastContainer } from 'react-toastify'; import 'react-toastify/dist/ReactToastify.css'; -import { engineOptionStyles } from './styles'; +import { + engineOptionStyles, + sortingKeyPillContainerStyle, + sortingKeyPillStyle, +} from './styles'; interface SortingKeysProps { columns: string[]; @@ -42,7 +46,7 @@ const SelectSortingKeys = ({ (col: string, action: 'add' | 'remove') => { setSortingKeysSelections((prev) => { if (action === 'add' && !prev.some((key) => key === col)) { - return [col, ...prev]; + return [...prev, col]; } else if (action === 'remove') { return prev.filter((prevCol) => prevCol !== col); } @@ -147,30 +151,10 @@ const SelectSortingKeys = ({ theme={SelectTheme} isClearable /> -
+
{sortingKeysSelections.map((col: string) => { return ( -
+

{col}

); } - -export default LagGraph; diff --git a/ui/package-lock.json b/ui/package-lock.json index 00613c421..636f50209 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -77,12 +77,13 @@ } }, "node_modules/@babel/code-frame": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.25.7.tgz", - "integrity": "sha512-0xZJFNE5XMpENsgfHYTw8FbX4kv53mFLn2i3XPoq69LyhYSCBJtitaHx9QnsVTrsogI4Z3+HtEfZ2/GFPOtf5g==", + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.0.tgz", + "integrity": "sha512-INCKxTtbXtcNbUZ3YXutwMpEleqttcswhAdee7dhuoVrD2cnuc3PqtERBtxkX5nziX9vnBL8WXmSGwv8CuPV6g==", "license": "MIT", "dependencies": { - "@babel/highlight": "^7.25.7", + "@babel/helper-validator-identifier": "^7.25.9", + "js-tokens": "^4.0.0", "picocolors": "^1.0.0" }, "engines": { @@ -90,12 +91,13 @@ } }, "node_modules/@babel/generator": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.25.7.tgz", - "integrity": "sha512-5Dqpl5fyV9pIAD62yK9P7fcA768uVPUyrQmqpqstHWgMma4feF1x/oFysBCVZLY5wJ2GkMUCdsNDnGZrPoR6rA==", + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.0.tgz", + "integrity": "sha512-/AIkAmInnWwgEAJGQr9vY0c66Mj6kjkE2ZPB1PurTRaRAh3U+J45sAQMjQDJdh4WbR3l0x5xkimXBKyBXXAu2w==", "license": "MIT", "dependencies": { - "@babel/types": "^7.25.7", + "@babel/parser": "^7.26.0", + "@babel/types": "^7.26.0", "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.25", "jsesc": "^3.0.2" @@ -105,129 +107,43 @@ } }, "node_modules/@babel/helper-module-imports": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.25.7.tgz", - "integrity": "sha512-o0xCgpNmRohmnoWKQ0Ij8IdddjyBFE4T2kagL/x6M3+4zUgc+4qTOUBoNe4XxDskt1HPKO007ZPiMgLDq2s7Kw==", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.25.9.tgz", + "integrity": "sha512-tnUA4RsrmflIM6W6RFTLFSXITtl0wKjgpnLgXyowocVPrbYrLUXSBXDgTs8BlbmIzIdlBySRQjINYs2BAkiLtw==", "license": "MIT", "dependencies": { - "@babel/traverse": "^7.25.7", - "@babel/types": "^7.25.7" + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-string-parser": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.7.tgz", - "integrity": "sha512-CbkjYdsJNHFk8uqpEkpCvRs3YRp9tY6FmFY7wLMSYuGYkrdUi7r2lc4/wqsvlHoMznX3WJ9IP8giGPq68T/Y6g==", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", + "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.7.tgz", - "integrity": "sha512-AM6TzwYqGChO45oiuPqwL2t20/HdMC1rTPAesnBCgPCSF1x3oN9MVUwQV2iyz4xqWrctwK5RNC8LV22kaQCNYg==", - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/highlight": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.25.7.tgz", - "integrity": "sha512-iYyACpW3iW8Fw+ZybQK+drQre+ns/tKpXbNESfrhNnPLIklLbXr7MYJ6gPEd0iETGLOK+SxMjVvKb/ffmk+FEw==", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", + "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.25.7", - "chalk": "^2.4.2", - "js-tokens": "^4.0.0", - "picocolors": "^1.0.0" - }, "engines": { "node": ">=6.9.0" } }, - "node_modules/@babel/highlight/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "license": "MIT", - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "license": "MIT", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/@babel/highlight/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "license": "MIT" - }, - "node_modules/@babel/highlight/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "license": "MIT", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@babel/highlight/node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "license": "MIT", - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/@babel/parser": { - "version": "7.25.8", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.25.8.tgz", - "integrity": "sha512-HcttkxzdPucv3nNFmfOOMfFf64KgdJVqm1KaCm25dPGMLElo9nsLvXeJECQg8UzPuBGLyTSA0ZzqCtDSzKTEoQ==", + "version": "7.26.1", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.1.tgz", + "integrity": "sha512-reoQYNiAJreZNsJzyrDNzFQ+IQ5JFiIzAHJg9bn94S3l+4++J7RsIhNMoB+lgP/9tpmiAQqspv+xfdxTSzREOw==", "license": "MIT", "dependencies": { - "@babel/types": "^7.25.8" + "@babel/types": "^7.26.0" }, "bin": { "parser": "bin/babel-parser.js" @@ -237,9 +153,9 @@ } }, "node_modules/@babel/runtime": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.25.7.tgz", - "integrity": "sha512-FjoyLe754PMiYsFaN5C94ttGiOmBNYTf6pLr4xXHAT5uctHb092PBszndLDR5XA/jghQvn4n7JMHl7dmTgbm9w==", + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.0.tgz", + "integrity": "sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw==", "license": "MIT", "dependencies": { "regenerator-runtime": "^0.14.0" @@ -249,30 +165,30 @@ } }, "node_modules/@babel/template": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.7.tgz", - "integrity": "sha512-wRwtAgI3bAS+JGU2upWNL9lSlDcRCqD05BZ1n3X2ONLH1WilFP6O1otQjeMK/1g0pvYcXC7b/qVUB1keofjtZA==", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.9.tgz", + "integrity": "sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==", "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.25.7", - "@babel/parser": "^7.25.7", - "@babel/types": "^7.25.7" + "@babel/code-frame": "^7.25.9", + "@babel/parser": "^7.25.9", + "@babel/types": "^7.25.9" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.25.7.tgz", - "integrity": "sha512-jatJPT1Zjqvh/1FyJs6qAHL+Dzb7sTb+xr7Q+gM1b+1oBsMsQQ4FkVKb6dFlJvLlVssqkRzV05Jzervt9yhnzg==", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.25.9.tgz", + "integrity": "sha512-ZCuvfwOwlz/bawvAuvcj8rrithP2/N55Tzz342AkTvq4qaWbGfmCk/tKhNaV2cthijKrPAA8SRJV5WWe7IBMJw==", "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.25.7", - "@babel/generator": "^7.25.7", - "@babel/parser": "^7.25.7", - "@babel/template": "^7.25.7", - "@babel/types": "^7.25.7", + "@babel/code-frame": "^7.25.9", + "@babel/generator": "^7.25.9", + "@babel/parser": "^7.25.9", + "@babel/template": "^7.25.9", + "@babel/types": "^7.25.9", "debug": "^4.3.1", "globals": "^11.1.0" }, @@ -290,14 +206,13 @@ } }, "node_modules/@babel/types": { - "version": "7.25.8", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.25.8.tgz", - "integrity": "sha512-JWtuCu8VQsMladxVz/P4HzHUGCAwpuqacmowgXFs5XjxIgKuNjnLokQzuVjlTvIzODaDmpjT3oxcC48vyk9EWg==", + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.0.tgz", + "integrity": "sha512-Z/yiTPj+lDVnF7lWeKCIJzaIkI0vYO87dMpZ4bg4TDrFe4XXLFWL1TbXU27gBP3QccxV9mZICCrnjnYlJjXHOA==", "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^7.25.7", - "@babel/helper-validator-identifier": "^7.25.7", - "to-fast-properties": "^2.0.0" + "@babel/helper-string-parser": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9" }, "engines": { "node": ">=6.9.0" @@ -442,25 +357,28 @@ "license": "MIT" }, "node_modules/@eslint-community/eslint-utils": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", - "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.1.tgz", + "integrity": "sha512-s3O3waFUrMV8P/XaF/+ZTp1X9XBZW1a4B97ZnjQF2KYWaFD2A8KyFBsrsfSjEmjn3RGWAIuvlneuZm3CUK3jbA==", "dev": true, "license": "MIT", "dependencies": { - "eslint-visitor-keys": "^3.3.0" + "eslint-visitor-keys": "^3.4.3" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, + "funding": { + "url": "https://opencollective.com/eslint" + }, "peerDependencies": { "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" } }, "node_modules/@eslint-community/regexpp": { - "version": "4.11.1", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.11.1.tgz", - "integrity": "sha512-m4DVN9ZqskZoLU5GlWZadwDnYo3vAEydiUayB9widCl9ffWx2IvPnp6n3on5rJmziJSw9Bv+Z3ChDVdMwXCY8Q==", + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.0.tgz", + "integrity": "sha512-gh7PdNombP8ftL8TinYC8Xd7WEypB8EKV4PI2h0eMzndKjPCXuo2zUiZtD2Hu+MSPt02Ty2MdS788ADl9ai1rA==", "dev": true, "license": "MIT", "engines": { @@ -781,15 +699,15 @@ } }, "node_modules/@next/env": { - "version": "14.2.15", - "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.15.tgz", - "integrity": "sha512-S1qaj25Wru2dUpcIZMjxeMVSwkt8BK4dmWHHiBuRstcIyOsMapqT4A4jSB6onvqeygkSSmOkyny9VVx8JIGamQ==", + "version": "14.2.16", + "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.16.tgz", + "integrity": "sha512-fLrX5TfJzHCbnZ9YUSnGW63tMV3L4nSfhgOQ0iCcX21Pt+VSTDuaLsSuL8J/2XAiVA5AnzvXDpf6pMs60QxOag==", "license": "MIT" }, "node_modules/@next/eslint-plugin-next": { - "version": "14.2.15", - "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-14.2.15.tgz", - "integrity": "sha512-pKU0iqKRBlFB/ocOI1Ip2CkKePZpYpnw5bEItEkuZ/Nr9FQP1+p7VDWr4VfOdff4i9bFmrOaeaU1bFEyAcxiMQ==", + "version": "14.2.16", + "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-14.2.16.tgz", + "integrity": "sha512-noORwKUMkKc96MWjTOwrsUCjky0oFegHbeJ1yEnQBGbMHAaTEIgLZIIfsYF0x3a06PiS+2TXppfifR+O6VWslg==", "dev": true, "license": "MIT", "dependencies": { @@ -797,9 +715,9 @@ } }, "node_modules/@next/swc-darwin-arm64": { - "version": "14.2.15", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.15.tgz", - "integrity": "sha512-Rvh7KU9hOUBnZ9TJ28n2Oa7dD9cvDBKua9IKx7cfQQ0GoYUwg9ig31O2oMwH3wm+pE3IkAQ67ZobPfEgurPZIA==", + "version": "14.2.16", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.16.tgz", + "integrity": "sha512-uFT34QojYkf0+nn6MEZ4gIWQ5aqGF11uIZ1HSxG+cSbj+Mg3+tYm8qXYd3dKN5jqKUm5rBVvf1PBRO/MeQ6rxw==", "cpu": [ "arm64" ], @@ -813,9 +731,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "14.2.15", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.15.tgz", - "integrity": "sha512-5TGyjFcf8ampZP3e+FyCax5zFVHi+Oe7sZyaKOngsqyaNEpOgkKB3sqmymkZfowy3ufGA/tUgDPPxpQx931lHg==", + "version": "14.2.16", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.16.tgz", + "integrity": "sha512-mCecsFkYezem0QiZlg2bau3Xul77VxUD38b/auAjohMA22G9KTJneUYMv78vWoCCFkleFAhY1NIvbyjj1ncG9g==", "cpu": [ "x64" ], @@ -829,9 +747,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "14.2.15", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.15.tgz", - "integrity": "sha512-3Bwv4oc08ONiQ3FiOLKT72Q+ndEMyLNsc/D3qnLMbtUYTQAmkx9E/JRu0DBpHxNddBmNT5hxz1mYBphJ3mfrrw==", + "version": "14.2.16", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.16.tgz", + "integrity": "sha512-yhkNA36+ECTC91KSyZcgWgKrYIyDnXZj8PqtJ+c2pMvj45xf7y/HrgI17hLdrcYamLfVt7pBaJUMxADtPaczHA==", "cpu": [ "arm64" ], @@ -845,9 +763,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "14.2.15", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.15.tgz", - "integrity": "sha512-k5xf/tg1FBv/M4CMd8S+JL3uV9BnnRmoe7F+GWC3DxkTCD9aewFRH1s5rJ1zkzDa+Do4zyN8qD0N8c84Hu96FQ==", + "version": "14.2.16", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.16.tgz", + "integrity": "sha512-X2YSyu5RMys8R2lA0yLMCOCtqFOoLxrq2YbazFvcPOE4i/isubYjkh+JCpRmqYfEuCVltvlo+oGfj/b5T2pKUA==", "cpu": [ "arm64" ], @@ -861,9 +779,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "14.2.15", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.15.tgz", - "integrity": "sha512-kE6q38hbrRbKEkkVn62reLXhThLRh6/TvgSP56GkFNhU22TbIrQDEMrO7j0IcQHcew2wfykq8lZyHFabz0oBrA==", + "version": "14.2.16", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.16.tgz", + "integrity": "sha512-9AGcX7VAkGbc5zTSa+bjQ757tkjr6C/pKS7OK8cX7QEiK6MHIIezBLcQ7gQqbDW2k5yaqba2aDtaBeyyZh1i6Q==", "cpu": [ "x64" ], @@ -877,9 +795,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "14.2.15", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.15.tgz", - "integrity": "sha512-PZ5YE9ouy/IdO7QVJeIcyLn/Rc4ml9M2G4y3kCM9MNf1YKvFY4heg3pVa/jQbMro+tP6yc4G2o9LjAz1zxD7tQ==", + "version": "14.2.16", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.16.tgz", + "integrity": "sha512-Klgeagrdun4WWDaOizdbtIIm8khUDQJ/5cRzdpXHfkbY91LxBXeejL4kbZBrpR/nmgRrQvmz4l3OtttNVkz2Sg==", "cpu": [ "x64" ], @@ -893,9 +811,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "14.2.15", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.15.tgz", - "integrity": "sha512-2raR16703kBvYEQD9HNLyb0/394yfqzmIeyp2nDzcPV4yPjqNUG3ohX6jX00WryXz6s1FXpVhsCo3i+g4RUX+g==", + "version": "14.2.16", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.16.tgz", + "integrity": "sha512-PwW8A1UC1Y0xIm83G3yFGPiOBftJK4zukTmk7DI1CebyMOoaVpd8aSy7K6GhobzhkjYvqS/QmzcfsWG2Dwizdg==", "cpu": [ "arm64" ], @@ -909,9 +827,9 @@ } }, "node_modules/@next/swc-win32-ia32-msvc": { - "version": "14.2.15", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.15.tgz", - "integrity": "sha512-fyTE8cklgkyR1p03kJa5zXEaZ9El+kDNM5A+66+8evQS5e/6v0Gk28LqA0Jet8gKSOyP+OTm/tJHzMlGdQerdQ==", + "version": "14.2.16", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.16.tgz", + "integrity": "sha512-jhPl3nN0oKEshJBNDAo0etGMzv0j3q3VYorTSFqH1o3rwv1MQRdor27u1zhkgsHPNeY1jxcgyx1ZsCkDD1IHgg==", "cpu": [ "ia32" ], @@ -925,9 +843,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "14.2.15", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.15.tgz", - "integrity": "sha512-SzqGbsLsP9OwKNUG9nekShTwhj6JSB9ZLMWQ8g1gG6hdE5gQLncbnbymrwy2yVmH9nikSLYRYxYMFu78Ggp7/g==", + "version": "14.2.16", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.16.tgz", + "integrity": "sha512-OA7NtfxgirCjfqt+02BqxC3MIgM/JaGjw9tOe4fyZgPsqfseNiMPnCRP44Pfs+Gpo9zPN+SXaFsgP6vk8d571A==", "cpu": [ "x64" ], @@ -2315,12 +2233,12 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.7.7", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.7.7.tgz", - "integrity": "sha512-SRxCrrg9CL/y54aiMCG3edPKdprgMVGDXjA3gB8UmmBW5TcXzRUYAh8EWzTnSJFAd1rgImPELza+A3bJ+qxz8Q==", + "version": "22.8.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.8.1.tgz", + "integrity": "sha512-k6Gi8Yyo8EtrNtkHXutUu2corfDf9su95VYVP10aGYMMROM6SAItZi0w1XszA6RtWTHSVp5OeFof37w0IEqCQg==", "license": "MIT", "dependencies": { - "undici-types": "~6.19.2" + "undici-types": "~6.19.8" } }, "node_modules/@types/parse-json": { @@ -2336,9 +2254,9 @@ "license": "MIT" }, "node_modules/@types/react": { - "version": "18.3.11", - "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.11.tgz", - "integrity": "sha512-r6QZ069rFTjrEYgFdOck1gK7FLVsgJE7tTz0pQBczlBNUhBNk0MQH4UbnFSwjpQLMkLzgqvBBa+qGpLje16eTQ==", + "version": "18.3.12", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.12.tgz", + "integrity": "sha512-D2wOSq/d6Agt28q7rSI3jhU7G6aiuzljDGZ2hTZHIkrTLUI+AF3WMeKkEZ9nN2fkBAlcktT6vcZjDFiIhMYEQw==", "license": "MIT", "dependencies": { "@types/prop-types": "*", @@ -2370,17 +2288,17 @@ "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.10.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.10.0.tgz", - "integrity": "sha512-phuB3hoP7FFKbRXxjl+DRlQDuJqhpOnm5MmtROXyWi3uS/Xg2ZXqiQfcG2BJHiN4QKyzdOJi3NEn/qTnjUlkmQ==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.11.0.tgz", + "integrity": "sha512-KhGn2LjW1PJT2A/GfDpiyOfS4a8xHQv2myUagTM5+zsormOmBlYsnQ6pobJ8XxJmh6hnHwa2Mbe3fPrDJoDhbA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "8.10.0", - "@typescript-eslint/type-utils": "8.10.0", - "@typescript-eslint/utils": "8.10.0", - "@typescript-eslint/visitor-keys": "8.10.0", + "@typescript-eslint/scope-manager": "8.11.0", + "@typescript-eslint/type-utils": "8.11.0", + "@typescript-eslint/utils": "8.11.0", + "@typescript-eslint/visitor-keys": "8.11.0", "graphemer": "^1.4.0", "ignore": "^5.3.1", "natural-compare": "^1.4.0", @@ -2404,16 +2322,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.10.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.10.0.tgz", - "integrity": "sha512-E24l90SxuJhytWJ0pTQydFT46Nk0Z+bsLKo/L8rtQSL93rQ6byd1V/QbDpHUTdLPOMsBCcYXZweADNCfOCmOAg==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.11.0.tgz", + "integrity": "sha512-lmt73NeHdy1Q/2ul295Qy3uninSqi6wQI18XwSpm8w0ZbQXUpjCAWP1Vlv/obudoBiIjJVjlztjQ+d/Md98Yxg==", "dev": true, "license": "BSD-2-Clause", "dependencies": { - "@typescript-eslint/scope-manager": "8.10.0", - "@typescript-eslint/types": "8.10.0", - "@typescript-eslint/typescript-estree": "8.10.0", - "@typescript-eslint/visitor-keys": "8.10.0", + "@typescript-eslint/scope-manager": "8.11.0", + "@typescript-eslint/types": "8.11.0", + "@typescript-eslint/typescript-estree": "8.11.0", + "@typescript-eslint/visitor-keys": "8.11.0", "debug": "^4.3.4" }, "engines": { @@ -2433,14 +2351,14 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.10.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.10.0.tgz", - "integrity": "sha512-AgCaEjhfql9MDKjMUxWvH7HjLeBqMCBfIaBbzzIcBbQPZE7CPh1m6FF+L75NUMJFMLYhCywJXIDEMa3//1A0dw==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.11.0.tgz", + "integrity": "sha512-Uholz7tWhXmA4r6epo+vaeV7yjdKy5QFCERMjs1kMVsLRKIrSdM6o21W2He9ftp5PP6aWOVpD5zvrvuHZC0bMQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.10.0", - "@typescript-eslint/visitor-keys": "8.10.0" + "@typescript-eslint/types": "8.11.0", + "@typescript-eslint/visitor-keys": "8.11.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2451,14 +2369,14 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.10.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.10.0.tgz", - "integrity": "sha512-PCpUOpyQSpxBn230yIcK+LeCQaXuxrgCm2Zk1S+PTIRJsEfU6nJ0TtwyH8pIwPK/vJoA+7TZtzyAJSGBz+s/dg==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.11.0.tgz", + "integrity": "sha512-ItiMfJS6pQU0NIKAaybBKkuVzo6IdnAhPFZA/2Mba/uBjuPQPet/8+zh5GtLHwmuFRShZx+8lhIs7/QeDHflOg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/typescript-estree": "8.10.0", - "@typescript-eslint/utils": "8.10.0", + "@typescript-eslint/typescript-estree": "8.11.0", + "@typescript-eslint/utils": "8.11.0", "debug": "^4.3.4", "ts-api-utils": "^1.3.0" }, @@ -2476,9 +2394,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "8.10.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.10.0.tgz", - "integrity": "sha512-k/E48uzsfJCRRbGLapdZgrX52csmWJ2rcowwPvOZ8lwPUv3xW6CcFeJAXgx4uJm+Ge4+a4tFOkdYvSpxhRhg1w==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.11.0.tgz", + "integrity": "sha512-tn6sNMHf6EBAYMvmPUaKaVeYvhUsrE6x+bXQTxjQRp360h1giATU0WvgeEys1spbvb5R+VpNOZ+XJmjD8wOUHw==", "dev": true, "license": "MIT", "engines": { @@ -2490,14 +2408,14 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.10.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.10.0.tgz", - "integrity": "sha512-3OE0nlcOHaMvQ8Xu5gAfME3/tWVDpb/HxtpUZ1WeOAksZ/h/gwrBzCklaGzwZT97/lBbbxJ16dMA98JMEngW4w==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.11.0.tgz", + "integrity": "sha512-yHC3s1z1RCHoCz5t06gf7jH24rr3vns08XXhfEqzYpd6Hll3z/3g23JRi0jM8A47UFKNc3u/y5KIMx8Ynbjohg==", "dev": true, "license": "BSD-2-Clause", "dependencies": { - "@typescript-eslint/types": "8.10.0", - "@typescript-eslint/visitor-keys": "8.10.0", + "@typescript-eslint/types": "8.11.0", + "@typescript-eslint/visitor-keys": "8.11.0", "debug": "^4.3.4", "fast-glob": "^3.3.2", "is-glob": "^4.0.3", @@ -2545,16 +2463,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.10.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.10.0.tgz", - "integrity": "sha512-Oq4uZ7JFr9d1ZunE/QKy5egcDRXT/FrS2z/nlxzPua2VHFtmMvFNDvpq1m/hq0ra+T52aUezfcjGRIB7vNJF9w==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.11.0.tgz", + "integrity": "sha512-CYiX6WZcbXNJV7UNB4PLDIBtSdRmRI/nb0FMyqHPTQD1rMjA0foPLaPUV39C/MxkTd/QKSeX+Gb34PPsDVC35g==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "8.10.0", - "@typescript-eslint/types": "8.10.0", - "@typescript-eslint/typescript-estree": "8.10.0" + "@typescript-eslint/scope-manager": "8.11.0", + "@typescript-eslint/types": "8.11.0", + "@typescript-eslint/typescript-estree": "8.11.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2568,13 +2486,13 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.10.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.10.0.tgz", - "integrity": "sha512-k8nekgqwr7FadWk548Lfph6V3r9OVqjzAIVskE7orMZR23cGJjAOVazsZSJW+ElyjfTM4wx/1g88Mi70DDtG9A==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.11.0.tgz", + "integrity": "sha512-EaewX6lxSjRJnc+99+dqzTeoDZUfyrA52d2/HRrkI830kgovWsmIiTfmr0NZorzqic7ga+1bS60lRBUgR3n/Bw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.10.0", + "@typescript-eslint/types": "8.11.0", "eslint-visitor-keys": "^3.4.3" }, "engines": { @@ -2768,9 +2686,9 @@ "license": "Apache-2.0" }, "node_modules/acorn": { - "version": "8.13.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.13.0.tgz", - "integrity": "sha512-8zSiw54Oxrdym50NlZ9sUusyO1Z1ZchgRLWRaK6c86XJFClyCgFKetdowBg5bKxyp/u+CDBJG4Mpp0m3HLZl9w==", + "version": "8.14.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", + "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", "dev": true, "license": "MIT", "bin": { @@ -3167,9 +3085,9 @@ } }, "node_modules/axe-core": { - "version": "4.10.1", - "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.10.1.tgz", - "integrity": "sha512-qPC9o+kD8Tir0lzNGLeghbOrWMr3ZJpaRlCIb6Uobt/7N4FiEDvqUMnxzCHRHmg8vOg14kr5gVNyScRmbMaJ9g==", + "version": "4.10.2", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.10.2.tgz", + "integrity": "sha512-RE3mdQ7P3FRSe7eqCWoeQ/Z9QXrtniSjp1wUjt5nRC3WIpz5rSCve6o3fsZ2aCpJtrZjSZgjwXAoTO5k4tEI0w==", "dev": true, "license": "MPL-2.0", "engines": { @@ -3243,9 +3161,9 @@ } }, "node_modules/browserslist": { - "version": "4.24.0", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.0.tgz", - "integrity": "sha512-Rmb62sR1Zpjql25eSanFGEhAxcFwfA1K0GuQcLoaJBAcENegrQut3hYdhXFF1obQfiDyqIW/cLM5HSJ/9k884A==", + "version": "4.24.2", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.2.tgz", + "integrity": "sha512-ZIc+Q62revdMcqC6aChtW4jz3My3klmCO1fEmINZY/8J3EpBg5/A/D0AKmBveUh6pgoeycoMkVMko84tuYS+Gg==", "dev": true, "funding": [ { @@ -3263,10 +3181,10 @@ ], "license": "MIT", "dependencies": { - "caniuse-lite": "^1.0.30001663", - "electron-to-chromium": "^1.5.28", + "caniuse-lite": "^1.0.30001669", + "electron-to-chromium": "^1.5.41", "node-releases": "^2.0.18", - "update-browserslist-db": "^1.1.0" + "update-browserslist-db": "^1.1.1" }, "bin": { "browserslist": "cli.js" @@ -3341,9 +3259,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001669", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001669.tgz", - "integrity": "sha512-DlWzFDJqstqtIVx1zeSpIMLjunf5SmwOw0N2Ck/QSQdS8PLS4+9HrLaYei4w8BIAL7IB/UEDu889d8vhCTPA0w==", + "version": "1.0.30001672", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001672.tgz", + "integrity": "sha512-XhW1vRo1ob6aeK2w3rTohwTPBLse/rvjq+s3RTSBwnlZqoFFjx9cHsShJjAIbLsLjyoacaTxpLZy9v3gg6zypw==", "funding": [ { "type": "opencollective", @@ -3485,9 +3403,9 @@ "license": "MIT" }, "node_modules/cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", "license": "MIT", "engines": { "node": ">= 0.6" @@ -3914,9 +3832,9 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.41", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.41.tgz", - "integrity": "sha512-dfdv/2xNjX0P8Vzme4cfzHqnPm5xsZXwsolTYr0eyW18IUmNyG08vL+fttvinTfhKfIKdRoqkDIC9e9iWQCNYQ==", + "version": "1.5.47", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.47.tgz", + "integrity": "sha512-zS5Yer0MOYw4rtK2iq43cJagHZ8sXN0jDHDKzB+86gSBSAI4v07S97mcq+Gs2vclAxSh1j7vOAHxSVgduiiuVQ==", "dev": true, "license": "ISC" }, @@ -4216,13 +4134,13 @@ } }, "node_modules/eslint-config-next": { - "version": "14.2.15", - "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-14.2.15.tgz", - "integrity": "sha512-mKg+NC/8a4JKLZRIOBplxXNdStgxy7lzWuedUaCc8tev+Al9mwDUTujQH6W6qXDH9kycWiVo28tADWGvpBsZcQ==", + "version": "14.2.16", + "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-14.2.16.tgz", + "integrity": "sha512-HOcnCJsyLXR7B8wmjaCgkTSpz+ijgOyAkP8OlvANvciP8PspBYFEBTmakNMxOf71fY0aKOm/blFIiKnrM4K03Q==", "dev": true, "license": "MIT", "dependencies": { - "@next/eslint-plugin-next": "14.2.15", + "@next/eslint-plugin-next": "14.2.16", "@rushstack/eslint-patch": "^1.3.3", "@typescript-eslint/eslint-plugin": "^5.4.2 || ^6.0.0 || ^7.0.0 || ^8.0.0", "@typescript-eslint/parser": "^5.4.2 || ^6.0.0 || ^7.0.0 || ^8.0.0", @@ -4410,9 +4328,9 @@ } }, "node_modules/eslint-plugin-jsx-a11y": { - "version": "6.10.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.10.1.tgz", - "integrity": "sha512-zHByM9WTUMnfsDTafGXRiqxp6lFtNoSOWBY6FonVRn3A+BUwN1L/tdBXT40BcBJi0cZjOGTXZ0eD/rTG9fEJ0g==", + "version": "6.10.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.10.2.tgz", + "integrity": "sha512-scB3nz4WmG75pV8+3eRUQOHZlNSUhFNq37xnpgRkCCELU3XMvXAxLk1eqWWyE22Ki4Q01Fnsw9BA3cJHDPgn2Q==", "dev": true, "license": "MIT", "dependencies": { @@ -4424,7 +4342,6 @@ "axobject-query": "^4.1.0", "damerau-levenshtein": "^1.0.8", "emoji-regex": "^9.2.2", - "es-iterator-helpers": "^1.1.0", "hasown": "^2.0.2", "jsx-ast-utils": "^3.3.5", "language-tags": "^1.0.9", @@ -4441,9 +4358,9 @@ } }, "node_modules/eslint-plugin-react": { - "version": "7.37.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.37.1.tgz", - "integrity": "sha512-xwTnwDqzbDRA8uJ7BMxPs/EXRB3i8ZfnOIp8BsxEQkT0nHPp+WWceqGgo6rKb9ctNi8GJLDT4Go5HAWELa/WMg==", + "version": "7.37.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.37.2.tgz", + "integrity": "sha512-EsTAnj9fLVr/GZleBLFbj/sSuXeWmp1eXIN60ceYnZveqEaUCyW4X+Vh4WTdUhCkW4xutXYqTXCUSyqD4rB75w==", "dev": true, "license": "MIT", "dependencies": { @@ -4452,7 +4369,7 @@ "array.prototype.flatmap": "^1.3.2", "array.prototype.tosorted": "^1.1.4", "doctrine": "^2.1.0", - "es-iterator-helpers": "^1.0.19", + "es-iterator-helpers": "^1.1.0", "estraverse": "^5.3.0", "hasown": "^2.0.2", "jsx-ast-utils": "^2.4.1 || ^3.0.0", @@ -6311,12 +6228,12 @@ "license": "MIT" }, "node_modules/next": { - "version": "14.2.15", - "resolved": "https://registry.npmjs.org/next/-/next-14.2.15.tgz", - "integrity": "sha512-h9ctmOokpoDphRvMGnwOJAedT6zKhwqyZML9mDtspgf4Rh3Pn7UTYKqePNoDvhsWBAO5GoPNYshnAUGIazVGmw==", + "version": "14.2.16", + "resolved": "https://registry.npmjs.org/next/-/next-14.2.16.tgz", + "integrity": "sha512-LcO7WnFu6lYSvCzZoo1dB+IO0xXz5uEv52HF1IUN0IqVTUIZGHuuR10I5efiLadGt+4oZqTcNZyVVEem/TM5nA==", "license": "MIT", "dependencies": { - "@next/env": "14.2.15", + "@next/env": "14.2.16", "@swc/helpers": "0.5.5", "busboy": "1.6.0", "caniuse-lite": "^1.0.30001579", @@ -6331,15 +6248,15 @@ "node": ">=18.17.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "14.2.15", - "@next/swc-darwin-x64": "14.2.15", - "@next/swc-linux-arm64-gnu": "14.2.15", - "@next/swc-linux-arm64-musl": "14.2.15", - "@next/swc-linux-x64-gnu": "14.2.15", - "@next/swc-linux-x64-musl": "14.2.15", - "@next/swc-win32-arm64-msvc": "14.2.15", - "@next/swc-win32-ia32-msvc": "14.2.15", - "@next/swc-win32-x64-msvc": "14.2.15" + "@next/swc-darwin-arm64": "14.2.16", + "@next/swc-darwin-x64": "14.2.16", + "@next/swc-linux-arm64-gnu": "14.2.16", + "@next/swc-linux-arm64-musl": "14.2.16", + "@next/swc-linux-x64-gnu": "14.2.16", + "@next/swc-linux-x64-musl": "14.2.16", + "@next/swc-win32-arm64-msvc": "14.2.16", + "@next/swc-win32-ia32-msvc": "14.2.16", + "@next/swc-win32-x64-msvc": "14.2.16" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", @@ -6361,14 +6278,14 @@ } }, "node_modules/next-auth": { - "version": "4.24.8", - "resolved": "https://registry.npmjs.org/next-auth/-/next-auth-4.24.8.tgz", - "integrity": "sha512-SLt3+8UCtklsotnz2p+nB4aN3IHNmpsQFAZ24VLxGotWGzSxkBh192zxNhm/J5wgkcrDWVp0bwqvW0HksK/Lcw==", + "version": "4.24.10", + "resolved": "https://registry.npmjs.org/next-auth/-/next-auth-4.24.10.tgz", + "integrity": "sha512-8NGqiRO1GXBcVfV8tbbGcUgQkAGsX4GRzzXXea4lDikAsJtD5KiEY34bfhUOjHLvr6rT6afpcxw2H8EZqOV6aQ==", "license": "ISC", "dependencies": { "@babel/runtime": "^7.20.13", "@panva/hkdf": "^1.0.2", - "cookie": "^0.5.0", + "cookie": "^0.7.0", "jose": "^4.15.5", "oauth": "^0.9.15", "openid-client": "^5.4.0", @@ -6378,7 +6295,7 @@ }, "peerDependencies": { "@auth/core": "0.34.2", - "next": "^12.2.5 || ^13 || ^14", + "next": "^12.2.5 || ^13 || ^14 || ^15", "nodemailer": "^6.6.5", "react": "^17.0.2 || ^18", "react-dom": "^17.0.2 || ^18" @@ -7235,9 +7152,9 @@ } }, "node_modules/react-select": { - "version": "5.8.1", - "resolved": "https://registry.npmjs.org/react-select/-/react-select-5.8.1.tgz", - "integrity": "sha512-RT1CJmuc+ejqm5MPgzyZujqDskdvB9a9ZqrdnVLsvAHjJ3Tj0hELnLeVPQlmYdVKCdCpxanepl6z7R5KhXhWzg==", + "version": "5.8.2", + "resolved": "https://registry.npmjs.org/react-select/-/react-select-5.8.2.tgz", + "integrity": "sha512-a/LkOckoI62710gGPQSQqUp7A10fGbH/ya3/IR49qaq3XoBvwymgD5mJgtiHxBDsutyEQfdKNycWVh8Cg8UCjw==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.12.0", @@ -8474,15 +8391,6 @@ "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==", "license": "MIT" }, - "node_modules/to-fast-properties": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", - "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",