From bb3ad271298d8c6eb1d3905e8611a064056b2964 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Tue, 23 Jul 2024 12:11:55 -0700 Subject: [PATCH 1/5] x/sdk/test: Upgrade testcontainer-go And redpanda module. To grab support for arbitrary bootstrap configs. Also removes usage of deprecated methods and makes some minor tweaks to account for new version. Signed-off-by: Oren Leiman --- src/transform-sdk/tests/go.mod | 43 ++++++++++++++++------------ src/transform-sdk/tests/go.sum | 37 ++++++++++++++++++++++++ src/transform-sdk/tests/test_util.go | 10 ++++--- 3 files changed, 67 insertions(+), 23 deletions(-) diff --git a/src/transform-sdk/tests/go.mod b/src/transform-sdk/tests/go.mod index 8b7bde7af451..d599c65930f2 100644 --- a/src/transform-sdk/tests/go.mod +++ b/src/transform-sdk/tests/go.mod @@ -5,32 +5,37 @@ go 1.21.5 require ( github.com/hamba/avro v1.8.0 github.com/stretchr/testify v1.9.0 - github.com/testcontainers/testcontainers-go v0.29.1 - github.com/testcontainers/testcontainers-go/modules/redpanda v0.29.1 + github.com/testcontainers/testcontainers-go v0.33.0 + github.com/testcontainers/testcontainers-go/modules/redpanda v0.33.0 github.com/twmb/franz-go v1.16.1 github.com/twmb/franz-go/pkg/kadm v1.11.0 ) +require ( + github.com/containerd/platforms v0.2.1 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect +) + require ( dario.cat/mergo v1.0.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/Microsoft/hcsshim v0.11.4 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Microsoft/hcsshim v0.11.5 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/containerd/containerd v1.7.12 // indirect + github.com/containerd/containerd v1.7.18 // indirect github.com/containerd/log v0.1.0 // indirect github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/distribution/reference v0.5.0 // indirect - github.com/docker/docker v25.0.3+incompatible // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v27.1.1+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/go-logr/logr v1.2.4 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.4 // indirect @@ -57,17 +62,17 @@ require ( github.com/twmb/franz-go/pkg/kmsg v1.7.0 // indirect github.com/twmb/franz-go/pkg/sr v0.0.0-20240307025822-351e7fae879c github.com/yusufpapurcu/wmi v1.2.3 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect - go.opentelemetry.io/otel v1.19.0 // indirect - go.opentelemetry.io/otel/metric v1.19.0 // indirect - go.opentelemetry.io/otel/trace v1.19.0 // indirect - golang.org/x/crypto v0.17.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + golang.org/x/crypto v0.24.0 // indirect golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect golang.org/x/mod v0.16.0 // indirect - golang.org/x/sys v0.16.0 // indirect + golang.org/x/sys v0.21.0 // indirect golang.org/x/tools v0.13.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect - google.golang.org/grpc v1.58.3 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/grpc v1.64.1 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/src/transform-sdk/tests/go.sum b/src/transform-sdk/tests/go.sum index db8552725879..c4bf5a7aa155 100644 --- a/src/transform-sdk/tests/go.sum +++ b/src/transform-sdk/tests/go.sum @@ -6,14 +6,21 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= +github.com/Microsoft/hcsshim v0.11.5/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/containerd/containerd v1.7.12 h1:+KQsnv4VnzyxWcfO9mlxxELaoztsDEjOuCMPAuPqgU0= github.com/containerd/containerd v1.7.12/go.mod h1:/5OMpE1p0ylxtEUGY8kuCYkDRzJm9NO1TFMWjUpdevk= +github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= +github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -23,17 +30,25 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/docker v25.0.3+incompatible h1:D5fy/lYmY7bvZa0XTZ5/UJPljor41F+vdyJG5luQLfQ= github.com/docker/docker v25.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= +github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -43,6 +58,7 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -70,6 +86,8 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= @@ -120,8 +138,12 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/testcontainers/testcontainers-go v0.29.1 h1:z8kxdFlovA2y97RWx98v/TQ+tR+SXZm6p35M+xB92zk= github.com/testcontainers/testcontainers-go v0.29.1/go.mod h1:SnKnKQav8UcgtKqjp/AD8bE1MqZm+3TDb/B8crE3XnI= +github.com/testcontainers/testcontainers-go v0.33.0 h1:zJS9PfXYT5O0ZFXM2xxXfk4J5UMw/kRiISng037Gxdw= +github.com/testcontainers/testcontainers-go v0.33.0/go.mod h1:W80YpTa8D5C3Yy16icheD01UTDu+LmXIA2Keo+jWtT8= github.com/testcontainers/testcontainers-go/modules/redpanda v0.29.1 h1:wqmZd30jKS1L87Va/zl7QLgjVaE/PUwjZ8XfVAAie/I= github.com/testcontainers/testcontainers-go/modules/redpanda v0.29.1/go.mod h1:URq0cHQv6JId/oO+ErrrTFhN6QV+TDr6D32sz7xwV5Y= +github.com/testcontainers/testcontainers-go/modules/redpanda v0.33.0 h1:VscyHm3YRhU7MeaDFTCkesifv4Ff9NfqCrrQA056JkI= +github.com/testcontainers/testcontainers-go/modules/redpanda v0.33.0/go.mod h1:VLRAi4dprZ4nFAhgaqUlIxhfOVicz8SG33Ol54OEZug= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -140,18 +162,26 @@ github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFi github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -159,6 +189,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4= golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -188,6 +220,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= @@ -209,12 +243,15 @@ google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1: google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/src/transform-sdk/tests/test_util.go b/src/transform-sdk/tests/test_util.go index bba45c7f9acb..2c1ffa3ab9ab 100644 --- a/src/transform-sdk/tests/test_util.go +++ b/src/transform-sdk/tests/test_util.go @@ -49,15 +49,17 @@ func (lc *stdoutLogConsumer) Accept(l testcontainers.Log) { // startRedpanda runs the Redpanda binary with a data transforms enabled. func startRedpanda(ctx context.Context) (*redpanda.Container, context.CancelFunc) { - redpandaContainer, err := redpanda.RunContainer(ctx, + redpandaContainer, err := redpanda.Run( + ctx, + "redpandadata/redpanda-nightly:latest", testcontainers.WithLogger(log.Default()), - testcontainers.WithImage("redpandadata/redpanda-nightly:latest"), - testcontainers.CustomizeRequestOption(func(req *testcontainers.GenericContainerRequest) { + testcontainers.CustomizeRequestOption(func(req *testcontainers.GenericContainerRequest) error { if req.LogConsumerCfg == nil { req.LogConsumerCfg = &testcontainers.LogConsumerConfig{} } // Uncomment this to get broker logs - // req.LogConsumerCfg.Consumers = append(req.LogConsumerCfg.Consumers, &stdoutLogConsumer{}) + req.LogConsumerCfg.Consumers = append(req.LogConsumerCfg.Consumers, &stdoutLogConsumer{}) + return nil }), redpanda.WithEnableWasmTransform(), ) From cb5514bc62dd51c9d82bcc4a32d45c4eb82e8b53 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 24 Jul 2024 12:26:21 -0700 Subject: [PATCH 2/5] x/sdk/test: Crank the transform memory limits in the testcontainer This is mostly for JavaScript, but shouldn't have any ill effect on other SDKs. Signed-off-by: Oren Leiman --- src/transform-sdk/tests/test_util.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transform-sdk/tests/test_util.go b/src/transform-sdk/tests/test_util.go index 2c1ffa3ab9ab..1c88c216ef78 100644 --- a/src/transform-sdk/tests/test_util.go +++ b/src/transform-sdk/tests/test_util.go @@ -62,6 +62,8 @@ func startRedpanda(ctx context.Context) (*redpanda.Container, context.CancelFunc return nil }), redpanda.WithEnableWasmTransform(), + redpanda.WithBootstrapConfig("data_transforms_per_core_memory_reservation", 135000000), + redpanda.WithBootstrapConfig("data_transforms_per_function_memory_limit", 16777216), ) if err != nil { log.Fatalf("failed to start container: %s", err) From a89af685ebe934ebf83e71cfddfd33b8203b93ef Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Fri, 19 Jul 2024 11:50:52 -0700 Subject: [PATCH 3/5] x/sdk: schema_registry_example Depends on latest avsc, which is unreleased. In the next commit we point directly to the head of avsc master branch as of 2024.08.22. Signed-off-by: Oren Leiman --- .../js/schema_registry_example/.gitignore | 3 + .../js/schema_registry_example/esbuild.js | 29 + .../schema_registry_example/package-lock.json | 500 ++++++++++++++++++ .../js/schema_registry_example/package.json | 18 + .../js/schema_registry_example/src/index.ts | 41 ++ 5 files changed, 591 insertions(+) create mode 100644 src/transform-sdk/js/schema_registry_example/.gitignore create mode 100644 src/transform-sdk/js/schema_registry_example/esbuild.js create mode 100644 src/transform-sdk/js/schema_registry_example/package-lock.json create mode 100644 src/transform-sdk/js/schema_registry_example/package.json create mode 100644 src/transform-sdk/js/schema_registry_example/src/index.ts diff --git a/src/transform-sdk/js/schema_registry_example/.gitignore b/src/transform-sdk/js/schema_registry_example/.gitignore new file mode 100644 index 000000000000..dd4bb9d11f82 --- /dev/null +++ b/src/transform-sdk/js/schema_registry_example/.gitignore @@ -0,0 +1,3 @@ +node_modules +node_modules/**/* +user_code.js diff --git a/src/transform-sdk/js/schema_registry_example/esbuild.js b/src/transform-sdk/js/schema_registry_example/esbuild.js new file mode 100644 index 000000000000..c180124363f4 --- /dev/null +++ b/src/transform-sdk/js/schema_registry_example/esbuild.js @@ -0,0 +1,29 @@ +import * as esbuild from 'esbuild' +import { polyfillNode } from "esbuild-plugin-polyfill-node"; + +await esbuild.build({ + entryPoints: ["src/index.ts"], + outfile: "dist/schema_registry_example.js", + bundle: true, + external: [ + // This package is provided by the Redpanda JavaScript runtime. + "@redpanda-data/transform-sdk", + "@redpanda-data/transform-sdk-sr", + ], + target: "es2022", + platform: "neutral", // We're running in Wasm + plugins: [ + polyfillNode({ + globals: { + // Allow a global Buffer variable if referenced. + buffer: true, + // Don't inject the process global, the Redpanda JavaScript runtime + // does that. + process: false, + }, + polyfills: { + // Any NodeJS APIs that need to polyfilled can be added here. + }, + }), + ], +}); diff --git a/src/transform-sdk/js/schema_registry_example/package-lock.json b/src/transform-sdk/js/schema_registry_example/package-lock.json new file mode 100644 index 000000000000..f48f2dfa8dea --- /dev/null +++ b/src/transform-sdk/js/schema_registry_example/package-lock.json @@ -0,0 +1,500 @@ +{ + "name": "schema_registry_example", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "schema_registry_example", + "dependencies": { + "@redpanda-data/transform-sdk": "file:../module", + "@redpanda-data/transform-sdk-sr": "file:../sr-module", + "avsc": "^5.7.7" + }, + "devDependencies": { + "esbuild": "0.20.x", + "esbuild-plugin-polyfill-node": "0.3.x", + "typescript": "5.x" + } + }, + "../module": { + "name": "@redpanda-data/transform-sdk", + "version": "0.1.0", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "../sr-module": { + "name": "@redpanda-data/transform-sdk-sr", + "version": "0.1.0", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.20.2.tgz", + "integrity": "sha512-D+EBOJHXdNZcLJRBkhENNG8Wji2kgc9AZ9KiPr1JuZjsNtyHzrsfLRrY0tk2H2aoFu6RANO1y1iPPUCDYWkb5g==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.20.2.tgz", + "integrity": "sha512-t98Ra6pw2VaDhqNWO2Oph2LXbz/EJcnLmKLGBJwEwXX/JAN83Fym1rU8l0JUWK6HkIbWONCSSatf4sf2NBRx/w==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.20.2.tgz", + "integrity": "sha512-mRzjLacRtl/tWU0SvD8lUEwb61yP9cqQo6noDZP/O8VkwafSYwZ4yWy24kan8jE/IMERpYncRt2dw438LP3Xmg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.20.2.tgz", + "integrity": "sha512-btzExgV+/lMGDDa194CcUQm53ncxzeBrWJcncOBxuC6ndBkKxnHdFJn86mCIgTELsooUmwUm9FkhSp5HYu00Rg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.20.2.tgz", + "integrity": "sha512-4J6IRT+10J3aJH3l1yzEg9y3wkTDgDk7TSDFX+wKFiWjqWp/iCfLIYzGyasx9l0SAFPT1HwSCR+0w/h1ES/MjA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.20.2.tgz", + "integrity": "sha512-tBcXp9KNphnNH0dfhv8KYkZhjc+H3XBkF5DKtswJblV7KlT9EI2+jeA8DgBjp908WEuYll6pF+UStUCfEpdysA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.20.2.tgz", + "integrity": "sha512-d3qI41G4SuLiCGCFGUrKsSeTXyWG6yem1KcGZVS+3FYlYhtNoNgYrWcvkOoaqMhwXSMrZRl69ArHsGJ9mYdbbw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.20.2.tgz", + "integrity": "sha512-d+DipyvHRuqEeM5zDivKV1KuXn9WeRX6vqSqIDgwIfPQtwMP4jaDsQsDncjTDDsExT4lR/91OLjRo8bmC1e+Cw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.20.2.tgz", + "integrity": "sha512-VhLPeR8HTMPccbuWWcEUD1Az68TqaTYyj6nfE4QByZIQEQVWBB8vup8PpR7y1QHL3CpcF6xd5WVBU/+SBEvGTg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.20.2.tgz", + "integrity": "sha512-9pb6rBjGvTFNira2FLIWqDk/uaf42sSyLE8j1rnUpuzsODBq7FvpwHYZxQ/It/8b+QOS1RYfqgGFNLRI+qlq2A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.20.2.tgz", + "integrity": "sha512-o10utieEkNPFDZFQm9CoP7Tvb33UutoJqg3qKf1PWVeeJhJw0Q347PxMvBgVVFgouYLGIhFYG0UGdBumROyiig==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.20.2.tgz", + "integrity": "sha512-PR7sp6R/UC4CFVomVINKJ80pMFlfDfMQMYynX7t1tNTeivQ6XdX5r2XovMmha/VjR1YN/HgHWsVcTRIMkymrgQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.20.2.tgz", + "integrity": "sha512-4BlTqeutE/KnOiTG5Y6Sb/Hw6hsBOZapOVF6njAESHInhlQAghVVZL1ZpIctBOoTFbQyGW+LsVYZ8lSSB3wkjA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.20.2.tgz", + "integrity": "sha512-rD3KsaDprDcfajSKdn25ooz5J5/fWBylaaXkuotBDGnMnDP1Uv5DLAN/45qfnf3JDYyJv/ytGHQaziHUdyzaAg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.20.2.tgz", + "integrity": "sha512-snwmBKacKmwTMmhLlz/3aH1Q9T8v45bKYGE3j26TsaOVtjIag4wLfWSiZykXzXuE1kbCE+zJRmwp+ZbIHinnVg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.20.2.tgz", + "integrity": "sha512-wcWISOobRWNm3cezm5HOZcYz1sKoHLd8VL1dl309DiixxVFoFe/o8HnwuIwn6sXre88Nwj+VwZUvJf4AFxkyrQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.20.2.tgz", + "integrity": "sha512-1MdwI6OOTsfQfek8sLwgyjOXAu+wKhLEoaOLTjbijk6E2WONYpH9ZU2mNtR+lZ2B4uwr+usqGuVfFT9tMtGvGw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.20.2.tgz", + "integrity": "sha512-K8/DhBxcVQkzYc43yJXDSyjlFeHQJBiowJ0uVL6Tor3jGQfSGHNNJcWxNbOI8v5k82prYqzPuwkzHt3J1T1iZQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.20.2.tgz", + "integrity": "sha512-eMpKlV0SThJmmJgiVyN9jTPJ2VBPquf6Kt/nAoo6DgHAoN57K15ZghiHaMvqjCye/uU4X5u3YSMgVBI1h3vKrQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.20.2.tgz", + "integrity": "sha512-2UyFtRC6cXLyejf/YEld4Hajo7UHILetzE1vsRcGL3earZEW77JxrFjH4Ez2qaTiEfMgAXxfAZCm1fvM/G/o8w==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.20.2.tgz", + "integrity": "sha512-GRibxoawM9ZCnDxnP3usoUDO9vUkpAxIIZ6GQI+IlVmr5kP3zUq+l17xELTHMWTWzjxa2guPNyrpq1GWmPvcGQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.20.2.tgz", + "integrity": "sha512-HfLOfn9YWmkSKRQqovpnITazdtquEW8/SoHW7pWpuEeguaZI4QnCRW6b+oZTztdBnZOS2hqJ6im/D5cPzBTTlQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.20.2.tgz", + "integrity": "sha512-N49X4lJX27+l9jbLKSqZ6bKNjzQvHaT8IIFUy+YIqmXQdjYCToGWwOItDrfby14c78aDd5NHQl29xingXfCdLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@jspm/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@jspm/core/-/core-2.0.1.tgz", + "integrity": "sha512-Lg3PnLp0QXpxwLIAuuJboLeRaIhrgJjeuh797QADg3xz8wGLugQOS5DpsE8A6i6Adgzf+bacllkKZG3J0tGfDw==", + "dev": true + }, + "node_modules/@redpanda-data/transform-sdk": { + "resolved": "../module", + "link": true + }, + "node_modules/@redpanda-data/transform-sdk-sr": { + "resolved": "../sr-module", + "link": true + }, + "node_modules/avsc": { + "version": "5.7.7", + "resolved": "https://registry.npmjs.org/avsc/-/avsc-5.7.7.tgz", + "integrity": "sha512-9cYNccliXZDByFsFliVwk5GvTq058Fj513CiR4E60ndDwmuXzTJEp/Bp8FyuRmGyYupLjHLs+JA9/CBoVS4/NQ==", + "engines": { + "node": ">=0.11" + } + }, + "node_modules/esbuild": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.2.tgz", + "integrity": "sha512-WdOOppmUNU+IbZ0PaDiTst80zjnrOkyJNHoKupIcVyU8Lvla3Ugx94VzkQ32Ijqd7UhHJy75gNWDMUekcrSJ6g==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.20.2", + "@esbuild/android-arm": "0.20.2", + "@esbuild/android-arm64": "0.20.2", + "@esbuild/android-x64": "0.20.2", + "@esbuild/darwin-arm64": "0.20.2", + "@esbuild/darwin-x64": "0.20.2", + "@esbuild/freebsd-arm64": "0.20.2", + "@esbuild/freebsd-x64": "0.20.2", + "@esbuild/linux-arm": "0.20.2", + "@esbuild/linux-arm64": "0.20.2", + "@esbuild/linux-ia32": "0.20.2", + "@esbuild/linux-loong64": "0.20.2", + "@esbuild/linux-mips64el": "0.20.2", + "@esbuild/linux-ppc64": "0.20.2", + "@esbuild/linux-riscv64": "0.20.2", + "@esbuild/linux-s390x": "0.20.2", + "@esbuild/linux-x64": "0.20.2", + "@esbuild/netbsd-x64": "0.20.2", + "@esbuild/openbsd-x64": "0.20.2", + "@esbuild/sunos-x64": "0.20.2", + "@esbuild/win32-arm64": "0.20.2", + "@esbuild/win32-ia32": "0.20.2", + "@esbuild/win32-x64": "0.20.2" + } + }, + "node_modules/esbuild-plugin-polyfill-node": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/esbuild-plugin-polyfill-node/-/esbuild-plugin-polyfill-node-0.3.0.tgz", + "integrity": "sha512-SHG6CKUfWfYyYXGpW143NEZtcVVn8S/WHcEOxk62LuDXnY4Zpmc+WmxJKN6GMTgTClXJXhEM5KQlxKY6YjbucQ==", + "dev": true, + "dependencies": { + "@jspm/core": "^2.0.1", + "import-meta-resolve": "^3.0.0" + }, + "peerDependencies": { + "esbuild": "*" + } + }, + "node_modules/import-meta-resolve": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-3.1.1.tgz", + "integrity": "sha512-qeywsE/KC3w9Fd2ORrRDUw6nS/nLwZpXgfrOc2IILvZYnCaEMd+D56Vfg9k4G29gIeVi3XKql1RQatME8iYsiw==", + "dev": true, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/typescript": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", + "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + } + } +} diff --git a/src/transform-sdk/js/schema_registry_example/package.json b/src/transform-sdk/js/schema_registry_example/package.json new file mode 100644 index 000000000000..e109f712d317 --- /dev/null +++ b/src/transform-sdk/js/schema_registry_example/package.json @@ -0,0 +1,18 @@ +{ + "name": "schema_registry_example", + "type": "module", + "private": true, + "scripts": { + "build": "node esbuild.js" + }, + "dependencies": { + "@redpanda-data/transform-sdk": "file:../module", + "@redpanda-data/transform-sdk-sr": "file:../sr-module", + "avsc": "^5.7.7" + }, + "devDependencies": { + "esbuild": "0.20.x", + "esbuild-plugin-polyfill-node": "0.3.x", + "typescript": "5.x" + } +} diff --git a/src/transform-sdk/js/schema_registry_example/src/index.ts b/src/transform-sdk/js/schema_registry_example/src/index.ts new file mode 100644 index 000000000000..cb124fe917ca --- /dev/null +++ b/src/transform-sdk/js/schema_registry_example/src/index.ts @@ -0,0 +1,41 @@ +import { onRecordWritten, OnRecordWrittenEvent, RecordWriter } from "@redpanda-data/transform-sdk" +import { newClient, decodeSchemaID } from "@redpanda-data/transform-sdk-sr"; +import { Type } from "avsc/lib/index.js" +import { Buffer } from "buffer" + +var sr_client = newClient(); +const schema = { + type: "record", + name: "Example", + fields: [ + { "name": "a", "type": "long", "default": 0 }, + { "name": "b", "type": "string", "default": "" } + ] +}; + +const subj_schema = sr_client.createSchema( + "avro-value", + { + schema: JSON.stringify(schema), + format: 0,//SchemaFormat.Avro, + references: [], + } +); + +const orig_t = Type.forSchema(JSON.parse(subj_schema.schema.schema)); + +onRecordWritten((event: OnRecordWrittenEvent, writer: RecordWriter) => { + var arr = event.record.value.array(); + const decoded = decodeSchemaID(arr); + var schema = sr_client.lookupSchemaById(decoded.id); + const t = Type.forSchema(JSON.parse(schema["schema"])); + var buf = Buffer.from(decoded.rest); + var val = t.fromBuffer(buf); + buf = orig_t.toBuffer(val); + val = orig_t.fromBuffer(buf); + writer.write({ + key: event.record.key, + value: JSON.stringify(val), + headers: [], + }); +}); From 4a1b3ad3243ec5421de237a3eeb62b893b49182b Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 24 Jul 2024 12:32:56 -0700 Subject: [PATCH 4/5] x/sdk/sr_example: Point avsc at main branch head The main branch has some unreleased improvements that we need to run in a wasm env. When (if) avsc v6 drops, we can remove this and point the npm repos. Signed-off-by: Oren Leiman --- .../schema_registry_example/package-lock.json | 46 +++++++++++++++---- .../js/schema_registry_example/package.json | 2 +- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/transform-sdk/js/schema_registry_example/package-lock.json b/src/transform-sdk/js/schema_registry_example/package-lock.json index f48f2dfa8dea..0175bd57c93d 100644 --- a/src/transform-sdk/js/schema_registry_example/package-lock.json +++ b/src/transform-sdk/js/schema_registry_example/package-lock.json @@ -8,7 +8,7 @@ "dependencies": { "@redpanda-data/transform-sdk": "file:../module", "@redpanda-data/transform-sdk-sr": "file:../sr-module", - "avsc": "^5.7.7" + "avsc": "github:mtth/avsc#7cb76a6eb7b0d5b5a71c9cfb446acf3eff763006" }, "devDependencies": { "esbuild": "0.20.x", @@ -17,7 +17,6 @@ } }, "../module": { - "name": "@redpanda-data/transform-sdk", "version": "0.1.0", "license": "Apache-2.0", "engines": { @@ -25,7 +24,6 @@ } }, "../sr-module": { - "name": "@redpanda-data/transform-sdk-sr", "version": "0.1.0", "license": "Apache-2.0", "engines": { @@ -40,6 +38,7 @@ "ppc64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "aix" @@ -56,6 +55,7 @@ "arm" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "android" @@ -72,6 +72,7 @@ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "android" @@ -88,6 +89,7 @@ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "android" @@ -104,6 +106,7 @@ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "darwin" @@ -120,6 +123,7 @@ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "darwin" @@ -136,6 +140,7 @@ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "freebsd" @@ -152,6 +157,7 @@ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "freebsd" @@ -168,6 +174,7 @@ "arm" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -184,6 +191,7 @@ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -200,6 +208,7 @@ "ia32" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -216,6 +225,7 @@ "loong64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -232,6 +242,7 @@ "mips64el" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -248,6 +259,7 @@ "ppc64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -264,6 +276,7 @@ "riscv64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -280,6 +293,7 @@ "s390x" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -296,6 +310,7 @@ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -312,6 +327,7 @@ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "netbsd" @@ -328,6 +344,7 @@ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "openbsd" @@ -344,6 +361,7 @@ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "sunos" @@ -360,6 +378,7 @@ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" @@ -376,6 +395,7 @@ "ia32" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" @@ -392,6 +412,7 @@ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" @@ -415,11 +436,12 @@ "link": true }, "node_modules/avsc": { - "version": "5.7.7", - "resolved": "https://registry.npmjs.org/avsc/-/avsc-5.7.7.tgz", - "integrity": "sha512-9cYNccliXZDByFsFliVwk5GvTq058Fj513CiR4E60ndDwmuXzTJEp/Bp8FyuRmGyYupLjHLs+JA9/CBoVS4/NQ==", + "version": "6.0.0-rc.1", + "resolved": "git+ssh://git@github.com/mtth/avsc.git#7cb76a6eb7b0d5b5a71c9cfb446acf3eff763006", + "integrity": "sha512-ZTglgwKEW1f3yGP2vogFv0sTz6um10wo1f1uPbk4cfKRU/QJfWDAnp2/KkYWr79m/pZ+8Q8vvUGOQbgzQG1amQ==", + "license": "MIT", "engines": { - "node": ">=0.11" + "node": ">=6.0.0" } }, "node_modules/esbuild": { @@ -428,6 +450,7 @@ "integrity": "sha512-WdOOppmUNU+IbZ0PaDiTst80zjnrOkyJNHoKupIcVyU8Lvla3Ugx94VzkQ32Ijqd7UhHJy75gNWDMUekcrSJ6g==", "dev": true, "hasInstallScript": true, + "license": "MIT", "bin": { "esbuild": "bin/esbuild" }, @@ -465,6 +488,7 @@ "resolved": "https://registry.npmjs.org/esbuild-plugin-polyfill-node/-/esbuild-plugin-polyfill-node-0.3.0.tgz", "integrity": "sha512-SHG6CKUfWfYyYXGpW143NEZtcVVn8S/WHcEOxk62LuDXnY4Zpmc+WmxJKN6GMTgTClXJXhEM5KQlxKY6YjbucQ==", "dev": true, + "license": "MIT", "dependencies": { "@jspm/core": "^2.0.1", "import-meta-resolve": "^3.0.0" @@ -478,16 +502,18 @@ "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-3.1.1.tgz", "integrity": "sha512-qeywsE/KC3w9Fd2ORrRDUw6nS/nLwZpXgfrOc2IILvZYnCaEMd+D56Vfg9k4G29gIeVi3XKql1RQatME8iYsiw==", "dev": true, + "license": "MIT", "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" } }, "node_modules/typescript": { - "version": "5.5.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", - "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==", + "version": "5.5.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", + "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", "dev": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/src/transform-sdk/js/schema_registry_example/package.json b/src/transform-sdk/js/schema_registry_example/package.json index e109f712d317..6ff4b2d4031e 100644 --- a/src/transform-sdk/js/schema_registry_example/package.json +++ b/src/transform-sdk/js/schema_registry_example/package.json @@ -8,7 +8,7 @@ "dependencies": { "@redpanda-data/transform-sdk": "file:../module", "@redpanda-data/transform-sdk-sr": "file:../sr-module", - "avsc": "^5.7.7" + "avsc": "github:mtth/avsc#7cb76a6eb7b0d5b5a71c9cfb446acf3eff763006" }, "devDependencies": { "esbuild": "0.20.x", From 48f4edb0765b736cc9d7d572ccbcb754854fd943 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 17 Jul 2024 16:01:29 -0700 Subject: [PATCH 5/5] x/sdk: Enable SR integration test for JavaScript SDK Signed-off-by: Oren Leiman --- .github/workflows/transform-sdk-build.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/transform-sdk-build.yml b/.github/workflows/transform-sdk-build.yml index 3bae4b9cc82a..ffdf25589284 100644 --- a/.github/workflows/transform-sdk-build.yml +++ b/.github/workflows/transform-sdk-build.yml @@ -258,6 +258,10 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 + - name: Install node + uses: actions/setup-node@v4 + with: + node-version: 20 - name: Build integration tests working-directory: src/transform-sdk/js run: | @@ -267,6 +271,8 @@ jobs: cmake --build --preset release-static -- redpanda_js_transform cat identity.js | python3 generate_js_provider.py > identity_source.wat cat identity_logging.js | python3 generate_js_provider.py > logging_source.wat + (cd schema_registry_example && npm install && npm run build) + cat schema_registry_example/dist/schema_registry_example.js | python3 generate_js_provider.py > sr_source.wat curl -SLO https://github.com/WebAssembly/binaryen/releases/download/version_${{ env.BINARYEN_VERSION }}/binaryen-version_${{ env.BINARYEN_VERSION }}-x86_64-linux.tar.gz tar xf binaryen-version_${{ env.BINARYEN_VERSION }}-x86_64-linux.tar.gz rm binaryen-version_${{ env.BINARYEN_VERSION }}-x86_64-linux.tar.gz @@ -280,6 +286,11 @@ jobs: ./logging_source.wat redpanda_js_provider \ -mvp --enable-simd --enable-bulk-memory --enable-multimemory \ -o logging.wasm + ./binaryen-version_117/bin/wasm-merge \ + ./build/release-static/redpanda_js_transform js_vm \ + ./sr_source.wat redpanda_js_provider \ + -mvp --enable-simd --enable-bulk-memory --enable-multimemory \ + -o sr.wasm - name: Download integration test suite uses: actions/download-artifact@v4 with: @@ -291,7 +302,7 @@ jobs: IDENTITY: identity.wasm LOGGING: logging.wasm TEE: "@UNIMPLEMENTED@" - SCHEMA_REGISTRY: "@UNIMPLEMENTED@" + SCHEMA_REGISTRY: sr.wasm run: | chmod +x wasm-integration-test ./wasm-integration-test -test.v