[PT-5027] Add ruby 3.0 and 3.1 test matrix support

.circleci/config.yml

@@ -1,99 +1,48 @@
-version: 2
+version: 2.1
 jobs:
   unit:
+    parameters:
+      ruby-version:
+        type: string
     docker:
-      - image: circleci/ruby:2.4.1-node
+      - image: cimg/ruby:<< parameters.ruby-version >>
         environment:
           LOG_LEVEL: DEBUG
     steps:
       - checkout
+      - run: ruby --version
+      - run: sudo apt-get update && sudo apt-get install -y cmake # For installing snappy
       - run: bundle install --path vendor/bundle
       - run: bundle exec rspec
       - run: bundle exec rubocop
 
-  kafka-0.10:
+  kafka-with-wurstmeister:
+  #TODO: replace with confluentinc/cp-kafka which is used in production
+    parameters:
+      kafka-version:
+        type: string
+      ruby-version:
+        type: string
     docker:
-      - image: circleci/ruby:2.4.1-node
+      - image: cimg/ruby:<< parameters.ruby-version >>
         environment:
           LOG_LEVEL: DEBUG
       - image: wurstmeister/zookeeper
-      - image: wurstmeister/kafka:0.10.2.1
+      - image: wurstmeister/kafka:<< parameters.kafka-version >>
         environment:
           KAFKA_ADVERTISED_HOST_NAME: localhost
           KAFKA_ADVERTISED_PORT: 9092
           KAFKA_PORT: 9092
           KAFKA_ZOOKEEPER_CONNECT: localhost:2181
           KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:0.10.2.1
+      - image: wurstmeister/kafka:<< parameters.kafka-version >>
         environment:
           KAFKA_ADVERTISED_HOST_NAME: localhost
           KAFKA_ADVERTISED_PORT: 9093
           KAFKA_PORT: 9093
           KAFKA_ZOOKEEPER_CONNECT: localhost:2181
           KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:0.10.2.1
-        environment:
-          KAFKA_ADVERTISED_HOST_NAME: localhost
-          KAFKA_ADVERTISED_PORT: 9094
-          KAFKA_PORT: 9094
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_DELETE_TOPIC_ENABLE: true
-    steps:
-      - checkout
-      - run: bundle install --path vendor/bundle
-      - run: bundle exec rspec --profile --tag functional spec/functional
-
-  kafka-0.11:
-    docker:
-      - image: circleci/ruby:2.4.1-node
-        environment:
-          LOG_LEVEL: DEBUG
-      - image: wurstmeister/zookeeper
-      - image: wurstmeister/kafka:0.11.0.1
-        environment:
-          KAFKA_ADVERTISED_HOST_NAME: localhost
-          KAFKA_ADVERTISED_PORT: 9092
-          KAFKA_PORT: 9092
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:0.11.0.1
-        environment:
-          KAFKA_PORT: 9093
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:0.11.0.1
-        environment:
-          KAFKA_ADVERTISED_HOST_NAME: localhost
-          KAFKA_ADVERTISED_PORT: 9094
-          KAFKA_PORT: 9094
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_DELETE_TOPIC_ENABLE: true
-    steps:
-      - checkout
-      - run: bundle install --path vendor/bundle
-      - run: bundle exec rspec --profile --tag functional spec/functional
-
-  kafka-1.0:
-    docker:
-      - image: circleci/ruby:2.4.1-node
-        environment:
-          LOG_LEVEL: DEBUG
-      - image: wurstmeister/zookeeper
-      - image: wurstmeister/kafka:1.0.0
-        environment:
-          KAFKA_ADVERTISED_HOST_NAME: localhost
-          KAFKA_ADVERTISED_PORT: 9092
-          KAFKA_PORT: 9092
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:1.0.0
-        environment:
-          KAFKA_ADVERTISED_HOST_NAME: localhost
-          KAFKA_ADVERTISED_PORT: 9093
-          KAFKA_PORT: 9093
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:1.0.0
+      - image: wurstmeister/kafka:<< parameters.kafka-version >>
         environment:
           KAFKA_ADVERTISED_HOST_NAME: localhost
           KAFKA_ADVERTISED_PORT: 9094
@@ -102,14 +51,32 @@ jobs:
           KAFKA_DELETE_TOPIC_ENABLE: true
     steps:
       - checkout
+      - run: sudo apt-get update && sudo apt-get install -y cmake # For installing snappy
       - run: bundle install --path vendor/bundle
       - run: bundle exec rspec --profile --tag functional spec/functional
 
 workflows:
   version: 2
   test:
     jobs:
-      - unit
-      - kafka-0.10
-      - kafka-0.11
-      - kafka-1.0
+      - unit:
+          name: unit-ruby-<< matrix.ruby-version >>
+          matrix:
+            parameters:
+              ruby-version:
+                - "2.7"
+                - "3.0"
+                - "3.1"
+      - kafka-with-wurstmeister:
+          name: wurstmeister-kafka-<< matrix.kafka-version >>-ruby-<< matrix.ruby-version >>
+          matrix:
+            parameters:
+              kafka-version:
+                - "2.11-0.10.2.2"
+                # 2.11 is the version of the docker image
+                # 0.10.2.2 is the kafka version associated to the docker image
+                # The image link is https://hub.docker.com/layers/wurstmeister/kafka/2.11-1.1.1/images/sha256-f72ab373c599d4253f7467601223a75313d2044ba477c41bef9ccc9d02bb04f7?context=explore
+              ruby-version:
+                - "2.7"
+                - "3.0"
+                - "3.1"

.rubocop.yml

@@ -1,13 +1,13 @@
+inherit_from: .rubocop_todo.yml
+
 AllCops:
   DisplayCopNames: true
-  TargetRubyVersion: 2.1
+  TargetRubyVersion: 3.0.5
 
 Lint:
   Enabled: false
 Metrics:
   Enabled: false
-Performance:
-  Enabled: false
 Style:
   Enabled: false
 
@@ -17,24 +17,22 @@ Layout/CaseIndentation:
   EnforcedStyle: end
 Layout/FirstParameterIndentation:
   EnforcedStyle: consistent
-Layout/IndentArray:
+Layout/FirstArrayElementIndentation:
   EnforcedStyle: consistent
-Layout/IndentHash:
+Layout/FirstHashElementIndentation:
   EnforcedStyle: consistent
 Layout/MultilineMethodCallIndentation:
   EnforcedStyle: indented
 Layout/MultilineOperationIndentation:
   EnforcedStyle: indented
-Lint/EndAlignment:
-  EnforcedStyleAlignWith: variable
 
 #
 # Disabled cops
 #
 
-Layout/AlignHash:
+Layout/HashAlignment:
   Enabled: false
-Layout/AlignParameters:
+Layout/ParameterAlignment:
   Enabled: false
 Layout/EmptyLinesAroundClassBody:
   Enabled: false

.rubocop_todo.yml

@@ -0,0 +1,109 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2023-02-17 11:26:49 UTC using RuboCop version 1.45.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+# Configuration parameters: Severity, Include.
+# Include: **/*.gemspec
+Gemspec/RequiredRubyVersion:
+  Exclude:
+    - 'ruby-kafka.gemspec'
+
+# Offense count: 4
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: EnforcedStyle, IndentationWidth.
+# SupportedStyles: with_first_argument, with_fixed_indentation
+Layout/ArgumentAlignment:
+  Exclude:
+    - 'lib/kafka/async_producer.rb'
+    - 'lib/kafka/producer.rb'
+    - 'lib/kafka/snappy_codec.rb'
+
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+Layout/ClosingParenthesisIndentation:
+  Exclude:
+    - 'lib/kafka/client.rb'
+
+# Offense count: 6
+# This cop supports safe autocorrection (--autocorrect).
+Layout/EmptyLineAfterGuardClause:
+  Exclude:
+    - 'lib/kafka/broker_pool.rb'
+    - 'lib/kafka/protocol/alter_configs_response.rb'
+    - 'lib/kafka/protocol/describe_configs_response.rb'
+    - 'lib/kafka/protocol/sasl_handshake_request.rb'
+    - 'lib/kafka/sasl/gssapi.rb'
+    - 'spec/producer_spec.rb'
+
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: EnforcedStyle, IndentationWidth.
+# SupportedStyles: consistent, align_parentheses
+Layout/FirstParameterIndentation:
+  Exclude:
+    - 'lib/kafka/client.rb'
+
+# Offense count: 2
+Naming/AccessorMethodName:
+  Exclude:
+    - 'lib/kafka/socket_with_timeout.rb'
+    - 'lib/kafka/ssl_socket_with_timeout.rb'
+
+# Offense count: 7
+# Configuration parameters: ExpectMatchingDefinition, CheckDefinitionPathHierarchy, CheckDefinitionPathHierarchyRoots, Regex, IgnoreExecutableScripts, AllowedAcronyms.
+# CheckDefinitionPathHierarchyRoots: lib, spec, test, src
+# AllowedAcronyms: CLI, DSL, ACL, API, ASCII, CPU, CSS, DNS, EOF, GUID, HTML, HTTP, HTTPS, ID, IP, JSON, LHS, QPS, RAM, RHS, RPC, SLA, SMTP, SQL, SSH, TCP, TLS, TTL, UDP, UI, UID, UUID, URI, URL, UTF8, VM, XML, XMPP, XSRF, XSS
+Naming/FileName:
+  Exclude:
+    - 'examples/consumer-group.rb'
+    - 'examples/firehose-consumer.rb'
+    - 'examples/firehose-producer.rb'
+    - 'examples/simple-consumer.rb'
+    - 'examples/simple-producer.rb'
+    - 'examples/ssl-producer.rb'
+    - 'lib/ruby-kafka.rb'
+
+# Offense count: 1
+# Configuration parameters: EnforcedStyleForLeadingUnderscores.
+# SupportedStylesForLeadingUnderscores: disallowed, required, optional
+Naming/MemoizedInstanceVariableName:
+  Exclude:
+    - 'lib/kafka/async_producer.rb'
+
+# Offense count: 1
+# Configuration parameters: NamePrefix, ForbiddenPrefixes, AllowedMethods, MethodDefinitionMacros.
+# NamePrefix: is_, has_, have_
+# ForbiddenPrefixes: is_, has_, have_
+# AllowedMethods: is_a?
+# MethodDefinitionMacros: define_method, define_singleton_method
+Naming/PredicateName:
+  Exclude:
+    - 'spec/**/*'
+    - 'lib/kafka/client.rb'
+
+# Offense count: 2
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: PreferredName.
+Naming/RescuedExceptionsVariableName:
+  Exclude:
+    - 'lib/kafka/ssl_socket_with_timeout.rb'
+
+# Offense count: 2
+# Configuration parameters: EnforcedStyle, CheckMethodNames, CheckSymbols, AllowedIdentifiers, AllowedPatterns.
+# SupportedStyles: snake_case, normalcase, non_integer
+# AllowedIdentifiers: capture3, iso8601, rfc1123_date, rfc822, rfc2822, rfc3339
+Naming/VariableNumber:
+  Exclude:
+    - 'spec/functional/consumer_group_spec.rb'
+
+# Offense count: 21
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns.
+# URISchemes: http, https
+Layout/LineLength:
+  Max: 193

Gemfile

@@ -7,20 +7,20 @@ gemspec
 group :development, :test do
   gem "activesupport"
   gem "bundler", ">= 1.9.5"
+  gem "colored"
   gem "docker-api"
+  gem "dogstatsd-ruby", ">= 3.0.0", "< 4.7.0"
   gem "dotenv"
+  gem "extlz4"
+  gem "gssapi", ">= 1.2.0"
   gem "pry"
   gem "rake", "~> 10.0"
   gem "rspec"
   gem "rspec-benchmark"
-  gem "snappy"
-  gem "colored"
-  gem "dogstatsd-ruby", ">= 3.0.0", "< 5.0.0"
-  gem "extlz4"
-  gem "gssapi", ">= 1.2.0"
   gem "rspec_junit_formatter", "0.2.2"
-  gem "rubocop", "~> 0.49.1"
+  gem "rubocop", "~> 1.0"
   gem "ruby-prof"
+  gem "snappy"
   gem "stackprof"
   gem "statsd-ruby"
   gem "timecop"

lib/kafka/async_producer.rb

@@ -195,7 +195,7 @@ def run
 
           case operation
           when :produce
-            produce(*payload)
+            produce(payload[0], **payload[1])
             deliver_messages if threshold_reached?
           when :deliver_messages
             deliver_messages
@@ -232,8 +232,8 @@ def run
 
       private
 
-      def produce(...)
-        @producer.produce(...)
+      def produce(value, **kwargs)
+        @producer.produce(value, **kwargs)
       rescue BufferOverflow
         deliver_messages
         retry

lib/kafka/datadog.rb

@@ -96,8 +96,8 @@ class StatsdSubscriber < ActiveSupport::Subscriber
       private
 
       %w[increment histogram count timing gauge].each do |type|
-        define_method(type) do |*args|
-          emit(type, *args)
+        define_method(type) do |*args, **kwargs|
+          emit(type, *args, **kwargs)
         end
       end
 

spec/datadog_spec.rb

@@ -24,7 +24,7 @@
 
     agent.wait_for_metrics
 
-    expect(agent.metrics.count).to eq 1
+    expect(agent.metrics.count).to eq 7
 
     metric = agent.metrics.first
 

spec/functional/client_spec.rb

@@ -111,19 +111,6 @@
     expect(message.key).to eq "xoxo"
   end
 
-  example "delivering a message to a topic that doesn't yet exist" do
-    topic = "unknown-topic-#{rand(1000)}"
-    now = Time.now
-
-    expect {
-      Timecop.freeze(now) do
-        kafka.deliver_message("yolo", topic: topic, key: "xoxo", partition: 0)
-      end
-    }.to raise_exception(Kafka::DeliveryFailed) {|exception|
-      expect(exception.failed_messages).to eq [Kafka::PendingMessage.new("yolo", "xoxo", topic, 0, nil, now)]
-    }
-  end
-
   example "enumerating the messages in a topic" do
     values = (1..10).to_a