From f7761cfa6572bf68c1f0ff146f6418dddbd8d226 Mon Sep 17 00:00:00 2001 From: Maciej Mensfeld Date: Fri, 1 Sep 2023 18:06:03 +0200 Subject: [PATCH 1/2] support routing patterns discovery in web-ui --- CHANGELOG.md | 3 +- Gemfile.lock | 4 +- karafka-web.gemspec | 2 +- lib/karafka/web/ui/pro/controllers/routing.rb | 44 +++++++++++++++++++ .../ui/pro/views/routing/_consumer_group.erb | 35 +++++++++++++++ .../web/ui/pro/views/routing/_detail.erb | 25 +++++++++++ .../web/ui/pro/views/routing/_topic.erb | 23 ++++++++++ .../web/ui/pro/views/routing/index.erb | 10 +++++ lib/karafka/web/ui/pro/views/routing/show.erb | 26 +++++++++++ lib/karafka/web/ui/views/routing/_detail.erb | 4 +- 10 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 lib/karafka/web/ui/pro/views/routing/_consumer_group.erb create mode 100644 lib/karafka/web/ui/pro/views/routing/_detail.erb create mode 100644 lib/karafka/web/ui/pro/views/routing/_topic.erb create mode 100644 lib/karafka/web/ui/pro/views/routing/index.erb create mode 100644 lib/karafka/web/ui/pro/views/routing/show.erb diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a0a1d1b..bf0975a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - **[Feature]** Introduce `bundle exec karafka-web migrate` that can be used to bootstrap the proper topics and initial data in environments where Karafka Web-UI should be used but is missing the initial setup. - **[Feature]** Replace `decrypt` with a pluggable API for deciding which topics data to display. - **[Feature]** Make sure, that the karafka server process that is materializing UI states is not processing any data having unsupported (newer) schemas. This state will be also visible in the status page. +- [Improvement] Support pattern subscriptions details in the routing view both by displaying the pattern as well as expanded routing details. - [Improvement] Collect total number of threads per process for the process details view. - [Improvement] Normalize naming of metrics to better reflect what they do (in reports and in the Web UI). - [Improvement] Link error reported first and last offset to the explorer. @@ -59,7 +60,7 @@ - [Refactor] Limit usage of UI models for data intense computation to speed up states materialization under load. - [Refactor] Reorganize pagination engine to support offset based pagination. - [Refactor] Use Roda `custom_block_results` plugin for controllers results handling. -- [Maintenance] Require `karafka` `2.1.8` due to fixes in the Iterator API and routing API extensions. +- [Maintenance] Require `karafka` `2.2.0` due to fixes in the Iterator API and routing API extensions. ### Upgrade Notes diff --git a/Gemfile.lock b/Gemfile.lock index 288accfe..069a0687 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,7 +3,7 @@ PATH specs: karafka-web (0.7.0) erubi (~> 1.4) - karafka (>= 2.1.13, < 3.0.0) + karafka (>= 2.2.0, < 3.0.0) karafka-core (>= 2.0.13, < 3.0.0) roda (~> 3.68, >= 3.69) tilt (~> 2.0) @@ -26,7 +26,7 @@ GEM ffi (1.15.5) i18n (1.14.1) concurrent-ruby (~> 1.0) - karafka (2.1.13) + karafka (2.2.0) karafka-core (>= 2.1.1, < 2.2.0) thor (>= 0.20) waterdrop (>= 2.6.6, < 3.0.0) diff --git a/karafka-web.gemspec b/karafka-web.gemspec index 50f60a6a..860b6146 100644 --- a/karafka-web.gemspec +++ b/karafka-web.gemspec @@ -17,7 +17,7 @@ Gem::Specification.new do |spec| spec.licenses = %w[LGPL-3.0 Commercial] spec.add_dependency 'erubi', '~> 1.4' - spec.add_dependency 'karafka', '>= 2.1.13', '< 3.0.0' + spec.add_dependency 'karafka', '>= 2.2.0', '< 3.0.0' spec.add_dependency 'karafka-core', '>= 2.0.13', '< 3.0.0' spec.add_dependency 'roda', '~> 3.68', '>= 3.69' spec.add_dependency 'tilt', '~> 2.0' diff --git a/lib/karafka/web/ui/pro/controllers/routing.rb b/lib/karafka/web/ui/pro/controllers/routing.rb index b95b31d0..8b3ad02c 100644 --- a/lib/karafka/web/ui/pro/controllers/routing.rb +++ b/lib/karafka/web/ui/pro/controllers/routing.rb @@ -18,6 +18,50 @@ module Pro module Controllers # Routing details - same as in OSS class Routing < Ui::Controllers::Routing + # Routing list + def index + detect_patterns_routes + + @routes = Karafka::App.routes + + respond + end + + # Given route details + # + # @param topic_id [String] topic id + def show(topic_id) + detect_patterns_routes + + @topic = Karafka::Routing::Router.find_by(id: topic_id) + + @topic || raise(::Karafka::Web::Errors::Ui::NotFoundError, topic_id) + + respond + end + + private + + # Checks list of topics and tries to match them against the available patterns + # Uses the Pro detector to expand routes in the Web-UI so we include topics that are + # or will be matched using our regular expressions + def detect_patterns_routes + detector = ::Karafka::Pro::Routing::Features::Patterns::Detector.new + topics_names = Models::ClusterInfo.topics.map(&:topic_name) + + Karafka::App + .routes + .flat_map(&:subscription_groups) + .each do |subscription_group| + sg_topics = subscription_group.topics + + # Reject topics that are already part of routing for given subscription groups + # and then for remaining try to apply patterns and expand routes + topics_names + .reject { |t_name| sg_topics.any? { |rtopic| rtopic.name == t_name } } + .each { |t_name| detector.expand(sg_topics, t_name) } + end + end end end end diff --git a/lib/karafka/web/ui/pro/views/routing/_consumer_group.erb b/lib/karafka/web/ui/pro/views/routing/_consumer_group.erb new file mode 100644 index 00000000..3b3858ae --- /dev/null +++ b/lib/karafka/web/ui/pro/views/routing/_consumer_group.erb @@ -0,0 +1,35 @@ +
+
+

+ <%= consumer_group.id %> +

+
+
+
+ +
+
+ + + + + + + + + + + + <% consumer_group.subscription_groups.each do |subscription_group| %> + <%== + each_partial( + subscription_group.topics, + 'routing/topic', + locals: { subscription_group: subscription_group } + ) + %> + <% end %> + +
Subscription groupTopicTypeActive
+
+
diff --git a/lib/karafka/web/ui/pro/views/routing/_detail.erb b/lib/karafka/web/ui/pro/views/routing/_detail.erb new file mode 100644 index 00000000..1573f835 --- /dev/null +++ b/lib/karafka/web/ui/pro/views/routing/_detail.erb @@ -0,0 +1,25 @@ +<% if v.is_a?(Hash) || v.respond_to?(:to_h) %> + <% v.to_h.each do |k2, v2| %> + + + <%= "#{k}.#{k2}" %> + + + <% if %w[sasl ssl].any? { |scope| k2.to_s.include?(scope) } %> + *** + <% else %> + <%= v2 %> + <% end %> + + + <% end %> +<% else %> + + + <%= k %> + + + <%= object_value_to_s(v) %> + + +<% end %> diff --git a/lib/karafka/web/ui/pro/views/routing/_topic.erb b/lib/karafka/web/ui/pro/views/routing/_topic.erb new file mode 100644 index 00000000..2df0477b --- /dev/null +++ b/lib/karafka/web/ui/pro/views/routing/_topic.erb @@ -0,0 +1,23 @@ + + + <%= subscription_group.id %> + + + <%= topic.name %> + + + + <%= topic.patterns.type %> + + + + + <%= topic.active? %> + + + + + Details + + + diff --git a/lib/karafka/web/ui/pro/views/routing/index.erb b/lib/karafka/web/ui/pro/views/routing/index.erb new file mode 100644 index 00000000..52055783 --- /dev/null +++ b/lib/karafka/web/ui/pro/views/routing/index.erb @@ -0,0 +1,10 @@ +<%== view_title('Routing details') %> + +
+ <%== + each_partial( + @routes, + 'routing/consumer_group' + ) + %> +
diff --git a/lib/karafka/web/ui/pro/views/routing/show.erb b/lib/karafka/web/ui/pro/views/routing/show.erb new file mode 100644 index 00000000..4022e9c7 --- /dev/null +++ b/lib/karafka/web/ui/pro/views/routing/show.erb @@ -0,0 +1,26 @@ +<%== + title = [@topic.consumer_group.name, @topic.subscription_group, @topic.name].join(': ') + view_title(title, hr: true) +%> + +
+
+
+ + + <% flat_hash(@topic.to_h).each do |k, v| %> + <%== + partial( + 'routing/detail', + locals: { + k: k, + v: v + } + ) + %> + <% end %> + +
+
+
+
diff --git a/lib/karafka/web/ui/views/routing/_detail.erb b/lib/karafka/web/ui/views/routing/_detail.erb index 61717925..1573f835 100644 --- a/lib/karafka/web/ui/views/routing/_detail.erb +++ b/lib/karafka/web/ui/views/routing/_detail.erb @@ -1,5 +1,5 @@ -<% if v.is_a?(Hash) %> - <% v.each do |k2, v2| %> +<% if v.is_a?(Hash) || v.respond_to?(:to_h) %> + <% v.to_h.each do |k2, v2| %> <%= "#{k}.#{k2}" %> From 63a7f21648ce39313ff8833c70200326a6bf4142 Mon Sep 17 00:00:00 2001 From: Maciej Mensfeld Date: Fri, 1 Sep 2023 18:38:31 +0200 Subject: [PATCH 2/2] spec fix --- spec/lib/karafka/web/tracking/sampler_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/karafka/web/tracking/sampler_spec.rb b/spec/lib/karafka/web/tracking/sampler_spec.rb index 1a8a335d..f6ebd279 100644 --- a/spec/lib/karafka/web/tracking/sampler_spec.rb +++ b/spec/lib/karafka/web/tracking/sampler_spec.rb @@ -4,7 +4,7 @@ subject(:sampler) { described_class.new } it { expect(sampler.ruby_version).to start_with('ruby ') } - it { expect(sampler.karafka_version).to include('2.1.') } + it { expect(sampler.karafka_version).to include('2.2.') } it { expect(sampler.karafka_web_version).to include('0.7.') } it { expect(sampler.karafka_core_version).to include('2.1.') } it { expect(sampler.rdkafka_version).to start_with('0.1') }