From 2eff3b21a49165f6622c8749b574d296da23d8de Mon Sep 17 00:00:00 2001 From: Maciej Mensfeld Date: Tue, 12 Sep 2023 14:40:54 +0200 Subject: [PATCH] remars --- CHANGELOG.md | 2 +- lib/karafka/web/ui/lib/hash_proxy.rb | 31 +++++++++++-------- .../ui/pro/views/health/_partition_offset.erb | 3 ++ .../web/ui/pro/views/health/offsets.erb | 1 + spec/fixtures/consumer_report.json | 21 +++++++++++++ spec/lib/karafka/web/ui/models/health_spec.rb | 7 +++-- 6 files changed, 48 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0699e5f..7b5b3c07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ - **[Feature]** Provide ability to reproduce a given message to the same topic partition with all the details from the per message explorer view. - **[Feature]** Provide "surrounding" navigation link that allows to view the given message in the context of its surrounding. Useful for debugging of failures where the batch context may be relevant. - **[Feature]** Allow for time based lookups per topic partition. -- **[Feature]** Introduce Offsets Health inspection view for frozen LSO lookups. +- **[Feature]** Introduce Offsets Health inspection view for frozen LSO lookups with `lso_threshold` configuration option. - [Improvement] Support pattern subscriptions details in the routing view both by displaying the pattern as well as expanded routing details. - [Improvement] Collect total number of threads per process for the process details view. - [Improvement] Normalize naming of metrics to better reflect what they do (in reports and in the Web UI). diff --git a/lib/karafka/web/ui/lib/hash_proxy.rb b/lib/karafka/web/ui/lib/hash_proxy.rb index 8c07f12c..0ff88838 100644 --- a/lib/karafka/web/ui/lib/hash_proxy.rb +++ b/lib/karafka/web/ui/lib/hash_proxy.rb @@ -12,23 +12,17 @@ module Lib # It is mostly used for flat hashes. # # It is in a way similar to openstruct but has abilities to dive deep into objects + # + # It is not super fast but it is enough for the UI and how deep structures we have. class HashProxy + extend Forwardable + + def_delegators :@hash, :[], :[]=, :key?, :each, :find + # @param hash [Hash] hash we want to convert to a proxy def initialize(hash) @hash = hash - end - - # @param key [Object] hash key - # @return [Object] key content or nil if missing - def [](key) - @hash[key] - end - - # Assigns a new value to the key - # @param key [Object] hash key for assignment - # @param value [Object] anything we want to assign - def []=(key, value) - @hash[key] = value + @visited = [] end # @return [Original hash] @@ -42,7 +36,12 @@ def to_h def method_missing(method_name, *args, &block) return super unless args.empty? && block.nil? + @visited.clear + result = deep_find(@hash, method_name.to_sym) + + @visited.clear + result.nil? ? super : result end @@ -58,6 +57,12 @@ def respond_to_missing?(method_name, include_private = false) # @param obj [Object] local scope of iterating # @param key [Symbol, String] key we are looking for def deep_find(obj, key) + # Prevent circular dependency lookups by making sure we do not check the same object + # multiple times + return nil if @visited.include?(obj) + + @visited << obj + if obj.respond_to?(:key?) && obj.key?(key) obj[key] elsif obj.respond_to?(:each) diff --git a/lib/karafka/web/ui/pro/views/health/_partition_offset.erb b/lib/karafka/web/ui/pro/views/health/_partition_offset.erb index 11024c41..a360ba63 100644 --- a/lib/karafka/web/ui/pro/views/health/_partition_offset.erb +++ b/lib/karafka/web/ui/pro/views/health/_partition_offset.erb @@ -17,6 +17,9 @@ <%== relative_time(Time.now - details.stored_offset_fd / 1_000) %> + + <%== offset_with_label topic_name, partition_id, details.lo_offset %> + <%== offset_with_label topic_name, partition_id, details.hi_offset %> diff --git a/lib/karafka/web/ui/pro/views/health/offsets.erb b/lib/karafka/web/ui/pro/views/health/offsets.erb index c659d68d..b6f7e597 100644 --- a/lib/karafka/web/ui/pro/views/health/offsets.erb +++ b/lib/karafka/web/ui/pro/views/health/offsets.erb @@ -41,6 +41,7 @@ Committed offset change Stored offset Stored offset change + Low offset High offset High offset change Last stable offset diff --git a/spec/fixtures/consumer_report.json b/spec/fixtures/consumer_report.json index 14ad2877..07d58ebc 100644 --- a/spec/fixtures/consumer_report.json +++ b/spec/fixtures/consumer_report.json @@ -68,9 +68,16 @@ "lag_stored": 1, "lag_stored_d": -3, "committed_offset": 327343, + "committed_offset_fd": 5000, "stored_offset": 327355, + "stored_offset_fd": 5, "fetch_state": "active", "hi_offset": 327356, + "hi_offset_fd": 100, + "lo_offset": 0, + "eof_offset": 327356, + "ls_offset": 200, + "ls_offset_fd": 1000, "id": 0, "poll_state": "active" } @@ -85,9 +92,16 @@ "lag_stored": -1, "lag_stored_d": 0, "committed_offset": -1001, + "committed_offset_fd": 0, "stored_offset": -1001, + "stored_offset_fd": 0, "fetch_state": "active", "hi_offset": 0, + "hi_offset_fd": 100, + "lo_offset": 0, + "eof_offset": 0, + "ls_offset": 0, + "ls_offset_fd": 0, "id": 0, "poll_state": "active" } @@ -102,9 +116,16 @@ "lag_stored": -1, "lag_stored_d": 0, "committed_offset": 27, + "committed_offset_fd": 0, "stored_offset": -1001, + "stored_offset_fd": 0, "fetch_state": "active", "hi_offset": 27, + "hi_offset_fd": 100, + "lo_offset": 0, + "eof_offset": 0, + "ls_offset": 0, + "ls_offset_fd": 0, "id": 0, "poll_state": "active" } diff --git a/spec/lib/karafka/web/ui/models/health_spec.rb b/spec/lib/karafka/web/ui/models/health_spec.rb index c5f7883e..dc2622af 100644 --- a/spec/lib/karafka/web/ui/models/health_spec.rb +++ b/spec/lib/karafka/web/ui/models/health_spec.rb @@ -89,9 +89,10 @@ keys = %i[ lag lag_d lag_stored lag_stored_d committed_offset stored_offset fetch_state hi_offset id - poll_state process - ] - expect(sg[:topics][:default][:partitions]['0'.to_sym].keys).to eq(keys) + poll_state process hi_offset_fd stored_offset_fd lo_offset ls_offset ls_offset_fd + eof_offset committed_offset_fd + ].sort + expect(sg[:topics][:default][:partitions]['0'.to_sym].keys.sort).to eq(keys) end end end