Skip to content

Commit

Permalink
add nsqd server side messageCount topology stats
Browse files Browse the repository at this point in the history
  • Loading branch information
@zoemccormick
zoemccormick committed Apr 17, 2024
1 parent 178da24 commit d0cd516
Show file tree
Hide file tree
Showing 13 changed files with 320 additions and 103 deletions.
2 changes: 2 additions & 0 deletions internal/clusterinfo/data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -590,6 +590,7 @@ func (c *ClusterInfo) GetNSQDStats(producers Producers,
topic.Node = addr
topic.Hostname = p.Hostname
topic.MemoryDepth = topic.Depth - topic.BackendDepth
topic.DeliveryMsgCount = topic.ZoneLocalMsgCount + topic.RegionLocalMsgCount + topic.GlobalMsgCount
if selectedTopic != "" && topic.TopicName != selectedTopic {
continue
}
Expand All @@ -600,6 +601,7 @@ func (c *ClusterInfo) GetNSQDStats(producers Producers,
channel.Hostname = p.Hostname
channel.TopicName = topic.TopicName
channel.MemoryDepth = channel.Depth - channel.BackendDepth
channel.DeliveryMsgCount = channel.ZoneLocalMsgCount + channel.RegionLocalMsgCount + channel.GlobalMsgCount
key := channel.ChannelName
if selectedTopic == "" {
key = fmt.Sprintf("%s:%s", topic.TopicName, channel.ChannelName)
Expand Down
70 changes: 43 additions & 27 deletions internal/clusterinfo/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,16 +98,20 @@ func (p *Producer) IsInconsistent(numLookupd int) bool {
}

type TopicStats struct {
Node string `json:"node"`
Hostname string `json:"hostname"`
TopicName string `json:"topic_name"`
Depth int64 `json:"depth"`
MemoryDepth int64 `json:"memory_depth"`
BackendDepth int64 `json:"backend_depth"`
MessageCount int64 `json:"message_count"`
NodeStats []*TopicStats `json:"nodes"`
Channels []*ChannelStats `json:"channels"`
Paused bool `json:"paused"`
Node string `json:"node"`
Hostname string `json:"hostname"`
TopicName string `json:"topic_name"`
Depth int64 `json:"depth"`
MemoryDepth int64 `json:"memory_depth"`
BackendDepth int64 `json:"backend_depth"`
MessageCount int64 `json:"message_count"`
DeliveryMsgCount int64 `json:"delivery_msg_count"`
ZoneLocalMsgCount int64 `json:"zone_local_msg_count,omitempty"`
RegionLocalMsgCount int64 `json:"region_local_msg_count,omitempty"`
GlobalMsgCount int64 `json:"global_msg_count,omitempty"`
NodeStats []*TopicStats `json:"nodes"`
Channels []*ChannelStats `json:"channels"`
Paused bool `json:"paused"`

E2eProcessingLatency *quantile.E2eProcessingLatencyAggregate `json:"e2e_processing_latency"`
}
Expand All @@ -118,6 +122,10 @@ func (t *TopicStats) Add(a *TopicStats) {
t.MemoryDepth += a.MemoryDepth
t.BackendDepth += a.BackendDepth
t.MessageCount += a.MessageCount
t.DeliveryMsgCount += a.DeliveryMsgCount
t.ZoneLocalMsgCount += a.ZoneLocalMsgCount
t.RegionLocalMsgCount += a.RegionLocalMsgCount
t.GlobalMsgCount += a.GlobalMsgCount
if a.Paused {
t.Paused = a.Paused
}
Expand Down Expand Up @@ -145,23 +153,27 @@ func (t *TopicStats) Add(a *TopicStats) {
}

type ChannelStats struct {
Node string `json:"node"`
Hostname string `json:"hostname"`
TopicName string `json:"topic_name"`
ChannelName string `json:"channel_name"`
Depth int64 `json:"depth"`
MemoryDepth int64 `json:"memory_depth"`
BackendDepth int64 `json:"backend_depth"`
InFlightCount int64 `json:"in_flight_count"`
DeferredCount int64 `json:"deferred_count"`
RequeueCount int64 `json:"requeue_count"`
TimeoutCount int64 `json:"timeout_count"`
MessageCount int64 `json:"message_count"`
ClientCount int `json:"client_count"`
Selected bool `json:"-"`
NodeStats []*ChannelStats `json:"nodes"`
Clients []*ClientStats `json:"clients"`
Paused bool `json:"paused"`
Node string `json:"node"`
Hostname string `json:"hostname"`
TopicName string `json:"topic_name"`
ChannelName string `json:"channel_name"`
Depth int64 `json:"depth"`
MemoryDepth int64 `json:"memory_depth"`
BackendDepth int64 `json:"backend_depth"`
InFlightCount int64 `json:"in_flight_count"`
DeferredCount int64 `json:"deferred_count"`
RequeueCount int64 `json:"requeue_count"`
TimeoutCount int64 `json:"timeout_count"`
MessageCount int64 `json:"message_count"`
DeliveryMsgCount int64 `json:"delivery_msg_count,omitempty"`
ZoneLocalMsgCount int64 `json:"zone_local_msg_count,omitempty"`
RegionLocalMsgCount int64 `json:"region_local_msg_count,omitempty"`
GlobalMsgCount int64 `json:"global_msg_count,omitempty"`
ClientCount int `json:"client_count"`
Selected bool `json:"-"`
NodeStats []*ChannelStats `json:"nodes"`
Clients []*ClientStats `json:"clients"`
Paused bool `json:"paused"`

E2eProcessingLatency *quantile.E2eProcessingLatencyAggregate `json:"e2e_processing_latency"`
}
Expand All @@ -176,6 +188,10 @@ func (c *ChannelStats) Add(a *ChannelStats) {
c.RequeueCount += a.RequeueCount
c.TimeoutCount += a.TimeoutCount
c.MessageCount += a.MessageCount
c.DeliveryMsgCount += a.DeliveryMsgCount
c.ZoneLocalMsgCount += a.ZoneLocalMsgCount
c.RegionLocalMsgCount += a.RegionLocalMsgCount
c.GlobalMsgCount += a.GlobalMsgCount
c.ClientCount += a.ClientCount
if a.Paused {
c.Paused = a.Paused
Expand Down
66 changes: 66 additions & 0 deletions nsqadmin/static/build/base.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,4 +163,70 @@
margin-right: 5px;
margin-top: -5px;
display: inline;
}

.popup {
position: relative;
display: inline-block;
cursor: pointer;
}

/* The actual popup (appears on top) */
.popup .popuptext {
visibility: hidden;
width: 180px;
height: 27px;
background-color: white;
color: #4b4b4b;
text-align: center;
border-radius: 6px;
border-right: 1px solid #777;
border-left: 1px solid #777;
border-top: 1px solid #777;
border-bottom: 1px solid #777;
padding: 2px 0;
position: absolute;
z-index: 1;
top: 60%;
left: 50%;
margin-left: -85%;
}

/* Popup arrow */
.popup .popuptext::after {
content: "";
position: absolute;
top: -5px;
left: 50%;
margin-left: -5px;
width: 0;
height: 0;
border-left: 5px solid transparent;
border-right: 5px solid transparent;
border-bottom: 5px solid #777;
}

/* Toggle this class when clicking on the popup container (hide and show the popup) */
.popup .show {
visibility: visible;
-webkit-animation: fadeIn 1s;
animation: fadeIn 1s;
}

/* Add animation (fade in the popup) */
@-webkit-keyframes fadeIn {
from {
opacity: 0;
}
to {
opacity: 1;
}
}
@keyframes fadeIn {
from {
opacity: 0;
}
to {
opacity: 1;
}
}
2 changes: 1 addition & 1 deletion nsqadmin/static/build/main.js
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion nsqadmin/static/build/main.js.map
View file

Large diffs are not rendered by default.

59 changes: 59 additions & 0 deletions nsqadmin/static/css/base.scss
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,4 +166,63 @@
margin-right: 5px;
margin-top: -5px;
display: inline;
}

.popup {
position: relative;
display: inline-block;
cursor: pointer;
}

/* The actual popup (appears on top) */
.popup .popuptext {
visibility: hidden;
width: 180px;
height: 27px;
background-color: white;
color: #4b4b4b;
text-align: center;
border-radius: 6px;
border-right: 1px solid #777;
border-left: 1px solid #777;
border-top: 1px solid #777;
border-bottom: 1px solid #777;
padding: 2px 0;
position: absolute;
z-index: 1;
top: 60%;
left: 50%;
margin-left: -85%;
}

/* Popup arrow */
.popup .popuptext::after {
content: "";
position: absolute;
top: -5px;
left: 50%;
margin-left: -5px;
width: 0;
height: 0;
border-left: 5px solid transparent;
border-right: 5px solid transparent;
border-bottom: 5px solid #777;
}

/* Toggle this class when clicking on the popup container (hide and show the popup) */
.popup .show {
visibility: visible;
-webkit-animation: fadeIn 1s;
animation: fadeIn 1s
}

/* Add animation (fade in the popup) */
@-webkit-keyframes fadeIn {
from {opacity: 0;}
to {opacity: 1;}
}

@keyframes fadeIn {
from {opacity: 0;}
to {opacity:1 ;}
}
4 changes: 4 additions & 0 deletions nsqadmin/static/js/lib/handlebars_helpers.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,10 @@ Handlebars.registerHelper('floatToPercent', function(f) {
return Math.floor(f * 100);
});

Handlebars.registerHelper('floatToDecimalPercent', function(f) {
return parseFloat((f * 100).toFixed(2));
});

Handlebars.registerHelper('percSuffix', function(f) {
var v = Math.floor(f * 100) % 10;
if (v === 1) {
Expand Down
16 changes: 16 additions & 0 deletions nsqadmin/static/js/models/channel.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,24 @@ var Channel = Backbone.Model.extend({
var port = nodeParts.pop();
var address = nodeParts.join(':');
var hostname = node['hostname'];
var zonecount = node['zone_local_msg_count'];
var deliverycount = node['delivery_msg_count'];
var regioncount = node['region_local_msg_count'];
var globalcount = node['global_msg_count'];
node['show_broadcast_address'] = hostname.toLowerCase() !== address.toLowerCase();
node['hostname_port'] = hostname + ':' + port;
node['zone_local_percentage'] = zonecount / deliverycount;
node['region_local_percentage'] = regioncount / deliverycount;
node['global_percentage'] = globalcount / deliverycount;
if (isNaN(node['zone_local_percentage'])) {
node['zone_local_percentage'] = 0;
}
if (isNaN(node['region_local_percentage'])) {
node['region_local_percentage'] = 0;
}
if (isNaN(node['global_percentage'])) {
node['global_percentage'] = 0;
}
return node;
});

Expand Down
24 changes: 23 additions & 1 deletion nsqadmin/static/js/views/channel.hbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
<tr>
<th>&nbsp;</th>
<th colspan="4" class="text-center">Message Queues</th>
<th colspan="{{#if graph_active}}5{{else}}4{{/if}}" class="text-center">Statistics</th>
<th colspan="{{#if graph_active}}6{{else}}5{{/if}}" class="text-center">Statistics</th>
{{#if e2e_processing_latency.percentiles.length}}
<th colspan="{{e2e_processing_latency.percentiles.length}}">E2E Processing Latency</th>
{{/if}}
Expand All @@ -67,6 +67,7 @@
<th>Messages</th>
{{#if graph_active}}<th>Rate</th>{{/if}}
<th>Connections</th>
{{#if delivery_msg_count}}<th style="width: 130px">Delivery</th>{{/if}}
{{#each e2e_processing_latency.percentiles}}
<th>{{floatToPercent quantile}}<sup>{{percSuffix quantile}}</sup></th>
{{/each}}
Expand All @@ -92,6 +93,24 @@
<td class="bold rate" target="{{rate "topic" node topic_name ""}}"></td>
{{/if}}
<td>{{commafy client_count}}</td>
{{#if delivery_msg_count}}
<td>
<div class="popup" data-id="popup-{{node}}">
<div class="progress" style="width: 120px; height: 20px; border-right: .01em solid #777; border-left: .01em solid #777; border-top: .01em solid #777; border-bottom: .01em solid #777">
<div class="progress-bar progress-bar-success" style="background-color:rgb(221,255,221); width: {{floatToPercent zone_local_percentage}}%">
<span class="sr-only">{{floatToPercent zone_local_percentage}}%</span>
</div>
<div class="progress-bar progress-bar-warning progress-bar-striped" style="background-color:rgb(254,254,194); width: {{floatToPercent region_local_percentage}}%">
<span class="sr-only">{{floatToPercent region_local_percentage}}%</span>
</div>
<div class="progress-bar progress-bar-danger" style="background-color: white; width: {{floatToPercent global_percentage}}%">
<span class="sr-only">{{floatToPercent global_percentage}}%</span>
</div>
</div>
<span class="popuptext" id="popup-{{node}}"><span style="background-color:rgb(221,255,221); color:#4b4b4b">{{floatToDecimalPercent zone_local_percentage}}%</span> | <span style="background-color:rgb(254,254,194); color:#4b4b4b">{{floatToDecimalPercent region_local_percentage}}%</span> | <span style="color:#4b4b4b">{{floatToDecimalPercent global_percentage}}%</span></span>
</div>
</td>
{{/if}}
{{#if e2e_processing_latency.percentiles.length}}
{{#each e2e_processing_latency.percentiles}}
<td>
Expand All @@ -112,6 +131,7 @@
<td><a href="{{large_graph "channel" node topic_name channel_name "message_count"}}"><img width="120" height="20" src="{{sparkline "channel" node topic_name channel_name "message_count"}}"></a></td>
<td></td>
<td><a href="{{large_graph "channel" node topic_name channel_name "clients"}}"><img width="120" height="20" src="{{sparkline "channel" node topic_name channel_name "clients"}}"></a></td>
{{#if delivery_msg_count}}<td></td>{{/if}}
{{#if e2e_processing_latency.percentiles.length}}
<td colspan="{{e2e_processing_latency.percentiles.length}}">
<a href="{{large_graph "e2e" node e2e_processing_latency "" "e2e_processing_latency"}}"><img width="120" height="20" src="{{sparkline "e2e" node e2e_processing_latency "" "e2e_processing_latency"}}"></a>
Expand All @@ -133,6 +153,7 @@
<td class="bold rate" target="{{rate "topic" node topic_name ""}}"></td>
{{/if}}
<td>{{commafy client_count}}</td>
{{#if delivery_msg_count}}<td></td>{{/if}}
{{#if e2e_processing_latency.percentiles.length}}
{{#each e2e_processing_latency.percentiles}}
<td>
Expand All @@ -153,6 +174,7 @@
<td><a href="{{large_graph "channel" node topic_name channel_name "message_count"}}"><img width="120" height="20" src="{{sparkline "channel" node topic_name channel_name "message_count"}}"></a></td>
<td></td>
<td><a href="{{large_graph "channel" node topic_name channel_name "clients"}}"><img width="120" height="20" src="{{sparkline "channel" node topic_name channel_name "clients"}}"></a></td>
{{#if delivery_msg_count}}<td></td>{{/if}}
{{#if e2e_processing_latency.percentiles.length}}
<td colspan="{{e2e_processing_latency.percentiles.length}}">
<a href="{{large_graph "e2e" node e2e_processing_latency "" "e2e_processing_latency"}}"><img width="120" height="20" src="{{sparkline "e2e" node e2e_processing_latency "" "e2e_processing_latency"}}"></a>
Expand Down
10 changes: 9 additions & 1 deletion nsqadmin/static/js/views/channel.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ var ChannelView = BaseView.extend({
template: require('./spinner.hbs'),

events: {
'click .channel-actions button': 'channelAction'
'click .channel-actions button': 'channelAction',
'click .popup': 'showDeliveryBreakdown'
},

initialize: function() {
Expand All @@ -31,6 +32,13 @@ var ChannelView = BaseView.extend({
.always(Pubsub.trigger.bind(Pubsub, 'view:ready'));
},

showDeliveryBreakdown: function(e) {
e.preventDefault();
e.stopPropagation();
var popup = document.getElementById($(e.currentTarget).data('id'));
popup.classList.toggle("show");
},

channelAction: function(e) {
e.preventDefault();
e.stopPropagation();
Expand Down
Loading

0 comments on commit d0cd516

Please sign in to comment.