Bump ruff and add log report of database queue size

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ ci:
   autoupdate_schedule: quarterly
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.1.11"
+    rev: "v0.4.2"
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

app/applib/ldmbridge.py

@@ -1,15 +1,22 @@
 """Async bridge."""
-from twisted.internet import stdio
-from twisted.python import log
-from twisted.internet import reactor
+
+from twisted.internet import reactor, stdio
 from twisted.protocols import basic
+from twisted.python import log
+
 from applib import rtstats
 
 
 class RTStatsIngestor(basic.LineReceiver):
     # default delimiter is \r\n
     delimiter = b"\n"
 
+    def check_queue_length(self):
+        """Check the database queue length"""
+        # no public API, last checked
+        sz = self.dbpool.threadpool._queue.qsize()  # noqa
+        log.msg(f"Database write queue size: {sz}")
+
     def connectionLost(self, reason):
         """Called when the STDIN connection is lost"""
         log.msg("connectionLost")

app/applib/rtstats.py

@@ -1,9 +1,10 @@
 """Main rtstats processing work lies here."""
+
 import datetime
 from distutils.version import LooseVersion
 
-from twisted.python import log
 import pytz
+from twisted.python import log
 
 # Versions that had the origin 32 byte truncation bug, see @akrherz/rtstats#1
 TRUNC_BUG_FLOOR = LooseVersion("6.11.7")

app/run.py

@@ -1,16 +1,15 @@
-"""A Twisted rtstats ingest + server
+"""A Twisted rtstats ingest + server"""
 
-
-"""
+import datetime
 import json
 import os
-import datetime
 from syslog import LOG_LOCAL2
 
-from twisted.internet import reactor
+from applib import ldmbridge
 from twisted.enterprise import adbapi
+from twisted.internet import reactor
+from twisted.internet.task import LoopingCall
 from twisted.python import syslog
-from applib import ldmbridge
 
 # This is a hack that prevents a strange exception with datetime and threading
 datetime.datetime.strptime("2017", "%Y")
@@ -25,12 +24,17 @@ def ready(_, dbpool):
     protocol.dbpool = dbpool
     ldmbridge.LDMProductFactory(protocol)
 
+    # Start a LoopingCall every 5 minutes that checks on database queues
+    lc = LoopingCall(protocol.check_queue_length)
+    lc.start(300, now=False)
+
 
 def load_dbtables(cursor):
     pass
 
 
-if __name__ == "__main__":
+def main():
+    """Setup and run the processor."""
     fn = "%s/settings.json" % (
         os.path.join(os.path.dirname(__file__), "../config"),
     )
@@ -50,3 +54,7 @@ def load_dbtables(cursor):
     df.addCallback(ready, dbpool)
 
     reactor.run()
+
+
+if __name__ == "__main__":
+    main()

cgi-bin/rtstats.py

@@ -2,19 +2,20 @@
 # pylint: disable=cell-var-from-loop
 """I should answer the following URIs
 
-    .../feedindex [list of feedtypes]
-    .../siteindex
-    .../iddstats_nc?EXP+server1.smn.gov.ar
-    .../iddbinstats_nc?EXP+server1.smn.gov.ar  [latency histogram]
-    .../iddstats_vol_nc?EXP+server1.smn.gov.ar [volume]
-    .../iddstats_num_nc?HDS+server1.smn.gov.ar [products]
-    .../iddstats_topo_nc?HDS+metfs1.agron.iastate.edu [topology]
-    .../rtstats_summary_volume?metfs1.agron.iastate.edu [text stats]
-    .../topoindex?tree [feedtype listing]
-    .../rtstats_feedtree?EXP [reverse topology]
-    .../iddstats_vol_nc1?EXP+10.100.69.110 [volume summaries]
-    .../rtstats_summary_volume1?10.100.69.110+GRAPH [volume summary]
+.../feedindex [list of feedtypes]
+.../siteindex
+.../iddstats_nc?EXP+server1.smn.gov.ar
+.../iddbinstats_nc?EXP+server1.smn.gov.ar  [latency histogram]
+.../iddstats_vol_nc?EXP+server1.smn.gov.ar [volume]
+.../iddstats_num_nc?HDS+server1.smn.gov.ar [products]
+.../iddstats_topo_nc?HDS+metfs1.agron.iastate.edu [topology]
+.../rtstats_summary_volume?metfs1.agron.iastate.edu [text stats]
+.../topoindex?tree [feedtype listing]
+.../rtstats_feedtree?EXP [reverse topology]
+.../iddstats_vol_nc1?EXP+10.100.69.110 [volume summaries]
+.../rtstats_summary_volume1?10.100.69.110+GRAPH [volume summary]
 """
+
 import datetime
 import re
 import sys

database/schema_manager.py

@@ -1,12 +1,13 @@
 """
- My goal in life is to manage the database schema, so when things change, this
- script can handle it all.  I am run like so:
+My goal in life is to manage the database schema, so when things change, this
+script can handle it all.  I am run like so:
 
-     python schema_manager.py
+    python schema_manager.py
 """
+
+import json
 import os
 import sys
-import json
 
 import psycopg2
 

htdocs/services/feedtype.py

@@ -1,15 +1,16 @@
 #!/usr/bin/env python
 """Emit JSON of feedtype
 
-    /services/feedtype/<feedtype>/topology.json
+/services/feedtype/<feedtype>/topology.json
 
 """
-import json
+
 import datetime
+import json
 
 import memcache
-from paste.request import parse_formvars
 import rtstats_util as util
+from paste.request import parse_formvars
 
 
 def handle_topology(feedtype, reverse=False):

htdocs/services/feedtypes.py

@@ -1,14 +1,15 @@
 #!/usr/bin/env python
 """Emit JSON of feedtypes
 
-    /services/feedtypes.json
+/services/feedtypes.json
 
 """
+
 import json
 
 import memcache
-from paste.request import parse_formvars
 import rtstats_util as util
+from paste.request import parse_formvars
 
 
 def run():

htdocs/services/host.py

@@ -1,19 +1,20 @@
 #!/usr/bin/env python
 """Emit JSON of host
 
-    /services/host/<hostname>/feedtypes.json
-    /services/host/<hostname>/rtstats.json
-    /services/host/<hostname>/topology.json
+/services/host/<hostname>/feedtypes.json
+/services/host/<hostname>/rtstats.json
+/services/host/<hostname>/topology.json
 
 """
+
 import collections
 import datetime
 import json
 
 import memcache
 import pandas as pd
-from paste.request import parse_formvars
 import rtstats_util as util
+from paste.request import parse_formvars
 
 
 def Tree():

htdocs/services/hosts.py

@@ -5,12 +5,13 @@
 
 This service caches for 1 hour before refreshing.
 """
-import json
+
 import datetime
+import json
 
 import memcache
-from paste.request import parse_formvars
 import rtstats_util as util
+from paste.request import parse_formvars
 
 
 def run(feedtype):

htdocs/services/idd.py

@@ -5,13 +5,14 @@
 
 This service is cached for 3 minutes via memcached
 """
-import json
+
 import datetime
+import json
 
 import memcache
 import pytz
-from paste.request import parse_formvars
 import rtstats_util as util
+from paste.request import parse_formvars
 
 
 def run(feedtype):

pylib/myview.py

@@ -1,4 +1,5 @@
 """Simple templating"""
+
 import os
 
 

pylib/rtstats_util.py

@@ -3,15 +3,12 @@
 import json
 import os
 
-os.environ["MPLCONFIGDIR"] = "/tmp"  # hack
-import psycopg2
-
 import matplotlib
-from matplotlib.ticker import FuncFormatter
 import matplotlib.dates as mdates
+import psycopg2
+from matplotlib.ticker import FuncFormatter
 
 matplotlib.use("agg")
-import matplotlib.pyplot as plt
 from pandas.plotting import register_matplotlib_converters
 
 register_matplotlib_converters()

pyproject.toml

@@ -1,6 +1,11 @@
 
 [tool.ruff]
 line-length = 79
-select = ["E", "F", "I"]
 target-version = "py39"
 
+[tool.ruff.lint]
+select = [
+    "E",
+    "F",
+    "I",
+]

scripts/agg_rtstats.py

@@ -1,8 +1,9 @@
 """Aggregate the hourly rtstats"""
+
 import datetime
 
-import rtstats_util as util
 import pytz
+import rtstats_util as util
 
 
 def daily(pgconn):

scripts/assign_hostname_geom.py

@@ -1,12 +1,14 @@
 """Use GeoIP and assign lat/lon to hostnames"""
+
 from __future__ import print_function
-import os
+
 import json
+import os
 import re
 import socket
 
-import psycopg2
 import geoip2.database
+import psycopg2
 
 RE_IP = re.compile(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$")