Ensure compatibility with Python 3.5.3

This commit replaces multiple occurences of new features which were not yet implemented with Python 3.5.3, which is the reference backwards compatibility version for this package. The version is based on the current Python version in Debian Stretch (oldstable). According to pkgs.org, all other distros use 3.6+, so 3.5.3 is the lower boundary. Changes: * Add maxsize argument to functools.lru_cache decorator * Replace f"" with .format() * Replace variable type hints "var: type = val" with "# type:" comments * Replace pstats.SortKey enum with strings in performance tests Additionally, various styling fixes were applied. The version compatibility was tested with tox, pyenv and Python 3.5.3, but there is no tox.ini yet which automates this test. Bump patch version number to 0.10.3 Update author's email address. Resolves bitkeks#27
dolevha · Apr 24, 2020 · 5cdb514 · 5cdb514
1 parent 5d1c5b8
commit 5cdb514
Show file tree

Hide file tree

Showing 15 changed files with 77 additions and 65 deletions.
diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@ This package contains libraries and tools for **NetFlow versions 1, 5 and 9, and
 
 Version 9 is the first NetFlow version using templates. Templates make dynamically sized and configured NetFlow data flowsets possible, which makes the collector's job harder. The library provides the `netflow.parse_packet()` function as the main API point (see below). By importing `netflow.v1`, `netflow.v5` or `netflow.v9` you have direct access to the respective parsing objects, but at the beginning you probably will have more success by running the reference collector (example below) and look into its code. IPFIX (IP Flow Information Export) is based on NetFlow v9 and standardized by the IETF. All related classes are contained in `netflow.ipfix`. 
 
-Copyright 2016-2020 Dominik Pataky <dev@bitkeks.eu>
+Copyright 2016-2020 Dominik Pataky <software+pynetflow@dpataky.eu>
 
 Licensed under MIT License. See LICENSE.
 

diff --git a/netflow/__init__.py b/netflow/__init__.py
@@ -3,7 +3,7 @@
 """
 This file belongs to https://github.com/bitkeks/python-netflow-v9-softflowd.
 
-Copyright 2016-2020 Dominik Pataky <dev@bitkeks.eu>
+Copyright 2016-2020 Dominik Pataky <software+pynetflow@dpataky.eu>
 Licensed under MIT License. See LICENSE.
 """
 

diff --git a/netflow/analyzer.py b/netflow/analyzer.py
@@ -4,14 +4,12 @@
 Reference analyzer script for NetFlow Python package.
 This file belongs to https://github.com/bitkeks/python-netflow-v9-softflowd.
 
-Copyright 2016-2020 Dominik Pataky <dev@bitkeks.eu>
+Copyright 2016-2020 Dominik Pataky <software+pynetflow@dpataky.eu>
 Licensed under MIT License. See LICENSE.
 """
 
 import argparse
-from collections import namedtuple
 import contextlib
-from datetime import datetime
 import functools
 import gzip
 import ipaddress
@@ -20,7 +18,8 @@
 import os.path
 import socket
 import sys
-
+from collections import namedtuple
+from datetime import datetime
 
 IP_PROTOCOLS = {
     1: "ICMP",
@@ -65,10 +64,10 @@ def human_size(size_bytes):
         return "%dB" % size_bytes
     elif size_bytes / 1024. < 1024:
         return "%.2fK" % (size_bytes / 1024.)
-    elif size_bytes / 1024.**2 < 1024:
-        return "%.2fM" % (size_bytes / 1024.**2)
+    elif size_bytes / 1024. ** 2 < 1024:
+        return "%.2fM" % (size_bytes / 1024. ** 2)
     else:
-        return "%.2fG" % (size_bytes / 1024.**3)
+        return "%.2fG" % (size_bytes / 1024. ** 3)
 
 
 def human_duration(seconds):
@@ -78,7 +77,7 @@ def human_duration(seconds):
         return "%d sec" % seconds
     if seconds / 60 > 60:
         # hours
-        return "%d:%02d.%02d hours" % (seconds / 60**2, seconds % 60**2 / 60, seconds % 60)
+        return "%d:%02d.%02d hours" % (seconds / 60 ** 2, seconds % 60 ** 2 / 60, seconds % 60)
     # minutes
     return "%02d:%02d min" % (seconds / 60, seconds % 60)
 
@@ -90,6 +89,7 @@ class Connection:
     'src' describes the peer which sends more data towards the other. This
     does NOT have to mean that 'src' was the initiator of the connection.
     """
+
     def __init__(self, flow1, flow2):
         if not flow1 or not flow2:
             raise Exception("A connection requires two flows")
@@ -129,7 +129,7 @@ def __init__(self, flow1, flow2):
         if self.duration < 0:
             # 32 bit int has its limits. Handling overflow here
             # TODO: Should be handled in the collection phase
-            self.duration = (2**32 - src['FIRST_SWITCHED']) + src['LAST_SWITCHED']
+            self.duration = (2 ** 32 - src['FIRST_SWITCHED']) + src['LAST_SWITCHED']
 
     def __repr__(self):
         return "<Connection from {} to {}, size {}>".format(
@@ -298,25 +298,27 @@ def total_packets(self):
                 continue
 
             if first_line:
-                print("{:19} | {:14} | {:8} | {:9} | {:7} | Involved hosts".format("Timestamp", "Service", "Size", "Duration", "Packets"))
+                print("{:19} | {:14} | {:8} | {:9} | {:7} | Involved hosts".format("Timestamp", "Service", "Size",
+                                                                                   "Duration", "Packets"))
                 print("-" * 100)
                 first_line = False
 
             print("{timestamp} | {service:<14} | {size:8} | {duration:9} | {packets:7} | "
-                  "Between {src_host} ({src}) and {dest_host} ({dest})" \
+                  "Between {src_host} ({src}) and {dest_host} ({dest})"
                   .format(timestamp=timestamp, service=con.service.upper(), src_host=con.hostnames.src, src=con.src,
                           dest_host=con.hostnames.dest, dest=con.dest, size=con.human_size, duration=con.human_duration,
                           packets=con.total_packets))
 
     if skipped > 0:
-        print(f"{skipped} connections skipped, because they had less than {skipped_threshold} packets (this value can be set with the -p flag).")
+        print("{skipped} connections skipped, because they had less than {skipped_threshold} packets "
+              "(this value can be set with the -p flag).".format(skipped=skipped, skipped_threshold=skipped_threshold))
 
     if not args.verbose:
         # Exit here if no debugging session was wanted
         exit(0)
 
     if len(pending) > 0:
-        print(f"\nThere are {len(pending)} first_switched entries left in the pending dict!")
+        print("\nThere are {pending} first_switched entries left in the pending dict!".format(pending=len(pending)))
         all_noise = True
         for first_switched, flows in sorted(pending.items(), key=lambda x: x[0]):
             for peer, flow in flows.items():
@@ -327,19 +329,21 @@ def total_packets(self):
 
                 src = flow.get("IPV4_SRC_ADDR") or flow.get("IPV6_SRC_ADDR")
                 src_host = resolve_hostname(src)
-                src_text = f"{src}" if src == src_host else f"{src_host} ({src})"
+                src_text = "{}".format(src) if src == src_host else "{} ({})".format(src_host, src)
                 dst = flow.get("IPV4_DST_ADDR") or flow.get("IPV6_DST_ADDR")
                 dst_host = resolve_hostname(dst)
-                dst_text = f"{dst}" if dst == dst_host else f"{dst_host} ({dst})"
+                dst_text = "{}".format(dst) if dst == dst_host else "{} ({})".format(dst_host, dst)
                 proto = flow["PROTOCOL"]
                 size = flow["IN_BYTES"]
                 packets = flow["IN_PKTS"]
                 src_port = flow.get("L4_SRC_PORT", 0)
                 dst_port = flow.get("L4_DST_PORT", 0)
 
-                print(f"From {src_text}:{src_port} to {dst_text}:{dst_port} with "
-                      f"proto {IP_PROTOCOLS.get(proto, 'UNKNOWN')} and size {human_size(size)}"
-                      f" ({packets} packets)")
+                print("From {src_text}:{src_port} to {dst_text}:{dst_port} with "
+                      "proto {proto} and size {size}"
+                      " ({packets} packets)".format(src_text=src_text, src_port=src_port, dst_text=dst_text,
+                                                    dst_port=dst_port, proto=IP_PROTOCOLS.get(proto, 'UNKNOWN'),
+                                                    size=human_size(size), packets=packets))
 
         if all_noise:
             print("They were all noise!")
diff --git a/netflow/collector.py b/netflow/collector.py
@@ -4,23 +4,23 @@
 Reference collector script for NetFlow v1, v5, and v9 Python package.
 This file belongs to https://github.com/bitkeks/python-netflow-v9-softflowd.
 
-Copyright 2016-2020 Dominik Pataky <dev@bitkeks.eu>
+Copyright 2016-2020 Dominik Pataky <software+pynetflow@dpataky.eu>
 Licensed under MIT License. See LICENSE.
 """
 import argparse
 import gzip
 import json
-from collections import namedtuple
-import queue
 import logging
+import queue
 import socket
 import socketserver
 import threading
 import time
+from collections import namedtuple
 
+from .ipfix import IPFIXTemplateNotRecognized
 from .utils import UnknownExportVersion, parse_packet
 from .v9 import V9TemplateNotRecognized
-from .ipfix import IPFIXTemplateNotRecognized
 
 RawPacket = namedtuple('RawPacket', ['ts', 'client', 'data'])
 ParsedPacket = namedtuple('ParsedPacket', ['ts', 'client', 'export'])
@@ -118,7 +118,7 @@ def run(self):
             while not self._shutdown.is_set():
                 try:
                     # 0.5s delay to limit CPU usage while waiting for new packets
-                    pkt: RawPacket = self.input.get(block=True, timeout=0.5)
+                    pkt = self.input.get(block=True, timeout=0.5)  # type: RawPacket
                 except queue.Empty:
                     continue
 

diff --git a/netflow/ipfix.py b/netflow/ipfix.py
@@ -4,12 +4,12 @@
 This file belongs to https://github.com/bitkeks/python-netflow-v9-softflowd.
 Reference: https://tools.ietf.org/html/rfc7011
 
-Copyright 2016-2020 Dominik Pataky <dev@bitkeks.eu>
+Copyright 2016-2020 Dominik Pataky <software+pynetflow@dpataky.eu>
 Licensed under MIT License. See LICENSE.
 """
-from collections import namedtuple
 import functools
 import struct
+from collections import namedtuple
 from typing import Optional, Union, List, Dict
 
 FieldType = namedtuple("FieldType", ["id", "name", "type"])
@@ -488,23 +488,23 @@ class IPFIXFieldTypes:
     ]
 
     @classmethod
-    @functools.lru_cache
+    @functools.lru_cache(maxsize=128)
     def by_id(cls, id_: int) -> Optional[FieldType]:
         for item in cls.iana_field_types:
             if item[0] == id_:
                 return FieldType(*item)
         return None
 
     @classmethod
-    @functools.lru_cache
+    @functools.lru_cache(maxsize=128)
     def by_name(cls, key: str) -> Optional[FieldType]:
         for item in cls.iana_field_types:
             if item[1] == key:
                 return FieldType(*item)
         return None
 
     @classmethod
-    @functools.lru_cache
+    @functools.lru_cache(maxsize=128)
     def get_type_unpack(cls, key: Union[int, str]) -> Optional[DataType]:
         """
         This method covers the mapping from a field type to a struct.unpack format string.
@@ -555,7 +555,7 @@ class IPFIXDataTypes:
     ]
 
     @classmethod
-    @functools.lru_cache
+    @functools.lru_cache(maxsize=128)
     def by_name(cls, key: str) -> Optional[DataType]:
         """
         Get DataType by name if found, else None.
@@ -732,13 +732,13 @@ def __init__(self, data, template: List[Union[TemplateField, TemplateFieldEnterp
             # Here, reduced-size encoding of fields blocks the usage of IPFIXFieldTypes.get_type_unpack.
             # See comment in IPFIXFieldTypes.get_type_unpack for more information.
 
-            field_type: FieldType = IPFIXFieldTypes.by_id(field_type_id)
+            field_type = IPFIXFieldTypes.by_id(field_type_id)  # type: Optional[FieldType]
             if not field_type and type(field) is not TemplateFieldEnterprise:
                 # This should break, since the exporter seems to use a field identifier
                 # which is not standardized by IANA.
                 raise NotImplementedError("Field type with ID {} is not implemented".format(field_type_id))
 
-            datatype: str = field_type.type
+            datatype = field_type.type  # type: str
             discovered_fields.append((field_type.name, field_type_id))
 
             # Catch fields which are meant to be raw bytes and skip the rest
@@ -749,7 +749,7 @@ def __init__(self, data, template: List[Union[TemplateField, TemplateFieldEnterp
             # Go into int, uint, float types
             issigned = IPFIXDataTypes.is_signed(datatype)
             isfloat = IPFIXDataTypes.is_float(datatype)
-            assert not(all([issigned, isfloat]))  # signed int and float are exclusive
+            assert not (all([issigned, isfloat]))  # signed int and float are exclusive
 
             if field_length == 1:
                 unpacker += "b" if issigned else "B"
@@ -833,7 +833,8 @@ def __init__(self, data: bytes, templates):
 
         elif self.header.set_id >= 256:  # data set, set_id is template id
             while offset < self.header.length:
-                template: List[Union[TemplateField, TemplateFieldEnterprise]] = templates.get(self.header.set_id)
+                template = templates.get(
+                    self.header.set_id)  # type: List[Union[TemplateField, TemplateFieldEnterprise]]
                 if not template:
                     raise IPFIXTemplateNotRecognized
                 data_record = IPFIXDataRecord(data[offset:], template)
@@ -889,6 +890,7 @@ def __repr__(self):
 class IPFIXExportPacket:
     """IPFIX export packet with header, templates, options and data flowsets
     """
+
     def __init__(self, data: bytes, templates: Dict[int, list]):
         self.header = IPFIXHeader(data[:IPFIXHeader.size])
         self.sets = []
@@ -945,8 +947,8 @@ def parse_fields(data: bytes, count: int) -> (list, int):
     :param count:
     :return: List of fields and the new offset.
     """
-    offset: int = 0
-    fields: List[Union[TemplateField, TemplateFieldEnterprise]] = []
+    offset = 0
+    fields = []  # type: List[Union[TemplateField, TemplateFieldEnterprise]]
     for ctr in range(count):
         if data[offset] & 1 << 7 != 0:  # enterprise flag set
             pack = struct.unpack("!HHI", data[offset:offset + 8])

diff --git a/netflow/utils.py b/netflow/utils.py
@@ -3,17 +3,17 @@
 """
 This file belongs to https://github.com/bitkeks/python-netflow-v9-softflowd.
 
-Copyright 2016-2020 Dominik Pataky <dev@bitkeks.eu>
+Copyright 2016-2020 Dominik Pataky <software+pynetflow@dpataky.eu>
 Licensed under MIT License. See LICENSE.
 """
 
 import struct
 from typing import Union
 
+from .ipfix import IPFIXExportPacket
 from .v1 import V1ExportPacket
 from .v5 import V5ExportPacket
 from .v9 import V9ExportPacket
-from .ipfix import IPFIXExportPacket
 
 
 class UnknownExportVersion(Exception):

diff --git a/netflow/v1.py b/netflow/v1.py
@@ -83,4 +83,4 @@ def __init__(self, data):
 
     def __repr__(self):
         return "<ExportPacket v{} with {} records>".format(
-                self.header.version, self.header.count)
+            self.header.version, self.header.count)
diff --git a/netflow/v5.py b/netflow/v5.py
@@ -77,6 +77,7 @@ def to_dict(self):
 class V5ExportPacket:
     """The flow record holds the header and data flowsets.
     """
+
     def __init__(self, data):
         self.flows = []
         self.header = V5Header(data)
@@ -90,4 +91,4 @@ def __init__(self, data):
 
     def __repr__(self):
         return "<ExportPacket v{} with {} records>".format(
-                self.header.version, self.header.count)
+            self.header.version, self.header.count)