new app subnet-flowcap

analyzers/subnet-flowcap/README.md

@@ -0,0 +1,64 @@
+# Subnet Flow cap
+
+Remove flow storage cap for selected subnets 
+
+
+## Flow Cap concept
+
+In Trisul, user can specify a _Flow Volume Cutoff_. Only flows transmitting or receiving a total volume
+greater than the _Flow Volume Cutoff_  will be stored in the database. Smaller flows will be discarded.
+This is to prevent customer databases from exploding in size. 
+
+See [Configuring Session Cutoff](https://www.trisul.org/docs/ug/flow/tuning.html#optimize_flow_handling)
+
+With this app, you gain ability to use a volume cutoff but also allow selected subnets to store all flows. 
+
+1. Set the Volume Cutoff Bytes option to 0 in Session Groups admin screen
+2. Specify a list of subnets which will store all flows as shown below
+3. Specify a new Volume Cutoff that will apply to all flows not matching the subnets listed 
+
+
+## Installing 
+
+To install this APP logon as admin, then select APP from _Web Admin > Manage > Apps._
+
+
+Config Parameters
+==============
+
+The config settings you can customize on a per Probe basis
+
+To supply your own custom settings, 
+
+1. create a new config file named `trisulnsm_subnet-flowcap.lua` in the probe config directory
+`/usr/local/var/lib/trisul-probe/domain0/probe0/context0/config` directory with the following
+2. You only supply new values for parameters you want to replace 
+
+
+````lua
+
+return {
+    -- which subnets
+    Subnets =  {
+
+        "209.85.175.96/30",
+        "209.85.175.160/29",
+
+
+    },
+
+    -- volume cutoff
+    VolumeCutOff =10000,
+}
+
+````
+
+
+UPDATES
+=======
+
+````
+1.0.0   Jul 5 2023     Initial release 
+````
+
+

analyzers/subnet-flowcap/iputils.lua

@@ -0,0 +1,34 @@
+local bit=require 'bit'
+
+-- ip number to trisulkey format
+function ipnum_tokey(ipnum)
+	return string.format("%02X.%02X.%02X.%02X", 
+		bit.rshift(ipnum,24), bit.band(bit.rshift(ipnum,16),0xff), bit.band(bit.rshift(ipnum,8),0xff), bit.band(bit.rshift(ipnum,0),0xff))
+end
+
+function ipnum_todotted(ipnum)
+	return string.format("%d.%d.%d.%d", 
+		bit.rshift(ipnum,24), bit.band(bit.rshift(ipnum,16),0xff), bit.band(bit.rshift(ipnum,8),0xff), bit.band(bit.rshift(ipnum,0),0xff))
+end
+
+function key_toipnum(key)
+  local pmatch,_, b1,b2,b3,b4= key:find("(%x+)%.(%x+)%.(%x+)%.(%x+)")
+  return  tonumber(b1,16)*16777216+tonumber(b2,16)*65536+tonumber(b3,16)*256+tonumber(b4,16) 
+end
+
+function ipstr_tokey(ipstr)
+  local pmatch,_, b1,b2,b3,b4= ipstr:find("(%d+)%.(%d+)%.(%d+)%.(%d+)")
+  return  string.format("%02X.%02X.%02X.%02X", b1,b2,b3,b4)
+end
+
+function cidr_range( ip_range)
+    local _,_, b1,b2,b3,b4,cidr = ip_range:find("(%d+)%.(%d+)%.(%d+)%.(%d+)/*(%d*)")
+	if b1 == nil then return; end  
+    local num_start = b1*math.pow(2,24) + b2*math.pow(2,16) + b3*math.pow(2,8) + b4*math.pow(2,0) 
+    local num_end = num_start
+    if #cidr > 0  then 
+      num_end = num_start + math.pow(2, 32-tonumber(cidr)) -1 
+    end
+
+	return num_start, num_end 
+end

analyzers/subnet-flowcap/mkconfig.lua

@@ -0,0 +1,26 @@
+function file_exists(name)
+  local f=io.open(name,"r")
+  if f~=nil then io.close(f) return true else return false end
+end
+
+
+function  make_config( custom_config_file, default_config_table)
+
+    -- load custom config if present 
+  local active_config = default_config_table
+
+    if file_exists(custom_config_file) then 
+      local newsettings = dofile(custom_config_file) 
+      T.loginfo("Loading custom settings from ".. custom_config_file)
+      for k,v in pairs(newsettings) do 
+        active_config[k]=v
+        T.loginfo("Loaded new setting "..k.."="..tostring(v))
+      end
+    else 
+      T.loginfo("Loaded default settings")
+    end
+
+  return active_config
+
+end 
+

analyzers/subnet-flowcap/pkg.yaml

@@ -0,0 +1,16 @@
+name: Subnet Flowcap
+
+version: 1.0.0
+
+author: trisul
+
+description: 
+   short: Remove flow storage caps for subnets
+   long:  |
+     For selected subnets remove flow storage caps.  
+
+apptype: lua
+
+appresources: subnet-flowcap.lua mkconfig.lua iputils.lua
+
+dependencies:  

analyzers/subnet-flowcap/subnet-flowcap.lua

@@ -0,0 +1,69 @@
+--
+-- subnet-flowcapp.lua
+--
+-- TYPE:        BACKEND SCRIPT
+-- PURPOSE:     Enforces flow cap except for a multiple subnet 
+-- 
+require 'mkconfig' 
+require 'iputils'
+
+TrisulPlugin = { 
+
+  id =  {
+    name = "Subnet Flowcap",
+    description = "Monitor IP flows and enforce flow cap except some subnets",
+  },
+
+
+
+  -- config load 
+  onload = function()
+
+    -- override by trisulnsm_subnet-flowcap.lua 
+    -- in probe config directory /usr/local/var/lib/trisul-probe/dX/pX/contextX/config 
+    --  
+    T.active_config = make_config(
+            T.env.get_config("App>DBRoot").."/config/trisulnsm_subnet-flowcap.lua",
+            {   
+                -- which subnets
+                Subnets =  { },
+
+                -- volume cutoff
+                VolumeCutOff =0,
+
+				-- numbers
+				SubnetNumbers = { } 
+            })
+
+
+  	-- convert to ipnum range
+	for _,ipcidr in pairs(T.active_config.Subnets) do 
+		local ns, ne = cidr_range( ipcidr) 
+		table.insert(T.active_config.SubnetNumbers, { numstart=ns, numend=ne } ) 
+	end 
+
+  end,
+
+  sg_monitor  = {
+
+    session_guid = '{99A78737-4B41-4387-8F31-8077DB917336}', -- optional
+
+    flushfilter = function(engine, flow) 
+
+		local ipna = key_toipnum(flow:flow():ipa())
+		local ipnz = key_toipnum(flow:flow():ipz())
+
+		for _,iprange in ipairs(T.active_config.SubnetNumbers) do 
+
+			if (ipna >= iprange.numstart and ipna <= iprange.numend ) or
+			   (ipnz >= iprange.numstart and ipnz <= iprange.numend ) then 
+				return true
+			end 
+		end 
+		return flow:az_bytes() + flow:za_bytes() > T.active_config.VolumeCutOff
+    end,
+
+
+  },
+
+}