feat: include source in script uploads

scripts/thunderstorm-collector.bat

@@ -46,6 +46,15 @@ SET /A MAX_AGE=30
 :: Debug
 SET DEBUG=0
 
+:: Source
+SET SOURCE=
+IF "%SOURCE%"=="" (
+    FOR /F "tokens=*" %%i IN ('hostname') DO SET HOSTNAME=%%i
+)
+IF "%SOURCE%" EQ "" AND "%HOSTNAME%" NEQ "" (
+    SOURCE="?source=%HOSTNAME%"
+)
+
 :: WELCOME -------------------------------------------------------
 
 ECHO =============================================================
@@ -112,7 +121,7 @@ FOR %%T IN (%COLLECT_DIRS%) DO (
                         :: Upload
                         ECHO Uploading %%F ..
                         :: We'll start the upload process in background to speed up the submission process 
-                        START /B curl -F file=@%%F -H "Content-Type: multipart/form-data" -o nul -s %URL_SCHEME%://%THUNDERSTORM_SERVER%:%THUNDERSTORM_PORT%/api/checkAsync
+                        START /B curl -F file=@%%F -H "Content-Type: multipart/form-data" -o nul -s %URL_SCHEME%://%THUNDERSTORM_SERVER%:%THUNDERSTORM_PORT%/api/checkAsync%SOURCE%
                     )
                 )
             )

scripts/thunderstorm-collector.pl

@@ -12,12 +12,14 @@
 # Usage examples:
 #   $> perl thunderstorm-collector.pl -- -s thunderstorm.internal.net
 #   $> perl thunderstorm-collector.pl -- --dir / --server thunderstorm.internal.net
+#   $> perl thunderstorm-collector.pl -- --dir / --server thunderstorm.internal.net --so "My Source"
 
 use warnings;
 use strict;
 use Getopt::Long;
 use LWP::UserAgent;
 use File::Spec::Functions qw( catfile );
+use Sys::Hostname;
 
 use Cwd; # module for finding the current working directory 
 
@@ -27,20 +29,32 @@
 my $server = "";
 my $port = 8080;
 my $scheme = "http";
-our $max_age = 3;       # in days
+my $source = "";
+our $max_age = 3000;       # in days
 our $max_size = 10;     # in megabytes
 our @skipElements = map { qr{$_} } ('^\/proc', '^\/mnt', '\.dat$', '\.npm');
 our @hardSkips = ('/proc', '/dev', '/sys');
 
 # Command Line Parameters
-GetOptions("dir=s"      => \$targetdir,  # same for --dir or -d
-           "server=s"   => \$server,     # same for --server or -s
-           "port=i"     => \$port,       # same for --port or -p
-           "debug"      => \$debug       # --debug
-          );
+GetOptions(
+    "dir|d=s"      => \$targetdir,  # --dir or -d
+    "server|s=s"   => \$server,     # --server or -s
+    "port|p=i"     => \$port,       # --port or -p
+    "source|so=s"  => \$source,     # --source or -so
+    "debug"        => \$debug       # --debug
+);
+
+# Use Hostname as Source if not set
+if ( $source eq "" ) {
+    $source = hostname;
+}
+# Add Source to URL if available
+if ( $source ne "" ) {
+    $source = "?source=$source";
+}
 
 # Composed Values
-our $api_endpoint = "$scheme://$server:$port/api/checkAsync";
+our $api_endpoint = "$scheme://$server:$port/api/checkAsync$source";
 our $current_date = time;
 
 # Stats
@@ -86,7 +100,7 @@ sub processDir {
             if ( $debug ) { print "[DEBUG] Checking $filepath ...\n"; }
         }
 
-        # Characteristics 
+        # Characteristics
         my $size = (stat($filepath))[7];
         my $mdate = (stat($filepath))[9];
         #print("SIZE: $size MDATE: $mdate\n");

scripts/thunderstorm-collector.ps1

@@ -19,6 +19,8 @@
         Server name (FQDN) or IP address of your Thunderstorm instance
     .PARAMETER ThunderstormPort 
         Port number on which the Thunderstorm service is listening (default: 8080)
+    .PARAMETER Source
+        Source of the submission (default: hostname of the system)
     .PARAMETER Folder 
         Folder to process (default: C:\)
     .PARAMETER MaxAge 
@@ -59,6 +61,10 @@ param
         [Alias('TP')]    
         [int]$ThunderstormPort = 8080,
 
+    [Parameter(HelpMessage="")]
+        [Alias('S')]
+        [string]$Source=$env:COMPUTERNAME,
+
     [Parameter(HelpMessage="Folder to process (default: C:\)")] 
         [ValidateNotNullOrEmpty()] 
         [Alias('F')]

scripts/thunderstorm-collector.py

@@ -7,6 +7,7 @@
 import ssl
 import time
 import uuid
+import socket
 
 # Configuration
 schema = "http"
@@ -206,14 +207,24 @@ def submit_sample(filepath):
         action="store_true",
         help="Skip TLS verification and proceed without checking.",
     )
+    parser.add_argument(
+        "-S",
+        "--source",
+        default=socket.gethostname(),
+        help="Source identifier to be used in the Thunderstorm submission.",
+    )
     parser.add_argument("--debug", action="store_true", help="Enable debug logging.")
 
     args = parser.parse_args()
 
     if args.tls:
         schema = "https"
 
-    api_endpoint = "{}://{}:{}/api/checkAsync".format(schema, args.server, args.port)
+    source = ""
+    if args.source:
+        source = f"?source={args.source}"
+
+    api_endpoint = "{}://{}:{}/api/checkAsync{}".format(schema, args.server, args.port, source)
 
     print("=" * 80)
     print("   Python Thunderstorm Collector")
@@ -227,6 +238,7 @@ def submit_sample(filepath):
     print("Maximum Age of Files: {}".format(max_age))
     print("Maximum File Size: {} MB".format(max_size))
     print("Excluded directories: {}".format(", ".join(hard_skips)))
+    print("Source Identifier: {}".format(args.source)) if args.source else None
     print()
 
     print("Starting the walk at: {} ...".format(", ".join(args.dirs)))

scripts/thunderstorm-collector.sh

@@ -16,11 +16,16 @@ LOG_TO_CMDLINE=1
 
 # Thunderstorm Server
 THUNDERSTORM_SERVER="ygdrasil.nextron"
+THUNDERSTORM_PORT=8080
 USE_SSL=0
 ASYNC_MODE=1
 
+# Source
+HOSTNAME=$(hostname -f)
+echo "$HOSTNAME"
+
 # Target selection 
-declare -a SCAN_FOLDERS=('/root' '/tmp' '/home' '/var' '/usr');  # folders to scan 
+declare -a SCAN_FOLDERS=('/root' '/tmp' '/home' '/var' '/usr');  # folders to scan
 MAX_AGE=14
 MAX_FILE_SIZE=2000  # max file size to check in kilobyte, default 2 MB
 
@@ -124,6 +129,11 @@ scheme="http"
 if [[ $USE_SSL -eq 1 ]]; then
     scheme="https"
 fi
+source=""
+if [[ -n $HOSTNAME ]]; then
+    source="?source=${HOSTNAME}"
+    echo "Source: $source"
+fi
 
 # Loop over filesystem
 for scandir in "${SCAN_FOLDERS[@]}";
@@ -142,7 +152,7 @@ do
             for retry in {1..3}; do
                 # Submit sample
                 result=$(curl -s -X POST \
-                        "$scheme://$THUNDERSTORM_SERVER:8080/api/$api_endpoint" \
+                        "$scheme://$THUNDERSTORM_SERVER:$THUNDERSTORM_PORT/api/$api_endpoint$source" \
                         --form "file=@${file_path};filename=${file_path}")
                 curl_exit=$?
                 if [ $curl_exit -ne 0 ]; then