Adding a new mechanism for logging jobs into a database in #13

AptecoPSFramework/AptecoPSFramework.psd1

@@ -158,7 +158,8 @@ PrivateData = @{
 
         # 'ReleaseNotes' des Moduls
         ReleaseNotes = '
-      Features: Adding an internal Function Add-HttpQueryPart to add new parts to a URI query
+      Feature: Adding a new local database with sqlite/DuckDB to log jobs into a database which will make it much easier to repeat jobs
+      Feature: Adding an internal Function Add-HttpQueryPart to add new parts to a URI query
       Feature: Improved Performance for DuckDB reader (e.g. 5 seconds for 150k rows instead of 70 seconds)
       Feature: Improved handling for sniffing csv files with DuckDB (example is in help)
 0.3.3 Fix: Setting a default logfile at the start of the module load named "logfile.log"

AptecoPSFramework/plugins/InxmailPro/Public/PeopleStage/invoke-upload.ps1

@@ -40,6 +40,19 @@ function Invoke-Upload{
         }
 
         # Log the job in the database
+        Set-JobLogDatabase
+        $jobId = Add-JobLog
+
+        $jobParams = [Hashtable]@{
+            "JobId" = $jobId
+            "Plugin" = $script:settings.plugin.guid
+            "Input" = ( ConvertTo-Json $InputHashtable -Depth 99 )
+            "Status" = "Starting"
+            "DebugMode" = $Script:debugMode
+            "Type" = $moduleName
+        }
+        Update-JobLog @jobParams
+
 
         #-----------------------------------------------
         # DEBUG MODE
@@ -104,6 +117,7 @@ function Invoke-Upload{
         If ( $Script:settings.upload.countRowsInputFile -eq $true ) {
             $rowsCount = Measure-Rows -Path $file.FullName -SkipFirstRow
             Write-Log -Message "Got a file with $( $rowsCount ) rows"
+            Update-JobLog -JobId $jobId -Inputrecords $rowsCount
         } else {
             Write-Log -Message "RowCount of input file not activated"
         }
@@ -523,6 +537,19 @@ function Invoke-Upload{
             Write-Log -message "    $( $param ) = '$( $return[$param] )'" -writeToHostToo $false
         }
 
+        # log the return into database and close connection
+        $jobReturnParams = [Hashtable]@{
+            "JobId" = $jobId
+            "Status" = "Finished"
+            "Finished" = $true
+            "Successful" = $check.successCount
+            "Failed" = 0 # TODO needs correction
+            "Totalseconds" = $processDuration.TotalSeconds
+            "Output" = ( ConvertTo-Json $return -Depth 99)
+        }
+        Update-JobLog @jobReturnParams
+        Close-DuckDBConnection -Name "JobLog"
+
         # return the results
         $return
 

AptecoPSFramework/private/jobs/Add-JobLog.ps1

@@ -0,0 +1,100 @@
+
+<#
+If ( $isDuckDBLoaded -eq $true )
+
+    # Resolve path first
+    $absolutePath = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($script:settings.joblogDB)
+    
+    # Build Connection string
+    $connString = "DataSource=$( $absolutePath )"
+
+    # Add connection to duckdb
+    Add-DuckDBConnection -Name "JobLog" -ConnectionString $connString
+
+Incremental ID
+Guid/ProcessID
+Plugin
+Debug Yes/No
+JobType (Upload, Messages, Lists, Preview)
+InputHashtable as JSON
+InputRecordsCount
+CreateDateTime
+UpdateDateTime
+FinishDateTime
+TotalTime
+output is the returned hashtable
+
+Return an ID
+
+
+CREATE TABLE IF NOT EXISTS joblog (
+     id              INTEGER PRIMARY KEY
+    ,created         TEXT DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW', 'localtime'))
+    ,updated         TEXT
+    ,finished        INTEGER
+    ,status          TEXT
+    ,process         TEXT
+    ,plugin          TEXT
+    ,debug           INTEGER
+    ,type            TEXT
+    ,input           TEXT
+    ,inputrecords    INTEGER
+    ,successful      INTEGER
+    ,failed          INTEGER
+    ,totalseconds    INTEGER
+    ,output          TEXT
+)
+
+CREATE TRIGGER IF NOT EXISTS update_joblog_trigger
+AFTER UPDATE On joblog
+BEGIN
+   UPDATE joblog SET updated = STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW', 'localtime') WHERE id = NEW.id;
+END;
+
+And then allow to execute the job type
+
+
+Then when creating a new row with
+insert into joblog (status) values ('abc');
+select last_insert_rowid()
+
+you get back the id value
+
+https://duckdb.org/docs/extensions/sqlite#writing-data-to-sqlite
+
+
+INSERT INTO sqlite_db.tbl VALUES (42, 'DuckDB');
+
+
+INSTALL sqlite;
+LOAD sqlite;
+ATTACH 'C:\Users\Florian\Downloads\a pet co\apetco.sqlite' AS apetco (TYPE sqlite);
+use apetco;
+select * from kontakte;
+-- select * from duckdb_extensions();
+
+#>
+
+
+Function Add-JobLog {
+    <#
+
+    ...
+
+    #>
+    [cmdletbinding()]
+    param(
+        # [Parameter(Mandatory=$true)][String]$Guid
+        #,[Parameter(Mandatory=$true)][String]$ConnectionString
+        # TODO also allow use other connection strings as input parameter?
+    )
+
+    Process {
+
+        # TODO check if connection is open?
+
+        Read-DuckDBQueryAsScalar -Name "JobLog" -Query "INSERT INTO joblog (process) values ('$( $Script:processId )'); SELECT last_insert_rowid()"
+
+    }
+
+}

AptecoPSFramework/private/jobs/Clean-LogDatabase.ps1

@@ -0,0 +1 @@
+# Delete some records if the log gets too big

AptecoPSFramework/private/jobs/Set-JobLogDatabase.ps1

@@ -0,0 +1,29 @@
+
+Function Set-JobLogDatabase {
+    [cmdletbinding()]
+    param(
+    )
+
+    Process {
+
+        # Resolve path first
+        $absolutePath = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($script:settings.joblogDB)
+
+        # Build Connection string
+        $connString = "DataSource=$( $absolutePath )"
+
+        # Add connection to duckdb
+        Add-DuckDBConnection -Name "JobLog" -ConnectionString $connString
+
+        # Open the connection
+        Open-DuckDBConnection -Name "JobLog"
+
+        # Create the database, if not exists
+        $joblogCreateStatementPath = Join-Path -Path $Script:moduleRoot -ChildPath "sql/joblog_create.sql"
+        $joblogCreateStatement = Get-Content -Path $joblogCreateStatementPath -Encoding UTF8 -Raw
+        Invoke-DuckDBQueryAsNonExecute -Query $joblogCreateStatement -ConnectionName "JobLog"
+
+
+    }
+
+}

AptecoPSFramework/private/jobs/Update-JobLog.ps1

@@ -0,0 +1,56 @@
+Function Update-JobLog {
+    <#
+
+    ...
+
+    #>
+    [cmdletbinding()]
+    param(
+
+          [Parameter(Mandatory=$true)][Int]$JobId                # The Job ID that you have already got from the database
+
+          # Values to change
+         ,[Parameter(Mandatory=$false)][Switch]$Finished = $false   # Finished like 0 or 1
+         ,[Parameter(Mandatory=$false)][String]$Status = ""     # Status like "Finished" and others
+         #,[Parameter(Mandatory=$false)][String]$Process = ""    # Process ID
+         ,[Parameter(Mandatory=$false)][String]$Plugin = ""     # Plugin guid
+         ,[Parameter(Mandatory=$false)][Int]$DebugMode = 0          # Debug mode like 0 or 1
+         ,[Parameter(Mandatory=$false)][String]$Type = ""       # Type like UPLOAD, MESSAGES, LISTS etc.
+         ,[Parameter(Mandatory=$false)][String]$Input = ""      # Input hashtable
+         ,[Parameter(Mandatory=$false)][Int]$Inputrecords = 0   # Number of records that have been put in
+         ,[Parameter(Mandatory=$false)][Int]$Successful = 0     # Successful records, only needed on uploads
+         ,[Parameter(Mandatory=$false)][Int]$Failed = 0         # Failed records, only needed on uploads
+         ,[Parameter(Mandatory=$false)][Int]$Totalseconds = 0   # Seconds in total, logged at the end
+         ,[Parameter(Mandatory=$false)][String]$Output = ""     # Output hashtable (if it is suitable)
+    )
+
+    Process {
+
+        $sb = [System.Text.StringBuilder]::new()
+        $sb.Append("UPDATE joblog SET ")
+
+        $upd = [System.Collections.ArrayList]@()
+        [void]$upd.Add("finished = $( $Finished )")
+        [void]$upd.Add("status = '$( $Status )'")
+        #[void]$upd.Add("process = '$( $Process )'")
+        [void]$upd.Add("plugin = '$( $Plugin )'")
+        [void]$upd.Add("debug = $( $DebugMode )")
+        [void]$upd.Add("type = '$( $Type )'")
+        [void]$upd.Add("input = '$( $Input )'")
+        [void]$upd.Add("inputrecords = $( $Inputrecords )")
+        [void]$upd.Add("successful = $( $Successful )")
+        [void]$upd.Add("failed = $( $Failed )")
+        [void]$upd.Add("totalseconds = $( $Totalseconds )")
+        [void]$upd.Add("output = '$( $Output )'")
+        $params = $upd -join ", "
+        $sb.Append($params)
+
+        $sb.Append("WHERE id = $( $JobId )")
+
+        #$query = "update logjob set process = 'abc' where id = $( $JobId )"
+        Invoke-DuckDBQueryAsNonExecute -Query $sb.ToString() -ConnectionName "JobLog"
+
+    }
+
+
+}

AptecoPSFramework/settings/defaultsettings.ps1

@@ -31,5 +31,8 @@
 
     # DuckDB
     "defaultDuckDBConnection" = "Data Source=:memory:;"     # Default DuckDB connection -> In-Memory connection, could also be a file
+
+    # LogJob database (sqlite file via DuckDB will directly made, not connected to defaultDuckDBConnection)
+    "joblogDB" = "./logjob.sqlite"
 
 }

AptecoPSFramework/sql/joblog_create.sql

@@ -0,0 +1,23 @@
+CREATE TABLE IF NOT EXISTS joblog (
+     id              INTEGER PRIMARY KEY
+    ,created         TEXT DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW', 'localtime'))
+    ,updated         TEXT
+    ,finished        INTEGER DEFAULT 0
+    ,status          TEXT
+    ,process         TEXT
+    ,plugin          TEXT
+    ,debug           INTEGER
+    ,type            TEXT
+    ,input           TEXT
+    ,inputrecords    INTEGER
+    ,successful      INTEGER
+    ,failed          INTEGER
+    ,totalseconds    INTEGER
+    ,output          TEXT
+);
+
+CREATE TRIGGER IF NOT EXISTS update_joblog_trigger
+AFTER UPDATE On joblog
+BEGIN
+   UPDATE joblog SET updated = STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW', 'localtime') WHERE id = NEW.id;
+END;