diff --git a/AptecoPSFramework/AptecoPSFramework.psd1 b/AptecoPSFramework/AptecoPSFramework.psd1 index e7a0042..b188528 100644 --- a/AptecoPSFramework/AptecoPSFramework.psd1 +++ b/AptecoPSFramework/AptecoPSFramework.psd1 @@ -158,7 +158,8 @@ PrivateData = @{ # 'ReleaseNotes' des Moduls ReleaseNotes = ' - Features: Adding an internal Function Add-HttpQueryPart to add new parts to a URI query + Feature: Adding a new local database with sqlite/DuckDB to log jobs into a database which will make it much easier to repeat jobs + Feature: Adding an internal Function Add-HttpQueryPart to add new parts to a URI query Feature: Improved Performance for DuckDB reader (e.g. 5 seconds for 150k rows instead of 70 seconds) Feature: Improved handling for sniffing csv files with DuckDB (example is in help) 0.3.3 Fix: Setting a default logfile at the start of the module load named "logfile.log" diff --git a/AptecoPSFramework/plugins/InxmailPro/Public/PeopleStage/invoke-upload.ps1 b/AptecoPSFramework/plugins/InxmailPro/Public/PeopleStage/invoke-upload.ps1 index f6dcba4..b458ac5 100644 --- a/AptecoPSFramework/plugins/InxmailPro/Public/PeopleStage/invoke-upload.ps1 +++ b/AptecoPSFramework/plugins/InxmailPro/Public/PeopleStage/invoke-upload.ps1 @@ -40,6 +40,19 @@ function Invoke-Upload{ } # Log the job in the database + Set-JobLogDatabase + $jobId = Add-JobLog + + $jobParams = [Hashtable]@{ + "JobId" = $jobId + "Plugin" = $script:settings.plugin.guid + "Input" = ( ConvertTo-Json $InputHashtable -Depth 99 ) + "Status" = "Starting" + "DebugMode" = $Script:debugMode + "Type" = $moduleName + } + Update-JobLog @jobParams + #----------------------------------------------- # DEBUG MODE @@ -104,6 +117,7 @@ function Invoke-Upload{ If ( $Script:settings.upload.countRowsInputFile -eq $true ) { $rowsCount = Measure-Rows -Path $file.FullName -SkipFirstRow Write-Log -Message "Got a file with $( $rowsCount ) rows" + Update-JobLog -JobId $jobId -Inputrecords $rowsCount } else { Write-Log -Message "RowCount of input file not activated" } @@ -523,6 +537,19 @@ function Invoke-Upload{ Write-Log -message " $( $param ) = '$( $return[$param] )'" -writeToHostToo $false } + # log the return into database and close connection + $jobReturnParams = [Hashtable]@{ + "JobId" = $jobId + "Status" = "Finished" + "Finished" = $true + "Successful" = $check.successCount + "Failed" = 0 # TODO needs correction + "Totalseconds" = $processDuration.TotalSeconds + "Output" = ( ConvertTo-Json $return -Depth 99) + } + Update-JobLog @jobReturnParams + Close-DuckDBConnection -Name "JobLog" + # return the results $return diff --git a/AptecoPSFramework/private/jobs/Add-JobLog.ps1 b/AptecoPSFramework/private/jobs/Add-JobLog.ps1 new file mode 100644 index 0000000..03e4a96 --- /dev/null +++ b/AptecoPSFramework/private/jobs/Add-JobLog.ps1 @@ -0,0 +1,100 @@ + +<# +If ( $isDuckDBLoaded -eq $true ) + + # Resolve path first + $absolutePath = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($script:settings.joblogDB) + + # Build Connection string + $connString = "DataSource=$( $absolutePath )" + + # Add connection to duckdb + Add-DuckDBConnection -Name "JobLog" -ConnectionString $connString + +Incremental ID +Guid/ProcessID +Plugin +Debug Yes/No +JobType (Upload, Messages, Lists, Preview) +InputHashtable as JSON +InputRecordsCount +CreateDateTime +UpdateDateTime +FinishDateTime +TotalTime +output is the returned hashtable + +Return an ID + + +CREATE TABLE IF NOT EXISTS joblog ( + id INTEGER PRIMARY KEY + ,created TEXT DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW', 'localtime')) + ,updated TEXT + ,finished INTEGER + ,status TEXT + ,process TEXT + ,plugin TEXT + ,debug INTEGER + ,type TEXT + ,input TEXT + ,inputrecords INTEGER + ,successful INTEGER + ,failed INTEGER + ,totalseconds INTEGER + ,output TEXT +) + +CREATE TRIGGER IF NOT EXISTS update_joblog_trigger +AFTER UPDATE On joblog +BEGIN + UPDATE joblog SET updated = STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW', 'localtime') WHERE id = NEW.id; +END; + +And then allow to execute the job type + + +Then when creating a new row with +insert into joblog (status) values ('abc'); +select last_insert_rowid() + +you get back the id value + +https://duckdb.org/docs/extensions/sqlite#writing-data-to-sqlite + + +INSERT INTO sqlite_db.tbl VALUES (42, 'DuckDB'); + + +INSTALL sqlite; +LOAD sqlite; +ATTACH 'C:\Users\Florian\Downloads\a pet co\apetco.sqlite' AS apetco (TYPE sqlite); +use apetco; +select * from kontakte; +-- select * from duckdb_extensions(); + +#> + + +Function Add-JobLog { + <# + + ... + + #> + [cmdletbinding()] + param( + # [Parameter(Mandatory=$true)][String]$Guid + #,[Parameter(Mandatory=$true)][String]$ConnectionString + # TODO also allow use other connection strings as input parameter? + ) + + Process { + + # TODO check if connection is open? + + Read-DuckDBQueryAsScalar -Name "JobLog" -Query "INSERT INTO joblog (process) values ('$( $Script:processId )'); SELECT last_insert_rowid()" + + } + +} \ No newline at end of file diff --git a/AptecoPSFramework/private/jobs/Clean-LogDatabase.ps1 b/AptecoPSFramework/private/jobs/Clean-LogDatabase.ps1 new file mode 100644 index 0000000..c8a197d --- /dev/null +++ b/AptecoPSFramework/private/jobs/Clean-LogDatabase.ps1 @@ -0,0 +1 @@ +# Delete some records if the log gets too big \ No newline at end of file diff --git a/AptecoPSFramework/private/jobs/Set-JobLogDatabase.ps1 b/AptecoPSFramework/private/jobs/Set-JobLogDatabase.ps1 new file mode 100644 index 0000000..f90b676 --- /dev/null +++ b/AptecoPSFramework/private/jobs/Set-JobLogDatabase.ps1 @@ -0,0 +1,29 @@ + +Function Set-JobLogDatabase { + [cmdletbinding()] + param( + ) + + Process { + + # Resolve path first + $absolutePath = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($script:settings.joblogDB) + + # Build Connection string + $connString = "DataSource=$( $absolutePath )" + + # Add connection to duckdb + Add-DuckDBConnection -Name "JobLog" -ConnectionString $connString + + # Open the connection + Open-DuckDBConnection -Name "JobLog" + + # Create the database, if not exists + $joblogCreateStatementPath = Join-Path -Path $Script:moduleRoot -ChildPath "sql/joblog_create.sql" + $joblogCreateStatement = Get-Content -Path $joblogCreateStatementPath -Encoding UTF8 -Raw + Invoke-DuckDBQueryAsNonExecute -Query $joblogCreateStatement -ConnectionName "JobLog" + + + } + +} \ No newline at end of file diff --git a/AptecoPSFramework/private/jobs/Update-JobLog.ps1 b/AptecoPSFramework/private/jobs/Update-JobLog.ps1 new file mode 100644 index 0000000..9445b02 --- /dev/null +++ b/AptecoPSFramework/private/jobs/Update-JobLog.ps1 @@ -0,0 +1,56 @@ +Function Update-JobLog { + <# + + ... + + #> + [cmdletbinding()] + param( + + [Parameter(Mandatory=$true)][Int]$JobId # The Job ID that you have already got from the database + + # Values to change + ,[Parameter(Mandatory=$false)][Switch]$Finished = $false # Finished like 0 or 1 + ,[Parameter(Mandatory=$false)][String]$Status = "" # Status like "Finished" and others + #,[Parameter(Mandatory=$false)][String]$Process = "" # Process ID + ,[Parameter(Mandatory=$false)][String]$Plugin = "" # Plugin guid + ,[Parameter(Mandatory=$false)][Int]$DebugMode = 0 # Debug mode like 0 or 1 + ,[Parameter(Mandatory=$false)][String]$Type = "" # Type like UPLOAD, MESSAGES, LISTS etc. + ,[Parameter(Mandatory=$false)][String]$Input = "" # Input hashtable + ,[Parameter(Mandatory=$false)][Int]$Inputrecords = 0 # Number of records that have been put in + ,[Parameter(Mandatory=$false)][Int]$Successful = 0 # Successful records, only needed on uploads + ,[Parameter(Mandatory=$false)][Int]$Failed = 0 # Failed records, only needed on uploads + ,[Parameter(Mandatory=$false)][Int]$Totalseconds = 0 # Seconds in total, logged at the end + ,[Parameter(Mandatory=$false)][String]$Output = "" # Output hashtable (if it is suitable) + ) + + Process { + + $sb = [System.Text.StringBuilder]::new() + $sb.Append("UPDATE joblog SET ") + + $upd = [System.Collections.ArrayList]@() + [void]$upd.Add("finished = $( $Finished )") + [void]$upd.Add("status = '$( $Status )'") + #[void]$upd.Add("process = '$( $Process )'") + [void]$upd.Add("plugin = '$( $Plugin )'") + [void]$upd.Add("debug = $( $DebugMode )") + [void]$upd.Add("type = '$( $Type )'") + [void]$upd.Add("input = '$( $Input )'") + [void]$upd.Add("inputrecords = $( $Inputrecords )") + [void]$upd.Add("successful = $( $Successful )") + [void]$upd.Add("failed = $( $Failed )") + [void]$upd.Add("totalseconds = $( $Totalseconds )") + [void]$upd.Add("output = '$( $Output )'") + $params = $upd -join ", " + $sb.Append($params) + + $sb.Append("WHERE id = $( $JobId )") + + #$query = "update logjob set process = 'abc' where id = $( $JobId )" + Invoke-DuckDBQueryAsNonExecute -Query $sb.ToString() -ConnectionName "JobLog" + + } + + +} \ No newline at end of file diff --git a/AptecoPSFramework/settings/defaultsettings.ps1 b/AptecoPSFramework/settings/defaultsettings.ps1 index 5476e6c..9b007d9 100644 --- a/AptecoPSFramework/settings/defaultsettings.ps1 +++ b/AptecoPSFramework/settings/defaultsettings.ps1 @@ -31,5 +31,8 @@ # DuckDB "defaultDuckDBConnection" = "Data Source=:memory:;" # Default DuckDB connection -> In-Memory connection, could also be a file + + # LogJob database (sqlite file via DuckDB will directly made, not connected to defaultDuckDBConnection) + "joblogDB" = "./logjob.sqlite" } \ No newline at end of file diff --git a/AptecoPSFramework/sql/joblog_create.sql b/AptecoPSFramework/sql/joblog_create.sql new file mode 100644 index 0000000..c101a84 --- /dev/null +++ b/AptecoPSFramework/sql/joblog_create.sql @@ -0,0 +1,23 @@ +CREATE TABLE IF NOT EXISTS joblog ( + id INTEGER PRIMARY KEY + ,created TEXT DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW', 'localtime')) + ,updated TEXT + ,finished INTEGER DEFAULT 0 + ,status TEXT + ,process TEXT + ,plugin TEXT + ,debug INTEGER + ,type TEXT + ,input TEXT + ,inputrecords INTEGER + ,successful INTEGER + ,failed INTEGER + ,totalseconds INTEGER + ,output TEXT +); + +CREATE TRIGGER IF NOT EXISTS update_joblog_trigger +AFTER UPDATE On joblog +BEGIN + UPDATE joblog SET updated = STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW', 'localtime') WHERE id = NEW.id; +END;