Various changes for emarsys in #10

- Return not always "data" when calling emarsys - Compressing json on calling emarsys - Added more examples in Readme - Show status progress bar when using Get-ListContact - Allow streaming when using Get-ListContact instead of a big array - Allow pipeline input for Get-ContactData and some more options to fetch the data that is requested
Apteco · Jun 21, 2024 · f9e83f3 · f9e83f3
1 parent d46098d
commit f9e83f3
Show file tree

Hide file tree

Showing 5 changed files with 166 additions and 70 deletions.
diff --git a/AptecoPSFramework/plugins/emarsys/Private/emarsys/Resolve-ContactData.ps1 b/AptecoPSFramework/plugins/emarsys/Private/emarsys/Resolve-ContactData.ps1
@@ -0,0 +1,62 @@
+
+function Resolve-ContactData {
+    [CmdletBinding()]
+    param (
+          [Parameter(Mandatory=$true,ValueFromPipeline=$true,Position=0)]$FetchList
+         ,[Parameter(Mandatory=$false)][Switch]$IgnoreErrors = $false
+    )
+
+    begin {
+
+        # fill fields into variable cache or just get it
+        If ( $Script:variableCache.fields -eq $null ) {
+
+            # Create a lookup hashtable for field names
+            $fieldHashtable = [hashtable]@{}            
+            get-field | ForEach-Object {
+                $fieldHashtable.add($_.id,$_.name)
+            }
+            $Script:variableCache.Add("fields",$fieldHashtable)
+
+        } else {
+
+            $fieldHashtable = $Script:variableCache.fields
+
+        }
+
+    }
+
+    process {
+
+        $res = [System.Collections.ArrayList]@()
+        $FetchList.result | ForEach-Object {
+
+            $row = $_
+
+            $newRow = [PSCustomObject]@{
+                "id" = $row.id      # always returned
+                "uid" = $row.uid    # always returned
+            }
+
+            $row.PSObject.Properties | Where-Object { $_.MemberType -eq "NoteProperty" -and $_.Name -notin "id","uid" } | ForEach-Object {
+                #$v = $_
+                $newRow | Add-Member -MemberType NoteProperty -Name $fieldHashtable[[int]$_.Name]  -Value $_.Value
+            }
+
+            [void]$res.Add($newRow)
+
+        }
+
+        # return directly
+        If ( $IgnoreErrors -eq $true ) {
+            $res
+        } else {
+            [PSCustomObject]@{
+                "errors" = $fetchList.errors
+                "result" =  $res
+            }
+        }
+
+    }
+
+}
diff --git a/AptecoPSFramework/plugins/emarsys/Public/emarsys/Get-ContactData.ps1 b/AptecoPSFramework/plugins/emarsys/Public/emarsys/Get-ContactData.ps1
@@ -6,7 +6,7 @@
 function Get-ContactData {
     [CmdletBinding()]
     param (
-        [Parameter(Mandatory=$true)][Array]$KeyValues
+         [Parameter(Mandatory=$true,ValueFromPipeline=$true,Position=0)][String[]]$KeyValues
         ,[Parameter(Mandatory=$false)][String]$KeyId = "email" # Identifies the contact by their id, uid, or the name/integer id of a custom field, such as email
         ,[Parameter(Mandatory=$false)][Array]$Fields = @("id","email")
         ,[Parameter(Mandatory=$false)][Switch]$ResolveFields = $false
@@ -16,35 +16,15 @@ function Get-ContactData {
 
     begin {
 
-        #Invoke-EmarsysLogin
-
-    }
-
-    process {
-
-        #$emarsys = $Script:variableCache.emarsys
-        #$fields = [System.Collections.ArrayList]@(1,2,3,31,9,46,11,12)
-
-        <#
-        $keys = [System.Collections.ArrayList]@(378808151,378808960)
-        $fetch = $emarsys.getContactData("id",$fields,$keys)
-        #>
-
-        #$keys = [System.Collections.ArrayList]@("[email protected]","[email protected]")
-        #ä$fetch = $emarsys.getContactData("3",$fields,$InputEmail)
-        #$fetch = $emarsys.getContactData("id",$fields,$InputEmail)
-
-        #$fetch
-
-
+        # build object for call
         $params = [Hashtable]@{
             "Object" = "contact"
             "Method" = "POST"
             "Path" = "getdata"
             "Body" = [PSCustomObject]@{
                 'fields' = $Fields
                 'keyId' = $KeyId
-                'keyValues' = $KeyValues
+                'keyValues' = $null
             }
         }
 
@@ -53,62 +33,96 @@ function Get-ContactData {
             $params.Add("Verbose", $true)
         }
 
-        # Request list creation
-        $fetchList = Invoke-EmarsysCore @params
+        $arr = [System.Collections.ArrayList]@()
+        $total = 0
+        $i = 0
+
+    }
 
-        # Rewrite result
-        If ( $ResolveFields -eq $true  ) {
+    process {
 
-            # Create a lookup hashtable for field names
-            $fieldHashtable = [hashtable]@{}
-            get-field | ForEach-Object { $fieldHashtable.add($_.id,$_.name) }
+        # Support for parameter input
+        foreach ($KeyValue in $KeyValues) {
 
-            $res = [System.Collections.ArrayList]@()
-            $fetchList.result | ForEach-Object {
-
-                $row = $_
-
-                $newRow = [PSCustomObject]@{
-                    "id" = $row.id      # always returned
-                    "uid" = $row.uid    # always returned
-                }
-
-                $row.PSObject.Properties | Where-Object { $_.MemberType -eq "NoteProperty" -and $_.Name -notin "id","uid" } | ForEach-Object {
-                    #$v = $_
-                    $newRow | Add-Member -MemberType NoteProperty -Name $fieldHashtable[[int]$_.Name]  -Value $_.Value
-                }
+            [void]$arr.Add($KeyValue)
+            $i += 1
+            $total += 1
 
-                [void]$res.Add($newRow)
-
-            }
+            Write-Verbose "Added at i $( $i ) and total $( $total ) of $( $KeyValues.Count )"
+
+            # Request list creation every n
+            If ( $i % 1000 -eq 0 ) {
+
+                Write-Verbose "Calling emarsys at i $( $i ) and total $( $total ) of $( $KeyValues.Count )"
+
+                # Get the data from emarsys
+                $params.Body.keyValues = $arr
+                $fetchList = Invoke-EmarsysCore @params
+
+                # Rewrite result
+                If ( $ResolveFields -eq $true  ) {
+
+                    $rewriteParams = [Hashtable]@{
+                        "FetchList" = $fetchList
+                        "IgnoreErrors" = $IgnoreErrors
+                    }
+                    Resolve-ContactData @rewriteParams
+
+                } else {
+
+                    If ( $IgnoreErrors -eq $true ) {
+                        $fetchList.result
+                    } else {
+                        $fetchList
+                    }
 
-            # return
-            If ( $IgnoreErrors -eq $true ) {
-                $res
-            } else {
-                [PSCustomObject]@{
-                    "errors" = $fetchList.errors
-                    "result" =  $res
                 }
+
+                # Empty the cached values
+                $arr.Clear()
+                $i = 0
+
             }
-
-        } else {
-            If ( $IgnoreErrors -eq $true ) {
-                $res.result
-            } else {
-                $fetchList
-            }
+
         }
 
+    }
 
+    end {
 
-
+        # Get a last call if there is something left
+        If ( $i -gt 0 ) {
 
-
+            Write-Verbose "Calling emarsys at i $( $i ) and total $( $total ) of $( $KeyValues.Count )"
 
-    }
+            # Get the data from emarsys
+            $params.Body.keyValues = $arr
+            $fetchList = Invoke-EmarsysCore @params
 
-    end {
+            # Rewrite result
+            If ( $ResolveFields -eq $true  ) {
+
+                $rewriteParams = [Hashtable]@{
+                    "FetchList" = $fetchList
+                    "IgnoreErrors" = $IgnoreErrors
+                }
+                Resolve-ContactData @rewriteParams
+
+            } else {
+
+                If ( $IgnoreErrors -eq $true ) {
+                    $fetchList.result
+                } else {
+                    $fetchList
+                }
+
+            }
+
+            # Empty the cached values
+            $arr.Clear()
+            $i = 0
+
+        }
 
     }
 

diff --git a/AptecoPSFramework/plugins/emarsys/Public/emarsys/Get-ListContact.ps1 b/AptecoPSFramework/plugins/emarsys/Public/emarsys/Get-ListContact.ps1
@@ -21,13 +21,18 @@ function Get-ListContact {
 
     begin {
 
-        $res =  [System.Collections.ArrayList]@()
+        # Count list
+        $listCount = Get-ListCount -ListId $ListId
+
+        #$res =  [System.Collections.ArrayList]@()
 
         if ($PSCmdlet.ParameterSetName -eq 'AllPages') {
             $SkipToken = 0
             $Top = 10000
         }
 
+        $i = 0
+
     }
 
     process {
@@ -53,7 +58,8 @@ function Get-ListContact {
 
             # Request list creation
             $fetchList = Invoke-EmarsysCore @params
-            $res.AddRange($fetchList.value)
+            $fetchList.value #return directly
+            #$res.AddRange($fetchList.value)
 
             # Setup next page
             If ( $fetchList.next -ne $null ) {
@@ -63,10 +69,14 @@ function Get-ListContact {
                 #$Top = $nextQueryParams['$top']
             }
 
+            $i += $fetchList.value.count
+            $percent = [math]::Floor(($i/$listCount)*100)
+            Write-Progress -Activity "Loading list contacts" -Status "$( $percent )% complete" -PercentComplete $percent    
+
         } While ( $All -eq $true -and $fetchList.next -ne $null )
 
         # return
-        $res
+        #$res
 
     }
 

diff --git a/AptecoPSFramework/plugins/emarsys/Public/emarsys/Invoke-EmarsysCore.ps1 b/AptecoPSFramework/plugins/emarsys/Public/emarsys/Invoke-EmarsysCore.ps1
@@ -201,7 +201,7 @@ function Invoke-EmarsysCore {
 
             # Prepare Body
             If ( $updatedParameters.ContainsKey("Body") -eq $true ) {
-                $bodyJson = ConvertTo-Json -InputObject $Body -Depth 99
+                $bodyJson = ConvertTo-Json -InputObject $Body -Depth 99 -Compress
                 $updatedParameters.Body = $bodyJson
             }
 
@@ -299,7 +299,7 @@ function Invoke-EmarsysCore {
         If ( $Paging -eq $true ) {
             $res
         } else {
-            If ( $wr.data ) {
+            If ( $wr.data -ne $null ) {
                 $wr.data
             } else {
                 $wr

diff --git a/AptecoPSFramework/plugins/emarsys/README.md b/AptecoPSFramework/plugins/emarsys/README.md
@@ -66,4 +66,14 @@ Get-ListCount -ListId 1932108413
 # Get contacts by email address, resolve fields, ignore errors (e.g. not existing email addresses)
 Get-ContactData @( "[email protected]","[email protected]" ) -Fields "first_name", "last_name","email" -ResolveFields -IgnoreErrors
 
+Get-ContactData -KeyValues "10596","10764","13919" -KeyId "id" -Fields "email" -IgnoreErrors
+
+# Or via pipeline input
+"10596", "10764","13919" | Get-ContactData -KeyId "id" -Fields "email" -IgnoreErrors
+
+# Or connect the commands together
+# This process is running with 1 thread. In my tests around 200k contacts need around 300 seconds to load
+$c = Get-ListContact -ListId 1801153297 -all | Get-ContactData -KeyId "id" -Fields "email", "first_name" -ResolveFields -IgnoreErrors
+$c | Select -First 1000 | Out-Gridview
+
 ```