Add job document comparator to ignore pre-signed difference in s3 url

source/jobs/JobsFeature.cpp

@@ -527,9 +527,87 @@ void JobsFeature::copyJobsNotification(Iotjobs::JobExecutionData job)
     latestJobsNotification.ExecutionNumber = job.ExecutionNumber.value();
 }
 
+
+bool JobsFeature::compareJobDocuments(const Aws::Crt::JsonObject& job1, const Aws::Crt::JsonObject& job2) {
+    std::string str1 = job1.View().WriteCompact().c_str();
+    std::string str2 = job2.View().WriteCompact().c_str();
+
+    // Regular expression to match S3 URLs and capture the non-presigned parts
+    std::regex s3UrlRegex(R"((https://[^.\s"]+\.s3[.-](?:[^.\s"]+\.)?amazonaws\.com/[^?\s"]+)(\?[^"\s]+)?)");
+
+    // Function to replace only the pre-signed portion of S3 URLs
+    auto processPresignedUrls = [&s3UrlRegex](std::string& s) -> int {
+        int count = 0;
+        std::string result;
+        std::sregex_iterator it(s.begin(), s.end(), s3UrlRegex);
+        std::sregex_iterator end;
+
+        size_t lastPos = 0;
+        for (; it != end; ++it) {
+            count++;
+            result += s.substr(lastPos, it->position() - lastPos);
+            result += it->str(1);
+            // Add the non-presigned part of the URL
+            result += "?aws:iot:s3-presigned-suffix:PLACEHOLDER"; 
+            // Replace only the presigned portion
+            lastPos = it->position() + it->length();
+        }
+        result += s.substr(lastPos);
+
+        s = result;
+        return count;
+    };
+
+    int count1 = processPresignedUrls(str1);
+    int count2 = processPresignedUrls(str2);
+
+    // If the number of pre-signed URLs differs, the documents are different
+    if (count1 != count2) {
+        return false;
+    }
+
+    std::cout << "Processed str1: " << str1 << std::endl;
+    std::cout << "Processed str2: " << str2 << std::endl;
+
+    return str1 == str2;
+}
 bool JobsFeature::isDuplicateNotification(JobExecutionData job)
 {
     unique_lock<mutex> readLatestNotificationLock(latestJobsNotificationLock);
+
+    // Basic logging at the start of the function
+    std::cout << "DEBUG: Entering isDuplicateNotification function" << std::endl;
+
+
+
+    // Log Job IDs being compared
+    std::cout << "INFO: Comparing Job IDs - New: " << (job.JobId.has_value() ? job.JobId.value().c_str() : "NULL")
+              << ", Latest: " << (latestJobsNotification.JobId.has_value() ? latestJobsNotification.JobId.value().c_str() : "NULL") << std::endl;
+
+
+
+
+
+    // Log Job Documents being compared
+    std::cout << "INFO: Comparing Job Documents - New: " 
+              << (job.JobDocument.has_value() ? job.JobDocument.value().View().WriteCompact().c_str() : "NULL")
+              << ", Latest: " 
+              << (latestJobsNotification.JobDocument.has_value() ? latestJobsNotification.JobDocument.value().View().WriteCompact().c_str() : "NULL") 
+              << std::endl;
+
+
+
+
+
+        // Log Execution Numbers being compared
+    std::cout << "INFO: Comparing Execution Numbers - New: " 
+              << (job.ExecutionNumber.has_value() ? std::to_string(job.ExecutionNumber.value()) : "NULL")
+              << ", Latest: " 
+              << (latestJobsNotification.ExecutionNumber.has_value() ? std::to_string(latestJobsNotification.ExecutionNumber.value()) : "NULL") 
+              << std::endl;
+
+
+
     if (!latestJobsNotification.JobId.has_value())
     {
         // We have not seen a job yet
@@ -543,14 +621,19 @@ bool JobsFeature::isDuplicateNotification(JobExecutionData job)
         return false;
     }
 
-    if (strcmp(
-            job.JobDocument.value().View().WriteCompact().c_str(),
-            latestJobsNotification.JobDocument.value().View().WriteCompact().c_str()) != 0)
-    {
+    if (!compareJobDocuments(
+            job.JobDocument.value(),
+            latestJobsNotification.JobDocument.value())) {
         LOG_DEBUG(TAG, "Job document differs");
         return false;
     }
 
+        std::cout << "INFO: Comparing Job Documents after replacement - New: " 
+              << (job.JobDocument.has_value() ? job.JobDocument.value().View().WriteCompact().c_str() : "NULL")
+              << ", Latest: " 
+              << (latestJobsNotification.JobDocument.has_value() ? latestJobsNotification.JobDocument.value().View().WriteCompact().c_str() : "NULL") 
+              << std::endl;
+
     if (job.ExecutionNumber.value() != latestJobsNotification.ExecutionNumber.value())
     {
         LOG_DEBUG(TAG, "Execution number differs");

source/jobs/JobsFeature.h

@@ -15,6 +15,7 @@
 #include "IotJobsClientWrapper.h"
 #include "JobDocument.h"
 #include "JobEngine.h"
+#include <regex>
 
 namespace Aws
 {
@@ -77,6 +78,9 @@ namespace Aws
                     virtual int stop() override;
 
                   protected:
+
+                    bool compareJobDocuments(const Aws::Crt::JsonObject& job1, const Aws::Crt::JsonObject& job2);
+
                     /**
                      * \brief Begins running the Jobs feature
                      */
@@ -313,6 +317,17 @@ namespace Aws
                      */
                     bool isDuplicateNotification(Iotjobs::JobExecutionData job);
 
+                    /**
+                     * \brief Compares two job documents, ignoring differences in pre-signed URLs.
+                     *
+                     * @param job1 The first job document as a JsonObject
+                     * @param job2 The second job document as a JsonObject
+                     * @return true if the documents are equivalent (ignoring pre-signed URLs), false otherwise
+                     */
+
+                    // Add this line to declare the test class as a friend
+                    friend class TestJobsFeaturePrivate;
+
                     /**
                      * \brief Stores information about a job notification
                      *

test/jobs/TestJobsFeature.cpp

@@ -15,6 +15,7 @@
 #include <aws/iotjobs/RejectedError.h>
 #include <aws/iotjobs/StartNextJobExecutionResponse.h>
 #include <aws/iotjobs/UpdateJobExecutionResponse.h>
+#include <aws/crt/Api.h>
 
 using namespace std;
 using namespace testing;
@@ -280,6 +281,25 @@ class TestJobsFeature : public ::testing::Test
     shared_ptr<MockJobEngine> mockEngine;
 };
 
+class TestJobsFeaturePrivate : public ::testing::Test, public JobsFeature
+{
+protected:
+    Aws::Crt::ApiHandle apiHandle;
+    std::unique_ptr<JobsFeature> jobsFeature;
+
+    void SetUp() override
+    {
+        // The ApiHandle constructor initializes the AWS CRT library
+        jobsFeature.reset(new JobsFeature());
+    }
+
+    void TearDown() override
+    {
+        jobsFeature.reset();
+        // The ApiHandle destructor cleans up the AWS CRT library
+    }
+};
+
 MATCHER_P(ThingNameEq, ThingName, "Matcher ThingName for all Aws request Objects using Aws::Crt::String")
 {
     return arg.ThingName.value() == ThingName;
@@ -710,3 +730,39 @@ TEST_F(TestJobsFeature, InvalidJobDocument)
     jobsMock->init(std::shared_ptr<Mqtt::MqttConnection>(), notifier, config);
     jobsMock->invokeRunJobs();
 }
+
+    TEST_F(TestJobsFeaturePrivate, CompareJobDocuments)
+{
+    // Test case 1: Identical documents
+    Aws::Crt::JsonObject doc1("{\"key\": \"value\"}");
+    Aws::Crt::JsonObject doc2("{\"key\": \"value\"}");
+    EXPECT_TRUE(compareJobDocuments(doc1, doc2));
+
+    // Test case 2: Different documents
+    Aws::Crt::JsonObject doc3("{\"key\": \"different_value\"}");
+    EXPECT_FALSE(compareJobDocuments(doc1, doc3));
+
+    // Test case 3: Documents with pre-signed URLs
+    Aws::Crt::JsonObject doc4("{\"url\": \"https://bucket.s3.amazonaws.com/file?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=1234567890&Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX\"}");
+    Aws::Crt::JsonObject doc5("{\"url\": \"https://bucket.s3.amazonaws.com/file?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=9876543210&Signature=YYYYYYYYYYYYYYYYYYYYYYYYYYY\"}");
+    EXPECT_TRUE(compareJobDocuments(doc4, doc5));
+
+    // Test case 4: Documents with multiple pre-signed URLs
+    Aws::Crt::JsonObject doc6("{\"url1\": \"https://bucket1.s3.amazonaws.com/file1?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=1234567890&Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX\", \"url2\": \"https://bucket2.s3.amazonaws.com/file2?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=1234567890&Signature=ZZZZZZZZZZZZZZZZZZZZZZZZZZZ\"}");
+    Aws::Crt::JsonObject doc7("{\"url1\": \"https://bucket1.s3.amazonaws.com/file1?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=9876543210&Signature=YYYYYYYYYYYYYYYYYYYYYYYYYYY\", \"url2\": \"https://bucket2.s3.amazonaws.com/file2?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=9876543210&Signature=WWWWWWWWWWWWWWWWWWWWWWWWWWW\"}");
+    EXPECT_TRUE(compareJobDocuments(doc6, doc7));
+
+    // Test case 6: Documents with multiple pre-signed URLs in one doc and single url in the other
+    Aws::Crt::JsonObject doc8("{\"url1\": \"https://bucket1.s3.amazonaws.com/file1?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=9876543210&Signature=YYYYYYYYYYYYYYYYYYYYYYYYYYY\"}");
+    EXPECT_FALSE(compareJobDocuments(doc6, doc8));
+
+    // Test case 7: Documents with different structure
+    Aws::Crt::JsonObject doc9("{\"key1\": \"value1\", \"key2\": \"value2\"}");
+    Aws::Crt::JsonObject doc10("{\"key1\": \"value1\", \"key3\": \"value3\"}");
+    EXPECT_FALSE(compareJobDocuments(doc9, doc10));
+
+    // Test case 8: Documents with same pre-signed URLs but different buckets
+    Aws::Crt::JsonObject doc11("{\"url\": \"https://bucket1.s3.amazonaws.com/file?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=1234567890&Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX\"}");
+    Aws::Crt::JsonObject doc12("{\"url\": \"https://bucket2.s3.amazonaws.com/file?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=1234567890&Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX\"}");
+    EXPECT_FALSE(compareJobDocuments(doc11, doc12));
+}