feat(s3): Don't upload if file of same name and size already exists

src/nwp_consumer/internal/outputs/s3/client.py

@@ -74,6 +74,17 @@ def store(self, *, src: pathlib.Path, dst: pathlib.Path) -> pathlib.Path:
             src=src.as_posix(),
             dst=(self.__bucket / dst).as_posix(),
         )
+
+        # If file already exists in store and is of the same size, skip the upload
+        if self.exists(dst=dst) and self.__fs.du((self.__bucket / dst).as_posix()) == src.stat().st_size:
+            log.debug(
+                event="file of same size already exists in s3, skipping",
+                src=src.as_posix(),
+                dst=(self.__bucket / dst).as_posix(),
+            )
+            return dst
+
+        # Upload the file to the store
         self.__fs.put(lpath=src.as_posix(), rpath=(self.__bucket / dst).as_posix(), recursive=True)
         # Don't delete cached file as user may want to do further processing locally.
         nbytes = self.__fs.du((self.__bucket / dst).as_posix())

src/nwp_consumer/internal/outputs/s3/test_client.py

@@ -122,6 +122,31 @@ def test_store(self) -> None:
         self.testS3.delete_object(Bucket=BUCKET, Key=dst.as_posix())
         src.unlink(missing_ok=True)
 
+        ## Test the store doesn't overwrite an existing file of equivalent size
+
+        # Create a mock file in the store
+        self.testS3.put_object(
+            Bucket=BUCKET,
+            Key=dst.as_posix(),
+            Body=bytes(fileName, "utf-8"),
+        )
+
+        # Create a temporary file with the same data
+        src.write_bytes(bytes(fileName, "utf-8"))
+
+        # Get the modified date of the file in the store
+        response = self.testS3.head_object(Bucket=BUCKET, Key=dst.as_posix())
+        lastModified = response["LastModified"]
+
+        # Call the store method on the file
+        name = self.client.store(src=src, dst=dst)
+
+        # Verify the file in the store was not overwritten
+        response = self.testS3.get_object(Bucket=BUCKET, Key=dst.as_posix())
+        self.assertEqual(response["Body"].read(), bytes(fileName, "utf-8"))
+        self.assertEqual(lastModified, response["LastModified"])
+
+
     def test_listInitTimes(self) -> None:
         # Create mock folders/files in the raw directory
         self.testS3.put_object(