From b1f8aba72dc959d932b36a5abb102a68e1398f0e Mon Sep 17 00:00:00 2001 From: Terry Case Date: Tue, 12 Mar 2024 14:02:49 -0700 Subject: [PATCH] Allow users of IMPAssembler to supply pre-computed SHA-1 hashes for track entries to facilitate map reduce use cases. --- .../imflibrary/writerTools/IMPAssembler.java | 48 +++++++++++++++++-- .../writerTools/IMPAssemblerTest.java | 6 ++- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/netflix/imflibrary/writerTools/IMPAssembler.java b/src/main/java/com/netflix/imflibrary/writerTools/IMPAssembler.java index fdb019a5..9aea926c 100644 --- a/src/main/java/com/netflix/imflibrary/writerTools/IMPAssembler.java +++ b/src/main/java/com/netflix/imflibrary/writerTools/IMPAssembler.java @@ -56,6 +56,7 @@ public AssembledIMPResult assembleIMFFromFiles(SimpleTimeline simpleTimeline, Fi Map trackFileIdToResourceMap = new HashMap<>(); Map> sampleRateMap = new HashMap<>(); Map sampleCountMap = new HashMap<>(); + Map hashMap = new HashMap<>(); for (Track track : simpleTimeline.getTracks()) { @@ -70,7 +71,22 @@ public AssembledIMPResult assembleIMFFromFiles(SimpleTimeline simpleTimeline, Fi throw new IOException("Could not get header partition for file: " + trackEntry.getFile().getAbsolutePath()); } byte[] headerPartitionBytes = headerPartitionPayloadRecord.getPayload(); - byte[] hash = IMFUtils.generateSHA1Hash(resourceByteRangeProvider); + + + // get sha-1 hash or use cached value + byte[] hash = null; + if (trackEntry.getHash() != null) { + logger.info("Using hash from user: {}", trackEntry.getHash()); + hash = trackEntry.getHash(); + hashMap.put(IMPFixer.getTrackFileId(headerPartitionPayloadRecord), trackEntry.getHash()); + } else if (hashMap.containsKey(IMPFixer.getTrackFileId(headerPartitionPayloadRecord))) { + logger.info("Using cached hash: {}", hashMap.get(IMPFixer.getTrackFileId(headerPartitionPayloadRecord))); + hash = hashMap.get(IMPFixer.getTrackFileId(headerPartitionPayloadRecord)); + } else { + logger.info("Generating hash for file: {}", trackEntry.getFile().getAbsolutePath()); + hash = IMFUtils.generateSHA1Hash(resourceByteRangeProvider); + hashMap.put(IMPFixer.getTrackFileId(headerPartitionPayloadRecord), hash); + } UUID trackFileId = IMPFixer.getTrackFileId(headerPartitionPayloadRecord); logger.info("UUID read from file: {}: {}", trackEntry.getFile().getName(), trackFileId.toString()); @@ -78,7 +94,7 @@ public AssembledIMPResult assembleIMFFromFiles(SimpleTimeline simpleTimeline, Fi imfTrackFileMetadataMap.put( trackFileId, new IMPBuilder.IMFTrackFileMetadata(headerPartitionBytes, - hash, + hash, // a byte[] containing the SHA-1, Base64 encoded hash of the IMFTrack file CompositionPlaylistBuilder_2016.defaultHashAlgorithm, trackEntry.getFile().getName(), resourceByteRangeProvider.getResourceSize()) @@ -387,14 +403,30 @@ public static class TrackEntry { * @param entryPoint - the entry point, if null, defaults to 0 * @param duration - the duration, if null, defaults to intrinsic duration * @param repeatCount - the repeat count, if null, defaults to 1 + * @param hash - the SHA-1 hash of the file, optional, introspected if null */ - public TrackEntry(@Nonnull File file, @Nullable Composition.EditRate sampleRate, @Nullable BigInteger intrinsicDuration, @Nullable BigInteger entryPoint, @Nullable BigInteger duration, @Nullable BigInteger repeatCount) { + public TrackEntry(@Nonnull File file, @Nullable Composition.EditRate sampleRate, @Nullable BigInteger intrinsicDuration, @Nullable BigInteger entryPoint, @Nullable BigInteger duration, @Nullable BigInteger repeatCount, @Nullable byte[] hash) { this.file = file; this.sampleRate = sampleRate; this.intrinsicDuration = intrinsicDuration; this.entryPoint = entryPoint; this.duration = duration; this.repeatCount = repeatCount; + this.hash = hash; + } + + public TrackEntry(@Nonnull File file, @Nullable Composition.EditRate sampleRate, @Nullable BigInteger intrinsicDuration, @Nullable BigInteger entryPoint, @Nullable BigInteger duration, @Nullable BigInteger repeatCount) { + this(file, sampleRate, intrinsicDuration, entryPoint, duration, repeatCount, null); + } + + public TrackEntry() { + this.file = null; + this.sampleRate = null; + this.intrinsicDuration = null; + this.entryPoint = null; + this.duration = null; + this.repeatCount = null; + this.hash = null; } public File getFile() { @@ -451,5 +483,15 @@ public void setRepeatCount(BigInteger repeatCount) { public BigInteger entryPoint; public BigInteger duration; public BigInteger repeatCount; + + public byte[] getHash() { + return hash; + } + + public void setHash(byte[] hash) { + this.hash = hash; + } + + public byte[] hash; } } diff --git a/src/test/java/com/netflix/imflibrary/writerTools/IMPAssemblerTest.java b/src/test/java/com/netflix/imflibrary/writerTools/IMPAssemblerTest.java index 1302e009..1cc94a20 100644 --- a/src/test/java/com/netflix/imflibrary/writerTools/IMPAssemblerTest.java +++ b/src/test/java/com/netflix/imflibrary/writerTools/IMPAssemblerTest.java @@ -40,7 +40,8 @@ private Object[][] trackEntries() { BigInteger.valueOf(10), BigInteger.valueOf(0), BigInteger.valueOf(10), - BigInteger.valueOf(1) + BigInteger.valueOf(1), + java.util.Base64.getDecoder().decode("fL7SnTeNskm71I4otXqr/T0D5LQ=") ), new IMPAssembler.TrackEntry( TestHelper.findResourceByPath("TestIMP/MERIDIAN_Netflix_Photon_161006/MERIDIAN_Netflix_Photon_161006_ENG-51_00.mxf"), @@ -48,7 +49,8 @@ private Object[][] trackEntries() { BigInteger.valueOf(8008), BigInteger.valueOf(0), BigInteger.valueOf(8008), - BigInteger.valueOf(1) + BigInteger.valueOf(1), + java.util.Base64.getDecoder().decode("X6GxGHTavnlIRLZiD7hHe5/CUh4=") ) },