diff --git a/chunky/src/java/se/llbit/chunky/renderer/export/Tiff32ExportFormat.java b/chunky/src/java/se/llbit/chunky/renderer/export/Tiff32ExportFormat.java index 1cd18b9159..19d2606835 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/export/Tiff32ExportFormat.java +++ b/chunky/src/java/se/llbit/chunky/renderer/export/Tiff32ExportFormat.java @@ -17,9 +17,18 @@ */ package se.llbit.chunky.renderer.export; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + import se.llbit.chunky.renderer.scene.Scene; +import se.llbit.imageformats.tiff.CompressionType; import se.llbit.imageformats.tiff.TiffFileWriter; import se.llbit.util.TaskTracker; @@ -50,9 +59,37 @@ public boolean isTransparencySupported() { @Override public void write(OutputStream out, Scene scene, TaskTracker taskTracker) throws IOException { - try (TaskTracker.Task task = taskTracker.task("Writing TIFF"); - TiffFileWriter writer = new TiffFileWriter(out)) { - writer.export(scene, task); + try (TaskTracker.Task task = taskTracker.task("Writing TIFF")) { + if (out instanceof FileOutputStream) { + write(((FileOutputStream) out).getChannel(), scene, task); + } else { + // fallback for the case, that the output stream was not created on a file + Path tempFile = Files.createTempFile(scene.name + "-", getExtension()); + try (FileChannel fileChannel = FileChannel.open(tempFile, StandardOpenOption.DELETE_ON_CLOSE, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.READ, StandardOpenOption.WRITE + )) { + write(fileChannel, scene, task); + // rewind channel + fileChannel.position(0); + try (InputStream inputStream = Channels.newInputStream(fileChannel)) { + // copy temp file to output + inputStream.transferTo(out); + } + } + } } } + + /** + * Note: does not (!) close the file channel after writing + */ + private void write(FileChannel fileChannel, Scene scene, TaskTracker.Task task) throws IOException { + TiffFileWriter writer = new TiffFileWriter( + fileChannel, + CompressionType.DEFLATE + ); + writer.export(scene, task); + writer.doFinalization(); + } } diff --git a/chunky/src/java/se/llbit/chunky/renderer/renderdump/ClassicDumpFormat.java b/chunky/src/java/se/llbit/chunky/renderer/renderdump/ClassicDumpFormat.java index 0aba0ebf34..daa9aa62ae 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/renderdump/ClassicDumpFormat.java +++ b/chunky/src/java/se/llbit/chunky/renderer/renderdump/ClassicDumpFormat.java @@ -17,7 +17,7 @@ package se.llbit.chunky.renderer.renderdump; import se.llbit.chunky.renderer.scene.Scene; -import se.llbit.util.IsolatedOutputStream; +import se.llbit.util.io.IsolatedOutputStream; import se.llbit.util.TaskTracker; import java.io.DataInputStream; diff --git a/chunky/src/java/se/llbit/chunky/renderer/renderdump/GzipDumpFormat.java b/chunky/src/java/se/llbit/chunky/renderer/renderdump/GzipDumpFormat.java index b8914101fd..f79fe4723e 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/renderdump/GzipDumpFormat.java +++ b/chunky/src/java/se/llbit/chunky/renderer/renderdump/GzipDumpFormat.java @@ -1,7 +1,7 @@ package se.llbit.chunky.renderer.renderdump; import se.llbit.chunky.renderer.scene.Scene; -import se.llbit.util.IsolatedOutputStream; +import se.llbit.util.io.IsolatedOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; diff --git a/chunky/src/java/se/llbit/chunky/renderer/renderdump/HuffmanDumpFormat.java b/chunky/src/java/se/llbit/chunky/renderer/renderdump/HuffmanDumpFormat.java index 35f2bdef92..3b5fcc5a60 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/renderdump/HuffmanDumpFormat.java +++ b/chunky/src/java/se/llbit/chunky/renderer/renderdump/HuffmanDumpFormat.java @@ -1,7 +1,7 @@ package se.llbit.chunky.renderer.renderdump; import se.llbit.chunky.renderer.scene.Scene; -import se.llbit.util.IsolatedOutputStream; +import se.llbit.util.io.IsolatedOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/Scene.java b/chunky/src/java/se/llbit/chunky/renderer/scene/Scene.java index fd1c4536d7..8ebe411106 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/Scene.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/Scene.java @@ -58,6 +58,8 @@ import se.llbit.nbt.Tag; import se.llbit.util.*; import se.llbit.util.annotation.NotNull; +import se.llbit.util.io.PositionalInputStream; +import se.llbit.util.io.ZipExport; import se.llbit.util.mojangapi.MinecraftProfile; import java.io.*; diff --git a/chunky/src/java/se/llbit/imageformats/tiff/BasicIFD.java b/chunky/src/java/se/llbit/imageformats/tiff/BasicIFD.java new file mode 100644 index 0000000000..5f60cedf1d --- /dev/null +++ b/chunky/src/java/se/llbit/imageformats/tiff/BasicIFD.java @@ -0,0 +1,50 @@ +package se.llbit.imageformats.tiff; + +import java.io.IOException; + +public class BasicIFD extends ImageFileDirectory { + final CompressionType compressionType; + + public BasicIFD( + int width, int height, + CompressionType compressionType + ) throws IOException { + this.compressionType = compressionType; + + // RGB full color + addTag(IFDTag.TAG_PHOTOMETRIC_INTERPRETATION, (short) 2); + // Store pixel components contiguously [RGBRGBRGB...] + addTag(IFDTag.TAG_PLANAR_CONFIGURATION, (short) 1); + + assert (width <= Short.MAX_VALUE); + addTag(IFDTag.TAG_IMAGE_WIDTH, (short) width); + assert (height <= Short.MAX_VALUE); + addTag(IFDTag.TAG_IMAGE_HEIGHT, (short) height); + // The 0th row represents the visual top of the image, and the 0th column represents the visual left-hand side. + addTag(IFDTag.TAG_ORIENTATION, (short) 1); + + // No compression, but pack data into bytes as tightly as possible, leaving no unused + // bits (except at the end of a row). The component values are stored as an array of + // type BYTE. Each scan line (row) is padded to the next BYTE boundary. + addTag(IFDTag.TAG_COMPRESSION_TYPE, compressionType.id); + + // Image does not have a physical size + addTag(IFDTag.TAG_RESOLUTION_UNIT, (short) 1); // not an absolute unit + addMultiTag(IFDTag.TAG_X_RESOLUTION, new int[]{1, 1}); + addMultiTag(IFDTag.TAG_Y_RESOLUTION, new int[]{1, 1}); + + // "Compressed or uncompressed image data can be stored almost anywhere in a + // TIFF file. TIFF also supports breaking an image into separate strips for increased + // editing flexibility and efficient I/O buffering." + // We will use exactly 1 strip, therefore the relevant tags have only 1 entry with all rows in 1 strip. + addTag(IFDTag.TAG_ROWS_PER_STRIP, height); + } + + @Override + void writePixelData( + FinalizableBFCOutputStream out, + PixelDataWriter writer + ) throws IOException { + compressionType.writePixelData(out, writer); + } +} diff --git a/chunky/src/java/se/llbit/imageformats/tiff/CompressionType.java b/chunky/src/java/se/llbit/imageformats/tiff/CompressionType.java new file mode 100644 index 0000000000..7fc0cebf27 --- /dev/null +++ b/chunky/src/java/se/llbit/imageformats/tiff/CompressionType.java @@ -0,0 +1,37 @@ +package se.llbit.imageformats.tiff; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; + +public enum CompressionType { + NONE(0x0001), + DEFLATE(0x80B2); + + final short id; + + CompressionType(int id) { + this.id = (short) id; + } + + void writePixelData( + FinalizableBFCOutputStream out, + ImageFileDirectory.PixelDataWriter writer + ) throws IOException { + switch (this) { + case NONE: + writer.writePixelData(out); + out.flush(); + break; + + case DEFLATE: + Deflater deflater = new Deflater(Deflater.BEST_COMPRESSION, false); + DeflaterOutputStream deflOut = new DeflaterOutputStream(out, deflater, 16 * 1024, true); + writer.writePixelData(new DataOutputStream(deflOut)); + deflOut.finish(); + deflater.end(); + break; + } + } +} diff --git a/chunky/src/java/se/llbit/imageformats/tiff/FinalizableBFCOutputStream.java b/chunky/src/java/se/llbit/imageformats/tiff/FinalizableBFCOutputStream.java new file mode 100644 index 0000000000..8a3a4c2211 --- /dev/null +++ b/chunky/src/java/se/llbit/imageformats/tiff/FinalizableBFCOutputStream.java @@ -0,0 +1,87 @@ +package se.llbit.imageformats.tiff; + +import se.llbit.util.io.BufferedFileChannelOutputStream; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.util.ArrayDeque; +import java.util.Deque; + +class FinalizableBFCOutputStream extends BufferedFileChannelOutputStream { + + private static final Deque> finalizationQueue = new ArrayDeque<>(); + + public FinalizableBFCOutputStream(FileChannel fileChannel) { + super(fileChannel); + } + + void ensureAlignment() throws IOException { + if ((position() & 0b1) != 0) + write((byte) 0); + } + + UnfinalizedData.Int writeUnfinalizedInt() throws IOException { + return writeUnfinalized(new UnfinalizedData.Int((int) position()), 4); + } + private > T writeUnfinalized(T ud, int byteCount) throws IOException { + finalizationQueue.add(ud); + skip(byteCount); + return ud; + } + + /** + * writes remaining unfinalized data + */ + public void doFinalization() throws IOException { + for(UnfinalizedData data : finalizationQueue) { + data.write(this); + } + finalizationQueue.clear(); + } + + /** + * does finalization, then closes the output stream + */ + @Override + public void close() throws IOException { + doFinalization(); + super.close(); + } + + static abstract class UnfinalizedData { + final long position; + protected T data; + + UnfinalizedData(long position) { + this.position = position; + } + + public void setData(T data) { + this.data = data; + } + + public T getData() { + return data; + } + + public void write(FinalizableBFCOutputStream out) throws IOException { + out.position(position); + if(data != null) { + writeData(out); + } + } + + abstract void writeData(FinalizableBFCOutputStream out) throws IOException; + + static class Int extends UnfinalizedData { + Int(long position) { + super(position); + } + + @Override + void writeData(FinalizableBFCOutputStream out) throws IOException { + out.writeInt(data); + } + } + } +} \ No newline at end of file diff --git a/chunky/src/java/se/llbit/imageformats/tiff/IFDTag.java b/chunky/src/java/se/llbit/imageformats/tiff/IFDTag.java new file mode 100644 index 0000000000..6c4ddac2a5 --- /dev/null +++ b/chunky/src/java/se/llbit/imageformats/tiff/IFDTag.java @@ -0,0 +1,188 @@ +package se.llbit.imageformats.tiff; + +import java.io.DataOutput; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +public abstract class IFDTag { + final TagFieldType type; + final short tagId; + + enum TagFieldType { + ASCII(2, 1), + SHORT(3, 2), + LONG(4, 4), + RATIONAL(5, 8); + final short id; + final int byteSize; + + TagFieldType(int id, int byteSize) { + this.id = (short) id; + this.byteSize = byteSize; + } + } + + IFDTag(TagFieldType type, short tagId) { + this.type = type; + this.tagId = tagId; + } + + abstract int write(DataOutput out, SingleType data) throws IOException; + + int writeMultiple(DataOutput out, ArrayType data) throws IOException { + throw new UnsupportedOperationException("cannot store multiple data entries"); + } + + abstract int valueCount(SingleType st, ArrayType at); + + /** + * width / columns / pixels per scanline + */ + static final ShortTag TAG_IMAGE_WIDTH = new ShortTag(0x0100); + /** + * height / rows / length / scanline count + */ + static final ShortTag TAG_IMAGE_HEIGHT = new ShortTag(0x0101); + static final ShortTag TAG_BITS_PER_SAMPLE = new ShortTag(0x0102); + static final ShortTag TAG_SAMPLE_FORMAT = new ShortTag(0x0153); + + /** + * defines details of subfile using 32 flag bits + */ + static final LongTag TAG_NEW_SUBFILE_TYPE = new LongTag(0x00FE); + + static final ShortTag TAG_COMPRESSION_TYPE = new ShortTag(0x0103); + static final ShortTag TAG_PHOTOMETRIC_INTERPRETATION = new ShortTag(0x0106); + static final ShortTag TAG_PLANAR_CONFIGURATION = new ShortTag(0x011C); + + /** + * number of rows in each strip (except possibly the last strip) + */ + static final LongTag TAG_ROWS_PER_STRIP = new LongTag(0x0116); + /** + * for each strip, the byte offset of that strip + */ + static final LongTag TAG_STRIP_OFFSETS = new LongTag(0x0111); + /** + * for each strip, the number of bytes in that strip after any compression + */ + static final LongTag TAG_STRIP_BYTE_COUNTS = new LongTag(0x0117); + static final ShortTag TAG_ORIENTATION = new ShortTag(0x0112); + static final ShortTag TAG_SAMPLES_PER_PIXEL = new ShortTag(0x0115); + + static final RationalTag TAG_X_RESOLUTION = new RationalTag(0x011A); + static final RationalTag TAG_Y_RESOLUTION = new RationalTag(0x011B); + static final ShortTag TAG_RESOLUTION_UNIT = new ShortTag(0x0128); + + static final ASCIITag TAG_SOFTWARE = new ASCIITag(0x0131); + static final ASCIITag TAG_DATETIME = new ASCIITag(0x0132); + + /** + * 7-bit ASCII code, 0-terminated + */ + static class ASCIITag extends IFDTag { + ASCIITag(int tagID) { + super(TagFieldType.ASCII, (short) tagID); + } + + @Override + int write(DataOutput out, String data) throws IOException { + byte[] strBuf = data.getBytes(StandardCharsets.US_ASCII); + out.write(strBuf); + out.writeByte(0); + return strBuf.length + 1; + } + + @Override + int valueCount(String st, Void at) { + return st.getBytes(StandardCharsets.US_ASCII).length + 1; + } + } + + /** + * 16-bit unsigned(!) integer + */ + static class ShortTag extends IFDTag { + ShortTag(int tagID) { + super(TagFieldType.SHORT, (short) tagID); + } + + @Override + int write(DataOutput out, Short data) throws IOException { + out.writeShort(data); + return 2; + } + + @Override + int writeMultiple(DataOutput out, short[] data) throws IOException { + for (short s : data) { + out.writeShort(s); + } + return data.length * 2; + } + + @Override + int valueCount(Short st, short[] at) { + return at != null ? at.length : 1; + } + } + + /** + * 32-bit unsigned(!) integer + */ + static class LongTag extends IFDTag { + LongTag(int tagID) { + super(TagFieldType.LONG, (short) tagID); + } + + @Override + int write(DataOutput out, Integer data) throws IOException { + out.writeInt(data); + return 4; + } + + @Override + int writeMultiple(DataOutput out, int[] data) throws IOException { + for (int i : data) { + out.writeInt(i); + } + return data.length * 4; + } + + @Override + int valueCount(Integer st, int[] at) { + return at != null ? at.length : 1; + } + } + + /** + * fraction using: + * - 32-bit unsigned(!) integer numerator + * - 32-bit unsigned(!) integer denominator + */ + static class RationalTag extends IFDTag { + RationalTag(int tagID) { + super(TagFieldType.RATIONAL, (short) tagID); + } + + @Override + int write(DataOutput out, Void data) { + throw new UnsupportedOperationException("fraction requires numerator denominator pairs"); + } + + @Override + int writeMultiple(DataOutput out, int[] numeratorDenominatorPairs) throws IOException { + for (int nd : numeratorDenominatorPairs) { + out.writeInt(nd); + } + return numeratorDenominatorPairs.length * 4; + } + + @Override + int valueCount(Void st, int[] numeratorDenominatorPairs) { + if (numeratorDenominatorPairs.length % 2 != 0) + throw new IllegalArgumentException("fraction requires pairs of numerators and denominators"); + return numeratorDenominatorPairs.length; + } + } +} diff --git a/chunky/src/java/se/llbit/imageformats/tiff/ImageFileDirectory.java b/chunky/src/java/se/llbit/imageformats/tiff/ImageFileDirectory.java new file mode 100644 index 0000000000..dc4cab2904 --- /dev/null +++ b/chunky/src/java/se/llbit/imageformats/tiff/ImageFileDirectory.java @@ -0,0 +1,156 @@ +package se.llbit.imageformats.tiff; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +abstract class ImageFileDirectory { + + private final List> tagEntries = new ArrayList<>(20); + private final ByteArrayOutputStream tagDataBuffer = new ByteArrayOutputStream(72); + private final DataOutput tagDataBufferOutput = new DataOutputStream(tagDataBuffer); + + void addTag(IFDTag tag, SingleType data) { + tagEntries.add(new TagEntryData<>(tag, data)); + } + void addMultiTag(IFDTag tag, ArrayType data) { + tagEntries.add(new TagEntryData<>(tag, data, null)); + } + private class TagEntryData implements Comparable> { + final IFDTag tag; + final SingleType st; + final ArrayType at; + + @Override + public int compareTo(TagEntryData o) { + return Short.compareUnsigned(tag.tagId, o.tag.tagId); + } + TagEntryData(IFDTag tag, SingleType st) { + this.tag = tag; + this.st = st; + this.at = null; + } + TagEntryData(IFDTag tag, ArrayType at, Void unused) { + this.tag = tag; + this.st = null; + this.at = at; + } + + int writeHeader( + FinalizableBFCOutputStream out + ) throws IOException { + out.writeShort(tag.tagId); + out.writeShort(tag.type.id); + int valueCount = tag.valueCount(st, at); + out.writeInt(valueCount); + return valueCount; + } + + void writeData( + FinalizableBFCOutputStream out, + int valueCount, + int bufferAddress + ) throws IOException { + if (valueCount * tag.type.byteSize <= 4) { + // store in tag + int byteCount = writeTagData(out); + // pad tag + out.skip(4 - byteCount); + } else { + // store in buffer + int bufferDataAddress = bufferAddress + tagDataBuffer.size(); + out.writeInt(bufferDataAddress); + int byteCount = writeTagData(tagDataBufferOutput); + // pad address + if (byteCount % 2 != 0) + tagDataBufferOutput.writeByte(0); + } + } + + private int writeTagData(DataOutput out) throws IOException { + if(at == null) { + return tag.write(out, st); + } else { + return tag.writeMultiple(out, at); + } + } + } + + /** + * IFD structure: + * - 2 bytes: tag count + * - x * 12 bytes: tags + * - 4 bytes: next IFD address + */ + FinalizableBFCOutputStream.UnfinalizedData.Int write( + FinalizableBFCOutputStream out, + FinalizableBFCOutputStream.UnfinalizedData.Int ifdPointer, + PixelDataWriter writer + ) throws IOException { + out.ensureAlignment(); + // update current pointer to location + ifdPointer.data = (int) out.position(); + + // Absolute strip address + addTag(IFDTag.TAG_STRIP_OFFSETS, 0); + FinalizableBFCOutputStream.UnfinalizedData.Int pixelDataPointer = null; + // Strip length + addTag(IFDTag.TAG_STRIP_BYTE_COUNTS, 0); + FinalizableBFCOutputStream.UnfinalizedData.Int pixelDataByteCount = null; + + // write tag count + int tagCount = tagEntries.size(); + out.writeShort(tagCount); + + // address for extended header data + int bufferAddress = (int) out.position() + tagCount * 12 + 4; + + // write tags + tagEntries.sort(Comparator.naturalOrder()); + IFDTag lastTag = null; + for (TagEntryData tagEntry : tagEntries) { + if(lastTag == tagEntry.tag) + throw new IllegalStateException("duplicate IFD tag"); + lastTag = tagEntry.tag; + + int valueCount = tagEntry.writeHeader(out); + if(tagEntry.tag == IFDTag.TAG_STRIP_OFFSETS) { + pixelDataPointer = out.writeUnfinalizedInt(); + } else if(tagEntry.tag == IFDTag.TAG_STRIP_BYTE_COUNTS) { + pixelDataByteCount = out.writeUnfinalizedInt(); + } else { + tagEntry.writeData(out, valueCount, bufferAddress); + } + } + // write pointer to next IFD + FinalizableBFCOutputStream.UnfinalizedData.Int nextIFDPointer = out.writeUnfinalizedInt(); + + // write extended header data + tagDataBuffer.writeTo(out); + out.flush(); + + // write pixel data + out.ensureAlignment(); + assert pixelDataPointer != null; + pixelDataPointer.setData((int) out.position()); + writePixelData(out, writer); + assert pixelDataByteCount != null; + pixelDataByteCount.setData((int) out.position() - pixelDataPointer.getData()); + + return nextIFDPointer; + } + + abstract void writePixelData( + FinalizableBFCOutputStream out, + PixelDataWriter writer + ) throws IOException; + + @FunctionalInterface + interface PixelDataWriter { + void writePixelData(DataOutput out) throws IOException; + } +} diff --git a/chunky/src/java/se/llbit/imageformats/tiff/TiffFileWriter.java b/chunky/src/java/se/llbit/imageformats/tiff/TiffFileWriter.java index fc3d7f82b9..dd0265e512 100644 --- a/chunky/src/java/se/llbit/imageformats/tiff/TiffFileWriter.java +++ b/chunky/src/java/se/llbit/imageformats/tiff/TiffFileWriter.java @@ -17,18 +17,11 @@ */ package se.llbit.imageformats.tiff; -import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; +import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; +import java.nio.channels.FileChannel; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; import se.llbit.chunky.main.Version; import se.llbit.chunky.renderer.postprocessing.PixelPostProcessingFilter; @@ -45,297 +38,87 @@ */ public class TiffFileWriter implements AutoCloseable { - private final DataOutputStream out; + private final FinalizableBFCOutputStream out; + private final CompressionType compressionType; + private FinalizableBFCOutputStream.UnfinalizedData.Int nextIFDOffset; - public TiffFileWriter(OutputStream outputStream) throws IOException { - out = new DataOutputStream(outputStream); + public TiffFileWriter( + FileChannel fileChannel, + CompressionType compressionType + ) throws IOException { + this.compressionType = compressionType; + out = new FinalizableBFCOutputStream(fileChannel); // "MM\0*" // - MM -> magic bytes // - \0* -> magic number 42 for big-endian byte order out.writeInt(0x4D4D002A); + nextIFDOffset = out.writeUnfinalizedInt(); } - @Override - public void close() throws IOException { - out.close(); + public TiffFileWriter(FileOutputStream outputStream) throws IOException { + this(outputStream.getChannel(), CompressionType.NONE); } - static class IFDWriter { - /** - * Tag structure (12 bytes): - * - 2 bytes: tag ID - * - 2 bytes: field type - * - 4 bytes: value count, not byte count! - * - 4 bytes: value if it fits otherwise address in file - */ - List tagEntries = new ArrayList<>(); - List unfinalizedTags = new ArrayList<>(); - ByteArrayOutputStream tagDataBuffer = new ByteArrayOutputStream(128); - - enum TagFieldType { - ASCII(2, 1), - SHORT(3, 2), - LONG(4, 4), - RATIONAL(5, 8); - final short id; - final int byteSize; - - TagFieldType(int id, int byteSize) { - this.id = (short) id; - this.byteSize = byteSize; - } - } - - /** - * Writes a string as 7-bit ASCII code, 0-terminated - */ - public void addAsciiTagEntry(short tagID, String data) throws IOException { - byte[] bytes = data.getBytes(StandardCharsets.US_ASCII); - // append 0-terminator - bytes = Arrays.copyOf(bytes, bytes.length+1); - addTagEntry(tagID, TagFieldType.ASCII, bytes.length, bytes); - } - - /** - * Writes single 16-bit unsigned(!) integer - */ - public void addShortTagEntry(short tagID, short data) throws IOException { - ByteBuffer buf = ByteBuffer.allocate(12); - buf.putShort(tagID); - buf.putShort(TagFieldType.SHORT.id); - buf.putInt(1); - buf.putShort(data); - buf.putShort((short) 0); - tagEntries.add(buf); - } - - /** - * Writes multiple 16-bit unsigned(!) integer - */ - public void addShortTagEntry(short tagID, short[] data) throws IOException { - ByteBuffer buf = ByteBuffer.allocate(data.length*2); - buf.asShortBuffer().put(data); - addTagEntry(tagID, TagFieldType.SHORT, data.length, buf.array()); - } - - /** - * Writes single 32-bit unsigned(!) integer - */ - public void addLongTagEntry(short tagID, int data) throws IOException { - ByteBuffer buf = ByteBuffer.allocate(12); - buf.putShort(tagID); - buf.putShort(TagFieldType.LONG.id); - buf.putInt(1); - buf.putInt(data); - tagEntries.add(buf); - } - - /** - * Writes multiple 32-bit unsigned(!) integer - */ - private void addLongTagEntry(short tagID, int[] data) throws IOException { - ByteBuffer buf = ByteBuffer.allocate(data.length*4); - buf.asIntBuffer().put(data); - addTagEntry(tagID, TagFieldType.LONG, data.length, buf.array()); - } - - /** - * Writes a fraction using an - * - 32-bit unsigned(!) integer numerator - * - 32-bit unsigned(!) integer denominator - * @param numeratorDenominatorPairs interleaved array of fraction numerator and denominator [n,d,n,d,...] - */ - private void addRationalTagEntry(short tagID, int[] numeratorDenominatorPairs) throws IOException { - ByteBuffer buf = ByteBuffer.allocate(numeratorDenominatorPairs.length*4); - buf.asIntBuffer().put(numeratorDenominatorPairs); - addTagEntry(tagID, TagFieldType.RATIONAL, numeratorDenominatorPairs.length, buf.array()); - } - - private void addTagEntry(short tagID, TagFieldType fieldType, int valueCount, byte[] data) throws IOException { - ByteBuffer buf = ByteBuffer.allocate(12); - buf.putShort(tagID); - buf.putShort(fieldType.id); - buf.putInt(valueCount); - if(valueCount * fieldType.byteSize <= 4) { - // store in tag - buf.put(data); - // pad tag - while(buf.position() < 12) { - buf.put((byte) 0); - } - } else { - // finalize later - buf.putInt(0); - int tagIndex = tagEntries.size(); - int bufferOffset = tagDataBuffer.size(); - unfinalizedTags.add((long) tagIndex << 32 | bufferOffset); - tagDataBuffer.write(data); - } - tagEntries.add(buf); - } - - /** - * IFD structure: - * - 2 bytes: tag count - * - x * 12 bytes: tags - * - 4 bytes: next IFD address - */ - void write(DataOutputStream out, int expectedAddress, boolean lastIFD) throws IOException { - assert(expectedAddress >= out.size()); - int padding = expectedAddress - out.size(); - out.write(new byte[padding]); - - // write tag count - out.writeShort(tagEntries.size()); - - // address for buffered data - int addressAfterIFD = out.size() + tagEntries.size() * 12 + 4; - // finalize tag addresses - for(long unfinalizedTag : unfinalizedTags) { - int unfinalizedTagIndex = (int) (unfinalizedTag >> 32); - int bufferOffset = (int) unfinalizedTag; - ByteBuffer tagEntry = tagEntries.get(unfinalizedTagIndex); - tagEntry.putInt(8, addressAfterIFD + bufferOffset); - } - // sort tag entries by tagID - tagEntries.sort( - Comparator.comparingInt((ByteBuffer tagEntry) -> (int) tagEntry.getShort(0)) - ); - - // write tags - for(ByteBuffer tagEntry : tagEntries) { - out.write(tagEntry.array()); - } - // write pointer to next IFD - if(!lastIFD) { - int nextIFDAddress = addressAfterIFD + tagDataBuffer.size(); - if ((nextIFDAddress & 0b1) != 0) { - // align to 16-bit address - nextIFDAddress++; - } - out.writeInt(nextIFDAddress); - } else { - // zero-pointer represents last IFD - out.writeInt(0); - } - out.flush(); - - // write buffered data - out.write(tagDataBuffer.toByteArray()); - out.flush(); - } - - /** width / columns / pixels per scanline */ - static final short TAG_IMAGE_WIDTH = 0x0100; - /** height / rows / length / scanline count */ - static final short TAG_IMAGE_HEIGHT = 0x0101; - static final short TAG_BITS_PER_SAMPLE = 0x0102; - static final short TAG_SAMPLE_FORMAT = 0x0153; - - /** defines details of subfile using 32 flag bits */ - static final short TAG_NEW_SUBFILE_TYPE = 0x00FE; - - static final short TAG_COMPRESSION_TYPE = 0x0103; - static final short TAG_PHOTOMETRIC_INTERPRETATION = 0x0106; - static final short TAG_PLANAR_CONFIGURATION = 0x011C; - - /** number of rows in each strip (except possibly the last strip) */ - static final short TAG_ROWS_PER_STRIP = 0x0116; - /** for each strip, the byte offset of that strip */ - static final short TAG_STRIP_OFFSETS = 0x0111; - /** for each strip, the number of bytes in that strip after any compression */ - static final short TAG_STRIP_BYTE_COUNTS = 0x0117; - static final short TAG_ORIENTATION = 0x0112; - static final short TAG_SAMPLES_PER_PIXEL = 0x0115; - - static final short TAG_X_RESOLUTION = 0x011A; - static final short TAG_Y_RESOLUTION = 0x011B; - static final short TAG_RESOLUTION_UNIT = 0x0128; - - static final short TAG_SOFTWARE = 0x0131; - static final short TAG_DATETIME = 0x0132; + public void doFinalization() throws IOException { + out.doFinalization(); } - private static final int BYTES_PER_SAMPLE = 4; - private static final DateTimeFormatter DATETIME_FORMAT = DateTimeFormatter.ofPattern("yyyy:MM:dd HH:mm:ss"); + @Override + public void close() throws IOException { + out.close(); + } /** * Export sample buffer as Baseline TIFF RGB image / TIFF Class R image * with 32 bits per color component. + * + *

Note: This method does not close the output stream, and can be called multiple times for multiple layers. + * Use {@link #doFinalization()} to complete the export. */ public void export(Scene scene, TaskTracker.Task task) throws IOException { + nextIFDOffset = writePrimaryIDF(nextIFDOffset, scene, task); + } + + private static final DateTimeFormatter DATETIME_FORMAT = DateTimeFormatter.ofPattern("yyyy:MM:dd HH:mm:ss"); + + private static final int BYTES_PER_SAMPLE = 4; + private FinalizableBFCOutputStream.UnfinalizedData.Int writePrimaryIDF( + FinalizableBFCOutputStream.UnfinalizedData.Int ifdOffset, + Scene scene, + TaskTracker.Task task + ) throws IOException { int width = scene.canvasWidth(); int height = scene.canvasHeight(); - int pixelDataOffset = out.size() + 4; // header + ifd address - int pixelDataByteCount = width * height * 3 * BYTES_PER_SAMPLE; - int ifdOffset = pixelDataOffset + pixelDataByteCount; - out.writeInt(ifdOffset); + BasicIFD idf = new BasicIFD(width, height, compressionType); - writePixelData(width, height, scene, task); - - IFDWriter idf = new IFDWriter(); - // RGB full color - idf.addShortTagEntry(IFDWriter.TAG_PHOTOMETRIC_INTERPRETATION, (short) 2); - // Store pixel components contiguously [RGBRGBRGB...] - idf.addShortTagEntry(IFDWriter.TAG_PLANAR_CONFIGURATION, (short) 1); // Number of components per pixel (R, G, B) - idf.addShortTagEntry(IFDWriter.TAG_SAMPLES_PER_PIXEL, (short) 3); - - assert(width <= Short.MAX_VALUE); - idf.addShortTagEntry(IFDWriter.TAG_IMAGE_WIDTH, (short) width); - assert(height <= Short.MAX_VALUE); - idf.addShortTagEntry(IFDWriter.TAG_IMAGE_HEIGHT, (short) height); + idf.addTag(IFDTag.TAG_SAMPLES_PER_PIXEL, (short) 3); short bitsPerSample = (short) (8 * BYTES_PER_SAMPLE); - idf.addShortTagEntry(IFDWriter.TAG_BITS_PER_SAMPLE, new short[]{ bitsPerSample, bitsPerSample, bitsPerSample }); + idf.addMultiTag(IFDTag.TAG_BITS_PER_SAMPLE, new short[]{ bitsPerSample, bitsPerSample, bitsPerSample }); // Interpret each component as IEEE754 float32 - idf.addShortTagEntry(IFDWriter.TAG_SAMPLE_FORMAT, new short[]{ 3, 3, 3 }); - - // No compression, but pack data into bytes as tightly as possible, leaving no unused - // bits (except at the end of a row). The component values are stored as an array of - // type BYTE. Each scan line (row) is padded to the next BYTE boundary. - idf.addShortTagEntry(IFDWriter.TAG_COMPRESSION_TYPE, (short) 1); - - // "Compressed or uncompressed image data can be stored almost anywhere in a - // TIFF file. TIFF also supports breaking an image into separate strips for increased - // editing flexibility and efficient I/O buffering." - // We will use exactly 1 strip, therefore the relevant tags have only 1 entry. - // All rows in 1 strip - idf.addLongTagEntry(IFDWriter.TAG_ROWS_PER_STRIP, height); - // Absolute strip address - idf.addLongTagEntry(IFDWriter.TAG_STRIP_OFFSETS, pixelDataOffset); - // Strip length - idf.addLongTagEntry(IFDWriter.TAG_STRIP_BYTE_COUNTS, pixelDataByteCount); - // The 0th row represents the visual top of the image, and the 0th column represents the visual left-hand side. - idf.addShortTagEntry(IFDWriter.TAG_ORIENTATION, (short) 1); - - // Image does not have a physical size - idf.addShortTagEntry(IFDWriter.TAG_RESOLUTION_UNIT, (short) 1); // not an absolute unit - idf.addRationalTagEntry(IFDWriter.TAG_X_RESOLUTION, new int[]{ 1, 1 }); - idf.addRationalTagEntry(IFDWriter.TAG_Y_RESOLUTION, new int[]{ 1, 1 }); - - idf.addAsciiTagEntry(IFDWriter.TAG_SOFTWARE, "Chunky " + Version.getVersion()); - idf.addAsciiTagEntry(IFDWriter.TAG_DATETIME, DATETIME_FORMAT.format(LocalDateTime.now())); - - idf.write(out, ifdOffset, true); - } - - private void writePixelData(int width, int height, Scene scene, TaskTracker.Task task) throws IOException { - PixelPostProcessingFilter filter = requirePixelPostProcessingFilter(scene); - double[] sampleBuffer = scene.getSampleBuffer(); - double[] pixelBuffer = new double[3]; - for (int y = 0; y < height; ++y) { - task.update(height, y); - for (int x = 0; x < width; ++x) { - // TODO: refactor pixel access to remove duplicate post processing code from here - filter.processPixel(width, height, sampleBuffer, x, y, scene.getExposure(), pixelBuffer); - out.writeFloat((float) pixelBuffer[0]); - out.writeFloat((float) pixelBuffer[1]); - out.writeFloat((float) pixelBuffer[2]); + idf.addMultiTag(IFDTag.TAG_SAMPLE_FORMAT, new short[]{ 3, 3, 3 }); + + idf.addTag(IFDTag.TAG_SOFTWARE, "Chunky " + Version.getVersion()); + idf.addTag(IFDTag.TAG_DATETIME, DATETIME_FORMAT.format(LocalDateTime.now())); + + return idf.write(out, ifdOffset, (out) -> { + PixelPostProcessingFilter filter = requirePixelPostProcessingFilter(scene); + double[] sampleBuffer = scene.getSampleBuffer(); + double[] pixelBuffer = new double[3]; + for (int y = 0; y < height; ++y) { + task.update(height, y); + for (int x = 0; x < width; ++x) { + // TODO: refactor pixel access to remove duplicate post processing code from here + filter.processPixel(width, height, sampleBuffer, x, y, scene.getExposure(), pixelBuffer); + out.writeFloat((float) pixelBuffer[0]); + out.writeFloat((float) pixelBuffer[1]); + out.writeFloat((float) pixelBuffer[2]); + } + } + task.update(height, height); } - } - out.flush(); - task.update(height, height); + ); } private PixelPostProcessingFilter requirePixelPostProcessingFilter(Scene scene) { diff --git a/chunky/src/java/se/llbit/math/Octree.java b/chunky/src/java/se/llbit/math/Octree.java index 6a90a3ad94..201bdde496 100644 --- a/chunky/src/java/se/llbit/math/Octree.java +++ b/chunky/src/java/se/llbit/math/Octree.java @@ -37,8 +37,8 @@ import se.llbit.chunky.renderer.scene.Scene; import se.llbit.chunky.world.Material; import se.llbit.log.Log; -import se.llbit.util.PositionalInputStream; -import se.llbit.util.PositionalOutputStream; +import se.llbit.util.io.PositionalInputStream; +import se.llbit.util.io.PositionalOutputStream; import se.llbit.util.TaskTracker; /** diff --git a/chunky/src/java/se/llbit/util/io/BufferedFileChannelOutputStream.java b/chunky/src/java/se/llbit/util/io/BufferedFileChannelOutputStream.java new file mode 100644 index 0000000000..f520f83af7 --- /dev/null +++ b/chunky/src/java/se/llbit/util/io/BufferedFileChannelOutputStream.java @@ -0,0 +1,203 @@ +package se.llbit.util.io; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.Objects; + +/** + * Buffered output stream which writes into a file channel. + * + *

Note: Buffering is implemented with assumption that no data larger that the buffer size + * is written in a single operation. Otherwise a {@link java.nio.BufferOverflowException} + * will be thrown. + */ +public class BufferedFileChannelOutputStream extends OutputStream + implements RepositionableMeasurableDataOutput { + + private final FileChannel fileChannel; + private final ByteBuffer buffer; + + public BufferedFileChannelOutputStream( + FileChannel fileChannel, + int bufferSize + ) { + this.fileChannel = fileChannel; + assert bufferSize >= 8; + buffer = ByteBuffer.allocate(bufferSize); + } + + public BufferedFileChannelOutputStream( + FileChannel fileChannel + ) { + this(fileChannel, 64 * 1024); + } + + public FileChannel getChannel() { + return fileChannel; + } + + /** + * Repositions the stream. + *

Note: This method performs a {@link #flush()} before changing the underlying {@link FileChannel} position. + */ + @Override + public void position(long newPosition) throws IOException { + flushBuffer(); + fileChannel.position(newPosition); + } + + @Override + public long position() throws IOException { + return fileChannel.position() + buffer.position(); + } + + /** + * Returns the length of the underlying {@link FileChannel}. + *

Note: This method performs a {@link #flush()} before detecting the length. + * @return the size of the underlying {@link FileChannel}. + */ + @Override + public long length() throws IOException { + flush(); + return fileChannel.size(); + } + + @Override + public void write(int b) throws IOException { + if(!buffer.hasRemaining()) { + flushBuffer(); + } + buffer.put((byte) b); + } + + @Override + public void write(byte[] b) throws IOException { + if(buffer.remaining() < b.length) { + flushBuffer(); + } + buffer.put(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + Objects.checkFromIndexSize(off, len, b.length); + if(buffer.remaining() < len) { + flushBuffer(); + } + buffer.put(b, off, len); + } + + @Override + public void skip(int byteCount) throws IOException { + if(buffer.remaining() >= byteCount) { + buffer.position(buffer.position() + byteCount); + } else { + flushBuffer(); + fileChannel.position(fileChannel.position() + byteCount); + } + } + + @Override + public void writeBoolean(boolean v) throws IOException { + writeByte(v ? 1 : 0); + } + + @Override + public void writeByte(int v) throws IOException { + if(!buffer.hasRemaining()) { + flushBuffer(); + } + buffer.put((byte) v); + } + + @Override + public void writeShort(int v) throws IOException { + if(buffer.remaining() < 2) { + flushBuffer(); + } + buffer.putShort((short) v); + } + + @Override + public void writeChar(int v) throws IOException { + if(buffer.remaining() < 2) { + flushBuffer(); + } + buffer.putChar((char) v); + } + + @Override + public void writeInt(int v) throws IOException { + if(buffer.remaining() < 4) { + flushBuffer(); + } + buffer.putInt(v); + } + + @Override + public void writeLong(long v) throws IOException { + if(buffer.remaining() < 8) { + flushBuffer(); + } + buffer.putLong(v); + } + + @Override + public void writeFloat(float v) throws IOException { + if(buffer.remaining() < 4) { + flushBuffer(); + } + buffer.putFloat(v); + } + + @Override + public void writeDouble(double v) throws IOException { + if(buffer.remaining() < 8) { + flushBuffer(); + } + buffer.putDouble(v); + } + + @Override + public void writeBytes(String s) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void writeChars(String s) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void writeUTF(String s) throws IOException { + throw new UnsupportedOperationException(); + } + + private void flushBuffer() throws IOException { + buffer.flip(); + fileChannel.write(buffer); + buffer.clear(); + } + + /** + * Flushes this output stream and forces any buffered output bytes to be written out to the storage device from which the {@link FileChannel} was created. + * If this channel's file resides on a local storage device then when this method returns it is guaranteed that all changes made to the file will have been written to that device. + */ + @Override + public void flush() throws IOException { + flushBuffer(); + fileChannel.force(false); + } + + /** + * {@link #flush()} this stream, then closes the associated {@link FileChannel} of this stream. + * The closed stream cannot perform output operations and cannot be reopened. + */ + @Override + public void close() throws IOException { + flush(); + fileChannel.close(); + } +} diff --git a/chunky/src/java/se/llbit/util/IsolatedOutputStream.java b/chunky/src/java/se/llbit/util/io/IsolatedOutputStream.java similarity index 93% rename from chunky/src/java/se/llbit/util/IsolatedOutputStream.java rename to chunky/src/java/se/llbit/util/io/IsolatedOutputStream.java index 55d0db6332..ad2fa1dcc4 100644 --- a/chunky/src/java/se/llbit/util/IsolatedOutputStream.java +++ b/chunky/src/java/se/llbit/util/io/IsolatedOutputStream.java @@ -1,4 +1,4 @@ -package se.llbit.util; +package se.llbit.util.io; import java.io.FilterOutputStream; import java.io.IOException; diff --git a/chunky/src/java/se/llbit/util/PositionalInputStream.java b/chunky/src/java/se/llbit/util/io/PositionalInputStream.java similarity index 99% rename from chunky/src/java/se/llbit/util/PositionalInputStream.java rename to chunky/src/java/se/llbit/util/io/PositionalInputStream.java index 9113d7cf74..ffcc19bf7f 100644 --- a/chunky/src/java/se/llbit/util/PositionalInputStream.java +++ b/chunky/src/java/se/llbit/util/io/PositionalInputStream.java @@ -14,7 +14,7 @@ * You should have received a copy of the GNU General Public License * along with Chunky. If not, see . */ -package se.llbit.util; +package se.llbit.util.io; import java.io.IOException; import java.io.InputStream; diff --git a/chunky/src/java/se/llbit/util/PositionalOutputStream.java b/chunky/src/java/se/llbit/util/io/PositionalOutputStream.java similarity index 98% rename from chunky/src/java/se/llbit/util/PositionalOutputStream.java rename to chunky/src/java/se/llbit/util/io/PositionalOutputStream.java index 5ec9ce2bd0..5cfe888c6f 100644 --- a/chunky/src/java/se/llbit/util/PositionalOutputStream.java +++ b/chunky/src/java/se/llbit/util/io/PositionalOutputStream.java @@ -16,7 +16,7 @@ * along with Chunky. If not, see . */ -package se.llbit.util; +package se.llbit.util.io; import se.llbit.util.annotation.NotNull; diff --git a/chunky/src/java/se/llbit/util/io/RepositionableMeasurableDataOutput.java b/chunky/src/java/se/llbit/util/io/RepositionableMeasurableDataOutput.java new file mode 100644 index 0000000000..95349c2562 --- /dev/null +++ b/chunky/src/java/se/llbit/util/io/RepositionableMeasurableDataOutput.java @@ -0,0 +1,19 @@ +package se.llbit.util.io; + +import it.unimi.dsi.fastutil.io.MeasurableStream; +import it.unimi.dsi.fastutil.io.RepositionableStream; + +import java.io.DataOutput; +import java.io.IOException; + +/** + * A repositionable {@link DataOutput} stream whose size can be measured. + */ +public interface RepositionableMeasurableDataOutput + extends RepositionableStream, MeasurableStream, DataOutput { + + /** + * Skips n bytes (should be equivalent to writing n 0x00 bytes). + */ + void skip(int byteCount) throws IOException; +} diff --git a/chunky/src/java/se/llbit/util/ZipExport.java b/chunky/src/java/se/llbit/util/io/ZipExport.java similarity index 98% rename from chunky/src/java/se/llbit/util/ZipExport.java rename to chunky/src/java/se/llbit/util/io/ZipExport.java index 1cabbaaf3b..6e52a22e64 100644 --- a/chunky/src/java/se/llbit/util/ZipExport.java +++ b/chunky/src/java/se/llbit/util/io/ZipExport.java @@ -14,7 +14,7 @@ * You should have received a copy of the GNU General Public License * along with Chunky. If not, see . */ -package se.llbit.util; +package se.llbit.util.io; import se.llbit.log.Log;