From b9d2fb36a0bc94c109201035183ae7f1b9b33721 Mon Sep 17 00:00:00 2001 From: ArthurHeitmann <37270165+ArthurHeitmann@users.noreply.github.com> Date: Tue, 27 Aug 2024 08:55:58 +0200 Subject: [PATCH] Faster cpk extraction --- lib/fileTypeUtils/cpk/cpk.dart | 26 ++++----- .../utils/ByteDataWrapperRA.dart | 56 ++++++++++++++++--- 2 files changed, 62 insertions(+), 20 deletions(-) diff --git a/lib/fileTypeUtils/cpk/cpk.dart b/lib/fileTypeUtils/cpk/cpk.dart index e2bc0abd..2b0d7535 100644 --- a/lib/fileTypeUtils/cpk/cpk.dart +++ b/lib/fileTypeUtils/cpk/cpk.dart @@ -73,7 +73,7 @@ class CpkTable { int rowsStart = bytes.position; List> rows = []; for (var i = 0; i < header.rowCount; i++) { - await bytes.setPosition(rowsStart); + bytes.setPosition(rowsStart); List row = []; for (var j = 0; j < header.fieldCount; j++) row.add(await CpkField.read(bytes, header)); @@ -110,29 +110,29 @@ class CpkTableDataPool { Future readString(int offset) async { int pos = bytes.position; - await bytes.setPosition(stringPoolPosition + offset); + bytes.setPosition(stringPoolPosition + offset); String str = await bytes.readStringZeroTerminated(); - await bytes.setPosition(pos); + bytes.setPosition(pos); return str; } Future> readData(int offset, int length) async { int pos = bytes.position; - await bytes.setPosition(dataPoolPosition + offset); + bytes.setPosition(dataPoolPosition + offset); List data; if (offset > 0 && length == 0) { print("WARNING: Untested code (:"); int subLength = 0; if (await bytes.readString(4) == "@UTF") { subLength = await bytes.readUint32(); - await bytes.setPosition(bytes.position - 4); + bytes.setPosition(bytes.position - 4); } - await bytes.setPosition(bytes.position - 4); + bytes.setPosition(bytes.position - 4); data = await bytes.readUint8List(subLength); } else { data = await bytes.readUint8List(length); } - await bytes.setPosition(pos); + bytes.setPosition(pos); return data; } @@ -182,9 +182,9 @@ class CpkField { value = await _readValue(field, bytes, dataPool, false); } else if (isRowStorage) { int pos = bytes.position; - await bytes.setPosition(dataPool.currentRowStorePos); + bytes.setPosition(dataPool.currentRowStorePos); value = await _readValue(field, bytes, dataPool, true); - await bytes.setPosition(pos); + bytes.setPosition(pos); } field.value = value; @@ -298,7 +298,7 @@ class CpkFileUncompressed extends CpkFile { @override Future readData(ByteDataWrapperRA bytes) async { - await bytes.setPosition(dataOffset); + bytes.setPosition(dataOffset); return await bytes.readUint8List(dataSize); } } @@ -321,7 +321,7 @@ class CpkFileCompressed extends CpkFile { static Future read(String path, String name, ByteDataWrapperRA bytes, int compressedDataOffset) async { bytes.endian = Endian.little; - await bytes.setPosition(compressedDataOffset); + bytes.setPosition(compressedDataOffset); var id = await bytes.readString(8); var uncompressedSize = await bytes.readUint32(); var compressedSize = await bytes.readUint32(); @@ -330,7 +330,7 @@ class CpkFileCompressed extends CpkFile { @override Future readData(ByteDataWrapperRA bytes) async { - await bytes.setPosition(compressedDataOffset); + bytes.setPosition(compressedDataOffset); var compressedData = await bytes.readUint8List(compressedSize); var uncompressedHeader = await bytes.readUint8List(0x100); return decompress(compressedData, uncompressedHeader); @@ -438,7 +438,7 @@ class Cpk { throw Exception("TocOffset field not found in header table"); int tocOffset = tocOffsetField.value! as int; - await bytes.setPosition(tocOffset); + bytes.setPosition(tocOffset); var toc = await CpkSection.read(bytes); int contentDelta = min(tocOffset, contentOffset); diff --git a/lib/fileTypeUtils/utils/ByteDataWrapperRA.dart b/lib/fileTypeUtils/utils/ByteDataWrapperRA.dart index 24e575bf..625d8b2c 100644 --- a/lib/fileTypeUtils/utils/ByteDataWrapperRA.dart +++ b/lib/fileTypeUtils/utils/ByteDataWrapperRA.dart @@ -4,13 +4,55 @@ import 'dart:typed_data'; import 'ByteDataWrapper.dart'; +class _BufferedReader { + final RandomAccessFile _file; + final Uint8List _buffer = Uint8List(_bufferCapacity); + static const int _bufferCapacity = 1024 * 1024; + int _position = 0; + int _bufferStart = 0; + int _bufferSize = -1; + + _BufferedReader(this._file); + + Future close() { + return _file.close(); + } + + void setPosition(int pos) { + _position = pos; + } + + Future read(int length) async { + if (length > _bufferCapacity) { + if (_position != _bufferStart) + await _file.setPosition(_position); + return _file.read(length); + } + if (_position < _bufferStart || _position + length > _bufferStart + _bufferCapacity || _bufferSize == -1) { + await _fillBuffer(); + } + var bufferOffset = _position - _bufferStart; + if (bufferOffset >= _bufferSize) { + throw Exception("Reached end of file! Tried to read $length bytes at $_position when file only has ${_bufferStart + _bufferSize} bytes"); + } + return _buffer.sublist(bufferOffset, bufferOffset + length); + } + + Future _fillBuffer() async { + await _file.setPosition(_position); + _bufferStart = _position; + _bufferSize = await _file.readInto(_buffer); + } +} + class ByteDataWrapperRA { - final RandomAccessFile file; + final _BufferedReader _file; Endian endian; final int length; int _position = 0; - ByteDataWrapperRA(this.file, this.length, { this.endian = Endian.little }); + ByteDataWrapperRA(RandomAccessFile file, this.length, { this.endian = Endian.little }) + : _file = _BufferedReader(file); static Future fromFile(String path) async { @@ -20,22 +62,22 @@ class ByteDataWrapperRA { } Future close() async { - await file.close(); + await _file.close(); } int get position => _position; - Future setPosition(int value) async { + void setPosition(int value) { if (value > length) throw RangeError.range(value, 0, length, "File size"); - await file.setPosition(value); + _file.setPosition(value); _position = value; } Future _read(int length) async { - var bytes = await file.read(length); - await setPosition(_position + length); + var bytes = await _file.read(length); + setPosition(_position + length); return ByteData.view(bytes.buffer); }