Faster cpk extraction

ArthurHeitmann · Sep 7, 2024 · b9d2fb3 · b9d2fb3
1 parent 0d06277
commit b9d2fb3
Show file tree

Hide file tree

Showing 2 changed files with 62 additions and 20 deletions.
diff --git a/lib/fileTypeUtils/cpk/cpk.dart b/lib/fileTypeUtils/cpk/cpk.dart
@@ -73,7 +73,7 @@ class CpkTable {
     int rowsStart = bytes.position;
     List<List<CpkField>> rows = [];
     for (var i = 0; i < header.rowCount; i++) {
-      await bytes.setPosition(rowsStart);
+      bytes.setPosition(rowsStart);
       List<CpkField> row = [];
       for (var j = 0; j < header.fieldCount; j++)
         row.add(await CpkField.read(bytes, header));
@@ -110,29 +110,29 @@ class CpkTableDataPool {
 
   Future<String> readString(int offset) async {
     int pos = bytes.position;
-    await bytes.setPosition(stringPoolPosition + offset);
+    bytes.setPosition(stringPoolPosition + offset);
     String str = await bytes.readStringZeroTerminated();
-    await bytes.setPosition(pos);
+    bytes.setPosition(pos);
     return str;
   }
 
   Future<List<int>> readData(int offset, int length) async {
     int pos = bytes.position;
-    await bytes.setPosition(dataPoolPosition + offset);
+    bytes.setPosition(dataPoolPosition + offset);
     List<int> data;
     if (offset > 0 && length == 0) {
       print("WARNING: Untested code (:");
       int subLength = 0;
       if (await bytes.readString(4) == "@UTF") {
         subLength = await bytes.readUint32();
-        await bytes.setPosition(bytes.position - 4);
+        bytes.setPosition(bytes.position - 4);
       }
-      await bytes.setPosition(bytes.position - 4);
+      bytes.setPosition(bytes.position - 4);
       data = await bytes.readUint8List(subLength);
     } else {
       data = await bytes.readUint8List(length);
     }
-    await bytes.setPosition(pos);
+    bytes.setPosition(pos);
     return data;
   }
 
@@ -182,9 +182,9 @@ class CpkField {
       value = await _readValue(field, bytes, dataPool, false);
     } else if (isRowStorage) {
       int pos = bytes.position;
-      await bytes.setPosition(dataPool.currentRowStorePos);
+      bytes.setPosition(dataPool.currentRowStorePos);
       value = await _readValue(field, bytes, dataPool, true);
-      await bytes.setPosition(pos);
+      bytes.setPosition(pos);
     }
     field.value = value;
 
@@ -298,7 +298,7 @@ class CpkFileUncompressed extends CpkFile {
 
   @override
   Future<Uint8List> readData(ByteDataWrapperRA bytes) async {
-    await bytes.setPosition(dataOffset);
+    bytes.setPosition(dataOffset);
     return await bytes.readUint8List(dataSize);
   }
 }
@@ -321,7 +321,7 @@ class CpkFileCompressed extends CpkFile {
 
   static Future<CpkFileCompressed> read(String path, String name, ByteDataWrapperRA bytes, int compressedDataOffset) async {
     bytes.endian = Endian.little;
-    await bytes.setPosition(compressedDataOffset);
+    bytes.setPosition(compressedDataOffset);
     var id = await bytes.readString(8);
     var uncompressedSize = await bytes.readUint32();
     var compressedSize = await bytes.readUint32();
@@ -330,7 +330,7 @@ class CpkFileCompressed extends CpkFile {
 
   @override
   Future<Uint8List> readData(ByteDataWrapperRA bytes) async {
-    await bytes.setPosition(compressedDataOffset);
+    bytes.setPosition(compressedDataOffset);
     var compressedData = await bytes.readUint8List(compressedSize);
     var uncompressedHeader = await bytes.readUint8List(0x100);
     return decompress(compressedData, uncompressedHeader);
@@ -438,7 +438,7 @@ class Cpk {
       throw Exception("TocOffset field not found in header table");
     int tocOffset = tocOffsetField.value! as int;
 
-    await bytes.setPosition(tocOffset);
+    bytes.setPosition(tocOffset);
     var toc = await CpkSection.read(bytes);
 
     int contentDelta = min(tocOffset, contentOffset);

diff --git a/lib/fileTypeUtils/utils/ByteDataWrapperRA.dart b/lib/fileTypeUtils/utils/ByteDataWrapperRA.dart
@@ -4,13 +4,55 @@ import 'dart:typed_data';
 import 'ByteDataWrapper.dart';
 
 
+class _BufferedReader {
+  final RandomAccessFile _file;
+  final Uint8List _buffer = Uint8List(_bufferCapacity);
+  static const int _bufferCapacity = 1024 * 1024;
+  int _position = 0;
+  int _bufferStart = 0;
+  int _bufferSize = -1;
+
+  _BufferedReader(this._file);
+
+  Future<void> close() {
+    return _file.close();
+  }
+
+  void setPosition(int pos) {
+    _position = pos;
+  }
+
+  Future<Uint8List> read(int length) async {
+    if (length > _bufferCapacity) {
+      if (_position != _bufferStart)
+        await _file.setPosition(_position);
+      return _file.read(length);
+    }
+    if (_position < _bufferStart || _position + length > _bufferStart + _bufferCapacity || _bufferSize == -1) {
+      await _fillBuffer();
+    }
+    var bufferOffset = _position - _bufferStart;
+    if (bufferOffset >= _bufferSize) {
+      throw Exception("Reached end of file! Tried to read $length bytes at $_position when file only has ${_bufferStart + _bufferSize} bytes");
+    }
+    return _buffer.sublist(bufferOffset, bufferOffset + length);
+  }
+
+  Future<void> _fillBuffer() async {
+    await _file.setPosition(_position);
+    _bufferStart = _position;
+    _bufferSize = await _file.readInto(_buffer);
+  }
+}
+
 class ByteDataWrapperRA {
-  final RandomAccessFile file;
+  final _BufferedReader _file;
   Endian endian;
   final int length;
   int _position = 0;
 
-  ByteDataWrapperRA(this.file, this.length, { this.endian = Endian.little });
+  ByteDataWrapperRA(RandomAccessFile file, this.length, { this.endian = Endian.little })
+    : _file = _BufferedReader(file);
 
 
   static Future<ByteDataWrapperRA> fromFile(String path) async {
@@ -20,22 +62,22 @@ class ByteDataWrapperRA {
   }
 
   Future<void> close() async {
-    await file.close();
+    await _file.close();
   }
 
   int get position => _position;
 
-  Future<void> setPosition(int value) async {
+  void setPosition(int value) {
     if (value > length)
       throw RangeError.range(value, 0, length, "File size");
 
-    await file.setPosition(value);
+    _file.setPosition(value);
     _position = value;
   }
 
   Future<ByteData> _read(int length) async {
-    var bytes = await file.read(length);
-    await setPosition(_position + length);
+    var bytes = await _file.read(length);
+    setPosition(_position + length);
     return ByteData.view(bytes.buffer);
   }