diff --git a/.gitignore b/.gitignore index 4fffb2f..93ad9b0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ /target /Cargo.lock +/files/ +/packages/ diff --git a/Cargo.lock b/Cargo.lock index 0a17aab..8ab3b0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,55 @@ dependencies = [ "subtle", ] +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" + +[[package]] +name = "anstyle-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle", + "windows-sys", +] + [[package]] name = "anyhow" version = "1.0.71" @@ -49,6 +98,12 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + [[package]] name = "binrw" version = "0.11.2" @@ -70,15 +125,27 @@ dependencies = [ "owo-colors", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] +[[package]] +name = "bitflags" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" + [[package]] name = "bytemuck" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + [[package]] name = "cfg-if" version = "1.0.0" @@ -95,6 +162,62 @@ dependencies = [ "inout", ] +[[package]] +name = "clap" +version = "4.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1640e5cc7fb47dbb8338fd471b105e7ed6c3cb2aeb00c2e067127ffd3764a05d" +dependencies = [ + "clap_builder", + "clap_derive", + "once_cell", +] + +[[package]] +name = "clap-num" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488557e97528174edaa2ee268b23a809e0c598213a4bbcb4f34575a46fda147e" +dependencies = [ + "num-traits", +] + +[[package]] +name = "clap_builder" +version = "4.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98c59138d527eeaf9b53f35a77fcc1fad9d883116070c63d5de1c7dc7b00c72b" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.25", +] + +[[package]] +name = "clap_lex" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "cpufeatures" version = "0.2.9" @@ -104,6 +227,49 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -125,16 +291,19 @@ dependencies = [ ] [[package]] -name = "destiny2-pkg" -version = "0.1.0" +name = "destiny-pkg" +version = "0.2.0" dependencies = [ "aes", "aes-gcm", "anyhow", "binrw", + "clap", + "clap-num", "lazy_static", "libloading", "nohash-hasher", + "rayon", ] [[package]] @@ -143,6 +312,27 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -174,6 +364,18 @@ dependencies = [ "polyval", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" + [[package]] name = "inout" version = "0.1.3" @@ -183,6 +385,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi", + "rustix", + "windows-sys", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -205,12 +418,52 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "nohash-hasher" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + [[package]] name = "opaque-debug" version = "0.3.0" @@ -262,6 +515,53 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "rustix" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "subtle" version = "2.5.0" @@ -279,6 +579,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "typenum" version = "1.16.0" @@ -301,6 +612,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "version_check" version = "0.9.4" diff --git a/Cargo.toml b/Cargo.toml index ba1d790..8cc53a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "destiny2-pkg" -version = "0.1.0" +name = "destiny-pkg" +version = "0.2.0" edition = "2021" authors = ["Lucas Cohaereo "] @@ -12,3 +12,6 @@ binrw = "0.11" lazy_static = "1.4.0" libloading = "0.8" nohash-hasher = "0.2.0" +rayon = "1.7.0" +clap = { version = "4.3.11", features = ["derive"] } +clap-num = "1.0.2" diff --git a/README.md b/README.md new file mode 100644 index 0000000..7dfa672 --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +To run this, you will need to get oo2core_3_win64.dll from somewhere (an old game for example), and place it in the +directory where you run destinypkgtool from. + +## Version support + +| Version | Platform | Works? | +|---------------------------------|----------|--------| +| Destiny Legacy (The Taken King) | PS3/X360 | ✅ | +| Destiny Legacy (The Taken King) | PS4/XONE | ❔ | +| Destiny (Rise of Iron) | PS4/XONE | ❌ | +| Destiny 2 (Pre-BL) | PC | ✅ | +| Destiny 2 (Lightfall) | PC | ❌ | diff --git a/src/bin/unpack.rs b/src/bin/unpack.rs index 301e345..c5552ad 100644 --- a/src/bin/unpack.rs +++ b/src/bin/unpack.rs @@ -1,88 +1,80 @@ -use destiny2_pkg::package::Package; +use clap::Parser; +use destiny_pkg::{PackageVersion, TagHash}; use std::fs::File; use std::io::Write; +use std::path::PathBuf; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None, disable_version_flag(true))] +struct Args { + /// Package to extract + package: String, + + /// Don't extract any files, just print them + #[arg(short, long, default_value = "false")] + dry_run: bool, + + /// Directory to extract to (default: ./out/pkg_name) + #[arg(short)] + output_dir: Option, + + /// Version of the package to extract + #[arg(short, value_enum)] + version: PackageVersion, +} fn main() -> anyhow::Result<()> { - let package = Package::open(&std::env::args().nth(1).unwrap())?; - std::fs::create_dir("./files/").ok(); + let args = Args::parse(); + + let pkg_name = PathBuf::from(args.package.clone()) + .file_stem() + .unwrap() + .to_string_lossy() + .to_string(); - println!( - "PKG {:04x}_{}", - package.header.pkg_id, package.header.patch_id - ); - for (i, e) in package.entries().enumerate() { + let package = args.version.open(&args.package)?; + + let out_dir = args + .output_dir + .unwrap_or_else(|| format!("./out/{pkg_name}")); + + std::fs::create_dir_all(&out_dir).ok(); + + println!("PKG {:04x}_{}", package.pkg_id(), package.patch_id()); + for (i, e) in package.entries().iter().enumerate() { print!("{}/{} - ", e.file_type, e.file_subtype); - if e.reference != u32::MAX { - print!( - "{i} 0x{:x} - p={:x} f={} / r=0x{:x} ", - e.file_size, - (e.reference & !0x80800000) >> 13, - e.reference & 0x1fff, - e.reference + let ref_hash = TagHash(e.reference); + if ref_hash.is_pkg_file() { + println!( + "{i} 0x{:04x} - Reference {ref_hash:?} / r=0x{:x} (type={}, subtype={})", + e.file_size, ref_hash.0, e.file_type, e.file_subtype ); } else { - print!("{i} 0x{:x} - ", e.file_size); + println!( + "{i} 0x{:04x} - r=0x{:x} (type={}, subtype={})", + e.file_size, ref_hash.0, e.file_type, e.file_subtype + ); } - let ext = match (e.file_type, e.file_subtype) { - (26, 6) => { - println!("WWise WAVE Audio"); - "wem".to_string() - } - (26, 7) => { - println!("Havok File"); - "hkf".to_string() - } - (27, _) => { - println!("CriWare USM Video"); - "usm".to_string() - } - (33, _) => { - println!("DirectX Bytecode Header"); - "cso.header".to_string() - } - (32, _) => { - println!("Texture Header"); - "texture.header".to_string() - } - (40, _) | (48, 1) | (48, 2) => { - println!("Texture Data"); - "texture.data".to_string() - } - (41, _) => { - let ty = match e.file_subtype { - 0 => "fragment".to_string(), - 1 => "vertex".to_string(), - 6 => "compute".to_string(), - u => format!("unk{u}"), - }; - println!("DirectX Bytecode Data ({})", ty); - - format!("cso.{ty}") - } - (8, _) => { - println!("8080 structure file"); - "8080".to_string() - } - _ => { - println!("Unknown {}/{}", e.file_type, e.file_subtype); - "bin".to_string() - } - }; - let data = match package.read_entry(i) { - Ok(data) => data, - Err(e) => { - eprintln!( - "Failed to extract entry {}/{}: {e}", - i, - package.entries().count() - 1 - ); - continue; - } - }; + if !args.dry_run { + let data: Vec = match package.read_entry(i) { + Ok(data) => data, + Err(e) => { + eprintln!( + "Failed to extract entry {}/{}: {e}", + i, + package.entries().len() - 1 + ); + continue; + } + }; - let mut o = File::create(format!("files/{i}.{ext}"))?; - o.write_all(&data)?; + let mut o = File::create(format!( + "{out_dir}/{i}_{:08x}_t{}_s{}.bin", + e.reference, e.file_type, e.file_subtype + ))?; + o.write_all(&data)?; + } } Ok(()) diff --git a/src/bin/unpack_refs.rs b/src/bin/unpack_refs.rs new file mode 100644 index 0000000..52c062f --- /dev/null +++ b/src/bin/unpack_refs.rs @@ -0,0 +1,83 @@ +use clap::Parser; +use clap_num::maybe_hex; +use destiny_pkg::{PackageManager, PackageVersion, TagHash}; +use std::fs::File; +use std::io::Write; +use std::path::PathBuf; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None, disable_version_flag(true))] +struct Args { + /// Path to packages directory + packages_path: String, + + #[arg(value_parser = maybe_hex::)] + reference: u32, + + /// Don't extract any files, just print them + #[arg(short, long, default_value = "false")] + dry_run: bool, + + /// Directory to extract to (default: ./out/pkg_name) + #[arg(short)] + output_dir: Option, + + /// Version of the package to extract + #[arg(short, value_enum)] + version: PackageVersion, +} + +fn main() -> anyhow::Result<()> { + let args = Args::parse(); + let mut package_manager = PackageManager::new(args.packages_path, args.version, true)?; + + for (p, i, e) in package_manager.get_all_by_reference(args.reference) { + let pkg_path = package_manager.package_paths.get(&p).unwrap(); + let pkg_name = PathBuf::from(pkg_path) + .file_stem() + .unwrap() + .to_string_lossy() + .to_string(); + + let out_dir = args + .output_dir + .clone() + .unwrap_or_else(|| format!("./out/{pkg_name}")); + + std::fs::create_dir_all(&out_dir).ok(); + println!( + "{} - entry {} type {} subtype {}", + pkg_name, i, e.file_type, e.file_subtype + ); + let ref_hash = TagHash(e.reference); + if ref_hash.is_pkg_file() { + println!( + "{:04x}/{i} 0x{:04x} - Reference {ref_hash:?} / r=0x{:x} (type={}, subtype={})", + p, e.file_size, ref_hash.0, e.file_type, e.file_subtype + ); + } else { + println!( + "{:04x}/{i} 0x{:04x} - r=0x{:x} (type={}, subtype={})", + p, e.file_size, ref_hash.0, e.file_type, e.file_subtype + ); + } + + if !args.dry_run { + let data = match package_manager.read_entry(p, i) { + Ok(data) => data, + Err(e) => { + eprintln!("Failed to extract entry {:04x}/{}: {e}", p, i,); + continue; + } + }; + + let mut o = File::create(format!( + "{out_dir}/{i}_{:08x}_t{}_s{}.bin", + e.reference, e.file_type, e.file_subtype + ))?; + o.write_all(&data)?; + } + } + + Ok(()) +} diff --git a/src/d1_legacy/impl.rs b/src/d1_legacy/impl.rs new file mode 100644 index 0000000..9edde92 --- /dev/null +++ b/src/d1_legacy/impl.rs @@ -0,0 +1,183 @@ +use crate::d1_legacy::structs::{BlockHeader, EntryHeader, PackageHeader}; +use crate::oodle; +use crate::package::{Package, ReadSeek, UEntryHeader, UHashTableEntry, BLOCK_CACHE_SIZE}; +use anyhow::Context; +use binrw::{BinReaderExt, Endian, VecArgs}; +use nohash_hasher::IntMap; +use std::cell::RefCell; +use std::collections::hash_map::Entry; +use std::fs::File; +use std::io::{BufReader, Read, Seek, SeekFrom}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +pub const BLOCK_SIZE: usize = 0x40000; + +// TODO(cohae): Ensure Send+Sync so packages can be multithreaded, should be enforced on `Package` as well +pub struct PackageD1Legacy { + pub header: PackageHeader, + entries: Vec, + blocks: Vec, + + reader: RefCell>, + path_base: String, + + block_counter: AtomicUsize, + block_cache: RefCell>)>>, +} + +impl PackageD1Legacy { + pub fn open(path: &str) -> anyhow::Result { + let reader = BufReader::new(File::open(path)?); + + Self::from_reader(path, reader) + } + + pub fn from_reader( + path: &str, + reader: R, + ) -> anyhow::Result { + let mut reader = reader; + let header: PackageHeader = reader.read_be()?; + + reader.seek(SeekFrom::Start(header.entry_table_offset as u64))?; + let entries = reader.read_be_args( + VecArgs::builder() + .count(header.entry_table_size as usize) + .finalize(), + )?; + + reader.seek(SeekFrom::Start(header.block_table_offset as u64))?; + let blocks = reader.read_be_args( + VecArgs::builder() + .count(header.block_table_size as usize) + .finalize(), + )?; + + let last_underscore_pos = path.rfind('_').unwrap(); + let path_base = path[..last_underscore_pos].to_owned(); + + Ok(PackageD1Legacy { + path_base, + reader: RefCell::new(Box::new(reader)), + header, + entries, + blocks, + block_counter: AtomicUsize::default(), + block_cache: Default::default(), + }) + } + + fn get_block_raw(&self, block_index: usize) -> anyhow::Result> { + let bh = &self.blocks[block_index]; + let mut data = vec![0u8; bh.size as usize]; + + if self.header.patch_id == bh.patch_id { + self.reader + .borrow_mut() + .seek(SeekFrom::Start(bh.offset as u64))?; + self.reader.borrow_mut().read_exact(&mut data)?; + } else { + let mut f = + File::open(format!("{}_{}.pkg", self.path_base, bh.patch_id)).context(format!( + "Failed to open package file {}_{}.pkg", + self.path_base, bh.patch_id + ))?; + + f.seek(SeekFrom::Start(bh.offset as u64))?; + f.read_exact(&mut data)?; + }; + + Ok(data) + } + + fn read_block(&self, block_index: usize) -> anyhow::Result> { + let bh = &self.blocks[block_index]; + let block_data = self.get_block_raw(block_index)?.to_vec(); + + Ok(if (bh.flags & 0x100) != 0 { + let mut buffer = vec![0u8; BLOCK_SIZE]; + let _decompressed_size = oodle::decompress(&block_data, &mut buffer); + buffer + } else { + block_data + }) + } +} + +impl Package for PackageD1Legacy { + fn endianness(&self) -> Endian { + Endian::Big // TODO(cohae): Not necessarily + } + + fn pkg_id(&self) -> u16 { + self.header.pkg_id + } + + fn patch_id(&self) -> u16 { + self.header.patch_id + } + + fn hashes64(&self) -> Vec { + vec![] + } + + fn entries(&self) -> Vec { + self.entries + .iter() + .map(|e| UEntryHeader { + reference: e.reference, + file_type: e.file_type, + file_subtype: e.file_subtype, + starting_block: e.starting_block, + starting_block_offset: e.starting_block_offset, + file_size: e.file_size, + }) + .collect() + } + + fn entry(&self, index: usize) -> Option { + self.entries.get(index).map(|e| UEntryHeader { + reference: e.reference, + file_type: e.file_type, + file_subtype: e.file_subtype, + starting_block: e.starting_block, + starting_block_offset: e.starting_block_offset, + file_size: e.file_size, + }) + } + + fn get_block(&self, block_index: usize) -> anyhow::Result>> { + let (_, b) = match self.block_cache.borrow_mut().entry(block_index) { + Entry::Occupied(o) => o.get().clone(), + Entry::Vacant(v) => { + let block = self.read_block(*v.key())?; + let b = v + .insert((self.block_counter.load(Ordering::Relaxed), Arc::new(block))) + .clone(); + + self.block_counter.store( + self.block_counter.load(Ordering::Relaxed) + 1, + Ordering::Relaxed, + ); + + b + } + }; + + while self.block_cache.borrow().len() > BLOCK_CACHE_SIZE { + let bc = self.block_cache.borrow(); + let (oldest, _) = bc + .iter() + .min_by(|(_, (at, _)), (_, (bt, _))| at.cmp(bt)) + .unwrap(); + + let oldest = *oldest; + drop(bc); + + self.block_cache.borrow_mut().remove(&oldest); + } + + Ok(b) + } +} diff --git a/src/d1_legacy/mod.rs b/src/d1_legacy/mod.rs new file mode 100644 index 0000000..eef8081 --- /dev/null +++ b/src/d1_legacy/mod.rs @@ -0,0 +1,4 @@ +mod r#impl; +mod structs; + +pub use r#impl::PackageD1Legacy; diff --git a/src/d1_legacy/structs.rs b/src/d1_legacy/structs.rs new file mode 100644 index 0000000..9b7966e --- /dev/null +++ b/src/d1_legacy/structs.rs @@ -0,0 +1,81 @@ +use binrw::{binrw, BinRead}; +use std::io::SeekFrom; + +#[derive(BinRead, Debug)] +#[br(repr = u16)] +pub enum PackageLanguage { + None = 0, + English = 1, + French = 2, + Italian = 3, + German = 4, + Spanish = 5, + Japanese = 6, + Portuguese = 7, +} + +#[derive(BinRead, Debug)] +#[br(big)] +pub struct PackageHeader { + pub magic: u32, + pub pkg_id: u16, + pub _unk6: u16, + pub _unk8: u64, + pub build_time: u64, + pub _unk_buildid: u32, + pub _unk1c: u32, + pub patch_id: u16, + pub language: PackageLanguage, + + #[brw(count = 128)] + #[br(map = |s: Vec| String::from_utf8_lossy(&s).to_string().trim_end_matches('\0').to_string())] + pub tool_string: String, + + pub _unka4: u32, + pub _unka8: u32, + pub _unkac: u32, + pub header_signature_offset: u32, + + pub entry_table_size: u32, + pub entry_table_offset: u32, + pub entry_table_hash: [u8; 20], + + pub block_table_size: u32, + pub block_table_offset: u32, + pub block_table_hash: [u8; 20], + + #[br(seek_before = SeekFrom::Start(0x13c))] + pub file_size: u32, +} + +#[derive(BinRead, Debug)] +#[br(big)] +pub struct EntryHeader { + pub reference: u32, + + pub flags: u16, + pub file_type: u8, + pub file_subtype: u8, + + _block_info: u64, + + #[br(calc = _block_info as u32 & 0x3fff)] + pub starting_block: u32, + + #[br(calc = (_block_info >> 14) as u32 & 0x3FFF)] + pub starting_block_offset: u32, + + #[br(calc = (_block_info >> 28) as u32)] + pub file_size: u32, +} + +#[derive(Debug)] +#[binrw] +#[br(big)] +pub struct BlockHeader { + pub offset: u32, + pub size: u32, + pub patch_id: u16, + pub flags: u16, + pub hash: [u8; 20], +} diff --git a/src/d2_prebl/impl.rs b/src/d2_prebl/impl.rs new file mode 100644 index 0000000..c195f36 --- /dev/null +++ b/src/d2_prebl/impl.rs @@ -0,0 +1,224 @@ +use crate::crypto::PkgGcmState; +use crate::d2_prebl::structs::{BlockHeader, EntryHeader, HashTableEntry, PackageHeader}; +use crate::oodle; +use crate::package::{Package, ReadSeek, UEntryHeader, UHashTableEntry, BLOCK_CACHE_SIZE}; +use anyhow::Context; +use binrw::{BinReaderExt, Endian, VecArgs}; +use nohash_hasher::IntMap; +use std::borrow::Cow; +use std::cell::RefCell; +use std::collections::hash_map::Entry; +use std::fs::File; +use std::io::{BufReader, Read, Seek, SeekFrom}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +pub const BLOCK_SIZE: usize = 0x40000; + +// TODO(cohae): Ensure Send+Sync so packages can be multithreaded, should be enforced on `Package` as well +pub struct PackageD2PreBL { + gcm: RefCell, + + pub header: PackageHeader, + entries: Vec, + blocks: Vec, + pub hashes: Vec, + + reader: RefCell>, + path_base: String, + + /// Used for purging old blocks + block_counter: AtomicUsize, + block_cache: RefCell>)>>, +} + +impl PackageD2PreBL { + pub fn open(path: &str) -> anyhow::Result { + let reader = BufReader::new(File::open(path)?); + + Self::from_reader(path, reader) + } + + pub fn from_reader( + path: &str, + reader: R, + ) -> anyhow::Result { + let mut reader = reader; + let header: PackageHeader = reader.read_le()?; + + reader.seek(SeekFrom::Start(header.entry_table_offset as u64 - 16))?; + let entry_table_size_bytes = reader.read_le::()? * 16; + + reader.seek(SeekFrom::Start(header.entry_table_offset as _))?; + let entries = reader.read_le_args(VecArgs { + count: header.entry_table_size as _, + inner: (), + })?; + + reader.seek(SeekFrom::Start( + (header.entry_table_offset + entry_table_size_bytes + 32) as _, + ))?; + let blocks = reader.read_le_args(VecArgs { + count: header.block_table_size as _, + inner: (), + })?; + + let hashes: Vec = if header.unkf0_table_offset != 0 { + reader.seek(SeekFrom::Start((header.unkf0_table_offset + 48) as _))?; + let h64_table_size: u64 = reader.read_le()?; + let real_h64_table_offset: u64 = reader.read_le()?; + reader.seek(SeekFrom::Current(-8 + real_h64_table_offset as i64 + 16))?; + reader.read_le_args(VecArgs { + count: h64_table_size as _, + inner: (), + })? + } else { + vec![] + }; + + let last_underscore_pos = path.rfind('_').unwrap(); + let path_base = path[..last_underscore_pos].to_owned(); + + Ok(PackageD2PreBL { + path_base, + reader: RefCell::new(Box::new(reader)), + gcm: RefCell::new(PkgGcmState::new(header.pkg_id)), + header, + entries, + blocks, + hashes, + block_counter: AtomicUsize::default(), + block_cache: Default::default(), + }) + } + + fn get_block_raw(&self, block_index: usize) -> anyhow::Result> { + let bh = &self.blocks[block_index]; + let mut data = vec![0u8; bh.size as usize]; + + if self.header.patch_id == bh.patch_id { + self.reader + .borrow_mut() + .seek(SeekFrom::Start(bh.offset as u64))?; + self.reader.borrow_mut().read_exact(&mut data)?; + } else { + // TODO(cohae): Can we cache these? + let mut f = + File::open(format!("{}_{}.pkg", self.path_base, bh.patch_id)).context(format!( + "Failed to open package file {}_{}.pkg", + self.path_base, bh.patch_id + ))?; + + f.seek(SeekFrom::Start(bh.offset as u64))?; + f.read_exact(&mut data)?; + }; + + Ok(Cow::Owned(data)) + } + + /// Reads, decrypts and decompresses the specified block + fn read_block(&self, block_index: usize) -> anyhow::Result> { + let bh = self.blocks[block_index].clone(); + let mut block_data = self.get_block_raw(block_index)?.to_vec(); + + if (bh.flags & 0x2) != 0 { + self.gcm + .borrow_mut() + .decrypt_block_in_place(bh.flags, &bh.gcm_tag, &mut block_data)?; + }; + + let decompressed_data = if (bh.flags & 0x1) != 0 { + let mut buffer = vec![0u8; BLOCK_SIZE]; + let _decompressed_size = oodle::decompress(&block_data, &mut buffer); + buffer + } else { + block_data + }; + + Ok(decompressed_data) + } +} + +impl Package for PackageD2PreBL { + fn endianness(&self) -> Endian { + Endian::Little // TODO(cohae): Not necessarily + } + + fn pkg_id(&self) -> u16 { + self.header.pkg_id + } + + fn patch_id(&self) -> u16 { + self.header.patch_id + } + + fn hashes64(&self) -> Vec { + self.hashes + .iter() + .map(|h| UHashTableEntry { + hash32: h.hash32, + hash64: h.hash64, + reference: h.reference, + }) + .collect() + } + + fn entries(&self) -> Vec { + self.entries + .iter() + .map(|e| UEntryHeader { + reference: e.reference, + file_type: e.file_type, + file_subtype: e.file_subtype, + starting_block: e.starting_block, + starting_block_offset: e.starting_block_offset, + file_size: e.file_size, + }) + .collect() + } + + fn entry(&self, index: usize) -> Option { + self.entries.get(index).map(|e| UEntryHeader { + reference: e.reference, + file_type: e.file_type, + file_subtype: e.file_subtype, + starting_block: e.starting_block, + starting_block_offset: e.starting_block_offset, + file_size: e.file_size, + }) + } + + fn get_block(&self, block_index: usize) -> anyhow::Result>> { + let (_, b) = match self.block_cache.borrow_mut().entry(block_index) { + Entry::Occupied(o) => o.get().clone(), + Entry::Vacant(v) => { + let block = self.read_block(*v.key())?; + let b = v + .insert((self.block_counter.load(Ordering::Relaxed), Arc::new(block))) + .clone(); + + self.block_counter.store( + self.block_counter.load(Ordering::Relaxed) + 1, + Ordering::Relaxed, + ); + + b + } + }; + + while self.block_cache.borrow().len() > BLOCK_CACHE_SIZE { + let bc = self.block_cache.borrow(); + let (oldest, _) = bc + .iter() + .min_by(|(_, (at, _)), (_, (bt, _))| at.cmp(bt)) + .unwrap(); + + let oldest = *oldest; + drop(bc); + + self.block_cache.borrow_mut().remove(&oldest); + } + + Ok(b) + } +} diff --git a/src/d2_prebl/mod.rs b/src/d2_prebl/mod.rs new file mode 100644 index 0000000..7fe9f2f --- /dev/null +++ b/src/d2_prebl/mod.rs @@ -0,0 +1,4 @@ +mod r#impl; +mod structs; + +pub use r#impl::PackageD2PreBL; diff --git a/src/structs.rs b/src/d2_prebl/structs.rs similarity index 85% rename from src/structs.rs rename to src/d2_prebl/structs.rs index 5a417b9..de18a37 100644 --- a/src/structs.rs +++ b/src/d2_prebl/structs.rs @@ -1,4 +1,5 @@ -use binrw::BinRead; +use crate::TagHash; +use binrw::{BinRead, BinWrite}; use std::fmt::Debug; use std::io::SeekFrom; @@ -27,6 +28,9 @@ pub struct PackageHeader { #[br(seek_before = SeekFrom::Start(0xd0))] pub block_table_size: u32, + #[br(seek_before = SeekFrom::Start(0xf0))] + pub unkf0_table_offset: u32, + #[br(seek_before = SeekFrom::Start(0x110))] #[br(map(|v: u32| v + 96))] pub entry_table_offset: u32, @@ -67,3 +71,10 @@ pub struct BlockHeader { pub hash: [u8; 20], pub gcm_tag: [u8; 16], } + +#[derive(BinRead, BinWrite, Debug, Clone)] +pub struct HashTableEntry { + pub hash64: u64, + pub hash32: TagHash, + pub reference: TagHash, +} diff --git a/src/lib.rs b/src/lib.rs index 121c54e..b6952ea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,16 @@ extern crate core; mod crypto; +mod d1_legacy; +mod d2_prebl; +pub mod manager; mod oodle; pub mod package; -pub mod structs; +pub mod tag; +pub use d2_prebl::PackageD2PreBL; + +pub use manager::PackageManager; pub use package::Package; +pub use package::PackageVersion; +pub use tag::TagHash; diff --git a/src/manager.rs b/src/manager.rs new file mode 100644 index 0000000..8cb85ab --- /dev/null +++ b/src/manager.rs @@ -0,0 +1,155 @@ +use crate::package::{Package, PackageVersion, UEntryHeader}; +use crate::TagHash; +use nohash_hasher::IntMap; +use rayon::prelude::*; +use std::collections::hash_map::Entry; +use std::fs; +use std::path::Path; +use std::sync::Arc; + +#[derive(Clone, Copy)] +pub struct HashTableEntryShort { + pub hash32: TagHash, + pub reference: TagHash, +} + +pub struct PackageManager { + pub package_paths: IntMap, + pub version: PackageVersion, + + // TODO(cohae): Should these be grouped by package? + /// Every entry + pub package_entry_index: IntMap>, + pub hash64_table: IntMap, + + /// Packages that are currently open for reading + pkgs: IntMap>, +} + +impl PackageManager { + pub fn new>( + packages_dir: P, + version: PackageVersion, + build_index: bool, + ) -> anyhow::Result { + let path = packages_dir.as_ref(); + // Every package in the given directory, including every patch + let mut packages_all = vec![]; + for entry in fs::read_dir(path)? { + let entry = entry?; + let path = entry.path(); + if path.is_file() { + packages_all.push(path.to_string_lossy().to_string()); + } + } + + packages_all.sort(); + + // All the latest packages + let mut packages: IntMap = Default::default(); + for p in packages_all { + let parts: Vec<&str> = p.split("_").collect(); + if let Some(Ok(pkg_id)) = parts + .get(parts.len() - 2) + .map(|s| u16::from_str_radix(s, 16)) + { + packages.insert(pkg_id, p); + } else { + // Take the long route and extract the package ID from the header + if let Ok(pkg) = version.open(&p) { + packages.insert(pkg.pkg_id(), p); + } + } + } + + let mut s = Self { + package_paths: packages, + version, + package_entry_index: Default::default(), + hash64_table: Default::default(), + pkgs: Default::default(), + }; + + if build_index { + s.rebuild_tables(); + } + + Ok(s) + } + + pub fn rebuild_tables(&mut self) { + let (entries, hashes): (IntMap>, Vec<_>) = self + .package_paths + .par_iter() + .map(|(_, p)| { + let pkg = self.version.open(p).unwrap(); + let entries = (pkg.pkg_id(), pkg.entries()); + + let hashes = (pkg + .hashes64() + .iter() + .map(|h| { + ( + h.hash64, + HashTableEntryShort { + hash32: h.hash32, + reference: h.reference, + }, + ) + }) + .collect::>(),); + + (entries, hashes) + }) + .unzip(); + + self.package_entry_index = entries; + self.hash64_table = hashes.iter().flat_map(|(v,)| v.clone()).collect(); + + println!("Loaded {} packages", self.package_entry_index.len()); + } + + pub fn get_all_by_reference(&self, reference: u32) -> Vec<(u16, usize, UEntryHeader)> { + self.package_entry_index + .par_iter() + .map(|(p, e)| { + e.iter() + .enumerate() + .filter(|(_, e)| e.reference == reference) + .map(|(i, e)| (*p, i, e.clone())) + .collect::>() + }) + .flatten() + .collect() + } + + fn get_or_load_pkg(&mut self, pkg_id: u16) -> anyhow::Result> { + Ok(match self.pkgs.entry(pkg_id) { + Entry::Occupied(o) => o.get().clone(), + Entry::Vacant(v) => v + .insert( + self.version + .open(self.package_paths.get(&pkg_id).ok_or_else(|| { + anyhow::anyhow!("Couldn't get a path for package id {pkg_id:04x}") + })?)?, + ) + .clone(), + }) + } + + pub fn read_entry(&mut self, pkg_id: u16, index: usize) -> anyhow::Result> { + Ok(self.get_or_load_pkg(pkg_id)?.read_entry(index)?.to_vec()) + } + + pub fn read_tag(&mut self, tag: TagHash) -> anyhow::Result> { + self.read_entry(tag.pkg_id(), tag.entry_index() as usize) + } + + pub fn get_entry_by_tag(&mut self, tag: TagHash) -> anyhow::Result { + self.get_or_load_pkg(tag.pkg_id())? + .entries() + .get(tag.entry_index() as usize) + .cloned() + .ok_or_else(|| anyhow::anyhow!("Entry does not exist in pkg {:04x}", tag.pkg_id())) + } +} diff --git a/src/package.rs b/src/package.rs index 44bbefc..6e7a6d7 100644 --- a/src/package.rs +++ b/src/package.rs @@ -1,141 +1,88 @@ -use crate::crypto::PkgGcmState; -use crate::oodle; -use crate::structs::{BlockHeader, EntryHeader, PackageHeader}; -use anyhow::{anyhow, Context}; -use binrw::{BinReaderExt, VecArgs}; -use nohash_hasher::IntMap; -use std::borrow::Cow; -use std::cell::RefCell; -use std::collections::hash_map::Entry; -use std::fs::File; -use std::io::{BufReader, Read, Seek, SeekFrom}; -use std::rc::Rc; -use std::slice::Iter; - -pub const BLOCK_SIZE: usize = 0x40000; +use crate::d1_legacy::PackageD1Legacy; +use crate::{PackageD2PreBL, TagHash}; +use anyhow::{anyhow, ensure}; +use std::io::{Read, Seek}; +use std::sync::Arc; + +pub const BLOCK_CACHE_SIZE: usize = 128; pub trait ReadSeek: Read + Seek {} impl ReadSeek for R {} -pub struct Package { - gcm: RefCell, +#[derive(Clone)] +pub struct UEntryHeader { + pub reference: u32, + pub file_type: u8, + pub file_subtype: u8, + pub starting_block: u32, + pub starting_block_offset: u32, + pub file_size: u32, +} - pub header: PackageHeader, - entries: Vec, - blocks: Vec, +#[derive(Clone)] +pub struct UHashTableEntry { + pub hash64: u64, + pub hash32: TagHash, + pub reference: TagHash, +} - reader: RefCell>, - path_base: String, +#[derive(clap::ValueEnum, PartialEq, Debug, Clone)] +pub enum PackageVersion { + /// PS3/X360 version of Destiny (The Taken King) + #[value(name = "d1_legacy")] + DestinyLegacy, - block_cache: RefCell>>>, -} + /// The latest version of Destiny (Rise of Iron) + #[value(name = "d1")] + Destiny, -impl Package { - pub fn open(path: &str) -> anyhow::Result { - let reader = BufReader::new(File::open(path)?); + /// The last version of Destiny before Beyond Light (Shadowkeep/Season of Arrivals) + #[value(name = "d2_prebl")] + Destiny2PreBeyondLight, - Self::from_reader(path, reader) - } + /// The latest version of Destiny 2 (currently Lightfall) + #[value(name = "d2")] + Destiny2, +} - pub fn from_reader(path: &str, reader: R) -> anyhow::Result { - let mut reader = reader; - let header: PackageHeader = reader.read_le()?; - - reader.seek(SeekFrom::Start(header.entry_table_offset as u64 - 16))?; - let entry_table_size_bytes = reader.read_le::()? * 16; - - reader.seek(SeekFrom::Start(header.entry_table_offset as _))?; - let entries = reader.read_le_args(VecArgs { - count: header.entry_table_size as _, - inner: (), - })?; - - reader.seek(SeekFrom::Start( - (header.entry_table_offset + entry_table_size_bytes + 32) as _, - ))?; - let blocks = reader.read_le_args(VecArgs { - count: header.block_table_size as _, - inner: (), - })?; - - let last_underscore_pos = path.rfind('_').unwrap(); - let path_base = path[..last_underscore_pos].to_owned(); - - Ok(Package { - path_base, - reader: RefCell::new(Box::new(reader)), - gcm: RefCell::new(PkgGcmState::new(header.pkg_id)), - header, - entries, - blocks, - block_cache: Default::default(), +impl PackageVersion { + pub fn open(&self, path: &str) -> anyhow::Result> { + Ok(match self { + PackageVersion::DestinyLegacy => Arc::new(PackageD1Legacy::open(path)?), + PackageVersion::Destiny => { + anyhow::bail!("The latest version of Destiny is not supported yet") + } + PackageVersion::Destiny2PreBeyondLight => Arc::new(PackageD2PreBL::open(path)?), + PackageVersion::Destiny2 => { + anyhow::bail!("The latest version of Destiny 2 is not supported yet") + } }) } +} - pub fn entries(&self) -> Iter { - self.entries.iter() - } +// TODO(cohae): Package language +pub trait Package { + fn endianness(&self) -> binrw::Endian; - fn get_block_raw(&self, block_index: usize) -> anyhow::Result> { - let bh = &self.blocks[block_index]; - let mut data = vec![0u8; bh.size as usize]; - - if self.header.patch_id == bh.patch_id { - self.reader - .borrow_mut() - .seek(SeekFrom::Start(bh.offset as u64))?; - self.reader.borrow_mut().read_exact(&mut data)?; - } else { - let mut f = - File::open(format!("{}_{}.pkg", self.path_base, bh.patch_id)).context(format!( - "Failed to open package file {}_{}.pkg", - self.path_base, bh.patch_id - ))?; - - f.seek(SeekFrom::Start(bh.offset as u64))?; - f.read_exact(&mut data)?; - }; - - Ok(Cow::Owned(data)) - } + fn pkg_id(&self) -> u16; + fn patch_id(&self) -> u16; - /// Reads, decrypts and decompresses the specified block - fn read_block(&self, block_index: usize) -> anyhow::Result> { - let bh = self.blocks[block_index].clone(); - let mut block_data = self.get_block_raw(block_index)?.to_vec(); - - if (bh.flags & 0x2) != 0 { - self.gcm - .borrow_mut() - .decrypt_block_in_place(bh.flags, &bh.gcm_tag, &mut block_data)?; - }; - - let decompressed_data = if (bh.flags & 0x1) != 0 { - let mut buffer = vec![0u8; BLOCK_SIZE]; - let _decompressed_size = oodle::decompress(&block_data, &mut buffer); - buffer - } else { - block_data - }; - - Ok(decompressed_data) - } + /// Every hash64 in this package. + /// Does not apply to Destiny 1 + fn hashes64(&self) -> Vec; - /// Gets the specified block from the cache or reads it - pub fn get_block(&self, block_index: usize) -> anyhow::Result>> { - Ok(match self.block_cache.borrow_mut().entry(block_index) { - Entry::Occupied(o) => o.get().clone(), - Entry::Vacant(v) => { - let block = self.read_block(*v.key())?; - v.insert(Rc::new(block)).clone() - } - }) - } + fn entries(&self) -> Vec; - pub fn read_entry(&self, index: usize) -> anyhow::Result> { + fn entry(&self, index: usize) -> Option; + + /// Gets/reads a specific block from the file. + /// It's recommended that the implementation caches blocks to prevent re-reads + fn get_block(&self, index: usize) -> anyhow::Result>>; + + /// Reads the entire specified entry's data + fn read_entry(&self, index: usize) -> anyhow::Result> { let entry = self - .entries - .get(index) + .entry(index) .ok_or(anyhow!("Entry index is out of range"))?; let mut buffer = Vec::with_capacity(entry.file_size as usize); @@ -173,6 +120,33 @@ impl Package { current_block += 1; } - Ok(Cow::Owned(buffer)) + Ok(buffer) + } + + /// Reads the entire specified entry's data + /// Tag needs to be in this package + fn read_tag(&self, tag: TagHash) -> anyhow::Result> { + ensure!(tag.pkg_id() == self.pkg_id()); + self.read_entry(tag.entry_index() as _) + } + + fn get_all_by_reference(&self, reference: u32) -> Vec<(usize, UEntryHeader)> { + self.entries() + .iter() + .enumerate() + .filter(|(_, e)| e.reference == reference) + .map(|(i, e)| (i, e.clone())) + .collect() + } + + fn get_all_by_type(&self, etype: u8, esubtype: Option) -> Vec<(usize, UEntryHeader)> { + self.entries() + .iter() + .enumerate() + .filter(|(_, e)| { + e.file_type == etype && esubtype.map(|t| t == e.file_subtype).unwrap_or(true) + }) + .map(|(i, e)| (i, e.clone())) + .collect() } } diff --git a/src/tag.rs b/src/tag.rs new file mode 100644 index 0000000..eb2d6fd --- /dev/null +++ b/src/tag.rs @@ -0,0 +1,66 @@ +use binrw::{BinRead, BinWrite}; +use std::fmt::{Debug, Display, Formatter}; + +#[derive(BinRead, BinWrite, Copy, Clone, PartialEq, PartialOrd, Hash, Eq)] +pub struct TagHash(pub u32); + +impl From for u32 { + fn from(value: TagHash) -> Self { + value.0 + } +} + +impl From for TagHash { + fn from(value: u32) -> Self { + Self(value) + } +} + +impl From<(u16, u16)> for TagHash { + fn from((pkg_id, index): (u16, u16)) -> Self { + Self::new(pkg_id, index) + } +} + +impl TagHash { + pub fn new(pkg_id: u16, entry: u16) -> TagHash { + TagHash(0x80800000 | ((pkg_id as u32) << 13) | entry as u32) + } + + pub fn is_valid(&self) -> bool { + self.0 != u32::MAX && (self.0 > 0x80800000) + } + + /// Does this hash look like a pkg hash? + pub fn is_pkg_file(&self) -> bool { + self.is_valid() && (0x10..0xa00).contains(&self.pkg_id()) + } + + pub fn pkg_id(&self) -> u16 { + ((self.0 - 0x80800000) >> 13) as u16 + } + + pub fn entry_index(&self) -> u16 { + ((self.0 & 0x1fff) % 8192) as u16 + } +} + +impl Debug for TagHash { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + if !self.is_valid() { + f.write_fmt(format_args!("TagHash(INVALID(0x{:x}))", self.0)) + } else { + f.write_fmt(format_args!( + "TagHash(pkg={:04x}, entry={})", + self.pkg_id(), + self.entry_index() + )) + } + } +} + +impl Display for TagHash { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!("TagHash(0x{:x})", self.0)) + } +}