Skip to content

Commit

Permalink
Many fixes to compressor, decompressor now works, added a benchmark h…
Browse files Browse the repository at this point in the history 
…arness
  • Loading branch information
@maxim-zhao
maxim-zhao committed Nov 13, 2022
1 parent e8d159b commit dd4cd71
Show file tree
Hide file tree
Showing 6 changed files with 164 additions and 139 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,4 @@ benchmark/*.sms
benchmark/data.*
benchmark/expected.bin
/benchmark/venv
/compressors/data.sonic2compr
28 changes: 28 additions & 0 deletions benchmark/benchmark-sonic2.sms.asm
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
; { "technology": "Sonic 2", "extension": "sonic2compr" }

.memorymap
defaultslot 0
slotsize $4000
slot 0 $0000
.endme

.rombankmap
bankstotal 1
banksize $4000
banks 1
.endro

.bank 0 slot 0

.org 0
ld hl,data
ld de,$4000
call Sonic2TileLoader_Decompress
ret ; ends the test

.define Sonic2TileLoaderMemory $c000
.block "decompressor"
.include "../decompressors/Sonic 2 decompressor.asm"
.endb

data: .incbin "data.sonic2compr"
2 changes: 1 addition & 1 deletion benchmark/benchmark.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import statistics
import numpy

WLA_PATH = "C:\\Users\\maxim\\Documents\\Code\\C\\wla-dx\\binaries"
WLA_PATH = "C:\\Users\\maxim\\Documents\\Code\\C\\wla-dx\\binaries\\Release"
BMP2TILE_PATH = "C:\\Users\\maxim\\Documents\\Code\\C#\\bmp2tile"
Z80BENCH_PATH = "C:\\Users\\maxim\\Documents\\Code\\C#\\z80bench\\bin\\Release"

Expand Down
217 changes: 113 additions & 104 deletions compressors/gfxcomp_sonic2.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,7 @@ void writeWord(std::vector<uint8_t>& buffer, const uint16_t word)
buffer.push_back((word >> 8) & 0xff);
}

bool allZero(const std::vector<uint8_t>& tile)
{
return std::ranges::all_of(
tile,
[](auto x)
{
return x == 0;
});
}

bool shouldCompress(const std::vector<uint8_t>& tile)
int countZeroes(const std::vector<uint8_t>& tile)
{
int count = 0;
for (const uint8_t value : tile)
Expand All @@ -43,13 +33,14 @@ bool shouldCompress(const std::vector<uint8_t>& tile)
++count;
}
}
return count > 4;
return count;
}

std::vector<uint8_t> xorTile(const std::vector<uint8_t>& tile)
{
std::vector<uint8_t> copy(tile);
for (int i = 0; i < 7; ++i)
// The original works forwards so we work backwards...
auto copy(tile);
for (int i = 12; i >= 0; i -= 2)
{
copy[i + 2] ^= copy[i + 0];
copy[i + 3] ^= copy[i + 1];
Expand All @@ -59,24 +50,27 @@ std::vector<uint8_t> xorTile(const std::vector<uint8_t>& tile)
return copy;
}


std::vector<uint8_t> compress(const std::vector<uint8_t>& tile)
{
std::vector<uint8_t> result;
// Emit bitmask for zero bits
uint32_t bitmask = 0;
uint8_t bitmask = 1;
uint8_t accumulator = 0;
for (const uint8_t value : tile)
{
bitmask <<= 1;
if (value != 0)
{
bitmask &= 1;
accumulator |= bitmask;
}
bitmask <<= 1;
if (bitmask == 0)
{
// Emit byte
result.push_back(accumulator);
bitmask = 1;
accumulator = 0;
}
}
result.push_back((bitmask >> 0) & 0xff);
result.push_back((bitmask >> 8) & 0xff);
result.push_back((bitmask >> 16) & 0xff);
result.push_back((bitmask >> 24) & 0xff);
// Then the non-zero bytes
for (const uint8_t value : tile)
{
Expand All @@ -98,102 +92,117 @@ extern "C" __declspec(dllexport) int compressTiles(
{
return -1; // error
}
std::vector<uint8_t> destination; // the output
destination.reserve(destinationLength); // avoid reallocation

// Format is:
// dw 0001 ; meaningless header (little-endian)
// dw TileCount ; Tile count (little-endian)
// dw OffsetToBitStream ; Offset of stream 2, relative to start of data
// dsb n Data ; stream 1
// dsb n CompressionData ; stream 2
// Compression data holds 2 bits per tile, right-aligned in CompressionData:
// 00 = all zeroes
// 01 = raw, copy 32 bytes from Data
// 02 = compressed tile, see below
// 03 = XORed compressed data, see below
// Compressed data then consists of a run of bytes in Data:
// 4B: 32 bits, 1 = emit a byte from Data, 0 = emit a 0
// So it makes sense to use this is more than 4 bytes of the 32 are 0.
// XOR compressed data is the same except after decoding, the bytes are XORed against each other within bitplanes.
// So we need to pre-process the same way to check if it yields some 0s.

// First we make the header
writeWord(destination, 0x0001);
writeWord(destination, static_cast<uint16_t>(numTiles));
writeWord(destination, 0x0000); // Offset to fill in later

// Then we make a buffer for the compression bitstream. This will hold a byte per tile which we'll squash later.
std::vector<uint8_t> bitStream;
bitStream.reserve(numTiles);

// Then for each tile...
std::vector<uint8_t> tile(32);
for (auto i = 0u; i < numTiles; ++i)
{
// Copy to temp buffer
std::copy_n(pSource, 32, tile.begin());

// If all 0, it's easy
if (allZero(tile))
{
bitStream.push_back(0);
}
else if (shouldCompress(tile))
{
bitStream.push_back(2);
const auto& compressed = compress(tile);
std::ranges::copy(compressed, destination.end());
}
else
try
{
std::vector<uint8_t> destination; // the output
destination.reserve(destinationLength); // avoid reallocation

// Format is:
// dw 0001 ; meaningless header (little-endian)
// dw TileCount ; Tile count (little-endian)
// dw OffsetToBitStream ; Offset of stream 2, relative to start of data
// dsb n Data ; stream 1
// dsb n CompressionData ; stream 2
// Compression data holds 2 bits per tile, right-aligned in CompressionData:
// 00 = all zeroes
// 01 = raw, copy 32 bytes from Data
// 02 = compressed tile, see below
// 03 = XORed compressed data, see below
// Compressed data then consists of a run of bytes in Data:
// 4B: 32 bits, 1 = emit a byte from Data, 0 = emit a 0
// So it makes sense to use this is more than 4 bytes of the 32 are 0.
// XOR compressed data is the same except after decoding, the bytes are XORed against each other within bitplanes.
// So we need to pre-process the opposite way to check if it yields more 0s.

// First we make the header
writeWord(destination, 0x0001);
writeWord(destination, static_cast<uint16_t>(numTiles));
writeWord(destination, 0x0000); // Offset to fill in later

// Then we make a buffer for the compression bitstream. This will hold a byte per tile which we'll squash later.
std::vector<uint8_t> bitStream;
bitStream.reserve(numTiles);

// Then for each tile...
std::vector<uint8_t> tile(32);
for (auto i = 0u; i < numTiles; ++i)
{
if (const auto& xored = xorTile(tile); shouldCompress(xored))
// Copy to temp buffer
std::copy_n(pSource, 32, tile.begin());
// Move pointer on
pSource += 32;

const int zeroCount = countZeroes(tile);
if (zeroCount == 32)
{
bitStream.push_back(3);
const auto& compressed = compress(xored);
std::ranges::copy(compressed, destination.end());
// All 0
bitStream.push_back(0);
continue;
}
else

const auto& xored = xorTile(tile);
const int xoredZeroCount = countZeroes(xored);
if (zeroCount <= 4 && xoredZeroCount <= 4)
{
// Uncompressed
// Uncompressed as compression will not save space
bitStream.push_back(1);
std::ranges::copy(tile, destination.end());
std::ranges::copy(tile, std::back_inserter(destination));
continue;
}

if (zeroCount >= xoredZeroCount)
{
// Compressed
bitStream.push_back(2);
const auto& compressed = compress(tile);
std::ranges::copy(compressed, std::back_inserter(destination));
}
else
{
// XORed compressed
bitStream.push_back(3);
const auto& compressed = compress(xored);
std::ranges::copy(compressed, std::back_inserter(destination));
}
}
}

// Update the offset
const auto offsetOfBitstream = static_cast<uint16_t>(destination.size());
destination[4] = (offsetOfBitstream >> 0) & 0xff;
destination[5] = (offsetOfBitstream >> 8) & 0xff;
// Update the offset
const auto offsetOfBitstream = static_cast<uint16_t>(destination.size());
destination[4] = (offsetOfBitstream >> 0) & 0xff;
destination[5] = (offsetOfBitstream >> 8) & 0xff;

// And then append it
int nibbleCount = 0;
uint8_t accumulator = 0;
for (const uint8_t b : bitStream)
{
accumulator <<= 2;
accumulator |= b;
if (++nibbleCount == 4)
// And then append it
int nibbleCount = 0;
uint8_t accumulator = 0;
for (const uint8_t b : bitStream)
{
accumulator |= b << (nibbleCount * 2);
if (++nibbleCount == 4)
{
destination.push_back(accumulator);
accumulator = 0;
nibbleCount = 0;
}
}
if (nibbleCount != 0)
{
// Left over bits in accumulator
destination.push_back(accumulator);
accumulator = 0;
nibbleCount = 0;
}
}
if (nibbleCount != 0)
{
// Left over bits in accumulator
destination.push_back(accumulator);
}

// check length
if (destination.size() > destinationLength)
// check length
if (destination.size() > destinationLength)
{
return 0;
}
// copy to dest
memcpy_s(pDestination, destinationLength, destination.data(), destination.size());
// return length
return static_cast<int>(destination.size());
}
catch (const std::exception&)
{
return 0;
return -1;
}
// copy to dest
memcpy_s(pDestination, destinationLength, destination.data(), destination.size());
// return length
return static_cast<int>(destination.size());
}
3 changes: 2 additions & 1 deletion compressors/tilecompressordlls.sln.DotSettings
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,5 @@
<s:Boolean x:Key="/Default/UserDictionary/Words/=optimise/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Pucrunch/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Pucrunch_0027s/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=tilemap/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>
<s:Boolean x:Key="/Default/UserDictionary/Words/=tilemap/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=xored/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>
Loading

0 comments on commit dd4cd71

Please sign in to comment.