Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

mkclean: add Zstd compression support #150

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
+271 −7
Open

mkclean: add Zstd compression support #150

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0460ed1
libmatroska2: add Zstandard compression support
robUx4 Oct 25, 2024
9906093
mkclean: add support for Zstd compression via new --compalgo option
robUx4 Dec 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions libmatroska2/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ project("matroska2" VERSION 0.22.3 LANGUAGES C)
option(CONFIG_ZLIB "Enable zlib (de)compression" ON)
option(CONFIG_BZLIB "Enable bzlib decompression in libmatroska2" ON)
option(CONFIG_LZO1X "Enable lzo decompression in libmatroska2" ON)
option(CONFIG_ZSTD "Enable Zstandard/zstd (de)compression in libmatroska2" ON)
option(CONFIG_NOCODEC_HELPER "Enable Vorbis frame durations in libmatroska2" ON)

if (CONFIG_ZLIB)
Expand Down Expand Up @@ -66,6 +67,28 @@ if (CONFIG_BZLIB)
add_subdirectory("bzip2")
target_link_libraries("matroska2" PRIVATE $<BUILD_INTERFACE:bzlib>)
endif(CONFIG_BZLIB)
if (CONFIG_ZSTD)
include(FetchContent)

set(ZSTD_BUILD_STATIC ON)
set(ZSTD_BUILD_SHARED OFF)
set(ZSTD_BUILD_PROGRAMS OFF)
set(ZSTD_BUILD_TESTS OFF)
set(ZSTD_LEGACY_SUPPORT OFF)

FetchContent_Declare(
zstd
URL "https://github.com/facebook/zstd/releases/download/v1.5.6/zstd-1.5.6.tar.gz"
URL_HASH SHA512=54a578f2484da0520a6e9a24f501b9540a3fe3806785d6bc9db79fc095b7c142a7c121387c7eecd460ca71446603584ef1ba4d29a33ca90873338c9ffbd04f14
DOWNLOAD_EXTRACT_TIMESTAMP TRUE
SOURCE_SUBDIR build/cmake
)

FetchContent_MakeAvailable(zstd)

target_include_directories("matroska2" PRIVATE ${zstd_SOURCE_DIR}/lib )
target_link_libraries("matroska2" PRIVATE $<BUILD_INTERFACE:libzstd_static>)
endif(CONFIG_ZSTD)
if (NOT CONFIG_NOCODEC_HELPER)
add_subdirectory("tremor")
target_link_libraries("matroska2" PRIVATE $<BUILD_INTERFACE:tremor>)
Expand Down
8 changes: 8 additions & 0 deletions libmatroska2/matroska2/matroska.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,14 @@ static INLINE err_t CompressFrameZLib(const uint8_t *Cursor, size_t CursorSize,
return ERR_NOT_SUPPORTED;
}
#endif // !CONFIG_ZLIB
#if defined(CONFIG_ZSTD)
MATROSKA_DLL err_t CompressFrameZstd(const uint8_t *Cursor, size_t CursorSize, uint8_t **OutBuf, size_t *OutSize);
#else // !CONFIG_ZLIB
static INLINE err_t CompressFrameZstd(const uint8_t *Cursor, size_t CursorSize, uint8_t **OutBuf, size_t *OutSize)
{
return ERR_NOT_SUPPORTED;
}
#endif // !CONFIG_ZLIB
#endif

MATROSKA_DLL void MATROSKA_ClusterSort(matroska_cluster *Cluster); // not good with P frames!!!
Expand Down
1 change: 1 addition & 0 deletions libmatroska2/matroska2/matroska_sem.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,7 @@ typedef enum {
MATROSKA_TRACK_ENCODING_COMP_BZLIB = 1, // bzip2 compression (BZIP2) **SHOULD NOT** be used.
MATROSKA_TRACK_ENCODING_COMP_LZO1X = 2, // Lempel-Ziv-Oberhumer compression (LZO) **SHOULD NOT** be used.
MATROSKA_TRACK_ENCODING_COMP_HEADERSTRIP = 3, // Octets in `ContentCompSettings` ((#contentcompsettings-element)) have been stripped from each frame.
MATROSKA_TRACK_ENCODING_COMP_ZSTD = 4, // Zstandard (zstd).
} MatroskaTrackEncodingCompAlgo;

/**
Expand Down
1 change: 1 addition & 0 deletions libmatroska2/matroska_config.h.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#cmakedefine CONFIG_ZLIB
#cmakedefine CONFIG_LZO1X
#cmakedefine CONFIG_BZLIB
#cmakedefine CONFIG_ZSTD
#cmakedefine CONFIG_NOCODEC_HELPER

#define LIBMATROSKA2_PROJECT_VERSION T("@matroska2_VERSION_MAJOR@.@matroska2_VERSION_MINOR@.@matroska2_VERSION_PATCH@")
Expand Down
200 changes: 196 additions & 4 deletions libmatroska2/matroskamain.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@
#if defined(CONFIG_LZO1X)
#include "minilzo.h"
#endif
#if defined(CONFIG_ZSTD)
#include <zstd.h>
#endif
#include <corec/helpers/file/streams.h>
#include <corec/str/str.h>

Expand Down Expand Up @@ -223,6 +226,94 @@ err_t CompressFrameZLib(const uint8_t *Cursor, size_t CursorSize, uint8_t **OutB
#endif // CONFIG_EBML_WRITING
#endif // CONFIG_ZLIB

#if defined(CONFIG_ZSTD)
err_t UnCompressFrameZstd(const uint8_t *Cursor, size_t CursorSize, array *OutBuf, size_t *FrameSize, size_t *ArrayOffset)
{
uint8_t Buffer[4+14]; // 14: max frame header size
STORE32LE(Buffer, ZSTD_MAGICNUMBER);
memcpy(&Buffer[4], Cursor, min(sizeof(Buffer)-4, CursorSize));

unsigned long long outSize = ZSTD_getFrameContentSize(Buffer, sizeof(Buffer));
if (ZSTD_isError(outSize))
return ERR_INVALID_DATA;
if (!ArrayResize(OutBuf, outSize, 0))
return ERR_OUT_OF_MEMORY;

err_t Err = ERR_NONE;
ZSTD_DStream *dstream = ZSTD_createDStream();
if (dstream == NULL)
return ERR_OUT_OF_MEMORY;
size_t Count;
Count = ZSTD_initDStream(dstream);
if (ZSTD_isError(Count))
Err = ERR_INVALID_DATA;
else
{
ZSTD_inBuffer in;
ZSTD_outBuffer out;
out.dst = ARRAYBEGIN(*OutBuf,uint8_t);
out.size = ARRAYCOUNT(*OutBuf,uint8_t);
out.pos = 0;

// feed the header
in.src = Buffer;
in.size = 4;
in.pos = 0;
Count = ZSTD_decompressStream(dstream, &out, &in);
if (ZSTD_isError(Count))
Err = ERR_INVALID_DATA;
else
{
// feed the data
in.src = Cursor;
in.size = CursorSize;
in.pos = 0;
Count = ZSTD_decompressStream(dstream, &out, &in);
if (ZSTD_isError(Count))
Err = ERR_INVALID_DATA;
else
*FrameSize = out.size;
}
}
ZSTD_freeDStream(dstream);
return Err;
}

#if defined(CONFIG_EBML_WRITING)
err_t CompressFrameZstd(const uint8_t *Cursor, size_t CursorSize, uint8_t **OutBuf, size_t *OutSize)
{
ZSTD_CCtx* cctx = ZSTD_createCCtx();
if (cctx == NULL)
return ERR_OUT_OF_MEMORY;
void *output;
if (OutBuf)
output = *OutBuf;
else
{
output = malloc(CursorSize + 14); // extra header in case the compression is bad
if (output == NULL)
{
ZSTD_freeCCtx(cctx);
return ERR_OUT_OF_MEMORY;
}
*OutSize += 14;
}
size_t result = ZSTD_compressCCtx(cctx, output, *OutSize, Cursor, CursorSize, ZSTD_CLEVEL_DEFAULT);
ZSTD_freeCCtx(cctx);
assert(ZSTD_isError(result) || *((fourcc_t*)output) == ZSTD_MAGICNUMBER);
if (!OutBuf)
free(output);
if (ZSTD_isError(result))
return ERR_WRITE;
// strip the magic number
if (OutBuf)
*OutBuf = *OutBuf + 4;
*OutSize = result - 4;
return ERR_NONE;
}
#endif // CONFIG_EBML_WRITING
#endif // CONFIG_ZSTD

static err_t CheckCompression(matroska_block *Block, int ForProfile)
{
ebml_master *Elt, *Header;
Expand Down Expand Up @@ -259,6 +350,10 @@ static err_t CheckCompression(matroska_block *Block, int ForProfile)
#if defined(CONFIG_BZLIB)
else if (CompressionAlgo == MATROSKA_TRACK_ENCODING_COMP_BZLIB)
CanDecompress = 1;
#endif
#if defined(CONFIG_ZSTD)
else if (CompressionAlgo == MATROSKA_TRACK_ENCODING_COMP_ZSTD)
CanDecompress = 1;
#endif
if (!CanDecompress)
return ERR_INVALID_DATA;
Expand Down Expand Up @@ -1023,6 +1118,10 @@ err_t MATROSKA_BlockReadData(matroska_block *Element, struct stream *Input, int
#if defined(CONFIG_BZLIB)
else if (CompressionAlgo == MATROSKA_TRACK_ENCODING_COMP_BZLIB)
CanDecompress = 1;
#endif
#if defined(CONFIG_ZSTD)
else if (CompressionAlgo == MATROSKA_TRACK_ENCODING_COMP_ZSTD)
CanDecompress = 1;
#endif
if (!CanDecompress)
return ERR_INVALID_DATA;
Expand All @@ -1032,7 +1131,7 @@ err_t MATROSKA_BlockReadData(matroska_block *Element, struct stream *Input, int
}
}

#if !defined(CONFIG_ZLIB) && !defined(CONFIG_LZO1X) && !defined(CONFIG_BZLIB)
#if !defined(CONFIG_ZLIB) && !defined(CONFIG_LZO1X) && !defined(CONFIG_BZLIB) && !defined(CONFIG_ZSTD)
if (Header && Header->Context==MATROSKA_getContextContentCompAlgo())
return ERR_NOT_SUPPORTED;
#endif
Expand All @@ -1046,7 +1145,7 @@ err_t MATROSKA_BlockReadData(matroska_block *Element, struct stream *Input, int
switch (Element->Lacing)
{
case LACING_NONE:
#if defined(CONFIG_ZLIB) || defined(CONFIG_LZO1X) || defined(CONFIG_BZLIB)
#if defined(CONFIG_ZLIB) || defined(CONFIG_LZO1X) || defined(CONFIG_BZLIB) || defined(CONFIG_ZSTD)
if (Header && Header->Context==MATROSKA_getContextContentCompAlgo())
{
// zlib handling, read the buffer in temp memory
Expand Down Expand Up @@ -1075,6 +1174,19 @@ err_t MATROSKA_BlockReadData(matroska_block *Element, struct stream *Input, int
}
}
#endif
#if defined(CONFIG_ZSTD)
if (EBML_IntegerValue((ebml_integer*)Header)==MATROSKA_TRACK_ENCODING_COMP_ZSTD)
{
size_t UncompressedSize;
size_t Offset = 0;
Err = UnCompressFrameZstd(InBuf, ARRAYBEGIN(Element->SizeList,int32_t)[0], &Element->Data, &UncompressedSize, &Offset);
if (Err == ERR_NONE)
{
ArrayResize(&Element->Data, UncompressedSize, 0);
ARRAYBEGIN(Element->SizeList,int32_t)[0] = UncompressedSize;
}
}
#endif
#if defined(CONFIG_LZO1X)
if (EBML_IntegerValue((ebml_integer*)Header)==MATROSKA_TRACK_ENCODING_COMP_LZO1X)
{
Expand Down Expand Up @@ -1179,7 +1291,7 @@ err_t MATROSKA_BlockReadData(matroska_block *Element, struct stream *Input, int
BufSize = 0;
for (NumFrame=0;NumFrame<ARRAYCOUNT(Element->SizeList,int32_t);++NumFrame)
BufSize += ARRAYBEGIN(Element->SizeList,int32_t)[NumFrame];
#if defined(CONFIG_ZLIB) || defined(CONFIG_LZO1X) || defined(CONFIG_BZLIB)
#if defined(CONFIG_ZLIB) || defined(CONFIG_LZO1X) || defined(CONFIG_BZLIB) || defined(CONFIG_ZSTD)
if (Header && Header->Context==MATROSKA_getContextContentCompAlgo())
{
// zlib handling, read the buffer in temp memory
Expand Down Expand Up @@ -1219,6 +1331,19 @@ err_t MATROSKA_BlockReadData(matroska_block *Element, struct stream *Input, int
}
}
#endif
#if defined(CONFIG_ZSTD)
if (EBML_IntegerValue((ebml_integer*)Header)==MATROSKA_TRACK_ENCODING_COMP_ZSTD)
{
size_t UncompressedSize;
size_t Offset = 0;
Err = UnCompressFrameZstd(InBuf, FrameSize, &Element->Data, &UncompressedSize, &Offset);
if (Err == ERR_NONE)
{
OutSize += UncompressedSize;
ARRAYBEGIN(Element->SizeList,int32_t)[NumFrame] = UncompressedSize;
}
}
#endif
#if defined(CONFIG_LZO1X)
if (EBML_IntegerValue((ebml_integer*)Header)==MATROSKA_TRACK_ENCODING_COMP_LZO1X)
{
Expand Down Expand Up @@ -1608,6 +1733,10 @@ static filepos_t GetBlockFrameSize(const matroska_block *Element, size_t Frame,
if (CompAlgo == MATROSKA_TRACK_ENCODING_COMP_ZLIB && CompressFrameZLib(Data,ARRAYBEGIN(Element->SizeList,int32_t)[Frame],NULL,&OutSize)!=ERR_NONE)
return ARRAYBEGIN(Element->SizeList,int32_t)[Frame]; // we can't tell the final size without decoding the data
#endif
#if defined(CONFIG_ZSTD)
if (CompAlgo == MATROSKA_TRACK_ENCODING_COMP_ZSTD && CompressFrameZstd(Data,ARRAYBEGIN(Element->SizeList,int32_t)[Frame],NULL,&OutSize)!=ERR_NONE)
return ARRAYBEGIN(Element->SizeList,int32_t)[Frame]; // we can't tell the final size without decoding the data
#endif
#endif
return OutSize;
}
Expand Down Expand Up @@ -1664,6 +1793,10 @@ static char GetBestLacingType(const matroska_block *Element, int ForProfile)
#if defined(CONFIG_ZLIB)
else if (CompressionAlgo == MATROSKA_TRACK_ENCODING_COMP_ZLIB)
CanCompress = 1;
#endif
#if defined(CONFIG_ZSTD)
else if (CompressionAlgo == MATROSKA_TRACK_ENCODING_COMP_ZSTD)
CanCompress = 1;
#endif
if (!CanCompress)
return 0;
Expand Down Expand Up @@ -1776,6 +1909,10 @@ static err_t RenderBlockData(matroska_block *Element, struct stream *Output, boo
#if defined(CONFIG_ZLIB)
else if (CompressionAlgo == MATROSKA_TRACK_ENCODING_COMP_ZLIB)
CanCompress = 1;
#endif
#if defined(CONFIG_ZSTD)
else if (CompressionAlgo == MATROSKA_TRACK_ENCODING_COMP_ZSTD)
CanCompress = 1;
#endif
if (!CanCompress)
{
Expand Down Expand Up @@ -1886,6 +2023,39 @@ static err_t RenderBlockData(matroska_block *Element, struct stream *Output, boo
}
}
#endif
#if defined(CONFIG_ZSTD)
if (CompressionAlgo == MATROSKA_TRACK_ENCODING_COMP_ZSTD)
{
uint8_t *OutBuf;
array TmpBuf;
ArrayInit(&TmpBuf);
for (i=ARRAYBEGIN(Element->SizeList,int32_t);i!=ARRAYEND(Element->SizeList,int32_t);++i)
{
if (!ArrayResize(&TmpBuf,*i + 100,0))
{
ArrayClear(&TmpBuf);
Err = ERR_OUT_OF_MEMORY;
break;
}
OutBuf = ARRAYBEGIN(TmpBuf,uint8_t);
ToWrite = ARRAYCOUNT(TmpBuf,uint8_t);
if (CompressFrameZstd(Cursor, *i, &OutBuf, &ToWrite) != ERR_NONE)
{
ArrayClear(&TmpBuf);
Err = ERR_OUT_OF_MEMORY;
break;
}

Err = Stream_Write(Output,OutBuf,ToWrite,&Written);
ArrayClear(&TmpBuf);
if (Rendered)
*Rendered += Written;
Cursor += *i;
if (Err!=ERR_NONE)
break;
}
}
#endif
}
else
{
Expand Down Expand Up @@ -1990,6 +2160,10 @@ static filepos_t UpdateBlockSize(matroska_block *Element, bool_t bWithDefault, b
#if defined(CONFIG_ZLIB)
else if (CompressionAlgo == MATROSKA_TRACK_ENCODING_COMP_ZLIB)
CanCompress = 1;
#endif
#if defined(CONFIG_ZSTD)
else if (CompressionAlgo == MATROSKA_TRACK_ENCODING_COMP_ZSTD)
CanCompress = 1;
#endif
if (!CanCompress)
return ERR_INVALID_DATA;
Expand Down Expand Up @@ -2297,6 +2471,14 @@ static filepos_t UpdateDataSizeTrackEntry(matroska_trackentry *Element, bool_t b
if (EBML_BinarySetData(CodecPrivate, NewCompressed, CompressedSize)==ERR_NONE)
Element->CodecPrivateCompressionAlgo = MATROSKA_TRACK_ENCODING_COMP_ZLIB;
}
#endif
#if defined(CONFIG_ZSTD)
if (CompressionAlgo == MATROSKA_TRACK_ENCODING_COMP_ZSTD &&
CompressFrameZstd(ARRAYBEGIN(CodecPrivate->Data,uint8_t), (size_t)CodecPrivate->Base.DataSize, &NewCompressed, &CompressedSize)==ERR_NONE)
{
if (EBML_BinarySetData(CodecPrivate, NewCompressed, CompressedSize)==ERR_NONE)
Element->CodecPrivateCompressionAlgo = MATROSKA_TRACK_ENCODING_COMP_ZSTD;
}
#endif
free(Compressed);
}
Expand All @@ -2319,6 +2501,14 @@ static filepos_t UpdateDataSizeTrackEntry(matroska_trackentry *Element, bool_t b
if (EBML_BinarySetData(CodecPrivate, ARRAYBEGIN(Compressed,uint8_t), CompressedSize)==ERR_NONE)
Element->CodecPrivateCompressionAlgo = MATROSKA_TRACK_ENCODING_COMP_NONE;
}
#endif
#if defined(CONFIG_ZSTD)
if (Element->CodecPrivateCompressionAlgo == MATROSKA_TRACK_ENCODING_COMP_ZSTD &&
UnCompressFrameZstd(ARRAYBEGIN(CodecPrivate->Data,uint8_t), (size_t)CodecPrivate->Base.DataSize, &Compressed, &CompressedSize, &Offset)==ERR_NONE)
{
if (EBML_BinarySetData(CodecPrivate, ARRAYBEGIN(Compressed,uint8_t), CompressedSize)==ERR_NONE)
Element->CodecPrivateCompressionAlgo = MATROSKA_TRACK_ENCODING_COMP_NONE;
}
#endif
ArrayClear(&Compressed);
}
Expand Down Expand Up @@ -2364,10 +2554,12 @@ MatroskaTrackEncodingCompAlgo MATROSKA_TrackGetBlockCompression(const matroska_t

bool_t MATROSKA_TrackSetCompressionAlgo(matroska_trackentry *TrackEntry, MatroskaContentEncodingScope Scope, int ForProfile, MatroskaTrackEncodingCompAlgo algo)
{
// force zlib compression
// force zlib or Zstd compression
bool_t HadEncoding;
ebml_element *Encodings, *Elt, *Elt2;
assert(Node_IsPartOf(TrackEntry, MATROSKA_TRACKENTRY_CLASS));
if (Scope != 0 && algo != MATROSKA_TRACK_ENCODING_COMP_ZLIB && algo != MATROSKA_TRACK_ENCODING_COMP_ZSTD)
return 0;
// remove the previous compression and the new optimized one
Encodings = EBML_MasterFindChild(TrackEntry,MATROSKA_getContextContentEncodings());
HadEncoding = Encodings!=NULL;
Expand Down
Loading
Loading