/*
** Command & Conquer Generals Zero Hour(tm)
** Copyright 2025 Electronic Arts Inc.
**
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU General Public License as published by
** the Free Software Foundation, either version 3 of the License, or
** (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU General Public License for more details.
**
** You should have received a copy of the GNU General Public License
** along with this program. If not, see .
*/
// FILE: Compression.cpp /////////////////////////////////////////////////////
// Author: Matthew D. Campbell
// LZH wrapper taken from Nox, originally from Jeff Brown
//////////////////////////////////////////////////////////////////////////////
#include "Compression.h"
#include "LZHCompress/NoxCompress.h"
extern "C" {
#include "ZLib/zlib.h"
}
#include "EAC/codex.h"
#include "EAC/btreecodex.h"
#include "EAC/huffcodex.h"
#include "EAC/refcodex.h"
#ifdef _INTERNAL
// for occasional debugging...
//#pragma optimize("", off)
//#pragma message("************************************** WARNING, optimization disabled for debugging purposes")
#endif
#define DEBUG_LOG(x) {}
const char *CompressionManager::getCompressionNameByType( CompressionType compType )
{
static const char *s_compressionNames[COMPRESSION_MAX+1] = {
"No compression",
"RefPack",
/*
"LZHL",
"ZLib 1 (fast)",
"ZLib 2",
"ZLib 3",
"ZLib 4",
"ZLib 5 (default)",
"ZLib 6",
"ZLib 7",
"ZLib 8",
"ZLib 9 (slow)",
"BTree",
"Huff",
*/
};
return s_compressionNames[compType];
}
// For perf timers, so we can have separate ones for compression/decompression
const char *CompressionManager::getDecompressionNameByType( CompressionType compType )
{
static const char *s_decompressionNames[COMPRESSION_MAX+1] = {
"d_None",
"d_RefPack",
/*
"d_NoxLZW",
"d_ZLib1",
"d_ZLib2",
"d_ZLib3",
"d_ZLib4",
"d_ZLib5",
"d_ZLib6",
"d_ZLib7",
"d_ZLib8",
"d_ZLib9",
"d_BTree",
"d_Huff",
*/
};
return s_decompressionNames[compType];
}
// ---------------------------------------------------------------------------------------
Bool CompressionManager::isDataCompressed( const void *mem, Int len )
{
CompressionType t = getCompressionType(mem, len);
return t != COMPRESSION_NONE;
}
CompressionType CompressionManager::getPreferredCompression( void )
{
return COMPRESSION_REFPACK;
}
CompressionType CompressionManager::getCompressionType( const void *mem, Int len )
{
if (len < 8)
return COMPRESSION_NONE;
if ( memcmp( mem, "NOX\0", 4 ) == 0 )
return COMPRESSION_NOXLZH;
if ( memcmp( mem, "ZL1\0", 4 ) == 0 )
return COMPRESSION_ZLIB1;
if ( memcmp( mem, "ZL2\0", 4 ) == 0 )
return COMPRESSION_ZLIB2;
if ( memcmp( mem, "ZL3\0", 4 ) == 0 )
return COMPRESSION_ZLIB3;
if ( memcmp( mem, "ZL4\0", 4 ) == 0 )
return COMPRESSION_ZLIB4;
if ( memcmp( mem, "ZL5\0", 4 ) == 0 )
return COMPRESSION_ZLIB5;
if ( memcmp( mem, "ZL6\0", 4 ) == 0 )
return COMPRESSION_ZLIB6;
if ( memcmp( mem, "ZL7\0", 4 ) == 0 )
return COMPRESSION_ZLIB7;
if ( memcmp( mem, "ZL8\0", 4 ) == 0 )
return COMPRESSION_ZLIB8;
if ( memcmp( mem, "ZL9\0", 4 ) == 0 )
return COMPRESSION_ZLIB9;
if ( memcmp( mem, "EAB\0", 4 ) == 0 )
return COMPRESSION_BTREE;
if ( memcmp( mem, "EAH\0", 4 ) == 0 )
return COMPRESSION_HUFF;
if ( memcmp( mem, "EAR\0", 4 ) == 0 )
return COMPRESSION_REFPACK;
return COMPRESSION_NONE;
}
Int CompressionManager::getMaxCompressedSize( Int uncompressedLen, CompressionType compType )
{
switch (compType)
{
case COMPRESSION_NOXLZH:
return CalcNewSize(uncompressedLen) + 8;
case COMPRESSION_BTREE: // guessing here
case COMPRESSION_HUFF: // guessing here
case COMPRESSION_REFPACK: // guessing here
return uncompressedLen + 8;
case COMPRESSION_ZLIB1:
case COMPRESSION_ZLIB2:
case COMPRESSION_ZLIB3:
case COMPRESSION_ZLIB4:
case COMPRESSION_ZLIB5:
case COMPRESSION_ZLIB6:
case COMPRESSION_ZLIB7:
case COMPRESSION_ZLIB8:
case COMPRESSION_ZLIB9:
return (Int)(ceil(uncompressedLen * 1.1 + 12 + 8));
}
return 0;
}
Int CompressionManager::getUncompressedSize( const void *mem, Int len )
{
if (len < 8)
return len;
CompressionType compType = getCompressionType( mem, len );
switch (compType)
{
case COMPRESSION_NOXLZH:
case COMPRESSION_ZLIB1:
case COMPRESSION_ZLIB2:
case COMPRESSION_ZLIB3:
case COMPRESSION_ZLIB4:
case COMPRESSION_ZLIB5:
case COMPRESSION_ZLIB6:
case COMPRESSION_ZLIB7:
case COMPRESSION_ZLIB8:
case COMPRESSION_ZLIB9:
case COMPRESSION_BTREE:
case COMPRESSION_HUFF:
case COMPRESSION_REFPACK:
return *(Int *)(((UnsignedByte *)mem)+4);
}
return len;
}
Int CompressionManager::compressData( CompressionType compType, void *srcVoid, Int srcLen, void *destVoid, Int destLen )
{
if (destLen < 8)
return 0;
destLen -= 8;
UnsignedByte *src = (UnsignedByte *)srcVoid;
UnsignedByte *dest = (UnsignedByte *)destVoid;
if (compType == COMPRESSION_BTREE)
{
memcpy(dest, "EAB\0", 4);
*(Int *)(dest+4) = 0;
Int ret = BTREE_encode(dest+8, src, srcLen);
if (ret)
{
*(Int *)(dest+4) = srcLen;
return ret + 8;
}
else
return 0;
}
if (compType == COMPRESSION_HUFF)
{
memcpy(dest, "EAH\0", 4);
*(Int *)(dest+4) = 0;
Int ret = HUFF_encode(dest+8, src, srcLen);
if (ret)
{
*(Int *)(dest+4) = srcLen;
return ret + 8;
}
else
return 0;
}
if (compType == COMPRESSION_REFPACK)
{
memcpy(dest, "EAR\0", 4);
*(Int *)(dest+4) = 0;
Int ret = REF_encode(dest+8, src, srcLen);
if (ret)
{
*(Int *)(dest+4) = srcLen;
return ret + 8;
}
else
return 0;
}
if (compType == COMPRESSION_NOXLZH)
{
memcpy(dest, "NOX\0", 4);
*(Int *)(dest+4) = 0;
Bool ret = CompressMemory(src, srcLen, dest+8, destLen);
if (ret)
{
*(Int *)(dest+4) = srcLen;
return destLen + 8;
}
else
return 0;
}
if (compType >= COMPRESSION_ZLIB1 && compType <= COMPRESSION_ZLIB9)
{
Int level = compType - COMPRESSION_ZLIB1 + 1; // 1-9
memcpy(dest, "ZL0\0", 4);
dest[2] = '0' + level;
*(Int *)(dest+4) = 0;
unsigned long outLen = destLen;
Int err = z_compress2( dest+8, &outLen, src, srcLen, level );
if (err == Z_OK || err == Z_STREAM_END)
{
*(Int *)(dest+4) = srcLen;
return outLen + 8;
}
else
{
DEBUG_LOG(("ZLib compression error (level is %d, src len is %d) %d\n", level, srcLen, err));
return 0;
}
}
return 0;
}
Int CompressionManager::decompressData( void *srcVoid, Int srcLen, void *destVoid, Int destLen )
{
if (srcLen < 8)
return 0;
UnsignedByte *src = (UnsignedByte *)srcVoid;
UnsignedByte *dest = (UnsignedByte *)destVoid;
CompressionType compType = getCompressionType(src, srcLen);
if (compType == COMPRESSION_BTREE)
{
Int slen = srcLen - 8;
Int ret = BTREE_decode(dest, src+8, &slen);
if (ret)
return ret;
else
return 0;
}
if (compType == COMPRESSION_HUFF)
{
Int slen = srcLen - 8;
Int ret = HUFF_decode(dest, src+8, &slen);
if (ret)
return ret;
else
return 0;
}
if (compType == COMPRESSION_REFPACK)
{
Int slen = srcLen - 8;
Int ret = REF_decode(dest, src+8, &slen);
if (ret)
return ret;
else
return 0;
}
if (compType == COMPRESSION_NOXLZH)
{
Bool ret = DecompressMemory(src+8, srcLen-8, dest, destLen);
if (ret)
return destLen;
else
return 0;
}
if (compType >= COMPRESSION_ZLIB1 && compType <= COMPRESSION_ZLIB9)
{
#ifdef DEBUG_LOGGING
Int level = compType - COMPRESSION_ZLIB1 + 1; // 1-9
#endif
unsigned long outLen = destLen;
Int err = z_uncompress(dest, &outLen, src+8, srcLen-8);
if (err == Z_OK || err == Z_STREAM_END)
{
return outLen;
}
else
{
DEBUG_LOG(("ZLib decompression error (src is level %d, %d bytes long) %d\n", level, srcLen, err));
return 0;
}
}
return 0;
}
///////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////
///// Performance Testing ///////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////
#ifdef TEST_COMPRESSION
#include "GameClient/MapUtil.h"
#include "Common/FileSystem.h"
#include "Common/File.h"
#include "Common/PerfTimer.h"
enum { NUM_TIMES = 10 };
struct CompData
{
public:
Int origSize;
Int compressedSize[COMPRESSION_MAX+1];
};
void DoCompressTest( void )
{
Int i;
PerfGather *s_compressGathers[COMPRESSION_MAX+1];
PerfGather *s_decompressGathers[COMPRESSION_MAX+1];
for (i = 0; i < COMPRESSION_MAX+1; ++i)
{
s_compressGathers[i] = new PerfGather(CompressionManager::getCompressionNameByType((CompressionType)i));
s_decompressGathers[i] = new PerfGather(CompressionManager::getDecompressionNameByType((CompressionType)i));
}
std::map s_sizes;
std::map::const_iterator it = TheMapCache->begin();
while (it != TheMapCache->end())
{
//if (it->second.m_isOfficial)
//{
//++it;
//continue;
//}
//static Int count = 0;
//if (count++ > 2)
//break;
File *f = TheFileSystem->openFile(it->first.str());
if (f)
{
DEBUG_LOG(("***************************\nTesting '%s'\n\n", it->first.str()));
Int origSize = f->size();
UnsignedByte *buf = (UnsignedByte *)f->readEntireAndClose();
UnsignedByte *uncompressedBuf = NEW UnsignedByte[origSize];
CompData d = s_sizes[it->first];
d.origSize = origSize;
d.compressedSize[COMPRESSION_NONE] = origSize;
for (i=COMPRESSION_MIN; i<=COMPRESSION_MAX; ++i)
{
DEBUG_LOG(("=================================================\n"));
DEBUG_LOG(("Compression Test %d\n", i));
Int maxCompressedSize = CompressionManager::getMaxCompressedSize( origSize, (CompressionType)i );
DEBUG_LOG(("Orig size is %d, max compressed size is %d bytes\n", origSize, maxCompressedSize));
UnsignedByte *compressedBuf = NEW UnsignedByte[maxCompressedSize];
memset(compressedBuf, 0, maxCompressedSize);
memset(uncompressedBuf, 0, origSize);
Int compressedLen, decompressedLen;
for (Int j=0; j < NUM_TIMES; ++j)
{
s_compressGathers[i]->startTimer();
compressedLen = CompressionManager::compressData((CompressionType)i, buf, origSize, compressedBuf, maxCompressedSize);
s_compressGathers[i]->stopTimer();
s_decompressGathers[i]->startTimer();
decompressedLen = CompressionManager::decompressData(compressedBuf, compressedLen, uncompressedBuf, origSize);
s_decompressGathers[i]->stopTimer();
}
d.compressedSize[i] = compressedLen;
DEBUG_LOG(("Compressed len is %d (%g%% of original size)\n", compressedLen, (double)compressedLen/(double)origSize*100.0));
DEBUG_ASSERTCRASH(compressedLen, ("Failed to compress\n"));
DEBUG_LOG(("Decompressed len is %d (%g%% of original size)\n", decompressedLen, (double)decompressedLen/(double)origSize*100.0));
DEBUG_ASSERTCRASH(decompressedLen == origSize, ("orig size does not match compressed+uncompressed output\n"));
if (decompressedLen == origSize)
{
Int ret = memcmp(buf, uncompressedBuf, origSize);
if (ret != 0)
{
DEBUG_CRASH(("orig buffer does not match compressed+uncompressed output - ret was %d\n", ret));
}
}
delete compressedBuf;
compressedBuf = NULL;
}
DEBUG_LOG(("d = %d -> %d\n", d.origSize, d.compressedSize[i]));
s_sizes[it->first] = d;
DEBUG_LOG(("s_sizes[%s] = %d -> %d\n", it->first.str(), s_sizes[it->first].origSize, s_sizes[it->first].compressedSize[i]));
delete[] buf;
buf = NULL;
delete[] uncompressedBuf;
uncompressedBuf = NULL;
}
++it;
}
for (i=COMPRESSION_MIN; i<=COMPRESSION_MAX; ++i)
{
Real maxCompression = 1000.0f;
Real minCompression = 0.0f;
Int totalUncompressedBytes = 0;
Int totalCompressedBytes = 0;
for (std::map::iterator cd = s_sizes.begin(); cd != s_sizes.end(); ++cd)
{
CompData d = cd->second;
Real ratio = d.compressedSize[i]/(Real)d.origSize;
maxCompression = min(maxCompression, ratio);
minCompression = max(minCompression, ratio);
totalUncompressedBytes += d.origSize;
totalCompressedBytes += d.compressedSize[i];
}
DEBUG_LOG(("***************************************************\n"));
DEBUG_LOG(("Compression method %s:\n", CompressionManager::getCompressionNameByType((CompressionType)i)));
DEBUG_LOG(("%d bytes compressed to %d (%g%%)\n", totalUncompressedBytes, totalCompressedBytes,
totalCompressedBytes/(Real)totalUncompressedBytes*100.0f));
DEBUG_LOG(("Min ratio: %g%%, Max ratio: %g%%\n",
minCompression*100.0f, maxCompression*100.0f));
DEBUG_LOG(("\n"));
}
PerfGather::dumpAll(10000);
//PerfGather::displayGraph(TheGameLogic->getFrame());
PerfGather::resetAll();
CopyFile( "AAAPerfStats.csv", "AAACompressPerfStats.csv", FALSE );
for (i = 0; i < COMPRESSION_MAX+1; ++i)
{
delete s_compressGathers[i];
s_compressGathers[i] = NULL;
delete s_decompressGathers[i];
s_decompressGathers[i] = NULL;
}
}
#endif // TEST_COMPRESSION