diff --git a/src/net/torvald/terrarum/DefaultConfig.kt b/src/net/torvald/terrarum/DefaultConfig.kt index 2182abb6a..f9f2afe79 100644 --- a/src/net/torvald/terrarum/DefaultConfig.kt +++ b/src/net/torvald/terrarum/DefaultConfig.kt @@ -21,6 +21,7 @@ object DefaultConfig { "screenheight" to TerrarumScreenSize.defaultH, "fullscreen" to false, "atlastexsize" to 2048, + "savegamecomp" to "zstd", "audio_buffer_size" to 512, "audio_dynamic_source_max" to 128, diff --git a/src/net/torvald/terrarum/savegame/ByteArray64.kt b/src/net/torvald/terrarum/savegame/ByteArray64.kt index 11d4f58ca..ff1f1f52c 100644 --- a/src/net/torvald/terrarum/savegame/ByteArray64.kt +++ b/src/net/torvald/terrarum/savegame/ByteArray64.kt @@ -279,23 +279,7 @@ class ByteArray64(initialSize: Long = BANK_SIZE.toLong()) { fun writeToFile(file: File) { var fos = FileOutputStream(file, false) - // following code writes in-chunk basis - /*fos.write(__data[0]) - fos.flush() - fos.close() - if (__data.size > 1) { - fos = FileOutputStream(file, true) - for (i in 1..__data.lastIndex) { - fos.write(__data[i]) - fos.flush() - } - fos.close() - }*/ - - /*forEach { - fos.write(it.toInt()) - }*/ forEachUsedBanks { count, bytes -> fos.write(bytes, 0, count) } diff --git a/src/net/torvald/terrarum/serialise/Common.kt b/src/net/torvald/terrarum/serialise/Common.kt index 126dd1332..29adae0eb 100644 --- a/src/net/torvald/terrarum/serialise/Common.kt +++ b/src/net/torvald/terrarum/serialise/Common.kt @@ -3,9 +3,12 @@ package net.torvald.terrarum.serialise import com.badlogic.gdx.utils.Json import com.badlogic.gdx.utils.JsonValue import com.badlogic.gdx.utils.JsonWriter +import io.airlift.compress.snappy.SnappyFramedInputStream +import io.airlift.compress.snappy.SnappyFramedOutputStream import io.airlift.compress.zstd.ZstdInputStream import io.airlift.compress.zstd.ZstdOutputStream import net.torvald.random.HQRNG +import net.torvald.terrarum.App import net.torvald.terrarum.TerrarumAppConfiguration import net.torvald.terrarum.console.EchoError import net.torvald.terrarum.gameworld.BlockLayerI16 @@ -484,7 +487,7 @@ object Common { zo.flush(); zo.close() return bo.toByteArray64() } - fun zip(byteIterator: Iterator): ByteArray64 { + private fun zipZ(byteIterator: Iterator): ByteArray64 { val bo = ByteArray64GrowableOutputStream() val zo = ZstdOutputStream(bo) @@ -495,6 +498,36 @@ object Common { zo.flush(); zo.close() return bo.toByteArray64() } + private fun zipS(byteIterator: Iterator): ByteArray64 { + val bo = ByteArray64GrowableOutputStream() + val zo = SnappyFramedOutputStream(bo) + + // zip + byteIterator.forEach { + zo.write(it.toInt()) + } + zo.flush(); zo.close() + return bo.toByteArray64() + } + /*private fun zipNull(byteIterator: Iterator): ByteArray64 { + val bo = ByteArray64GrowableOutputStream() + + bo.write(byteArrayOf(0xfe.toByte(), 0xed.toByte(), 0xda.toByte(), 0x7a.toByte())) + + // zip + byteIterator.forEach { + bo.write(it.toInt()) + } + return bo.toByteArray64() + }*/ + + fun zip(byteIterator: Iterator): ByteArray64 { + return when (App.getConfigString("savegamecomp")) { + "snappy" -> zipS(byteIterator) +// "null" -> zipNull(byteIterator) + else -> zipZ(byteIterator) + } + } fun enasciiToString(ba: ByteArray64): String = enasciiToString(ba.iterator()) @@ -541,12 +574,32 @@ object Common { return unzipdBytes } + private fun unzipS(bytes: ByteArray64): ByteArray64 { + val unzipdBytes = ByteArray64() + val zi = SnappyFramedInputStream(ByteArray64InputStream(bytes)) + while (true) { + val byte = zi.read() + if (byte == -1) break + unzipdBytes.appendByte(byte.toByte()) + } + zi.close() + return unzipdBytes + } + + /*private fun unzipNull(bytes: ByteArray64): ByteArray64 { + return bytes.sliceArray64(4 until bytes.size) + }*/ + fun unzip(bytes: ByteArray64): ByteArray64 { val header = bytes[0].toUint().shl(24) or bytes[1].toUint().shl(16) or bytes[2].toUint().shl(8) or bytes[3].toUint() + // to save yourself from the curiosity: load time of the null compression is no faster than the snappy + return when (header) { - in 0x1F8B0000..0x1F8B08FF -> unzipG(bytes) + in 0x1F8B0800..0x1F8B08FF -> unzipG(bytes) 0x28B52FFD -> unzipZ(bytes) + 0xFF060000.toInt() -> unzipS(bytes) +// 0xFEEDDA7A.toInt() -> unzipNull(bytes) else -> throw IllegalArgumentException("Unknown archive with header ${header.toHex()}") } } diff --git a/src/net/torvald/terrarum/tests/ZipTest.kt b/src/net/torvald/terrarum/tests/ZipTest.kt index 02b439838..74c2ff356 100644 --- a/src/net/torvald/terrarum/tests/ZipTest.kt +++ b/src/net/torvald/terrarum/tests/ZipTest.kt @@ -1,5 +1,6 @@ package net.torvald.terrarum.tests +import com.badlogic.gdx.utils.compression.Lzma import io.airlift.compress.snappy.SnappyFramedInputStream import io.airlift.compress.snappy.SnappyFramedOutputStream import io.airlift.compress.zstd.ZstdInputStream @@ -10,6 +11,8 @@ import net.torvald.terrarum.realestate.LandUtil.CHUNK_W import net.torvald.terrarum.savegame.ByteArray64 import net.torvald.terrarum.savegame.ByteArray64GrowableOutputStream import net.torvald.terrarum.savegame.ByteArray64InputStream +import net.torvald.terrarum.serialise.toUint +import net.torvald.terrarum.toHex import java.io.InputStream import java.io.OutputStream import java.util.zip.GZIPInputStream @@ -75,6 +78,7 @@ class ZipTest(val mode: String) { private val testInput0 = Array(TEST_COUNT) { dataGenerator(CHUNKSIZE) } private val testInputG = testInput0.copyOf().also { it.shuffle() } + private val testInputL = testInput0.copyOf().also { it.shuffle() } private val testInputZ = testInput0.copyOf().also { it.shuffle() } private val testInputS = testInput0.copyOf().also { it.shuffle() } @@ -128,7 +132,6 @@ class ZipTest(val mode: String) { } } - val zstdCompTime = measureNanoTime { for (i in 0 until TEST_COUNT) { compBufZ[i] = compZstd(testInputZ[i]) @@ -169,12 +172,29 @@ class ZipTest(val mode: String) { println("Snpy comp: $snappyCompTime ns") println("Snpy decomp: $snappyDecompTime ns; ratio: $ratioS% (avr size: $compSizeS)") println() + + repeat(2) { sg.add(compBufG.random()!!.sliceArray(0..15).joinToString { it.toUint().toHex().takeLast(2) }) } + repeat(2) { sz.add(compBufZ.random()!!.sliceArray(0..15).joinToString { it.toUint().toHex().takeLast(2) }) } + repeat(2) { ss.add(compBufS.random()!!.sliceArray(0..15).joinToString { it.toUint().toHex().takeLast(2) }) } + + + } } +private val sg = ArrayList() +private val sz = ArrayList() +private val ss = ArrayList() fun main() { ZipTest("Simulated Real-World").main() ZipTest("Zero-Filled").main() ZipTest("Random").main() + + println("Gzip samples:") + sg.forEach { println(it) } + println("Zstd samples:") + sz.forEach { println(it) } + println("Snappy samples:") + ss.forEach { println(it) } } \ No newline at end of file