From d4fe903273ed6cdb91b796bec26ec31bba3d729e Mon Sep 17 00:00:00 2001 From: minjaesong Date: Wed, 20 Dec 2023 02:12:03 +0900 Subject: [PATCH] zstd test --- .idea/libraries/io_airlift_aircompressor.xml | 14 ++ ModuleComputers/ModuleComputers.iml | 1 + TerrarumBuild.iml | 1 + lib/aircompressor-0.25-javadoc.jar | 3 + lib/aircompressor-0.25-sources.jar | 3 + lib/aircompressor-0.25.jar | 3 + .../torvald/terrarum/gameitems/GameItem.kt | 3 - src/net/torvald/terrarum/serialise/Common.kt | 3 +- src/net/torvald/terrarum/tests/ZipTest.kt | 160 ++++++++++++++++++ src/net/torvald/terrarum/utils/HashArray.kt | 1 - 10 files changed, 187 insertions(+), 5 deletions(-) create mode 100644 .idea/libraries/io_airlift_aircompressor.xml create mode 100644 lib/aircompressor-0.25-javadoc.jar create mode 100644 lib/aircompressor-0.25-sources.jar create mode 100644 lib/aircompressor-0.25.jar create mode 100644 src/net/torvald/terrarum/tests/ZipTest.kt diff --git a/.idea/libraries/io_airlift_aircompressor.xml b/.idea/libraries/io_airlift_aircompressor.xml new file mode 100644 index 000000000..33fab4d3f --- /dev/null +++ b/.idea/libraries/io_airlift_aircompressor.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ModuleComputers/ModuleComputers.iml b/ModuleComputers/ModuleComputers.iml index bf956ac44..020f9564c 100644 --- a/ModuleComputers/ModuleComputers.iml +++ b/ModuleComputers/ModuleComputers.iml @@ -18,5 +18,6 @@ + \ No newline at end of file diff --git a/TerrarumBuild.iml b/TerrarumBuild.iml index 41e651879..2ee128eb3 100644 --- a/TerrarumBuild.iml +++ b/TerrarumBuild.iml @@ -29,5 +29,6 @@ + \ No newline at end of file diff --git a/lib/aircompressor-0.25-javadoc.jar b/lib/aircompressor-0.25-javadoc.jar new file mode 100644 index 000000000..d4c84e081 --- /dev/null +++ b/lib/aircompressor-0.25-javadoc.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2eda6ba06fbd6d5146acf9dd6d5b91f7b295549a97b5e6b7f3b79731f0b57522 +size 272002 diff --git a/lib/aircompressor-0.25-sources.jar b/lib/aircompressor-0.25-sources.jar new file mode 100644 index 000000000..01f383a8e --- /dev/null +++ b/lib/aircompressor-0.25-sources.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0d2ec4c3f36aab30cc03dd19c0e35bbb04d9dc8217653da6e640ddbf7c34021 +size 207038 diff --git a/lib/aircompressor-0.25.jar b/lib/aircompressor-0.25.jar new file mode 100644 index 000000000..7decf3bcd --- /dev/null +++ b/lib/aircompressor-0.25.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:610938768aee8e4279af55f35bba5276d34cac447d4614897e26deb907e3115d +size 254041 diff --git a/src/net/torvald/terrarum/gameitems/GameItem.kt b/src/net/torvald/terrarum/gameitems/GameItem.kt index 186b95662..2d381c7c1 100644 --- a/src/net/torvald/terrarum/gameitems/GameItem.kt +++ b/src/net/torvald/terrarum/gameitems/GameItem.kt @@ -13,9 +13,6 @@ import net.torvald.terrarum.itemproperties.Material import net.torvald.terrarum.langpack.Lang import net.torvald.terrarum.modulebasegame.gameactors.ActorInventory import net.torvald.terrarum.modulebasegame.gameactors.Pocketed -import net.torvald.terrarum.savegame.ByteArray64 -import net.torvald.terrarum.utils.HashArray -import net.torvald.terrarum.utils.ZipCodedStr import org.dyn4j.geometry.Vector2 import kotlin.math.min diff --git a/src/net/torvald/terrarum/serialise/Common.kt b/src/net/torvald/terrarum/serialise/Common.kt index dc09e0284..98a47cfd8 100644 --- a/src/net/torvald/terrarum/serialise/Common.kt +++ b/src/net/torvald/terrarum/serialise/Common.kt @@ -36,7 +36,8 @@ object Common { const val GENVER = TerrarumAppConfiguration.VERSION_RAW const val COMP_NONE = 0 const val COMP_GZIP = 1 - const val COMP_LZMA = 2 +// const val COMP_LZMA = 2 + const val COMP_ZSTD = 3 val CHARSET = Charsets.UTF_8 diff --git a/src/net/torvald/terrarum/tests/ZipTest.kt b/src/net/torvald/terrarum/tests/ZipTest.kt new file mode 100644 index 000000000..82715db97 --- /dev/null +++ b/src/net/torvald/terrarum/tests/ZipTest.kt @@ -0,0 +1,160 @@ +package net.torvald.terrarum.tests + +import io.airlift.compress.zstd.ZstdInputStream +import io.airlift.compress.zstd.ZstdOutputStream +import net.torvald.random.HQRNG +import net.torvald.terrarum.realestate.LandUtil.CHUNK_H +import net.torvald.terrarum.realestate.LandUtil.CHUNK_W +import net.torvald.terrarum.savegame.ByteArray64 +import net.torvald.terrarum.savegame.ByteArray64GrowableOutputStream +import net.torvald.terrarum.savegame.ByteArray64InputStream +import net.torvald.terrarum.serialise.Common +import kotlin.math.roundToInt +import kotlin.system.measureNanoTime + +/** + * Created by minjaesong on 2023-12-20. + */ +class ZipTest(val mode: String) { + + val rnd = HQRNG() + + private val generateRLErandData = { size: Int -> + val r = ByteArray64() + var c = 0 + var payloadSize = 0 + var currentPayload1 = 0.toByte() + var currentPayload2 = 0.toByte() + var tiktok = 0 + while (c < size) { + if (payloadSize == 0) { + payloadSize = rnd.nextInt(1, 64) * 2 + currentPayload1 = rnd.nextInt(0, 256).toByte() + currentPayload2 = rnd.nextInt(0, 256).toByte() + } + + if (tiktok == 0) + r.appendByte(currentPayload1) + else + r.appendByte(currentPayload2) + + c++ + payloadSize-- + tiktok = 1 - tiktok + } + r + } + + private val generateZerofilled = { size: Int -> + val r = ByteArray64() + val zero = 0.toByte() + for (i in 0 until size) r.appendByte(zero) + r + } + + private val generateFullRandom = { size: Int -> + val r = ByteArray64() + for (i in 0 until size) r.appendByte(rnd.nextInt(0, 256).toByte()) + r + } + + val dataGenerator = when (mode) { + "Simulated Real-World" -> generateRLErandData + "Zero-Filled" -> generateZerofilled + "Random" -> generateFullRandom + else -> throw IllegalArgumentException() + } + + private val CHUNKSIZE = CHUNK_W * CHUNK_H + private val TEST_COUNT = 5000 + + private val testInput0 = Array(TEST_COUNT) { dataGenerator(CHUNKSIZE) } + private val testInputG = testInput0.copyOf().also { it.shuffle() } + private val testInputZ = testInput0.copyOf().also { it.shuffle() } + + private fun compGzip(bytes: ByteArray64): ByteArray64 { + return Common.zip(bytes) + } + + private fun decompGzip(bytes: ByteArray64): ByteArray64 { + return Common.unzip(bytes) + } + + private fun compZstd(bytes: ByteArray64): ByteArray64 { + val bo = ByteArray64GrowableOutputStream() + val zo = ZstdOutputStream(bo) + + bytes.iterator().forEach { + zo.write(it.toInt()) + } + zo.flush();zo.close() + return bo.toByteArray64() + } + + private fun decompZstd(bytes: ByteArray64): ByteArray64 { + val unzipdBytes = ByteArray64() + val zi = ZstdInputStream(ByteArray64InputStream(bytes)) + while (true) { + val byte = zi.read() + if (byte == -1) break + unzipdBytes.appendByte(byte.toByte()) + } + zi.close() + return unzipdBytes + } + + fun main() { + val compBufG = arrayOfNulls(TEST_COUNT) + val compBufZ = arrayOfNulls(TEST_COUNT) + val decompBufG = arrayOfNulls(TEST_COUNT) + val decompBufZ = arrayOfNulls(TEST_COUNT) + +// println("Compressing $TEST_COUNT samples of $CHUNKSIZE bytes using Gzip") + val gzipCompTime = measureNanoTime { + for (i in 0 until TEST_COUNT) { + compBufG[i] = compGzip(testInputG[i]) + } + } + +// println("Decompressing $TEST_COUNT samples of $CHUNKSIZE bytes using Gzip") + val gzipDecompTime = measureNanoTime { + for (i in 0 until TEST_COUNT) { + decompBufG[i] = decompGzip(compBufG[i]!!) + } + } + +// println("Compressing $TEST_COUNT samples of $CHUNKSIZE bytes using Zstd") + val zstdCompTime = measureNanoTime { + for (i in 0 until TEST_COUNT) { + compBufZ[i] = compZstd(testInputZ[i]) + } + } + +// println("Decompressing $TEST_COUNT samples of $CHUNKSIZE bytes using Zstd") + val zstdDecompTime = measureNanoTime { + for (i in 0 until TEST_COUNT) { + decompBufZ[i] = decompZstd(compBufZ[i]!!) + } + } + + val compSizeG = compBufG.sumOf { it!!.size } / TEST_COUNT + val compSizeZ = compBufZ.sumOf { it!!.size } / TEST_COUNT + val origSize = testInput0.sumOf { it.size } / TEST_COUNT + val ratioG = ((1.0 - (compSizeG.toDouble() / origSize)) * 10000).roundToInt() / 100 + val ratioZ = ((1.0 - (compSizeZ.toDouble() / origSize)) * 10000).roundToInt() / 100 + + println("==== $mode Data ($origSize bytes x $TEST_COUNT samples) ====") + println("Gzip comp: $gzipCompTime ns") + println("Gzip decomp: $gzipDecompTime ns; ratio: $ratioG% (avr size: $compSizeG)") + println("Zstd comp: $zstdCompTime ns") + println("Zstd decomp: $zstdDecompTime ns; ratio: $ratioZ% (avr size: $compSizeZ)") + println() + } +} + + +fun main() { + ZipTest("Simulated Real-World").main() + ZipTest("Zero-Filled").main() + ZipTest("Random").main() +} \ No newline at end of file diff --git a/src/net/torvald/terrarum/utils/HashArray.kt b/src/net/torvald/terrarum/utils/HashArray.kt index bd667ac91..6af439201 100644 --- a/src/net/torvald/terrarum/utils/HashArray.kt +++ b/src/net/torvald/terrarum/utils/HashArray.kt @@ -21,7 +21,6 @@ class HashArray: HashMap() // primitives are working just fine tho class WiringGraphMap: HashMap() class HashedWirings: HashMap() class HashedWiringGraph: HashMap() -class MetaModuleCSVPair: HashMap() class PlayersLastStatus: HashMap() { operator fun get(uuid: UUID) = this[uuid.toString()] operator fun set(uuid: UUID, value: PlayerLastStatus) = this.set(uuid.toString(), value)