zstd test

.idea/libraries/io_airlift_aircompressor.xml

@@ -0,0 +1,14 @@
+<component name="libraryTable">
+  <library name="io.airlift.aircompressor" type="repository">
+    <properties maven-id="io.airlift:aircompressor:0.25" />
+    <CLASSES>
+      <root url="jar://$PROJECT_DIR$/lib/aircompressor-0.25.jar!/" />
+    </CLASSES>
+    <JAVADOC>
+      <root url="jar://$PROJECT_DIR$/lib/aircompressor-0.25-javadoc.jar!/" />
+    </JAVADOC>
+    <SOURCES>
+      <root url="jar://$PROJECT_DIR$/lib/aircompressor-0.25-sources.jar!/" />
+    </SOURCES>
+  </library>
+</component>

ModuleComputers/ModuleComputers.iml

@@ -18,5 +18,6 @@
     <orderEntry type="library" name="jetbrains.kotlin.test" level="project" />
     <orderEntry type="library" name="io.github.classgraph" level="project" />
     <orderEntry type="library" name="jetbrains.kotlinx.coroutines.core" level="project" />
+    <orderEntry type="library" name="io.airlift.aircompressor" level="project" />
   </component>
 </module>

TerrarumBuild.iml

@@ -29,5 +29,6 @@
     <orderEntry type="library" name="jetbrains.kotlinx.coroutines.core" level="project" />
     <orderEntry type="library" name="github.psambit9791.jdsp" level="project" />
     <orderEntry type="library" name="github.wendykierp.JTransforms" level="project" />
+    <orderEntry type="library" name="io.airlift.aircompressor" level="project" />
   </component>
 </module>

src/net/torvald/terrarum/gameitems/GameItem.kt

@@ -13,9 +13,6 @@ import net.torvald.terrarum.itemproperties.Material
 import net.torvald.terrarum.langpack.Lang
 import net.torvald.terrarum.modulebasegame.gameactors.ActorInventory
 import net.torvald.terrarum.modulebasegame.gameactors.Pocketed
-import net.torvald.terrarum.savegame.ByteArray64
-import net.torvald.terrarum.utils.HashArray
-import net.torvald.terrarum.utils.ZipCodedStr
 import org.dyn4j.geometry.Vector2
 import kotlin.math.min
 

src/net/torvald/terrarum/serialise/Common.kt

@@ -36,7 +36,8 @@ object Common {
     const val GENVER = TerrarumAppConfiguration.VERSION_RAW
     const val COMP_NONE = 0
     const val COMP_GZIP = 1
-    const val COMP_LZMA = 2
+//    const val COMP_LZMA = 2
+    const val COMP_ZSTD = 3
 
     val CHARSET = Charsets.UTF_8
 

src/net/torvald/terrarum/tests/ZipTest.kt

@@ -0,0 +1,160 @@
+package net.torvald.terrarum.tests
+
+import io.airlift.compress.zstd.ZstdInputStream
+import io.airlift.compress.zstd.ZstdOutputStream
+import net.torvald.random.HQRNG
+import net.torvald.terrarum.realestate.LandUtil.CHUNK_H
+import net.torvald.terrarum.realestate.LandUtil.CHUNK_W
+import net.torvald.terrarum.savegame.ByteArray64
+import net.torvald.terrarum.savegame.ByteArray64GrowableOutputStream
+import net.torvald.terrarum.savegame.ByteArray64InputStream
+import net.torvald.terrarum.serialise.Common
+import kotlin.math.roundToInt
+import kotlin.system.measureNanoTime
+
+/**
+ * Created by minjaesong on 2023-12-20.
+ */
+class ZipTest(val mode: String) {
+
+    val rnd = HQRNG()
+
+    private val generateRLErandData = { size: Int ->
+        val r = ByteArray64()
+        var c = 0
+        var payloadSize = 0
+        var currentPayload1 = 0.toByte()
+        var currentPayload2 = 0.toByte()
+        var tiktok = 0
+        while (c < size) {
+            if (payloadSize == 0) {
+                payloadSize = rnd.nextInt(1, 64) * 2
+                currentPayload1 = rnd.nextInt(0, 256).toByte()
+                currentPayload2 = rnd.nextInt(0, 256).toByte()
+            }
+
+            if (tiktok == 0)
+                r.appendByte(currentPayload1)
+            else
+                r.appendByte(currentPayload2)
+
+            c++
+            payloadSize--
+            tiktok = 1 - tiktok
+        }
+        r
+    }
+
+    private val generateZerofilled = { size: Int ->
+        val r = ByteArray64()
+        val zero = 0.toByte()
+        for (i in 0 until size) r.appendByte(zero)
+        r
+    }
+
+    private val generateFullRandom = { size: Int ->
+        val r = ByteArray64()
+        for (i in 0 until size) r.appendByte(rnd.nextInt(0, 256).toByte())
+        r
+    }
+
+    val dataGenerator = when (mode) {
+        "Simulated Real-World" -> generateRLErandData
+        "Zero-Filled" -> generateZerofilled
+        "Random" -> generateFullRandom
+        else -> throw IllegalArgumentException()
+    }
+
+    private val CHUNKSIZE = CHUNK_W * CHUNK_H
+    private val TEST_COUNT = 5000
+
+    private val testInput0 = Array(TEST_COUNT) { dataGenerator(CHUNKSIZE) }
+    private val testInputG = testInput0.copyOf().also { it.shuffle() }
+    private val testInputZ = testInput0.copyOf().also { it.shuffle() }
+
+    private fun compGzip(bytes: ByteArray64): ByteArray64 {
+        return Common.zip(bytes)
+    }
+
+    private fun decompGzip(bytes: ByteArray64): ByteArray64 {
+        return Common.unzip(bytes)
+    }
+
+    private fun compZstd(bytes: ByteArray64): ByteArray64 {
+        val bo = ByteArray64GrowableOutputStream()
+        val zo = ZstdOutputStream(bo)
+
+        bytes.iterator().forEach {
+            zo.write(it.toInt())
+        }
+        zo.flush();zo.close()
+        return bo.toByteArray64()
+    }
+
+    private fun decompZstd(bytes: ByteArray64): ByteArray64 {
+        val unzipdBytes = ByteArray64()
+        val zi = ZstdInputStream(ByteArray64InputStream(bytes))
+        while (true) {
+            val byte = zi.read()
+            if (byte == -1) break
+            unzipdBytes.appendByte(byte.toByte())
+        }
+        zi.close()
+        return unzipdBytes
+    }
+
+    fun main() {
+        val compBufG = arrayOfNulls<ByteArray64>(TEST_COUNT)
+        val compBufZ = arrayOfNulls<ByteArray64>(TEST_COUNT)
+        val decompBufG = arrayOfNulls<ByteArray64>(TEST_COUNT)
+        val decompBufZ = arrayOfNulls<ByteArray64>(TEST_COUNT)
+
+//        println("Compressing $TEST_COUNT samples of $CHUNKSIZE bytes using Gzip")
+        val gzipCompTime = measureNanoTime {
+            for (i in 0 until TEST_COUNT) {
+                compBufG[i] = compGzip(testInputG[i])
+            }
+        }
+
+//        println("Decompressing $TEST_COUNT samples of $CHUNKSIZE bytes using Gzip")
+        val gzipDecompTime = measureNanoTime {
+            for (i in 0 until TEST_COUNT) {
+                decompBufG[i] = decompGzip(compBufG[i]!!)
+            }
+        }
+
+//        println("Compressing $TEST_COUNT samples of $CHUNKSIZE bytes using Zstd")
+        val zstdCompTime = measureNanoTime {
+            for (i in 0 until TEST_COUNT) {
+                compBufZ[i] = compZstd(testInputZ[i])
+            }
+        }
+
+//        println("Decompressing $TEST_COUNT samples of $CHUNKSIZE bytes using Zstd")
+        val zstdDecompTime = measureNanoTime {
+            for (i in 0 until TEST_COUNT) {
+                decompBufZ[i] = decompZstd(compBufZ[i]!!)
+            }
+        }
+
+        val compSizeG = compBufG.sumOf { it!!.size } / TEST_COUNT
+        val compSizeZ = compBufZ.sumOf { it!!.size } / TEST_COUNT
+        val origSize =  testInput0.sumOf { it.size } / TEST_COUNT
+        val ratioG = ((1.0 - (compSizeG.toDouble() / origSize)) * 10000).roundToInt() / 100
+        val ratioZ = ((1.0 - (compSizeZ.toDouble() / origSize)) * 10000).roundToInt() / 100
+
+        println("==== $mode Data ($origSize bytes x $TEST_COUNT samples) ====")
+        println("Gzip   comp: $gzipCompTime ns")
+        println("Gzip decomp: $gzipDecompTime ns; ratio: $ratioG% (avr size: $compSizeG)")
+        println("Zstd   comp: $zstdCompTime ns")
+        println("Zstd decomp: $zstdDecompTime ns; ratio: $ratioZ% (avr size: $compSizeZ)")
+        println()
+    }
+}
+
+
+fun main() {
+    ZipTest("Simulated Real-World").main()
+    ZipTest("Zero-Filled").main()
+    ZipTest("Random").main()
+}

src/net/torvald/terrarum/utils/HashArray.kt

@@ -21,7 +21,6 @@ class HashArray<R>: HashMap<Long, R>() // primitives are working just fine tho
 class WiringGraphMap: HashMap<ItemID, GameWorld.WiringSimCell>()
 class HashedWirings: HashMap<BlockAddress, GameWorld.WiringNode>()
 class HashedWiringGraph: HashMap<BlockAddress, WiringGraphMap>()
-class MetaModuleCSVPair: HashMap<String, ZipCodedStr>()
 class PlayersLastStatus: HashMap<String, PlayerLastStatus>() {
     operator fun get(uuid: UUID) = this[uuid.toString()]
     operator fun set(uuid: UUID, value: PlayerLastStatus) = this.set(uuid.toString(), value)