seamless zstd integration

.idea/artifacts/TerrarumBuild.xml

@@ -119,6 +119,7 @@
       <element id="extracted-dir" path="$PROJECT_DIR$/lib/common-java5-2.19.1.jar" path-in-jar="/" />
       <element id="extracted-dir" path="$PROJECT_DIR$/lib/JTransforms-3.1.jar" path-in-jar="/" />
       <element id="extracted-dir" path="$PROJECT_DIR$/lib/JLargeArrays-1.5.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$PROJECT_DIR$/lib/aircompressor-0.25.jar" path-in-jar="/" />
     </root>
   </artifact>
 </component>

src/net/torvald/terrarum/modulebasegame/serialise/WriteWorld.kt

@@ -33,7 +33,7 @@ object WriteWorld {
         val world = ingame.world
         val currentPlayTime_t = time_t - ingame.loadedTime_t
 
-        world.comp = Common.COMP_GZIP
+        world.comp = Common.COMP_ZSTD
         world.lastPlayTime = time_t
         world.totalPlayTime += currentPlayTime_t
 

src/net/torvald/terrarum/serialise/Common.kt

@@ -3,6 +3,8 @@ package net.torvald.terrarum.serialise
 import com.badlogic.gdx.utils.Json
 import com.badlogic.gdx.utils.JsonValue
 import com.badlogic.gdx.utils.JsonWriter
+import io.airlift.compress.zstd.ZstdInputStream
+import io.airlift.compress.zstd.ZstdOutputStream
 import net.torvald.random.HQRNG
 import net.torvald.terrarum.TerrarumAppConfiguration
 import net.torvald.terrarum.console.EchoError
@@ -13,6 +15,7 @@ import net.torvald.terrarum.savegame.ByteArray64
 import net.torvald.terrarum.savegame.ByteArray64GrowableOutputStream
 import net.torvald.terrarum.savegame.ByteArray64InputStream
 import net.torvald.terrarum.savegame.ByteArray64Reader
+import net.torvald.terrarum.toHex
 import net.torvald.terrarum.utils.*
 import net.torvald.terrarum.weather.BaseModularWeather
 import net.torvald.terrarum.weather.WeatherDirBox
@@ -468,7 +471,9 @@ object Common {
     fun bytesToZipdStr(byteIterator: Iterator<Byte>): String = enasciiToString(zip(byteIterator))
 
     fun zip(ba: ByteArray64) = Common.zip(ba.iterator())
-    fun zip(byteIterator: Iterator<Byte>): ByteArray64 {
+
+    @Deprecated("New savegame standard should use Zstd")
+    private fun zipG(byteIterator: Iterator<Byte>): ByteArray64 {
         val bo = ByteArray64GrowableOutputStream()
         val zo = GZIPOutputStream(bo)
 
@@ -479,6 +484,18 @@ object Common {
         zo.flush(); zo.close()
         return bo.toByteArray64()
     }
+    fun zip(byteIterator: Iterator<Byte>): ByteArray64 {
+        val bo = ByteArray64GrowableOutputStream()
+        val zo = ZstdOutputStream(bo)
+
+        // zip
+        byteIterator.forEach {
+            zo.write(it.toInt())
+        }
+        zo.flush(); zo.close()
+        return bo.toByteArray64()
+    }
+
 
     fun enasciiToString(ba: ByteArray64): String = enasciiToString(ba.iterator())
     fun enasciiToString(ba: Iterator<Byte>): String {
@@ -500,7 +517,7 @@ object Common {
         return sb.toString()
     }
 
-    fun unzip(bytes: ByteArray64): ByteArray64 {
+    private fun unzipG(bytes: ByteArray64): ByteArray64 {
         val unzipdBytes = ByteArray64()
         val zi = GZIPInputStream(ByteArray64InputStream(bytes))
         while (true) {
@@ -512,6 +529,28 @@ object Common {
         return unzipdBytes
     }
 
+    private fun unzipZ(bytes: ByteArray64): ByteArray64 {
+        val unzipdBytes = ByteArray64()
+        val zi = ZstdInputStream(ByteArray64InputStream(bytes))
+        while (true) {
+            val byte = zi.read()
+            if (byte == -1) break
+            unzipdBytes.appendByte(byte.toByte())
+        }
+        zi.close()
+        return unzipdBytes
+    }
+
+    fun unzip(bytes: ByteArray64): ByteArray64 {
+        val header = bytes[0].toUint().shl(24) or bytes[1].toUint().shl(16) or bytes[2].toUint().shl(8) or bytes[3].toUint()
+
+        return when (header) {
+            in 0x1F8B0000..0x1F8B08FF -> unzipG(bytes)
+            0x28B52FFD -> unzipZ(bytes)
+            else -> throw IllegalArgumentException("Unknown archive with header ${header.toHex()}")
+        }
+    }
+
     fun unasciiToBytes(reader: Reader): ByteArray64 {
         val unasciidBytes = ByteArray64()
 

src/net/torvald/terrarum/serialise/ReadSimpleWorld.kt

@@ -45,7 +45,7 @@ object WriteSimpleWorld {
     private fun preWrite(ingame: IngameInstance, time_t: Long, world: SimpleGameWorld, actorsList: List<Actor>) {
         val currentPlayTime_t = time_t - ingame.loadedTime_t
 
-        world.comp = Common.COMP_GZIP
+        world.comp = Common.COMP_ZSTD
         world.lastPlayTime = time_t
         world.totalPlayTime += currentPlayTime_t
 

src/net/torvald/terrarum/tests/ZipTest.kt

@@ -8,7 +8,8 @@ import net.torvald.terrarum.realestate.LandUtil.CHUNK_W
 import net.torvald.terrarum.savegame.ByteArray64
 import net.torvald.terrarum.savegame.ByteArray64GrowableOutputStream
 import net.torvald.terrarum.savegame.ByteArray64InputStream
-import net.torvald.terrarum.serialise.Common
+import java.util.zip.GZIPInputStream
+import java.util.zip.GZIPOutputStream
 import kotlin.math.roundToInt
 import kotlin.system.measureNanoTime
 
@@ -73,11 +74,26 @@ class ZipTest(val mode: String) {
     private val testInputZ = testInput0.copyOf().also { it.shuffle() }
 
     private fun compGzip(bytes: ByteArray64): ByteArray64 {
-        return Common.zip(bytes)
+        val bo = ByteArray64GrowableOutputStream()
+        val zo = GZIPOutputStream(bo)
+
+        bytes.iterator().forEach {
+            zo.write(it.toInt())
+        }
+        zo.flush(); zo.close()
+        return bo.toByteArray64()
     }
 
     private fun decompGzip(bytes: ByteArray64): ByteArray64 {
-        return Common.unzip(bytes)
+        val unzipdBytes = ByteArray64()
+        val zi = GZIPInputStream(ByteArray64InputStream(bytes))
+        while (true) {
+            val byte = zi.read()
+            if (byte == -1) break
+            unzipdBytes.appendByte(byte.toByte())
+        }
+        zi.close()
+        return unzipdBytes
     }
 
     private fun compZstd(bytes: ByteArray64): ByteArray64 {