user-selectable compression methods: zstd and snappy

src/net/torvald/terrarum/DefaultConfig.kt

@@ -21,6 +21,7 @@ object DefaultConfig {
             "screenheight" to TerrarumScreenSize.defaultH,
             "fullscreen" to false,
             "atlastexsize" to 2048,
+            "savegamecomp" to "zstd",
 
             "audio_buffer_size" to 512,
             "audio_dynamic_source_max" to 128,

src/net/torvald/terrarum/savegame/ByteArray64.kt

@@ -279,23 +279,7 @@ class ByteArray64(initialSize: Long = BANK_SIZE.toLong()) {
 
     fun writeToFile(file: File) {
         var fos = FileOutputStream(file, false)
-        // following code writes in-chunk basis
-        /*fos.write(__data[0])
-        fos.flush()
-        fos.close()
 
-        if (__data.size > 1) {
-            fos = FileOutputStream(file, true)
-            for (i in 1..__data.lastIndex) {
-                fos.write(__data[i])
-                fos.flush()
-            }
-            fos.close()
-        }*/
-
-        /*forEach {
-            fos.write(it.toInt())
-        }*/
         forEachUsedBanks { count, bytes ->
             fos.write(bytes, 0, count)
         }

src/net/torvald/terrarum/serialise/Common.kt

@@ -3,9 +3,12 @@ package net.torvald.terrarum.serialise
 import com.badlogic.gdx.utils.Json
 import com.badlogic.gdx.utils.JsonValue
 import com.badlogic.gdx.utils.JsonWriter
+import io.airlift.compress.snappy.SnappyFramedInputStream
+import io.airlift.compress.snappy.SnappyFramedOutputStream
 import io.airlift.compress.zstd.ZstdInputStream
 import io.airlift.compress.zstd.ZstdOutputStream
 import net.torvald.random.HQRNG
+import net.torvald.terrarum.App
 import net.torvald.terrarum.TerrarumAppConfiguration
 import net.torvald.terrarum.console.EchoError
 import net.torvald.terrarum.gameworld.BlockLayerI16
@@ -484,7 +487,7 @@ object Common {
         zo.flush(); zo.close()
         return bo.toByteArray64()
     }
-    fun zip(byteIterator: Iterator<Byte>): ByteArray64 {
+    private fun zipZ(byteIterator: Iterator<Byte>): ByteArray64 {
         val bo = ByteArray64GrowableOutputStream()
         val zo = ZstdOutputStream(bo)
 
@@ -495,6 +498,36 @@ object Common {
         zo.flush(); zo.close()
         return bo.toByteArray64()
     }
+    private fun zipS(byteIterator: Iterator<Byte>): ByteArray64 {
+        val bo = ByteArray64GrowableOutputStream()
+        val zo = SnappyFramedOutputStream(bo)
+
+        // zip
+        byteIterator.forEach {
+            zo.write(it.toInt())
+        }
+        zo.flush(); zo.close()
+        return bo.toByteArray64()
+    }
+    /*private fun zipNull(byteIterator: Iterator<Byte>): ByteArray64 {
+        val bo = ByteArray64GrowableOutputStream()
+
+        bo.write(byteArrayOf(0xfe.toByte(), 0xed.toByte(), 0xda.toByte(), 0x7a.toByte()))
+
+        // zip
+        byteIterator.forEach {
+            bo.write(it.toInt())
+        }
+        return bo.toByteArray64()
+    }*/
+
+    fun zip(byteIterator: Iterator<Byte>): ByteArray64 {
+        return when (App.getConfigString("savegamecomp")) {
+            "snappy" -> zipS(byteIterator)
+//            "null" -> zipNull(byteIterator)
+            else -> zipZ(byteIterator)
+        }
+    }
 
 
     fun enasciiToString(ba: ByteArray64): String = enasciiToString(ba.iterator())
@@ -541,12 +574,32 @@ object Common {
         return unzipdBytes
     }
 
+    private fun unzipS(bytes: ByteArray64): ByteArray64 {
+        val unzipdBytes = ByteArray64()
+        val zi = SnappyFramedInputStream(ByteArray64InputStream(bytes))
+        while (true) {
+            val byte = zi.read()
+            if (byte == -1) break
+            unzipdBytes.appendByte(byte.toByte())
+        }
+        zi.close()
+        return unzipdBytes
+    }
+
+    /*private fun unzipNull(bytes: ByteArray64): ByteArray64 {
+        return bytes.sliceArray64(4 until bytes.size)
+    }*/
+
     fun unzip(bytes: ByteArray64): ByteArray64 {
         val header = bytes[0].toUint().shl(24) or bytes[1].toUint().shl(16) or bytes[2].toUint().shl(8) or bytes[3].toUint()
 
+        // to save yourself from the curiosity: load time of the null compression is no faster than the snappy
+
         return when (header) {
-            in 0x1F8B0000..0x1F8B08FF -> unzipG(bytes)
+            in 0x1F8B0800..0x1F8B08FF -> unzipG(bytes)
             0x28B52FFD -> unzipZ(bytes)
+            0xFF060000.toInt() -> unzipS(bytes)
+//            0xFEEDDA7A.toInt() -> unzipNull(bytes)
             else -> throw IllegalArgumentException("Unknown archive with header ${header.toHex()}")
         }
     }

src/net/torvald/terrarum/tests/ZipTest.kt

@@ -1,5 +1,6 @@
 package net.torvald.terrarum.tests
 
+import com.badlogic.gdx.utils.compression.Lzma
 import io.airlift.compress.snappy.SnappyFramedInputStream
 import io.airlift.compress.snappy.SnappyFramedOutputStream
 import io.airlift.compress.zstd.ZstdInputStream
@@ -10,6 +11,8 @@ import net.torvald.terrarum.realestate.LandUtil.CHUNK_W
 import net.torvald.terrarum.savegame.ByteArray64
 import net.torvald.terrarum.savegame.ByteArray64GrowableOutputStream
 import net.torvald.terrarum.savegame.ByteArray64InputStream
+import net.torvald.terrarum.serialise.toUint
+import net.torvald.terrarum.toHex
 import java.io.InputStream
 import java.io.OutputStream
 import java.util.zip.GZIPInputStream
@@ -75,6 +78,7 @@ class ZipTest(val mode: String) {
 
     private val testInput0 = Array(TEST_COUNT) { dataGenerator(CHUNKSIZE) }
     private val testInputG = testInput0.copyOf().also { it.shuffle() }
+    private val testInputL = testInput0.copyOf().also { it.shuffle() }
     private val testInputZ = testInput0.copyOf().also { it.shuffle() }
     private val testInputS = testInput0.copyOf().also { it.shuffle() }
 
@@ -128,7 +132,6 @@ class ZipTest(val mode: String) {
             }
         }
 
-
         val zstdCompTime = measureNanoTime {
             for (i in 0 until TEST_COUNT) {
                 compBufZ[i] = compZstd(testInputZ[i])
@@ -169,12 +172,29 @@ class ZipTest(val mode: String) {
         println("Snpy   comp: $snappyCompTime ns")
         println("Snpy decomp: $snappyDecompTime ns; ratio: $ratioS% (avr size: $compSizeS)")
         println()
+
+        repeat(2) { sg.add(compBufG.random()!!.sliceArray(0..15).joinToString { it.toUint().toHex().takeLast(2) }) }
+        repeat(2) { sz.add(compBufZ.random()!!.sliceArray(0..15).joinToString { it.toUint().toHex().takeLast(2) }) }
+        repeat(2) { ss.add(compBufS.random()!!.sliceArray(0..15).joinToString { it.toUint().toHex().takeLast(2) }) }
+
+
+
     }
 }
 
+private val sg = ArrayList<String>()
+private val sz = ArrayList<String>()
+private val ss = ArrayList<String>()
 
 fun main() {
     ZipTest("Simulated Real-World").main()
     ZipTest("Zero-Filled").main()
     ZipTest("Random").main()
+
+    println("Gzip samples:")
+    sg.forEach { println(it) }
+    println("Zstd samples:")
+    sz.forEach { println(it) }
+    println("Snappy samples:")
+    ss.forEach { println(it) }
 }