let's compress everything because we can

src/net/torvald/UnsafePtr.kt

@@ -2,6 +2,9 @@ package net.torvald
 
 import net.torvald.terrarum.printStackTrace
 import sun.misc.Unsafe
+import java.io.IOException
+import java.io.InputStream
+import java.io.OutputStream
 
 /**
  * Further read:
@@ -184,4 +187,38 @@ internal class UnsafePtr(pointer: Long, allocSize: Long) {
 
     override fun toString() = "0x${ptr.toString(16)} with size $size"
     override fun equals(other: Any?) = this.ptr == (other as UnsafePtr).ptr && this.size == other.size
+}
+
+internal class UnsafePtrInputStream(val ptr: UnsafePtr): InputStream() {
+    private var p = 0L
+
+    override fun reset() {
+        p = 0L
+    }
+
+    override fun read(): Int {
+        if (p < ptr.size) {
+            p += 1
+            return ptr[p - 1].toInt().and(255)
+        }
+        else return -1
+    }
+}
+
+internal class UnsafePtrOutputStream(val ptr: UnsafePtr): OutputStream() {
+    private var p = 0L
+
+    override fun write(p0: Int) {
+        if (p < ptr.size) {
+            p += 1
+            ptr[p - 1] = p0.toByte()
+        }
+        else throw IOException("Buffer overflow: $p for allocated size ${ptr.size}")
+    }
+
+    override fun write(b: ByteArray, off: Int, len: Int) {
+        if (p + len >= ptr.size) throw IOException("Buffer overflow: ${p+len} for allocated size ${ptr.size}")
+        UnsafeHelper.unsafe.copyMemory(b, off.toLong(), null, ptr.ptr + p, len.toLong())
+        p += len
+    }
 }

src/net/torvald/terrarum/gameworld/GameWorld.kt

@@ -5,6 +5,9 @@ import com.badlogic.gdx.utils.Disposable
 import com.badlogic.gdx.utils.Json
 import com.badlogic.gdx.utils.JsonValue
 import com.badlogic.gdx.utils.JsonWriter
+import com.badlogic.gdx.utils.compression.Lzma
+import net.torvald.UnsafePtr
+import net.torvald.UnsafePtrInputStream
 import net.torvald.gdx.graphics.Cvec
 import net.torvald.terrarum.*
 import net.torvald.terrarum.AppLoader.printdbg
@@ -16,6 +19,8 @@ import net.torvald.terrarum.gameitem.ItemID
 import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64GrowableOutputStream
 import net.torvald.terrarum.realestate.LandUtil
 import net.torvald.terrarum.serialise.Ascii85
+import net.torvald.terrarum.serialise.bytesToLzmadStr
+import net.torvald.terrarum.serialise.bytesToZipdStr
 import net.torvald.util.SortedArrayList
 import org.apache.commons.codec.digest.DigestUtils
 import org.dyn4j.geometry.Vector2
@@ -699,11 +704,14 @@ open class GameWorld : Disposable {
         layerWall.bytesIterator().forEachRemaining { digester.update(it) }
         val wallhash = StringBuilder().let { sb -> digester.digest().forEach { sb.append(it.tostr()) }; sb.toString() }
 
+        // use gzip; lzma's slower and larger for some reason
         return arrayListOf(
                 """"worldname": "$worldName"""",
-                """"comp": 1""",
+                """"comp": "gzip"""",
                 """"width": $width""",
                 """"height": $height""",
+                """"spawnx": $spawnX""",
+                """"spawny": $spawnY""",
                 """"genver": 4""",
                 """"time_t": ${worldTime.TIME_T}""",
                 """"terr": {
@@ -714,22 +722,22 @@ open class GameWorld : Disposable {
                     |"b": "${blockLayerToStr(layerWall)}"}""".trimMargin(),
                 """"tdmg": {
                     |"h": "${StringBuilder().let { sb -> digester.digest(tdmgstr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}",
-                    |"b": $tdmgstr}""".trimMargin(),
+                    |"b": "${bytesToZipdStr(tdmgstr.toByteArray())}"}""".trimMargin(),
                 """"wdmg": {
                     |"h": "${StringBuilder().let { sb -> digester.digest(wdmgstr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}",
-                    |"b": $wdmgstr}""".trimMargin(),
+                    |"b": "${bytesToZipdStr(wdmgstr.toByteArray())}"}""".trimMargin(),
                 """"flut": {
                     |"h": "${StringBuilder().let { sb -> digester.digest(flutstr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}",
-                    |"b": $flutstr}""".trimMargin(),
+                    |"b": "${bytesToZipdStr(flutstr.toByteArray())}"}""".trimMargin(),
                 """"fluf": {
                     |"h": "${StringBuilder().let { sb -> digester.digest(flufstr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}",
-                    |"b": $flufstr}""".trimMargin(),
+                    |"b": "${bytesToZipdStr(flufstr.toByteArray())}"}""".trimMargin(),
                 """"wire": {
                     |"h": "${StringBuilder().let { sb -> digester.digest(wirestr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}",
-                    |"b": $wirestr}""".trimMargin(),
+                    |"b": "${bytesToZipdStr(wirestr.toByteArray())}"}""".trimMargin(),
                 """"wirg": {
                     |"h": "${StringBuilder().let { sb -> digester.digest(wirgstr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}",
-                    |"b": $wirgstr}""".trimMargin()
+                    |"b": "${bytesToZipdStr(wirgstr.toByteArray())}"}""".trimMargin()
         )
     }
 }
@@ -772,5 +780,33 @@ fun blockLayerToStr(b: BlockLayer): String {
         bai += 1
     }; sb.append(Ascii85.encode(buf[0], buf[1], buf[2], buf[3]))
 
+    return sb.toString()
+}
+
+/**
+ * @param b a BlockLayer
+ * @return Bytes in [b] which are LZMA'd then Ascii85-encoded
+ */
+fun blockLayerToStr2(b: BlockLayer): String {
+    val sb = StringBuilder()
+    val bi = UnsafePtrInputStream(b.ptr)
+    val bo = ByteArray64GrowableOutputStream()
+
+    Lzma.compress(bi, bo); bo.flush(); bo.close()
+
+    val ba = bo.toByteArray64()
+    var bai = 0
+    val buf = IntArray(4) { Ascii85.PAD_BYTE }
+    ba.forEach {
+        if (bai > 0 && bai % 4 == 0) {
+            sb.append(Ascii85.encode(buf[0], buf[1], buf[2], buf[3]))
+            buf.fill(Ascii85.PAD_BYTE)
+        }
+
+        buf[bai % 4] = it.toInt() and 255
+
+        bai += 1
+    }; sb.append(Ascii85.encode(buf[0], buf[1], buf[2], buf[3]))
+
     return sb.toString()
 }

src/net/torvald/terrarum/serialise/WriteMeta.kt

@@ -3,12 +3,14 @@ package net.torvald.terrarum.serialise
 import com.badlogic.gdx.utils.Json
 import com.badlogic.gdx.utils.JsonValue
 import com.badlogic.gdx.utils.JsonWriter
+import com.badlogic.gdx.utils.compression.Lzma
 import net.torvald.terrarum.ModMgr
 import net.torvald.terrarum.gameworld.BlockLayer
 import net.torvald.terrarum.modulebasegame.TerrarumIngame
 import net.torvald.terrarum.modulebasegame.worldgenerator.RoguelikeRandomiser
 import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64GrowableOutputStream
 import net.torvald.terrarum.weather.WeatherMixer
+import java.io.ByteArrayInputStream
 import java.util.zip.GZIPOutputStream
 
 /**
@@ -95,5 +97,33 @@ fun bytesToZipdStr(b: ByteArray): String {
         bai += 1
     }; sb.append(Ascii85.encode(buf[0], buf[1], buf[2], buf[3]))
 
+    return sb.toString()
+}
+
+/**
+ * @param b a ByteArray
+ * @return Bytes in [b] which are LZMA'd then Ascii85-encoded
+ */
+fun bytesToLzmadStr(b: ByteArray): String {
+    val sb = StringBuilder()
+    val bi = ByteArrayInputStream(b)
+    val bo = ByteArray64GrowableOutputStream()
+
+    Lzma.compress(bi, bo); bo.flush(); bo.close()
+
+    val ba = bo.toByteArray64()
+    var bai = 0
+    val buf = IntArray(4) { Ascii85.PAD_BYTE }
+    ba.forEach {
+        if (bai > 0 && bai % 4 == 0) {
+            sb.append(Ascii85.encode(buf[0], buf[1], buf[2], buf[3]))
+            buf.fill(Ascii85.PAD_BYTE)
+        }
+
+        buf[bai % 4] = it.toInt() and 255
+
+        bai += 1
+    }; sb.append(Ascii85.encode(buf[0], buf[1], buf[2], buf[3]))
+
     return sb.toString()
 }

work_files/DataFormats/just-json-it-saveformat.md

@@ -32,7 +32,7 @@ File is named as `"world"+world_index+".json"`
 ```
 {
     worldname: "New World",
-    comp: <0 for uncompressed, 1 for GZip, 2 for LZMA>,
+    comp: <null, "gzip">,
     width: 8192,
     height: 2048,
     spawnx: 4096,