working blocklayer gzip writer and reader

src/net/torvald/terrarum/IngameInstance.kt

@@ -2,6 +2,7 @@ package net.torvald.terrarum
 
 import com.badlogic.gdx.Screen
 import com.badlogic.gdx.graphics.g2d.SpriteBatch
+import com.badlogic.gdx.utils.Queue
 import net.torvald.terrarum.AppLoader.printdbg
 import net.torvald.terrarum.gameactors.Actor
 import net.torvald.terrarum.gameactors.BlockMarkerActor
@@ -40,11 +41,13 @@ open class IngameInstance(val batch: SpriteBatch) : Screen {
 
     open var world: GameWorld = GameWorld.makeNullWorld()
         set(value) {
+            val oldWorld = field
             newWorldLoadedLatch = true
             printdbg(this, "Ingame instance ${this.hashCode()}, accepting new world ${value.layerTerrain}; called from")
             printStackTrace(this)
             field = value
             IngameRenderer.setRenderedWorld(value)
+            oldWorld.dispose()
         }
     /** how many different planets/stages/etc. are thenre. Whole stages must be manually managed by YOU. */
     var gameworldIndices = ArrayList<Int>()

src/net/torvald/terrarum/gameworld/GameWorld.kt

@@ -17,6 +17,7 @@ import net.torvald.terrarum.gameitem.ItemID
 import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64GrowableOutputStream
 import net.torvald.terrarum.realestate.LandUtil
 import net.torvald.terrarum.serialise.Ascii85
+import net.torvald.terrarum.serialise.Common
 import net.torvald.terrarum.serialise.bytesToZipdStr
 import net.torvald.terrarum.utils.*
 import net.torvald.util.SortedArrayList
@@ -112,6 +113,9 @@ class GameWorld : Disposable {
 
     val extraFields = HashMap<String, Any?>()
 
+    internal var genver = -1
+    internal var comp = -1
+
     /**
      * Create new world
      */
@@ -679,70 +683,6 @@ class GameWorld : Disposable {
     open fun updateWorldTime(delta: Float) {
         worldTime.update(delta)
     }
-
-    /**
-     * Returns lines that are part of the entire JSON
-     *
-     * To extend this function, you can code something like this:
-     * ```
-     * return super.getJsonFields() + arrayListOf(
-     *     """"<myModuleName>.<myNewObject>": ${Json(JsonWriter.OutputType.json).toJson(<myNewObject>)}"""
-     * )
-     * ```
-     */
-    open fun getJsonFields(): List<String> {
-        fun Byte.tostr() = this.toInt().and(255).toString(16).padStart(2,'0')
-
-        val tdmgstr = Json(JsonWriter.OutputType.json).toJson(terrainDamages)
-        val wdmgstr = Json(JsonWriter.OutputType.json).toJson(wallDamages)
-        val flutstr = Json(JsonWriter.OutputType.json).toJson(fluidTypes)
-        val flufstr = Json(JsonWriter.OutputType.json).toJson(fluidFills)
-        val wirestr = Json(JsonWriter.OutputType.json).toJson(wirings)
-        val wirgstr = Json(JsonWriter.OutputType.json).toJson(wiringGraph)
-
-        val digester = DigestUtils.getSha256Digest()
-
-        layerTerrain.bytesIterator().forEachRemaining { digester.update(it) }
-        val terrhash = StringBuilder().let { sb -> digester.digest().forEach { sb.append(it.tostr()) }; sb.toString() }
-        layerWall.bytesIterator().forEachRemaining { digester.update(it) }
-        val wallhash = StringBuilder().let { sb -> digester.digest().forEach { sb.append(it.tostr()) }; sb.toString() }
-
-        // use gzip; lzma's slower and larger for some reason
-        return arrayListOf(
-                """"worldname": "$worldName"""",
-                """"comp": "gzip"""",
-                """"width": $width""",
-                """"height": $height""",
-                """"spawnx": $spawnX""",
-                """"spawny": $spawnY""",
-                """"genver": 4""",
-                """"time_t": ${worldTime.TIME_T}""",
-                """"terr": {
-                    |"h": "$terrhash",
-                    |"b": "${blockLayerToStr(layerTerrain)}"}""".trimMargin(),
-                """"wall": {
-                    |"h": "$wallhash",
-                    |"b": "${blockLayerToStr(layerWall)}"}""".trimMargin(),
-                """"tdmg": {
-                    |"h": "${StringBuilder().let { sb -> digester.digest(tdmgstr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}",
-                    |"b": "${bytesToZipdStr(tdmgstr.toByteArray())}"}""".trimMargin(),
-                """"wdmg": {
-                    |"h": "${StringBuilder().let { sb -> digester.digest(wdmgstr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}",
-                    |"b": "${bytesToZipdStr(wdmgstr.toByteArray())}"}""".trimMargin(),
-                """"flut": {
-                    |"h": "${StringBuilder().let { sb -> digester.digest(flutstr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}",
-                    |"b": "${bytesToZipdStr(flutstr.toByteArray())}"}""".trimMargin(),
-                """"fluf": {
-                    |"h": "${StringBuilder().let { sb -> digester.digest(flufstr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}",
-                    |"b": "${bytesToZipdStr(flufstr.toByteArray())}"}""".trimMargin(),
-                """"wire": {
-                    |"h": "${StringBuilder().let { sb -> digester.digest(wirestr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}",
-                    |"b": "${bytesToZipdStr(wirestr.toByteArray())}"}""".trimMargin(),
-                """"wirg": {
-                    |"h": "${StringBuilder().let { sb -> digester.digest(wirgstr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}",
-                    |"b": "${bytesToZipdStr(wirgstr.toByteArray())}"}""".trimMargin()
-        )
-    }
 }
 
 infix fun Int.fmod(other: Int) = Math.floorMod(this, other)

src/net/torvald/terrarum/modulebasegame/console/PrintWorld.kt

@@ -1,6 +1,8 @@
 package net.torvald.terrarum.modulebasegame.console
 
 import net.torvald.terrarum.Terrarum
+import net.torvald.terrarum.ccG
+import net.torvald.terrarum.ccO
 import net.torvald.terrarum.console.ConsoleCommand
 import net.torvald.terrarum.console.Echo
 import net.torvald.terrarum.gameworld.BlockAddress
@@ -18,12 +20,8 @@ object PrintWorld : ConsoleCommand {
             val fieldAccessibility = field.isAccessible
 
             field.isAccessible = true
-            Echo(field.get(w).toString())
+            Echo("$ccO${field.get(w).javaClass.simpleName}")
-            Echo(field.get(w).javaClass.simpleName)
+            Echo("$ccG${field.get(w)}")
-            w.wirings.forEach { i, node ->
-                Echo(i.toString())
-            }
-
             field.isAccessible = fieldAccessibility
         }
         else {

src/net/torvald/terrarum/modulebasegame/ui/UIProxyNewRandomGame.kt

@@ -41,10 +41,10 @@ class UIProxyNewRandomGame : UICanvas() {
         val worldParam = TerrarumIngame.NewWorldParameters(2400, 1280, 0x51621DL)
         //val worldParam = TerrarumIngame.NewWorldParameters(2400, 1280, HQRNG().nextLong())
 
-        //val worldParam = TerrarumIngame.NewWorldParameters(6000, 1800, 0x51621DL)
+        //val worldParam = TerrarumIngame.NewWorldParameters(6000, 1800, 0x51621DL) // small
-        //val worldParam = TerrarumIngame.NewWorldParameters(9000, 2250, 0x51621DL)
+//        val worldParam = TerrarumIngame.NewWorldParameters(9000, 2250, 0x51621DL) // normal
-        //val worldParam = TerrarumIngame.NewWorldParameters(13500, 3000, 0x51621DL)
+        //val worldParam = TerrarumIngame.NewWorldParameters(13500, 3000, 0x51621DL) // large
-        //val worldParam = TerrarumIngame.NewWorldParameters(22500, 4500, 0x51621DL)
+        //val worldParam = TerrarumIngame.NewWorldParameters(22500, 4500, 0x51621DL) // huge
         ingame.gameLoadInfoPayload = worldParam
         ingame.gameLoadMode = TerrarumIngame.GameLoadMode.CREATE_NEW
 

src/net/torvald/terrarum/serialise/Common.kt

@@ -3,23 +3,29 @@ package net.torvald.terrarum.serialise
 import com.badlogic.gdx.utils.Json
 import com.badlogic.gdx.utils.JsonValue
 import com.badlogic.gdx.utils.JsonWriter
-import net.torvald.terrarum.AppLoader
-import net.torvald.terrarum.AppLoader.printdbg
 import net.torvald.terrarum.console.EchoError
 import net.torvald.terrarum.gameworld.BlockLayer
 import net.torvald.terrarum.gameworld.GameWorld
 import net.torvald.terrarum.gameworld.WorldTime
 import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64
 import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64GrowableOutputStream
+import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64InputStream
 import net.torvald.terrarum.utils.*
 import org.apache.commons.codec.digest.DigestUtils
 import java.math.BigInteger
+import java.util.zip.GZIPInputStream
 import java.util.zip.GZIPOutputStream
 
 /**
  * Created by minjaesong on 2021-08-26.
  */
 object Common {
+
+    const val GENVER = 4
+    const val COMP_NONE = 0
+    const val COMP_GZIP = 1
+    const val COMP_LZMA = 2
+
     /** dispose of the `offendingObject` after rejection! */
     class BlockLayerHashMismatchError(val oldHash: String, val newHash: String, val offendingObject: BlockLayer) : Error("Old Hash $oldHash != New Hash $newHash")
 
@@ -51,7 +57,7 @@ object Common {
 
                 val layer = LayerInfo(hash, blockLayerToStr(obj), obj.width, obj.height)
 
-                AppLoader.printdbg(this, "pre: ${(0L..1023L).map { obj.ptr[it].tostr() }.joinToString(" ")}")
+//                printdbg(this, "pre: ${(0L..1023L).map { obj.ptr[it].tostr() }.joinToString(" ")}")
 
 
                 json.writeValue(layer)
@@ -98,10 +104,8 @@ object Common {
 
             override fun read(json: Json, jsonData: JsonValue, type: Class<*>?): HashArray<*> {
                 val hashMap = HashArray<Any>()
-                printdbg(type?.canonicalName, "deserialising '$jsonData'")
                 JsonFetcher.forEach(jsonData) { key, obj ->
                     hashMap[key.toLong()] = json.readValue(null, obj)
-                    printdbg(this, "key: $key, value: ${hashMap[key.toLong()]}")
                 }
                 return hashMap
             }
@@ -118,10 +122,8 @@ object Common {
 
             override fun read(json: Json, jsonData: JsonValue, type: Class<*>?): HashedWirings {
                 val hashMap = HashedWirings()
-                printdbg(type?.canonicalName, "deserialising '$jsonData'")
                 JsonFetcher.forEach(jsonData) { key, obj ->
                     hashMap[key.toLong()] = json.readValue(GameWorld.WiringNode::class.java, obj)
-                    printdbg(this, "key: $key, value: ${hashMap[key.toLong()]}")
                 }
                 return hashMap
             }
@@ -138,10 +140,8 @@ object Common {
 
             override fun read(json: Json, jsonData: JsonValue, type: Class<*>?): HashedWiringGraph {
                 val hashMap = HashedWiringGraph()
-                printdbg(type?.canonicalName, "deserialising '$jsonData'")
                 JsonFetcher.forEach(jsonData) { key, obj ->
                     hashMap[key.toLong()] = json.readValue(WiringGraphMap::class.java, obj)
-                    printdbg(this, "key: $key, value: ${hashMap[key.toLong()]}")
                 }
                 return hashMap
             }
@@ -158,10 +158,8 @@ object Common {
 
             override fun read(json: Json, jsonData: JsonValue, type: Class<*>?): WiringGraphMap {
                 val hashMap = WiringGraphMap()
-                printdbg(type?.canonicalName, "deserialising '$jsonData'")
                 JsonFetcher.forEach(jsonData) { key, obj ->
                     hashMap[key] = json.readValue(GameWorld.WiringSimCell::class.java, obj)
-                    printdbg(this, "key: $key, value: ${hashMap[key]}")
                 }
                 return hashMap
             }
@@ -180,17 +178,17 @@ object Common {
         val zo = GZIPOutputStream(bo)
 
         // zip
-        /*b.bytesIterator().forEachRemaining {
+        b.bytesIterator().forEach {
             zo.write(it.toInt())
         }
-        zo.flush(); zo.close()*/
+        zo.flush(); zo.close()
 
         // enascii
         val ba = bo.toByteArray64()
         var bai = 0
         val buf = IntArray(4) { Ascii85.PAD_BYTE }
-//            ba.forEach {
+        ba.forEach {
-        b.bytesIterator().forEachRemaining {
+//        b.bytesIterator().forEachRemaining {
             if (bai > 0 && bai % 4 == 0) {
                 sb.append(Ascii85.encode(buf[0], buf[1], buf[2], buf[3]))
                 buf.fill(Ascii85.PAD_BYTE)
@@ -224,20 +222,20 @@ object Common {
         }; Ascii85.decode(buf[0], buf[1], buf[2], buf[3], buf[4]).forEach { unasciidBytes.add(it) }
 
         // unzip
-        /*val zi = GZIPInputStream(ByteArray64InputStream(unasciidBytes))
+        val zi = GZIPInputStream(ByteArray64InputStream(unasciidBytes))
         while (true) {
             val byte = zi.read()
             if (byte == -1) break
             unzipdBytes.add(byte.toByte())
         }
-        zi.close()*/
+        zi.close()
 
         // write to blocklayer and the digester
         digester.reset()
         var writeCursor = 0L
         val sb = StringBuilder()
-//            unzipdBytes.forEach {
+        unzipdBytes.forEach {
-        unasciidBytes.forEach {
+//        unasciidBytes.forEach {
             if (writeCursor < layer.ptr.size) {
 
                 if (writeCursor < 1024) {
@@ -252,7 +250,7 @@ object Common {
         }
 
 
-        AppLoader.printdbg(this, "post: $sb")
+//        printdbg(this, "post: $sb")
 
 
         // check hash

src/net/torvald/terrarum/serialise/WriteWorld.kt

@@ -2,6 +2,8 @@ package net.torvald.terrarum.serialise
 
 import net.torvald.terrarum.modulebasegame.TerrarumIngame
 import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64
+import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64OutputStream
+import java.io.Writer
 
 /**
  * Created by minjaesong on 2021-08-23.
@@ -10,22 +12,32 @@ open class WriteWorld(val ingame: TerrarumIngame) {
 
     open fun invoke(): String {
         val world = ingame.world
-        //return "{${world.getJsonFields().joinToString(",\n")}}"
+        world.genver = Common.GENVER
+        world.comp = Common.COMP_GZIP
         return Common.jsoner.toJson(world)
     }
 
     fun encodeToByteArray64(): ByteArray64 {
         val world = ingame.world
+        world.genver = Common.GENVER
+        world.comp = Common.COMP_GZIP
+
         val ba = ByteArray64()
-        ba.add('{'.code.toByte())
+        val bao = ByteArray64OutputStream(ba)
-        world.getJsonFields().forEachIndexed { index, str ->
+        val wr = object : Writer() {
-            if (index > 0) {
+            override fun close() {
-                ba.add(','.code.toByte())
+            }
-                ba.add('\n'.code.toByte())
+
+            override fun flush() {
+            }
+
+            override fun write(cbuf: CharArray, off: Int, len: Int) {
+                bao.write(cbuf.copyOfRange(off, off + len).toString().toByteArray())
             }
-            str.toByteArray().forEach { ba.add(it) }
         }
-        ba.add('}'.code.toByte())
+        Common.jsoner.toJson(world, wr)
+        wr.flush(); wr.close()
+
         return ba
     }