quicksave (part of the autosaving) is working

src/net/torvald/terrarum/IngameInstance.kt

@@ -21,7 +21,9 @@ import org.khelekore.prtree.*
 import java.io.File
 import java.io.FileNotFoundException
 import java.io.IOException
+import java.util.*
 import java.util.concurrent.locks.Lock
+import kotlin.collections.ArrayList
 
 /**
  * Although the game (as product) can have infinitely many stages/planets/etc., those stages must be manually managed by YOU;
@@ -123,7 +125,7 @@ open class IngameInstance(val batch: SpriteBatch) : Screen {
     val wallChangeQueue = ArrayList<BlockChangeQueueItem>()
     val wireChangeQueue = ArrayList<BlockChangeQueueItem>() // if 'old' is set and 'new' is blank, it's a wire cutter
 
-    val modifiedChunks = Array(16) { HashSet<Int>() }
+    val modifiedChunks = Array(16) { TreeSet<Int>() }
 
     internal var creationTime = App.getTIME_T() // cumulative value for the savegame
     internal var lastPlayTime = App.getTIME_T() // cumulative value for the savegame
@@ -378,6 +380,8 @@ open class IngameInstance(val batch: SpriteBatch) : Screen {
         } catch (e: NoSuchFileException) {}
     }
 
+    fun getSaveFileMain() = File(App.defaultSaveDir, savegameNickname)
+
 
 
     // simple euclidean norm, squared

src/net/torvald/terrarum/Terrarum.kt

@@ -33,6 +33,7 @@ import net.torvald.terrarumsansbitmap.gdx.TextureRegionPack
 import net.torvald.util.CircularArray
 import java.io.File
 import java.io.PrintStream
+import java.util.logging.Level
 import kotlin.math.absoluteValue
 import kotlin.math.round
 
@@ -682,7 +683,9 @@ fun AppUpdateListOfSavegames() {
     App.savegames.clear()
     File(App.defaultSaveDir).listFiles().filter { !it.isDirectory && !it.name.contains('.') }.map { file ->
         try {
-            VDUtil.readDiskArchive(file, charset = Common.CHARSET)
+            VDUtil.readDiskArchive(file, Level.INFO) {
+                printdbgerr("Terrarum", "Possibly corrupted savefile '${file.absolutePath}':\n$it")
+            }
         }
         catch (e: Throwable) {
             e.printStackTrace()

src/net/torvald/terrarum/console/CommandInterpreter.kt

@@ -25,7 +25,8 @@ internal object CommandInterpreter {
             "resize",
             "echo",
             "error",
-            "seed"
+            "seed",
+            "quicksave"
     )
 
     internal fun execute(command: String) {

src/net/torvald/terrarum/debuggerapp/SavegameCracker.kt

@@ -140,7 +140,7 @@ class SavegameCracker(
     @Command("Loads a disk archive", "path-to-file")
     fun load(args: List<String>) {
         file = File(args[1])
-        disk = VDUtil.readDiskArchive(file!!, Level.SEVERE, { printerrln("# Warning: $it") }, charset)
+        disk = VDUtil.readDiskArchive(file!!, Level.INFO) { printerrln("# Warning: $it") }
         file!!.copyTo(File(file!!.absolutePath + ".bak"), true)
     }
 

src/net/torvald/terrarum/modulebasegame/TerrarumIngame.kt

@@ -313,7 +313,7 @@ open class TerrarumIngame(batch: SpriteBatch) : IngameInstance(batch) {
         )
 
         // make initial savefile
-        WriteSavegame.immediate(savegameArchive, File(App.defaultSaveDir, savegameNickname), this) {
+        WriteSavegame.immediate(savegameArchive, getSaveFileMain(), this) {
             makeSavegameBackupCopy()
         }
     }

src/net/torvald/terrarum/modulebasegame/console/Load.kt

@@ -11,6 +11,7 @@ import net.torvald.terrarum.tvda.VDUtil
 import net.torvald.terrarum.serialise.*
 import java.io.File
 import java.io.IOException
+import java.util.logging.Level
 
 /**
  * Created by minjaesong on 2021-08-30.
@@ -24,7 +25,7 @@ object Load : ConsoleCommand {
 
                 val charset = Common.CHARSET
                 val file = File(App.defaultDir + "/Exports/${args[1]}")
-                val disk = VDUtil.readDiskArchive(file, charset = charset)
+                val disk = VDUtil.readDiskArchive(file, Level.INFO)
 
                 LoadSavegame(disk)
             }

src/net/torvald/terrarum/modulebasegame/console/Save.kt

@@ -22,18 +22,6 @@ import java.io.IOException
  */
 object Save : ConsoleCommand {
 
-    private fun acceptable(actor: Actor): Boolean {
-        return actor.referenceID !in ReferencingRanges.ACTORS_WIRES &&
-               actor.referenceID !in ReferencingRanges.ACTORS_WIRES_HELPER &&
-               actor != (CommonResourcePool.get("blockmarking_actor") as BlockMarkerActor)
-    }
-
-    private fun addFile(disk: VirtualDisk, file: DiskEntry) {
-        VDUtil.getAsDirectory(disk, 0).add(file.entryID)
-        disk.entries[file.entryID] = file
-        file.parentEntryID = 0
-    }
-
     override fun execute(args: Array<String>) {
         if (args.size == 2) {
             try {
@@ -59,4 +47,18 @@ object Save : ConsoleCommand {
         Echo("Usage: save <new-savegame-name>")
     }
 
+}
+
+object Quicksave : ConsoleCommand {
+    override fun execute(args: Array<String>) {
+        val ingame = Terrarum.ingame!! as TerrarumIngame
+
+        WriteSavegame.quick(ingame.savegameArchive, ingame.getSaveFileMain(), ingame) {
+
+        }
+    }
+
+    override fun printUsage() {
+        Echo("Usage: quicksave")
+    }
 }

src/net/torvald/terrarum/realestate/LandUtil.kt

@@ -22,15 +22,24 @@ object LandUtil {
     fun toChunkNum(world: GameWorld, x: Int, y: Int): Int {
         // coercing and fmod-ing follows ROUNDWORLD rule. See: GameWorld.coerceXY()
         val (x, y) = world.coerceXY(x, y)
-        return (x / CHUNK_W) + (y / CHUNK_H) * (world.width / CHUNK_W)
+        return chunkXYtoChunkNum(world, x / CHUNK_W, y / CHUNK_H)
     }
 
-    fun toChunkIndices(world: GameWorld, x: Int, y: Int): Point2i {
+    fun toChunkXY(world: GameWorld, x: Int, y: Int): Point2i {
         // coercing and fmod-ing follows ROUNDWORLD rule. See: GameWorld.coerceXY()
         val (x, y) = world.coerceXY(x, y)
         return Point2i(x / CHUNK_W, y / CHUNK_H)
     }
 
+    fun chunkXYtoChunkNum(world: GameWorld, cx: Int, cy: Int): Int {
+        val ch = world.height / CHUNK_H
+        return cx * ch + cy
+    }
+    fun chunkNumToChunkXY(world: GameWorld, chunkNum: Int): Point2i {
+        val ch = world.height / CHUNK_H
+        return Point2i(chunkNum / ch, chunkNum % ch)
+    }
+
     fun getBlockAddr(world: GameWorld, x: Int, y: Int): BlockAddress {
         // coercing and fmod-ing follows ROUNDWORLD rule. See: GameWorld.coerceXY()
         val (x, y) = world.coerceXY(x, y)

src/net/torvald/terrarum/serialise/GameSavingThread.kt

@@ -119,7 +119,7 @@ class GameSavingThread(val disk: VirtualDisk, val outFile: File, val ingame: Ter
         for (layer in layers.indices) {
             for (cx in 0 until cw) {
                 for (cy in 0 until ch) {
-                    val chunkNumber = (cx * ch + cy).toLong()
+                    val chunkNumber = LandUtil.chunkXYtoChunkNum(ingame.world, cx, cy).toLong()
 
                     Echo("Writing chunks... ${(cw*ch*layer) + chunkNumber + 1}/${cw*ch*layers.size}")
 

src/net/torvald/terrarum/serialise/QuickSaveThread.kt

@@ -0,0 +1,145 @@
+package net.torvald.terrarum.serialise
+
+import net.torvald.gdx.graphics.PixmapIO2
+import net.torvald.terrarum.App
+import net.torvald.terrarum.ccG
+import net.torvald.terrarum.ccW
+import net.torvald.terrarum.console.Echo
+import net.torvald.terrarum.modulebasegame.IngameRenderer
+import net.torvald.terrarum.modulebasegame.TerrarumIngame
+import net.torvald.terrarum.realestate.LandUtil
+import net.torvald.terrarum.tvda.*
+import java.io.File
+import java.util.zip.GZIPOutputStream
+
+/**
+ * Created by minjaesong on 2021-09-29.
+ */
+class QuickSaveThread(val disk: VirtualDisk, val file: File, val ingame: TerrarumIngame, val hasThumbnail: Boolean, val callback: () -> Unit) : Runnable {
+
+    /**
+     * Will happily overwrite existing entry
+     */
+    private fun addFile(disk: VirtualDisk, file: DiskEntry) {
+        disk.entries[file.entryID] = file
+        file.parentEntryID = 0
+        val dir = VDUtil.getAsDirectory(disk, 0)
+        if (!dir.contains(file.entryID)) dir.add(file.entryID)
+    }
+
+
+    private val chunkProgressMultiplier = 1f
+    private val actorProgressMultiplier = 1f
+
+
+    override fun run() {
+        val skimmer = DiskSkimmer(file, Common.CHARSET)
+
+        if (hasThumbnail) {
+            while (!IngameRenderer.fboRGBexportedLatch) {
+                Thread.sleep(1L)
+            }
+        }
+
+        val actorsList = listOf(ingame.actorContainerActive).flatMap { it.filter { WriteWorld.actorAcceptable(it) } }
+        val chunks = ingame.modifiedChunks
+
+        val chunkCount = chunks.map { it.size }.sum()
+
+        WriteSavegame.saveProgress = 0f
+        WriteSavegame.saveProgressMax = 2f +
+                                        (chunkCount) * chunkProgressMultiplier +
+                                        actorsList.size * actorProgressMultiplier
+
+
+        val tgaout = ByteArray64GrowableOutputStream()
+        val gzout = GZIPOutputStream(tgaout)
+
+        Echo("Writing metadata...")
+
+        val creation_t = ingame.creationTime
+        val time_t = App.getTIME_T()
+        
+
+        // Write Meta //
+        val metaContent = EntryFile(WriteMeta.encodeToByteArray64(ingame, time_t))
+        val meta = DiskEntry(-1, 0, creation_t, time_t, metaContent)
+        addFile(disk, meta); skimmer.appendEntryOnly(meta)
+
+        if (hasThumbnail) {
+            PixmapIO2._writeTGA(gzout, IngameRenderer.fboRGBexport, true, true)
+            IngameRenderer.fboRGBexport.dispose()
+
+            val thumbContent = EntryFile(tgaout.toByteArray64())
+            val thumb = DiskEntry(-2, 0, creation_t, time_t, thumbContent)
+            addFile(disk, thumb); skimmer.appendEntryOnly(thumb)
+        }
+
+        WriteSavegame.saveProgress += 1f
+
+
+        // Write World //
+        val worldNum = ingame.world.worldIndex
+        val worldMeta = EntryFile(WriteWorld.encodeToByteArray64(ingame, time_t))
+        val world = DiskEntry(worldNum.toLong(), 0, creation_t, time_t, worldMeta)
+        addFile(disk, world); skimmer.appendEntryOnly(world)
+
+        WriteSavegame.saveProgress += 1f
+
+        var chunksWrote = 1
+        chunks.forEachIndexed { layerNum, chunks ->
+
+            if (chunks.size != 0) {
+                val layer = ingame.world.getLayer(layerNum)
+
+                chunks.forEach { chunkNumber ->
+
+                    Echo("Writing chunks... $chunksWrote/$chunkCount")
+
+                    val chunkXY = LandUtil.chunkNumToChunkXY(ingame.world, chunkNumber)
+
+//                    println("Chunk xy from number $chunkNumber -> (${chunkXY.x}, ${chunkXY.y})")
+
+                    val chunkBytes = WriteWorld.encodeChunk(layer, chunkXY.x, chunkXY.y)
+                    val entryID = worldNum.toLong().shl(32) or layerNum.toLong().shl(24) or chunkNumber.toLong()
+
+                    val entryContent = EntryFile(chunkBytes)
+                    val entry = DiskEntry(entryID, 0, creation_t, time_t, entryContent)
+                    // "W1L0-92,15"
+                    addFile(disk, entry); skimmer.appendEntryOnly(entry)
+
+                    WriteSavegame.saveProgress += chunkProgressMultiplier
+                    chunksWrote += 1
+
+                }
+            }
+        }
+
+
+        // Write Actors //
+        actorsList.forEachIndexed { count, it ->
+            Echo("Writing actors... ${count+1}/${actorsList.size}")
+
+            val actorContent = EntryFile(WriteActor.encodeToByteArray64(it))
+            val actor = DiskEntry(it.referenceID.toLong(), 0, creation_t, time_t, actorContent)
+            addFile(disk, actor); skimmer.appendEntryOnly(actor)
+
+            WriteSavegame.saveProgress += actorProgressMultiplier
+        }
+
+
+        skimmer.rewriteDirectories()
+        skimmer.injectDiskCRC(disk.hashCode())
+
+
+        Echo ("${ccW}Game saved with size of $ccG${file.length()}$ccW bytes")
+
+
+        if (hasThumbnail) IngameRenderer.fboRGBexportedLatch = false
+        WriteSavegame.savingStatus = 255
+        ingame.modifiedChunks.forEach { it.clear() }
+
+        callback()
+    }
+
+}

src/net/torvald/terrarum/serialise/WriteSavegame.kt

@@ -67,6 +67,33 @@ object WriteSavegame {
         // it is caller's job to keep the game paused or keep a "save in progress" ui up
         // use field 'savingStatus' to know when the saving is done
     }
+
+    fun quick(disk: VirtualDisk, file: File, ingame: TerrarumIngame, callback: () -> Unit = {}) {
+        savingStatus = 0
+
+        Echo("Quicksave queued")
+
+        IngameRenderer.fboRGBexportCallback = {
+            Echo("Generating thumbnail...")
+
+            val w = 960
+            val h = 640
+            val p = Pixmap.createFromFrameBuffer((it.width - w).ushr(1), (it.height - h).ushr(1), w, h)
+            IngameRenderer.fboRGBexport = p
+            //PixmapIO2._writeTGA(gzout, p, true, true)
+            //p.dispose()
+            IngameRenderer.fboRGBexportedLatch = true
+
+            Echo("Done thumbnail generation")
+        }
+        IngameRenderer.fboRGBexportRequested = true
+
+        val savingThread = Thread(QuickSaveThread(disk, file, ingame, true, callback), "TerrarumBasegameGameSaveThread")
+        savingThread.start()
+
+        // it is caller's job to keep the game paused or keep a "save in progress" ui up
+        // use field 'savingStatus' to know when the saving is done
+    }
 }
 
 
@@ -128,9 +155,8 @@ object LoadSavegame {
 
             // load all the world blocklayer chunks
             val worldnum = world.worldIndex.toLong()
-            val cw = LandUtil.CHUNK_W;
+            val cw = LandUtil.CHUNK_W
             val ch = LandUtil.CHUNK_H
-            val chunksY = world.height / ch
             val chunkCount = world.width * world.height / (cw * ch)
             val worldLayer = arrayOf(world.getLayer(0), world.getLayer(1))
             for (chunk in 0L until (world.width * world.height) / (cw * ch)) {
@@ -138,10 +164,9 @@ object LoadSavegame {
                     loadscreen.addMessage("${Lang["MENU_IO_LOADING"]}  ${chunk*worldLayer.size+layer+1}/${chunkCount*2}")
 
                     val chunkFile = VDUtil.getAsNormalFile(disk, worldnum.shl(32) or layer.toLong().shl(24) or chunk)
-                    val cy = chunk % chunksY
+                    val chunkXY = LandUtil.chunkNumToChunkXY(world, chunk.toInt())
-                    val cx = chunk / chunksY
 
-                    ReadWorld.decodeChunkToLayer(chunkFile.getContent(), worldLayer[layer], cx.toInt(), cy.toInt())
+                    ReadWorld.decodeChunkToLayer(chunkFile.getContent(), worldLayer[layer], chunkXY.x, chunkXY.y)
                 }
             }
 

src/net/torvald/terrarum/tvda/DiskSkimmer.kt

@@ -3,6 +3,7 @@ package net.torvald.terrarum.tvda
 import java.io.*
 import java.nio.charset.Charset
 import java.util.*
+import java.util.logging.Level
 import kotlin.experimental.and
 
 /**
@@ -120,19 +121,19 @@ removefile:
         val currentLength = diskFile.length()
         while (currentPosition < currentLength) {
 
-            val entryID = readLongBig() // at this point, cursor is 4 bytes past to the entry head
+            val entryID = readLongBig() // at this point, cursor is 8 bytes past to the entry head
 
             // fill up the offset table
             val offset = currentPosition
 
-            skipRead(8)
+            skipRead(8) // parent ID
             val typeFlag = readByte()
             skipRead(3)
             skipRead(16) // skip rest of the header
 
             val entrySize = when (typeFlag and 127) {
                 DiskEntry.NORMAL_FILE -> readInt48()
-                DiskEntry.DIRECTORY -> readIntBig().toLong()
+                DiskEntry.DIRECTORY -> readIntBig().toLong() * 8L
                 else -> 0
             }
 
@@ -270,17 +271,59 @@ removefile:
     }*/
 
     fun invalidateEntry(id: EntryID) {
-        fa.seek(entryToOffsetTable[id]!! + 8)
+        entryToOffsetTable[id]?.let {
-        val type = fa.read()
+            fa.seek(it + 8)
-        fa.seek(entryToOffsetTable[id]!! + 8)
+            val type = fa.read()
-        fa.write(type or 128)
+            fa.seek(it + 8)
-        entryToOffsetTable.remove(id)
+            fa.write(type or 128)
+            entryToOffsetTable.remove(id)
+        }
+    }
+
+
+    fun injectDiskCRC(crc: Int) {
+        fa.seek(42L)
+        fa.write(crc.toBigEndian())
+    }
+
+    private val modifiedDirectories = TreeSet<DiskEntry>()
+
+    fun rewriteDirectories() {
+        modifiedDirectories.forEach {
+            invalidateEntry(it.entryID)
+
+            val appendAt = fa.length()
+            fa.seek(appendAt)
+
+            // append new file
+            entryToOffsetTable[it.entryID] = appendAt + 8
+            it.serialize().forEach { fa.writeByte(it.toInt()) }
+        }
     }
 
     ///////////////////////////////////////////////////////
     // THESE ARE METHODS TO SUPPORT ON-LINE MODIFICATION //
     ///////////////////////////////////////////////////////
 
+    fun appendEntryOnly(entry: DiskEntry) {
+        val parentDir = requestFile(entry.parentEntryID)!!
+        val id = entry.entryID
+
+        // add the entry to its parent directory if there was none
+        val dirContent = (parentDir.contents as EntryDirectory)
+        if (!dirContent.contains(id)) dirContent.add(id)
+        modifiedDirectories.add(parentDir)
+
+        invalidateEntry(id)
+
+        val appendAt = fa.length()
+        fa.seek(appendAt)
+
+        // append new file
+        entryToOffsetTable[id] = appendAt + 8
+        entry.serialize().forEach { fa.writeByte(it.toInt()) }
+    }
+
     fun appendEntry(entry: DiskEntry) {
         val parentDir = requestFile(entry.parentEntryID)!!
         val id = entry.entryID
@@ -331,7 +374,7 @@ removefile:
      */
     fun sync(): VirtualDisk {
         // rebuild VirtualDisk out of this and use it to write out
-        return VDUtil.readDiskArchive(diskFile, charset = charset)
+        return VDUtil.readDiskArchive(diskFile, Level.INFO)
     }
 
 

src/net/torvald/terrarum/tvda/VDUtil.kt

@@ -47,7 +47,7 @@ object VDUtil {
      *
      * @param crcWarnLevel Level.OFF -- no warning, Level.WARNING -- print out warning, Level.SEVERE -- throw error
      */
-    fun readDiskArchive(infile: File, crcWarnLevel: Level = Level.SEVERE, warningFunc: ((String) -> Unit)? = null, charset: Charset): VirtualDisk {
+    fun readDiskArchive(infile: File, crcWarnLevel: Level = Level.SEVERE, warningFunc: ((String) -> Unit)? = null): VirtualDisk {
         val inbytes = infile.readBytes64()
 
 
@@ -100,7 +100,7 @@ object VDUtil {
             }
 
             if (DEBUG_PRINT_READ) {
-                println("== Entry deserialise debugprint for entry ID $entryID (child of $entryParentID)")
+                println("[tvda.VDUtil] == Entry deserialise debugprint for entry ID $entryID (child of $entryParentID)")
                 println("Entry type flag: ${entryTypeFlag and 127}${if (entryTypeFlag < 0) "*" else ""}")
                 println("Entry raw contents bytes: (len: ${entryData.size})")
                 entryData.forEachIndexed { i, it ->
@@ -121,7 +121,7 @@ object VDUtil {
 
 
             // check for the discard bit
-            if (entryTypeFlag in 1..127) {
+            if (entryTypeFlag > 0) {
 
                 // create entry
                 val diskEntry = DiskEntry(
@@ -158,7 +158,7 @@ object VDUtil {
                         (diskEntry.contents as? EntryDirectory)?.forEach {
                             println("entry: ${it.toHex()}")
                         }
-                        println("bytes to calculate crc against:")
+                        println("[tvda.VDUtil] bytes to calculate crc against:")
                         testbytes.forEachIndexed { i, it ->
                             if (i % 4 == 0L) print(" ")
                             print(it.toInt().toHex().substring(6))
@@ -175,7 +175,7 @@ object VDUtil {
 
                     if (calculatedCRC != entryCRC) {
 
-                        println("CRC failed; entry info:\n$diskEntry")
+                        println("[tvda.VDUtil] CRC failed; entry info:\n$diskEntry")
 
                         if (crcWarnLevel == Level.SEVERE)
                             throw IOException(crcMsg)
@@ -187,6 +187,11 @@ object VDUtil {
                 // add entry to disk
                 vdisk.entries[entryID] = diskEntry
             }
+            else {
+                if (DEBUG_PRINT_READ) {
+                    println("[tvda.VDUtil] Discarding entry ${entryID.toHex()} (raw type flag: $entryTypeFlag)")
+                }
+            }
         }
 
         // check CRC of disk

src/net/torvald/terrarum/tvda/VirtualDisk.kt

@@ -234,7 +234,10 @@ class DiskEntry(
 
         // content
         val contents: DiskEntryContent
-) {
+): Comparable<DiskEntry> {
+
+    override fun compareTo(other: DiskEntry) = entryID.compareTo(other.entryID)
+
     val serialisedSize: Long
         get() = contents.getSizeEntry() + HEADER_SIZE
 

src/net/torvald/terrarum/tvda/finder/VirtualDiskCracker.kt

@@ -205,7 +205,7 @@ class VirtualDiskCracker(val sysCharset: Charset = Charsets.UTF_8) : JFrame() {
                     fileChooser.showOpenDialog(null)
                     if (fileChooser.selectedFile != null) {
                         try {
-                            vdisk = VDUtil.readDiskArchive(fileChooser.selectedFile, Level.WARNING, { popupWarning(it) }, sysCharset)
+                            vdisk = VDUtil.readDiskArchive(fileChooser.selectedFile, Level.WARNING) { popupWarning(it) }
                             if (vdisk != null) {
                                 gotoRoot()
                                 updateDiskInfo()