chunk pool wip

src/net/torvald/terrarum/gameworld/ChunkPool.kt

@@ -0,0 +1,114 @@
+package net.torvald.terrarum.gameworld
+
+import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.archivers.ClusteredFormatDOM
+import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.archivers.Clustfile
+import net.torvald.terrarum.realestate.LandUtil.CHUNK_H
+import net.torvald.terrarum.realestate.LandUtil.CHUNK_W
+import net.torvald.terrarum.serialise.Common
+import net.torvald.terrarum.serialise.toUint
+import net.torvald.unsafe.UnsafeHelper
+import net.torvald.unsafe.UnsafePtr
+import java.util.TreeMap
+
+/**
+ * Single layer gets single Chunk Pool.
+ *
+ * Created by minjaesong on 2024-09-07.
+ */
+open class ChunkPool(
+    val DOM: ClusteredFormatDOM,
+    val wordSizeInBytes: Int,
+    val chunkNumToFileNum: (Int) -> String,
+    val renumberFun: (Int) -> Int,
+) {
+    private val pointers = TreeMap<Int, Long>()
+    private var allocCap = 32
+    private var allocMap = ArrayList<Int>(allocCap)
+    private var allocCounter = 0
+
+    private val chunkSize = (wordSizeInBytes.toLong() * CHUNK_W * CHUNK_H)
+    private val pool = UnsafeHelper.allocate(chunkSize * allocCap)
+
+    init {
+        allocMap.fill(-1)
+    }
+
+    private fun createPointerViewOfChunk(chunkNumber: Int): UnsafePtr {
+        val baseAddr = pointers[chunkNumber]!!
+        return UnsafePtr(baseAddr, chunkSize)
+    }
+
+    private fun createPointerViewOfChunk(chunkNumber: Int, offsetX: Int, offsetY: Int): Pair<UnsafePtr, Long> {
+        val baseAddr = pointers[chunkNumber]!!
+        return UnsafePtr(baseAddr, chunkSize) to wordSizeInBytes.toLong() * (offsetY * CHUNK_W + offsetX)
+    }
+
+    private fun allocate(chunkNumber: Int): UnsafePtr {
+        // expand the pool if needed
+        if (allocCounter >= allocCap) {
+            allocCap *= 2
+            allocMap.ensureCapacity(allocCap)
+            pool.realloc(chunkSize * allocCap)
+        }
+
+        // find the empty spot
+        val idx = allocMap.indexOfFirst { it == -1 }
+        val ptr = pool.ptr + idx * chunkSize
+
+        allocCounter += 1
+
+        pointers[chunkNumber] = ptr
+        return UnsafePtr(ptr, chunkSize)
+    }
+
+    private fun deallocate(chunkNumber: Int) {
+        val ptr = pointers[chunkNumber] ?: return
+        storeToDisk(chunkNumber)
+        pointers.remove(chunkNumber)
+        allocMap[chunkNumber] = -1
+        allocCounter -= 1
+    }
+
+    private fun renumber(ptr: UnsafePtr) {
+        for (i in 0 until ptr.size step wordSizeInBytes.toLong()) {
+            val numIn = (0 until wordSizeInBytes).fold(0) { acc, off ->
+                acc or (ptr[i + off].toUint().shl(8 * off))
+            }
+            val numOut = renumberFun(numIn)
+            (0 until wordSizeInBytes).forEach { off ->
+                ptr[i + off] = numOut.ushr(8 * off).toByte()
+            }
+        }
+    }
+
+    private fun fetchFromDisk(chunkNumber: Int) {
+        // read data from the disk
+        val fileName = chunkNumToFileNum(chunkNumber)
+        Clustfile(DOM, fileName).let {
+            val bytes = Common.unzip(it.readBytes())
+            val ptr = allocate(chunkNumber)
+            UnsafeHelper.memcpyFromArrToPtr(bytes, 0, ptr.ptr, bytes.size)
+            renumber(ptr)
+        }
+    }
+
+    private fun storeToDisk(chunkNumber: Int) {
+        TODO()
+    }
+
+    private fun checkForChunk(chunkNumber: Int) {
+        if (!pointers.containsKey(chunkNumber)) {
+            fetchFromDisk(chunkNumber)
+        }
+    }
+
+    fun getTileRaw(chunkNumber: Int, offX: Int, offY: Int): Int {
+        checkForChunk(chunkNumber)
+        val (ptr, ptrOff) = createPointerViewOfChunk(chunkNumber, offX, offY)
+        val numIn = (0 until wordSizeInBytes).fold(0) { acc, off ->
+            acc or (ptr[ptrOff].toUint().shl(8 * off))
+        }
+        return numIn
+    }
+
+}

src/net/torvald/terrarum/serialise/Common.kt

@@ -20,10 +20,7 @@ import net.torvald.terrarum.savegame.ByteArray64Reader
 import net.torvald.terrarum.utils.*
 import net.torvald.terrarum.weather.*
 import org.apache.commons.codec.digest.DigestUtils
-import java.io.File
-import java.io.InputStream
-import java.io.Reader
-import java.io.StringReader
+import java.io.*
 import java.math.BigInteger
 import java.util.*
 import java.util.zip.GZIPInputStream
@@ -562,6 +559,18 @@ object Common {
         return unzipdBytes
     }
 
+    private fun unzipG(bytes: ByteArray): ByteArray {
+        val unzipdBytes = ByteArray64()
+        val zi = GZIPInputStream(ByteArrayInputStream(bytes))
+        while (true) {
+            val byte = zi.read()
+            if (byte == -1) break
+            unzipdBytes.appendByte(byte.toByte())
+        }
+        zi.close()
+        return unzipdBytes.toByteArray()
+    }
+
     private fun unzipZ(bytes: ByteArray64): ByteArray64 {
         val unzipdBytes = ByteArray64()
         val zi = ZstdInputStream(ByteArray64InputStream(bytes))
@@ -574,6 +583,18 @@ object Common {
         return unzipdBytes
     }
 
+    private fun unzipZ(bytes: ByteArray): ByteArray {
+        val unzipdBytes = ByteArray64()
+        val zi = ZstdInputStream(ByteArrayInputStream(bytes))
+        while (true) {
+            val byte = zi.read()
+            if (byte == -1) break
+            unzipdBytes.appendByte(byte.toByte())
+        }
+        zi.close()
+        return unzipdBytes.toByteArray()
+    }
+
     private fun unzipS(bytes: ByteArray64): ByteArray64 {
         val unzipdBytes = ByteArray64()
         val zi = SnappyFramedInputStream(ByteArray64InputStream(bytes))
@@ -586,6 +607,18 @@ object Common {
         return unzipdBytes
     }
 
+    private fun unzipS(bytes: ByteArray): ByteArray {
+        val unzipdBytes = ByteArray64()
+        val zi = SnappyFramedInputStream(ByteArrayInputStream(bytes))
+        while (true) {
+            val byte = zi.read()
+            if (byte == -1) break
+            unzipdBytes.appendByte(byte.toByte())
+        }
+        zi.close()
+        return unzipdBytes.toByteArray()
+    }
+
     /*private fun unzipNull(bytes: ByteArray64): ByteArray64 {
         return bytes.sliceArray64(4 until bytes.size)
     }*/
@@ -604,6 +637,20 @@ object Common {
         }
     }
 
+    fun unzip(bytes: ByteArray): ByteArray {
+        val header = bytes[0].toUint().shl(24) or bytes[1].toUint().shl(16) or bytes[2].toUint().shl(8) or bytes[3].toUint()
+
+        // to save yourself from the curiosity: load time of the null compression is no faster than the snappy
+
+        return when (header) {
+            in 0x1F8B0800..0x1F8B08FF -> unzipG(bytes)
+            0x28B52FFD -> unzipZ(bytes)
+            0xFF060000.toInt() -> unzipS(bytes)
+//            0xFEEDDA7A.toInt() -> unzipNull(bytes)
+            else -> throw IllegalArgumentException("Unknown archive with header ${header.toHex()}")
+        }
+    }
+
     fun unasciiToBytes(reader: Reader): ByteArray64 {
         val unasciidBytes = ByteArray64()
 

src/net/torvald/unsafe/UnsafePtr.kt

@@ -41,11 +41,15 @@ internal object UnsafeHelper {
     }
 
     fun memcpy(src: UnsafePtr, fromIndex: Long, dest: UnsafePtr, toIndex: Long, copyLength: Long) =
-            unsafe.copyMemory(src.ptr + fromIndex, dest.ptr + toIndex, copyLength)
+        unsafe.copyMemory(src.ptr + fromIndex, dest.ptr + toIndex, copyLength)
     fun memcpy(srcAddress: Long, destAddress: Long, copyLength: Long) =
-            unsafe.copyMemory(srcAddress, destAddress, copyLength)
+        unsafe.copyMemory(srcAddress, destAddress, copyLength)
     fun memcpyRaw(srcObj: Any?, srcPos: Long, destObj: Any?, destPos: Long, len: Long) =
-            unsafe.copyMemory(srcObj, srcPos, destObj, destPos, len)
+        unsafe.copyMemory(srcObj, srcPos, destObj, destPos, len)
+    fun memcpyFromArrToPtr(srcObj: Any, startIndex: Int, destPos: Long, len: Int) =
+        unsafe.copyMemory(srcObj, getArrayOffset(srcObj) + startIndex, null, destPos, len.toLong())
+    fun memcpyFromArrToPtr(srcObj: Any, startIndex: Int, destPos: Long, len: Long) =
+        unsafe.copyMemory(srcObj, getArrayOffset(srcObj) + startIndex, null, destPos, len)
 
     /**
      * The array object in JVM is stored in this memory map: