diff --git a/src/net/torvald/terrarum/gameworld/ChunkPool.kt b/src/net/torvald/terrarum/gameworld/ChunkPool.kt new file mode 100644 index 000000000..f0bcf99ec --- /dev/null +++ b/src/net/torvald/terrarum/gameworld/ChunkPool.kt @@ -0,0 +1,114 @@ +package net.torvald.terrarum.gameworld + +import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.archivers.ClusteredFormatDOM +import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.archivers.Clustfile +import net.torvald.terrarum.realestate.LandUtil.CHUNK_H +import net.torvald.terrarum.realestate.LandUtil.CHUNK_W +import net.torvald.terrarum.serialise.Common +import net.torvald.terrarum.serialise.toUint +import net.torvald.unsafe.UnsafeHelper +import net.torvald.unsafe.UnsafePtr +import java.util.TreeMap + +/** + * Single layer gets single Chunk Pool. + * + * Created by minjaesong on 2024-09-07. + */ +open class ChunkPool( + val DOM: ClusteredFormatDOM, + val wordSizeInBytes: Int, + val chunkNumToFileNum: (Int) -> String, + val renumberFun: (Int) -> Int, +) { + private val pointers = TreeMap() + private var allocCap = 32 + private var allocMap = ArrayList(allocCap) + private var allocCounter = 0 + + private val chunkSize = (wordSizeInBytes.toLong() * CHUNK_W * CHUNK_H) + private val pool = UnsafeHelper.allocate(chunkSize * allocCap) + + init { + allocMap.fill(-1) + } + + private fun createPointerViewOfChunk(chunkNumber: Int): UnsafePtr { + val baseAddr = pointers[chunkNumber]!! + return UnsafePtr(baseAddr, chunkSize) + } + + private fun createPointerViewOfChunk(chunkNumber: Int, offsetX: Int, offsetY: Int): Pair { + val baseAddr = pointers[chunkNumber]!! + return UnsafePtr(baseAddr, chunkSize) to wordSizeInBytes.toLong() * (offsetY * CHUNK_W + offsetX) + } + + private fun allocate(chunkNumber: Int): UnsafePtr { + // expand the pool if needed + if (allocCounter >= allocCap) { + allocCap *= 2 + allocMap.ensureCapacity(allocCap) + pool.realloc(chunkSize * allocCap) + } + + // find the empty spot + val idx = allocMap.indexOfFirst { it == -1 } + val ptr = pool.ptr + idx * chunkSize + + allocCounter += 1 + + pointers[chunkNumber] = ptr + return UnsafePtr(ptr, chunkSize) + } + + private fun deallocate(chunkNumber: Int) { + val ptr = pointers[chunkNumber] ?: return + storeToDisk(chunkNumber) + pointers.remove(chunkNumber) + allocMap[chunkNumber] = -1 + allocCounter -= 1 + } + + private fun renumber(ptr: UnsafePtr) { + for (i in 0 until ptr.size step wordSizeInBytes.toLong()) { + val numIn = (0 until wordSizeInBytes).fold(0) { acc, off -> + acc or (ptr[i + off].toUint().shl(8 * off)) + } + val numOut = renumberFun(numIn) + (0 until wordSizeInBytes).forEach { off -> + ptr[i + off] = numOut.ushr(8 * off).toByte() + } + } + } + + private fun fetchFromDisk(chunkNumber: Int) { + // read data from the disk + val fileName = chunkNumToFileNum(chunkNumber) + Clustfile(DOM, fileName).let { + val bytes = Common.unzip(it.readBytes()) + val ptr = allocate(chunkNumber) + UnsafeHelper.memcpyFromArrToPtr(bytes, 0, ptr.ptr, bytes.size) + renumber(ptr) + } + } + + private fun storeToDisk(chunkNumber: Int) { + TODO() + } + + private fun checkForChunk(chunkNumber: Int) { + if (!pointers.containsKey(chunkNumber)) { + fetchFromDisk(chunkNumber) + } + } + + fun getTileRaw(chunkNumber: Int, offX: Int, offY: Int): Int { + checkForChunk(chunkNumber) + val (ptr, ptrOff) = createPointerViewOfChunk(chunkNumber, offX, offY) + val numIn = (0 until wordSizeInBytes).fold(0) { acc, off -> + acc or (ptr[ptrOff].toUint().shl(8 * off)) + } + return numIn + } + +} \ No newline at end of file diff --git a/src/net/torvald/terrarum/serialise/Common.kt b/src/net/torvald/terrarum/serialise/Common.kt index 0cbbbdfd8..75113c9dc 100644 --- a/src/net/torvald/terrarum/serialise/Common.kt +++ b/src/net/torvald/terrarum/serialise/Common.kt @@ -20,10 +20,7 @@ import net.torvald.terrarum.savegame.ByteArray64Reader import net.torvald.terrarum.utils.* import net.torvald.terrarum.weather.* import org.apache.commons.codec.digest.DigestUtils -import java.io.File -import java.io.InputStream -import java.io.Reader -import java.io.StringReader +import java.io.* import java.math.BigInteger import java.util.* import java.util.zip.GZIPInputStream @@ -562,6 +559,18 @@ object Common { return unzipdBytes } + private fun unzipG(bytes: ByteArray): ByteArray { + val unzipdBytes = ByteArray64() + val zi = GZIPInputStream(ByteArrayInputStream(bytes)) + while (true) { + val byte = zi.read() + if (byte == -1) break + unzipdBytes.appendByte(byte.toByte()) + } + zi.close() + return unzipdBytes.toByteArray() + } + private fun unzipZ(bytes: ByteArray64): ByteArray64 { val unzipdBytes = ByteArray64() val zi = ZstdInputStream(ByteArray64InputStream(bytes)) @@ -574,6 +583,18 @@ object Common { return unzipdBytes } + private fun unzipZ(bytes: ByteArray): ByteArray { + val unzipdBytes = ByteArray64() + val zi = ZstdInputStream(ByteArrayInputStream(bytes)) + while (true) { + val byte = zi.read() + if (byte == -1) break + unzipdBytes.appendByte(byte.toByte()) + } + zi.close() + return unzipdBytes.toByteArray() + } + private fun unzipS(bytes: ByteArray64): ByteArray64 { val unzipdBytes = ByteArray64() val zi = SnappyFramedInputStream(ByteArray64InputStream(bytes)) @@ -586,6 +607,18 @@ object Common { return unzipdBytes } + private fun unzipS(bytes: ByteArray): ByteArray { + val unzipdBytes = ByteArray64() + val zi = SnappyFramedInputStream(ByteArrayInputStream(bytes)) + while (true) { + val byte = zi.read() + if (byte == -1) break + unzipdBytes.appendByte(byte.toByte()) + } + zi.close() + return unzipdBytes.toByteArray() + } + /*private fun unzipNull(bytes: ByteArray64): ByteArray64 { return bytes.sliceArray64(4 until bytes.size) }*/ @@ -604,6 +637,20 @@ object Common { } } + fun unzip(bytes: ByteArray): ByteArray { + val header = bytes[0].toUint().shl(24) or bytes[1].toUint().shl(16) or bytes[2].toUint().shl(8) or bytes[3].toUint() + + // to save yourself from the curiosity: load time of the null compression is no faster than the snappy + + return when (header) { + in 0x1F8B0800..0x1F8B08FF -> unzipG(bytes) + 0x28B52FFD -> unzipZ(bytes) + 0xFF060000.toInt() -> unzipS(bytes) +// 0xFEEDDA7A.toInt() -> unzipNull(bytes) + else -> throw IllegalArgumentException("Unknown archive with header ${header.toHex()}") + } + } + fun unasciiToBytes(reader: Reader): ByteArray64 { val unasciidBytes = ByteArray64() diff --git a/src/net/torvald/unsafe/UnsafePtr.kt b/src/net/torvald/unsafe/UnsafePtr.kt index 9495dd76e..e7aceeb1b 100644 --- a/src/net/torvald/unsafe/UnsafePtr.kt +++ b/src/net/torvald/unsafe/UnsafePtr.kt @@ -41,11 +41,15 @@ internal object UnsafeHelper { } fun memcpy(src: UnsafePtr, fromIndex: Long, dest: UnsafePtr, toIndex: Long, copyLength: Long) = - unsafe.copyMemory(src.ptr + fromIndex, dest.ptr + toIndex, copyLength) + unsafe.copyMemory(src.ptr + fromIndex, dest.ptr + toIndex, copyLength) fun memcpy(srcAddress: Long, destAddress: Long, copyLength: Long) = - unsafe.copyMemory(srcAddress, destAddress, copyLength) + unsafe.copyMemory(srcAddress, destAddress, copyLength) fun memcpyRaw(srcObj: Any?, srcPos: Long, destObj: Any?, destPos: Long, len: Long) = - unsafe.copyMemory(srcObj, srcPos, destObj, destPos, len) + unsafe.copyMemory(srcObj, srcPos, destObj, destPos, len) + fun memcpyFromArrToPtr(srcObj: Any, startIndex: Int, destPos: Long, len: Int) = + unsafe.copyMemory(srcObj, getArrayOffset(srcObj) + startIndex, null, destPos, len.toLong()) + fun memcpyFromArrToPtr(srcObj: Any, startIndex: Int, destPos: Long, len: Long) = + unsafe.copyMemory(srcObj, getArrayOffset(srcObj) + startIndex, null, destPos, len) /** * The array object in JVM is stored in this memory map: