diff --git a/src/net/torvald/terrarum/modulebasegame/serialise/WriteWorld.kt b/src/net/torvald/terrarum/modulebasegame/serialise/WriteWorld.kt index 9c44974f0..03b2ee47e 100644 --- a/src/net/torvald/terrarum/modulebasegame/serialise/WriteWorld.kt +++ b/src/net/torvald/terrarum/modulebasegame/serialise/WriteWorld.kt @@ -83,8 +83,8 @@ object WriteWorld { for (y in cy * LandUtil.CHUNK_H until (cy + 1) * LandUtil.CHUNK_H) { for (x in cx * LandUtil.CHUNK_W until (cx + 1) * LandUtil.CHUNK_W) { val tilenum = layer.unsafeGetTile(x, y) - ba.add(tilenum.ushr(8).and(255).toByte()) - ba.add(tilenum.and(255).toByte()) + ba.appendByte(tilenum.ushr(8).and(255).toByte()) + ba.appendByte(tilenum.and(255).toByte()) } } diff --git a/src/net/torvald/terrarum/savegame/ByteArray64.kt b/src/net/torvald/terrarum/savegame/ByteArray64.kt index befe74315..7d97887f7 100644 --- a/src/net/torvald/terrarum/savegame/ByteArray64.kt +++ b/src/net/torvald/terrarum/savegame/ByteArray64.kt @@ -18,7 +18,7 @@ import java.nio.charset.UnsupportedCharsetException * * Created by Minjaesong on 2017-04-12. */ -class ByteArray64(initialSize: Long = bankSize.toLong()) { +class ByteArray64(initialSize: Long = BANK_SIZE.toLong()) { var internalCapacity: Long = initialSize private set @@ -28,7 +28,7 @@ class ByteArray64(initialSize: Long = bankSize.toLong()) { private var finalised = false companion object { - val bankSize: Int = 8192 + val BANK_SIZE: Int = 8192 fun fromByteArray(byteArray: ByteArray): ByteArray64 { val ba64 = ByteArray64(byteArray.size.toLong()) @@ -47,16 +47,16 @@ class ByteArray64(initialSize: Long = bankSize.toLong()) { if (internalCapacity < 0) throw IllegalArgumentException("Invalid array size: $internalCapacity") else if (internalCapacity == 0L) // signalling empty array - internalCapacity = bankSize.toLong() + internalCapacity = BANK_SIZE.toLong() val requiredBanks: Int = (initialSize - 1).toBankNumber() + 1 __data = ArrayList(requiredBanks) - repeat(requiredBanks) { __data.add(ByteArray(bankSize)) } + repeat(requiredBanks) { __data.add(ByteArray(BANK_SIZE)) } } - private fun Long.toBankNumber(): Int = (this / bankSize).toInt() - private fun Long.toBankOffset(): Int = (this % bankSize).toInt() + private fun Long.toBankNumber(): Int = (this / BANK_SIZE).toInt() + private fun Long.toBankOffset(): Int = (this % BANK_SIZE).toInt() operator fun set(index: Long, value: Byte) { checkMutability() @@ -74,7 +74,62 @@ class ByteArray64(initialSize: Long = bankSize.toLong()) { } } - fun add(value: Byte) = set(size, value) + fun appendByte(value: Byte) = set(size, value) + + fun appendBytes(bytes: ByteArray64) { + checkMutability() + ensureCapacity(size + bytes.size) + + val bankOffset = size.toBankOffset() + val initialBankNumber = size.toBankNumber() + val remaining = BANK_SIZE - bankOffset + + bytes.forEachUsedBanksIndexed { index, bytesInBank, srcBank -> + // as the data must be written bank-aligned, each bank copy requires two separate copies, split by the + // 'remaining' below + if (remaining < bytesInBank) { // 'remaining' should never be less than zero + System.arraycopy(srcBank, 0, __data[initialBankNumber + index], bankOffset, remaining) + System.arraycopy(srcBank, remaining, __data[initialBankNumber + index + 1], 0, bytesInBank - remaining) + } + else if (bytesInBank > 0) { + System.arraycopy(srcBank, 0, __data[initialBankNumber + index], bankOffset, bytesInBank) + } + } + + size += bytes.size + } + + fun appendBytes(bytes: ByteArray) { + checkMutability() + ensureCapacity(size + bytes.size) + val bankOffset = size.toBankOffset() + var currentBankNumber = size.toBankNumber() + val remainingInHeadBank = BANK_SIZE - bankOffset // how much space left in the current (= head) bank + var remainingBytesToCopy = bytes.size + var srcCursor = 0 + + // as the source is single contiguous byte array, we only need three separate copies: + // 1. Copy over some bytes so that the current bank is fully filled + // 2. Copy over 8192*n bytes to fill a chunk in single operation + // 3. Copy over the remaining bytes + + // 1. + var actualBytesToCopy = minOf(remainingBytesToCopy, remainingInHeadBank) // it is possible that size of the bytes is smaller than the remainingInHeadBank + System.arraycopy(bytes, srcCursor, __data[currentBankNumber], bankOffset, actualBytesToCopy) + remainingBytesToCopy -= actualBytesToCopy + srcCursor += actualBytesToCopy + if (remainingBytesToCopy <= 0) { size += bytes.size; return } + + // 2. and 3. + while (remainingBytesToCopy > 0) { + currentBankNumber += 1 + actualBytesToCopy = minOf(remainingBytesToCopy, BANK_SIZE) // it is possible that size of the bytes is smaller than the remainingInHeadBank + System.arraycopy(bytes, srcCursor, __data[currentBankNumber], 0, actualBytesToCopy) + remainingBytesToCopy -= actualBytesToCopy + srcCursor += actualBytesToCopy + } + size += bytes.size; return + } operator fun get(index: Long): Byte { if (index < 0 || index >= size) @@ -93,8 +148,8 @@ class ByteArray64(initialSize: Long = bankSize.toLong()) { } private fun addOneBank() { - __data.add(ByteArray(bankSize)) - internalCapacity = __data.size * bankSize.toLong() + __data.add(ByteArray(BANK_SIZE)) + internalCapacity = __data.size * BANK_SIZE.toLong() } /** @@ -171,20 +226,20 @@ class ByteArray64(initialSize: Long = bankSize.toLong()) { /** * @param consumer (Int, Int, ByteArray)-to-Unit function where first Int is index; - * second Int is actual number of bytes written in that bank, 0 to BankSize inclusive. + * second Int is actual number of bytes written in that bank, 0..BANK_SIZE (0 means that the bank is unused) */ fun forEachUsedBanksIndexed(consumer: (Int, Int, ByteArray) -> Unit) { __data.forEachIndexed { index, bytes -> - consumer(index, (size - bankSize * index).coerceIn(0, bankSize.toLong()).toInt(), bytes) + consumer(index, (size - BANK_SIZE * index).coerceIn(0, BANK_SIZE.toLong()).toInt(), bytes) } } /** - * @param consumer (Int, Int, ByteArray)-to-Unit function where Int is actual number of bytes written in that bank, 0 to BankSize inclusive. + * @param consumer (Int, ByteArray)-to-Unit function where Int is actual number of bytes written in that bank, 0..BANK_SIZE (0 means that the bank is unused) */ fun forEachUsedBanks(consumer: (Int, ByteArray) -> Unit) { __data.forEachIndexed { index, bytes -> - consumer((size - bankSize * index).coerceIn(0, bankSize.toLong()).toInt(), bytes) + consumer((size - BANK_SIZE * index).coerceIn(0, BANK_SIZE.toLong()).toInt(), bytes) } } @@ -260,7 +315,7 @@ open class ByteArray64OutputStream(val byteArray64: ByteArray64): OutputStream() override fun write(b: Int) { try { - byteArray64.add(b.toByte()) + byteArray64.appendByte(b.toByte()) writeCounter += 1 } catch (e: ArrayIndexOutOfBoundsException) { @@ -275,7 +330,7 @@ open class ByteArray64OutputStream(val byteArray64: ByteArray64): OutputStream() /** Just like Java's ByteArrayOutputStream, except its size grows if you exceed the initial size */ -open class ByteArray64GrowableOutputStream(size: Long = ByteArray64.bankSize.toLong()): OutputStream() { +open class ByteArray64GrowableOutputStream(size: Long = ByteArray64.BANK_SIZE.toLong()): OutputStream() { protected open var buf = ByteArray64(size) protected open var count = 0L @@ -290,7 +345,7 @@ open class ByteArray64GrowableOutputStream(size: Long = ByteArray64.bankSize.toL throw IllegalStateException("This output stream is finalised and cannot be modified.") } else { - buf.add(b.toByte()) + buf.appendByte(b.toByte()) count += 1 } } @@ -360,7 +415,7 @@ open class ByteArray64Writer(val charset: Charset) : Writer() { throw IllegalStateException("Surrogate high: ${surrogateBuf.toUcode()}, surrogate low: ${c.toUcode()}") } Charset.forName("CP437") -> { - ba.add(c.toByte()) + ba.appendByte(c.toByte()) } else -> throw UnsupportedCharsetException(charset.name()) } @@ -368,21 +423,21 @@ open class ByteArray64Writer(val charset: Charset) : Writer() { fun writeUtf8Codepoint(codepoint: Int) { when (codepoint) { - in 0..127 -> ba.add(codepoint.toByte()) + in 0..127 -> ba.appendByte(codepoint.toByte()) in 128..2047 -> { - ba.add((0xC0 or codepoint.ushr(6).and(31)).toByte()) - ba.add((0x80 or codepoint.and(63)).toByte()) + ba.appendByte((0xC0 or codepoint.ushr(6).and(31)).toByte()) + ba.appendByte((0x80 or codepoint.and(63)).toByte()) } in 2048..65535 -> { - ba.add((0xE0 or codepoint.ushr(12).and(15)).toByte()) - ba.add((0x80 or codepoint.ushr(6).and(63)).toByte()) - ba.add((0x80 or codepoint.and(63)).toByte()) + ba.appendByte((0xE0 or codepoint.ushr(12).and(15)).toByte()) + ba.appendByte((0x80 or codepoint.ushr(6).and(63)).toByte()) + ba.appendByte((0x80 or codepoint.and(63)).toByte()) } in 65536..1114111 -> { - ba.add((0xF0 or codepoint.ushr(18).and(7)).toByte()) - ba.add((0x80 or codepoint.ushr(12).and(63)).toByte()) - ba.add((0x80 or codepoint.ushr(6).and(63)).toByte()) - ba.add((0x80 or codepoint.and(63)).toByte()) + ba.appendByte((0xF0 or codepoint.ushr(18).and(7)).toByte()) + ba.appendByte((0x80 or codepoint.ushr(12).and(63)).toByte()) + ba.appendByte((0x80 or codepoint.ushr(6).and(63)).toByte()) + ba.appendByte((0x80 or codepoint.and(63)).toByte()) } else -> throw IllegalArgumentException("Not a unicode code point: U+${codepoint.toString(16).toUpperCase()}") } @@ -395,7 +450,7 @@ open class ByteArray64Writer(val charset: Charset) : Writer() { override fun write(str: String) { checkOpen() - str.toByteArray(charset).forEach { ba.add(it) } + ba.appendBytes(str.toByteArray(charset)) } override fun write(cbuf: CharArray, off: Int, len: Int) { diff --git a/src/net/torvald/terrarum/savegame/DiskSkimmer.kt b/src/net/torvald/terrarum/savegame/DiskSkimmer.kt index 72b5fbd92..050d6bc76 100644 --- a/src/net/torvald/terrarum/savegame/DiskSkimmer.kt +++ b/src/net/torvald/terrarum/savegame/DiskSkimmer.kt @@ -9,9 +9,9 @@ import java.util.logging.Level import kotlin.experimental.and /** - * Skimming allows modifying the Virtual Disk without loading entire disk onto the memory. + * Skimmer allows modifications of the Virtual Disk without building a DOM (disk object model). * - * Skimmer will just scan through the raw bytes of the Virtual Disk to get the file requested with its Entry ID; + * Skimmer will scan through the raw bytes of the Virtual Disk to get the file requested with its Entry ID; * modifying/removing files will edit the Virtual Disk in "dirty" way, where old entries are simply marked as deletion * and leaves the actual contents untouched, then will simply append modified files at the end. * diff --git a/src/net/torvald/terrarum/savegame/VDUtil.kt b/src/net/torvald/terrarum/savegame/VDUtil.kt index 6e4855364..cd26ad42b 100644 --- a/src/net/torvald/terrarum/savegame/VDUtil.kt +++ b/src/net/torvald/terrarum/savegame/VDUtil.kt @@ -38,7 +38,7 @@ object VDUtil { } fun dumpToRealMachine(disk: VirtualDisk, outfile: File) { - outfile.writeBytes64(disk.serialize().array) + outfile.writeBytes64(disk.serialize()) } private const val DEBUG_PRINT_READ = false @@ -156,7 +156,7 @@ object VDUtil { // test print if (DEBUG_PRINT_READ) { val testbytez = diskEntry.contents.serialize() - val testbytes = testbytez.array + val testbytes = testbytez (diskEntry.contents as? EntryDirectory)?.forEach { println("entry: ${it.toHex()}") } @@ -646,7 +646,7 @@ object VDUtil { while (true) { val byte = zi.read() if (byte == -1) break - unzipdBytes.add(byte.toByte()) + unzipdBytes.appendByte(byte.toByte()) } zi.close() return unzipdBytes @@ -657,10 +657,10 @@ fun magicMismatch(magic: ByteArray, array: ByteArray): Boolean { return !Arrays.equals(array, magic) } fun String.toEntryName(length: Int, charset: Charset): ByteArray { - val buffer = AppendableByteBuffer(length.toLong()) + val buffer = ByteArray64(length.toLong()) val stringByteArray = this.toByteArray(charset) - buffer.put(stringByteArray.sliceArray(0..minOf(length, stringByteArray.size) - 1)) - return buffer.array.toByteArray() + buffer.appendBytes(stringByteArray.sliceArray(0 until minOf(length, stringByteArray.size))) + return buffer.toByteArray() } fun ByteArray.toCanonicalString(charset: Charset): String { var lastIndexOfRealStr = 0 diff --git a/src/net/torvald/terrarum/savegame/VirtualDisk.kt b/src/net/torvald/terrarum/savegame/VirtualDisk.kt index 8295267a8..208f1d799 100644 --- a/src/net/torvald/terrarum/savegame/VirtualDisk.kt +++ b/src/net/torvald/terrarum/savegame/VirtualDisk.kt @@ -124,14 +124,16 @@ NOTES: */ -/** - * Created by minjaesong on 2021-09-10. - */ typealias EntryID = Long val specversion = 254.toByte() +/** + * This class provides DOM (disk object model) of the TEVD virtual filesystem. + * + * Created by minjaesong on 2021-09-10. + */ class VirtualDisk( /** capacity of 0 makes the disk read-only */ var capacity: Long, @@ -170,40 +172,40 @@ class VirtualDisk( // make sure to write root directory first entries[0L]!!.let { rootDir -> - rootDir.serialize().forEach { buffer.add(it) } + buffer.appendBytes(rootDir.serialize()) printdbg(this, "Writing disk ${getDiskName(Common.CHARSET)}") printdbg(this, "Root creation: ${rootDir.creationDate}, modified: ${rootDir.modificationDate}") } entries.forEach { if (it.key != 0L) { - it.value.serialize().forEach { buffer.add(it) } + buffer.appendBytes(it.value.serialize()) } } return buffer } - fun serialize(): AppendableByteBuffer { + fun serialize(): ByteArray64 { val entriesBuffer = serializeEntriesOnly() - val buffer = AppendableByteBuffer(HEADER_SIZE + entriesBuffer.size) + val buffer = ByteArray64(HEADER_SIZE + entriesBuffer.size) val crc = hashCode().toBigEndian() val diskName0 = diskName.forceSize(NAME_LENGTH) val diskName1 = diskName0.sliceArray(0..31).forceSize(32) val diskName2 = diskName0.sliceArray(32 until NAME_LENGTH).forceSize(NAME_LENGTH - 32) - buffer.put(MAGIC) + buffer.appendBytes(MAGIC) - buffer.put(capacity.toInt48()) - buffer.put(diskName1) - buffer.put(crc) - buffer.put(specversion) - buffer.put(0xFE.toByte()) - buffer.put(extraInfoBytes) - buffer.put(diskName2) + buffer.appendBytes(capacity.toInt48()) + buffer.appendBytes(diskName1) + buffer.appendBytes(crc) + buffer.appendByte(specversion) + buffer.appendByte(0xFE.toByte()) + buffer.appendBytes(extraInfoBytes) + buffer.appendBytes(diskName2) - buffer.put(entriesBuffer) + buffer.appendBytes(entriesBuffer) return buffer } @@ -334,19 +336,19 @@ class DiskEntry( } } - fun serialize(): AppendableByteBuffer { + fun serialize(): ByteArray64 { val serialisedContents = contents.serialize() - val buffer = AppendableByteBuffer(HEADER_SIZE + serialisedContents.size) + val buffer = ByteArray64(HEADER_SIZE + serialisedContents.size) - buffer.put(entryID.toBigEndian()) - buffer.put(parentEntryID.toBigEndian()) - buffer.put(contents.getTypeFlag()) - buffer.put(0); buffer.put(0); buffer.put(0) - buffer.put(creationDate.toInt48()) - buffer.put(modificationDate.toInt48()) - buffer.put(this.hashCode().toBigEndian()) + buffer.appendBytes(entryID.toBigEndian()) + buffer.appendBytes(parentEntryID.toBigEndian()) + buffer.appendByte(contents.getTypeFlag()) + buffer.appendByte(0); buffer.appendByte(0); buffer.appendByte(0) + buffer.appendBytes(creationDate.toInt48()) + buffer.appendBytes(modificationDate.toInt48()) + buffer.appendBytes(this.hashCode().toBigEndian()) - buffer.put(serialisedContents.array) + buffer.appendBytes(serialisedContents) return buffer } @@ -363,7 +365,7 @@ fun ByteArray.forceSize(size: Int): ByteArray { return ByteArray(size) { if (it < this.size) this[it] else 0.toByte() } } interface DiskEntryContent { - fun serialize(): AppendableByteBuffer + fun serialize(): ByteArray64 fun getSizePure(): Long fun getSizeEntry(): Long fun getContent(): Any @@ -381,10 +383,10 @@ open class EntryFile(internal var bytes: ByteArray64) : DiskEntryContent { /** Create new blank file */ constructor(size: Long): this(ByteArray64(size)) - override fun serialize(): AppendableByteBuffer { - val buffer = AppendableByteBuffer(getSizeEntry()) - buffer.put(getSizePure().toInt48()) - buffer.put(bytes) + override fun serialize(): ByteArray64 { + val buffer = ByteArray64(getSizeEntry()) + buffer.appendBytes(getSizePure().toInt48()) + buffer.appendBytes(bytes) return buffer } @@ -415,10 +417,10 @@ class EntryDirectory(private val entries: ArrayList = ArrayList buffer.put(indexNumber.toBigEndian()) } + override fun serialize(): ByteArray64 { + val buffer = ByteArray64(getSizeEntry()) + buffer.appendBytes(entries.size.toBigEndian()) + entries.sorted().forEach { indexNumber -> buffer.appendBytes(indexNumber.toBigEndian()) } return buffer } @@ -433,9 +435,10 @@ class EntrySymlink(val target: EntryID) : DiskEntryContent { override fun getSizePure() = 8L override fun getSizeEntry() = 8L - override fun serialize(): AppendableByteBuffer { - val buffer = AppendableByteBuffer(getSizeEntry()) - return buffer.put(target.toBigEndian()) + override fun serialize(): ByteArray64 { + val buffer = ByteArray64(getSizeEntry()) + buffer.appendBytes(target.toBigEndian()) + return buffer } override fun getContent() = target @@ -460,29 +463,8 @@ fun Short.toBigEndian(): ByteArray { ) } -fun AppendableByteBuffer.getCRC32(): Int { +fun ByteArray64.getCRC32(): Int { val crc = CRC32() - this.array.forEach { crc.update(it.toInt()) } + this.forEach { crc.update(it.toInt()) } return crc.value.toInt() } -class AppendableByteBuffer(val size: Long) { - val array = ByteArray64(size) - private var offset = 0L - - fun put(byteArray64: ByteArray64): AppendableByteBuffer { - // it's slow but works - // can't do system.arrayCopy directly - byteArray64.forEach { put(it) } - return this - } - fun put(byteArray: ByteArray): AppendableByteBuffer { - byteArray.forEach { put(it) } - return this - } - fun put(byte: Byte): AppendableByteBuffer { - array[offset] = byte - offset += 1 - return this - } - fun forEach(consumer: (Byte) -> Unit) = array.forEach(consumer) -} diff --git a/src/net/torvald/terrarum/serialise/Common.kt b/src/net/torvald/terrarum/serialise/Common.kt index 57bb17e05..f2b4491a7 100644 --- a/src/net/torvald/terrarum/serialise/Common.kt +++ b/src/net/torvald/terrarum/serialise/Common.kt @@ -291,7 +291,7 @@ object Common { while (true) { val byte = zi.read() if (byte == -1) break - unzipdBytes.add(byte.toByte()) + unzipdBytes.appendByte(byte.toByte()) } zi.close() return unzipdBytes @@ -307,14 +307,14 @@ object Common { val char = reader.read() if (char < 0) break if (bai > 0 && bai % 5 == 0) { - Ascii85.decode(buf[0], buf[1], buf[2], buf[3], buf[4]).forEach { unasciidBytes.add(it) } + unasciidBytes.appendBytes(Ascii85.decode(buf[0], buf[1], buf[2], buf[3], buf[4])) buf.fill(Ascii85.PAD_CHAR) } buf[bai % 5] = char.toChar() bai += 1 - }; Ascii85.decode(buf[0], buf[1], buf[2], buf[3], buf[4]).forEach { unasciidBytes.add(it) } + }; unasciidBytes.appendBytes(Ascii85.decode(buf[0], buf[1], buf[2], buf[3], buf[4])) return unasciidBytes } diff --git a/src/net/torvald/unicode/UniTextShortcuts.kt b/src/net/torvald/unicode/UniTextShortcuts.kt index 99deeb744..6a83b8c70 100644 --- a/src/net/torvald/unicode/UniTextShortcuts.kt +++ b/src/net/torvald/unicode/UniTextShortcuts.kt @@ -100,21 +100,21 @@ fun List.toUTF8Bytes64(): ByteArray64 { val ba = ByteArray64() this.forEach { codepoint -> when (codepoint) { - in 0..127 -> ba.add(codepoint.toByte()) + in 0..127 -> ba.appendByte(codepoint.toByte()) in 128..2047 -> { - ba.add((0xC0 or codepoint.ushr(6).and(31)).toByte()) - ba.add((0x80 or codepoint.and(63)).toByte()) + ba.appendByte((0xC0 or codepoint.ushr(6).and(31)).toByte()) + ba.appendByte((0x80 or codepoint.and(63)).toByte()) } in 2048..65535 -> { - ba.add((0xE0 or codepoint.ushr(12).and(15)).toByte()) - ba.add((0x80 or codepoint.ushr(6).and(63)).toByte()) - ba.add((0x80 or codepoint.and(63)).toByte()) + ba.appendByte((0xE0 or codepoint.ushr(12).and(15)).toByte()) + ba.appendByte((0x80 or codepoint.ushr(6).and(63)).toByte()) + ba.appendByte((0x80 or codepoint.and(63)).toByte()) } in 65536..1114111 -> { - ba.add((0xF0 or codepoint.ushr(18).and(7)).toByte()) - ba.add((0x80 or codepoint.ushr(12).and(63)).toByte()) - ba.add((0x80 or codepoint.ushr(6).and(63)).toByte()) - ba.add((0x80 or codepoint.and(63)).toByte()) + ba.appendByte((0xF0 or codepoint.ushr(18).and(7)).toByte()) + ba.appendByte((0x80 or codepoint.ushr(12).and(63)).toByte()) + ba.appendByte((0x80 or codepoint.ushr(6).and(63)).toByte()) + ba.appendByte((0x80 or codepoint.and(63)).toByte()) } else -> throw IllegalArgumentException("Not a unicode code point: U+${codepoint.toString(16).toUpperCase()}") }