curioustorvald/Terrarum
1
0
Fork 0
mirror of https://github.com/curioustorvald/Terrarum.git synced 2026-03-10 22:01:52 +09:00
Code Issues Packages Projects Releases Wiki Activity

tevd: bytearray64 update

Browse Source
This commit is contained in:
minjaesong
2022-12-17 05:31:37 +09:00
parent fab1b168aa
commit 60c0dd4c06
7 changed files with 149 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/net/torvald/terrarum/modulebasegame/serialise/WriteWorld.kt
View File

@@ -83,8 +83,8 @@ object WriteWorld {
for (y in cy * LandUtil.CHUNK_H until (cy + 1) * LandUtil.CHUNK_H) {
for (x in cx * LandUtil.CHUNK_W until (cx + 1) * LandUtil.CHUNK_W) {
val tilenum = layer.unsafeGetTile(x, y)
ba.add(tilenum.ushr(8).and(255).toByte())
ba.add(tilenum.and(255).toByte())
ba.appendByte(tilenum.ushr(8).and(255).toByte())
ba.appendByte(tilenum.and(255).toByte())
}
}

111
src/net/torvald/terrarum/savegame/ByteArray64.kt
View File

@@ -18,7 +18,7 @@ import java.nio.charset.UnsupportedCharsetException
*
* Created by Minjaesong on 2017-04-12.
*/
class ByteArray64(initialSize: Long = bankSize.toLong()) {
class ByteArray64(initialSize: Long = BANK_SIZE.toLong()) {
var internalCapacity: Long = initialSize
private set
@@ -28,7 +28,7 @@ class ByteArray64(initialSize: Long = bankSize.toLong()) {
private var finalised = false
companion object {
val bankSize: Int = 8192
val BANK_SIZE: Int = 8192
fun fromByteArray(byteArray: ByteArray): ByteArray64 {
val ba64 = ByteArray64(byteArray.size.toLong())
@@ -47,16 +47,16 @@ class ByteArray64(initialSize: Long = bankSize.toLong()) {
if (internalCapacity < 0)
throw IllegalArgumentException("Invalid array size: $internalCapacity")
else if (internalCapacity == 0L) // signalling empty array
internalCapacity = bankSize.toLong()
internalCapacity = BANK_SIZE.toLong()
val requiredBanks: Int = (initialSize - 1).toBankNumber() + 1
__data = ArrayList<ByteArray>(requiredBanks)
repeat(requiredBanks) { __data.add(ByteArray(bankSize)) }
repeat(requiredBanks) { __data.add(ByteArray(BANK_SIZE)) }
}
private fun Long.toBankNumber(): Int = (this / bankSize).toInt()
private fun Long.toBankOffset(): Int = (this % bankSize).toInt()
private fun Long.toBankNumber(): Int = (this / BANK_SIZE).toInt()
private fun Long.toBankOffset(): Int = (this % BANK_SIZE).toInt()
operator fun set(index: Long, value: Byte) {
checkMutability()
@@ -74,7 +74,62 @@ class ByteArray64(initialSize: Long = bankSize.toLong()) {
}
}
fun add(value: Byte) = set(size, value)
fun appendByte(value: Byte) = set(size, value)
fun appendBytes(bytes: ByteArray64) {
checkMutability()
ensureCapacity(size + bytes.size)
val bankOffset = size.toBankOffset()
val initialBankNumber = size.toBankNumber()
val remaining = BANK_SIZE - bankOffset
bytes.forEachUsedBanksIndexed { index, bytesInBank, srcBank ->
// as the data must be written bank-aligned, each bank copy requires two separate copies, split by the
// 'remaining' below
if (remaining < bytesInBank) { // 'remaining' should never be less than zero
System.arraycopy(srcBank, 0, __data[initialBankNumber + index], bankOffset, remaining)
System.arraycopy(srcBank, remaining, __data[initialBankNumber + index + 1], 0, bytesInBank - remaining)
}
else if (bytesInBank > 0) {
System.arraycopy(srcBank, 0, __data[initialBankNumber + index], bankOffset, bytesInBank)
}
}
size += bytes.size
}
fun appendBytes(bytes: ByteArray) {
checkMutability()
ensureCapacity(size + bytes.size)
val bankOffset = size.toBankOffset()
var currentBankNumber = size.toBankNumber()
val remainingInHeadBank = BANK_SIZE - bankOffset // how much space left in the current (= head) bank
var remainingBytesToCopy = bytes.size
var srcCursor = 0
// as the source is single contiguous byte array, we only need three separate copies:
// 1. Copy over some bytes so that the current bank is fully filled
// 2. Copy over 8192*n bytes to fill a chunk in single operation
// 3. Copy over the remaining bytes
// 1.
var actualBytesToCopy = minOf(remainingBytesToCopy, remainingInHeadBank) // it is possible that size of the bytes is smaller than the remainingInHeadBank
System.arraycopy(bytes, srcCursor, __data[currentBankNumber], bankOffset, actualBytesToCopy)
remainingBytesToCopy -= actualBytesToCopy
srcCursor += actualBytesToCopy
if (remainingBytesToCopy <= 0) { size += bytes.size; return }
// 2. and 3.
while (remainingBytesToCopy > 0) {
currentBankNumber += 1
actualBytesToCopy = minOf(remainingBytesToCopy, BANK_SIZE) // it is possible that size of the bytes is smaller than the remainingInHeadBank
System.arraycopy(bytes, srcCursor, __data[currentBankNumber], 0, actualBytesToCopy)
remainingBytesToCopy -= actualBytesToCopy
srcCursor += actualBytesToCopy
}
size += bytes.size; return
}
operator fun get(index: Long): Byte {
if (index < 0 || index >= size)
@@ -93,8 +148,8 @@ class ByteArray64(initialSize: Long = bankSize.toLong()) {
}
private fun addOneBank() {
__data.add(ByteArray(bankSize))
internalCapacity = __data.size * bankSize.toLong()
__data.add(ByteArray(BANK_SIZE))
internalCapacity = __data.size * BANK_SIZE.toLong()
}
/**
@@ -171,20 +226,20 @@ class ByteArray64(initialSize: Long = bankSize.toLong()) {
/**
* @param consumer (Int, Int, ByteArray)-to-Unit function where first Int is index;
* second Int is actual number of bytes written in that bank, 0 to BankSize inclusive.
* second Int is actual number of bytes written in that bank, 0..BANK_SIZE (0 means that the bank is unused)
*/
fun forEachUsedBanksIndexed(consumer: (Int, Int, ByteArray) -> Unit) {
__data.forEachIndexed { index, bytes ->
consumer(index, (size - bankSize * index).coerceIn(0, bankSize.toLong()).toInt(), bytes)
consumer(index, (size - BANK_SIZE * index).coerceIn(0, BANK_SIZE.toLong()).toInt(), bytes)
}
}
/**
* @param consumer (Int, Int, ByteArray)-to-Unit function where Int is actual number of bytes written in that bank, 0 to BankSize inclusive.
* @param consumer (Int, ByteArray)-to-Unit function where Int is actual number of bytes written in that bank, 0..BANK_SIZE (0 means that the bank is unused)
*/
fun forEachUsedBanks(consumer: (Int, ByteArray) -> Unit) {
__data.forEachIndexed { index, bytes ->
consumer((size - bankSize * index).coerceIn(0, bankSize.toLong()).toInt(), bytes)
consumer((size - BANK_SIZE * index).coerceIn(0, BANK_SIZE.toLong()).toInt(), bytes)
}
}
@@ -260,7 +315,7 @@ open class ByteArray64OutputStream(val byteArray64: ByteArray64): OutputStream()
override fun write(b: Int) {
try {
byteArray64.add(b.toByte())
byteArray64.appendByte(b.toByte())
writeCounter += 1
}
catch (e: ArrayIndexOutOfBoundsException) {
@@ -275,7 +330,7 @@ open class ByteArray64OutputStream(val byteArray64: ByteArray64): OutputStream()
/** Just like Java's ByteArrayOutputStream, except its size grows if you exceed the initial size
*/
open class ByteArray64GrowableOutputStream(size: Long = ByteArray64.bankSize.toLong()): OutputStream() {
open class ByteArray64GrowableOutputStream(size: Long = ByteArray64.BANK_SIZE.toLong()): OutputStream() {
protected open var buf = ByteArray64(size)
protected open var count = 0L
@@ -290,7 +345,7 @@ open class ByteArray64GrowableOutputStream(size: Long = ByteArray64.bankSize.toL
throw IllegalStateException("This output stream is finalised and cannot be modified.")
}
else {
buf.add(b.toByte())
buf.appendByte(b.toByte())
count += 1
}
}
@@ -360,7 +415,7 @@ open class ByteArray64Writer(val charset: Charset) : Writer() {
throw IllegalStateException("Surrogate high: ${surrogateBuf.toUcode()}, surrogate low: ${c.toUcode()}")
}
Charset.forName("CP437") -> {
ba.add(c.toByte())
ba.appendByte(c.toByte())
}
else -> throw UnsupportedCharsetException(charset.name())
}
@@ -368,21 +423,21 @@ open class ByteArray64Writer(val charset: Charset) : Writer() {
fun writeUtf8Codepoint(codepoint: Int) {
when (codepoint) {
in 0..127 -> ba.add(codepoint.toByte())
in 0..127 -> ba.appendByte(codepoint.toByte())
in 128..2047 -> {
ba.add((0xC0 or codepoint.ushr(6).and(31)).toByte())
ba.add((0x80 or codepoint.and(63)).toByte())
ba.appendByte((0xC0 or codepoint.ushr(6).and(31)).toByte())
ba.appendByte((0x80 or codepoint.and(63)).toByte())
}
in 2048..65535 -> {
ba.add((0xE0 or codepoint.ushr(12).and(15)).toByte())
ba.add((0x80 or codepoint.ushr(6).and(63)).toByte())
ba.add((0x80 or codepoint.and(63)).toByte())
ba.appendByte((0xE0 or codepoint.ushr(12).and(15)).toByte())
ba.appendByte((0x80 or codepoint.ushr(6).and(63)).toByte())
ba.appendByte((0x80 or codepoint.and(63)).toByte())
}
in 65536..1114111 -> {
ba.add((0xF0 or codepoint.ushr(18).and(7)).toByte())
ba.add((0x80 or codepoint.ushr(12).and(63)).toByte())
ba.add((0x80 or codepoint.ushr(6).and(63)).toByte())
ba.add((0x80 or codepoint.and(63)).toByte())
ba.appendByte((0xF0 or codepoint.ushr(18).and(7)).toByte())
ba.appendByte((0x80 or codepoint.ushr(12).and(63)).toByte())
ba.appendByte((0x80 or codepoint.ushr(6).and(63)).toByte())
ba.appendByte((0x80 or codepoint.and(63)).toByte())
}
else -> throw IllegalArgumentException("Not a unicode code point: U+${codepoint.toString(16).toUpperCase()}")
}
@@ -395,7 +450,7 @@ open class ByteArray64Writer(val charset: Charset) : Writer() {
override fun write(str: String) {
checkOpen()
str.toByteArray(charset).forEach { ba.add(it) }
ba.appendBytes(str.toByteArray(charset))
}
override fun write(cbuf: CharArray, off: Int, len: Int) {

4
src/net/torvald/terrarum/savegame/DiskSkimmer.kt
View File

@@ -9,9 +9,9 @@ import java.util.logging.Level
import kotlin.experimental.and
/**
* Skimming allows modifying the Virtual Disk without loading entire disk onto the memory.
* Skimmer allows modifications of the Virtual Disk without building a DOM (disk object model).
*
* Skimmer will just scan through the raw bytes of the Virtual Disk to get the file requested with its Entry ID;
* Skimmer will scan through the raw bytes of the Virtual Disk to get the file requested with its Entry ID;
* modifying/removing files will edit the Virtual Disk in "dirty" way, where old entries are simply marked as deletion
* and leaves the actual contents untouched, then will simply append modified files at the end.
*

12
src/net/torvald/terrarum/savegame/VDUtil.kt
View File

@@ -38,7 +38,7 @@ object VDUtil {
}
fun dumpToRealMachine(disk: VirtualDisk, outfile: File) {
outfile.writeBytes64(disk.serialize().array)
outfile.writeBytes64(disk.serialize())
}
private const val DEBUG_PRINT_READ = false
@@ -156,7 +156,7 @@ object VDUtil {
// test print
if (DEBUG_PRINT_READ) {
val testbytez = diskEntry.contents.serialize()
val testbytes = testbytez.array
val testbytes = testbytez
(diskEntry.contents as? EntryDirectory)?.forEach {
println("entry: ${it.toHex()}")
}
@@ -646,7 +646,7 @@ object VDUtil {
while (true) {
val byte = zi.read()
if (byte == -1) break
unzipdBytes.add(byte.toByte())
unzipdBytes.appendByte(byte.toByte())
}
zi.close()
return unzipdBytes
@@ -657,10 +657,10 @@ fun magicMismatch(magic: ByteArray, array: ByteArray): Boolean {
return !Arrays.equals(array, magic)
}
fun String.toEntryName(length: Int, charset: Charset): ByteArray {
val buffer = AppendableByteBuffer(length.toLong())
val buffer = ByteArray64(length.toLong())
val stringByteArray = this.toByteArray(charset)
buffer.put(stringByteArray.sliceArray(0..minOf(length, stringByteArray.size) - 1))
return buffer.array.toByteArray()
buffer.appendBytes(stringByteArray.sliceArray(0 until minOf(length, stringByteArray.size)))
return buffer.toByteArray()
}
fun ByteArray.toCanonicalString(charset: Charset): String {
var lastIndexOfRealStr = 0

104
src/net/torvald/terrarum/savegame/VirtualDisk.kt
View File

@@ -124,14 +124,16 @@ NOTES:
*/
/**
* Created by minjaesong on 2021-09-10.
*/
typealias EntryID = Long
val specversion = 254.toByte()
/**
* This class provides DOM (disk object model) of the TEVD virtual filesystem.
*
* Created by minjaesong on 2021-09-10.
*/
class VirtualDisk(
/** capacity of 0 makes the disk read-only */
var capacity: Long,
@@ -170,40 +172,40 @@ class VirtualDisk(
// make sure to write root directory first
entries[0L]!!.let { rootDir ->
rootDir.serialize().forEach { buffer.add(it) }
buffer.appendBytes(rootDir.serialize())
printdbg(this, "Writing disk ${getDiskName(Common.CHARSET)}")
printdbg(this, "Root creation: ${rootDir.creationDate}, modified: ${rootDir.modificationDate}")
}
entries.forEach {
if (it.key != 0L) {
it.value.serialize().forEach { buffer.add(it) }
buffer.appendBytes(it.value.serialize())
}
}
return buffer
}
fun serialize(): AppendableByteBuffer {
fun serialize(): ByteArray64 {
val entriesBuffer = serializeEntriesOnly()
val buffer = AppendableByteBuffer(HEADER_SIZE + entriesBuffer.size)
val buffer = ByteArray64(HEADER_SIZE + entriesBuffer.size)
val crc = hashCode().toBigEndian()
val diskName0 = diskName.forceSize(NAME_LENGTH)
val diskName1 = diskName0.sliceArray(0..31).forceSize(32)
val diskName2 = diskName0.sliceArray(32 until NAME_LENGTH).forceSize(NAME_LENGTH - 32)
buffer.put(MAGIC)
buffer.appendBytes(MAGIC)
buffer.put(capacity.toInt48())
buffer.put(diskName1)
buffer.put(crc)
buffer.put(specversion)
buffer.put(0xFE.toByte())
buffer.put(extraInfoBytes)
buffer.put(diskName2)
buffer.appendBytes(capacity.toInt48())
buffer.appendBytes(diskName1)
buffer.appendBytes(crc)
buffer.appendByte(specversion)
buffer.appendByte(0xFE.toByte())
buffer.appendBytes(extraInfoBytes)
buffer.appendBytes(diskName2)
buffer.put(entriesBuffer)
buffer.appendBytes(entriesBuffer)
return buffer
}
@@ -334,19 +336,19 @@ class DiskEntry(
}
}
fun serialize(): AppendableByteBuffer {
fun serialize(): ByteArray64 {
val serialisedContents = contents.serialize()
val buffer = AppendableByteBuffer(HEADER_SIZE + serialisedContents.size)
val buffer = ByteArray64(HEADER_SIZE + serialisedContents.size)
buffer.put(entryID.toBigEndian())
buffer.put(parentEntryID.toBigEndian())
buffer.put(contents.getTypeFlag())
buffer.put(0); buffer.put(0); buffer.put(0)
buffer.put(creationDate.toInt48())
buffer.put(modificationDate.toInt48())
buffer.put(this.hashCode().toBigEndian())
buffer.appendBytes(entryID.toBigEndian())
buffer.appendBytes(parentEntryID.toBigEndian())
buffer.appendByte(contents.getTypeFlag())
buffer.appendByte(0); buffer.appendByte(0); buffer.appendByte(0)
buffer.appendBytes(creationDate.toInt48())
buffer.appendBytes(modificationDate.toInt48())
buffer.appendBytes(this.hashCode().toBigEndian())
buffer.put(serialisedContents.array)
buffer.appendBytes(serialisedContents)
return buffer
}
@@ -363,7 +365,7 @@ fun ByteArray.forceSize(size: Int): ByteArray {
return ByteArray(size) { if (it < this.size) this[it] else 0.toByte() }
}
interface DiskEntryContent {
fun serialize(): AppendableByteBuffer
fun serialize(): ByteArray64
fun getSizePure(): Long
fun getSizeEntry(): Long
fun getContent(): Any
@@ -381,10 +383,10 @@ open class EntryFile(internal var bytes: ByteArray64) : DiskEntryContent {
/** Create new blank file */
constructor(size: Long): this(ByteArray64(size))
override fun serialize(): AppendableByteBuffer {
val buffer = AppendableByteBuffer(getSizeEntry())
buffer.put(getSizePure().toInt48())
buffer.put(bytes)
override fun serialize(): ByteArray64 {
val buffer = ByteArray64(getSizeEntry())
buffer.appendBytes(getSizePure().toInt48())
buffer.appendBytes(bytes)
return buffer
}
@@ -415,10 +417,10 @@ class EntryDirectory(private val entries: ArrayList<EntryID> = ArrayList<EntryID
val entryCount: Int
get() = entries.size
override fun serialize(): AppendableByteBuffer {
val buffer = AppendableByteBuffer(getSizeEntry())
buffer.put(entries.size.toBigEndian())
entries.sorted().forEach { indexNumber -> buffer.put(indexNumber.toBigEndian()) }
override fun serialize(): ByteArray64 {
val buffer = ByteArray64(getSizeEntry())
buffer.appendBytes(entries.size.toBigEndian())
entries.sorted().forEach { indexNumber -> buffer.appendBytes(indexNumber.toBigEndian()) }
return buffer
}
@@ -433,9 +435,10 @@ class EntrySymlink(val target: EntryID) : DiskEntryContent {
override fun getSizePure() = 8L
override fun getSizeEntry() = 8L
override fun serialize(): AppendableByteBuffer {
val buffer = AppendableByteBuffer(getSizeEntry())
return buffer.put(target.toBigEndian())
override fun serialize(): ByteArray64 {
val buffer = ByteArray64(getSizeEntry())
buffer.appendBytes(target.toBigEndian())
return buffer
}
override fun getContent() = target
@@ -460,29 +463,8 @@ fun Short.toBigEndian(): ByteArray {
)
}
fun AppendableByteBuffer.getCRC32(): Int {
fun ByteArray64.getCRC32(): Int {
val crc = CRC32()
this.array.forEach { crc.update(it.toInt()) }
this.forEach { crc.update(it.toInt()) }
return crc.value.toInt()
}
class AppendableByteBuffer(val size: Long) {
val array = ByteArray64(size)
private var offset = 0L
fun put(byteArray64: ByteArray64): AppendableByteBuffer {
// it's slow but works
// can't do system.arrayCopy directly
byteArray64.forEach { put(it) }
return this
}
fun put(byteArray: ByteArray): AppendableByteBuffer {
byteArray.forEach { put(it) }
return this
}
fun put(byte: Byte): AppendableByteBuffer {
array[offset] = byte
offset += 1
return this
}
fun forEach(consumer: (Byte) -> Unit) = array.forEach(consumer)
}

6
src/net/torvald/terrarum/serialise/Common.kt
View File

@@ -291,7 +291,7 @@ object Common {
while (true) {
val byte = zi.read()
if (byte == -1) break
unzipdBytes.add(byte.toByte())
unzipdBytes.appendByte(byte.toByte())
}
zi.close()
return unzipdBytes
@@ -307,14 +307,14 @@ object Common {
val char = reader.read()
if (char < 0) break
if (bai > 0 && bai % 5 == 0) {
Ascii85.decode(buf[0], buf[1], buf[2], buf[3], buf[4]).forEach { unasciidBytes.add(it) }
unasciidBytes.appendBytes(Ascii85.decode(buf[0], buf[1], buf[2], buf[3], buf[4]))
buf.fill(Ascii85.PAD_CHAR)
}
buf[bai % 5] = char.toChar()
bai += 1
}; Ascii85.decode(buf[0], buf[1], buf[2], buf[3], buf[4]).forEach { unasciidBytes.add(it) }
}; unasciidBytes.appendBytes(Ascii85.decode(buf[0], buf[1], buf[2], buf[3], buf[4]))
return unasciidBytes
}

20
src/net/torvald/unicode/UniTextShortcuts.kt
View File

@@ -100,21 +100,21 @@ fun List<Int>.toUTF8Bytes64(): ByteArray64 {
val ba = ByteArray64()
this.forEach { codepoint ->
when (codepoint) {
in 0..127 -> ba.add(codepoint.toByte())
in 0..127 -> ba.appendByte(codepoint.toByte())
in 128..2047 -> {
ba.add((0xC0 or codepoint.ushr(6).and(31)).toByte())
ba.add((0x80 or codepoint.and(63)).toByte())
ba.appendByte((0xC0 or codepoint.ushr(6).and(31)).toByte())
ba.appendByte((0x80 or codepoint.and(63)).toByte())
}
in 2048..65535 -> {
ba.add((0xE0 or codepoint.ushr(12).and(15)).toByte())
ba.add((0x80 or codepoint.ushr(6).and(63)).toByte())
ba.add((0x80 or codepoint.and(63)).toByte())
ba.appendByte((0xE0 or codepoint.ushr(12).and(15)).toByte())
ba.appendByte((0x80 or codepoint.ushr(6).and(63)).toByte())
ba.appendByte((0x80 or codepoint.and(63)).toByte())
}
in 65536..1114111 -> {
ba.add((0xF0 or codepoint.ushr(18).and(7)).toByte())
ba.add((0x80 or codepoint.ushr(12).and(63)).toByte())
ba.add((0x80 or codepoint.ushr(6).and(63)).toByte())
ba.add((0x80 or codepoint.and(63)).toByte())
ba.appendByte((0xF0 or codepoint.ushr(18).and(7)).toByte())
ba.appendByte((0x80 or codepoint.ushr(12).and(63)).toByte())
ba.appendByte((0x80 or codepoint.ushr(6).and(63)).toByte())
ba.appendByte((0x80 or codepoint.and(63)).toByte())
}
else -> throw IllegalArgumentException("Not a unicode code point: U+${codepoint.toString(16).toUpperCase()}")
}