still wip modularisation, game somehow boots

This commit is contained in:
minjaesong
2018-06-21 17:33:22 +09:00
parent f0a6f8b9c2
commit a6ea2b4e18
266 changed files with 2409 additions and 1122 deletions

View File

@@ -0,0 +1,157 @@
package net.torvald.terrarum.modulecomputers.virtualcomputer.tvd
import java.io.BufferedOutputStream
import java.io.File
import java.io.FileOutputStream
import java.io.InputStream
/**
* ByteArray that can hold larger than 2 GiB of Data.
*
* Works kind of like Bank Switching of old game console's cartridges which does same thing.
*
* Created by Minjaesong on 2017-04-12.
*/
class ByteArray64(val size: Long) {
companion object {
val bankSize: Int = 8192
}
private val data: Array<ByteArray>
init {
if (size < 0)
throw IllegalArgumentException("Invalid array size!")
val requiredBanks: Int = 1 + ((size - 1) / bankSize).toInt()
data = Array<ByteArray>(
requiredBanks,
{ bankIndex ->
kotlin.ByteArray(
if (bankIndex == requiredBanks - 1)
size.toBankOffset()
else
bankSize,
{ 0.toByte() }
)
}
)
}
private fun Long.toBankNumber(): Int = (this / bankSize).toInt()
private fun Long.toBankOffset(): Int = (this % bankSize).toInt()
operator fun set(index: Long, value: Byte) {
if (index < 0 || index >= size)
throw ArrayIndexOutOfBoundsException("size $size, index $index")
data[index.toBankNumber()][index.toBankOffset()] = value
}
operator fun get(index: Long): Byte {
if (index < 0 || index >= size)
throw ArrayIndexOutOfBoundsException("size $size, index $index")
return data[index.toBankNumber()][index.toBankOffset()]
}
operator fun iterator(): ByteIterator {
return object : ByteIterator() {
var iterationCounter = 0L
override fun nextByte(): Byte {
iterationCounter += 1
return this@ByteArray64[iterationCounter - 1]
}
override fun hasNext() = iterationCounter < this@ByteArray64.size
}
}
fun iteratorChoppedToInt(): IntIterator {
return object : IntIterator() {
var iterationCounter = 0L
val iteratorSize = 1 + ((this@ByteArray64.size - 1) / 4).toInt()
override fun nextInt(): Int {
var byteCounter = iterationCounter * 4L
var int = 0
(0..3).forEach {
if (byteCounter + it < this@ByteArray64.size) {
int += this@ByteArray64[byteCounter + it].toInt() shl (it * 8)
}
else {
int += 0 shl (it * 8)
}
}
iterationCounter += 1
return int
}
override fun hasNext() = iterationCounter < iteratorSize
}
}
fun forEach(consumer: (Byte) -> Unit) = iterator().forEach { consumer(it) }
fun forEachInt32(consumer: (Int) -> Unit) = iteratorChoppedToInt().forEach { consumer(it) }
fun forEachBanks(consumer: (ByteArray) -> Unit) = data.forEach(consumer)
fun sliceArray64(range: LongRange): ByteArray64 {
val newarr = ByteArray64(range.last - range.first + 1)
range.forEach { index ->
newarr[index - range.first] = this[index]
}
return newarr
}
fun sliceArray(range: IntRange): ByteArray {
val newarr = ByteArray(range.last - range.first + 1)
range.forEach { index ->
newarr[index - range.first] = this[index.toLong()]
}
return newarr
}
fun toByteArray(): ByteArray {
if (this.size > Integer.MAX_VALUE - 8) // according to OpenJDK; the size itself is VM-dependent
throw TypeCastException("Impossible cast; too large to fit")
return ByteArray(this.size.toInt(), { this[it.toLong()] })
}
fun writeToFile(file: File) {
var fos = FileOutputStream(file, false)
fos.write(data[0])
fos.flush()
fos.close()
if (data.size > 1) {
fos = FileOutputStream(file, true)
for (i in 1..data.lastIndex) {
fos.write(data[i])
fos.flush()
}
fos.close()
}
}
}
class ByteArray64InputStream(val byteArray64: ByteArray64): InputStream() {
private var readCounter = 0L
override fun read(): Int {
readCounter += 1
return try {
byteArray64[readCounter - 1].toUint()
}
catch (e: ArrayIndexOutOfBoundsException) {
-1
}
}
}

View File

@@ -0,0 +1,102 @@
package net.torvald.terrarum.modulecomputers.virtualcomputer.tvd
import java.io.File
import java.io.FileInputStream
/**
* Creates entry-to-offset tables to allow streaming from the disk, without storing whole VD file to the memory.
*
* Created by minjaesong on 2017-11-17.
*/
class DiskSkimmer(diskFile: File) {
class EntryOffsetPair(val entryID: Int, val offset: Long)
val entryToOffsetTable = ArrayList<EntryOffsetPair>()
init {
val fis = FileInputStream(diskFile)
var currentPosition = fis.skip(47) // skip disk header
fun skipRead(bytes: Long) {
currentPosition += fis.skip(bytes)
}
/**
* Reads a byte and adds up the position var
*/
fun readByte(): Byte {
currentPosition++
val read = fis.read()
if (read < 0) throw InternalError("Unexpectedly reached EOF")
return read.toByte()
}
/**
* Reads specific bytes to the buffer and adds up the position var
*/
fun readBytes(buffer: ByteArray): Int {
val readStatus = fis.read(buffer)
currentPosition += readStatus
return readStatus
}
fun readIntBig(): Int {
val buffer = ByteArray(4)
val readStatus = readBytes(buffer)
if (readStatus != 4) throw InternalError("Unexpected error -- EOF reached? (expected 4, got $readStatus)")
return buffer.toIntBig()
}
fun readInt48(): Long {
val buffer = ByteArray(6)
val readStatus = readBytes(buffer)
if (readStatus != 6) throw InternalError("Unexpected error -- EOF reached? (expected 6, got $readStatus)")
return buffer.toInt48()
}
while (true) {
val entryID = readIntBig()
// footer
if (entryID == 0xFEFEFEFE.toInt()) break
// fill up table
entryToOffsetTable.add(EntryOffsetPair(entryID, currentPosition))
skipRead(4) // skip entryID of parent
val entryType = readByte()
skipRead(256 + 6 + 6 + 4) // skips rest of the header
// figure out the entry size so that we can skip
val entrySize: Long = when(entryType) {
0x01.toByte() -> readInt48()
0x11.toByte() -> readInt48() + 6 // size of compressed payload + 6 (header elem for uncompressed size)
0x02.toByte() -> readIntBig().shl(16).toLong() * 4 - 2 // #entris is 2 bytes, we read 4 bytes, so we subtract 2
0x03.toByte() -> 4 // symlink
else -> throw InternalError("Unknown entry type: ${entryType.toUint()}")
}
skipRead(entrySize) // skips rest of the entry's actual contents
}
}
private fun ByteArray.toIntBig(): Int {
return this[0].toUint().shl(24) or this[1].toUint().shl(16) or
this[2].toUint().shl(8) or this[2].toUint()
}
private fun ByteArray.toInt48(): Long {
return this[0].toUlong().shl(56) or this[1].toUlong().shl(48) or
this[2].toUlong().shl(40) or this[3].toUlong().shl(32) or
this[4].toUlong().shl(24) or this[5].toUlong().shl(16) or
this[6].toUlong().shl(8) or this[7].toUlong()
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,307 @@
package net.torvald.terrarum.modulecomputers.virtualcomputer.tvd
import java.io.IOException
import java.nio.charset.Charset
import java.util.*
import java.util.zip.CRC32
import kotlin.experimental.and
import kotlin.experimental.or
/**
* Created by minjaesong on 2017-03-31.
*/
typealias EntryID = Int
val specversion = 0x03.toByte()
class VirtualDisk(
/** capacity of 0 makes the disk read-only */
var capacity: Long,
var diskName: ByteArray = ByteArray(NAME_LENGTH),
footer: net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64 = net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64(8) // default to mandatory 8-byte footer
) {
var footerBytes: net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64 = footer
private set
val entries = HashMap<EntryID, DiskEntry>()
var isReadOnly: Boolean
set(value) { footerBytes[0] = (footerBytes[0] and 0xFE.toByte()) or value.toBit() }
get() = capacity == 0L || (footerBytes.size > 0 && footerBytes[0].and(1) == 1.toByte())
fun getDiskNameString(charset: Charset) = String(diskName, charset)
val root: DiskEntry
get() = entries[0]!!
private fun Boolean.toBit() = if (this) 1.toByte() else 0.toByte()
internal fun __internalSetFooter__(footer: net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64) {
footerBytes = footer
}
private fun serializeEntriesOnly(): net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64 {
val bufferList = ArrayList<Byte>() // FIXME this part would take up excessive memory for large files
entries.forEach {
val serialised = it.value.serialize()
serialised.forEach { bufferList.add(it) }
}
val byteArray = net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64(bufferList.size.toLong())
bufferList.forEachIndexed { index, byte -> byteArray[index.toLong()] = byte }
return byteArray
}
fun serialize(): AppendableByteBuffer {
val entriesBuffer = serializeEntriesOnly()
val buffer = AppendableByteBuffer(HEADER_SIZE + entriesBuffer.size + FOOTER_SIZE + footerBytes.size)
val crc = hashCode().toBigEndian()
buffer.put(MAGIC)
buffer.put(capacity.toInt48())
buffer.put(diskName.forceSize(NAME_LENGTH))
buffer.put(crc)
buffer.put(specversion)
buffer.put(entriesBuffer)
buffer.put(FOOTER_START_MARK)
buffer.put(footerBytes)
buffer.put(EOF_MARK)
return buffer
}
override fun hashCode(): Int {
val crcList = IntArray(entries.size)
var crcListAppendCursor = 0
entries.forEach { _, u ->
crcList[crcListAppendCursor] = u.hashCode()
crcListAppendCursor++
}
crcList.sort()
val crc = CRC32()
crcList.forEach { crc.update(it) }
return crc.value.toInt()
}
/** Expected size of the virtual disk */
val usedBytes: Long
get() = entries.map { it.value.serialisedSize }.sum() + HEADER_SIZE + FOOTER_SIZE
fun generateUniqueID(): Int {
var id: Int
do {
id = Random().nextInt()
} while (null != entries[id] || id == FOOTER_MARKER)
return id
}
override fun equals(other: Any?) = if (other == null) false else this.hashCode() == other.hashCode()
override fun toString() = "VirtualDisk(name: ${getDiskNameString(Charsets.UTF_8)}, capacity: $capacity bytes, crc: ${hashCode().toHex()})"
companion object {
val HEADER_SIZE = 47L // according to the spec
val FOOTER_SIZE = 6L // footer mark + EOF
val NAME_LENGTH = 32
val MAGIC = "TEVd".toByteArray()
val FOOTER_MARKER = 0xFEFEFEFE.toInt()
val FOOTER_START_MARK = FOOTER_MARKER.toBigEndian()
val EOF_MARK = byteArrayOf(0xFF.toByte(), 0x19.toByte())
}
}
class DiskEntry(
// header
var entryID: EntryID,
var parentEntryID: EntryID,
var filename: ByteArray = ByteArray(NAME_LENGTH),
var creationDate: Long,
var modificationDate: Long,
// content
val contents: DiskEntryContent
) {
fun getFilenameString(charset: Charset) = if (entryID == 0) ROOTNAME else filename.toCanonicalString(charset)
val serialisedSize: Long
get() = contents.getSizeEntry() + HEADER_SIZE
companion object {
val HEADER_SIZE = 281L // according to the spec
val ROOTNAME = "(root)"
val NAME_LENGTH = 256
val NORMAL_FILE = 1.toByte()
val DIRECTORY = 2.toByte()
val SYMLINK = 3.toByte()
val COMPRESSED_FILE = 0x11.toByte()
private fun DiskEntryContent.getTypeFlag() =
if (this is EntryFile) NORMAL_FILE
else if (this is EntryDirectory) DIRECTORY
else if (this is EntrySymlink) SYMLINK
else 0 // NULL
fun getTypeString(entry: DiskEntryContent) = when(entry.getTypeFlag()) {
NORMAL_FILE -> "File"
DIRECTORY -> "Directory"
SYMLINK -> "Symbolic Link"
else -> "(unknown type)"
}
}
fun serialize(): AppendableByteBuffer {
val serialisedContents = contents.serialize()
val buffer = AppendableByteBuffer(HEADER_SIZE + serialisedContents.size)
buffer.put(entryID.toBigEndian())
buffer.put(parentEntryID.toBigEndian())
buffer.put(contents.getTypeFlag())
buffer.put(filename.forceSize(NAME_LENGTH))
buffer.put(creationDate.toInt48())
buffer.put(modificationDate.toInt48())
buffer.put(this.hashCode().toBigEndian())
buffer.put(serialisedContents.array)
return buffer
}
override fun hashCode() = contents.serialize().getCRC32()
override fun equals(other: Any?) = if (other == null) false else this.hashCode() == other.hashCode()
override fun toString() = "DiskEntry(name: ${getFilenameString(Charsets.UTF_8)}, index: $entryID, type: ${contents.getTypeFlag()}, crc: ${hashCode().toHex()})"
}
fun ByteArray.forceSize(size: Int): ByteArray {
return ByteArray(size, { if (it < this.size) this[it] else 0.toByte() })
}
interface DiskEntryContent {
fun serialize(): AppendableByteBuffer
fun getSizePure(): Long
fun getSizeEntry(): Long
}
/**
* Do not retrieve bytes directly from this! Use VDUtil.retrieveFile(DiskEntry)
* And besides, the bytes could be compressed.
*/
open class EntryFile(internal var bytes: net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64) : DiskEntryContent {
override fun getSizePure() = bytes.size
override fun getSizeEntry() = getSizePure() + 6
/** Create new blank file */
constructor(size: Long): this(net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64(size))
override fun serialize(): AppendableByteBuffer {
val buffer = AppendableByteBuffer(getSizeEntry())
buffer.put(getSizePure().toInt48())
buffer.put(bytes)
return buffer
}
}
class EntryFileCompressed(internal var uncompressedSize: Long, bytes: net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64) : EntryFile(bytes) {
override fun getSizePure() = bytes.size
override fun getSizeEntry() = getSizePure() + 12
/* No new blank file for the compressed */
override fun serialize(): AppendableByteBuffer {
val buffer = AppendableByteBuffer(getSizeEntry())
buffer.put(getSizePure().toInt48())
buffer.put(uncompressedSize.toInt48())
buffer.put(bytes)
return buffer
}
}
class EntryDirectory(private val entries: ArrayList<EntryID> = ArrayList<EntryID>()) : DiskEntryContent {
override fun getSizePure() = entries.size * 4L
override fun getSizeEntry() = getSizePure() + 2
private fun checkCapacity(toAdd: Int = 1) {
if (entries.size + toAdd > 65535)
throw IOException("Directory entries limit exceeded.")
}
fun add(entryID: EntryID) {
checkCapacity()
entries.add(entryID)
}
fun remove(entryID: EntryID) {
entries.remove(entryID)
}
fun contains(entryID: EntryID) = entries.contains(entryID)
fun forEach(consumer: (EntryID) -> Unit) = entries.forEach(consumer)
val entryCount: Int
get() = entries.size
override fun serialize(): AppendableByteBuffer {
val buffer = AppendableByteBuffer(getSizeEntry())
buffer.put(entries.size.toShort().toBigEndian())
entries.forEach { indexNumber -> buffer.put(indexNumber.toBigEndian()) }
return buffer
}
companion object {
val NEW_ENTRY_SIZE = DiskEntry.HEADER_SIZE + 4L
}
}
class EntrySymlink(val target: EntryID) : DiskEntryContent {
override fun getSizePure() = 4L
override fun getSizeEntry() = 4L
override fun serialize(): AppendableByteBuffer {
val buffer = AppendableByteBuffer(4)
return buffer.put(target.toBigEndian())
}
}
fun Int.toHex() = this.toLong().and(0xFFFFFFFF).toString(16).padStart(8, '0').toUpperCase()
fun Int.toBigEndian(): ByteArray {
return ByteArray(4, { this.ushr(24 - (8 * it)).toByte() })
}
fun Long.toInt48(): ByteArray {
return ByteArray(6, { this.ushr(40 - (8 * it)).toByte() })
}
fun Short.toBigEndian(): ByteArray {
return byteArrayOf(
this.div(256).toByte(),
this.toByte()
)
}
fun AppendableByteBuffer.getCRC32(): Int {
val crc = CRC32()
this.array.forEachInt32 { crc.update(it) }
return crc.value.toInt()
}
class AppendableByteBuffer(val size: Long) {
val array = net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64(size)
private var offset = 0L
fun put(byteArray64: net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64): AppendableByteBuffer {
// it's slow but works
// can't do system.arrayCopy directly
byteArray64.forEach { put(it) }
return this
}
fun put(byteArray: ByteArray): AppendableByteBuffer {
byteArray.forEach { put(it) }
return this
}
fun put(byte: Byte): AppendableByteBuffer {
array[offset] = byte
offset += 1
return this
}
fun forEach(consumer: (Byte) -> Unit) = array.forEach(consumer)
}