package net.torvald.terrarum.savegame import net.torvald.terrarum.App.printdbg import net.torvald.terrarum.Snapshot import net.torvald.terrarum.savegame.VDSaveKind.PLAYER_DATA import net.torvald.terrarum.savegame.VDSaveKind.WORLD_DATA import net.torvald.terrarum.serialise.Common import net.torvald.terrarum.serialise.toUint import java.io.File import java.io.IOException import java.nio.charset.Charset import java.util.* import java.util.zip.CRC32 /* # Terran Virtual Disk Image Format Specification current specversion number: 254 ## Changes Version 254 is a customised version of TEVD tailored to be used as a savegame format for Terrarum. ### 254 - Removed Compressed File; a compression tool is provided instead - Footer moved up to the header (thus freeing the entry id 0xFEFEFEFE) - Entry IDs are extended to 8 bytes - Removed the file name field ### 0x03 - Option to compress file entry ### 0x02 - 48-Bit filesize and timestamp (Max 256 TiB / 8.9 million years) - 8 Reserved footer ### 0x01 **Note: this version was never released to public** - Doubly Linked List instead of Singly ## Specs * File structure Header ... * Order of the indices does not matter. Actual sorting is a job of the application. * Endianness: Big ## Header UInt8[4] Magic: TEVd Int48 Disk size in bytes (max 256 TiB) UInt8[32] Disk name Int32 CRC-32 1. create list of arrays that contains CRC 2. put all the CRCs of entries 3. sort the list (here's the catch -- you will treat CRCs as SIGNED integer) 4. for elems on list: update crc with the elem (crc = calculateCRC(crc, elem)) Int8 Version (0xFE) ----- alongside the Marker Byte, these 2 bytes are designed to raise errors when Int8 Marker Byte (0xFE) - the disk archive was read by the previous versions of the implementation. /* BEGIN extraInfoBytes */ Int8 Disk properties flag 1 0th bit: readonly Int8 Save type (0b 0000 00ab) b: unset - full save; set - quicksave (only applicable to worlds -- quicksave just means the disk is in dirty state) a: set - generated by autosave Int8 Kind of the Save file 0: Undefined (or very old version of the game) 1: Player Data 2: World Data Int8 Savefile Origin Flags (lower nybble: persistent, upper nybble: can be removed if conditions are met) 0: Created in-game 16: Imported (will be removed once the file is loaded by the player and saved in-game) Int16 Snapshot Number 0b A_yyyyyyy wwwwww_aa where: y: Current Year - 2000 (2023 -> 23) w: ISO Week Number (1-53) Aaa: Alphabet (a->000, b->001, ... e->100, f->101, ..., h->111) e.g. 23w40f is encoded as 1_0010111 101000_01 Int8 Game Mode (0b ww tttttt) t: 0 - Undecided 1 - Survival (0.4.0+) w: 0 - Singleplayer 2 - Multiplayer Int8[9] Extra info bytes reserved for future usage -- END extraInfoBytes -- UInt8[236] Rest of the long disk name (268 bytes total) (Header size: 300 bytes) ## IndexNumber and Contents NOTES: - entries are not guaranteed to be sorted, even though the Disk Cracker will show them sorted. - Root entry (ID=0) however, must be the first entry that comes right after the header. - Parent node of the root is undefined; do not make an assumption that parent of the root node is 0. ### Entry Header Int64 EntryID (random Long). This act as "jump" position for directory listing. NOTE: Index 0 must be a root "Directory" Int64 EntryID of parent directory UInt8 Type Marker 0b d000 00tt, where: tt: 01—Normal file, 10—Directory list, 11—Symlink d: discard the entry if the bit is set UInt8[3] Int48 Creation date in real-life UNIX timestamp Int48 Last modification date in real-life UNIX timestamp Int32 CRC-32 of Actual Entry (entrysize and the actual bytes concatenated) (Header size: 36 bytes) ### Entry of File Int48 File size in bytes (max 256 TiB) Actual Contents (Header size: 6 bytes) ### Entry of Directory UInt32 Number of entries (normal files, other directories, symlinks) Entry listing, contains IndexNumber (Header size: 4 bytes) */ typealias EntryID = Long val specversion = 254.toByte() /** * This class provides DOM (disk object model) of the TEVD virtual filesystem. * * Created by minjaesong on 2021-09-10. */ class VirtualDisk( /** capacity of 0 makes the disk read-only */ var capacity: Long, var diskName: ByteArray = ByteArray(NAME_LENGTH), var origin: File? = null ): SimpleFileSystem { override fun getBackingFile() = origin var extraInfoBytes = ByteArray(16) val entries = HashMap() val isReadOnly = false var saveMode: Int set(value) { extraInfoBytes[1] = value.toByte() } get() = extraInfoBytes[1].toUint() var saveKind: Int set(value) { extraInfoBytes[2] = value.toByte() } get() = extraInfoBytes[2].toUint() var saveOrigin: Int set(value) { extraInfoBytes[3] = value.toByte() } get() = extraInfoBytes[3].toUint() var snapshot: Snapshot? set(value) { if (value == null) { extraInfoBytes[4] = 0 extraInfoBytes[5] = 0 } else { value.toBytes().forEachIndexed { index, byte -> extraInfoBytes[4+index] = byte } } } get() { return if (extraInfoBytes[4] == extraInfoBytes[5] && extraInfoBytes[4] == 0.toByte()) null else { Snapshot(extraInfoBytes.sliceArray(4..5)) } } var gamemode: Int set(value) { extraInfoBytes[6] = value.toByte() } get() = extraInfoBytes[6].toUint() override fun getDiskName(charset: Charset) = diskName.toCanonicalString(charset) val root: DiskEntry get() = entries[0]!! private fun Boolean.toBit() = if (this) 1.toByte() else 0.toByte() internal fun __internalSetFooter__(footer: ByteArray64) { extraInfoBytes = footer.toByteArray() } override fun getEntry(id: EntryID) = entries[id] override fun getFile(id: EntryID) = try { VDUtil.getAsNormalFile(this, id) } catch (e: NullPointerException) { null } private fun serializeEntriesOnly(): ByteArray64 { val buffer = ByteArray64() // make sure to write root directory first entries[0L]!!.let { rootDir -> buffer.appendBytes(rootDir.serialize()) printdbg(this, "Writing disk ${getDiskName(Common.CHARSET)}") printdbg(this, "Root creation: ${rootDir.creationDate}, modified: ${rootDir.modificationDate}") } entries.forEach { if (it.key != 0L) { buffer.appendBytes(it.value.serialize()) } } return buffer } fun serialize(): ByteArray64 { val entriesBuffer = serializeEntriesOnly() val buffer = ByteArray64(HEADER_SIZE + entriesBuffer.size) val crc = hashCode().toBigEndian() val diskName0 = diskName.forceSize(NAME_LENGTH) val diskName1 = diskName0.sliceArray(0..31).forceSize(32) val diskName2 = diskName0.sliceArray(32 until NAME_LENGTH).forceSize(NAME_LENGTH - 32) buffer.appendBytes(MAGIC) buffer.appendBytes(capacity.toInt48()) buffer.appendBytes(diskName1) buffer.appendBytes(crc) buffer.appendByte(specversion) buffer.appendByte(0xFE.toByte()) buffer.appendBytes(extraInfoBytes) buffer.appendBytes(diskName2) buffer.appendBytes(entriesBuffer) return buffer } override fun hashCode(): Int { val crcList = IntArray(entries.size) var crcListAppendCursor = 0 entries.forEach { _, u -> crcList[crcListAppendCursor] = u.hashCode() crcListAppendCursor++ } crcList.sort() val crc = CRC32() crcList.forEach { crc.update(it) } return crc.value.toInt() } /** Expected size of the virtual disk */ val usedBytes: Long get() = entries.map { it.value.serialisedSize }.sum() + HEADER_SIZE fun generateUniqueID(): Long { var id: Long do { id = Random().nextLong() } while (null != entries[id]) return id } override fun equals(other: Any?) = if (other == null) false else this.hashCode() == other.hashCode() override fun toString() = "VirtualDisk(name: ${getDiskName(Charsets.UTF_8)}, capacity: $capacity bytes, crc: ${hashCode().toHex()})" companion object { val HEADER_SIZE = 300L // according to the spec val NAME_LENGTH = 268 val MAGIC = "TEVd".toByteArray() } } object VDSaveKind { const val UNDEFINED = 0 const val PLAYER_DATA = 1 const val WORLD_DATA = 2 } object VDSaveOrigin { const val INGAME = 0 const val IMPORTED = 16 } object VDSaveMode { operator fun invoke(actorvalue: String?) = when (actorvalue?.lowercase()) { "survival" -> 1 else -> 0 } } object VDFileID { const val ROOT = 0L const val SAVEGAMEINFO = -1L const val PLAYER_JSON = -1L const val WORLD_SCREENSHOT = -2L const val SPRITEDEF = -2L const val SPRITEDEF_GLOW = -3L const val LOADORDER = -4L const val PLAYER_SCREENSHOT = -5L const val SPRITEDEF_EMISSIVE = -6L const val BODYPART_TO_ENTRY_MAP = -1025L const val BODYPARTGLOW_TO_ENTRY_MAP = -1026L const val BODYPARTEMISSIVE_TO_ENTRY_MAP = -1027L } fun diskIDtoReadableFilename(id: EntryID, saveKind: Int?): String = when (id) { VDFileID.ROOT -> "root" VDFileID.SAVEGAMEINFO -> "savegameinfo.json" VDFileID.WORLD_SCREENSHOT, VDFileID.SPRITEDEF -> if (saveKind == PLAYER_DATA) "spritedef" else if (saveKind == WORLD_DATA) "thumbnail.tga.gz" else "thumbnail.tga.gz (world)/spritedef (player)" VDFileID.SPRITEDEF_GLOW -> if (saveKind == PLAYER_DATA) "spritedef-glow" else "file #$id" VDFileID.PLAYER_SCREENSHOT -> if (saveKind == PLAYER_DATA) "screenshot.tga.gz" else "file #$id" VDFileID.LOADORDER -> "loadOrder.txt" // -16L -> "blockcodex.json.gz" // -17L -> "itemcodex.json.gz" // -18L -> "wirecodex.json.gz" // -19L -> "materialcodex.json.gz" // -20L -> "factioncodex.json.gz" // -1024L -> "apocryphas.json.gz" VDFileID.BODYPART_TO_ENTRY_MAP -> "bodypart-to-entry.map" VDFileID.BODYPARTGLOW_TO_ENTRY_MAP -> "bodypartglow-to-entry.map" VDFileID.BODYPARTEMISSIVE_TO_ENTRY_MAP -> "bodypartemissive-to-entry.map" in 1..65535 -> if (saveKind == PLAYER_DATA) "bodypart #$id.tga.gz" else "file #$id" in 1048576..2147483647 -> "actor #$id.json" in 0x0000_0001_0000_0000L..0x0000_FFFF_FFFF_FFFFL -> "World${id.ushr(32)}-L${id.and(0xFF00_0000).ushr(24)}-C${id.and(0xFFFFFF)}.gz" else -> "file #$id" } class DiskEntry( // header var entryID: EntryID, var parentEntryID: EntryID, var creationDate: Long, var modificationDate: Long, // content val contents: DiskEntryContent ): Comparable { override fun compareTo(other: DiskEntry) = entryID.compareTo(other.entryID) val serialisedSize: Long get() = contents.getSizeEntry() + HEADER_SIZE companion object { val HEADER_SIZE = 36L // according to the spec val NORMAL_FILE = 1.toByte() val DIRECTORY = 2.toByte() val SYMLINK = 3.toByte() private fun DiskEntryContent.getTypeFlag() = if (this is EntryFile) NORMAL_FILE else if (this is EntryDirectory) DIRECTORY else if (this is EntrySymlink) SYMLINK else 0 // NULL fun getTypeString(entry: DiskEntryContent) = when(entry.getTypeFlag()) { NORMAL_FILE -> "File" DIRECTORY -> "Directory" SYMLINK -> "Symbolic Link" else -> "(unknown type)" } } fun serialize(): ByteArray64 { val serialisedContents = contents.serialize() val buffer = ByteArray64(HEADER_SIZE + serialisedContents.size) buffer.appendBytes(entryID.toBigEndian()) buffer.appendBytes(parentEntryID.toBigEndian()) buffer.appendByte(contents.getTypeFlag()) buffer.appendByte(0); buffer.appendByte(0); buffer.appendByte(0) buffer.appendBytes(creationDate.toInt48()) buffer.appendBytes(modificationDate.toInt48()) buffer.appendBytes(this.hashCode().toBigEndian()) buffer.appendBytes(serialisedContents) return buffer } override fun hashCode() = contents.serialize().getCRC32() override fun equals(other: Any?) = if (other == null) false else this.hashCode() == other.hashCode() override fun toString() = "DiskEntry(name: ${diskIDtoReadableFilename(entryID, null)}, ID: $entryID, parent: $parentEntryID, type: ${contents.getTypeFlag()}, contents size: ${contents.getSizeEntry()}, crc: ${hashCode().toHex()})" } fun ByteArray.forceSize(size: Int): ByteArray { return ByteArray(size) { if (it < this.size) this[it] else 0.toByte() } } interface DiskEntryContent { fun serialize(): ByteArray64 fun getSizePure(): Long fun getSizeEntry(): Long fun getContent(): Any } /** * Do not retrieve bytes directly from this! Use VDUtil.retrieveFile(DiskEntry) * And besides, the bytes could be compressed. */ open class EntryFile(internal var bytes: ByteArray64) : DiskEntryContent { override fun getSizePure() = bytes.size override fun getSizeEntry() = getSizePure() + 6 /** Create new blank file */ constructor(size: Long): this(ByteArray64(size)) override fun serialize(): ByteArray64 { val buffer = ByteArray64(getSizeEntry()) buffer.appendBytes(getSizePure().toInt48()) buffer.appendBytes(bytes) return buffer } override fun getContent() = bytes } class EntryDirectory(private val entries: ArrayList = ArrayList()) : DiskEntryContent { override fun getSizePure() = entries.size * 8L override fun getSizeEntry() = getSizePure() + 4 private fun checkCapacity(toAdd: Long = 1L) { if (entries.size + toAdd > 4294967295L) throw IOException("Directory entries limit exceeded.") } fun add(entryID: EntryID) { checkCapacity() entries.add(entryID) } fun remove(entryID: EntryID) { entries.remove(entryID) } fun contains(entryID: EntryID) = entries.contains(entryID) fun forEach(consumer: (EntryID) -> Unit) = entries.forEach(consumer) val entryCount: Int get() = entries.size override fun serialize(): ByteArray64 { val buffer = ByteArray64(getSizeEntry()) buffer.appendBytes(entries.size.toBigEndian()) entries.sorted().forEach { indexNumber -> buffer.appendBytes(indexNumber.toBigEndian()) } return buffer } override fun getContent() = entries.toLongArray() companion object { val NEW_ENTRY_SIZE = DiskEntry.HEADER_SIZE + 12L } } class EntrySymlink(val target: EntryID) : DiskEntryContent { override fun getSizePure() = 8L override fun getSizeEntry() = 8L override fun serialize(): ByteArray64 { val buffer = ByteArray64(getSizeEntry()) buffer.appendBytes(target.toBigEndian()) return buffer } override fun getContent() = target } fun Int.toHex() = this.toLong().and(0xFFFFFFFF).toString(16).padStart(8, '0').toUpperCase() fun Long.toHex() = this.ushr(32).toInt().toHex() + "_" + this.toInt().toHex() fun Int.toBigEndian(): ByteArray { return ByteArray(4) { this.ushr(24 - (8 * it)).toByte() } } fun Long.toBigEndian(): ByteArray { return ByteArray(8) { this.ushr(56 - (8 * it)).toByte() } } fun Long.toInt48(): ByteArray { return ByteArray(6) { this.ushr(40 - (8 * it)).toByte() } } fun Short.toBigEndian(): ByteArray { return byteArrayOf( this.div(256).toByte(), this.toByte() ) } fun ByteArray64.getCRC32(): Int { val crc = CRC32() this.forEach { crc.update(it.toInt()) } return crc.value.toInt() }