diff --git a/lib/TerranVirtualDisk-src.jar b/lib/TerranVirtualDisk-src.jar deleted file mode 100644 index c8ea1d371..000000000 Binary files a/lib/TerranVirtualDisk-src.jar and /dev/null differ diff --git a/lib/TerranVirtualDisk.jar b/lib/TerranVirtualDisk.jar deleted file mode 100644 index 78cf9edc0..000000000 Binary files a/lib/TerranVirtualDisk.jar and /dev/null differ diff --git a/src/net/torvald/terrarum/IngameInstance.kt b/src/net/torvald/terrarum/IngameInstance.kt index e6a93177c..044ff8751 100644 --- a/src/net/torvald/terrarum/IngameInstance.kt +++ b/src/net/torvald/terrarum/IngameInstance.kt @@ -13,14 +13,11 @@ import net.torvald.terrarum.modulebasegame.IngameRenderer import net.torvald.terrarum.modulebasegame.gameactors.ActorHumanoid import net.torvald.terrarum.modulebasegame.ui.Notification import net.torvald.terrarum.modulebasegame.ui.UITooltip -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.VirtualDisk +import net.torvald.terrarum.tvda.VirtualDisk +import net.torvald.terrarum.realestate.LandUtil import net.torvald.terrarum.ui.ConsoleWindow import net.torvald.util.SortedArrayList -import org.khelekore.prtree.DistanceCalculator -import org.khelekore.prtree.DistanceResult -import org.khelekore.prtree.MBRConverter -import org.khelekore.prtree.PRTree -import org.khelekore.prtree.PointND +import org.khelekore.prtree.* import java.util.concurrent.locks.Lock /** @@ -121,6 +118,8 @@ open class IngameInstance(val batch: SpriteBatch) : Screen { val wallChangeQueue = ArrayList() val wireChangeQueue = ArrayList() // if 'old' is set and 'new' is blank, it's a wire cutter + val modifiedChunks = Array(16) { HashSet() } + var loadedTime_t = App.getTIME_T() protected set @@ -227,6 +226,14 @@ open class IngameInstance(val batch: SpriteBatch) : Screen { } + open fun modified(layer: Int, x: Int, y: Int) { + modifiedChunks[layer].add(LandUtil.toChunkNum(world, x, y)) + } + + open fun clearModifiedChunks() { + modifiedChunks.forEach { it.clear() } + } + /////////////////////// // UTILITY FUNCTIONS // diff --git a/src/net/torvald/terrarum/debuggerapp/SavegameCracker.kt b/src/net/torvald/terrarum/debuggerapp/SavegameCracker.kt index 9110f3d54..ace483e29 100644 --- a/src/net/torvald/terrarum/debuggerapp/SavegameCracker.kt +++ b/src/net/torvald/terrarum/debuggerapp/SavegameCracker.kt @@ -1,19 +1,19 @@ package net.torvald.terrarum.debuggerapp import net.torvald.terrarum.TerrarumAppConfiguration -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.* import net.torvald.terrarum.serialise.Common +import net.torvald.terrarum.tvda.EntryFile +import net.torvald.terrarum.tvda.VDUtil +import net.torvald.terrarum.tvda.VirtualDisk +import net.torvald.terrarum.tvda.diskIDtoReadableFilename import java.io.File -import java.io.InputStream import java.io.PrintStream import java.nio.charset.Charset import java.util.* import java.util.logging.Level import kotlin.reflect.KFunction import kotlin.reflect.full.declaredFunctions -import kotlin.reflect.full.declaredMemberFunctions import kotlin.reflect.full.findAnnotation -import kotlin.reflect.jvm.isAccessible private val ESC = 27.toChar() @@ -151,7 +151,7 @@ class SavegameCracker( if (i != 0L) println( ccNoun + i.toString(10).padStart(11, ' ') + " " + - ccNoun2 + (entry.filename.toCanonicalString(charset) + cc0).padEnd(18) { if (it == 0) ' ' else '.' } + + ccNoun2 + (diskIDtoReadableFilename(entry.entryID) + cc0).padEnd(24) { if (it == 0) ' ' else '.' } + ccConst + " " + entry.contents.getSizePure() + " bytes" ) } @@ -195,16 +195,6 @@ class SavegameCracker( } } - @Command("Renames one file into another", "entry-id new-name") - fun mv(args: List) { - letdisk { - val id = args[1].toLong(10) - val newname = args[2] - it.entries[id]!!.filename = newname.toByteArray(charset) - return@letdisk null - } - } - @Command("Imports a real file onto the savefile", "input-file entry-id") fun import(args: List) { letdisk { diff --git a/src/net/torvald/terrarum/gameworld/GameWorld.kt b/src/net/torvald/terrarum/gameworld/GameWorld.kt index add55be29..c683408bc 100644 --- a/src/net/torvald/terrarum/gameworld/GameWorld.kt +++ b/src/net/torvald/terrarum/gameworld/GameWorld.kt @@ -252,8 +252,10 @@ class GameWorld() : Disposable { layerWall.unsafeSetTile(x, y, tilenum) wallDamages.remove(LandUtil.getBlockAddr(this, x, y)) - if (!bypassEvent) + if (!bypassEvent) { Terrarum.ingame?.queueWallChangedEvent(oldWall, itemID, x, y) + Terrarum.ingame?.modified(LandUtil.LAYER_WALL, x, y) + } } /** @@ -282,8 +284,10 @@ class GameWorld() : Disposable { } // fluid tiles-item should be modified so that they will also place fluid onto their respective map - if (!bypassEvent) + if (!bypassEvent) { Terrarum.ingame?.queueTerrainChangedEvent(oldTerrain, itemID, x, y) + Terrarum.ingame?.modified(LandUtil.LAYER_TERR, x, y) + } } fun setTileWire(x: Int, y: Int, tile: ItemID, bypassEvent: Boolean) { @@ -297,8 +301,10 @@ class GameWorld() : Disposable { wirings[blockAddr]!!.ws.add(tile) - if (!bypassEvent) + if (!bypassEvent) { Terrarum.ingame?.queueWireChangedEvent(tile, false, x, y) + Terrarum.ingame?.modified(LandUtil.LAYER_WIRE, x, y) + } // figure out wiring graphs diff --git a/src/net/torvald/terrarum/modulebasegame/TerrarumIngame.kt b/src/net/torvald/terrarum/modulebasegame/TerrarumIngame.kt index 1afe817cb..4cdb0c7bb 100644 --- a/src/net/torvald/terrarum/modulebasegame/TerrarumIngame.kt +++ b/src/net/torvald/terrarum/modulebasegame/TerrarumIngame.kt @@ -34,7 +34,8 @@ import net.torvald.terrarum.modulebasegame.ui.* import net.torvald.terrarum.modulebasegame.worldgenerator.RoguelikeRandomiser import net.torvald.terrarum.modulebasegame.worldgenerator.Worldgen import net.torvald.terrarum.modulebasegame.worldgenerator.WorldgenParams -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.VDUtil +import net.torvald.terrarum.tvda.VDUtil +import net.torvald.terrarum.realestate.LandUtil import net.torvald.terrarum.serialise.Common import net.torvald.terrarum.serialise.WriteMeta import net.torvald.terrarum.ui.UICanvas @@ -232,7 +233,13 @@ open class TerrarumIngame(batch: SpriteBatch) : IngameInstance(batch) { val height: Int, val worldGenSeed: Long // other worldgen options - ) + ) { + init { + if (width % LandUtil.CHUNK_W != 0 || height % LandUtil.CHUNK_H != 0) { + throw IllegalArgumentException("World size is not a multiple of chunk size; World size: ($width, $height), Chunk size: (${LandUtil.CHUNK_W}, ${LandUtil.CHUNK_H})") + } + } + } data class Codices( val meta: WriteMeta.WorldMeta, diff --git a/src/net/torvald/terrarum/modulebasegame/console/Load.kt b/src/net/torvald/terrarum/modulebasegame/console/Load.kt index ce1af78d0..c969859e3 100644 --- a/src/net/torvald/terrarum/modulebasegame/console/Load.kt +++ b/src/net/torvald/terrarum/modulebasegame/console/Load.kt @@ -7,7 +7,7 @@ import net.torvald.terrarum.ccG import net.torvald.terrarum.ccR import net.torvald.terrarum.console.ConsoleCommand import net.torvald.terrarum.console.Echo -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.VDUtil +import net.torvald.terrarum.tvda.VDUtil import net.torvald.terrarum.serialise.* import java.io.File import java.io.IOException diff --git a/src/net/torvald/terrarum/modulebasegame/console/ReaderTest.kt b/src/net/torvald/terrarum/modulebasegame/console/ReaderTest.kt index c340110d8..89a0c022c 100644 --- a/src/net/torvald/terrarum/modulebasegame/console/ReaderTest.kt +++ b/src/net/torvald/terrarum/modulebasegame/console/ReaderTest.kt @@ -2,8 +2,8 @@ package net.torvald.terrarum.modulebasegame.console import net.torvald.terrarum.console.ConsoleCommand import net.torvald.terrarum.console.Echo -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64Reader -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64Writer +import net.torvald.terrarum.tvda.ByteArray64Reader +import net.torvald.terrarum.tvda.ByteArray64Writer import net.torvald.terrarum.serialise.Common import net.torvald.terrarum.serialise.toUint import java.io.File diff --git a/src/net/torvald/terrarum/modulebasegame/console/Save.kt b/src/net/torvald/terrarum/modulebasegame/console/Save.kt index b3d018803..6822cf90c 100644 --- a/src/net/torvald/terrarum/modulebasegame/console/Save.kt +++ b/src/net/torvald/terrarum/modulebasegame/console/Save.kt @@ -9,9 +9,9 @@ import net.torvald.terrarum.console.Echo import net.torvald.terrarum.gameactors.Actor import net.torvald.terrarum.gameactors.BlockMarkerActor import net.torvald.terrarum.modulebasegame.TerrarumIngame -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.DiskEntry -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.VDUtil -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.VirtualDisk +import net.torvald.terrarum.tvda.DiskEntry +import net.torvald.terrarum.tvda.VDUtil +import net.torvald.terrarum.tvda.VirtualDisk import net.torvald.terrarum.serialise.Common import net.torvald.terrarum.serialise.WriteSavegame import java.io.File diff --git a/src/net/torvald/terrarum/modulebasegame/ui/UILoadDemoSavefiles.kt b/src/net/torvald/terrarum/modulebasegame/ui/UILoadDemoSavefiles.kt index a8cf89956..7fb6beb5f 100644 --- a/src/net/torvald/terrarum/modulebasegame/ui/UILoadDemoSavefiles.kt +++ b/src/net/torvald/terrarum/modulebasegame/ui/UILoadDemoSavefiles.kt @@ -9,9 +9,9 @@ import com.badlogic.gdx.graphics.g2d.TextureRegion import net.torvald.terrarum.* import net.torvald.terrarum.App.printdbg import net.torvald.terrarum.langpack.Lang -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64InputStream -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.VDUtil -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.VirtualDisk +import net.torvald.terrarum.tvda.ByteArray64InputStream +import net.torvald.terrarum.tvda.VDUtil +import net.torvald.terrarum.tvda.VirtualDisk import net.torvald.terrarum.serialise.Common import net.torvald.terrarum.serialise.LoadSavegame import net.torvald.terrarum.serialise.ReadMeta diff --git a/src/net/torvald/terrarum/modulebasegame/ui/UIProxyNewRandomGame.kt b/src/net/torvald/terrarum/modulebasegame/ui/UIProxyNewRandomGame.kt index bb576baec..c605f4b32 100644 --- a/src/net/torvald/terrarum/modulebasegame/ui/UIProxyNewRandomGame.kt +++ b/src/net/torvald/terrarum/modulebasegame/ui/UIProxyNewRandomGame.kt @@ -2,7 +2,6 @@ package net.torvald.terrarum.modulebasegame.ui import com.badlogic.gdx.graphics.Camera import com.badlogic.gdx.graphics.g2d.SpriteBatch -import net.torvald.random.HQRNG import net.torvald.terrarum.App import net.torvald.terrarum.App.printdbg import net.torvald.terrarum.Second @@ -38,8 +37,8 @@ class UIProxyNewRandomGame : UICanvas() { val ingame = TerrarumIngame(App.batch) -// val worldParam = TerrarumIngame.NewWorldParameters(2400, 1280, 0x51621DL) - val worldParam = TerrarumIngame.NewWorldParameters(2400, 1280, HQRNG().nextLong()) +// val worldParam = TerrarumIngame.NewWorldParameters(2880, 1344, HQRNG().nextLong()) + val worldParam = TerrarumIngame.NewWorldParameters(2880, 1350, 0x51621D) //val worldParam = TerrarumIngame.NewWorldParameters(6000, 1800, 0x51621DL) // small // val worldParam = TerrarumIngame.NewWorldParameters(9000, 2250, 0x51621DL) // normal diff --git a/src/net/torvald/terrarum/realestate/LandUtil.kt b/src/net/torvald/terrarum/realestate/LandUtil.kt index 1e7da6cd5..bb9e7338d 100644 --- a/src/net/torvald/terrarum/realestate/LandUtil.kt +++ b/src/net/torvald/terrarum/realestate/LandUtil.kt @@ -1,27 +1,47 @@ package net.torvald.terrarum.realestate +import net.torvald.terrarum.FactionCodex +import net.torvald.terrarum.Point2i import net.torvald.terrarum.Terrarum -import net.torvald.terrarum.gameactors.faction.FactionCodex import net.torvald.terrarum.gameworld.BlockAddress import net.torvald.terrarum.gameworld.GameWorld -import net.torvald.terrarum.gameworld.fmod -import net.torvald.terrarum.* /** * Created by minjaesong on 2016-03-27. */ object LandUtil { + + const val CHUNK_W = 90 + const val CHUNK_H = 90 + + const val LAYER_TERR = 0 + const val LAYER_WALL = 1 + const val LAYER_WIRE = 2 + const val LAYER_FLUID = 3 + + fun toChunkNum(world: GameWorld, x: Int, y: Int): Int { + // coercing and fmod-ing follows ROUNDWORLD rule. See: GameWorld.coerceXY() + val (x, y) = world.coerceXY(x, y) + return (x / CHUNK_W) + (y / CHUNK_H) * (world.width / CHUNK_W) + } + + fun toChunkIndices(world: GameWorld, x: Int, y: Int): Point2i { + // coercing and fmod-ing follows ROUNDWORLD rule. See: GameWorld.coerceXY() + val (x, y) = world.coerceXY(x, y) + return Point2i(x / CHUNK_W, y / CHUNK_H) + } + fun getBlockAddr(world: GameWorld, x: Int, y: Int): BlockAddress { // coercing and fmod-ing follows ROUNDWORLD rule. See: GameWorld.coerceXY() val (x, y) = world.coerceXY(x, y) return (world.width.toLong() * y) + x } - fun resolveBlockAddr(world: GameWorld, t: BlockAddress): Pair = - Pair((t % world.width).toInt(), (t / world.width).toInt()) + fun resolveBlockAddr(world: GameWorld, t: BlockAddress): Point2i = + Point2i((t % world.width).toInt(), (t / world.width).toInt()) - fun resolveBlockAddr(width: Int, t: BlockAddress): Pair = - Pair((t % width).toInt(), (t / width).toInt()) + fun resolveBlockAddr(width: Int, t: BlockAddress): Point2i = + Point2i((t % width).toInt(), (t / width).toInt()) /** * Get owner ID as an Actor/Faction diff --git a/src/net/torvald/terrarum/serialise/Common.kt b/src/net/torvald/terrarum/serialise/Common.kt index c840feca7..c470f7d5a 100644 --- a/src/net/torvald/terrarum/serialise/Common.kt +++ b/src/net/torvald/terrarum/serialise/Common.kt @@ -7,10 +7,10 @@ import net.torvald.terrarum.console.EchoError import net.torvald.terrarum.gameworld.BlockLayer import net.torvald.terrarum.gameworld.GameWorld import net.torvald.terrarum.gameworld.WorldTime -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64 -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64GrowableOutputStream -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64InputStream -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64Reader +import net.torvald.terrarum.tvda.ByteArray64 +import net.torvald.terrarum.tvda.ByteArray64GrowableOutputStream +import net.torvald.terrarum.tvda.ByteArray64InputStream +import net.torvald.terrarum.tvda.ByteArray64Reader import net.torvald.terrarum.utils.* import org.apache.commons.codec.digest.DigestUtils import java.io.InputStream diff --git a/src/net/torvald/terrarum/serialise/WriteActor.kt b/src/net/torvald/terrarum/serialise/WriteActor.kt index b7cebd211..bd9f5a48e 100644 --- a/src/net/torvald/terrarum/serialise/WriteActor.kt +++ b/src/net/torvald/terrarum/serialise/WriteActor.kt @@ -8,8 +8,8 @@ import net.torvald.terrarum.gameactors.ActorWithBody import net.torvald.terrarum.modulebasegame.TerrarumIngame import net.torvald.terrarum.modulebasegame.gameactors.ActorHumanoid import net.torvald.terrarum.modulebasegame.gameactors.Pocketed -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64 -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64Writer +import net.torvald.terrarum.tvda.ByteArray64 +import net.torvald.terrarum.tvda.ByteArray64Writer import java.io.Reader /** diff --git a/src/net/torvald/terrarum/serialise/WriteMeta.kt b/src/net/torvald/terrarum/serialise/WriteMeta.kt index 4b55fbfbf..49c88cc39 100644 --- a/src/net/torvald/terrarum/serialise/WriteMeta.kt +++ b/src/net/torvald/terrarum/serialise/WriteMeta.kt @@ -4,10 +4,10 @@ import net.torvald.terrarum.ModMgr import net.torvald.terrarum.gameactors.ActorID import net.torvald.terrarum.modulebasegame.TerrarumIngame import net.torvald.terrarum.modulebasegame.worldgenerator.RoguelikeRandomiser -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64 -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64Reader -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.EntryFile -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.VirtualDisk +import net.torvald.terrarum.tvda.ByteArray64 +import net.torvald.terrarum.tvda.ByteArray64Reader +import net.torvald.terrarum.tvda.EntryFile +import net.torvald.terrarum.tvda.VirtualDisk import net.torvald.terrarum.weather.WeatherMixer /** diff --git a/src/net/torvald/terrarum/serialise/WriteSavegame.kt b/src/net/torvald/terrarum/serialise/WriteSavegame.kt index 7e4b39b1c..fc6bd9fb6 100644 --- a/src/net/torvald/terrarum/serialise/WriteSavegame.kt +++ b/src/net/torvald/terrarum/serialise/WriteSavegame.kt @@ -8,10 +8,11 @@ import net.torvald.terrarum.console.Echo import net.torvald.terrarum.modulebasegame.IngameRenderer import net.torvald.terrarum.modulebasegame.TerrarumIngame import net.torvald.terrarum.modulebasegame.gameactors.IngamePlayer -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.* +import net.torvald.terrarum.realestate.LandUtil import net.torvald.terrarum.serialise.Common.getUnzipInputStream import net.torvald.terrarum.serialise.Common.zip import net.torvald.terrarum.serialise.WriteWorld.actorAcceptable +import net.torvald.terrarum.tvda.* import java.io.File import java.io.Reader import java.util.zip.GZIPOutputStream @@ -51,12 +52,12 @@ object WriteSavegame { // Write Meta // val metaContent = EntryFile(WriteMeta.encodeToByteArray64(ingame, currentPlayTime_t)) - val meta = DiskEntry(-1, 0, "savegame".toByteArray(Common.CHARSET), creation_t, time_t, metaContent) + val meta = DiskEntry(-1, 0, creation_t, time_t, metaContent) addFile(disk, meta) val thumbContent = EntryFile(tgaout.toByteArray64()) - val thumb = DiskEntry(-2, 0, "thumb".toByteArray(Common.CHARSET), creation_t, time_t, thumbContent) + val thumb = DiskEntry(-2, 0, creation_t, time_t, thumbContent) addFile(disk, thumb) @@ -68,7 +69,7 @@ object WriteSavegame { // Write ItemCodex// val itemCodexContent = EntryFile(zip(ByteArray64.fromByteArray(Common.jsoner.toJson(ItemCodex).toByteArray(Common.CHARSET)))) - val items = DiskEntry(-17, 0, "items".toByteArray(Common.CHARSET), creation_t, time_t, itemCodexContent) + val items = DiskEntry(-17, 0, creation_t, time_t, itemCodexContent) addFile(disk, items) // Gotta save dynamicIDs @@ -91,21 +92,42 @@ object WriteSavegame { // Write Apocryphas// val apocryphasContent = EntryFile(zip(ByteArray64.fromByteArray(Common.jsoner.toJson(Apocryphas).toByteArray(Common.CHARSET)))) - val apocryphas = DiskEntry(-1024, 0, "modprops".toByteArray(Common.CHARSET), creation_t, time_t, apocryphasContent) + val apocryphas = DiskEntry(-1024, 0, creation_t, time_t, apocryphasContent) addFile(disk, apocryphas) // Write World // val worldNum = ingame.world.worldIndex - val worldContent = EntryFile(WriteWorld.encodeToByteArray64(ingame)) + /*val worldContent = EntryFile(WriteWorld.encodeToByteArray64(ingame)) val world = DiskEntry(worldNum.toLong(), 0, "world${worldNum}".toByteArray(Common.CHARSET), creation_t, time_t, worldContent) - addFile(disk, world) + addFile(disk, world)*/ + + val layers = arrayOf(ingame.world.layerTerrain, ingame.world.layerWall) + val cw = ingame.world.width / LandUtil.CHUNK_W + val ch = ingame.world.height / LandUtil.CHUNK_H + for (layer in layers.indices) { + for (cy in 0 until ch) { + for (cx in 0 until cw) { + val chunkNumber = (cy * cw + cx).toLong() + val chunkBytes = WriteWorld.encodeChunk(layers[layer], cx, cy) + val entryID = worldNum.toLong().shl(32) or layer.toLong().shl(24) or chunkNumber + + val entryContent = EntryFile(chunkBytes) + val entry = DiskEntry(entryID, 0, creation_t, time_t, entryContent) + // "W1L0-92,15" + addFile(disk, entry) + } + } + } + + // TODO save worldinfo + // Write Actors // listOf(ingame.actorContainerActive, ingame.actorContainerInactive).forEach { actors -> actors.forEach { if (actorAcceptable(it)) { val actorContent = EntryFile(WriteActor.encodeToByteArray64(it)) - val actor = DiskEntry(it.referenceID.toLong(), 0, "actor${it.referenceID}".toByteArray(Common.CHARSET), creation_t, time_t, actorContent) + val actor = DiskEntry(it.referenceID.toLong(), 0, creation_t, time_t, actorContent) addFile(disk, actor) } } diff --git a/src/net/torvald/terrarum/serialise/WriteWorld.kt b/src/net/torvald/terrarum/serialise/WriteWorld.kt index 49ccaef59..e2327fa0a 100644 --- a/src/net/torvald/terrarum/serialise/WriteWorld.kt +++ b/src/net/torvald/terrarum/serialise/WriteWorld.kt @@ -5,10 +5,12 @@ import net.torvald.terrarum.ReferencingRanges import net.torvald.terrarum.gameactors.Actor import net.torvald.terrarum.gameactors.ActorID import net.torvald.terrarum.gameactors.BlockMarkerActor +import net.torvald.terrarum.gameworld.BlockLayer import net.torvald.terrarum.gameworld.GameWorld import net.torvald.terrarum.modulebasegame.TerrarumIngame -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64 -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64Writer +import net.torvald.terrarum.tvda.ByteArray64 +import net.torvald.terrarum.tvda.ByteArray64Writer +import net.torvald.terrarum.realestate.LandUtil import java.io.Reader /** @@ -50,8 +52,22 @@ object WriteWorld { return baw.toByteArray64() } -} + /** + * @return Gzipped chunk. Tile numbers are stored in Big Endian. + */ + fun encodeChunk(layer: BlockLayer, cx: Int, cy: Int): ByteArray64 { + val ba = ByteArray64() + for (y in cy * LandUtil.CHUNK_H until (cy + 1) * LandUtil.CHUNK_H) { + for (x in cx * LandUtil.CHUNK_W until (cx + 1) * LandUtil.CHUNK_W) { + val tilenum = layer.unsafeGetTile(x, y) + ba.add(tilenum.ushr(8).and(255).toByte()) + ba.add(tilenum.and(255).toByte()) + } + } + return Common.zip(ba) + } +} /** @@ -76,4 +92,21 @@ object ReadWorld { return world } + private val cw = LandUtil.CHUNK_W + private val ch = LandUtil.CHUNK_H + + fun decodeToLayer(chunk: ByteArray64, targetLayer: BlockLayer, cx: Int, cy: Int) { + val bytes = Common.unzip(chunk) + if (bytes.size != cw * ch * 2L) + throw UnsupportedOperationException("Chunk size mismatch: decoded chunk size is ${bytes.size} bytes " + + "where ${LandUtil.CHUNK_W * LandUtil.CHUNK_H * 2L} bytes (Int16 of ${LandUtil.CHUNK_W}x${LandUtil.CHUNK_H}) were expected") + + for (k in 0 until cw * ch) { + val tilenum = bytes[2L*k].toUint().shl(8) or bytes[2L*k + 1].toUint() + val offx = k % cw + val offy = k / cw + targetLayer.unsafeSetTile(cx * cw + offx, cy * ch * offy, tilenum) + } + } + } \ No newline at end of file diff --git a/src/net/torvald/terrarum/tests/ByteArray64Test.kt b/src/net/torvald/terrarum/tests/ByteArray64Test.kt index d84f3525b..f99c13bd0 100644 --- a/src/net/torvald/terrarum/tests/ByteArray64Test.kt +++ b/src/net/torvald/terrarum/tests/ByteArray64Test.kt @@ -1,6 +1,6 @@ package net.torvald.terrarum.tests -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64GrowableOutputStream +import net.torvald.terrarum.tvda.ByteArray64GrowableOutputStream import net.torvald.terrarum.serialise.toLittle import net.torvald.terrarum.serialise.toULittle48 import java.util.zip.Deflater diff --git a/src/net/torvald/terrarum/tvd/ByteArray64.kt b/src/net/torvald/terrarum/tvd/ByteArray64.kt new file mode 100644 index 000000000..56d2bc48d --- /dev/null +++ b/src/net/torvald/terrarum/tvd/ByteArray64.kt @@ -0,0 +1,535 @@ +package net.torvald.terrarum.tvda + +import java.io.* +import java.nio.channels.ClosedChannelException +import java.nio.charset.Charset +import java.nio.charset.UnsupportedCharsetException + + +/** + * ByteArray that can hold larger than 2 GiB of Data. + * + * Works kind of like Bank Switching of old game console's cartridges which does same thing. + * + * Note that this class is just a fancy ArrayList. Internal size will grow accordingly + * + * @param initialSize Initial size of the array. If it's not specified, 8192 will be used instead. + * + * Created by Minjaesong on 2017-04-12. + */ +class ByteArray64(initialSize: Long = bankSize.toLong()) { + var internalCapacity: Long = initialSize + private set + + var size = 0L + internal set + + private var finalised = false + + companion object { + val bankSize: Int = 8192 + + fun fromByteArray(byteArray: ByteArray): ByteArray64 { + val ba64 = ByteArray64(byteArray.size.toLong()) + byteArray.forEachIndexed { i, byte -> ba64[i.toLong()] = byte } + return ba64 + } + } + + private val __data: ArrayList + + private fun checkMutability() { + if (finalised) throw IllegalStateException("ByteArray64 is finalised and cannot be modified") + } + + init { + if (internalCapacity < 0) + throw IllegalArgumentException("Invalid array size: $internalCapacity") + else if (internalCapacity == 0L) // signalling empty array + internalCapacity = bankSize.toLong() + + val requiredBanks: Int = (initialSize - 1).toBankNumber() + 1 + + __data = ArrayList(requiredBanks) + repeat(requiredBanks) { __data.add(ByteArray(bankSize)) } + } + + private fun Long.toBankNumber(): Int = (this / bankSize).toInt() + private fun Long.toBankOffset(): Int = (this % bankSize).toInt() + + operator fun set(index: Long, value: Byte) { + checkMutability() + ensureCapacity(index + 1) + + try { + __data[index.toBankNumber()][index.toBankOffset()] = value + size = maxOf(size, index + 1) + } + catch (e: IndexOutOfBoundsException) { + val msg = "index: $index -> bank ${index.toBankNumber()} offset ${index.toBankOffset()}\n" + + "But the array only contains ${__data.size} banks.\n" + + "InternalCapacity = $internalCapacity, Size = $size" + throw IndexOutOfBoundsException(msg) + } + } + + fun add(value: Byte) = set(size, value) + + operator fun get(index: Long): Byte { + if (index < 0 || index >= size) + throw ArrayIndexOutOfBoundsException("size $size, index $index") + + try { + val r = __data[index.toBankNumber()][index.toBankOffset()] + return r + } + catch (e: IndexOutOfBoundsException) { + System.err.println("index: $index -> bank ${index.toBankNumber()} offset ${index.toBankOffset()}") + System.err.println("But the array only contains ${__data.size} banks.") + + throw e + } + } + + private fun addOneBank() { + __data.add(ByteArray(bankSize)) + internalCapacity = __data.size * bankSize.toLong() + } + + /** + * Increases the capacity of it, if necessary, to ensure that it can hold at least the number of elements specified by the minimum capacity argument. + */ + fun ensureCapacity(minCapacity: Long) { + while (minCapacity > internalCapacity) { + addOneBank() + } + } + + + operator fun iterator(): ByteIterator { + return object : ByteIterator() { + var iterationCounter = 0L + + override fun nextByte(): Byte { + iterationCounter += 1 + return this@ByteArray64[iterationCounter - 1] + } + + override fun hasNext() = iterationCounter < this@ByteArray64.size + } + } + + fun iteratorChoppedToInt(): IntIterator { + return object : IntIterator() { + var iterationCounter = 0L + val iteratorSize = 1 + ((this@ByteArray64.size - 1) / 4).toInt() + + override fun nextInt(): Int { + var byteCounter = iterationCounter * 4L + var int = 0 + (0..3).forEach { + if (byteCounter + it < this@ByteArray64.size) { + int += this@ByteArray64[byteCounter + it].toInt() shl (it * 8) + } + else { + int += 0 shl (it * 8) + } + } + + + iterationCounter += 1 + return int + } + + override fun hasNext() = iterationCounter < iteratorSize + } + } + + /** Iterates over all written bytes. */ + fun forEach(consumer: (Byte) -> Unit) = iterator().forEach { consumer(it) } + /** Iterates over all written 32-bit words. */ + fun forEachInt32(consumer: (Int) -> Unit) = iteratorChoppedToInt().forEach { consumer(it) } + /** Iterates over all existing banks, even if they are not used. Please use [forEachUsedBanks] to iterate over banks that are actually been used. */ + fun forEachBanks(consumer: (ByteArray) -> Unit) = __data.forEach(consumer) + /** Iterates over all written bytes. */ + fun forEachIndexed(consumer: (Long, Byte) -> Unit) { + var cnt = 0L + iterator().forEach { + consumer(cnt, it) + cnt += 1 + } + } + /** Iterates over all written 32-bit words. */ + fun forEachInt32Indexed(consumer: (Long, Int) -> Unit) { + var cnt = 0L + iteratorChoppedToInt().forEach { + consumer(cnt, it) + cnt += 1 + } + } + + /** + * @param consumer (Int, Int, ByteArray)-to-Unit function where first Int is index; + * second Int is actual number of bytes written in that bank, 0 to BankSize inclusive. + */ + fun forEachUsedBanksIndexed(consumer: (Int, Int, ByteArray) -> Unit) { + __data.forEachIndexed { index, bytes -> + consumer(index, (size - bankSize * index).coerceIn(0, bankSize.toLong()).toInt(), bytes) + } + } + + /** + * @param consumer (Int, Int, ByteArray)-to-Unit function where Int is actual number of bytes written in that bank, 0 to BankSize inclusive. + */ + fun forEachUsedBanks(consumer: (Int, ByteArray) -> Unit) { + __data.forEachIndexed { index, bytes -> + consumer((size - bankSize * index).coerceIn(0, bankSize.toLong()).toInt(), bytes) + } + } + + fun sliceArray64(range: LongRange): ByteArray64 { + val newarr = ByteArray64(range.last - range.first + 1) + range.forEach { index -> + newarr[index - range.first] = this[index] + } + return newarr + } + + fun sliceArray(range: IntRange): ByteArray { + val newarr = ByteArray(range.last - range.first + 1) + range.forEach { index -> + newarr[index - range.first] = this[index.toLong()] + } + return newarr + } + + fun toByteArray(): ByteArray { + if (this.size > Integer.MAX_VALUE - 8) // according to OpenJDK; the size itself is VM-dependent + throw TypeCastException("Impossible cast; too large to fit") + + return ByteArray(this.size.toInt()) { this[it.toLong()] } + } + + fun writeToFile(file: File) { + var fos = FileOutputStream(file, false) + // following code writes in-chunk basis + /*fos.write(__data[0]) + fos.flush() + fos.close() + + if (__data.size > 1) { + fos = FileOutputStream(file, true) + for (i in 1..__data.lastIndex) { + fos.write(__data[i]) + fos.flush() + } + fos.close() + }*/ + + forEach { + fos.write(it.toInt()) + } + fos.flush() + fos.close() + } + + fun finalise() { + this.finalised = true + } +} + +open class ByteArray64InputStream(val byteArray64: ByteArray64): InputStream() { + protected open var readCounter = 0L + + override fun read(): Int { + readCounter += 1 + + return try { + byteArray64[readCounter - 1].toUint() + } + catch (e: ArrayIndexOutOfBoundsException) { + -1 + } + } +} + +/** Static ByteArray OutputStream. Less leeway, more stable. */ +open class ByteArray64OutputStream(val byteArray64: ByteArray64): OutputStream() { + protected open var writeCounter = 0L + + override fun write(b: Int) { + try { + byteArray64.add(b.toByte()) + writeCounter += 1 + } + catch (e: ArrayIndexOutOfBoundsException) { + throw IOException(e) + } + } + + override fun close() { + byteArray64.finalise() + } +} + +/** Just like Java's ByteArrayOutputStream, except its size grows if you exceed the initial size + */ +open class ByteArray64GrowableOutputStream(size: Long = ByteArray64.bankSize.toLong()): OutputStream() { + protected open var buf = ByteArray64(size) + protected open var count = 0L + + private var finalised = false + + init { + if (size <= 0L) throw IllegalArgumentException("Illegal array size: $size") + } + + override fun write(b: Int) { + if (finalised) { + throw IllegalStateException("This output stream is finalised and cannot be modified.") + } + else { + buf.add(b.toByte()) + count += 1 + } + } + + /** Unlike Java's, this does NOT create a copy of the internal buffer; this just returns its internal. + * This method also "finalises" the buffer inside of the output stream, making further modification impossible. + * + * The output stream must be flushed and closed, warning you of closing the stream is not possible. + */ + @Synchronized + fun toByteArray64(): ByteArray64 { + close() + buf.size = count + return buf + } + + override fun close() { + finalised = true + buf.finalise() + } +} + +open class ByteArray64Writer(val charset: Charset) : Writer() { + + /* writer must be able to handle nonstandard utf-8 surrogate representation, where + * each surrogate is encoded in single code point, resulting six utf-8 bytes instead of four. + */ + + private val acceptableCharsets = arrayOf(Charsets.UTF_8, Charset.forName("CP437")) + + init { + if (!acceptableCharsets.contains(charset)) + throw UnsupportedCharsetException(charset.name()) + } + + private val ba = ByteArray64() + private var closed = false + private var surrogateBuf = 0 + + init { + this.lock = ba + } + + private fun checkOpen() { + if (closed) throw ClosedChannelException() + } + + private fun Int.isSurroHigh() = this.ushr(10) == 0b110110 + private fun Int.isSurroLow() = this.ushr(10) == 0b110111 + private fun Int.toUcode() = 'u' + this.toString(16).toUpperCase().padStart(4,'0') + + /** + * @param c not a freakin' codepoint; just a Java's Char casted into Int + */ + override fun write(c: Int) { + checkOpen() + when (charset) { + Charsets.UTF_8 -> { + if (surrogateBuf == 0 && !c.isSurroHigh() && !c.isSurroLow()) + writeUtf8Codepoint(c) + else if (surrogateBuf == 0 && c.isSurroHigh()) + surrogateBuf = c + else if (surrogateBuf != 0 && c.isSurroLow()) + writeUtf8Codepoint(65536 + surrogateBuf.and(1023).shl(10) or c.and(1023)) + // invalid surrogate pair input + else + throw IllegalStateException("Surrogate high: ${surrogateBuf.toUcode()}, surrogate low: ${c.toUcode()}") + } + Charset.forName("CP437") -> { + ba.add(c.toByte()) + } + else -> throw UnsupportedCharsetException(charset.name()) + } + } + + fun writeUtf8Codepoint(codepoint: Int) { + when (codepoint) { + in 0..127 -> ba.add(codepoint.toByte()) + in 128..2047 -> { + ba.add((0xC0 or codepoint.ushr(6).and(31)).toByte()) + ba.add((0x80 or codepoint.and(63)).toByte()) + } + in 2048..65535 -> { + ba.add((0xE0 or codepoint.ushr(12).and(15)).toByte()) + ba.add((0x80 or codepoint.ushr(6).and(63)).toByte()) + ba.add((0x80 or codepoint.and(63)).toByte()) + } + in 65536..1114111 -> { + ba.add((0xF0 or codepoint.ushr(18).and(7)).toByte()) + ba.add((0x80 or codepoint.ushr(12).and(63)).toByte()) + ba.add((0x80 or codepoint.ushr(6).and(63)).toByte()) + ba.add((0x80 or codepoint.and(63)).toByte()) + } + else -> throw IllegalArgumentException("Not a unicode code point: U+${codepoint.toString(16).toUpperCase()}") + } + } + + override fun write(cbuf: CharArray) { + checkOpen() + write(String(cbuf)) + } + + override fun write(str: String) { + checkOpen() + str.toByteArray(charset).forEach { ba.add(it) } + } + + override fun write(cbuf: CharArray, off: Int, len: Int) { + write(cbuf.copyOfRange(off, off + len)) + } + + override fun write(str: String, off: Int, len: Int) { + write(str.substring(off, off + len)) + } + + override fun close() { closed = true } + override fun flush() {} + + fun toByteArray64() = if (closed) ba else throw IllegalAccessException("Writer not closed") +} + +open class ByteArray64Reader(val ba: ByteArray64, val charset: Charset) : Reader() { + + /* reader must be able to handle nonstandard utf-8 surrogate representation, where + * each surrogate is encoded in single code point, resulting six utf-8 bytes instead of four. + */ + + private val acceptableCharsets = arrayOf(Charsets.UTF_8, Charset.forName("CP437")) + + init { + if (!acceptableCharsets.contains(charset)) + throw UnsupportedCharsetException(charset.name()) + } + + private var readCursor = 0L + private val remaining + get() = ba.size - readCursor + + /** + * U+0000 .. U+007F 0xxxxxxx + * U+0080 .. U+07FF 110xxxxx 10xxxxxx + * U+0800 .. U+FFFF 1110xxxx 10xxxxxx 10xxxxxx + * U+10000 .. U+10FFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + */ + private fun utf8GetCharLen(head: Byte) = when (head.toInt() and 255) { + in 0b11110_000..0b11110_111 -> 4 + in 0b1110_0000..0b1110_1111 -> 3 + in 0b110_00000..0b110_11111 -> 2 + in 0b0_0000000..0b0_1111111 -> 1 + else -> throw IllegalArgumentException("Invalid UTF-8 Character head byte: ${head.toInt() and 255}") + } + + /** + * @param list of bytes that encodes one unicode character. Get required byte length using [utf8GetCharLen]. + * @return A codepoint of the character. + */ + private fun utf8decode(bytes0: List): Int { + val bytes = bytes0.map { it.toInt() and 255 } + var ret = when (bytes.size) { + 4 -> (bytes[0] and 7) shl 18 + 3 -> (bytes[0] and 15) shl 12 + 2 -> (bytes[0] and 31) shl 6 + 1 -> (bytes[0] and 127) + else -> throw IllegalArgumentException("Expected bytes size: 1..4, got ${bytes.size}") + } + bytes.subList(1, bytes.size).reversed().forEachIndexed { index, byte -> + ret = ret or (byte and 63).shl(6 * index) + } + return ret + } + + private var surrogateLeftover = ' ' + + override fun read(cbuf: CharArray, off: Int, len: Int): Int { + var readCount = 0 + + if (remaining <= 0L) return -1 + + when (charset) { + Charsets.UTF_8 -> { + while (readCount < len && remaining > 0) { + if (surrogateLeftover != ' ') { + cbuf[off + readCount] = surrogateLeftover + + readCount += 1 + surrogateLeftover = ' ' + } + else { + val bbuf = (0 until minOf(4L, remaining)).map { ba[readCursor + it] } + val charLen = utf8GetCharLen(bbuf[0]) + val codePoint = utf8decode(bbuf.subList(0, charLen)) + + if (codePoint < 65536) { + cbuf[off + readCount] = codePoint.toChar() + + readCount += 1 + readCursor += charLen + } + else { + /* + * U' = yyyyyyyyyyxxxxxxxxxx // U - 0x10000 + * W1 = 110110yyyyyyyyyy // 0xD800 + yyyyyyyyyy + * W2 = 110111xxxxxxxxxx // 0xDC00 + xxxxxxxxxx + */ + val codPoin = codePoint - 65536 + val surroLead = (0xD800 or codPoin.ushr(10)).toChar() + val surroTrail = (0xDC00 or codPoin.and(1023)).toChar() + + cbuf[off + readCount] = surroLead + + if (off + readCount + 1 < cbuf.size) { + cbuf[off + readCount + 1] = surroTrail + + readCount += 2 + readCursor += 4 + } + else { + readCount += 1 + readCursor += 4 + surrogateLeftover = surroTrail + } + } + } + } + } + Charset.forName("CP437") -> { + for (i in 0 until minOf(len.toLong(), remaining)) { + cbuf[(off + i).toInt()] = ba[readCursor].toChar() + readCursor += 1 + readCount += 1 + } + } + else -> throw UnsupportedCharsetException(charset.name()) + } + + return readCount + } + + override fun close() { readCursor = 0L } + override fun reset() { readCursor = 0L } + override fun markSupported() = false + +} \ No newline at end of file diff --git a/src/net/torvald/terrarum/tvd/DiskSkimmer.kt b/src/net/torvald/terrarum/tvd/DiskSkimmer.kt new file mode 100644 index 000000000..f21ec6b9f --- /dev/null +++ b/src/net/torvald/terrarum/tvd/DiskSkimmer.kt @@ -0,0 +1,426 @@ +package net.torvald.terrarum.tvda + +import java.io.* +import java.nio.charset.Charset +import java.util.* +import kotlin.experimental.and + +/** + * Skimming allows modifying the Virtual Disk without loading entire disk onto the memory. + * + * Skimmer will just scan through the raw bytes of the Virtual Disk to get the file requested with its Entry ID; + * modifying/removing files will edit the Virtual Disk in "dirty" way, where old entries are simply marked as deletion + * and leaves the actual contents untouched, then will simply append modified files at the end. + * + * To obtain "clean" version of the modified Virtual Disk, simply run [sync] function. + * + * Created by minjaesong on 2017-11-17. + */ +class DiskSkimmer(private val diskFile: File, val charset: Charset = Charset.defaultCharset()) { + + /* + +init: + +1. get the startingpoint of the entries (after the 8 byte ID space ofc) + +addfile/editfile: + +10. mark old parentdir as invalidated +11. mark old entryfile as invalidated +20. append new file +30. append modified parentdir +40. update startingpoint table + +removefile: + +10. mark old parentdir as invalidated +20. append modified parentdir +30. update startingpoint table + + */ + + /** + * EntryID to Offset. + * + * Offset is where the header begins, so first 4 bytes are exactly the same as the EntryID. + */ + private var entryToOffsetTable = HashMap() + + + /** temporary storage to store tree edges */ +// private var directoryStruct = ArrayList() + + /** root node of the directory tree */ +// private var directory = DirectoryNode(0, null, DiskEntry.DIRECTORY, "") + +// private data class DirectoryEdge(val nodeParent: EntryID, val node: EntryID, val type: Byte, val name: String) +// private data class DirectoryNode(var nodeThis: EntryID, val nodeParent: EntryID?, var type: Byte, var name: String) + + private val dirDelim = Regex("""[\\/]""") + private val DIR = "/" + + val fa = RandomAccessFile(diskFile, "rw") + + init { + val fis = FileInputStream(diskFile) + + println("[DiskSkimmer] loading the diskfile ${diskFile.canonicalPath}") + + var currentPosition = fis.skip(64) // skip disk header + + + fun skipRead(bytes: Long) { + currentPosition += fis.skip(bytes) + } + /** + * Reads a byte and adds up the position var + */ + fun readByte(): Byte { + currentPosition++ + val read = fis.read() + + if (read < 0) throw InternalError("Unexpectedly reached EOF") + return read.toByte() + } + + /** + * Reads specific bytes to the buffer and adds up the position var + */ + fun readBytes(buffer: ByteArray): Int { + val readStatus = fis.read(buffer) + currentPosition += readStatus + return readStatus + } + fun readUshortBig(): Int { + val buffer = ByteArray(2) + val readStatus = readBytes(buffer) + if (readStatus != 2) throw InternalError("Unexpected error -- EOF reached? (expected 4, got $readStatus)") + return buffer.toShortBig() + } + fun readIntBig(): Int { + val buffer = ByteArray(4) + val readStatus = readBytes(buffer) + if (readStatus != 4) throw InternalError("Unexpected error -- EOF reached? (expected 4, got $readStatus)") + return buffer.toIntBig() + } + fun readInt48(): Long { + val buffer = ByteArray(6) + val readStatus = readBytes(buffer) + if (readStatus != 6) throw InternalError("Unexpected error -- EOF reached? (expected 6, got $readStatus)") + return buffer.toInt48() + } + fun readLongBig(): Long { + val buffer = ByteArray(8) + val readStatus = readBytes(buffer) + if (readStatus != 8) throw InternalError("Unexpected error -- EOF reached? (expected 8, got $readStatus)") + return buffer.toLongBig() + } + + val currentLength = diskFile.length() + while (currentPosition < currentLength) { + + val entryID = readLongBig() // at this point, cursor is 4 bytes past to the entry head + + // fill up the offset table + val offset = currentPosition + + skipRead(8) + val typeFlag = readByte() + skipRead(3) + skipRead(16) // skip rest of the header + + val entrySize = when (typeFlag and 127) { + DiskEntry.NORMAL_FILE -> readInt48() + DiskEntry.DIRECTORY -> readIntBig().toLong() + else -> 0 + } + + skipRead(entrySize) // skips rest of the entry's actual contents + + if (typeFlag > 0) { + entryToOffsetTable[entryID] = offset + println("[DiskSkimmer] successfully read the entry $entryID at offset $offset (name: ${diskIDtoReadableFilename(entryID)})") + } + else { + println("[DiskSkimmer] discarding entry $entryID at offset $offset (name: ${diskIDtoReadableFilename(entryID)})") + } + } + + } + + + ////////////////////////////////////////////////// + // THESE ARE METHODS TO SUPPORT ON-LINE READING // + ////////////////////////////////////////////////// + + /** + * Using entryToOffsetTable, composes DiskEntry on the fly upon request. + * @return DiskEntry if the entry exists on the disk, `null` otherwise. + */ + fun requestFile(entryID: EntryID): DiskEntry? { + entryToOffsetTable[entryID].let { offset -> + if (offset == null) { + println("[DiskSkimmer.requestFile] entry $entryID does not exist on the table") + return null + } + else { + fa.seek(offset) + val parent = fa.read(8).toLongBig() + val fileFlag = fa.read(4)[0] + val creationTime = fa.read(6).toInt48() + val modifyTime = fa.read(6).toInt48() + val skip_crc = fa.read(4) + + // get entry size // TODO future me, is this kind of comment helpful or redundant? + val entrySize = when (fileFlag) { + DiskEntry.NORMAL_FILE -> { + fa.read(6).toInt48() + } + DiskEntry.DIRECTORY -> { + fa.read(4).toIntBig().toLong() + } + DiskEntry.SYMLINK -> 8L + else -> throw UnsupportedOperationException("Unsupported entry type: $fileFlag") // FIXME no support for compressed file + } + + + val entryContent = when (fileFlag) { + DiskEntry.NORMAL_FILE -> { + val byteArray = ByteArray64(entrySize) + // read one byte at a time + for (c in 0L until entrySize) { + byteArray[c] = fa.read().toByte() + } + + EntryFile(byteArray) + } + DiskEntry.DIRECTORY -> { + val dirContents = ArrayList() + // read 8 bytes at a time + val bytesBuffer8 = ByteArray(8) + for (c in 0L until entrySize) { + fa.read(bytesBuffer8) + dirContents.add(bytesBuffer8.toLongBig()) + } + + EntryDirectory(dirContents) + } + DiskEntry.SYMLINK -> { + val target = fa.read(8).toLongBig() + + EntrySymlink(target) + } + else -> throw UnsupportedOperationException("Unsupported entry type: $fileFlag") // FIXME no support for compressed file + } + + return DiskEntry(entryID, parent, creationTime, modifyTime, entryContent) + } + } + } + + /** + * Try to find a file with given path (which uses '/' as a separator). Is search is failed for whatever reason, + * `null` is returned. + * + * @param path A path to the file from the root, directory separated with '/' (and not '\') + * @return DiskEntry if the search was successful, `null` otherwise + */ + /*fun requestFile(path: String): DiskEntry? { + // fixme pretty much untested + + val path = path.split(dirDelim) + //println(path) + + // bunch-of-io-access approach (for reading) + var traversedDir = 0L // entry ID + var dirFile: DiskEntry? = null + path.forEachIndexed { index, dirName -> + println("[DiskSkimmer.requestFile] $index\t$dirName, traversedDir = $traversedDir") + + dirFile = requestFile(traversedDir) + if (dirFile == null) { + println("[DiskSkimmer.requestFile] requestFile($traversedDir) came up null") + return null + } // outright null + if (dirFile!!.contents !is EntryDirectory && index < path.lastIndex) { // unexpectedly encountered non-directory + return null // because other than the last path, everything should be directory (think about it!) + } + //if (index == path.lastIndex) return dirFile // reached the end of the search strings + + // still got more paths behind to traverse + var dirGotcha = false + // loop for current dir contents + (dirFile!!.contents as EntryDirectory).forEach { + if (!dirGotcha) { // alternative impl of 'break' as it's not allowed + // get name of the file + val childDirFile = requestFile(it)!! + if (childDirFile.filename.toCanonicalString(charset) == dirName) { + //println("[DiskSkimmer] found, $traversedDir -> $it") + dirGotcha = true + traversedDir = it + } + } + } + + if (!dirGotcha) return null // got null || directory empty || + } + + return requestFile(traversedDir) + }*/ + + fun invalidateEntry(id: EntryID) { + fa.seek(entryToOffsetTable[id]!! + 8) + val type = fa.read() + fa.seek(entryToOffsetTable[id]!! + 8) + fa.write(type or 128) + entryToOffsetTable.remove(id) + } + + /////////////////////////////////////////////////////// + // THESE ARE METHODS TO SUPPORT ON-LINE MODIFICATION // + /////////////////////////////////////////////////////// + + fun appendEntry(entry: DiskEntry) { + val parentDir = requestFile(entry.parentEntryID)!! + val id = entry.entryID + val parent = entry.parentEntryID + + // add the entry to its parent directory if there was none + val dirContent = (parentDir.contents as EntryDirectory) + if (!dirContent.contains(id)) dirContent.add(id) + + invalidateEntry(parent) + invalidateEntry(id) + + val appendAt = fa.length() + fa.seek(appendAt) + + // append new file + entryToOffsetTable[id] = appendAt + 8 + entry.serialize().forEach { fa.writeByte(it.toInt()) } + // append modified directory + entryToOffsetTable[parent] = fa.filePointer + 8 + parentDir.serialize().forEach { fa.writeByte(it.toInt()) } + } + + fun deleteEntry(id: EntryID) { + val entry = requestFile(id)!! + val parentDir = requestFile(entry.parentEntryID)!! + val parent = entry.parentEntryID + + invalidateEntry(parent) + + // remove the entry + val dirContent = (parentDir.contents as EntryDirectory) + dirContent.remove(id) + + val appendAt = fa.length() + fa.seek(appendAt) + + // append modified directory + entryToOffsetTable[id] = appendAt + 8 + parentDir.serialize().forEach { fa.writeByte(it.toInt()) } + } + + fun appendEntries(entries: List) = entries.forEach { appendEntry(it) } + fun deleteEntries(entries: List) = entries.forEach { deleteEntry(it) } + + /** + * Writes new clean file + */ + fun sync(): VirtualDisk { + // rebuild VirtualDisk out of this and use it to write out + return VDUtil.readDiskArchive(diskFile, charset = charset) + } + + + fun dispose() { + fa.close() + } + + + companion object { + fun InputStream.read(size: Int): ByteArray { + val ba = ByteArray(size) + this.read(ba) + return ba + } + fun RandomAccessFile.read(size: Int): ByteArray { + val ba = ByteArray(size) + this.read(ba) + return ba + } + } + + /** + * total size of the entry block. This size includes that of the header + */ + private fun getEntryBlockSize(id: EntryID): Long? { + val offset = entryToOffsetTable[id] ?: return null + + val HEADER_SIZE = DiskEntry.HEADER_SIZE + + println("[DiskSkimmer.getEntryBlockSize] offset for entry $id = $offset") + + val fis = FileInputStream(diskFile) + fis.skip(offset + 8) + val type = fis.read().toByte() + fis.skip(272) // skip name, timestamp and CRC + + + val ret: Long + when (type) { + DiskEntry.NORMAL_FILE -> { + ret = fis.read(6).toInt48() + HEADER_SIZE + 6 + } + DiskEntry.DIRECTORY -> { + ret = fis.read(2).toShortBig() * 4 + HEADER_SIZE + 2 + } + DiskEntry.SYMLINK -> { ret = 4 } + else -> throw UnsupportedOperationException("Unknown type $type for entry $id") + } + + fis.close() + + return ret + } + + private fun byteByByteCopy(size: Long, `in`: InputStream, out: OutputStream) { + for (i in 0L until size) { + out.write(`in`.read()) + } + } + + private fun ByteArray.toShortBig(): Int { + return this[0].toUint().shl(8) or + this[1].toUint() + } + + private fun ByteArray.toIntBig(): Int { + return this[0].toUint().shl(24) or + this[1].toUint().shl(16) or + this[2].toUint().shl(8) or + this[3].toUint() + } + + private fun ByteArray.toInt48(): Long { + return this[0].toUlong().shl(40) or + this[1].toUlong().shl(32) or + this[2].toUlong().shl(24) or + this[3].toUlong().shl(16) or + this[4].toUlong().shl(8) or + this[5].toUlong() + } + + private fun ByteArray.toLongBig(): Long { + return this[0].toUlong().shl(56) or + this[1].toUlong().shl(48) or + this[2].toUlong().shl(40) or + this[3].toUlong().shl(32) or + this[4].toUlong().shl(24) or + this[5].toUlong().shl(16) or + this[6].toUlong().shl(8) or + this[7].toUlong() + } +} \ No newline at end of file diff --git a/src/net/torvald/terrarum/tvd/DiskSkimmerTest.kt b/src/net/torvald/terrarum/tvd/DiskSkimmerTest.kt new file mode 100644 index 000000000..b0fd135b5 --- /dev/null +++ b/src/net/torvald/terrarum/tvd/DiskSkimmerTest.kt @@ -0,0 +1,50 @@ +package net.torvald.terrarum.tvda + +import java.io.File +import java.nio.file.Files +import java.nio.file.StandardCopyOption + +object DiskSkimmerTest { + + val fullBattery = listOf( + { invoke00() } + ) + + operator fun invoke() { + fullBattery.forEach { it.invoke() } + } + + /** + * Testing of DiskSkimmer + */ + fun invoke00() { + val _infile = File("./test-assets/tevd-test-suite-00.tevd") + val outfile = File("./test-assets/tevd-test-suite-00_results.tevd") + + Files.copy(_infile.toPath(), outfile.toPath(), StandardCopyOption.REPLACE_EXISTING) + +/* +Copied from instruction.txt + +1. Create a file named "World!.txt" in the root directory. +2. Append "This is not SimCity 3k" on the file ./01_preamble/append-after-me +3. Delete a file ./01_preamble/deleteme +4. Modify this very file, delete everything and simply replace with "Mischief Managed." +5. Read the file ./instruction.txt and print its contents. + +Expected console output: + +Mischief Managed. + */ + val skimmer = DiskSkimmer(outfile) + + + println("=============================") + + + } +} + +fun main(args: Array) { + DiskSkimmerTest() +} \ No newline at end of file diff --git a/src/net/torvald/terrarum/tvd/VDUtil.kt b/src/net/torvald/terrarum/tvd/VDUtil.kt new file mode 100644 index 000000000..145fdc2f3 --- /dev/null +++ b/src/net/torvald/terrarum/tvd/VDUtil.kt @@ -0,0 +1,780 @@ +package net.torvald.terrarum.tvda + +import java.io.* +import java.nio.charset.Charset +import java.util.* +import java.util.logging.Level +import java.util.zip.GZIPInputStream +import java.util.zip.GZIPOutputStream +import javax.naming.OperationNotSupportedException +import kotlin.experimental.and + +/** + * Temporarily disabling on-disk compression; it somehow does not work, compress the files by yourself! + * + * Created by minjaesong on 2017-04-01. + */ +object VDUtil { + + fun File.writeBytes64(array: net.torvald.terrarum.tvda.ByteArray64) { + array.writeToFile(this) + } + + fun File.readBytes64(): net.torvald.terrarum.tvda.ByteArray64 { + val inbytes = net.torvald.terrarum.tvda.ByteArray64(this.length()) + val inputStream = BufferedInputStream(FileInputStream(this)) + var readInt = inputStream.read() + var readInCounter = 0L + while (readInt != -1) { + inbytes[readInCounter] = readInt.toByte() + readInCounter += 1 + + readInt = inputStream.read() + } + inputStream.close() + + return inbytes + } + + fun dumpToRealMachine(disk: VirtualDisk, outfile: File) { + if (!outfile.exists()) outfile.createNewFile() + outfile.writeBytes64(disk.serialize().array) + } + + private const val DEBUG_PRINT_READ = false + + /** + * Reads serialised binary and returns corresponding VirtualDisk instance. + * + * @param crcWarnLevel Level.OFF -- no warning, Level.WARNING -- print out warning, Level.SEVERE -- throw error + */ + fun readDiskArchive(infile: File, crcWarnLevel: Level = Level.SEVERE, warningFunc: ((String) -> Unit)? = null, charset: Charset): VirtualDisk { + val inbytes = infile.readBytes64() + + + + if (magicMismatch(VirtualDisk.MAGIC, inbytes.sliceArray64(0L..3L).toByteArray())) + throw RuntimeException("Invalid Virtual Disk file!") + + val diskSize = inbytes.sliceArray64(4L..9L).toInt48Big() + val diskName = inbytes.sliceArray64(10L..10L + 31) + val diskCRC = inbytes.sliceArray64(10L + 32..10L + 32 + 3).toIntBig() // to check with completed vdisk + val diskSpecVersion = inbytes[10L + 32 + 4] + val footers = inbytes.sliceArray64(10L+32+6..10L+32+21) + + if (diskSpecVersion != specversion) + throw RuntimeException("Unsupported disk format version: current internal version is $specversion; the file's version is $diskSpecVersion") + + val vdisk = VirtualDisk(diskSize, diskName.toByteArray()) + + vdisk.__internalSetFooter__(footers) + + //println("[VDUtil] currentUnixtime = $currentUnixtime") + + var entryOffset = VirtualDisk.HEADER_SIZE + // not footer, entries + while (entryOffset < inbytes.size) { + //println("[VDUtil] entryOffset = $entryOffset") + // read and prepare all the shits + val entryID = inbytes.sliceArray64(entryOffset..entryOffset + 7).toLongBig() + val entryParentID = inbytes.sliceArray64(entryOffset + 8..entryOffset + 15).toLongBig() + val entryTypeFlag = inbytes[entryOffset + 16] + val entryCreationTime = inbytes.sliceArray64(entryOffset + 20..entryOffset + 25).toInt48Big() + val entryModifyTime = inbytes.sliceArray64(entryOffset + 26..entryOffset + 31).toInt48Big() + val entryCRC = inbytes.sliceArray64(entryOffset + 32..entryOffset + 35).toIntBig() // to check with completed entry + + val entryData = when (entryTypeFlag and 127) { + DiskEntry.NORMAL_FILE -> { + val filesize = inbytes.sliceArray64(entryOffset + DiskEntry.HEADER_SIZE..entryOffset + DiskEntry.HEADER_SIZE + 5).toInt48Big() + //println("[VDUtil] --> is file; filesize = $filesize") + inbytes.sliceArray64(entryOffset + DiskEntry.HEADER_SIZE + 6..entryOffset + DiskEntry.HEADER_SIZE + 5 + filesize) + } + DiskEntry.DIRECTORY -> { + val entryCount = inbytes.sliceArray64(entryOffset + DiskEntry.HEADER_SIZE..entryOffset + DiskEntry.HEADER_SIZE + 3).toIntBig() + //println("[VDUtil] --> is directory; entryCount = $entryCount") + inbytes.sliceArray64(entryOffset + DiskEntry.HEADER_SIZE + 4..entryOffset + DiskEntry.HEADER_SIZE + 3 + entryCount * 8) + } + DiskEntry.SYMLINK -> { + inbytes.sliceArray64(entryOffset + DiskEntry.HEADER_SIZE..entryOffset + DiskEntry.HEADER_SIZE + 7) + } + else -> throw RuntimeException("Unknown entry with type $entryTypeFlag at entryOffset $entryOffset") + } + + if (DEBUG_PRINT_READ) { + println("== Entry deserialise debugprint for entry ID $entryID (child of $entryParentID)") + println("Entry type flag: ${entryTypeFlag and 127}${if (entryTypeFlag < 0) "*" else ""}") + println("Entry raw contents bytes: (len: ${entryData.size})") + entryData.forEachIndexed { i, it -> + if (i > 0 && i % 8 == 0L) print(" ") + else if (i > 0 && i % 4 == 0L) print("_") + print(it.toInt().toHex().substring(6)) + }; println() + } + + + // update entryOffset so that we can fetch next entry in the binary + entryOffset += DiskEntry.HEADER_SIZE + entryData.size + when (entryTypeFlag and 127) { + DiskEntry.NORMAL_FILE -> 6 // PLEASE DO REFER TO Spec.md + DiskEntry.DIRECTORY -> 4 // PLEASE DO REFER TO Spec.md + DiskEntry.SYMLINK -> 0 // PLEASE DO REFER TO Spec.md + else -> throw RuntimeException("Unknown entry with type $entryTypeFlag") + } + + + // check for the discard bit + if (entryTypeFlag in 1..127) { + + // create entry + val diskEntry = DiskEntry( + entryID = entryID, + parentEntryID = entryParentID, + creationDate = entryCreationTime, + modificationDate = entryModifyTime, + contents = if (entryTypeFlag == DiskEntry.NORMAL_FILE) { + EntryFile(entryData) + } else if (entryTypeFlag == DiskEntry.DIRECTORY) { + + val entryList = ArrayList() + + (0 until entryData.size / 8).forEach { cnt -> + entryList.add(entryData.sliceArray64(8 * cnt until 8 * (cnt+1)).toLongBig()) + } + + entryList.sort() + + EntryDirectory(entryList) + } else if (entryTypeFlag == DiskEntry.SYMLINK) { + EntrySymlink(entryData.toLongBig()) + } else + throw RuntimeException("Unknown entry with type $entryTypeFlag") + ) + + // check CRC of entry + if (crcWarnLevel == Level.SEVERE || crcWarnLevel == Level.WARNING) { + + // test print + if (DEBUG_PRINT_READ) { + val testbytez = diskEntry.contents.serialize() + val testbytes = testbytez.array + (diskEntry.contents as? EntryDirectory)?.forEach { + println("entry: ${it.toHex()}") + } + println("bytes to calculate crc against:") + testbytes.forEachIndexed { i, it -> + if (i % 4 == 0L) print(" ") + print(it.toInt().toHex().substring(6)) + } + println("\nCRC: " + testbytez.getCRC32().toHex()) + } + // end of test print + + val calculatedCRC = diskEntry.contents.serialize().getCRC32() + + val crcMsg = + "CRC failed: stored value is ${entryCRC.toHex()}, but calculated value is ${calculatedCRC.toHex()}\n" + + "at file \"${diskIDtoReadableFilename(diskEntry.entryID)}\" (entry ID ${diskEntry.entryID})" + + if (calculatedCRC != entryCRC) { + + println("CRC failed; entry info:\n$diskEntry") + + if (crcWarnLevel == Level.SEVERE) + throw IOException(crcMsg) + else if (warningFunc != null) + warningFunc(crcMsg) + } + } + + // add entry to disk + vdisk.entries[entryID] = diskEntry + } + } + + // check CRC of disk + if (crcWarnLevel == Level.SEVERE || crcWarnLevel == Level.WARNING) { + val calculatedCRC = vdisk.hashCode() + + val crcMsg = "Disk CRC failed: expected ${diskCRC.toHex()}, got ${calculatedCRC.toHex()}" + + if (calculatedCRC != diskCRC) { + if (crcWarnLevel == Level.SEVERE) + throw IOException(crcMsg) + else if (warningFunc != null) + warningFunc(crcMsg) + } + } + + return vdisk + } + + + fun isFile(disk: VirtualDisk, entryID: EntryID) = disk.entries[entryID]?.contents is EntryFile + fun isDirectory(disk: VirtualDisk, entryID: EntryID) = disk.entries[entryID]?.contents is EntryDirectory + fun isSymlink(disk: VirtualDisk, entryID: EntryID) = disk.entries[entryID]?.contents is EntrySymlink + + /** + * Get list of entries of directory. + */ + fun getDirectoryEntries(disk: VirtualDisk, dirToSearch: DiskEntry): Array { + if (dirToSearch.contents !is EntryDirectory) + throw IllegalArgumentException("The entry is not directory") + + val entriesList = ArrayList() + dirToSearch.contents.forEach { + val entry = disk.entries[it] + if (entry != null) entriesList.add(entry) + } + + return entriesList.toTypedArray() + } + /** + * Get list of entries of directory. + */ + fun getDirectoryEntries(disk: VirtualDisk, entryID: EntryID): Array { + val entry = disk.entries[entryID] + if (entry == null) { + throw IOException("Entry does not exist") + } + else { + return getDirectoryEntries(disk, entry) + } + } + + /** + * SYNOPSIS disk.getFile("bin/msh.lua")!!.file.getAsNormalFile(disk) + * + * Use VirtualDisk.getAsNormalFile(path) + */ + private fun DiskEntry.getAsNormalFile(disk: VirtualDisk): EntryFile = + this.contents as? EntryFile ?: + if (this.contents is EntryDirectory) + throw RuntimeException("this is directory") + else if (this.contents is EntrySymlink) + disk.entries[this.contents.target]!!.getAsNormalFile(disk) + else + throw RuntimeException("Unknown entry type") + /** + * SYNOPSIS disk.getFile("bin/msh.lua")!!.first.getAsNormalFile(disk) + * + * Use VirtualDisk.getAsNormalFile(path) + */ + private fun DiskEntry.getAsDirectory(disk: VirtualDisk): EntryDirectory = + this.contents as? EntryDirectory ?: + if (this.contents is EntrySymlink) + disk.entries[this.contents.target]!!.getAsDirectory(disk) + else if (this.contents is EntryFile) + throw RuntimeException("this is not directory") + else + throw RuntimeException("Unknown entry type") + + /** + * Fetch the file and returns a instance of normal file. + */ + fun getAsNormalFile(disk: VirtualDisk, entryIndex: EntryID) = + disk.entries[entryIndex]!!.getAsNormalFile(disk) + /** + * Fetch the file and returns a instance of directory. + */ + fun getAsDirectory(disk: VirtualDisk, entryIndex: EntryID) = + disk.entries[entryIndex]!!.getAsDirectory(disk) + + /** + * Deletes file on the disk safely. + */ + fun deleteFile(disk: VirtualDisk, targetID: EntryID) { + disk.checkReadOnly() + + val file = disk.entries[targetID] + + if (file == null) { + throw FileNotFoundException("No such file to delete") + } + + val parentID = file.parentEntryID + val parentDir = getAsDirectory(disk, parentID) + + fun rollback() { + if (!disk.entries.contains(targetID)) { + disk.entries[targetID] = file + } + if (!parentDir.contains(targetID)) { + parentDir.add(targetID) + } + } + + // check if directory "parentID" has "targetID" in the first place + if (!directoryContains(disk, parentID, targetID)) { + throw FileNotFoundException("No such file to delete") + } + else if (targetID == 0L) { + throw IOException("Cannot delete root file system") + } + else { + try { + // delete file record + disk.entries.remove(targetID) + // unlist file from parent directly + parentDir.remove(targetID) + } + catch (e: Exception) { + rollback() + throw InternalError("Unknown error *sigh* It's annoying, I know.") + } + } + } + /** + * Changes the name of the entry. + */ + fun renameFile(disk: VirtualDisk, fileID: EntryID, newID: EntryID, charset: Charset) { + val file = disk.entries[fileID] + + if (file != null) { + file.entryID = newID + } + else { + throw FileNotFoundException() + } + } + /** + * Add file to the specified directory. + * The file will get new EntryID and its ParentID will be overwritten. + */ + fun addFile(disk: VirtualDisk, file: DiskEntry) { + disk.entries[file.entryID] = file + file.parentEntryID = 0 + val dir = VDUtil.getAsDirectory(disk, 0) + if (!dir.contains(file.entryID)) dir.add(file.entryID) + } + + fun randomBase62(length: Int): String { + val glyphs = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890" + val sb = StringBuilder() + + kotlin.repeat(length) { + sb.append(glyphs[(Math.random() * glyphs.length).toInt()]) + } + + return sb.toString() + } + + /** + * Add fully qualified DiskEntry to the disk, using file's own and its parent entryID. + * + * It's your job to ensure no ID collision. + */ + fun registerFile(disk: VirtualDisk, file: DiskEntry) { + disk.checkReadOnly() + disk.checkCapacity(file.serialisedSize) + + VDUtil.getAsDirectory(disk, file.parentEntryID).add(file.entryID) + disk.entries[file.entryID] = file + } + + /** + * Add file to the specified directory. ParentID of the file will be overwritten. + */ + fun addFile(disk: VirtualDisk, directoryID: EntryID, file: DiskEntry) {//}, compressTheFile: Boolean = false) { + disk.checkReadOnly() + disk.checkCapacity(file.serialisedSize) + + try { + // generate new ID for the file + file.entryID = disk.generateUniqueID() + // add record to the directory + getAsDirectory(disk, directoryID).add(file.entryID) + + // Gzip fat boy if marked as + /*if (compressTheFile && file.contents is EntryFile) { + val bo = ByteArray64GrowableOutputStream() + val zo = GZIPOutputStream(bo) + + // zip + file.contents.bytes.forEach { + zo.write(it.toInt()) + } + zo.flush(); zo.close() + + val newContent = EntryFileCompressed(file.contents.bytes.size, bo.toByteArray64()) + val newEntry = DiskEntry( + file.entryID, file.parentEntryID, file.filename, file.creationDate, file.modificationDate, + newContent + ) + + disk.entries[file.entryID] = newEntry + } + // just the add the boy to the house + else*/ + disk.entries[file.entryID] = file + + // make this boy recognise his new parent + file.parentEntryID = directoryID + } + catch (e: KotlinNullPointerException) { + throw FileNotFoundException("No such directory") + } + } + + /** + * Imports external file and returns corresponding DiskEntry. + */ + fun importFile(file: File, newID: EntryID, charset: Charset): DiskEntry { + if (file.isDirectory) { + throw IOException("The file is a directory") + } + + return DiskEntry( + entryID = newID, + parentEntryID = 0, // placeholder + creationDate = currentUnixtime, + modificationDate = currentUnixtime, + contents = EntryFile(file.readBytes64()) + ) + } + + /** + * Export file on the virtual disk into real disk. + */ + fun exportFile(entryFile: EntryFile, outfile: File) { + outfile.createNewFile() + + /*if (entryFile is EntryFileCompressed) { + entryFile.bytes.forEachBanks { + val fos = FileOutputStream(outfile) + val inflater = InflaterOutputStream(fos) + + inflater.write(it) + inflater.flush() + inflater.close() + } + } + else*/ + outfile.writeBytes64(entryFile.bytes) + } + + + /** + * Creates new disk with given name and capacity + */ + fun createNewDisk(diskSize: Long, diskName: String, charset: Charset): VirtualDisk { + val newdisk = VirtualDisk(diskSize, diskName.toEntryName(VirtualDisk.NAME_LENGTH, charset)) + val rootDir = DiskEntry( + entryID = 0, + parentEntryID = 0, + creationDate = currentUnixtime, + modificationDate = currentUnixtime, + contents = EntryDirectory() + ) + + newdisk.entries[0] = rootDir + + return newdisk + } + + + /** + * Throws an exception if the disk is read-only + */ + fun VirtualDisk.checkReadOnly() { + if (this.isReadOnly) + throw IOException("Disk is read-only") + } + /** + * Throws an exception if specified size cannot fit into the disk + */ + fun VirtualDisk.checkCapacity(newSize: Long) { + if (this.usedBytes + newSize > this.capacity) + throw IOException("Not enough space on the disk") + } + fun ByteArray64.toIntBig(): Int { + if (this.size != 4L) + throw OperationNotSupportedException("ByteArray is not Int") + + var i = 0 + var c = 0 + this.forEach { byte -> i = i or byte.toUint().shl(24 - c * 8); c += 1 } + return i + } + fun ByteArray64.toLongBig(): Long { + if (this.size != 8L) + throw OperationNotSupportedException("ByteArray is not Long") + + var i = 0L + var c = 0 + this.forEach { byte -> i = i or byte.toUlong().shl(56 - c * 8); c += 1 } + return i + } + fun ByteArray64.toInt48Big(): Long { + if (this.size != 6L) + throw OperationNotSupportedException("ByteArray is not Long") + + var i = 0L + var c = 0 + this.forEach { byte -> i = i or byte.toUlong().shl(40 - c * 8); c += 1 } + return i + } + fun ByteArray64.toShortBig(): Short { + if (this.size != 2L) + throw OperationNotSupportedException("ByteArray is not Short") + + return (this[0].toUint().shl(256) + this[1].toUint()).toShort() + } + fun String.sanitisePath(): String { + val invalidChars = Regex("""[<>:"|?*\u0000-\u001F]""") + if (this.contains(invalidChars)) + throw IOException("path contains invalid characters") + + val path1 = this.replace('\\', '/') + return path1 + } + + fun resolveIfSymlink(disk: VirtualDisk, indexNumber: EntryID, recurse: Boolean = false): DiskEntry { + var entry: DiskEntry? = disk.entries[indexNumber] + if (entry == null) throw IOException("File does not exist") + if (entry.contents !is EntrySymlink) return entry + if (recurse) { + while (entry!!.contents is EntrySymlink) { + entry = disk.entries[(entry.contents as EntrySymlink).target] + if (entry == null) break + } + } + else { + entry = disk.entries[(entry.contents as EntrySymlink).target] + } + if (entry == null) throw IOException("Pointing file does not exist") + return entry + } + + val currentUnixtime: Long + get() = System.currentTimeMillis() / 1000 + + fun directoryContains(disk: VirtualDisk, dirID: EntryID, targetID: EntryID): Boolean { + val dir = resolveIfSymlink(disk, dirID) + + if (dir.contents !is EntryDirectory) { + throw FileNotFoundException("Not a directory") + } + else { + return dir.contents.contains(targetID) + } + } + + /** + * Searches for disconnected nodes using its parent pointer. + * If the parent node is invalid, the node is considered orphan, and will be added + * to the list this function returns. + * + * @return List of orphan entries + */ + fun gcSearchOrphan(disk: VirtualDisk): List { + return disk.entries.filter { disk.entries[it.value.parentEntryID] == null }.keys.toList() + } + + /** + * Searches for null-pointing entries (phantoms) within every directory. + * + * @return List of search results, which is Pair(directory that contains null pointer, null pointer) + */ + fun gcSearchPhantomBaby(disk: VirtualDisk): List> { + // Pair + val phantoms = ArrayList>() + disk.entries.filter { it.value.contents is EntryDirectory }.values.forEach { directory -> + (directory.contents as EntryDirectory).forEach { dirEntryID -> + if (disk.entries[dirEntryID] == null) { + phantoms.add(Pair(directory.entryID, dirEntryID)) + } + } + } + return phantoms + } + + fun gcDumpOrphans(disk: VirtualDisk) { + try { + gcSearchOrphan(disk).forEach { + disk.entries.remove(it) + } + } + catch (e: Exception) { + e.printStackTrace() + throw InternalError("Aw, snap!") + } + } + + fun gcDumpAll(disk: VirtualDisk) { + try { + gcSearchPhantomBaby(disk).forEach { + getAsDirectory(disk, it.first).remove(it.second) + } + gcSearchOrphan(disk).forEach { + disk.entries.remove(it) + } + } + catch (e: Exception) { + e.printStackTrace() + throw InternalError("Aw, snap!") + } + } + + fun compress(ba: ByteArray64) = compress(ba.iterator()) + fun compress(byteIterator: Iterator): ByteArray64 { + val bo = ByteArray64GrowableOutputStream() + val zo = GZIPOutputStream(bo) + + // zip + byteIterator.forEach { + zo.write(it.toInt()) + } + zo.flush(); zo.close() + return bo.toByteArray64() + } + + fun decompress(bytes: ByteArray64): ByteArray64 { + val unzipdBytes = ByteArray64() + val zi = GZIPInputStream(ByteArray64InputStream(bytes)) + while (true) { + val byte = zi.read() + if (byte == -1) break + unzipdBytes.add(byte.toByte()) + } + zi.close() + return unzipdBytes + } +} + +fun Byte.toUint() = java.lang.Byte.toUnsignedInt(this) +fun Byte.toUlong() = java.lang.Byte.toUnsignedLong(this) +fun magicMismatch(magic: ByteArray, array: ByteArray): Boolean { + return !Arrays.equals(array, magic) +} +fun String.toEntryName(length: Int, charset: Charset): ByteArray { + val buffer = AppendableByteBuffer(length.toLong()) + val stringByteArray = this.toByteArray(charset) + buffer.put(stringByteArray.sliceArray(0..minOf(length, stringByteArray.size) - 1)) + return buffer.array.toByteArray() +} +fun ByteArray.toCanonicalString(charset: Charset): String { + var lastIndexOfRealStr = 0 + for (i in this.lastIndex downTo 0) { + if (this[i] != 0.toByte()) { + lastIndexOfRealStr = i + break + } + } + return String(this.sliceArray(0..lastIndexOfRealStr), charset) +} + +fun ByteArray.toByteArray64(): ByteArray64 { + val array = ByteArray64(this.size.toLong()) + this.forEachIndexed { index, byte -> + array[index.toLong()] = byte + } + return array +} + +/** + * Writes String to the file + * + * Note: this FileWriter cannot write more than 2 GiB + * + * @param fileEntry must be File, resolve symlink beforehand + * @param mode "w" or "a" + */ +class VDFileWriter(private val fileEntry: DiskEntry, private val append: Boolean, val charset: Charset) : Writer() { + + private @Volatile var newFileBuffer = ArrayList() + + private @Volatile var closed = false + + init { + if (fileEntry.contents !is EntryFile) { + throw FileNotFoundException("Not a file") + } + } + + override fun write(cbuf: CharArray, off: Int, len: Int) { + if (!closed) { + val newByteArray = String(cbuf).toByteArray(charset).toByteArray64() + newByteArray.forEach { newFileBuffer.add(it) } + } + else { + throw IOException() + } + } + + override fun flush() { + if (!closed) { + val newByteArray = newFileBuffer.toByteArray() + + if (!append) { + (fileEntry.contents as EntryFile).bytes = newByteArray.toByteArray64() + } + else { + val oldByteArray = (fileEntry.contents as EntryFile).bytes.toByteArray().copyOf() + val newFileBuffer = ByteArray(oldByteArray.size + newByteArray.size) + + System.arraycopy(oldByteArray, 0, newFileBuffer, 0, oldByteArray.size) + System.arraycopy(newByteArray, 0, newFileBuffer, oldByteArray.size, newByteArray.size) + + fileEntry.contents.bytes = newByteArray.toByteArray64() + } + + newFileBuffer = ArrayList() + + fileEntry.modificationDate = VDUtil.currentUnixtime + } + else { + throw IOException() + } + } + + override fun close() { + flush() + closed = true + } +} + +class VDFileOutputStream(private val fileEntry: DiskEntry, private val append: Boolean, val charset: Charset) : OutputStream() { + + private @Volatile var newFileBuffer = ArrayList() + + private @Volatile var closed = false + + override fun write(b: Int) { + if (!closed) { + newFileBuffer.add(b.toByte()) + } + else { + throw IOException() + } + } + + override fun flush() { + if (!closed) { + val newByteArray = newFileBuffer.toByteArray() + + if (!append) { + (fileEntry.contents as EntryFile).bytes = newByteArray.toByteArray64() + } + else { + val oldByteArray = (fileEntry.contents as EntryFile).bytes.toByteArray().copyOf() + val newFileBuffer = ByteArray(oldByteArray.size + newByteArray.size) + + System.arraycopy(oldByteArray, 0, newFileBuffer, 0, oldByteArray.size) + System.arraycopy(newByteArray, 0, newFileBuffer, oldByteArray.size, newByteArray.size) + + fileEntry.contents.bytes = newByteArray.toByteArray64() + } + + newFileBuffer = ArrayList() + + fileEntry.modificationDate = VDUtil.currentUnixtime + } + else { + throw IOException() + } + } + + override fun close() { + flush() + closed = true + } +} \ No newline at end of file diff --git a/src/net/torvald/terrarum/tvd/VirtualDisk.kt b/src/net/torvald/terrarum/tvd/VirtualDisk.kt new file mode 100644 index 000000000..aa3610557 --- /dev/null +++ b/src/net/torvald/terrarum/tvd/VirtualDisk.kt @@ -0,0 +1,306 @@ +package net.torvald.terrarum.tvda + +import java.io.IOException +import java.nio.charset.Charset +import java.util.* +import java.util.zip.CRC32 +import kotlin.experimental.and +import kotlin.experimental.or + +/** + * Created by minjaesong on 2017-03-31. + */ + +typealias EntryID = Long + +val specversion = 254.toByte() + +class VirtualDisk( + /** capacity of 0 makes the disk read-only */ + var capacity: Long, + var diskName: ByteArray = ByteArray(NAME_LENGTH) +) { + var extraInfoBytes = ByteArray(16) + val entries = HashMap() + var isReadOnly: Boolean + set(value) { extraInfoBytes[0] = (extraInfoBytes[0] and 0xFE.toByte()) or value.toBit() } + get() = capacity == 0L || (extraInfoBytes.size > 0 && extraInfoBytes[0].and(1) == 1.toByte()) + fun getDiskNameString(charset: Charset) = String(diskName, charset) + val root: DiskEntry + get() = entries[0]!! + + private fun Boolean.toBit() = if (this) 1.toByte() else 0.toByte() + + internal fun __internalSetFooter__(footer: ByteArray64) { + extraInfoBytes = footer.toByteArray() + } + + private fun serializeEntriesOnly(): ByteArray64 { + val buffer = ByteArray64() + entries.forEach { + val serialised = it.value.serialize() + serialised.forEach { buffer.add(it) } + } + + return buffer + } + + fun serialize(): AppendableByteBuffer { + val entriesBuffer = serializeEntriesOnly() + val buffer = AppendableByteBuffer(HEADER_SIZE + entriesBuffer.size) + val crc = hashCode().toBigEndian() + + buffer.put(MAGIC) + + buffer.put(capacity.toInt48()) + buffer.put(diskName.forceSize(NAME_LENGTH)) + buffer.put(crc) + buffer.put(specversion) + buffer.put(0xFE.toByte()) + buffer.put(extraInfoBytes) + + buffer.put(entriesBuffer) + + return buffer + } + + override fun hashCode(): Int { + val crcList = IntArray(entries.size) + var crcListAppendCursor = 0 + entries.forEach { _, u -> + crcList[crcListAppendCursor] = u.hashCode() + crcListAppendCursor++ + } + crcList.sort() + val crc = CRC32() + crcList.forEach { crc.update(it) } + + return crc.value.toInt() + } + + /** Expected size of the virtual disk */ + val usedBytes: Long + get() = entries.map { it.value.serialisedSize }.sum() + HEADER_SIZE + + fun generateUniqueID(): Long { + var id: Long + do { + id = Random().nextLong() + } while (null != entries[id]) + return id + } + + override fun equals(other: Any?) = if (other == null) false else this.hashCode() == other.hashCode() + override fun toString() = "VirtualDisk(name: ${getDiskNameString(Charsets.UTF_8)}, capacity: $capacity bytes, crc: ${hashCode().toHex()})" + + companion object { + val HEADER_SIZE = 64L // according to the spec + val NAME_LENGTH = 32 + + val MAGIC = "TEVd".toByteArray() + } +} + +fun diskIDtoReadableFilename(id: EntryID): String = when (id) { + 0L -> "root" + -1L -> "savegameinfo.json" + -2L -> "thumbnail.tga.gz" + -16L -> "blockcodex.json.gz" + -17L -> "itemcodex.json.gz" + -18L -> "wirecodex.json.gz" + -19L -> "materialcodex.json.gz" + -20L -> "factioncodex.json.gz" + -1024L -> "apocryphas.json.gz" + in 1..65535 -> "worldinfo-$id.json" + in 1048576..2147483647 -> "actor-$id.json" + in 0x0000_0001_0000_0000L..0x0000_FFFF_FFFF_FFFFL -> + "World${id.ushr(32)}-L${id.and(0xFF00_0000).ushr(24)}-C${id.and(0xFFFFFF)}.gz" + else -> "file-$id" +} + +class DiskEntry( + // header + var entryID: EntryID, + var parentEntryID: EntryID, + var creationDate: Long, + var modificationDate: Long, + + // content + val contents: DiskEntryContent +) { + val serialisedSize: Long + get() = contents.getSizeEntry() + HEADER_SIZE + + companion object { + val HEADER_SIZE = 36L // according to the spec + + val NORMAL_FILE = 1.toByte() + val DIRECTORY = 2.toByte() + val SYMLINK = 3.toByte() + + private fun DiskEntryContent.getTypeFlag() = + if (this is EntryFile) NORMAL_FILE + else if (this is EntryDirectory) DIRECTORY + else if (this is EntrySymlink) SYMLINK + else 0 // NULL + + fun getTypeString(entry: DiskEntryContent) = when(entry.getTypeFlag()) { + NORMAL_FILE -> "File" + DIRECTORY -> "Directory" + SYMLINK -> "Symbolic Link" + else -> "(unknown type)" + } + } + + fun serialize(): AppendableByteBuffer { + val serialisedContents = contents.serialize() + val buffer = AppendableByteBuffer(HEADER_SIZE + serialisedContents.size) + + buffer.put(entryID.toBigEndian()) + buffer.put(parentEntryID.toBigEndian()) + buffer.put(contents.getTypeFlag()) + buffer.put(0); buffer.put(0); buffer.put(0) + buffer.put(creationDate.toInt48()) + buffer.put(modificationDate.toInt48()) + buffer.put(this.hashCode().toBigEndian()) + + buffer.put(serialisedContents.array) + + return buffer + } + + override fun hashCode() = contents.serialize().getCRC32() + + override fun equals(other: Any?) = if (other == null) false else this.hashCode() == other.hashCode() + + override fun toString() = "DiskEntry(name: ${diskIDtoReadableFilename(entryID)}, ID: $entryID, parent: $parentEntryID, type: ${contents.getTypeFlag()}, contents size: ${contents.getSizeEntry()}, crc: ${hashCode().toHex()})" +} + + +fun ByteArray.forceSize(size: Int): ByteArray { + return ByteArray(size) { if (it < this.size) this[it] else 0.toByte() } +} +interface DiskEntryContent { + fun serialize(): AppendableByteBuffer + fun getSizePure(): Long + fun getSizeEntry(): Long + fun getContent(): Any +} + +/** + * Do not retrieve bytes directly from this! Use VDUtil.retrieveFile(DiskEntry) + * And besides, the bytes could be compressed. + */ +open class EntryFile(internal var bytes: ByteArray64) : DiskEntryContent { + + override fun getSizePure() = bytes.size + override fun getSizeEntry() = getSizePure() + 6 + + /** Create new blank file */ + constructor(size: Long): this(ByteArray64(size)) + + override fun serialize(): AppendableByteBuffer { + val buffer = AppendableByteBuffer(getSizeEntry()) + buffer.put(getSizePure().toInt48()) + buffer.put(bytes) + return buffer + } + + override fun getContent() = bytes +} +class EntryDirectory(private val entries: ArrayList = ArrayList()) : DiskEntryContent { + + override fun getSizePure() = entries.size * 8L + override fun getSizeEntry() = getSizePure() + 4 + private fun checkCapacity(toAdd: Long = 1L) { + if (entries.size + toAdd > 4294967295L) + throw IOException("Directory entries limit exceeded.") + } + + fun add(entryID: EntryID) { + checkCapacity() + entries.add(entryID) + } + + fun remove(entryID: EntryID) { + entries.remove(entryID) + } + + fun contains(entryID: EntryID) = entries.contains(entryID) + + fun forEach(consumer: (EntryID) -> Unit) = entries.forEach(consumer) + + val entryCount: Int + get() = entries.size + + override fun serialize(): AppendableByteBuffer { + val buffer = AppendableByteBuffer(getSizeEntry()) + buffer.put(entries.size.toBigEndian()) + entries.sorted().forEach { indexNumber -> buffer.put(indexNumber.toBigEndian()) } + return buffer + } + + override fun getContent() = entries.toLongArray() + + companion object { + val NEW_ENTRY_SIZE = DiskEntry.HEADER_SIZE + 12L + } +} +class EntrySymlink(val target: EntryID) : DiskEntryContent { + + override fun getSizePure() = 8L + override fun getSizeEntry() = 8L + + override fun serialize(): AppendableByteBuffer { + val buffer = AppendableByteBuffer(getSizeEntry()) + return buffer.put(target.toBigEndian()) + } + + override fun getContent() = target +} + + +fun Int.toHex() = this.toLong().and(0xFFFFFFFF).toString(16).padStart(8, '0').toUpperCase() +fun Long.toHex() = this.ushr(32).toInt().toHex() + "_" + this.toInt().toHex() +fun Int.toBigEndian(): ByteArray { + return ByteArray(4) { this.ushr(24 - (8 * it)).toByte() } +} +fun Long.toBigEndian(): ByteArray { + return ByteArray(8) { this.ushr(56 - (8 * it)).toByte() } +} +fun Long.toInt48(): ByteArray { + return ByteArray(6) { this.ushr(40 - (8 * it)).toByte() } +} +fun Short.toBigEndian(): ByteArray { + return byteArrayOf( + this.div(256).toByte(), + this.toByte() + ) +} + +fun AppendableByteBuffer.getCRC32(): Int { + val crc = CRC32() + this.array.forEach { crc.update(it.toInt()) } + return crc.value.toInt() +} +class AppendableByteBuffer(val size: Long) { + val array = ByteArray64(size) + private var offset = 0L + + fun put(byteArray64: ByteArray64): AppendableByteBuffer { + // it's slow but works + // can't do system.arrayCopy directly + byteArray64.forEach { put(it) } + return this + } + fun put(byteArray: ByteArray): AppendableByteBuffer { + byteArray.forEach { put(it) } + return this + } + fun put(byte: Byte): AppendableByteBuffer { + array[offset] = byte + offset += 1 + return this + } + fun forEach(consumer: (Byte) -> Unit) = array.forEach(consumer) +} diff --git a/src/net/torvald/terrarum/tvd/finder/Popups.kt b/src/net/torvald/terrarum/tvd/finder/Popups.kt new file mode 100644 index 000000000..7f834814a --- /dev/null +++ b/src/net/torvald/terrarum/tvd/finder/Popups.kt @@ -0,0 +1,107 @@ +package net.torvald.terrarum.tvda.finder + +import java.awt.BorderLayout +import java.awt.GridLayout +import javax.swing.* + +/** + * Created by SKYHi14 on 2017-04-01. + */ +object Popups { + val okCancel = arrayOf("OK", "Cancel") + +} + +class OptionDiskNameAndCap { + val name = JTextField(11) + val capacity = JSpinner(SpinnerNumberModel( + 368640L.toJavaLong(), + 0L.toJavaLong(), + (1L shl 38).toJavaLong(), + 1L.toJavaLong() + )) // default 360 KiB, MAX 256 GiB + val mainPanel = JPanel() + val settingPanel = JPanel() + + init { + mainPanel.layout = BorderLayout() + settingPanel.layout = GridLayout(2, 2, 2, 0) + + //name.text = "Unnamed" + + settingPanel.add(JLabel("Name (max 32 bytes)")) + settingPanel.add(name) + settingPanel.add(JLabel("Capacity (bytes)")) + settingPanel.add(capacity) + + mainPanel.add(settingPanel, BorderLayout.CENTER) + mainPanel.add(JLabel("Set capacity to 0 to make the disk read-only"), BorderLayout.SOUTH) + } + + /** + * returns either JOptionPane.OK_OPTION or JOptionPane.CANCEL_OPTION + */ + fun showDialog(title: String): Int { + return JOptionPane.showConfirmDialog(null, mainPanel, + title, JOptionPane.OK_CANCEL_OPTION) + } +} + +fun kotlin.Long.toJavaLong() = java.lang.Long(this) + +class OptionFileNameAndCap { + val name = JTextField(11) + val capacity = JSpinner(SpinnerNumberModel( + 4096L.toJavaLong(), + 0L.toJavaLong(), + ((1L shl 48) - 1L).toJavaLong(), + 1L.toJavaLong() + )) // default 360 KiB, MAX 256 TiB + val mainPanel = JPanel() + val settingPanel = JPanel() + + init { + mainPanel.layout = BorderLayout() + settingPanel.layout = GridLayout(2, 2, 2, 0) + + //name.text = "Unnamed" + + settingPanel.add(JLabel("Name (max 32 bytes)")) + settingPanel.add(name) + settingPanel.add(JLabel("Capacity (bytes)")) + settingPanel.add(capacity) + + mainPanel.add(settingPanel, BorderLayout.CENTER) + } + + /** + * returns either JOptionPane.OK_OPTION or JOptionPane.CANCEL_OPTION + */ + fun showDialog(title: String): Int { + return JOptionPane.showConfirmDialog(null, mainPanel, + title, JOptionPane.OK_CANCEL_OPTION) + } +} + +class OptionSize { + val capacity = JSpinner(SpinnerNumberModel( + 368640L.toJavaLong(), + 0L.toJavaLong(), + (1L shl 38).toJavaLong(), + 1L.toJavaLong() + )) // default 360 KiB, MAX 256 GiB + val settingPanel = JPanel() + + init { + settingPanel.add(JLabel("Size (bytes)")) + settingPanel.add(capacity) + } + + /** + * returns either JOptionPane.OK_OPTION or JOptionPane.CANCEL_OPTION + */ + fun showDialog(title: String): Int { + return JOptionPane.showConfirmDialog(null, settingPanel, + title, JOptionPane.OK_CANCEL_OPTION) + } +} diff --git a/src/net/torvald/terrarum/tvd/finder/VirtualDiskCracker.kt b/src/net/torvald/terrarum/tvd/finder/VirtualDiskCracker.kt new file mode 100644 index 000000000..261586926 --- /dev/null +++ b/src/net/torvald/terrarum/tvd/finder/VirtualDiskCracker.kt @@ -0,0 +1,680 @@ +package net.torvald.terrarum.tvda.finder + +import net.torvald.terrarum.tvda.* +import java.awt.BorderLayout +import java.awt.Dimension +import java.awt.event.KeyEvent +import java.awt.event.MouseAdapter +import java.awt.event.MouseEvent +import java.nio.charset.Charset +import java.time.Instant +import java.time.format.DateTimeFormatter +import java.util.* +import java.util.logging.Level +import javax.swing.* +import javax.swing.table.AbstractTableModel +import javax.swing.text.DefaultCaret + + +/** + * Created by SKYHi14 on 2017-04-01. + */ +class VirtualDiskCracker(val sysCharset: Charset = Charsets.UTF_8) : JFrame() { + + + private val annoyHackers = true // Jar build settings. Intended for Terrarum proj. + + + private val PREVIEW_MAX_BYTES = 4L * 1024 // 4 kBytes + + private val appName = "TerranVirtualDiskCracker" + private val copyright = "Copyright 2017-18 Torvald (minjaesong). Distributed under MIT license." + + private val magicOpen = "I solemnly swear that I am up to no good." + private val magicSave = "Mischief managed." + private val annoyWhenLaunchMsg = "Type in following to get started:\n$magicOpen" + private val annoyWhenSaveMsg = "Type in following to save:\n$magicSave" + + private val panelMain = JPanel() + private val menuBar = JMenuBar() + private val tableFiles: JTable + private val fileDesc = JTextArea() + private val diskInfo = JTextArea() + private val statBar = JLabel("Open a disk or create new to get started") + + private var vdisk: VirtualDisk? = null + private var clipboard: DiskEntry? = null + + private val labelPath = JLabel("(root)") + private var currentDirectoryEntries: Array? = null + private val directoryHierarchy = Stack(); init { directoryHierarchy.push(0) } + + val currentDirectory: EntryID + get() = directoryHierarchy.peek() + val upperDirectory: EntryID + get() = if (directoryHierarchy.lastIndex == 0) 0 + else directoryHierarchy[directoryHierarchy.lastIndex - 1] + private fun gotoRoot() { + directoryHierarchy.removeAllElements() + directoryHierarchy.push(0) + selectedFile = null + fileDesc.text = "" + updateDiskInfo() + } + private fun gotoParent() { + if (directoryHierarchy.size > 1) + directoryHierarchy.pop() + selectedFile = null + fileDesc.text = "" + updateDiskInfo() + } + + + + private var selectedFile: EntryID? = null + + val tableColumns = arrayOf("Name", "Date Modified", "Size") + val tableParentRecord = arrayOf(arrayOf("..", "", "")) + + init { + + if (annoyHackers) { + val mantra = JOptionPane.showInputDialog(annoyWhenLaunchMsg) + if (mantra != magicOpen) { + System.exit(1) + } + } + + + + panelMain.layout = BorderLayout() + this.defaultCloseOperation = JFrame.EXIT_ON_CLOSE + + + tableFiles = JTable(tableParentRecord, tableColumns) + tableFiles.addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent) { + val table = e.source as JTable + val row = table.rowAtPoint(e.point) + + + selectedFile = if (row > 0) + currentDirectoryEntries!![row - 1].entryID + else + null // clicked ".." + + + fileDesc.text = if (selectedFile != null) { + getFileInfoText(vdisk!!.entries[selectedFile!!]!!) + } + else + "" + + fileDesc.caretPosition = 0 + } + }) + tableFiles.selectionModel = object : DefaultListSelectionModel() { + init { selectionMode = ListSelectionModel.SINGLE_SELECTION } + override fun clearSelection() { } // required! + override fun removeSelectionInterval(index0: Int, index1: Int) { } // required! + override fun fireValueChanged(isAdjusting: Boolean) { } // required! + } + tableFiles.model = object : AbstractTableModel() { + override fun getRowCount(): Int { + return if (vdisk != null) + 1 + (currentDirectoryEntries?.size ?: 0) + else 1 + } + + override fun getColumnCount() = tableColumns.size + + override fun getColumnName(column: Int) = tableColumns[column] + + override fun getValueAt(rowIndex: Int, columnIndex: Int): Any { + if (rowIndex == 0) { + return tableParentRecord[0][columnIndex] + } + else { + if (vdisk != null) { + val entry = currentDirectoryEntries!![rowIndex - 1] + return when(columnIndex) { + 0 -> diskIDtoReadableFilename(entry.entryID) + 1 -> Instant.ofEpochSecond(entry.modificationDate). + atZone(TimeZone.getDefault().toZoneId()). + format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) + 2 -> entry.getEffectiveSize() + else -> "" + } + } + else { + return "" + } + } + } + } + + + + val menuFile = JMenu("File") + menuFile.mnemonic = KeyEvent.VK_F + menuFile.add("New Disk…").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + try { + val makeNewDisk: Boolean + if (vdisk != null) { + makeNewDisk = confirmedDiscard() + } + else { + makeNewDisk = true + } + if (makeNewDisk) { + // inquire new size + val dialogBox = OptionDiskNameAndCap() + val confirmNew = JOptionPane.OK_OPTION == dialogBox.showDialog("Set Property of New Disk") + + if (confirmNew) { + vdisk = VDUtil.createNewDisk( + (dialogBox.capacity.value as Long).toLong(), + dialogBox.name.text, + sysCharset + ) + gotoRoot() + updateDiskInfo() + setWindowTitleWithName(dialogBox.name.text) + setStat("Disk created") + } + } + } + catch (e: Exception) { + e.printStackTrace() + popupError(e.toString()) + } + } + }) + menuFile.add("Open Disk…").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + val makeNewDisk: Boolean + if (vdisk != null) { + makeNewDisk = confirmedDiscard() + } + else { + makeNewDisk = true + } + if (makeNewDisk) { + val fileChooser = JFileChooser("./") + fileChooser.showOpenDialog(null) + if (fileChooser.selectedFile != null) { + try { + vdisk = VDUtil.readDiskArchive(fileChooser.selectedFile, Level.WARNING, { popupWarning(it) }, sysCharset) + if (vdisk != null) { + gotoRoot() + updateDiskInfo() + setWindowTitleWithName(fileChooser.selectedFile.canonicalPath) + setStat("Disk loaded") + } + } + catch (e: Exception) { + e.printStackTrace() + popupError(e.toString()) + } + } + } + } + }) + menuFile.addSeparator() + menuFile.add("Save Disk as…").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + if (vdisk != null) { + + + if (annoyHackers) { + val mantra = JOptionPane.showInputDialog(annoyWhenSaveMsg) + if (mantra != magicSave) { + popupError("Nope!") + return + } + } + + + + val fileChooser = JFileChooser("./") + fileChooser.showSaveDialog(null) + if (fileChooser.selectedFile != null) { + try { + VDUtil.dumpToRealMachine(vdisk!!, fileChooser.selectedFile) + setStat("Disk saved") + } + catch (e: Exception) { + e.printStackTrace() + popupError(e.toString()) + } + } + } + } + }) + menuBar.add(menuFile) + + val menuEdit = JMenu("Edit") + menuEdit.mnemonic = KeyEvent.VK_E + menuEdit.add("Cut").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + // copy + clipboard = vdisk!!.entries[selectedFile] + + // delete + if (vdisk != null && selectedFile != null) { + try { + VDUtil.deleteFile(vdisk!!, selectedFile!!) + updateDiskInfo() + setStat("File deleted") + } + catch (e: Exception) { + e.printStackTrace() + popupError(e.toString()) + } + } + } + }) + menuEdit.add("Delete").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + if (vdisk != null && selectedFile != null) { + try { + VDUtil.deleteFile(vdisk!!, selectedFile!!) + updateDiskInfo() + setStat("File deleted") + } + catch (e: Exception) { + e.printStackTrace() + popupError(e.toString()) + } + } + } + }) + menuEdit.add("Renumber…").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + if (selectedFile != null) { + try { + val newID = JOptionPane.showInputDialog("Enter a new name:").toLong() + if (newID != null) { + if (vdisk!!.entries[newID] != null) { + popupError("The name already exists") + } + else { + val id0 = selectedFile!! + val id1 = newID + + val entry = vdisk!!.entries.remove(id0)!! + entry.entryID = id1 + vdisk!!.entries[id1] = entry + VDUtil.getAsDirectory(vdisk!!, 0).remove(id0) + VDUtil.getAsDirectory(vdisk!!, 0).add(id1) + + + updateDiskInfo() + setStat("File renumbered") + } + } + } + catch (e: Exception) { + e.printStackTrace() + popupError(e.toString()) + } + } + } + }) + menuEdit.add("Look Clipboard").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + popupMessage(if (clipboard != null) + "${clipboard ?: "(bug found)"}" + else "(nothing)", "Clipboard" + ) + } + + }) + menuEdit.addSeparator() + menuEdit.add("Import Files/Folders…").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + if (vdisk != null) { + val fileChooser = JFileChooser("./") + fileChooser.fileSelectionMode = JFileChooser.FILES_AND_DIRECTORIES + fileChooser.isMultiSelectionEnabled = true + fileChooser.showOpenDialog(null) + if (fileChooser.selectedFiles.isNotEmpty()) { + try { + fileChooser.selectedFiles.forEach { + if (!it.isDirectory) { + val entry = VDUtil.importFile(it, vdisk!!.generateUniqueID(), sysCharset) + + if (vdisk!!.entries[entry.entryID] != null) { + entry.entryID = JOptionPane.showInputDialog("The ID already exists. Enter a new ID:").toLong() + } + + VDUtil.addFile(vdisk!!, currentDirectory, entry) + } + else { + popupError("Cannot import a directory!") + } + } + updateDiskInfo() + setStat("File added") + } + catch (e: Exception) { + e.printStackTrace() + popupError(e.toString()) + } + } + + fileChooser.isMultiSelectionEnabled = false + } + } + }) + menuEdit.add("Export…").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + if (vdisk != null) { + val file = vdisk!!.entries[selectedFile ?: currentDirectory]!! + + val fileChooser = JFileChooser("./") + fileChooser.fileSelectionMode = JFileChooser.DIRECTORIES_ONLY + fileChooser.isMultiSelectionEnabled = false + fileChooser.showSaveDialog(null) + if (fileChooser.selectedFile != null) { + try { + val file = VDUtil.resolveIfSymlink(vdisk!!, file.entryID) + if (file.contents is EntryFile) { + VDUtil.exportFile(file.contents, fileChooser.selectedFile) + setStat("File exported") + } + else { + } + } + catch (e: Exception) { + e.printStackTrace() + popupError(e.toString()) + } + } + } + } + }) + menuEdit.addSeparator() + menuEdit.add("Rename Disk…").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + if (vdisk != null) { + try { + val newname = JOptionPane.showInputDialog("Enter a new disk name:") + if (newname != null) { + vdisk!!.diskName = newname.toEntryName(VirtualDisk.NAME_LENGTH, sysCharset) + updateDiskInfo() + setStat("Disk renamed") + } + } + catch (e: Exception) { + e.printStackTrace() + popupError(e.toString()) + } + } + } + }) + menuEdit.add("Resize Disk…").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + if (vdisk != null) { + try { + val dialog = OptionSize() + val confirmed = dialog.showDialog("Input") == JOptionPane.OK_OPTION + if (confirmed) { + vdisk!!.capacity = (dialog.capacity.value as Long).toLong() + updateDiskInfo() + setStat("Disk resized") + } + } + catch (e: Exception) { + e.printStackTrace() + popupError(e.toString()) + } + } + } + }) + menuEdit.addSeparator() + menuEdit.add("Set/Unset Write Protection").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + if (vdisk != null) { + try { + vdisk!!.isReadOnly = vdisk!!.isReadOnly.not() + updateDiskInfo() + setStat("Disk write protection ${if (vdisk!!.isReadOnly) "" else "dis"}engaged") + } + catch (e: Exception) { + e.printStackTrace() + popupError(e.toString()) + } + } + } + }) + menuBar.add(menuEdit) + + val menuManage = JMenu("Manage") + menuManage.add("Report Orphans…").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + if (vdisk != null) { + try { + val reports = VDUtil.gcSearchOrphan(vdisk!!) + val orphansCount = reports.size + val orphansSize = reports.map { vdisk!!.entries[it]!!.contents.getSizeEntry() }.sum() + val message = "Orphans count: $orphansCount\n" + + "Size: ${orphansSize.bytes()}" + popupMessage(message, "Orphans Report") + } + catch (e: Exception) { + e.printStackTrace() + popupError(e.toString()) + } + } + } + }) + menuManage.add("Report Phantoms…").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + if (vdisk != null) { + try { + val reports = VDUtil.gcSearchPhantomBaby(vdisk!!) + val phantomsSize = reports.size + val message = "Phantoms count: $phantomsSize" + popupMessage(message, "Phantoms Report") + } + catch (e: Exception) { + e.printStackTrace() + popupError(e.toString()) + } + } + } + }) + menuManage.addSeparator() + menuManage.add("Remove Orphans").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + if (vdisk != null) { + try { + val oldSize = vdisk!!.usedBytes + VDUtil.gcDumpOrphans(vdisk!!) + val newSize = vdisk!!.usedBytes + popupMessage("Saved ${(oldSize - newSize).bytes()}", "GC Report") + updateDiskInfo() + setStat("Orphan nodes removed") + } + catch (e: Exception) { + e.printStackTrace() + popupError(e.toString()) + } + } + } + }) + menuManage.add("Full Garbage Collect").addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + if (vdisk != null) { + try { + val oldSize = vdisk!!.usedBytes + VDUtil.gcDumpAll(vdisk!!) + val newSize = vdisk!!.usedBytes + popupMessage("Saved ${(oldSize - newSize).bytes()}", "GC Report") + updateDiskInfo() + setStat("Orphan nodes and null directory pointers removed") + } + catch (e: Exception) { + e.printStackTrace() + popupError(e.toString()) + } + } + } + }) + menuBar.add(menuManage) + + val menuAbout = JMenu("About") + menuAbout.addMouseListener(object : MouseAdapter() { + override fun mousePressed(e: MouseEvent?) { + popupMessage(copyright, "Copyright") + } + }) + menuBar.add(menuAbout) + + + + diskInfo.highlighter = null + diskInfo.text = "(Disk not loaded)" + diskInfo.preferredSize = Dimension(-1, 60) + + fileDesc.highlighter = null + fileDesc.text = "" + fileDesc.caret.isVisible = false + (fileDesc.caret as DefaultCaret).updatePolicy = DefaultCaret.NEVER_UPDATE + + val fileDescScroll = JScrollPane(fileDesc) + val tableFilesScroll = JScrollPane(tableFiles) + tableFilesScroll.size = Dimension(200, -1) + + val panelFinder = JPanel(BorderLayout()) + panelFinder.add(labelPath, BorderLayout.NORTH) + panelFinder.add(tableFilesScroll, BorderLayout.CENTER) + + val panelFileDesc = JPanel(BorderLayout()) + panelFileDesc.add(JLabel("Entry Information"), BorderLayout.NORTH) + panelFileDesc.add(fileDescScroll, BorderLayout.CENTER) + + val filesSplit = JSplitPane(JSplitPane.HORIZONTAL_SPLIT, panelFinder, panelFileDesc) + filesSplit.resizeWeight = 0.571428 + + + val panelDiskOp = JPanel(BorderLayout(2, 2)) + panelDiskOp.add(filesSplit, BorderLayout.CENTER) + panelDiskOp.add(diskInfo, BorderLayout.SOUTH) + + + panelMain.add(menuBar, BorderLayout.NORTH) + panelMain.add(panelDiskOp, BorderLayout.CENTER) + panelMain.add(statBar, BorderLayout.SOUTH) + + + this.title = appName + this.add(panelMain) + this.setSize(700, 700) + this.isVisible = true + } + + private fun confirmedDiscard() = 0 == JOptionPane.showOptionDialog( + null, // parent + "Any changes to current disk will be discarded. Continue?", + "Confirm Discard", // window title + JOptionPane.DEFAULT_OPTION, // option type + JOptionPane.WARNING_MESSAGE, // message type + null, // icon + Popups.okCancel, // options (provided by JOptionPane.OK_CANCEL_OPTION in this case) + Popups.okCancel[1] // default selection + ) + private fun popupMessage(message: String, title: String = "") { + JOptionPane.showOptionDialog( + null, + message, + title, + JOptionPane.DEFAULT_OPTION, + JOptionPane.INFORMATION_MESSAGE, + null, null, null + ) + } + private fun popupError(message: String, title: String = "Uh oh…") { + JOptionPane.showOptionDialog( + null, + message, + title, + JOptionPane.DEFAULT_OPTION, + JOptionPane.ERROR_MESSAGE, + null, null, null + ) + } + private fun popupWarning(message: String, title: String = "Careful…") { + JOptionPane.showOptionDialog( + null, + message, + title, + JOptionPane.DEFAULT_OPTION, + JOptionPane.WARNING_MESSAGE, + null, null, null + ) + } + private fun updateCurrentDirectory() { + currentDirectoryEntries = VDUtil.getDirectoryEntries(vdisk!!, currentDirectory) + } + private fun updateDiskInfo() { + val sb = StringBuilder() + directoryHierarchy.forEach { + sb.append(diskIDtoReadableFilename(it)) + sb.append('/') + } + sb.dropLast(1) + labelPath.text = sb.toString() + + diskInfo.text = if (vdisk == null) "(Disk not loaded)" else getDiskInfoText(vdisk!!) + tableFiles.revalidate() + tableFiles.repaint() + + + updateCurrentDirectory() + } + private fun getDiskInfoText(disk: VirtualDisk): String { + return """Name: ${String(disk.diskName, sysCharset)} +Capacity: ${disk.capacity} bytes (${disk.usedBytes} bytes used, ${disk.capacity - disk.usedBytes} bytes free) +Write protected: ${disk.isReadOnly.toEnglish()}""" + } + + + private fun Boolean.toEnglish() = if (this) "Yes" else "No" + + + private fun getFileInfoText(file: DiskEntry): String { + return """Name: ${diskIDtoReadableFilename(file.entryID)} +Size: ${file.getEffectiveSize()} +Type: ${DiskEntry.getTypeString(file.contents)} +CRC: ${file.hashCode().toHex()} +EntryID: ${file.entryID} +ParentID: ${file.parentEntryID}""" + if (file.contents is EntryFile) """ + +Contents: +${String(file.contents.bytes.sliceArray64(0L..minOf(PREVIEW_MAX_BYTES, file.contents.bytes.size) - 1).toByteArray(), sysCharset)}""" else "" + } + private fun setWindowTitleWithName(name: String) { + this.title = "$appName - $name" + } + + private fun Long.bytes() = if (this == 1L) "1 byte" else "$this bytes" + private fun Int.entries() = if (this == 1) "1 entry" else "$this entries" + private fun DiskEntry.getEffectiveSize() = if (this.contents is EntryFile) + this.contents.getSizePure().bytes() + else if (this.contents is EntryDirectory) + this.contents.entryCount.entries() + else if (this.contents is EntrySymlink) + "(symlink)" + else + "n/a" + private fun setStat(message: String) { + statBar.text = message + } +} + +fun main(args: Array) { + VirtualDiskCracker(Charset.forName("CP437")) +} \ No newline at end of file diff --git a/src/net/torvald/terrarum/utils/HashArray.kt b/src/net/torvald/terrarum/utils/HashArray.kt index 1e14019d1..766139752 100644 --- a/src/net/torvald/terrarum/utils/HashArray.kt +++ b/src/net/torvald/terrarum/utils/HashArray.kt @@ -7,7 +7,7 @@ import net.torvald.terrarum.gameitem.ItemID import net.torvald.terrarum.gameworld.BlockAddress import net.torvald.terrarum.gameworld.FluidType import net.torvald.terrarum.gameworld.GameWorld -import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64Reader +import net.torvald.terrarum.tvda.ByteArray64Reader import net.torvald.terrarum.serialise.Common import java.io.StringReader