bytearray64reader: read length of zero and EOF are properly distinguished (gdx is somewhat pedantic); changed an ascii85 charset; working meta (de)serialisation

This commit is contained in:
minjaesong
2021-09-01 17:23:12 +09:00
parent 07f26a7716
commit f08296b3be
11 changed files with 109 additions and 315 deletions

Binary file not shown.

Binary file not shown.

View File

@@ -18,7 +18,6 @@ import com.github.strikerx3.jxinput.XInputDevice;
import net.torvald.gdx.graphics.PixmapIO2;
import net.torvald.getcpuname.GetCpuName;
import net.torvald.terrarum.concurrent.ThreadExecutor;
import net.torvald.terrarum.console.ConsoleCommand;
import net.torvald.terrarum.controller.GdxControllerAdapter;
import net.torvald.terrarum.controller.TerrarumController;
import net.torvald.terrarum.controller.XinputControllerAdapter;
@@ -29,7 +28,6 @@ import net.torvald.terrarum.imagefont.TinyAlphNum;
import net.torvald.terrarum.langpack.Lang;
import net.torvald.terrarum.modulebasegame.IngameRenderer;
import net.torvald.terrarum.modulebasegame.TerrarumIngame;
import net.torvald.terrarum.modulebasegame.console.ToggleNoClip;
import net.torvald.terrarum.modulebasegame.ui.ItemSlotImageFactory;
import net.torvald.terrarum.utils.JsonFetcher;
import net.torvald.terrarum.utils.JsonWriter;

View File

@@ -2,10 +2,6 @@
package net.torvald.terrarum.gameworld
import com.badlogic.gdx.utils.Disposable
import com.badlogic.gdx.utils.Json
import com.badlogic.gdx.utils.JsonWriter
import com.badlogic.gdx.utils.compression.Lzma
import net.torvald.UnsafePtrInputStream
import net.torvald.gdx.graphics.Cvec
import net.torvald.terrarum.*
import net.torvald.terrarum.AppLoader.printdbg
@@ -14,16 +10,10 @@ import net.torvald.terrarum.blockproperties.BlockCodex
import net.torvald.terrarum.blockproperties.Fluid
import net.torvald.terrarum.gameactors.WireActor
import net.torvald.terrarum.gameitem.ItemID
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64GrowableOutputStream
import net.torvald.terrarum.realestate.LandUtil
import net.torvald.terrarum.serialise.Ascii85
import net.torvald.terrarum.serialise.Common
import net.torvald.terrarum.serialise.bytesToZipdStr
import net.torvald.terrarum.utils.*
import net.torvald.util.SortedArrayList
import org.apache.commons.codec.digest.DigestUtils
import org.dyn4j.geometry.Vector2
import java.util.zip.GZIPOutputStream
import kotlin.math.absoluteValue
typealias BlockAddress = Long
@@ -49,7 +39,7 @@ class GameWorld() : Disposable {
internal set
var lastPlayTime: Long = AppLoader.getTIME_T()
internal set // there's a case of save-and-continue-playing
var totalPlayTime: Int = 0
var totalPlayTime: Long = 0
internal set
/** Used to calculate play time */
@@ -119,7 +109,7 @@ class GameWorld() : Disposable {
/**
* Create new world
*/
constructor(worldIndex: Int, width: Int, height: Int, creationTIME_T: Long, lastPlayTIME_T: Long, totalPlayTime: Int): this() {
constructor(worldIndex: Int, width: Int, height: Int, creationTIME_T: Long, lastPlayTIME_T: Long, totalPlayTime: Long): this() {
if (width <= 0 || height <= 0) throw IllegalArgumentException("Non-positive width/height: ($width, $height)")
this.worldIndex = worldIndex

View File

@@ -1,20 +1,16 @@
package net.torvald.terrarum.modulebasegame.console
import com.badlogic.gdx.utils.JsonValue
import net.torvald.terrarum.AppLoader
import net.torvald.terrarum.AppLoader.printdbg
import net.torvald.terrarum.Terrarum
import net.torvald.terrarum.console.ConsoleCommand
import net.torvald.terrarum.console.Echo
import net.torvald.terrarum.modulebasegame.TerrarumIngame
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.DiskEntry
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64Reader
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.EntryFile
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.VDUtil
import net.torvald.terrarum.serialise.*
import net.torvald.terrarum.utils.JsonFetcher
import java.io.File
import java.io.IOException
import java.nio.ByteBuffer
import java.io.StringReader
import kotlin.reflect.full.declaredMemberProperties
/**
* Created by minjaesong on 2021-08-30.
@@ -29,12 +25,16 @@ object Load : ConsoleCommand {
val disk = VDUtil.readDiskArchive(file, charset = charset)
val metaFile = disk.entries[-1]!!
val metaReader = ByteArray64Reader(metaFile.contents.serialize().array, charset)
val meta = Common.jsoner.fromJson(JsonValue::class.java, metaReader)
JsonFetcher.forEach(meta) { key, value ->
println("$key\t$value")
val metaReader = ByteArray64Reader((metaFile.contents as EntryFile).getContent(), Common.CHARSET)
val meta = Common.jsoner.fromJson(WriteMeta.WorldMeta::class.java, metaReader)
WriteMeta.WorldMeta::class.declaredMemberProperties.forEach {
println("${it.name} = ${it.get(meta)}")
}
println(WriteMeta.unasciiAndUnzipStr(meta.blocks))
println(meta.loadorder.joinToString())
}
catch (e: IOException) {
Echo("Load: IOException raised.")

View File

@@ -2,8 +2,8 @@ package net.torvald.terrarum.modulebasegame.console
import net.torvald.terrarum.console.ConsoleCommand
import net.torvald.terrarum.console.Echo
import net.torvald.terrarum.serialise.ByteArray64Reader
import net.torvald.terrarum.serialise.ByteArray64Writer
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64Reader
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64Writer
import net.torvald.terrarum.serialise.Common
import net.torvald.terrarum.serialise.toUint
import java.io.File

View File

@@ -6,8 +6,13 @@ package net.torvald.terrarum.serialise
* just gzip the inputstream instead!
*/
object Ascii85 {
/** As per RFC-1924 */
private const val CHAR_TABLE = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#${'$'}%&()*+-;<=>?@^_`{|}~"
/** My own string set that:
* - no "/": avoids nonstandard JSON comment key which GDX will happily parse away
* - no "\": you know what I mean\\intention
* - no "$": avoids Kotlin string template
* - no "[{]},": even the dumbest parser can comprehend the output
*/
private const val CHAR_TABLE = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!#%&'()*+-.:;<=>?@^_`|~"
/** As per Adobe standard */
//private val CHAR_TABLE = (33 until (33+85)).toList().map { it.toChar() }.joinToString("") // testing only!

View File

@@ -3,7 +3,6 @@ package net.torvald.terrarum.serialise
import com.badlogic.gdx.utils.Json
import com.badlogic.gdx.utils.JsonValue
import com.badlogic.gdx.utils.JsonWriter
import net.torvald.terrarum.AppLoader.printdbg
import net.torvald.terrarum.console.EchoError
import net.torvald.terrarum.gameworld.BlockLayer
import net.torvald.terrarum.gameworld.GameWorld
@@ -272,215 +271,3 @@ object Common {
return unzipdBytes
}
}
class ByteArray64Writer(val charset: Charset) : Writer() {
private val acceptableCharsets = arrayOf(Charsets.UTF_8, Charset.forName("CP437"))
init {
if (!acceptableCharsets.contains(charset))
throw UnsupportedCharsetException(charset.name())
}
private val ba64 = ByteArray64()
private var closed = false
private var surrogateBuf = 0
init {
this.lock = ba64
}
private fun checkOpen() {
if (closed) throw ClosedChannelException()
}
private fun Int.isSurroHigh() = this.ushr(10) == 0b110110
private fun Int.isSurroLow() = this.ushr(10) == 0b110111
private fun Int.toUcode() = 'u' + this.toString(16).uppercase().padStart(4,'0')
/**
* @param c not a freakin' codepoint; just a Java's Char casted into Int
*/
override fun write(c: Int) {
checkOpen()
when (charset) {
Charsets.UTF_8 -> {
if (surrogateBuf == 0 && !c.isSurroHigh() && !c.isSurroLow())
writeUtf8Codepoint(c)
else if (surrogateBuf == 0 && c.isSurroHigh())
surrogateBuf = c
else if (surrogateBuf != 0 && c.isSurroLow())
writeUtf8Codepoint(65536 + surrogateBuf.and(1023).shl(10) or c.and(1023))
// invalid surrogate pair input
else
throw IllegalStateException("Surrogate high: ${surrogateBuf.toUcode()}, surrogate low: ${c.toUcode()}")
}
Charset.forName("CP437") -> {
ba64.add(c.toByte())
}
else -> throw UnsupportedCharsetException(charset.name())
}
}
fun writeUtf8Codepoint(codepoint: Int) {
when (codepoint) {
in 0..127 -> ba64.add(codepoint.toByte())
in 128..2047 -> {
ba64.add((0xC0 or codepoint.ushr(6).and(31)).toByte())
ba64.add((0x80 or codepoint.and(63)).toByte())
}
in 2048..65535 -> {
ba64.add((0xE0 or codepoint.ushr(12).and(15)).toByte())
ba64.add((0x80 or codepoint.ushr(6).and(63)).toByte())
ba64.add((0x80 or codepoint.and(63)).toByte())
}
in 65536..1114111 -> {
ba64.add((0xF0 or codepoint.ushr(18).and(7)).toByte())
ba64.add((0x80 or codepoint.ushr(12).and(63)).toByte())
ba64.add((0x80 or codepoint.ushr(6).and(63)).toByte())
ba64.add((0x80 or codepoint.and(63)).toByte())
}
else -> throw IllegalArgumentException("Not a unicode code point: U+${codepoint.toString(16).uppercase()}")
}
}
override fun write(cbuf: CharArray) {
checkOpen()
write(String(cbuf))
}
override fun write(str: String) {
checkOpen()
str.toByteArray(charset).forEach { ba64.add(it) }
}
override fun write(cbuf: CharArray, off: Int, len: Int) {
write(cbuf.copyOfRange(off, off + len))
}
override fun write(str: String, off: Int, len: Int) {
write(str.substring(off, off + len))
}
override fun close() { closed = true }
override fun flush() {}
fun toByteArray64() = if (closed) ba64 else throw IllegalAccessException("Writer not closed")
}
class ByteArray64Reader(val ba: ByteArray64, val charset: Charset) : Reader() {
private val acceptableCharsets = arrayOf(Charsets.UTF_8, Charset.forName("CP437"))
init {
if (!acceptableCharsets.contains(charset))
throw UnsupportedCharsetException(charset.name())
}
private var readCursor = 0L
private val remaining
get() = ba.size - readCursor
/**
* U+0000 .. U+007F 0xxxxxxx
* U+0080 .. U+07FF 110xxxxx 10xxxxxx
* U+0800 .. U+FFFF 1110xxxx 10xxxxxx 10xxxxxx
* U+10000 .. U+10FFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
*/
private fun utf8GetCharLen(head: Byte) = when (head.toInt() and 255) {
in 0b11110_000..0b11110_111 -> 4
in 0b1110_0000..0b1110_1111 -> 3
in 0b110_00000..0b110_11111 -> 2
in 0b0_0000000..0b0_1111111 -> 1
else -> throw IllegalArgumentException("Invalid UTF-8 Character head byte: ${head.toInt() and 255}")
}
/**
* @param list of bytes that encodes one unicode character. Get required byte length using [utf8GetCharLen].
* @return A codepoint of the character.
*/
private fun utf8decode(bytes0: List<Byte>): Int {
val bytes = bytes0.map { it.toInt() and 255 }
var ret = when (bytes.size) {
4 -> (bytes[0] and 7) shl 18
3 -> (bytes[0] and 15) shl 12
2 -> (bytes[0] and 31) shl 6
1 -> (bytes[0] and 127)
else -> throw IllegalArgumentException("Expected bytes size: 1..4, got ${bytes.size}")
}
bytes.tail().reversed().forEachIndexed { index, byte ->
ret = ret or (byte and 63).shl(6 * index)
}
return ret
}
private var surrogateLeftover = ' '
override fun read(cbuf: CharArray, off: Int, len: Int): Int {
var readCount = 0
when (charset) {
Charsets.UTF_8 -> {
while (readCount < len && remaining > 0) {
if (surrogateLeftover != ' ') {
cbuf[off + readCount] = surrogateLeftover
readCount += 1
surrogateLeftover = ' '
}
else {
val bbuf = (0 until minOf(4L, remaining)).map { ba[readCursor + it] }
val charLen = utf8GetCharLen(bbuf[0])
val codePoint = utf8decode(bbuf.subList(0, charLen))
if (codePoint < 65536) {
cbuf[off + readCount] = codePoint.toChar()
readCount += 1
readCursor += charLen
}
else {
/*
* U' = yyyyyyyyyyxxxxxxxxxx // U - 0x10000
* W1 = 110110yyyyyyyyyy // 0xD800 + yyyyyyyyyy
* W2 = 110111xxxxxxxxxx // 0xDC00 + xxxxxxxxxx
*/
val codPoin = codePoint - 65536
val surroLead = (0xD800 or codPoin.ushr(10)).toChar()
val surroTrail = (0xDC00 or codPoin.and(1023)).toChar()
cbuf[off + readCount] = surroLead
if (off + readCount + 1 < cbuf.size) {
cbuf[off + readCount + 1] = surroTrail
readCount += 2
readCursor += 4
}
else {
readCount += 1
readCursor += 4
surrogateLeftover = surroTrail
}
}
}
}
}
Charset.forName("CP437") -> {
for (i in 0 until minOf(len.toLong(), remaining)) {
cbuf[(off + i).toInt()] = ba[readCursor].toChar()
readCursor += 1
readCount += 1
}
}
else -> throw UnsupportedCharsetException(charset.name())
}
return if (readCount == 0) -1 else readCount
}
override fun close() { readCursor = 0L }
override fun reset() { readCursor = 0L }
override fun markSupported() = false
}

View File

@@ -1,12 +1,8 @@
package net.torvald.terrarum.serialise
import com.badlogic.gdx.utils.Json
import com.badlogic.gdx.utils.JsonValue
import com.badlogic.gdx.utils.JsonWriter
import net.torvald.terrarum.gameactors.Actor
import net.torvald.terrarum.modulebasegame.gameactors.IngamePlayer
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64
import java.math.BigInteger
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64Writer
/**
* Created by minjaesong on 2021-08-24.

View File

@@ -1,19 +1,13 @@
package net.torvald.terrarum.serialise
import com.badlogic.gdx.utils.compression.Lzma
import net.torvald.terrarum.ModMgr
import net.torvald.terrarum.gameactors.Actor
import net.torvald.terrarum.gameworld.BlockLayer
import net.torvald.terrarum.gameactors.ActorID
import net.torvald.terrarum.modulebasegame.TerrarumIngame
import net.torvald.terrarum.modulebasegame.worldgenerator.RoguelikeRandomiser
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64GrowableOutputStream
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64InputStream
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64Reader
import net.torvald.terrarum.weather.WeatherMixer
import java.io.ByteArrayInputStream
import java.io.StringReader
import java.util.zip.GZIPInputStream
import java.util.zip.GZIPOutputStream
/**
* Created by minjaesong on 2021-08-23.
@@ -23,51 +17,51 @@ open class WriteMeta(val ingame: TerrarumIngame) {
open fun invoke(): String {
val world = ingame.world
val json = """{
"genver": ${Common.GENVER},
"savename": "${world.worldName}",
"terrseed": ${world.generatorSeed},
"randseed0": ${RoguelikeRandomiser.RNG.state0},
"randseed1": ${RoguelikeRandomiser.RNG.state1},
"weatseed0": ${WeatherMixer.RNG.state0},
"weatseed1": ${WeatherMixer.RNG.state1},
"playerid": ${ingame.actorGamer.referenceID},
"creation_t": ${world.creationTime},
"lastplay_t": ${world.lastPlayTime},
"playtime_t": ${world.totalPlayTime},
"blocks": "${StringBuilder().let {
ModMgr.getFilesFromEveryMod("blocks/blocks.csv").forEach { (modname, file) ->
it.append("\n\n## module: $modname ##\n\n")
it.append(file.readText())
}
zipStrAndEnascii(it.toString())
}}",
"items": "${StringBuilder().let {
ModMgr.getFilesFromEveryMod("items/itemid.csv").forEach { (modname, file) ->
it.append("\n\n## module: $modname ##\n\n")
it.append(file.readText())
}
zipStrAndEnascii(it.toString())
}}",
"wires": "${StringBuilder().let {
ModMgr.getFilesFromEveryMod("wires/wires.csv").forEach { (modname, file) ->
it.append("\n\n## module: $modname ##\n\n")
it.append(file.readText())
}
zipStrAndEnascii(it.toString())
}}",
"materials": "${StringBuilder().let {
ModMgr.getFilesFromEveryMod("materials/materials.csv").forEach { (modname, file) ->
it.append("\n\n## module: $modname ##\n\n")
it.append(file.readText())
}
zipStrAndEnascii(it.toString())
}}",
"loadorder": [${ModMgr.loadOrder.map { "\"${it}\"" }.joinToString()}],
"worlds": [${ingame.gameworldIndices.joinToString()}]
}"""
return json
val meta = WorldMeta(
genver = Common.GENVER,
savename = world.worldName,
terrseed = world.generatorSeed,
randseed0 = RoguelikeRandomiser.RNG.state0,
randseed1 = RoguelikeRandomiser.RNG.state1,
weatseed0 = WeatherMixer.RNG.state0,
weatseed1 = WeatherMixer.RNG.state1,
playerid = ingame.actorGamer.referenceID,
creation_t = world.creationTime,
lastplay_t = world.lastPlayTime,
playtime_t = world.totalPlayTime,
blocks = StringBuilder().let {
ModMgr.getFilesFromEveryMod("blocks/blocks.csv").forEach { (modname, file) ->
it.append(modnameToOrnamentalHeader(modname))
it.append(file.readText())
}
zipStrAndEnascii(it.toString())
},
items = StringBuilder().let {
ModMgr.getFilesFromEveryMod("items/itemid.csv").forEach { (modname, file) ->
it.append(modnameToOrnamentalHeader(modname))
it.append(file.readText())
}
zipStrAndEnascii(it.toString())
},
wires = StringBuilder().let {
ModMgr.getFilesFromEveryMod("wires/wires.csv").forEach { (modname, file) ->
it.append(modnameToOrnamentalHeader(modname))
it.append(file.readText())
}
zipStrAndEnascii(it.toString())
},
materials = StringBuilder().let {
ModMgr.getFilesFromEveryMod("materials/materials.csv").forEach { (modname, file) ->
it.append(modnameToOrnamentalHeader(modname))
it.append(file.readText())
}
zipStrAndEnascii(it.toString())
},
loadorder = ModMgr.loadOrder.toTypedArray(),
worlds = ingame.gameworldIndices.toTypedArray()
)
return Common.jsoner.toJson(meta)
}
fun encodeToByteArray64(): ByteArray64 {
@@ -77,20 +71,46 @@ open class WriteMeta(val ingame: TerrarumIngame) {
}
data class WorldMeta(
val genver: Int,
val savename: String
)
/**
* @param [s] a String
* @return UTF-8 encoded [s] which are GZip'd then Ascii85-encoded
*/
fun zipStrAndEnascii(s: String): String {
return Common.bytesToZipdStr(s.toByteArray(Common.CHARSET).iterator())
val genver: Int = -1,
val savename: String = "",
val terrseed: Long = 0,
val randseed0: Long = 0,
val randseed1: Long = 0,
val weatseed0: Long = 0,
val weatseed1: Long = 0,
val playerid: ActorID = 0,
val creation_t: Long = 0,
val lastplay_t: Long = 0,
val playtime_t: Long = 0,
val blocks: String = "",
val items: String = "",
val wires: String = "",
val materials: String = "",
val loadorder: Array<String> = arrayOf(), // do not use list; Could not instantiate instance of class: java.util.Collections$SingletonList
val worlds: Array<Int> = arrayOf() // do not use list; Could not instantiate instance of class: java.util.Collections$SingletonList
) {
override fun equals(other: Any?): Boolean {
throw UnsupportedOperationException()
}
}
fun unasciiAndUnzipStr(s: String): String {
return ByteArray64Reader(Common.strToBytes(StringReader(s)), Common.CHARSET).readText()
companion object {
private fun modnameToOrnamentalHeader(s: String) =
"\n\n${"#".repeat(16 + s.length)}\n" +
"## module: $s ##\n" +
"${"#".repeat(16 + s.length)}\n\n"
/**
* @param [s] a String
* @return UTF-8 encoded [s] which are GZip'd then Ascii85-encoded
*/
fun zipStrAndEnascii(s: String): String {
return Common.bytesToZipdStr(s.toByteArray(Common.CHARSET).iterator())
}
fun unasciiAndUnzipStr(s: String): String {
return ByteArray64Reader(Common.strToBytes(StringReader(s)), Common.CHARSET).readText()
}
}
}

View File

@@ -2,9 +2,7 @@ package net.torvald.terrarum.serialise
import net.torvald.terrarum.modulebasegame.TerrarumIngame
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64GrowableOutputStream
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64OutputStream
import java.io.Writer
import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64Writer
/**
* Created by minjaesong on 2021-08-23.