diff --git a/src/net/torvald/UnsafePtr.kt b/src/net/torvald/UnsafePtr.kt index 62265df97..3f4ce9ca9 100644 --- a/src/net/torvald/UnsafePtr.kt +++ b/src/net/torvald/UnsafePtr.kt @@ -2,6 +2,9 @@ package net.torvald import net.torvald.terrarum.printStackTrace import sun.misc.Unsafe +import java.io.IOException +import java.io.InputStream +import java.io.OutputStream /** * Further read: @@ -184,4 +187,38 @@ internal class UnsafePtr(pointer: Long, allocSize: Long) { override fun toString() = "0x${ptr.toString(16)} with size $size" override fun equals(other: Any?) = this.ptr == (other as UnsafePtr).ptr && this.size == other.size +} + +internal class UnsafePtrInputStream(val ptr: UnsafePtr): InputStream() { + private var p = 0L + + override fun reset() { + p = 0L + } + + override fun read(): Int { + if (p < ptr.size) { + p += 1 + return ptr[p - 1].toInt().and(255) + } + else return -1 + } +} + +internal class UnsafePtrOutputStream(val ptr: UnsafePtr): OutputStream() { + private var p = 0L + + override fun write(p0: Int) { + if (p < ptr.size) { + p += 1 + ptr[p - 1] = p0.toByte() + } + else throw IOException("Buffer overflow: $p for allocated size ${ptr.size}") + } + + override fun write(b: ByteArray, off: Int, len: Int) { + if (p + len >= ptr.size) throw IOException("Buffer overflow: ${p+len} for allocated size ${ptr.size}") + UnsafeHelper.unsafe.copyMemory(b, off.toLong(), null, ptr.ptr + p, len.toLong()) + p += len + } } \ No newline at end of file diff --git a/src/net/torvald/terrarum/gameworld/GameWorld.kt b/src/net/torvald/terrarum/gameworld/GameWorld.kt index 87a921215..6433fae67 100644 --- a/src/net/torvald/terrarum/gameworld/GameWorld.kt +++ b/src/net/torvald/terrarum/gameworld/GameWorld.kt @@ -5,6 +5,9 @@ import com.badlogic.gdx.utils.Disposable import com.badlogic.gdx.utils.Json import com.badlogic.gdx.utils.JsonValue import com.badlogic.gdx.utils.JsonWriter +import com.badlogic.gdx.utils.compression.Lzma +import net.torvald.UnsafePtr +import net.torvald.UnsafePtrInputStream import net.torvald.gdx.graphics.Cvec import net.torvald.terrarum.* import net.torvald.terrarum.AppLoader.printdbg @@ -16,6 +19,8 @@ import net.torvald.terrarum.gameitem.ItemID import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64GrowableOutputStream import net.torvald.terrarum.realestate.LandUtil import net.torvald.terrarum.serialise.Ascii85 +import net.torvald.terrarum.serialise.bytesToLzmadStr +import net.torvald.terrarum.serialise.bytesToZipdStr import net.torvald.util.SortedArrayList import org.apache.commons.codec.digest.DigestUtils import org.dyn4j.geometry.Vector2 @@ -699,11 +704,14 @@ open class GameWorld : Disposable { layerWall.bytesIterator().forEachRemaining { digester.update(it) } val wallhash = StringBuilder().let { sb -> digester.digest().forEach { sb.append(it.tostr()) }; sb.toString() } + // use gzip; lzma's slower and larger for some reason return arrayListOf( """"worldname": "$worldName"""", - """"comp": 1""", + """"comp": "gzip"""", """"width": $width""", """"height": $height""", + """"spawnx": $spawnX""", + """"spawny": $spawnY""", """"genver": 4""", """"time_t": ${worldTime.TIME_T}""", """"terr": { @@ -714,22 +722,22 @@ open class GameWorld : Disposable { |"b": "${blockLayerToStr(layerWall)}"}""".trimMargin(), """"tdmg": { |"h": "${StringBuilder().let { sb -> digester.digest(tdmgstr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}", - |"b": $tdmgstr}""".trimMargin(), + |"b": "${bytesToZipdStr(tdmgstr.toByteArray())}"}""".trimMargin(), """"wdmg": { |"h": "${StringBuilder().let { sb -> digester.digest(wdmgstr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}", - |"b": $wdmgstr}""".trimMargin(), + |"b": "${bytesToZipdStr(wdmgstr.toByteArray())}"}""".trimMargin(), """"flut": { |"h": "${StringBuilder().let { sb -> digester.digest(flutstr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}", - |"b": $flutstr}""".trimMargin(), + |"b": "${bytesToZipdStr(flutstr.toByteArray())}"}""".trimMargin(), """"fluf": { |"h": "${StringBuilder().let { sb -> digester.digest(flufstr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}", - |"b": $flufstr}""".trimMargin(), + |"b": "${bytesToZipdStr(flufstr.toByteArray())}"}""".trimMargin(), """"wire": { |"h": "${StringBuilder().let { sb -> digester.digest(wirestr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}", - |"b": $wirestr}""".trimMargin(), + |"b": "${bytesToZipdStr(wirestr.toByteArray())}"}""".trimMargin(), """"wirg": { |"h": "${StringBuilder().let { sb -> digester.digest(wirgstr.toByteArray()).forEach { sb.append(it.tostr()) }; sb.toString() }}", - |"b": $wirgstr}""".trimMargin() + |"b": "${bytesToZipdStr(wirgstr.toByteArray())}"}""".trimMargin() ) } } @@ -772,5 +780,33 @@ fun blockLayerToStr(b: BlockLayer): String { bai += 1 }; sb.append(Ascii85.encode(buf[0], buf[1], buf[2], buf[3])) + return sb.toString() +} + +/** + * @param b a BlockLayer + * @return Bytes in [b] which are LZMA'd then Ascii85-encoded + */ +fun blockLayerToStr2(b: BlockLayer): String { + val sb = StringBuilder() + val bi = UnsafePtrInputStream(b.ptr) + val bo = ByteArray64GrowableOutputStream() + + Lzma.compress(bi, bo); bo.flush(); bo.close() + + val ba = bo.toByteArray64() + var bai = 0 + val buf = IntArray(4) { Ascii85.PAD_BYTE } + ba.forEach { + if (bai > 0 && bai % 4 == 0) { + sb.append(Ascii85.encode(buf[0], buf[1], buf[2], buf[3])) + buf.fill(Ascii85.PAD_BYTE) + } + + buf[bai % 4] = it.toInt() and 255 + + bai += 1 + }; sb.append(Ascii85.encode(buf[0], buf[1], buf[2], buf[3])) + return sb.toString() } \ No newline at end of file diff --git a/src/net/torvald/terrarum/serialise/WriteMeta.kt b/src/net/torvald/terrarum/serialise/WriteMeta.kt index d0217841c..499b0e764 100644 --- a/src/net/torvald/terrarum/serialise/WriteMeta.kt +++ b/src/net/torvald/terrarum/serialise/WriteMeta.kt @@ -3,12 +3,14 @@ package net.torvald.terrarum.serialise import com.badlogic.gdx.utils.Json import com.badlogic.gdx.utils.JsonValue import com.badlogic.gdx.utils.JsonWriter +import com.badlogic.gdx.utils.compression.Lzma import net.torvald.terrarum.ModMgr import net.torvald.terrarum.gameworld.BlockLayer import net.torvald.terrarum.modulebasegame.TerrarumIngame import net.torvald.terrarum.modulebasegame.worldgenerator.RoguelikeRandomiser import net.torvald.terrarum.modulecomputers.virtualcomputer.tvd.ByteArray64GrowableOutputStream import net.torvald.terrarum.weather.WeatherMixer +import java.io.ByteArrayInputStream import java.util.zip.GZIPOutputStream /** @@ -95,5 +97,33 @@ fun bytesToZipdStr(b: ByteArray): String { bai += 1 }; sb.append(Ascii85.encode(buf[0], buf[1], buf[2], buf[3])) + return sb.toString() +} + +/** + * @param b a ByteArray + * @return Bytes in [b] which are LZMA'd then Ascii85-encoded + */ +fun bytesToLzmadStr(b: ByteArray): String { + val sb = StringBuilder() + val bi = ByteArrayInputStream(b) + val bo = ByteArray64GrowableOutputStream() + + Lzma.compress(bi, bo); bo.flush(); bo.close() + + val ba = bo.toByteArray64() + var bai = 0 + val buf = IntArray(4) { Ascii85.PAD_BYTE } + ba.forEach { + if (bai > 0 && bai % 4 == 0) { + sb.append(Ascii85.encode(buf[0], buf[1], buf[2], buf[3])) + buf.fill(Ascii85.PAD_BYTE) + } + + buf[bai % 4] = it.toInt() and 255 + + bai += 1 + }; sb.append(Ascii85.encode(buf[0], buf[1], buf[2], buf[3])) + return sb.toString() } \ No newline at end of file diff --git a/work_files/DataFormats/just-json-it-saveformat.md b/work_files/DataFormats/just-json-it-saveformat.md index e0385c14b..621897e4c 100644 --- a/work_files/DataFormats/just-json-it-saveformat.md +++ b/work_files/DataFormats/just-json-it-saveformat.md @@ -32,7 +32,7 @@ File is named as `"world"+world_index+".json"` ``` { worldname: "New World", - comp: <0 for uncompressed, 1 for GZip, 2 for LZMA>, + comp: , width: 8192, height: 2048, spawnx: 4096,