Little endian updates, automatic format detection (#35)

* support LE varint nbt, nbt format detection, tagname sanity checks

* sanity check reading tag types, update tests

* cleanup

* update readme, add samples

* fix zigzag size reading, api back compat

* add EOF checks, handle bedrock level.dat headers

* Fix gzipped EOF checks

* use BigIntExt, seperate 32/64bit zigzag

* fix BigIntExt

* api update

* simplify zigzag

* fix test

require protodef from git

* zigzag sizeOf fixes

* update remote

* lint

* fix long zigzag overflow

Author: extremeheat

README.md

@@ -2,11 +2,35 @@
 [![NPM version](https://img.shields.io/npm/v/prismarine-nbt.svg)](http://npmjs.com/package/prismarine-nbt)
 [![Build Status](https://github.com/PrismarineJS/prismarine-nbt/workflows/CI/badge.svg)](https://github.com/PrismarineJS/prismarine-nbt/actions?query=workflow%3A%22CI%22)
 
-Prismarine-NBT is a JavaScript parser and serializer for [NBT](http://wiki.vg/NBT) archives, for use with [Node.js](http://nodejs.org/).
+Prismarine-NBT is a JavaScript parser and serializer for [NBT](http://wiki.vg/NBT) archives, for use with [Node.js](http://nodejs.org/). It supports big, little, and little-varint encoded NBT files.
 
 
 ## Usage
 
+#### as a async promise
+
+```js
+const fs = require('fs')
+const { parse, writeUncompressed } = require('prismarine-nbt')
+
+async function main(file) {
+    const buffer = await fs.promises.readFile(file)
+    const { parsed, type } = await parse(buffer)
+    const json = JSON.stringify(result, null, 2)
+    console.log('JSON serialized:', json)
+
+    // Write it back 
+    const outBuffer = fs.createWriteStream('file.nbt')
+    const newBuf = writeUncompressed(result, type)
+    outBuffer.write(newBuf)
+    outBuffer.end(() => console.log('written!'))
+}
+
+main(process.argv[2])
+```
+
+#### as a callback
+
 ```js
 var fs = require('fs'),
     nbt = require('prismarine-nbt');
@@ -21,32 +45,42 @@ fs.readFile('bigtest.nbt', function(error, data) {
 });
 ```
 
-If the data is gzipped, it is automatically decompressed first.
+If the data is gzipped, it is automatically decompressed, for the buffer see metadata.buffer
 
 ## API
 
-### writeUncompressed(value,[isLittleEndian])
+### parse(data, [format]): Promise<{ parsed, type, metadata: { size, buffer? } }>
+### parse(data, [format,] callback)
 
-Returns a buffer with a serialized nbt `value`. If isLittleEndian is passed and is true, write little endian nbt (mcpe).
+Takes an optionally compressed `data` buffer and reads the nbt data.
 
-### parseUncompressed(data,[isLittleEndian])
+If the endian `format` is known, it can be specified as 'big', 'little' or 'littleVarint'. If not specified, the library will
+try to sequentially load as big, little and little varint until the parse is successful. The deduced type is returned as `type`.
 
-Takes a buffer `data` and returns a parsed nbt value. If isLittleEndian is passed and is true, read little endian nbt (mcpe).
+Minecraft Java Edition uses big-endian format, and Bedrock uses little-endian.
 
-### parse(data,[isLittleEndian], callback)
+### writeUncompressed(value, format='big')
+
+Returns a buffer with a serialized nbt `value`. 
+
+### parseUncompressed(data, format='big')
+
+Takes a buffer `data` and returns a parsed nbt value.
 
-Takes an optionally compressed `data` and provide a parsed nbt value in the `callback(err,value)`.
-If isLittleEndian is passed and is true, read little endian nbt (mcpe).
 
 ### simplify(nbt)
 
 Returns a simplified nbt representation : keep only the value to remove one level.
 This loses the types so you cannot use the resulting representation to write it back to nbt.
 
+### protos : { big, little, littleVarint }
+
+Provides compiled protodef instances used to parse and serialize nbt
+
 ### proto
 
-Provide the protodef instance used to parse and serialize nbt.
+Provide the big-endian protodef instance used to parse and serialize nbt.
 
 ### protoLE
 
-Provide the protodef instance used to parse and serialize little endian nbt.
+Provide the little-endian protodef instance used to parse and serialize little endian nbt.

compiler-compound.js

@@ -6,13 +6,17 @@ module.exports = {
         value: {},
         size: 0
       }
-      while (true) {
+      while (offset !== buffer.length) {
         const typ = ctx.i8(buffer, offset)
         if (typ.value === 0) {
           results.size += typ.size
           break
         }
 
+        if (typ.value > 20) {
+          throw new Error(`Invalid tag: ${typ.value} > 20`)
+        }
+
         const readResults = ctx.nbt(buffer, offset)
         offset += readResults.size
         results.size += readResults.size

compiler-tagname.js

@@ -0,0 +1,22 @@
+/* global ctx */
+function readPString (buffer, offset) {
+  const { value, size } = ctx.shortString(buffer, offset)
+  for (var c of value) {
+    if (c === '\0') throw new Error('unexpected tag end')
+  }
+  return { value, size }
+}
+
+function writePString (...args) {
+  return ctx.shortString(...args)
+}
+
+function sizeOfPString (...args) {
+  return ctx.shortString(...args)
+}
+
+module.exports = {
+  Read: { nbtTagName: ['context', readPString] },
+  Write: { nbtTagName: ['context', writePString] },
+  SizeOf: { nbtTagName: ['context', sizeOfPString] }
+}

compiler-zigzag.js

@@ -0,0 +1,125 @@
+/**
+ * Reads the length for a VarInt
+ */
+function sizeOfVarInt (value) {
+  value = (value << 1) ^ (value >> 63)
+  let cursor = 0
+  while (value & ~0x7F) {
+    value >>>= 7
+    cursor++
+  }
+  return cursor + 1
+}
+
+function sizeOfVarLong (value) {
+  if (typeof value.valueOf() === 'object') {
+    value = (BigInt(value[0]) << 32n) | BigInt(value[1])
+  } else if (typeof value !== 'bigint') value = BigInt(value)
+
+  value = (value << 1n) ^ (value >> 63n)
+  let cursor = 0
+  while (value > 127n) {
+    value >>= 7n
+    cursor++
+  }
+  return cursor + 1
+}
+
+/**
+ * Reads a zigzag encoded 64-bit VarInt as a BigInt
+ */
+function readSignedVarLong (buffer, offset) {
+  let result = BigInt(0)
+  let shift = 0n
+  let cursor = offset
+  let size = 0
+
+  while (true) {
+    if (cursor + 1 > buffer.length) { throw new Error('unexpected buffer end') }
+    const b = buffer.readUInt8(cursor)
+    result |= (BigInt(b) & 0x7fn) << shift // Add the bits to our number, except MSB
+    cursor++
+    if (!(b & 0x80)) { // If the MSB is not set, we return the number
+      size = cursor - offset
+      break
+    }
+    shift += 7n // we only have 7 bits, MSB being the return-trigger
+    if (shift > 63n) throw new Error(`varint is too big: ${shift}`)
+  }
+
+  // in zigzag encoding, the sign bit is the LSB of the value - remove the bit,
+  // if 1, then flip the rest of the bits (xor) and set to negative
+  // Note: bigint has no sign bit; instead if we XOR -0 we get no-op, XOR -1 flips and sets negative
+  const zigzag = (result >> 1n) ^ -(result & 1n)
+  return { value: zigzag, size }
+}
+
+/**
+ * Writes a zigzag encoded 64-bit VarInt as a BigInt
+ */
+function writeSignedVarLong (value, buffer, offset) {
+  // if an array, turn it into a BigInt
+  if (typeof value.valueOf() === 'object') {
+    value = BigInt.asIntN(64, (BigInt(value[0]) << 32n)) | BigInt(value[1])
+  } else if (typeof value !== 'bigint') value = BigInt(value)
+
+  // shift value left and flip if negative (no sign bit, but right shifting beyond value will always be -0b1)
+  value = (value << 1n) ^ (value >> 63n)
+  let cursor = 0
+  while (value > 127n) { // keep writing in 7 bit slices
+    const num = Number(value & 0xFFn)
+    buffer.writeUInt8(num | 0x80, offset + cursor)
+    cursor++
+    value >>= 7n
+  }
+  buffer.writeUInt8(Number(value), offset + cursor)
+  return offset + cursor + 1
+}
+
+/**
+ * Reads a 32-bit zigzag encoded varint as a Number
+ */
+function readSignedVarInt (buffer, offset) {
+  let result = 0
+  let shift = 0
+  let cursor = offset
+  let size = 0
+
+  while (true) {
+    if (cursor + 1 > buffer.length) { throw new Error('unexpected buffer end') }
+    const b = buffer.readUInt8(cursor)
+    result |= ((b & 0x7f) << shift) // Add the bits to our number, except MSB
+    cursor++
+    if (!(b & 0x80)) { // If the MSB is not set, we return the number
+      size = cursor - offset
+      break
+    }
+    shift += 7 // we only have 7 bits, MSB being the return-trigger
+    if (shift > 31) throw new Error(`varint is too big: ${shift}`)
+  }
+
+  const zigzag = ((((result << 63) >> 63) ^ result) >> 1) ^ (result & (1 << 63))
+  return { value: zigzag, size }
+}
+
+/**
+ * Writes a 32-bit zigzag encoded varint
+ */
+function writeSignedVarInt (value, buffer, offset) {
+  value = (value << 1) ^ (value >> 31)
+  let cursor = 0
+  while (value & ~0x7F) {
+    const num = Number((value & 0xFF) | 0x80)
+    buffer.writeUInt8(num, offset + cursor)
+    cursor++
+    value >>>= 7
+  }
+  buffer.writeUInt8(value, offset + cursor)
+  return offset + cursor + 1
+}
+
+module.exports = {
+  Read: { zigzag64: ['native', readSignedVarLong], zigzag32: ['native', readSignedVarInt] },
+  Write: { zigzag64: ['native', writeSignedVarLong], zigzag32: ['native', writeSignedVarInt] },
+  SizeOf: { zigzag64: ['native', sizeOfVarLong], zigzag32: ['native', sizeOfVarInt] }
+}

nbt-varint.json

@@ -0,0 +1,130 @@
+{
+  "void": "native",
+  "container": "native",
+  "i8": "native",
+  "switch": "native",
+  "compound": "native",
+  "zigzag32": "native",
+  "zigzag64": "native",
+  "i16": "native",
+  "i32": "native",
+  "i64": "native",
+  "f32": "native",
+  "f64": "native",
+  "pstring": "native",
+  "shortString": [
+    "pstring",
+    {
+      "countType": "varint"
+    }
+  ],
+  "byteArray": [
+    "array",
+    {
+      "countType": "zigzag32",
+      "type": "i8"
+    }
+  ],
+  "list": [
+    "container",
+    [
+      {
+        "name": "type",
+        "type": "nbtMapper"
+      },
+      {
+        "name": "value",
+        "type": [
+          "array",
+          {
+            "countType": "zigzag32",
+            "type": [
+              "nbtSwitch",
+              {
+                "type": "type"
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  ],
+  "intArray": [
+    "array",
+    {
+      "countType": "zigzag32",
+      "type": "i32"
+    }
+  ],
+  "longArray": [
+    "array",
+    {
+      "countType": "zigzag32",
+      "type": "i64"
+    }
+  ],
+  "nbtMapper": [
+    "mapper",
+    {
+      "type": "i8",
+      "mappings": {
+        "0": "end",
+        "1": "byte",
+        "2": "short",
+        "3": "int",
+        "4": "long",
+        "5": "float",
+        "6": "double",
+        "7": "byteArray",
+        "8": "string",
+        "9": "list",
+        "10": "compound",
+        "11": "intArray",
+        "12": "longArray"
+      }
+    }
+  ],
+  "nbtSwitch": [
+    "switch",
+    {
+      "compareTo": "$type",
+      "fields": {
+        "end": "void",
+        "byte": "i8",
+        "short": "i16",
+        "int": "zigzag32",
+        "long": "zigzag64",
+        "float": "f32",
+        "double": "f64",
+        "byteArray": "byteArray",
+        "string": "shortString",
+        "list": "list",
+        "compound": "compound",
+        "intArray": "intArray",
+        "longArray": "longArray"
+      }
+    }
+  ],
+  "nbt": [
+    "container",
+    [
+      {
+        "name": "type",
+        "type": "nbtMapper"
+      },
+      {
+        "name": "name",
+        "type": "nbtTagName"
+      },
+      {
+        "name": "value",
+        "type": [
+          "nbtSwitch",
+          {
+            "type": "type"
+          }
+        ]
+      }
+    ]
+  ]
+}