2.0.0 - Hole punching in Windows

Author: xori

README.md

@@ -1,6 +1,6 @@
 # `set-sparse`
 
-A simple function that sets the `FSCTL_SET_SPARSE` windows flag via `DeviceIoControl`.
+A simple function that sets the `FSCTL_SET_SPARSE` or `FSCTL_SET_ZERO_DATA` windows flag via `DeviceIoControl`.
 
 ## Why
 
@@ -15,10 +15,26 @@ _this is a file with a lot of zeros, notice the Size vs Size on disk_
 ## Usage
 
 ```js
-const setSparse = require('set-sparse')
+const sparse = require('set-sparse')
 const create_if_nonexistant = true
 
-const success = setSparse('./file-to-set', create_if_nonexistant)
+const success = sparse.setSparse('./file-to-set', create_if_nonexistant)
+// you can now write to sections of your file
+const { open } = require('fs/promises')
+const fd = await open('./file-to-set', 'r+')
+await fd.write(Buffer.from([1]), 0, 1, 10000000) // 10mb file
+fd.close() // file is *still small!*
+
+// And if we start with a dense file
+const { open, stat } = require('fs/promises')
+const fd = await open('./other-file', 'w+')
+const TEN_MB = Buffer.alloc(10000000, 'repeating-bytes')
+await fd.write(TEN_MB, 0, TEN_MB.length, 0) // 10mb file
+fd.close() // file is TEN_MB even set as sparse
+sparse.setSparse('./other-file') // still 10mb
+// punch a hole leaving 1 non-zero byte on either side
+sparse.holePunch('./other-file', 1, TEN_MB.length)
+await stat('./other-file') // reclaim all those zeros, saving space!
 ```
 
 On Mac and Linux this library is a no-op and will always return `false`.
@@ -29,6 +45,29 @@ On Mac and Linux this library is a no-op and will always return `false`.
 * Setting the sparse flag does not tell Windows to go clear up space. You should set the flag before writing data to the file.
 * Do not open the file in node with `await open(file, 'w+')` this will disable sparse mode. Use `a+` or `r+`.
 
+## API
+
+### `setSparse(filepath[, create_if_nonexistant])`
+
+* `filepath String`: a relative or absolute path.
+* `create_if_nonexistant Boolean = false`: if `setSparse` should create the file if it doesn't exist
+
+This function is synchronous but not dependent on the file's existing size and is
+generally very fast.
+
+Returns `true` if operation succeeded.
+
+### `holePunch(filepath, start_byte, beyond_final_zero)`
+
+* `filepath String`: a relative or absolute path.
+* `start_byte Number`: the starting (inclusive) byte to set to zero
+* `beyond_final_zero Number`: the ending (exclusive) byte to set to zero
+
+This function is synchronous and runtime (whilst good) is dependent on the size
+of the new hole.
+
+Returns `true` if operation succeeded.
+
 ## Development
 
 ```bash

package.json

@@ -1,10 +1,10 @@
 {
   "name": "set-sparse",
-  "version": "1.3.1",
+  "version": "2.0.0",
   "description": "A simple function that sets the FSCTL_SET_SPARSE Windows flag to save space in sparse files.",
   "main": "./src/index.js",
   "scripts": {
-    "build": "node-gyp rebuild",
+    "build": "node-gyp rebuild --release",
     "postbuild": "cp ./build/Release/sparse.node ./src/sparse.node",
     "test": "tap --comments --no-coverage -R spec"
   },

src/index.js

@@ -1,7 +1,10 @@
 const noop = () => false;
 
 try {
-  module.exports = require('./sparse.node').setSparse;
+  module.exports = require('./sparse.node');
 } catch (_) {
-  module.exports = noop;
+  module.exports = {
+    setSparse: noop,
+    holePunch: noop
+  };
 }

src/sparse.cpp

@@ -21,8 +21,28 @@ bool set_sparse(const wchar_t *path, const bool create) {
   return result;
 }
 
+bool hole_punch(const wchar_t *path, const LONGLONG start, const LONGLONG end) {
+  bool result = false;
+  HANDLE hnd = CreateFileW(path, FILE_GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL);
+  if (hnd != INVALID_HANDLE_VALUE) {
+    DWORD d;
+    LARGE_INTEGER _start;
+    _start.QuadPart = start;
+    LARGE_INTEGER _end;
+    _end.QuadPart = end;
+    FILE_ZERO_DATA_INFORMATION info;
+    info.FileOffset = _start;
+    info.BeyondFinalZero = _end;
+    if (DeviceIoControl(hnd, FSCTL_SET_ZERO_DATA, &info, sizeof(info), NULL, 0, &d, NULL)) {
+      result = true;
+    }
+    CloseHandle(hnd);
+  }
+  return result;
+}
 
-napi_value wrap_function(napi_env env, napi_callback_info info) {
+
+napi_value wrap_set_function(napi_env env, napi_callback_info info) {
   napi_value result;
   napi_get_new_target(env, info, &result);
   if (result) {
@@ -54,12 +74,54 @@ napi_value wrap_function(napi_env env, napi_callback_info info) {
   return result;
 }
 
+napi_value wrap_punch_function(napi_env env, napi_callback_info info) {
+  napi_value result;
+  napi_get_new_target(env, info, &result);
+  if (result) {
+    result = NULL;
+    napi_throw_error(env, SYB_EXP_INVAL, SYB_ERR_NOT_A_CONSTRUCTOR);
+  } else {
+    napi_value argv[3];
+    size_t argc = 3;
+    napi_get_cb_info(env, info, &argc, argv, NULL, NULL);
+    if (argc < 3) {
+      napi_throw_error(env, SYB_EXP_INVAL, SYB_ERR_WRONG_ARGUMENTS);
+    } else {
+      size_t str_len;
+      napi_value tmp;
+      napi_coerce_to_string(env, argv[0], &tmp);
+      napi_get_value_string_utf16(env, tmp, NULL, 0, &str_len);
+      str_len += 1;
+      wchar_t *path = (wchar_t*)malloc(sizeof(wchar_t) * str_len);
+      napi_get_value_string_utf16(env, tmp, (char16_t*)path, str_len, NULL);
+
+      LONGLONG start = 0;
+      LONGLONG end = 0;
+      napi_coerce_to_number(env, argv[1], &tmp);
+      napi_get_value_int64(env, tmp, &start);
+      napi_coerce_to_number(env, argv[2], &tmp);
+      napi_get_value_int64(env, tmp, &end);
+
+      if (start < 0 || end <= 0) {
+        napi_throw_error(env, SYB_EXP_INVAL, SYB_ERR_WRONG_ARGUMENTS);
+      } else {
+        napi_get_boolean(env, hole_punch(path, start, end), &result);
+      }
+      free(path);
+    }
+  }
+  return result;
+}
+
 
 napi_value init(napi_env env, napi_value exports) {
   napi_value f;
-  napi_create_function(env, NULL, 0, wrap_function, NULL, &f);
+  napi_create_function(env, NULL, 0, wrap_set_function, NULL, &f);
   napi_set_named_property(env, exports, "setSparse", f);
 
+  napi_create_function(env, NULL, 0, wrap_punch_function, NULL, &f);
+  napi_set_named_property(env, exports, "holePunch", f);
+
   return exports;
 }
 

test/holepunch.test.js

@@ -0,0 +1,107 @@
+const t = require('tap')
+const path = require('path')
+const os = require('os')
+const fs = require('fs')
+const { open, stat } = require('fs/promises')
+const fswin = require('fswin')
+const sparse = require('..')
+const assert = require('assert');
+
+const A_ONE = Buffer.from([1])
+const TEN_MB = Buffer.alloc(100000000, 'helloworld')
+let i = 0
+
+t.autoend(true)
+
+if(process.platform === 'win32') {
+
+  t.test('basic functionality', async t => {
+    let fd; // file descriptor
+
+    const _path = t.testdir({})
+    const sparseFile = path.join(_path, 'sparse-file')
+
+    sparse.setSparse(sparseFile, true)
+    fd = await open(sparseFile, 'r+')
+    await fd.write(TEN_MB, 0, TEN_MB.length, 0)
+    fd.close()
+
+    let fullsize = fswin.ntfs.getCompressedSizeSync(sparseFile)
+
+    // will leave a file with a non-zero byte at either end.
+    sparse.holePunch(sparseFile, 1, TEN_MB.length)
+
+    let sparseSize = fswin.ntfs.getCompressedSizeSync(sparseFile)
+    t.ok(
+      sparseSize <= fullsize / 2,
+      'filesize should be significantly smaller\n' +
+      `(sparse size: ${sparseSize} regular size: ${fullsize})`
+    )
+    t.end()
+  })
+
+  t.test('holepunch a basic file.', async t => {
+    const _path=t.testdir({})
+    const basicFile = path.join(_path, 'basic-file')
+
+    const fd = await open(basicFile, 'w+')
+    await fd.write(TEN_MB, 0, TEN_MB.length, 0)
+    fd.close()
+
+    sparse.setSparse(basicFile)
+    sparse.holePunch(basicFile, 1, TEN_MB.length)
+
+    let stats = fswin.ntfs.getCompressedSizeSync(basicFile)
+    t.ok(stats < TEN_MB.length, `file should be much smaller than 10mb (${stats})`)
+  })
+
+  t.test('holepunch non-existant file.', async t => {
+    const _path=t.testdir({})
+    const basicFile = path.join(_path, 'basic-file')
+
+    t.notOk(sparse.holePunch(basicFile, 1, TEN_MB.length))
+  })
+
+  t.test('handle invalid cases like', async t => {
+    const _path = t.testdir({
+      basic: 'helloworld'
+    })
+    const file = path.join(_path, 'basic')
+
+    t.test('wrong number of arguments', async t => {
+      try {
+        sparse.holePunch(file)
+        t.fail('holePunch should throw with invalid parameters')
+      } catch (err) {
+        t.pass('ensure fails 1 arg')
+      }
+      try {
+        sparse.holePunch(file, 0)
+        t.fail('holePunch should throw with invalid parameters')
+      } catch (err) {
+        t.pass('ensure fails with 2 args')
+      }
+    })
+    t.test('negative numbers', async t => {
+      try {
+        sparse.holePunch(file, -10, 1000)
+        t.fail('holePunch should throw with invalid parameters')
+      } catch (err) {
+        t.pass('ensure fails with negative numbers')
+      }
+    })
+    t.test('ranges outside of filesize', async t => {
+      let stats = await stat(file)
+      const size_before = stats.size
+      assert(size_before < 999)
+      sparse.holePunch(file, 0, 1000)
+      stats = await stat(file)
+      t.ok(stats.size === size_before, 'file should not get larger')
+    })
+  })
+
+} else {
+
+  t.notOk(sparse.holePunch('./wont-be-created', 0, 1), 'On non-windows ensure no-op works')
+
+}

test/sparse.test.js

@@ -4,23 +4,21 @@ const os = require('os')
 const fs = require('fs')
 const { open } = require('fs/promises')
 const fswin = require('fswin')
-const setSparse = require('..')
+const setSparse = require('..').setSparse
 
-const tmp = path.join(os.tmpdir(), 'set-sparse-' + process.pid + '-' + Date.now())
 const A_ONE = Buffer.from([1])
 let i = 0
 
 t.autoend(true)
-t.comment(tmp)
-fs.mkdirSync(tmp, { recursive: true })
 
 if(process.platform === 'win32') {
 
   t.test('basic functionality', async t => {
     let fd; // file descriptor
+    const _path = t.testdir({})
 
-    const regularFile = path.join(tmp, String(i++))
-    const sparseFile = path.join(tmp, String(i++))
+    const regularFile = path.join(_path, String('regular-file'))
+    const sparseFile = path.join(_path, String('sparse-file'))
     fd = await open(regularFile, 'w+')
     await fd.write(A_ONE, 0, 1, 0)
     await fd.write(A_ONE, 0, 1, 10000000)
@@ -34,21 +32,24 @@ if(process.platform === 'win32') {
 
     const sparseSize = fswin.ntfs.getCompressedSizeSync(sparseFile)
     const regularSize = fswin.ntfs.getCompressedSizeSync(regularFile)
-    t.comment('sparse size: ' + sparseSize +
-             ' regular size: ' + regularSize)
-    t.ok(sparseSize < regularSize / 2, 'sparse file should use far less blocks')
-    t.end()
+    t.ok(
+      sparseSize < regularSize / 2,
+      'sparse file should use far less blocks\n'+
+      `(sparse size: ${sparseSize} regular size: ${regularSize})`
+    )
   })
 
   t.test('creation flag', t => {
+    const _path = t.testdir({})
+
     t.test('when true', async t => {
-      const file = path.join(tmp, String(i++))
+      const file = path.join(_path, String('when-true'))
       setSparse(file, true)
       t.ok(fs.accessSync(file) === undefined, 'ensure file is created')
       t.end()
     })
     t.test('when false', async t => {
-      const file = path.join(tmp, String(i++))
+      const file = path.join(_path, String('when-false'))
       setSparse(file, false)
       try {
         fs.accessSync(file)
@@ -59,7 +60,7 @@ if(process.platform === 'win32') {
       t.end()
     })
     t.test('when undefined', async t => {
-      const file = path.join(tmp, String(i++))
+      const file = path.join(_path, String('when-undefined'))
       setSparse(file)
       try {
         fs.accessSync(file)