Skip to content
This repository was archived by the owner on Apr 22, 2023. It is now read-only.

Commit 6f6a979

Browse files
eendeegochrisdickinson
eendeego
authored and
chrisdickinson
committed
zlib: support concatenated gzip files
Reviewed-By: Fedor Indutny <[email protected]> PR-URL: #6442
1 parent 5b9e5bd commit 6f6a979

5 files changed

+282
-8
lines changed

‎lib/zlib.js

+8-1
Original file line numberDiff line numberDiff line change
@@ -580,14 +580,21 @@ Zlib.prototype._processChunk = function(chunk, flushFlag, cb) {
580580
self._buffer = new Buffer(self._chunkSize);
581581
}
582582

583-
if (availOutAfter === 0) {
583+
if (availOutAfter === 0 || availInAfter > 0) {
584584
// Not actually done. Need to reprocess.
585585
// Also, update the availInBefore to the availInAfter value,
586586
// so that if we have to hit it a third (fourth, etc.) time,
587587
// it'll have the correct byte counts.
588588
inOff += (availInBefore - availInAfter);
589589
availInBefore = availInAfter;
590590

591+
if (availOutAfter !== 0) {
592+
// There is still some data available for reading.
593+
// This is usually a concatenated stream, so, reset and restart.
594+
self.reset();
595+
self._offset = 0;
596+
}
597+
591598
if (!async)
592599
return true;
593600

‎src/node_zlib.cc

+24-7
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,11 @@ enum node_zlib_mode {
6363
UNZIP
6464
};
6565

66+
enum node_zlib_error {
67+
NO_ERROR,
68+
FAILED,
69+
WRITE_PENDING
70+
};
6671

6772
void InitZlib(v8::Handle<v8::Object> target);
6873

@@ -207,7 +212,7 @@ class ZCtx : public AsyncWrap {
207212
if (!async) {
208213
// sync version
209214
Process(work_req);
210-
if (CheckError(ctx))
215+
if (CheckError(ctx) == NO_ERROR)
211216
AfterSync(ctx, args);
212217
return;
213218
}
@@ -292,7 +297,7 @@ class ZCtx : public AsyncWrap {
292297
}
293298

294299

295-
static bool CheckError(ZCtx* ctx) {
300+
static node_zlib_error CheckError(ZCtx* ctx) {
296301
// Acceptable error states depend on the type of zlib stream.
297302
switch (ctx->err_) {
298303
case Z_OK:
@@ -305,14 +310,18 @@ class ZCtx : public AsyncWrap {
305310
ZCtx::Error(ctx, "Missing dictionary");
306311
else
307312
ZCtx::Error(ctx, "Bad dictionary");
308-
return false;
313+
return FAILED;
309314
default:
310315
// something else.
311-
ZCtx::Error(ctx, "Zlib error");
312-
return false;
316+
if (ctx->strm_.total_out == 0) {
317+
ZCtx::Error(ctx, "Zlib error");
318+
return FAILED;
319+
} else {
320+
return WRITE_PENDING;
321+
}
313322
}
314323

315-
return true;
324+
return NO_ERROR;
316325
}
317326

318327

@@ -326,7 +335,8 @@ class ZCtx : public AsyncWrap {
326335
HandleScope handle_scope(env->isolate());
327336
Context::Scope context_scope(env->context());
328337

329-
if (!CheckError(ctx))
338+
node_zlib_error error = CheckError(ctx);
339+
if (error == FAILED)
330340
return;
331341

332342
Local<Integer> avail_out = Integer::New(env->isolate(),
@@ -340,6 +350,11 @@ class ZCtx : public AsyncWrap {
340350
Local<Value> args[2] = { avail_in, avail_out };
341351
ctx->MakeCallback(env->callback_string(), ARRAY_SIZE(args), args);
342352

353+
if (error == WRITE_PENDING) {
354+
ZCtx::Error(ctx, "Zlib error");
355+
return;
356+
}
357+
343358
ctx->Unref();
344359
if (ctx->pending_close_)
345360
ctx->Close();
@@ -557,10 +572,12 @@ class ZCtx : public AsyncWrap {
557572
switch (ctx->mode_) {
558573
case DEFLATE:
559574
case DEFLATERAW:
575+
case GZIP:
560576
ctx->err_ = deflateReset(&ctx->strm_);
561577
break;
562578
case INFLATE:
563579
case INFLATERAW:
580+
case GUNZIP:
564581
ctx->err_ = inflateReset(&ctx->strm_);
565582
break;
566583
default:

‎test/simple/test-zlib-from-multiple-gzip-with-garbage.js

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
// Copyright Joyent, Inc. and other Node contributors.
2+
//
3+
// Permission is hereby granted, free of charge, to any person obtaining a
4+
// copy of this software and associated documentation files (the
5+
// "Software"), to deal in the Software without restriction, including
6+
// without limitation the rights to use, copy, modify, merge, publish,
7+
// distribute, sublicense, and/or sell copies of the Software, and to permit
8+
// persons to whom the Software is furnished to do so, subject to the
9+
// following conditions:
10+
//
11+
// The above copyright notice and this permission notice shall be included
12+
// in all copies or substantial portions of the Software.
13+
//
14+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15+
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16+
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
17+
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
18+
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
19+
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20+
// USE OR OTHER DEALINGS IN THE SOFTWARE.
21+
22+
// test unzipping a file that was created by concatenating multiple gzip
23+
// streams.
24+
25+
var common = require('../common');
26+
var assert = require('assert');
27+
var zlib = require('zlib');
28+
29+
var util = require('util');
30+
31+
var gzipBuffer = new Buffer(128);
32+
var gzipOffset = 0;
33+
34+
var stream1 = '123\n';
35+
var stream2 = '456\n';
36+
var stream3 = '789\n';
37+
38+
function gzipAppend(data) {
39+
data.copy(gzipBuffer, gzipOffset);
40+
gzipOffset += data.length;
41+
}
42+
43+
function writeGzipStream(text, cb) {
44+
var gzip = zlib.createGzip();
45+
gzip.on('data', gzipAppend);
46+
gzip.write(text, function() {
47+
gzip.flush(function() {
48+
gzip.end(function() {
49+
cb();
50+
});
51+
});
52+
});
53+
}
54+
55+
function writeGarbageStream(text, cb) {
56+
gzipAppend(new Buffer(text));
57+
cb();
58+
}
59+
60+
writeGzipStream(stream1, function() {
61+
writeGzipStream(stream2, function() {
62+
writeGarbageStream(stream3, function() {
63+
var gunzip = zlib.createGunzip();
64+
var gunzippedData = new Buffer(2 * 1024);
65+
var gunzippedOffset = 0;
66+
gunzip.on('data', function (data) {
67+
data.copy(gunzippedData, gunzippedOffset);
68+
gunzippedOffset += data.length;
69+
});
70+
gunzip.on('error', function() {
71+
assert.equal(gunzippedData.toString('utf8', 0, gunzippedOffset),
72+
stream1 + stream2);
73+
});
74+
gunzip.on('end', function() {
75+
assert.fail('end event not expected');
76+
});
77+
78+
gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() {
79+
gunzip.end();
80+
});
81+
});
82+
});
83+
});

‎test/simple/test-zlib-from-multiple-gzip.js

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
// Copyright Joyent, Inc. and other Node contributors.
2+
//
3+
// Permission is hereby granted, free of charge, to any person obtaining a
4+
// copy of this software and associated documentation files (the
5+
// "Software"), to deal in the Software without restriction, including
6+
// without limitation the rights to use, copy, modify, merge, publish,
7+
// distribute, sublicense, and/or sell copies of the Software, and to permit
8+
// persons to whom the Software is furnished to do so, subject to the
9+
// following conditions:
10+
//
11+
// The above copyright notice and this permission notice shall be included
12+
// in all copies or substantial portions of the Software.
13+
//
14+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15+
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16+
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
17+
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
18+
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
19+
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20+
// USE OR OTHER DEALINGS IN THE SOFTWARE.
21+
22+
// test unzipping a file that was created by concatenating multiple gzip
23+
// streams.
24+
25+
var common = require('../common');
26+
var assert = require('assert');
27+
var zlib = require('zlib');
28+
29+
var util = require('util');
30+
31+
var gzipBuffer = new Buffer(128);
32+
var gzipOffset = 0;
33+
34+
var stream1 = '123\n';
35+
var stream2 = '456\n';
36+
var stream3 = '789\n';
37+
38+
function gzipAppend(data) {
39+
data.copy(gzipBuffer, gzipOffset);
40+
gzipOffset += data.length;
41+
}
42+
43+
function writeGzipStream(text, cb) {
44+
var gzip = zlib.createGzip();
45+
gzip.on('data', gzipAppend);
46+
gzip.write(text, function() {
47+
gzip.flush(function() {
48+
gzip.end(function() {
49+
cb();
50+
});
51+
});
52+
});
53+
}
54+
55+
writeGzipStream(stream1, function() {
56+
writeGzipStream(stream2, function() {
57+
writeGzipStream(stream3, function() {
58+
var gunzip = zlib.createGunzip();
59+
var gunzippedData = new Buffer(2 * 1024);
60+
var gunzippedOffset = 0;
61+
gunzip.on('data', function (data) {
62+
data.copy(gunzippedData, gunzippedOffset);
63+
gunzippedOffset += data.length;
64+
});
65+
gunzip.on('end', function() {
66+
assert.equal(gunzippedData.toString('utf8', 0, gunzippedOffset), stream1 + stream2 + stream3);
67+
});
68+
69+
gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() {
70+
gunzip.end();
71+
});
72+
});
73+
});
74+
});

‎test/simple/test-zlib-from-multiple-huge-gzip.js

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// Copyright Joyent, Inc. and other Node contributors.
2+
//
3+
// Permission is hereby granted, free of charge, to any person obtaining a
4+
// copy of this software and associated documentation files (the
5+
// "Software"), to deal in the Software without restriction, including
6+
// without limitation the rights to use, copy, modify, merge, publish,
7+
// distribute, sublicense, and/or sell copies of the Software, and to permit
8+
// persons to whom the Software is furnished to do so, subject to the
9+
// following conditions:
10+
//
11+
// The above copyright notice and this permission notice shall be included
12+
// in all copies or substantial portions of the Software.
13+
//
14+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15+
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16+
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
17+
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
18+
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
19+
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20+
// USE OR OTHER DEALINGS IN THE SOFTWARE.
21+
22+
// test unzipping a file that was created by concatenating multiple gzip
23+
// streams.
24+
25+
var common = require('../common');
26+
var assert = require('assert');
27+
var zlib = require('zlib');
28+
29+
var util = require('util');
30+
31+
var HUGE = 64 * 1024;
32+
33+
var originalBuffer = new Buffer(3 * HUGE);
34+
var originalOffset = 0;
35+
36+
var gzipBuffer = new Buffer(3 * HUGE);
37+
var gzipOffset = 0;
38+
39+
function getRandomLetter() {
40+
return (Math.random() * (122 - 97)) + 97;
41+
}
42+
43+
function generateHugeStream() {
44+
var buffer = new Buffer(HUGE);
45+
for (var i = 0; i < HUGE; i++)
46+
buffer.writeUInt8(getRandomLetter(), i);
47+
48+
buffer.copy(originalBuffer, originalOffset);
49+
originalOffset += HUGE;
50+
51+
return buffer;
52+
}
53+
54+
function gzipAppend(data) {
55+
data.copy(gzipBuffer, gzipOffset);
56+
gzipOffset += data.length;
57+
}
58+
59+
function writeGzipStream(text, cb) {
60+
var gzip = zlib.createGzip();
61+
gzip.on('data', gzipAppend);
62+
gzip.write(text, function() {
63+
gzip.flush(function() {
64+
gzip.end(function() {
65+
cb();
66+
});
67+
});
68+
});
69+
}
70+
71+
writeGzipStream(generateHugeStream(), function() {
72+
writeGzipStream(generateHugeStream(), function() {
73+
writeGzipStream(generateHugeStream(), function() {
74+
var gunzip = zlib.createGunzip();
75+
var gunzippedData = new Buffer(3 * HUGE);
76+
var gunzippedOffset = 0;
77+
gunzip.on('data', function (data) {
78+
data.copy(gunzippedData, gunzippedOffset);
79+
gunzippedOffset += data.length;
80+
});
81+
gunzip.on('end', function() {
82+
var gunzippedStr = gunzippedData.toString('utf8', 0, gunzippedOffset);
83+
var originalStr = originalBuffer.toString('utf8', 0, 3 * HUGE);
84+
85+
assert.equal(gunzippedStr, originalStr);
86+
});
87+
88+
gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() {
89+
gunzip.end();
90+
});
91+
});
92+
});
93+
});

0 commit comments

Comments
 (0)