accept gzip from uplinks, fix #54

lib/up-storage.js

@@ -1,6 +1,7 @@
 var URL = require('url')
   , request = require('request')
   , Stream = require('stream')
+  , zlib = require('zlib')
   , UError = require('./error').UserError
   , mystreams = require('./streams')
   , Logger = require('./logger')
@@ -110,7 +111,8 @@ Storage.prototype.request = function(options, cb) {
 
 	var self = this
 	  , headers = options.headers || {}
-	headers.Accept = headers.Accept || 'application/json'
+	headers['Accept'] = headers['Accept'] || 'application/json'
+	headers['Accept-Encoding'] = headers['Accept-Encoding'] || 'gzip'
 	headers['User-Agent'] = headers['User-Agent'] || this.userAgent
 
 	var method = options.method || 'GET'
@@ -133,15 +135,33 @@ Storage.prototype.request = function(options, cb) {
 		body: json,
 		ca: this.ca,
 		proxy: this.proxy,
-		timeout: this.timeout
+		encoding: null,
+		timeout: this.timeout,
 	}, function(err, res, body) {
 		var error
-		if (!err) {
-			var res_length = body.length
+		var res_length = err ? 0 : body.length
 
+		do_gunzip(function() {
+			do_decode()
+			do_log()
+			if (cb) cb(err, res, body)
+		})
+
+		function do_gunzip(cb) {
+			if (err) return cb()
+			if (res.headers['content-encoding'] !== 'gzip') return cb()
+			zlib.gunzip(body, function(er, buf) {
+				if (er) err = er
+				body = buf
+				return cb()
+			})
+		}
+
+		function do_decode() {
+			if (err) return error = err.message
 			if (options.json && res.statusCode < 300) {
 				try {
-					body = JSON.parse(body)
+					body = JSON.parse(body.toString('utf8'))
 				} catch(_err) {
 					body = {}
 					err = _err
@@ -154,28 +174,27 @@ Storage.prototype.request = function(options, cb) {
 					error = body.error
 				}
 			}
-		} else {
-			error = err.message
 		}
 
-		var msg = '@{!status}, req: \'@{request.method} @{request.url}\''
-		if (error) {
-			msg += ', error: @{!error}'
-		} else {
-			msg += ', bytes: @{bytes.in}/@{bytes.out}'
-		}
-		self.logger.warn({
-			err: err,
-			request: {method: method, url: uri},
-			level: 35, // http
-			status: res != null ? res.statusCode : 'ERR',
-			error: error,
-			bytes: {
-				in: json ? json.length : 0,
-				out: res_length || 0,
+		function do_log() {
+			var msg = '@{!status}, req: \'@{request.method} @{request.url}\''
+			if (error) {
+				msg += ', error: @{!error}'
+			} else {
+				msg += ', bytes: @{bytes.in}/@{bytes.out}'
 			}
-		}, msg)
-		if (cb) cb.apply(self, arguments)
+			self.logger.warn({
+				err: err,
+				request: {method: method, url: uri},
+				level: 35, // http
+				status: res != null ? res.statusCode : 'ERR',
+				error: error,
+				bytes: {
+					in: json ? json.length : 0,
+					out: res_length || 0,
+				}
+			}, msg)
+		}
 	})
 
 	var status_called = false

test/functional/gzip.js

@@ -0,0 +1,93 @@
+require('./lib/startup')
+
+var assert = require('assert')
+  , async = require('async')
+  , crypto = require('crypto')
+
+function readfile(x) {
+	return require('fs').readFileSync(__dirname + '/' + x)
+}
+
+module.exports = function() {
+	var server = process.server
+	var express = process.express
+
+	describe('testexp_gzip', function() {
+		before(function() {
+			express.get('/testexp_gzip', function(req, res) {
+				var x = eval(
+					'(' + readfile('fixtures/publish.json5')
+						.toString('utf8')
+						.replace(/__NAME__/g, 'testexp_gzip')
+						.replace(/__VERSION__/g, '0.0.1')
+					+ ')'
+				)
+
+				// overcoming compress threshold
+				x.versions['0.0.2'] = x.versions['0.0.1']
+				x.versions['0.0.3'] = x.versions['0.0.1']
+				x.versions['0.0.4'] = x.versions['0.0.1']
+				x.versions['0.0.5'] = x.versions['0.0.1']
+				x.versions['0.0.6'] = x.versions['0.0.1']
+				x.versions['0.0.7'] = x.versions['0.0.1']
+				x.versions['0.0.8'] = x.versions['0.0.1']
+				x.versions['0.0.9'] = x.versions['0.0.1']
+
+				require('zlib').gzip(JSON.stringify(x), function(err, buf) {
+					assert(!err)
+					assert.equal(req.headers['accept-encoding'], 'gzip')
+					res.header('content-encoding', 'gzip')
+					res.send(buf)
+				})
+			})
+
+			express.get('/testexp_baddata', function(req, res) {
+				assert.equal(req.headers['accept-encoding'], 'gzip')
+				res.header('content-encoding', 'gzip')
+				res.send(new Buffer([1,2,3,4,5,6,7,7,6,5,4,3,2,1]))
+			})
+		})
+
+		it('should not fail on bad gzip', function(cb) {
+			server.get_package('testexp_baddata', function(res, body) {
+				assert.equal(res.statusCode, 404)
+				cb()
+			})
+		})
+
+		it('should understand gzipped data from uplink', function(cb) {
+			server.get_package('testexp_gzip', function(res, body) {
+				assert.equal(res.statusCode, 200)
+				assert.equal(res.headers['content-encoding'], undefined)
+				assert.equal(body.name, 'testexp_gzip')
+				assert.equal(Object.keys(body.versions).length, 9)
+				cb()
+			})
+		})
+
+		it('should serve gzipped data', function(cb) {
+			server.request({
+				uri: '/testexp_gzip',
+				encoding: null,
+				headers: {
+					'Accept-encoding': 'gzip',
+				},
+				json: false,
+			}, function(err, res, body) {
+				assert.equal(res.statusCode, 200)
+				assert.equal(res.headers['content-encoding'], 'gzip')
+				assert.throws(function() {
+					JSON.parse(body.toString('utf8'))
+				})
+				require('zlib').gunzip(body, function(err, buf) {
+					assert(!err)
+					body = JSON.parse(buf)
+					assert.equal(body.name, 'testexp_gzip')
+					assert.equal(Object.keys(body.versions).length, 9)
+					cb()
+				})
+			})
+		})
+	})
+}
+

test/functional/index.js

@@ -39,6 +39,7 @@ describe('Func', function() {
 	require('./basic')()
 	require('./gh29')()
 	require('./tags')()
+	require('./gzip')()
 	require('./incomplete')()
 	require('./mirror')()
 	require('./race')()

test/functional/lib/server.js

@@ -25,7 +25,8 @@ Server.prototype.request = function(options, cb) {
 		url: this.url + options.uri,
 		method: options.method || 'GET',
 		headers: headers,
-		json: options.json || true,
+		encoding: options.encoding,
+		json: options.json != null ? options.json : true,
 	}, cb)
 }