662 lines
17 KiB
JavaScript
662 lines
17 KiB
JavaScript
'use strict';
|
|
const EventEmitter = require('events');
|
|
const http = require('http');
|
|
const https = require('https');
|
|
const PassThrough = require('stream').PassThrough;
|
|
const Transform = require('stream').Transform;
|
|
const urlLib = require('url');
|
|
const fs = require('fs');
|
|
const querystring = require('querystring');
|
|
const CacheableRequest = require('cacheable-request');
|
|
const duplexer3 = require('duplexer3');
|
|
const intoStream = require('into-stream');
|
|
const is = require('@sindresorhus/is');
|
|
const getStream = require('get-stream');
|
|
const timedOut = require('timed-out');
|
|
const urlParseLax = require('url-parse-lax');
|
|
const urlToOptions = require('url-to-options');
|
|
const lowercaseKeys = require('lowercase-keys');
|
|
const decompressResponse = require('decompress-response');
|
|
const mimicResponse = require('mimic-response');
|
|
const isRetryAllowed = require('is-retry-allowed');
|
|
const isURL = require('isurl');
|
|
const PCancelable = require('p-cancelable');
|
|
const pTimeout = require('p-timeout');
|
|
const pify = require('pify');
|
|
const Buffer = require('safe-buffer').Buffer;
|
|
const pkg = require('./package.json');
|
|
const errors = require('./errors');
|
|
|
|
const getMethodRedirectCodes = new Set([300, 301, 302, 303, 304, 305, 307, 308]);
|
|
const allMethodRedirectCodes = new Set([300, 303, 307, 308]);
|
|
|
|
const isFormData = body => is.nodeStream(body) && is.function(body.getBoundary);
|
|
|
|
const getBodySize = opts => {
|
|
const body = opts.body;
|
|
|
|
if (opts.headers['content-length']) {
|
|
return Number(opts.headers['content-length']);
|
|
}
|
|
|
|
if (!body && !opts.stream) {
|
|
return 0;
|
|
}
|
|
|
|
if (is.string(body)) {
|
|
return Buffer.byteLength(body);
|
|
}
|
|
|
|
if (isFormData(body)) {
|
|
return pify(body.getLength.bind(body))();
|
|
}
|
|
|
|
if (body instanceof fs.ReadStream) {
|
|
return pify(fs.stat)(body.path).then(stat => stat.size);
|
|
}
|
|
|
|
if (is.nodeStream(body) && is.buffer(body._buffer)) {
|
|
return body._buffer.length;
|
|
}
|
|
|
|
return null;
|
|
};
|
|
|
|
function requestAsEventEmitter(opts) {
|
|
opts = opts || {};
|
|
|
|
const ee = new EventEmitter();
|
|
const requestUrl = opts.href || urlLib.resolve(urlLib.format(opts), opts.path);
|
|
const redirects = [];
|
|
const agents = is.object(opts.agent) ? opts.agent : null;
|
|
let retryCount = 0;
|
|
let redirectUrl;
|
|
let uploadBodySize;
|
|
let uploaded = 0;
|
|
|
|
const get = opts => {
|
|
if (opts.protocol !== 'http:' && opts.protocol !== 'https:') {
|
|
ee.emit('error', new got.UnsupportedProtocolError(opts));
|
|
return;
|
|
}
|
|
|
|
let fn = opts.protocol === 'https:' ? https : http;
|
|
|
|
if (agents) {
|
|
const protocolName = opts.protocol === 'https:' ? 'https' : 'http';
|
|
opts.agent = agents[protocolName] || opts.agent;
|
|
}
|
|
|
|
if (opts.useElectronNet && process.versions.electron) {
|
|
const electron = require('electron');
|
|
fn = electron.net || electron.remote.net;
|
|
}
|
|
|
|
let progressInterval;
|
|
|
|
const cacheableRequest = new CacheableRequest(fn.request, opts.cache);
|
|
const cacheReq = cacheableRequest(opts, res => {
|
|
clearInterval(progressInterval);
|
|
|
|
ee.emit('uploadProgress', {
|
|
percent: 1,
|
|
transferred: uploaded,
|
|
total: uploadBodySize
|
|
});
|
|
|
|
const statusCode = res.statusCode;
|
|
|
|
res.url = redirectUrl || requestUrl;
|
|
res.requestUrl = requestUrl;
|
|
|
|
const followRedirect = opts.followRedirect && 'location' in res.headers;
|
|
const redirectGet = followRedirect && getMethodRedirectCodes.has(statusCode);
|
|
const redirectAll = followRedirect && allMethodRedirectCodes.has(statusCode);
|
|
|
|
if (redirectAll || (redirectGet && (opts.method === 'GET' || opts.method === 'HEAD'))) {
|
|
res.resume();
|
|
|
|
if (statusCode === 303) {
|
|
// Server responded with "see other", indicating that the resource exists at another location,
|
|
// and the client should request it from that location via GET or HEAD.
|
|
opts.method = 'GET';
|
|
}
|
|
|
|
if (redirects.length >= 10) {
|
|
ee.emit('error', new got.MaxRedirectsError(statusCode, redirects, opts), null, res);
|
|
return;
|
|
}
|
|
|
|
const bufferString = Buffer.from(res.headers.location, 'binary').toString();
|
|
|
|
redirectUrl = urlLib.resolve(urlLib.format(opts), bufferString);
|
|
|
|
redirects.push(redirectUrl);
|
|
|
|
const redirectOpts = Object.assign({}, opts, urlLib.parse(redirectUrl));
|
|
|
|
ee.emit('redirect', res, redirectOpts);
|
|
|
|
get(redirectOpts);
|
|
|
|
return;
|
|
}
|
|
|
|
setImmediate(() => {
|
|
try {
|
|
getResponse(res, opts, ee, redirects);
|
|
} catch (e) {
|
|
ee.emit('error', e);
|
|
}
|
|
});
|
|
});
|
|
|
|
cacheReq.on('error', err => {
|
|
if (err instanceof CacheableRequest.RequestError) {
|
|
ee.emit('error', new got.RequestError(err, opts));
|
|
} else {
|
|
ee.emit('error', new got.CacheError(err, opts));
|
|
}
|
|
});
|
|
|
|
cacheReq.once('request', req => {
|
|
let aborted = false;
|
|
req.once('abort', _ => {
|
|
aborted = true;
|
|
});
|
|
|
|
req.once('error', err => {
|
|
clearInterval(progressInterval);
|
|
|
|
if (aborted) {
|
|
return;
|
|
}
|
|
|
|
const backoff = opts.retries(++retryCount, err);
|
|
|
|
if (backoff) {
|
|
setTimeout(get, backoff, opts);
|
|
return;
|
|
}
|
|
|
|
ee.emit('error', new got.RequestError(err, opts));
|
|
});
|
|
|
|
ee.once('request', req => {
|
|
ee.emit('uploadProgress', {
|
|
percent: 0,
|
|
transferred: 0,
|
|
total: uploadBodySize
|
|
});
|
|
|
|
const socket = req.connection;
|
|
if (socket) {
|
|
// `._connecting` was the old property which was made public in node v6.1.0
|
|
const isConnecting = socket.connecting === undefined ? socket._connecting : socket.connecting;
|
|
|
|
const onSocketConnect = () => {
|
|
const uploadEventFrequency = 150;
|
|
|
|
progressInterval = setInterval(() => {
|
|
const lastUploaded = uploaded;
|
|
const headersSize = Buffer.byteLength(req._header);
|
|
uploaded = socket.bytesWritten - headersSize;
|
|
|
|
// Prevent the known issue of `bytesWritten` being larger than body size
|
|
if (uploadBodySize && uploaded > uploadBodySize) {
|
|
uploaded = uploadBodySize;
|
|
}
|
|
|
|
// Don't emit events with unchanged progress and
|
|
// prevent last event from being emitted, because
|
|
// it's emitted when `response` is emitted
|
|
if (uploaded === lastUploaded || uploaded === uploadBodySize) {
|
|
return;
|
|
}
|
|
|
|
ee.emit('uploadProgress', {
|
|
percent: uploadBodySize ? uploaded / uploadBodySize : 0,
|
|
transferred: uploaded,
|
|
total: uploadBodySize
|
|
});
|
|
}, uploadEventFrequency);
|
|
};
|
|
|
|
// Only subscribe to 'connect' event if we're actually connecting a new
|
|
// socket, otherwise if we're already connected (because this is a
|
|
// keep-alive connection) do not bother. This is important since we won't
|
|
// get a 'connect' event for an already connected socket.
|
|
if (isConnecting) {
|
|
socket.once('connect', onSocketConnect);
|
|
} else {
|
|
onSocketConnect();
|
|
}
|
|
}
|
|
});
|
|
|
|
if (opts.gotTimeout) {
|
|
clearInterval(progressInterval);
|
|
timedOut(req, opts.gotTimeout);
|
|
}
|
|
|
|
setImmediate(() => {
|
|
ee.emit('request', req);
|
|
});
|
|
});
|
|
};
|
|
|
|
setImmediate(() => {
|
|
Promise.resolve(getBodySize(opts))
|
|
.then(size => {
|
|
uploadBodySize = size;
|
|
get(opts);
|
|
})
|
|
.catch(err => {
|
|
ee.emit('error', err);
|
|
});
|
|
});
|
|
|
|
return ee;
|
|
}
|
|
|
|
function getResponse(res, opts, ee, redirects) {
|
|
const downloadBodySize = Number(res.headers['content-length']) || null;
|
|
let downloaded = 0;
|
|
|
|
const progressStream = new Transform({
|
|
transform(chunk, encoding, callback) {
|
|
downloaded += chunk.length;
|
|
|
|
const percent = downloadBodySize ? downloaded / downloadBodySize : 0;
|
|
|
|
// Let flush() be responsible for emitting the last event
|
|
if (percent < 1) {
|
|
ee.emit('downloadProgress', {
|
|
percent,
|
|
transferred: downloaded,
|
|
total: downloadBodySize
|
|
});
|
|
}
|
|
|
|
callback(null, chunk);
|
|
},
|
|
|
|
flush(callback) {
|
|
ee.emit('downloadProgress', {
|
|
percent: 1,
|
|
transferred: downloaded,
|
|
total: downloadBodySize
|
|
});
|
|
|
|
callback();
|
|
}
|
|
});
|
|
|
|
mimicResponse(res, progressStream);
|
|
progressStream.redirectUrls = redirects;
|
|
|
|
const response = opts.decompress === true &&
|
|
is.function(decompressResponse) &&
|
|
opts.method !== 'HEAD' ? decompressResponse(progressStream) : progressStream;
|
|
|
|
if (!opts.decompress && ['gzip', 'deflate'].indexOf(res.headers['content-encoding']) !== -1) {
|
|
opts.encoding = null;
|
|
}
|
|
|
|
ee.emit('response', response);
|
|
|
|
ee.emit('downloadProgress', {
|
|
percent: 0,
|
|
transferred: 0,
|
|
total: downloadBodySize
|
|
});
|
|
|
|
res.pipe(progressStream);
|
|
}
|
|
|
|
function asPromise(opts) {
|
|
const timeoutFn = requestPromise => opts.gotTimeout && opts.gotTimeout.request ?
|
|
pTimeout(requestPromise, opts.gotTimeout.request, new got.RequestError({message: 'Request timed out', code: 'ETIMEDOUT'}, opts)) :
|
|
requestPromise;
|
|
|
|
const proxy = new EventEmitter();
|
|
|
|
const cancelable = new PCancelable((resolve, reject, onCancel) => {
|
|
const ee = requestAsEventEmitter(opts);
|
|
let cancelOnRequest = false;
|
|
|
|
onCancel(() => {
|
|
cancelOnRequest = true;
|
|
});
|
|
|
|
ee.on('request', req => {
|
|
if (cancelOnRequest) {
|
|
req.abort();
|
|
}
|
|
|
|
onCancel(() => {
|
|
req.abort();
|
|
});
|
|
|
|
if (is.nodeStream(opts.body)) {
|
|
opts.body.pipe(req);
|
|
opts.body = undefined;
|
|
return;
|
|
}
|
|
|
|
req.end(opts.body);
|
|
});
|
|
|
|
ee.on('response', res => {
|
|
const stream = is.null(opts.encoding) ? getStream.buffer(res) : getStream(res, opts);
|
|
|
|
stream
|
|
.catch(err => reject(new got.ReadError(err, opts)))
|
|
.then(data => {
|
|
const statusCode = res.statusCode;
|
|
const limitStatusCode = opts.followRedirect ? 299 : 399;
|
|
|
|
res.body = data;
|
|
|
|
if (opts.json && res.body) {
|
|
try {
|
|
res.body = JSON.parse(res.body);
|
|
} catch (err) {
|
|
if (statusCode >= 200 && statusCode < 300) {
|
|
throw new got.ParseError(err, statusCode, opts, data);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (opts.throwHttpErrors && statusCode !== 304 && (statusCode < 200 || statusCode > limitStatusCode)) {
|
|
throw new got.HTTPError(statusCode, res.statusMessage, res.headers, opts);
|
|
}
|
|
|
|
resolve(res);
|
|
})
|
|
.catch(err => {
|
|
Object.defineProperty(err, 'response', {value: res});
|
|
reject(err);
|
|
});
|
|
});
|
|
|
|
ee.once('error', reject);
|
|
ee.on('redirect', proxy.emit.bind(proxy, 'redirect'));
|
|
ee.on('uploadProgress', proxy.emit.bind(proxy, 'uploadProgress'));
|
|
ee.on('downloadProgress', proxy.emit.bind(proxy, 'downloadProgress'));
|
|
});
|
|
|
|
// Preserve backwards-compatibility
|
|
// TODO: Remove this in the next major version
|
|
Object.defineProperty(cancelable, 'canceled', {
|
|
get() {
|
|
return cancelable.isCanceled;
|
|
}
|
|
});
|
|
|
|
const promise = timeoutFn(cancelable);
|
|
|
|
promise.cancel = cancelable.cancel.bind(cancelable);
|
|
|
|
promise.on = (name, fn) => {
|
|
proxy.on(name, fn);
|
|
return promise;
|
|
};
|
|
|
|
return promise;
|
|
}
|
|
|
|
function asStream(opts) {
|
|
opts.stream = true;
|
|
|
|
const input = new PassThrough();
|
|
const output = new PassThrough();
|
|
const proxy = duplexer3(input, output);
|
|
let timeout;
|
|
|
|
if (opts.gotTimeout && opts.gotTimeout.request) {
|
|
timeout = setTimeout(() => {
|
|
proxy.emit('error', new got.RequestError({message: 'Request timed out', code: 'ETIMEDOUT'}, opts));
|
|
}, opts.gotTimeout.request);
|
|
}
|
|
|
|
if (opts.json) {
|
|
throw new Error('Got can not be used as a stream when the `json` option is used');
|
|
}
|
|
|
|
if (opts.body) {
|
|
proxy.write = () => {
|
|
throw new Error('Got\'s stream is not writable when the `body` option is used');
|
|
};
|
|
}
|
|
|
|
const ee = requestAsEventEmitter(opts);
|
|
|
|
ee.on('request', req => {
|
|
proxy.emit('request', req);
|
|
|
|
if (is.nodeStream(opts.body)) {
|
|
opts.body.pipe(req);
|
|
return;
|
|
}
|
|
|
|
if (opts.body) {
|
|
req.end(opts.body);
|
|
return;
|
|
}
|
|
|
|
if (opts.method === 'POST' || opts.method === 'PUT' || opts.method === 'PATCH') {
|
|
input.pipe(req);
|
|
return;
|
|
}
|
|
|
|
req.end();
|
|
});
|
|
|
|
ee.on('response', res => {
|
|
clearTimeout(timeout);
|
|
|
|
const statusCode = res.statusCode;
|
|
|
|
res.on('error', err => {
|
|
proxy.emit('error', new got.ReadError(err, opts));
|
|
});
|
|
|
|
res.pipe(output);
|
|
|
|
if (opts.throwHttpErrors && statusCode !== 304 && (statusCode < 200 || statusCode > 299)) {
|
|
proxy.emit('error', new got.HTTPError(statusCode, res.statusMessage, res.headers, opts), null, res);
|
|
return;
|
|
}
|
|
|
|
proxy.emit('response', res);
|
|
});
|
|
|
|
ee.on('error', proxy.emit.bind(proxy, 'error'));
|
|
ee.on('redirect', proxy.emit.bind(proxy, 'redirect'));
|
|
ee.on('uploadProgress', proxy.emit.bind(proxy, 'uploadProgress'));
|
|
ee.on('downloadProgress', proxy.emit.bind(proxy, 'downloadProgress'));
|
|
|
|
return proxy;
|
|
}
|
|
|
|
function normalizeArguments(url, opts) {
|
|
if (!is.string(url) && !is.object(url)) {
|
|
throw new TypeError(`Parameter \`url\` must be a string or object, not ${is(url)}`);
|
|
} else if (is.string(url)) {
|
|
url = url.replace(/^unix:/, 'http://$&');
|
|
|
|
try {
|
|
decodeURI(url);
|
|
} catch (err) {
|
|
throw new Error('Parameter `url` must contain valid UTF-8 character sequences');
|
|
}
|
|
|
|
url = urlParseLax(url);
|
|
if (url.auth) {
|
|
throw new Error('Basic authentication must be done with the `auth` option');
|
|
}
|
|
} else if (isURL.lenient(url)) {
|
|
url = urlToOptions(url);
|
|
}
|
|
|
|
opts = Object.assign(
|
|
{
|
|
path: '',
|
|
retries: 2,
|
|
cache: false,
|
|
decompress: true,
|
|
useElectronNet: false,
|
|
throwHttpErrors: true
|
|
},
|
|
url,
|
|
{
|
|
protocol: url.protocol || 'http:' // Override both null/undefined with default protocol
|
|
},
|
|
opts
|
|
);
|
|
|
|
const headers = lowercaseKeys(opts.headers);
|
|
for (const key of Object.keys(headers)) {
|
|
if (is.nullOrUndefined(headers[key])) {
|
|
delete headers[key];
|
|
}
|
|
}
|
|
|
|
opts.headers = Object.assign({
|
|
'user-agent': `${pkg.name}/${pkg.version} (https://github.com/sindresorhus/got)`
|
|
}, headers);
|
|
|
|
if (opts.decompress) {
|
|
opts.headers['accept-encoding'] = 'gzip,deflate';
|
|
}
|
|
|
|
const query = opts.query;
|
|
|
|
if (query) {
|
|
if (!is.string(query)) {
|
|
opts.query = querystring.stringify(query);
|
|
}
|
|
|
|
opts.path = `${opts.path.split('?')[0]}?${opts.query}`;
|
|
delete opts.query;
|
|
}
|
|
|
|
if (opts.json && is.undefined(opts.headers.accept)) {
|
|
opts.headers.accept = 'application/json';
|
|
}
|
|
|
|
const body = opts.body;
|
|
if (is.nullOrUndefined(body)) {
|
|
opts.method = (opts.method || 'GET').toUpperCase();
|
|
} else {
|
|
const headers = opts.headers;
|
|
if (!is.nodeStream(body) && !is.string(body) && !is.buffer(body) && !(opts.form || opts.json)) {
|
|
throw new TypeError('The `body` option must be a stream.Readable, string, Buffer or plain Object');
|
|
}
|
|
|
|
const canBodyBeStringified = is.plainObject(body) || is.array(body);
|
|
if ((opts.form || opts.json) && !canBodyBeStringified) {
|
|
throw new TypeError('The `body` option must be a plain Object or Array when the `form` or `json` option is used');
|
|
}
|
|
|
|
if (isFormData(body)) {
|
|
// Special case for https://github.com/form-data/form-data
|
|
headers['content-type'] = headers['content-type'] || `multipart/form-data; boundary=${body.getBoundary()}`;
|
|
} else if (opts.form && canBodyBeStringified) {
|
|
headers['content-type'] = headers['content-type'] || 'application/x-www-form-urlencoded';
|
|
opts.body = querystring.stringify(body);
|
|
} else if (opts.json && canBodyBeStringified) {
|
|
headers['content-type'] = headers['content-type'] || 'application/json';
|
|
opts.body = JSON.stringify(body);
|
|
}
|
|
|
|
if (is.undefined(headers['content-length']) && is.undefined(headers['transfer-encoding']) && !is.nodeStream(body)) {
|
|
const length = is.string(opts.body) ? Buffer.byteLength(opts.body) : opts.body.length;
|
|
headers['content-length'] = length;
|
|
}
|
|
|
|
// Convert buffer to stream to receive upload progress events
|
|
// see https://github.com/sindresorhus/got/pull/322
|
|
if (is.buffer(body)) {
|
|
opts.body = intoStream(body);
|
|
opts.body._buffer = body;
|
|
}
|
|
|
|
opts.method = (opts.method || 'POST').toUpperCase();
|
|
}
|
|
|
|
if (opts.hostname === 'unix') {
|
|
const matches = /(.+?):(.+)/.exec(opts.path);
|
|
|
|
if (matches) {
|
|
opts.socketPath = matches[1];
|
|
opts.path = matches[2];
|
|
opts.host = null;
|
|
}
|
|
}
|
|
|
|
if (!is.function(opts.retries)) {
|
|
const retries = opts.retries;
|
|
|
|
opts.retries = (iter, err) => {
|
|
if (iter > retries || !isRetryAllowed(err)) {
|
|
return 0;
|
|
}
|
|
|
|
const noise = Math.random() * 100;
|
|
|
|
return ((1 << iter) * 1000) + noise;
|
|
};
|
|
}
|
|
|
|
if (is.undefined(opts.followRedirect)) {
|
|
opts.followRedirect = true;
|
|
}
|
|
|
|
if (opts.timeout) {
|
|
if (is.number(opts.timeout)) {
|
|
opts.gotTimeout = {request: opts.timeout};
|
|
} else {
|
|
opts.gotTimeout = opts.timeout;
|
|
}
|
|
delete opts.timeout;
|
|
}
|
|
|
|
return opts;
|
|
}
|
|
|
|
function got(url, opts) {
|
|
try {
|
|
const normalizedArgs = normalizeArguments(url, opts);
|
|
|
|
if (normalizedArgs.stream) {
|
|
return asStream(normalizedArgs);
|
|
}
|
|
|
|
return asPromise(normalizedArgs);
|
|
} catch (err) {
|
|
return Promise.reject(err);
|
|
}
|
|
}
|
|
|
|
got.stream = (url, opts) => asStream(normalizeArguments(url, opts));
|
|
|
|
const methods = [
|
|
'get',
|
|
'post',
|
|
'put',
|
|
'patch',
|
|
'head',
|
|
'delete'
|
|
];
|
|
|
|
for (const method of methods) {
|
|
got[method] = (url, opts) => got(url, Object.assign({}, opts, {method}));
|
|
got.stream[method] = (url, opts) => got.stream(url, Object.assign({}, opts, {method}));
|
|
}
|
|
|
|
Object.assign(got, errors);
|
|
|
|
module.exports = got;
|