'use strict'; const EventEmitter = require('events'); const http = require('http'); const https = require('https'); const PassThrough = require('stream').PassThrough; const Transform = require('stream').Transform; const urlLib = require('url'); const fs = require('fs'); const querystring = require('querystring'); const CacheableRequest = require('cacheable-request'); const duplexer3 = require('duplexer3'); const intoStream = require('into-stream'); const is = require('@sindresorhus/is'); const getStream = require('get-stream'); const timedOut = require('timed-out'); const urlParseLax = require('url-parse-lax'); const urlToOptions = require('url-to-options'); const lowercaseKeys = require('lowercase-keys'); const decompressResponse = require('decompress-response'); const mimicResponse = require('mimic-response'); const isRetryAllowed = require('is-retry-allowed'); const isURL = require('isurl'); const PCancelable = require('p-cancelable'); const pTimeout = require('p-timeout'); const pify = require('pify'); const Buffer = require('safe-buffer').Buffer; const pkg = require('./package.json'); const errors = require('./errors'); const getMethodRedirectCodes = new Set([300, 301, 302, 303, 304, 305, 307, 308]); const allMethodRedirectCodes = new Set([300, 303, 307, 308]); const isFormData = body => is.nodeStream(body) && is.function(body.getBoundary); const getBodySize = opts => { const body = opts.body; if (opts.headers['content-length']) { return Number(opts.headers['content-length']); } if (!body && !opts.stream) { return 0; } if (is.string(body)) { return Buffer.byteLength(body); } if (isFormData(body)) { return pify(body.getLength.bind(body))(); } if (body instanceof fs.ReadStream) { return pify(fs.stat)(body.path).then(stat => stat.size); } if (is.nodeStream(body) && is.buffer(body._buffer)) { return body._buffer.length; } return null; }; function requestAsEventEmitter(opts) { opts = opts || {}; const ee = new EventEmitter(); const requestUrl = opts.href || urlLib.resolve(urlLib.format(opts), opts.path); const redirects = []; const agents = is.object(opts.agent) ? opts.agent : null; let retryCount = 0; let redirectUrl; let uploadBodySize; let uploaded = 0; const get = opts => { if (opts.protocol !== 'http:' && opts.protocol !== 'https:') { ee.emit('error', new got.UnsupportedProtocolError(opts)); return; } let fn = opts.protocol === 'https:' ? https : http; if (agents) { const protocolName = opts.protocol === 'https:' ? 'https' : 'http'; opts.agent = agents[protocolName] || opts.agent; } if (opts.useElectronNet && process.versions.electron) { const electron = require('electron'); fn = electron.net || electron.remote.net; } let progressInterval; const cacheableRequest = new CacheableRequest(fn.request, opts.cache); const cacheReq = cacheableRequest(opts, res => { clearInterval(progressInterval); ee.emit('uploadProgress', { percent: 1, transferred: uploaded, total: uploadBodySize }); const statusCode = res.statusCode; res.url = redirectUrl || requestUrl; res.requestUrl = requestUrl; const followRedirect = opts.followRedirect && 'location' in res.headers; const redirectGet = followRedirect && getMethodRedirectCodes.has(statusCode); const redirectAll = followRedirect && allMethodRedirectCodes.has(statusCode); if (redirectAll || (redirectGet && (opts.method === 'GET' || opts.method === 'HEAD'))) { res.resume(); if (statusCode === 303) { // Server responded with "see other", indicating that the resource exists at another location, // and the client should request it from that location via GET or HEAD. opts.method = 'GET'; } if (redirects.length >= 10) { ee.emit('error', new got.MaxRedirectsError(statusCode, redirects, opts), null, res); return; } const bufferString = Buffer.from(res.headers.location, 'binary').toString(); redirectUrl = urlLib.resolve(urlLib.format(opts), bufferString); redirects.push(redirectUrl); const redirectOpts = Object.assign({}, opts, urlLib.parse(redirectUrl)); ee.emit('redirect', res, redirectOpts); get(redirectOpts); return; } setImmediate(() => { try { getResponse(res, opts, ee, redirects); } catch (e) { ee.emit('error', e); } }); }); cacheReq.on('error', err => { if (err instanceof CacheableRequest.RequestError) { ee.emit('error', new got.RequestError(err, opts)); } else { ee.emit('error', new got.CacheError(err, opts)); } }); cacheReq.once('request', req => { let aborted = false; req.once('abort', _ => { aborted = true; }); req.once('error', err => { clearInterval(progressInterval); if (aborted) { return; } const backoff = opts.retries(++retryCount, err); if (backoff) { setTimeout(get, backoff, opts); return; } ee.emit('error', new got.RequestError(err, opts)); }); ee.once('request', req => { ee.emit('uploadProgress', { percent: 0, transferred: 0, total: uploadBodySize }); const socket = req.connection; if (socket) { // `._connecting` was the old property which was made public in node v6.1.0 const isConnecting = socket.connecting === undefined ? socket._connecting : socket.connecting; const onSocketConnect = () => { const uploadEventFrequency = 150; progressInterval = setInterval(() => { const lastUploaded = uploaded; const headersSize = Buffer.byteLength(req._header); uploaded = socket.bytesWritten - headersSize; // Prevent the known issue of `bytesWritten` being larger than body size if (uploadBodySize && uploaded > uploadBodySize) { uploaded = uploadBodySize; } // Don't emit events with unchanged progress and // prevent last event from being emitted, because // it's emitted when `response` is emitted if (uploaded === lastUploaded || uploaded === uploadBodySize) { return; } ee.emit('uploadProgress', { percent: uploadBodySize ? uploaded / uploadBodySize : 0, transferred: uploaded, total: uploadBodySize }); }, uploadEventFrequency); }; // Only subscribe to 'connect' event if we're actually connecting a new // socket, otherwise if we're already connected (because this is a // keep-alive connection) do not bother. This is important since we won't // get a 'connect' event for an already connected socket. if (isConnecting) { socket.once('connect', onSocketConnect); } else { onSocketConnect(); } } }); if (opts.gotTimeout) { clearInterval(progressInterval); timedOut(req, opts.gotTimeout); } setImmediate(() => { ee.emit('request', req); }); }); }; setImmediate(() => { Promise.resolve(getBodySize(opts)) .then(size => { uploadBodySize = size; get(opts); }) .catch(err => { ee.emit('error', err); }); }); return ee; } function getResponse(res, opts, ee, redirects) { const downloadBodySize = Number(res.headers['content-length']) || null; let downloaded = 0; const progressStream = new Transform({ transform(chunk, encoding, callback) { downloaded += chunk.length; const percent = downloadBodySize ? downloaded / downloadBodySize : 0; // Let flush() be responsible for emitting the last event if (percent < 1) { ee.emit('downloadProgress', { percent, transferred: downloaded, total: downloadBodySize }); } callback(null, chunk); }, flush(callback) { ee.emit('downloadProgress', { percent: 1, transferred: downloaded, total: downloadBodySize }); callback(); } }); mimicResponse(res, progressStream); progressStream.redirectUrls = redirects; const response = opts.decompress === true && is.function(decompressResponse) && opts.method !== 'HEAD' ? decompressResponse(progressStream) : progressStream; if (!opts.decompress && ['gzip', 'deflate'].indexOf(res.headers['content-encoding']) !== -1) { opts.encoding = null; } ee.emit('response', response); ee.emit('downloadProgress', { percent: 0, transferred: 0, total: downloadBodySize }); res.pipe(progressStream); } function asPromise(opts) { const timeoutFn = requestPromise => opts.gotTimeout && opts.gotTimeout.request ? pTimeout(requestPromise, opts.gotTimeout.request, new got.RequestError({message: 'Request timed out', code: 'ETIMEDOUT'}, opts)) : requestPromise; const proxy = new EventEmitter(); const cancelable = new PCancelable((resolve, reject, onCancel) => { const ee = requestAsEventEmitter(opts); let cancelOnRequest = false; onCancel(() => { cancelOnRequest = true; }); ee.on('request', req => { if (cancelOnRequest) { req.abort(); } onCancel(() => { req.abort(); }); if (is.nodeStream(opts.body)) { opts.body.pipe(req); opts.body = undefined; return; } req.end(opts.body); }); ee.on('response', res => { const stream = is.null(opts.encoding) ? getStream.buffer(res) : getStream(res, opts); stream .catch(err => reject(new got.ReadError(err, opts))) .then(data => { const statusCode = res.statusCode; const limitStatusCode = opts.followRedirect ? 299 : 399; res.body = data; if (opts.json && res.body) { try { res.body = JSON.parse(res.body); } catch (err) { if (statusCode >= 200 && statusCode < 300) { throw new got.ParseError(err, statusCode, opts, data); } } } if (opts.throwHttpErrors && statusCode !== 304 && (statusCode < 200 || statusCode > limitStatusCode)) { throw new got.HTTPError(statusCode, res.statusMessage, res.headers, opts); } resolve(res); }) .catch(err => { Object.defineProperty(err, 'response', {value: res}); reject(err); }); }); ee.once('error', reject); ee.on('redirect', proxy.emit.bind(proxy, 'redirect')); ee.on('uploadProgress', proxy.emit.bind(proxy, 'uploadProgress')); ee.on('downloadProgress', proxy.emit.bind(proxy, 'downloadProgress')); }); // Preserve backwards-compatibility // TODO: Remove this in the next major version Object.defineProperty(cancelable, 'canceled', { get() { return cancelable.isCanceled; } }); const promise = timeoutFn(cancelable); promise.cancel = cancelable.cancel.bind(cancelable); promise.on = (name, fn) => { proxy.on(name, fn); return promise; }; return promise; } function asStream(opts) { opts.stream = true; const input = new PassThrough(); const output = new PassThrough(); const proxy = duplexer3(input, output); let timeout; if (opts.gotTimeout && opts.gotTimeout.request) { timeout = setTimeout(() => { proxy.emit('error', new got.RequestError({message: 'Request timed out', code: 'ETIMEDOUT'}, opts)); }, opts.gotTimeout.request); } if (opts.json) { throw new Error('Got can not be used as a stream when the `json` option is used'); } if (opts.body) { proxy.write = () => { throw new Error('Got\'s stream is not writable when the `body` option is used'); }; } const ee = requestAsEventEmitter(opts); ee.on('request', req => { proxy.emit('request', req); if (is.nodeStream(opts.body)) { opts.body.pipe(req); return; } if (opts.body) { req.end(opts.body); return; } if (opts.method === 'POST' || opts.method === 'PUT' || opts.method === 'PATCH') { input.pipe(req); return; } req.end(); }); ee.on('response', res => { clearTimeout(timeout); const statusCode = res.statusCode; res.on('error', err => { proxy.emit('error', new got.ReadError(err, opts)); }); res.pipe(output); if (opts.throwHttpErrors && statusCode !== 304 && (statusCode < 200 || statusCode > 299)) { proxy.emit('error', new got.HTTPError(statusCode, res.statusMessage, res.headers, opts), null, res); return; } proxy.emit('response', res); }); ee.on('error', proxy.emit.bind(proxy, 'error')); ee.on('redirect', proxy.emit.bind(proxy, 'redirect')); ee.on('uploadProgress', proxy.emit.bind(proxy, 'uploadProgress')); ee.on('downloadProgress', proxy.emit.bind(proxy, 'downloadProgress')); return proxy; } function normalizeArguments(url, opts) { if (!is.string(url) && !is.object(url)) { throw new TypeError(`Parameter \`url\` must be a string or object, not ${is(url)}`); } else if (is.string(url)) { url = url.replace(/^unix:/, 'http://$&'); try { decodeURI(url); } catch (err) { throw new Error('Parameter `url` must contain valid UTF-8 character sequences'); } url = urlParseLax(url); if (url.auth) { throw new Error('Basic authentication must be done with the `auth` option'); } } else if (isURL.lenient(url)) { url = urlToOptions(url); } opts = Object.assign( { path: '', retries: 2, cache: false, decompress: true, useElectronNet: false, throwHttpErrors: true }, url, { protocol: url.protocol || 'http:' // Override both null/undefined with default protocol }, opts ); const headers = lowercaseKeys(opts.headers); for (const key of Object.keys(headers)) { if (is.nullOrUndefined(headers[key])) { delete headers[key]; } } opts.headers = Object.assign({ 'user-agent': `${pkg.name}/${pkg.version} (https://github.com/sindresorhus/got)` }, headers); if (opts.decompress) { opts.headers['accept-encoding'] = 'gzip,deflate'; } const query = opts.query; if (query) { if (!is.string(query)) { opts.query = querystring.stringify(query); } opts.path = `${opts.path.split('?')[0]}?${opts.query}`; delete opts.query; } if (opts.json && is.undefined(opts.headers.accept)) { opts.headers.accept = 'application/json'; } const body = opts.body; if (is.nullOrUndefined(body)) { opts.method = (opts.method || 'GET').toUpperCase(); } else { const headers = opts.headers; if (!is.nodeStream(body) && !is.string(body) && !is.buffer(body) && !(opts.form || opts.json)) { throw new TypeError('The `body` option must be a stream.Readable, string, Buffer or plain Object'); } const canBodyBeStringified = is.plainObject(body) || is.array(body); if ((opts.form || opts.json) && !canBodyBeStringified) { throw new TypeError('The `body` option must be a plain Object or Array when the `form` or `json` option is used'); } if (isFormData(body)) { // Special case for https://github.com/form-data/form-data headers['content-type'] = headers['content-type'] || `multipart/form-data; boundary=${body.getBoundary()}`; } else if (opts.form && canBodyBeStringified) { headers['content-type'] = headers['content-type'] || 'application/x-www-form-urlencoded'; opts.body = querystring.stringify(body); } else if (opts.json && canBodyBeStringified) { headers['content-type'] = headers['content-type'] || 'application/json'; opts.body = JSON.stringify(body); } if (is.undefined(headers['content-length']) && is.undefined(headers['transfer-encoding']) && !is.nodeStream(body)) { const length = is.string(opts.body) ? Buffer.byteLength(opts.body) : opts.body.length; headers['content-length'] = length; } // Convert buffer to stream to receive upload progress events // see https://github.com/sindresorhus/got/pull/322 if (is.buffer(body)) { opts.body = intoStream(body); opts.body._buffer = body; } opts.method = (opts.method || 'POST').toUpperCase(); } if (opts.hostname === 'unix') { const matches = /(.+?):(.+)/.exec(opts.path); if (matches) { opts.socketPath = matches[1]; opts.path = matches[2]; opts.host = null; } } if (!is.function(opts.retries)) { const retries = opts.retries; opts.retries = (iter, err) => { if (iter > retries || !isRetryAllowed(err)) { return 0; } const noise = Math.random() * 100; return ((1 << iter) * 1000) + noise; }; } if (is.undefined(opts.followRedirect)) { opts.followRedirect = true; } if (opts.timeout) { if (is.number(opts.timeout)) { opts.gotTimeout = {request: opts.timeout}; } else { opts.gotTimeout = opts.timeout; } delete opts.timeout; } return opts; } function got(url, opts) { try { const normalizedArgs = normalizeArguments(url, opts); if (normalizedArgs.stream) { return asStream(normalizedArgs); } return asPromise(normalizedArgs); } catch (err) { return Promise.reject(err); } } got.stream = (url, opts) => asStream(normalizeArguments(url, opts)); const methods = [ 'get', 'post', 'put', 'patch', 'head', 'delete' ]; for (const method of methods) { got[method] = (url, opts) => got(url, Object.assign({}, opts, {method})); got.stream[method] = (url, opts) => got.stream(url, Object.assign({}, opts, {method})); } Object.assign(got, errors); module.exports = got;