Skip to content

refactor: axios http client instead of request #5473

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jul 22, 2021
Merged

refactor: axios http client instead of request #5473

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
df62c66
refactor: axios http client instead of request
janoshrubos Jan 17, 2021
96ee2c0
refactor: axios proxy settings
janoshrubos Jan 18, 2021
4c351f9
Merge branch 'master' into pr/5473
rigor789 Jul 22, 2021
9a0e3f9
Update yarn.lock
rigor789 Jul 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
333 changes: 58 additions & 275 deletions lib/common/http-client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,29 +1,17 @@
import * as url from "url";
import * as _ from "lodash";
import { EOL } from "os";
import { TLSSocket } from "tls";
import * as helpers from "./helpers";
import * as zlib from "zlib";
import * as util from "util";
import * as _ from "lodash";
import { HttpStatusCodes } from "./constants";
import * as request from "request";
import {
Server,
IProxyService,
IProxySettings,
IPromiseActions,
} from "./declarations";
import { Server, IProxySettings, IProxyService } from "./declarations";
import { injector } from "./yok";
import axios from "axios";
import { HttpStatusCodes } from "./constants";
import * as tunnel from "tunnel";

export class HttpClient implements Server.IHttpClient {
private static STATUS_CODE_REGEX = /statuscode=(\d+)/i;
private static STUCK_REQUEST_ERROR_MESSAGE =
"The request can't receive any response.";
private static STUCK_RESPONSE_ERROR_MESSAGE =
"Can't receive all parts of the response.";
private static STUCK_REQUEST_TIMEOUT = 60000;
// We receive multiple response packets every ms but we don't need to be very aggressive here.
private static STUCK_RESPONSE_CHECK_INTERVAL = 10000;

private defaultUserAgent: string;

Expand All @@ -39,7 +27,11 @@ export class HttpClient implements Server.IHttpClient {
): Promise<Server.IResponse> {
try {
const result = await this.httpRequestCore(options, proxySettings);
return result;
return {
response: result,
body: result.body,
headers: result.headers,
};
} catch (err) {
if (
err.message === HttpClient.STUCK_REQUEST_ERROR_MESSAGE ||
Expand All @@ -54,7 +46,11 @@ export class HttpClient implements Server.IHttpClient {
options.url || options
);
const retryResult = await this.httpRequestCore(options, proxySettings);
return retryResult;
return {
response: retryResult,
body: retryResult.body,
headers: retryResult.headers,
};
}

throw err;
Expand All @@ -72,46 +68,20 @@ export class HttpClient implements Server.IHttpClient {
};
}

const clonedOptions = _.cloneDeep(options);

if (clonedOptions.url) {
const urlParts = url.parse(clonedOptions.url);
if (urlParts.protocol) {
clonedOptions.proto = urlParts.protocol.slice(0, -1);
}
clonedOptions.host = urlParts.hostname;
clonedOptions.port = urlParts.port;
clonedOptions.path = urlParts.path;
}

const requestProto = clonedOptions.proto || "http";
const body = clonedOptions.body;
delete clonedOptions.body;
let pipeTo = options.pipeTo; // Use the real stream because the _.cloneDeep can't clone the internal state of a stream.
delete clonedOptions.pipeTo;

const cliProxySettings = await this.$proxyService.getCache();
const requestProto = options.proto || "http";

clonedOptions.headers = clonedOptions.headers || {};
const headers = clonedOptions.headers;
options.headers = options.headers || {};
const headers = options.headers;

await this.useProxySettings(
proxySettings,
cliProxySettings,
clonedOptions,
options,
headers,
requestProto
);

if (!headers.Accept || headers.Accept.indexOf("application/json") < 0) {
if (headers.Accept) {
headers.Accept += ", ";
} else {
headers.Accept = "";
}
headers.Accept += "application/json; charset=UTF-8, */*;q=0.8";
}

if (!headers["User-Agent"]) {
if (!this.defaultUserAgent) {
//TODO: the user agent client name is also passed explicitly during login and should be kept in sync
Expand All @@ -126,208 +96,51 @@ export class HttpClient implements Server.IHttpClient {
headers["Accept-Encoding"] = "gzip,deflate";
}

const result = new Promise<Server.IResponse>((resolve, reject) => {
let timerId: NodeJS.Timer;
const cleanupRequestData: ICleanupRequestData = Object.create({
timers: [],
});

const promiseActions: IPromiseActions<Server.IResponse> = {
resolve,
reject,
isResolved: () => false,
};

clonedOptions.url =
clonedOptions.url ||
`${clonedOptions.proto}://${clonedOptions.host}${clonedOptions.path}`;
if (clonedOptions.timeout) {
timerId = setTimeout(() => {
this.setResponseResult(promiseActions, cleanupRequestData, {
err: new Error(`Request to ${clonedOptions.url} timed out.`),
});
}, clonedOptions.timeout);
cleanupRequestData.timers.push(timerId);

delete clonedOptions.timeout;
}

clonedOptions.encoding = null;
clonedOptions.followAllRedirects = true;

this.$logger.trace("httpRequest: %s", util.inspect(clonedOptions));
const requestObj = request(clonedOptions);
cleanupRequestData.req = requestObj;

requestObj
.on("error", (err: any) => {
this.$logger.trace(
"An error occurred while sending the request:",
err
);
// In case we get a 4xx error code there seems to be no better way than this regex to get the error code
// the tunnel-agent module that request is using is obscuring the response and hence the statusCode by throwing an error message
// https://github.com/request/tunnel-agent/blob/eb2b1b19e09ee0e6a2b54eb2612755731b7301dc/index.js#L166
// in case there is a better way to obtain status code in future version do not hesitate to remove this code
const errorMessageMatch = err.message.match(
HttpClient.STATUS_CODE_REGEX
);
const errorMessageStatusCode =
errorMessageMatch && errorMessageMatch[1] && +errorMessageMatch[1];
const errorMessage = this.getErrorMessage(
errorMessageStatusCode,
null
);
err.proxyAuthenticationRequired =
errorMessageStatusCode ===
HttpStatusCodes.PROXY_AUTHENTICATION_REQUIRED;
err.message = errorMessage || err.message;
this.setResponseResult(promiseActions, cleanupRequestData, { err });
})
.on("socket", (s: TLSSocket) => {
let stuckRequestTimerId: NodeJS.Timer;

stuckRequestTimerId = setTimeout(() => {
this.setResponseResult(promiseActions, cleanupRequestData, {
err: new Error(HttpClient.STUCK_REQUEST_ERROR_MESSAGE),
});
}, clonedOptions.timeout || HttpClient.STUCK_REQUEST_TIMEOUT);

cleanupRequestData.timers.push(stuckRequestTimerId);

s.once("secureConnect", () => {
clearTimeout(stuckRequestTimerId);
stuckRequestTimerId = null;
});
})
.on("response", (responseData: Server.IRequestResponseData) => {
cleanupRequestData.res = responseData;
let lastChunkTimestamp = Date.now();
cleanupRequestData.stuckResponseIntervalId = setInterval(() => {
if (
Date.now() - lastChunkTimestamp >
HttpClient.STUCK_RESPONSE_CHECK_INTERVAL
) {
this.setResponseResult(promiseActions, cleanupRequestData, {
err: new Error(HttpClient.STUCK_RESPONSE_ERROR_MESSAGE),
});
}
}, HttpClient.STUCK_RESPONSE_CHECK_INTERVAL);
const successful =
helpers.isRequestSuccessful(responseData) ||
responseData.statusCode === HttpStatusCodes.NOT_MODIFIED;
if (!successful) {
pipeTo = undefined;
}

let responseStream: any = responseData;
responseStream.on("data", (chunk: string) => {
lastChunkTimestamp = Date.now();
});
switch (responseData.headers["content-encoding"]) {
case "gzip":
responseStream = responseStream.pipe(zlib.createGunzip());
break;
case "deflate":
responseStream = responseStream.pipe(zlib.createInflate());
break;
}

if (pipeTo) {
pipeTo.on("finish", () => {
this.$logger.trace(
"httpRequest: Piping done. code = %d",
responseData.statusCode.toString()
);
this.setResponseResult(promiseActions, cleanupRequestData, {
response: responseData,
});
});

responseStream.pipe(pipeTo);
} else {
const data: string[] = [];

responseStream.on("data", (chunk: string) => {
data.push(chunk);
});

responseStream.on("end", () => {
this.$logger.trace(
"httpRequest: Done. code = %d",
responseData.statusCode.toString()
);
const responseBody = data.join("");

if (successful) {
this.setResponseResult(promiseActions, cleanupRequestData, {
body: responseBody,
response: responseData,
});
} else {
const errorMessage = this.getErrorMessage(
responseData.statusCode,
responseBody
);
const err: any = new Error(errorMessage);
err.response = responseData;
err.body = responseBody;
this.setResponseResult(promiseActions, cleanupRequestData, {
err,
});
}
});
}
});
this.$logger.trace("httpRequest: %s", util.inspect(options));

this.$logger.trace(
"httpRequest: Sending:\n%s",
this.$logger.prepare(body)
);

if (!body || !body.pipe) {
requestObj.end(body);
const agent = tunnel.httpsOverHttp({
proxy: {
host: cliProxySettings.hostname,
port: parseInt(cliProxySettings.port),
},
});
const result = await axios({
url: options.url,
headers: options.headers,
method: options.method,
proxy: false,
httpAgent: agent,
}).catch((err) => {
this.$logger.trace("An error occurred while sending the request:", err);
if (err.response) {
// The request was made and the server responded with a status code
// that falls out of the range of 2xx
const errorMessage = this.getErrorMessage(err.response.status, null);

err.proxyAuthenticationRequired =
err.response.status === HttpStatusCodes.PROXY_AUTHENTICATION_REQUIRED;

err.message = errorMessage || err.message;
} else if (err.request) {
// The request was made but no response was received
// `err.request` is an instance of XMLHttpRequest in the browser and an instance of
} else {
body.pipe(requestObj);
// Something happened in setting up the request that triggered an Error
}
throw err;
});

const response = await result;

if (helpers.isResponseRedirect(response.response)) {
const unmodifiedOptions = _.cloneDeep(options);
if (response.response.statusCode === HttpStatusCodes.SEE_OTHER) {
unmodifiedOptions.method = "GET";
}

this.$logger.trace("Begin redirected to %s", response.headers.location);
unmodifiedOptions.url = response.headers.location;
return await this.httpRequest(unmodifiedOptions);
}

return response;
}

private setResponseResult(
result: IPromiseActions<Server.IResponse>,
cleanupRequestData: ICleanupRequestData,
resultData: {
response?: Server.IRequestResponseData;
body?: string;
err?: Error;
}
): void {
this.cleanupAfterRequest(cleanupRequestData);
if (!result.isResolved()) {
result.isResolved = () => true;
if (resultData.err || !resultData.response.complete) {
return result.reject(resultData.err || new Error("Request canceled"));
}

const finalResult: any = resultData;
finalResult.headers = resultData.response.headers;
if (result) {
this.$logger.trace(
"httpRequest: Done. code = %d",
result.status.toString()
);

result.resolve(finalResult);
return {
response: result,
body: JSON.stringify(result.data),
headers: result.headers,
};
}
}

Expand Down Expand Up @@ -410,35 +223,5 @@ export class HttpClient implements Server.IHttpClient {
this.$logger.trace("Using proxy: %s", options.proxy);
}
}

private cleanupAfterRequest(data: ICleanupRequestData): void {
data.timers.forEach((t) => {
if (t) {
clearTimeout(t);
t = null;
}
});

if (data.stuckResponseIntervalId) {
clearInterval(data.stuckResponseIntervalId);
data.stuckResponseIntervalId = null;
}

if (data.req) {
data.req.abort();
}

if (data.res) {
data.res.destroy();
}
}
}

interface ICleanupRequestData {
timers: NodeJS.Timer[];
stuckResponseIntervalId: NodeJS.Timer;
req: request.Request;
res: Server.IRequestResponseData;
}

injector.register("httpClient", HttpClient);
Loading