chore: retries improvements (#230)

* chore: retries improvements

* chore: improvements on retries

* chore: improvements on retries

* chore: add unit test

* chore: format

* chore: format

* chore: format

* fix: remove console
This commit is contained in:
Alexandro Castro 2023-10-03 15:50:52 +01:00 • committed by GitHub
parent 5246b67912
commit 22a9bc4112
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 1843 additions and 51 deletions

3
.gitignore vendored
View file

@ -1,5 +1,6 @@
.vscode .vscode
.env .env
.idea .idea
.lock deno.lock
*.sh *.sh
**/.DS_Store

12
deps.ts
View file

@ -6,7 +6,9 @@ import {
} from "https://deno.land/std@0.203.0/assert/mod.ts"; } from "https://deno.land/std@0.203.0/assert/mod.ts";
import { import {
assertSpyCalls, assertSpyCalls,
returnsNext,
spy, spy,
stub,
} from "https://deno.land/std@0.203.0/testing/mock.ts"; } from "https://deno.land/std@0.203.0/testing/mock.ts";
import { CONSTANTS } from "./src/utils.ts"; import { CONSTANTS } from "./src/utils.ts";
@ -18,4 +20,12 @@ const soxa = new ServiceProvider({
baseURL, baseURL,
}); });
export { assertEquals, assertRejects, assertSpyCalls, soxa, spy }; export {
assertEquals,
assertRejects,
assertSpyCalls,
returnsNext,
soxa,
spy,
stub,
};

BIN
src/.DS_Store vendored

Binary file not shown.

View file

@ -1,3 +1,4 @@
import { ServiceError } from "../Types/index.ts";
import { Logger } from "./Logger.ts"; import { Logger } from "./Logger.ts";
export type RetryCallbackProps = { export type RetryCallbackProps = {
@ -12,11 +13,17 @@ async function* createAsyncIterable<T>(
delay: number, delay: number,
) { ) {
for (let i = 0; i < retries; i++) { for (let i = 0; i < retries; i++) {
const isLastAttempt = i === retries - 1;
try { try {
const data = await callback({ attempt: i }); const data = await callback({ attempt: i });
yield data; yield data;
return; return;
} catch (e) { } catch (e) {
if (e instanceof ServiceError && isLastAttempt) {
yield e;
return;
}
yield null; yield null;
Logger.error(e); Logger.error(e);
await new Promise((resolve) => setTimeout(resolve, delay)); await new Promise((resolve) => setTimeout(resolve, delay));
@ -29,6 +36,7 @@ export class Retry {
async fetch<T = unknown>( async fetch<T = unknown>(
callback: callbackType<T>, callback: callbackType<T>,
) { ) {
let lastError = null;
for await ( for await (
const callbackResult of createAsyncIterable<T>( const callbackResult of createAsyncIterable<T>(
callback, callback,
@ -36,11 +44,19 @@ export class Retry {
this.retryDelay, this.retryDelay,
) )
) { ) {
if (callbackResult) { const isError = callbackResult instanceof Error;
if (callbackResult && !isError) {
return callbackResult as T; return callbackResult as T;
} }
if (isError) {
lastError = callbackResult;
}
} }
throw new Error(`Max retries (${this.maxRetries}) exceeded.`); throw new Error(`Max retries (${this.maxRetries}) exceeded.`, {
cause: lastError,
});
} }
} }

View file

@ -12,13 +12,12 @@ import {
queryUserPullRequest, queryUserPullRequest,
queryUserRepository, queryUserRepository,
} from "../Schemas/index.ts"; } from "../Schemas/index.ts";
import { soxa } from "../../deps.ts";
import { Retry } from "../Helpers/Retry.ts"; import { Retry } from "../Helpers/Retry.ts";
import { GithubError, QueryDefaultResponse } from "../Types/index.ts";
import { CONSTANTS } from "../utils.ts"; import { CONSTANTS } from "../utils.ts";
import { EServiceKindError } from "../Types/EServiceKindError.ts"; import { EServiceKindError } from "../Types/EServiceKindError.ts";
import { ServiceError } from "../Types/ServiceError.ts"; import { ServiceError } from "../Types/ServiceError.ts";
import { Logger } from "../Helpers/Logger.ts"; import { Logger } from "../Helpers/Logger.ts";
import { requestGithubData } from "./request.ts";
// Need to be here - Exporting from another file makes array of null // Need to be here - Exporting from another file makes array of null
export const TOKENS = [ export const TOKENS = [
@ -59,6 +58,7 @@ export class GithubApiService extends GithubRepository {
async requestUserInfo(username: string): Promise<UserInfo | ServiceError> { async requestUserInfo(username: string): Promise<UserInfo | ServiceError> {
// Avoid to call others if one of them is null // Avoid to call others if one of them is null
const repository = await this.requestUserRepository(username); const repository = await this.requestUserRepository(username);
if (repository instanceof ServiceError) { if (repository instanceof ServiceError) {
Logger.error(repository); Logger.error(repository);
return repository; return repository;
@ -78,7 +78,7 @@ export class GithubApiService extends GithubRepository {
if (status.includes("rejected")) { if (status.includes("rejected")) {
Logger.error(`Can not find a user with username:' ${username}'`); Logger.error(`Can not find a user with username:' ${username}'`);
return new ServiceError("not found", EServiceKindError.NOT_FOUND); return new ServiceError("Not found", EServiceKindError.NOT_FOUND);
} }
return new UserInfo( return new UserInfo(
@ -89,27 +89,6 @@ export class GithubApiService extends GithubRepository {
); );
} }
private handleError(responseErrors: GithubError[]): ServiceError {
const errors = responseErrors ?? [];
const isRateLimitExceeded = errors.some((error) =>
error.type.includes(EServiceKindError.RATE_LIMIT) ||
error.message.includes("rate limit")
);
if (isRateLimitExceeded) {
throw new ServiceError(
"Rate limit exceeded",
EServiceKindError.RATE_LIMIT,
);
}
throw new ServiceError(
"unknown error",
EServiceKindError.NOT_FOUND,
);
}
async executeQuery<T = unknown>( async executeQuery<T = unknown>(
query: string, query: string,
variables: { [key: string]: string }, variables: { [key: string]: string },
@ -120,36 +99,25 @@ export class GithubApiService extends GithubRepository {
CONSTANTS.DEFAULT_GITHUB_RETRY_DELAY, CONSTANTS.DEFAULT_GITHUB_RETRY_DELAY,
); );
const response = await retry.fetch<Promise<T>>(async ({ attempt }) => { const response = await retry.fetch<Promise<T>>(async ({ attempt }) => {
const res = await soxa.post("", {}, { return await requestGithubData(
data: { query: query, variables }, query,
headers: { variables,
Authorization: `bearer ${TOKENS[attempt]}`, TOKENS[attempt],
}, );
}); });
if (res?.data?.errors) {
return this.handleError(res?.data?.errors);
}
return res;
}) as QueryDefaultResponse<{ user: T }>;
return response?.data?.data?.user ?? return response;
new ServiceError("not found", EServiceKindError.NOT_FOUND);
} catch (error) { } catch (error) {
if (error instanceof ServiceError) { if (error.cause instanceof ServiceError) {
Logger.error(error); Logger.error(error.cause.message);
return error; return error.cause;
} }
// TODO: Move this to a logger instance later
if (error instanceof Error && error.cause) { if (error instanceof Error && error.cause) {
Logger.error(JSON.stringify(error.cause, null, 2)); Logger.error(JSON.stringify(error.cause, null, 2));
} else { } else {
Logger.error(error); Logger.error(error);
} }
return new ServiceError("not found", EServiceKindError.NOT_FOUND);
return new ServiceError(
"Rate limit exceeded",
EServiceKindError.RATE_LIMIT,
);
} }
} }
} }

View file

@ -0,0 +1,20 @@
{
"data": {
"user": null
},
"errors": [
{
"type": "NOT_FOUND",
"path": [
"user"
],
"locations": [
{
"line": 2,
"column": 5
}
],
"message": "Could not resolve to a User with the login of 'alekinho'."
}
]
}

View file

@ -0,0 +1,18 @@
{
"exceeded": {
"data": {
"documentation_url": "https://docs.github.com/en/free-pro-team@latest/rest/overview/resources-in-the-rest-api#secondary-rate-limits",
"message": "You have exceeded a secondary rate limit. Please wait a few minutes before you try again. If you reach out to GitHub Support for help, please include the request ID DBD8:FB98:31801A8:3222432:65195FDB."
}
},
"rate_limit": {
"data": {
"errors": [
{
"type": "RATE_LIMITED",
"message": "API rate limit exceeded for user ID 10711649."
}
]
}
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,145 @@
import { GithubApiService } from "../GithubApiService.ts";
import { assertEquals, returnsNext, soxa, stub } from "../../../deps.ts";
import { GitHubUserRepository } from "../../user_info.ts";
const rateLimitMock = await import("../__mocks__/rateLimitMock.json", {
assert: { type: "json" },
});
const successGithubResponseMock = await import(
"../__mocks__/successGithubResponse.json",
{ assert: { type: "json" } }
);
const notFoundGithubResponseMock = await import(
"../__mocks__/notFoundUserMock.json",
{ assert: { type: "json" } }
);
import { ServiceError } from "../../Types/index.ts";
// Unfortunatelly, The spy is a global instance
// We can't reset mock as Jest does.
stub(
soxa,
"post",
returnsNext([
// Should get data in first try
new Promise((resolve) => {
resolve(successGithubResponseMock.default);
}),
// // Should get data in second Retry
new Promise((resolve) => {
resolve(rateLimitMock.default.rate_limit);
}),
new Promise((resolve) => {
resolve(successGithubResponseMock.default);
}),
// Should throw NOT FOUND
new Promise((resolve) => {
resolve(notFoundGithubResponseMock.default);
}),
new Promise((resolve) => {
resolve(notFoundGithubResponseMock.default);
}),
// Should throw NOT FOUND even if request the user only
new Promise((resolve) => {
resolve(notFoundGithubResponseMock.default);
}),
new Promise((resolve) => {
resolve(notFoundGithubResponseMock.default);
}),
// Should throw RATE LIMIT
new Promise((resolve) => {
resolve(rateLimitMock.default.rate_limit);
}),
new Promise((resolve) => {
resolve(rateLimitMock.default.rate_limit);
}),
// Should throw RATE LIMIT Exceed
new Promise((resolve) => {
resolve(rateLimitMock.default.rate_limit);
}),
new Promise((resolve) => {
resolve(rateLimitMock.default.exceeded);
}),
]),
);
Deno.test("Should get data in first try", async () => {
const provider = new GithubApiService();
const data = await provider.requestUserRepository(
"test",
) as GitHubUserRepository;
assertEquals(data.repositories.totalCount, 128);
});
Deno.test("Should get data in second Retry", async () => {
const provider = new GithubApiService();
const data = await provider.requestUserRepository(
"test",
) as GitHubUserRepository;
assertEquals(data.repositories.totalCount, 128);
});
Deno.test("Should throw NOT FOUND", async () => {
const provider = new GithubApiService();
let error = null;
try {
error = await provider.requestUserInfo("test");
} catch (e) {
error = e;
}
assertEquals(error.code, 404);
assertEquals(error instanceof ServiceError, true);
});
Deno.test("Should throw NOT FOUND even if request the user only", async () => {
const provider = new GithubApiService();
let error = null;
try {
error = await provider.requestUserRepository("test");
} catch (e) {
error = e;
}
assertEquals(error.code, 404);
assertEquals(error instanceof ServiceError, true);
});
// The assertRejects() assertion is a little more complicated
// mainly because it deals with Promises.
// https://docs.deno.com/runtime/manual/basics/testing/assertions#throws
Deno.test("Should throw RATE LIMIT", async () => {
const provider = new GithubApiService();
let error = null;
try {
error = await provider.requestUserRepository("test");
} catch (e) {
error = e;
}
assertEquals(error.code, 419);
assertEquals(error instanceof ServiceError, true);
});
Deno.test("Should throw RATE LIMIT Exceed", async () => {
const provider = new GithubApiService();
let error = null;
try {
error = await provider.requestUserRepository("test");
} catch (e) {
error = e;
}
assertEquals(error.code, 419);
assertEquals(error instanceof ServiceError, true);
});

62
src/Services/request.ts Normal file
View file

@ -0,0 +1,62 @@
import { soxa } from "../../deps.ts";
import {
EServiceKindError,
GithubErrorResponse,
GithubExceedError,
QueryDefaultResponse,
ServiceError,
} from "../Types/index.ts";
export async function requestGithubData<T = unknown>(
query: string,
variables: { [key: string]: string },
token = "",
) {
const response = await soxa.post("", {}, {
data: { query, variables },
headers: {
Authorization: `bearer ${token}`,
},
}) as QueryDefaultResponse<{ user: T }>;
const responseData = response.data;
if (responseData?.data?.user) {
return responseData.data.user;
}
throw handleError(
responseData as unknown as GithubErrorResponse | GithubExceedError,
);
}
function handleError(
reponseErrors: GithubErrorResponse | GithubExceedError,
): ServiceError {
let isRateLimitExceeded = false;
const arrayErrors = (reponseErrors as GithubErrorResponse)?.errors || [];
const objectError = (reponseErrors as GithubExceedError) || {};
if (Array.isArray(arrayErrors)) {
isRateLimitExceeded = arrayErrors.some((error) =>
error.type.includes(EServiceKindError.RATE_LIMIT)
);
}
if (objectError?.message) {
isRateLimitExceeded = objectError?.message.includes(
"rate limit",
);
}
if (isRateLimitExceeded) {
throw new ServiceError(
"Rate limit exceeded",
EServiceKindError.RATE_LIMIT,
);
}
throw new ServiceError(
"unknown error",
EServiceKindError.NOT_FOUND,
);
}

View file

@ -3,9 +3,19 @@ export type GithubError = {
type: string; type: string;
}; };
export type GithubErrorResponse = {
errors: GithubError[];
};
export type GithubExceedError = {
documentation_url: string;
message: string;
};
export type QueryDefaultResponse<T = unknown> = { export type QueryDefaultResponse<T = unknown> = {
data: { data: {
data: T; data: T;
errors: GithubError[]; errors?: GithubErrorResponse;
message?: string;
}; };
}; };

View file

@ -3,6 +3,7 @@ import { EServiceKindError } from "./EServiceKindError.ts";
export class ServiceError extends Error { export class ServiceError extends Error {
constructor(message: string, kind: EServiceKindError) { constructor(message: string, kind: EServiceKindError) {
super(message); super(message);
this.message = message;
this.name = "ServiceError"; this.name = "ServiceError";
this.cause = kind; this.cause = kind;
} }