chore: retries improvements (#230)

* chore: retries improvements

* chore: improvements on retries

* chore: improvements on retries

* chore: add unit test

* chore: format

* chore: format

* chore: format

* fix: remove console
This commit is contained in:
Alexandro Castro 2023-10-03 15:50:52 +01:00 • committed by GitHub
parent 5246b67912
commit 22a9bc4112
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 1843 additions and 51 deletions

3
.gitignore vendored
View file

@ -1,5 +1,6 @@
.vscode
.env
.idea
.lock
deno.lock
*.sh
**/.DS_Store

12
deps.ts
View file

@ -6,7 +6,9 @@ import {
} from "https://deno.land/std@0.203.0/assert/mod.ts";
import {
assertSpyCalls,
returnsNext,
spy,
stub,
} from "https://deno.land/std@0.203.0/testing/mock.ts";
import { CONSTANTS } from "./src/utils.ts";
@ -18,4 +20,12 @@ const soxa = new ServiceProvider({
baseURL,
});
export { assertEquals, assertRejects, assertSpyCalls, soxa, spy };
export {
assertEquals,
assertRejects,
assertSpyCalls,
returnsNext,
soxa,
spy,
stub,
};

BIN
src/.DS_Store vendored

Binary file not shown.

View file

@ -1,3 +1,4 @@
import { ServiceError } from "../Types/index.ts";
import { Logger } from "./Logger.ts";
export type RetryCallbackProps = {
@ -12,11 +13,17 @@ async function* createAsyncIterable<T>(
delay: number,
) {
for (let i = 0; i < retries; i++) {
const isLastAttempt = i === retries - 1;
try {
const data = await callback({ attempt: i });
yield data;
return;
} catch (e) {
if (e instanceof ServiceError && isLastAttempt) {
yield e;
return;
}
yield null;
Logger.error(e);
await new Promise((resolve) => setTimeout(resolve, delay));
@ -29,6 +36,7 @@ export class Retry {
async fetch<T = unknown>(
callback: callbackType<T>,
) {
let lastError = null;
for await (
const callbackResult of createAsyncIterable<T>(
callback,
@ -36,11 +44,19 @@ export class Retry {
this.retryDelay,
)
) {
if (callbackResult) {
const isError = callbackResult instanceof Error;
if (callbackResult && !isError) {
return callbackResult as T;
}
if (isError) {
lastError = callbackResult;
}
}
throw new Error(`Max retries (${this.maxRetries}) exceeded.`);
throw new Error(`Max retries (${this.maxRetries}) exceeded.`, {
cause: lastError,
});
}
}

View file

@ -12,13 +12,12 @@ import {
queryUserPullRequest,
queryUserRepository,
} from "../Schemas/index.ts";
import { soxa } from "../../deps.ts";
import { Retry } from "../Helpers/Retry.ts";
import { GithubError, QueryDefaultResponse } from "../Types/index.ts";
import { CONSTANTS } from "../utils.ts";
import { EServiceKindError } from "../Types/EServiceKindError.ts";
import { ServiceError } from "../Types/ServiceError.ts";
import { Logger } from "../Helpers/Logger.ts";
import { requestGithubData } from "./request.ts";
// Need to be here - Exporting from another file makes array of null
export const TOKENS = [
@ -59,6 +58,7 @@ export class GithubApiService extends GithubRepository {
async requestUserInfo(username: string): Promise<UserInfo | ServiceError> {
// Avoid to call others if one of them is null
const repository = await this.requestUserRepository(username);
if (repository instanceof ServiceError) {
Logger.error(repository);
return repository;
@ -78,7 +78,7 @@ export class GithubApiService extends GithubRepository {
if (status.includes("rejected")) {
Logger.error(`Can not find a user with username:' ${username}'`);
return new ServiceError("not found", EServiceKindError.NOT_FOUND);
return new ServiceError("Not found", EServiceKindError.NOT_FOUND);
}
return new UserInfo(
@ -89,27 +89,6 @@ export class GithubApiService extends GithubRepository {
);
}
private handleError(responseErrors: GithubError[]): ServiceError {
const errors = responseErrors ?? [];
const isRateLimitExceeded = errors.some((error) =>
error.type.includes(EServiceKindError.RATE_LIMIT) ||
error.message.includes("rate limit")
);
if (isRateLimitExceeded) {
throw new ServiceError(
"Rate limit exceeded",
EServiceKindError.RATE_LIMIT,
);
}
throw new ServiceError(
"unknown error",
EServiceKindError.NOT_FOUND,
);
}
async executeQuery<T = unknown>(
query: string,
variables: { [key: string]: string },
@ -120,36 +99,25 @@ export class GithubApiService extends GithubRepository {
CONSTANTS.DEFAULT_GITHUB_RETRY_DELAY,
);
const response = await retry.fetch<Promise<T>>(async ({ attempt }) => {
const res = await soxa.post("", {}, {
data: { query: query, variables },
headers: {
Authorization: `bearer ${TOKENS[attempt]}`,
},
});
if (res?.data?.errors) {
return this.handleError(res?.data?.errors);
}
return res;
}) as QueryDefaultResponse<{ user: T }>;
return await requestGithubData(
query,
variables,
TOKENS[attempt],
);
});
return response?.data?.data?.user ??
new ServiceError("not found", EServiceKindError.NOT_FOUND);
return response;
} catch (error) {
if (error instanceof ServiceError) {
Logger.error(error);
return error;
if (error.cause instanceof ServiceError) {
Logger.error(error.cause.message);
return error.cause;
}
// TODO: Move this to a logger instance later
if (error instanceof Error && error.cause) {
Logger.error(JSON.stringify(error.cause, null, 2));
} else {
Logger.error(error);
}
return new ServiceError(
"Rate limit exceeded",
EServiceKindError.RATE_LIMIT,
);
return new ServiceError("not found", EServiceKindError.NOT_FOUND);
}
}
}

View file

@ -0,0 +1,20 @@
{
"data": {
"user": null
},
"errors": [
{
"type": "NOT_FOUND",
"path": [
"user"
],
"locations": [
{
"line": 2,
"column": 5
}
],
"message": "Could not resolve to a User with the login of 'alekinho'."
}
]
}

View file

@ -0,0 +1,18 @@
{
"exceeded": {
"data": {
"documentation_url": "https://docs.github.com/en/free-pro-team@latest/rest/overview/resources-in-the-rest-api#secondary-rate-limits",
"message": "You have exceeded a secondary rate limit. Please wait a few minutes before you try again. If you reach out to GitHub Support for help, please include the request ID DBD8:FB98:31801A8:3222432:65195FDB."
}
},
"rate_limit": {
"data": {
"errors": [
{
"type": "RATE_LIMITED",
"message": "API rate limit exceeded for user ID 10711649."
}
]
}
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,145 @@
import { GithubApiService } from "../GithubApiService.ts";
import { assertEquals, returnsNext, soxa, stub } from "../../../deps.ts";
import { GitHubUserRepository } from "../../user_info.ts";
const rateLimitMock = await import("../__mocks__/rateLimitMock.json", {
assert: { type: "json" },
});
const successGithubResponseMock = await import(
"../__mocks__/successGithubResponse.json",
{ assert: { type: "json" } }
);
const notFoundGithubResponseMock = await import(
"../__mocks__/notFoundUserMock.json",
{ assert: { type: "json" } }
);
import { ServiceError } from "../../Types/index.ts";
// Unfortunatelly, The spy is a global instance
// We can't reset mock as Jest does.
stub(
soxa,
"post",
returnsNext([
// Should get data in first try
new Promise((resolve) => {
resolve(successGithubResponseMock.default);
}),
// // Should get data in second Retry
new Promise((resolve) => {
resolve(rateLimitMock.default.rate_limit);
}),
new Promise((resolve) => {
resolve(successGithubResponseMock.default);
}),
// Should throw NOT FOUND
new Promise((resolve) => {
resolve(notFoundGithubResponseMock.default);
}),
new Promise((resolve) => {
resolve(notFoundGithubResponseMock.default);
}),
// Should throw NOT FOUND even if request the user only
new Promise((resolve) => {
resolve(notFoundGithubResponseMock.default);
}),
new Promise((resolve) => {
resolve(notFoundGithubResponseMock.default);
}),
// Should throw RATE LIMIT
new Promise((resolve) => {
resolve(rateLimitMock.default.rate_limit);
}),
new Promise((resolve) => {
resolve(rateLimitMock.default.rate_limit);
}),
// Should throw RATE LIMIT Exceed
new Promise((resolve) => {
resolve(rateLimitMock.default.rate_limit);
}),
new Promise((resolve) => {
resolve(rateLimitMock.default.exceeded);
}),
]),
);
Deno.test("Should get data in first try", async () => {
const provider = new GithubApiService();
const data = await provider.requestUserRepository(
"test",
) as GitHubUserRepository;
assertEquals(data.repositories.totalCount, 128);
});
Deno.test("Should get data in second Retry", async () => {
const provider = new GithubApiService();
const data = await provider.requestUserRepository(
"test",
) as GitHubUserRepository;
assertEquals(data.repositories.totalCount, 128);
});
Deno.test("Should throw NOT FOUND", async () => {
const provider = new GithubApiService();
let error = null;
try {
error = await provider.requestUserInfo("test");
} catch (e) {
error = e;
}
assertEquals(error.code, 404);
assertEquals(error instanceof ServiceError, true);
});
Deno.test("Should throw NOT FOUND even if request the user only", async () => {
const provider = new GithubApiService();
let error = null;
try {
error = await provider.requestUserRepository("test");
} catch (e) {
error = e;
}
assertEquals(error.code, 404);
assertEquals(error instanceof ServiceError, true);
});
// The assertRejects() assertion is a little more complicated
// mainly because it deals with Promises.
// https://docs.deno.com/runtime/manual/basics/testing/assertions#throws
Deno.test("Should throw RATE LIMIT", async () => {
const provider = new GithubApiService();
let error = null;
try {
error = await provider.requestUserRepository("test");
} catch (e) {
error = e;
}
assertEquals(error.code, 419);
assertEquals(error instanceof ServiceError, true);
});
Deno.test("Should throw RATE LIMIT Exceed", async () => {
const provider = new GithubApiService();
let error = null;
try {
error = await provider.requestUserRepository("test");
} catch (e) {
error = e;
}
assertEquals(error.code, 419);
assertEquals(error instanceof ServiceError, true);
});

62
src/Services/request.ts Normal file
View file

@ -0,0 +1,62 @@
import { soxa } from "../../deps.ts";
import {
EServiceKindError,
GithubErrorResponse,
GithubExceedError,
QueryDefaultResponse,
ServiceError,
} from "../Types/index.ts";
export async function requestGithubData<T = unknown>(
query: string,
variables: { [key: string]: string },
token = "",
) {
const response = await soxa.post("", {}, {
data: { query, variables },
headers: {
Authorization: `bearer ${token}`,
},
}) as QueryDefaultResponse<{ user: T }>;
const responseData = response.data;
if (responseData?.data?.user) {
return responseData.data.user;
}
throw handleError(
responseData as unknown as GithubErrorResponse | GithubExceedError,
);
}
function handleError(
reponseErrors: GithubErrorResponse | GithubExceedError,
): ServiceError {
let isRateLimitExceeded = false;
const arrayErrors = (reponseErrors as GithubErrorResponse)?.errors || [];
const objectError = (reponseErrors as GithubExceedError) || {};
if (Array.isArray(arrayErrors)) {
isRateLimitExceeded = arrayErrors.some((error) =>
error.type.includes(EServiceKindError.RATE_LIMIT)
);
}
if (objectError?.message) {
isRateLimitExceeded = objectError?.message.includes(
"rate limit",
);
}
if (isRateLimitExceeded) {
throw new ServiceError(
"Rate limit exceeded",
EServiceKindError.RATE_LIMIT,
);
}
throw new ServiceError(
"unknown error",
EServiceKindError.NOT_FOUND,
);
}

View file

@ -3,9 +3,19 @@ export type GithubError = {
type: string;
};
export type GithubErrorResponse = {
errors: GithubError[];
};
export type GithubExceedError = {
documentation_url: string;
message: string;
};
export type QueryDefaultResponse<T = unknown> = {
data: {
data: T;
errors: GithubError[];
errors?: GithubErrorResponse;
message?: string;
};
};

View file

@ -3,6 +3,7 @@ import { EServiceKindError } from "./EServiceKindError.ts";
export class ServiceError extends Error {
constructor(message: string, kind: EServiceKindError) {
super(message);
this.message = message;
this.name = "ServiceError";
this.cause = kind;
}