mirror of
https://github.com/reiyua/github-profile-trophy-reiyua-mirror.git
synced 2026-11-17 06:02:09 +11:00
chore: retries improvements (#230)
* chore: retries improvements * chore: improvements on retries * chore: improvements on retries * chore: add unit test * chore: format * chore: format * chore: format * fix: remove console
This commit is contained in:
parent
5246b67912
commit
22a9bc4112
12 changed files with 1843 additions and 51 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -1,5 +1,6 @@
|
||||||
.vscode
|
.vscode
|
||||||
.env
|
.env
|
||||||
.idea
|
.idea
|
||||||
.lock
|
deno.lock
|
||||||
*.sh
|
*.sh
|
||||||
|
**/.DS_Store
|
||||||
|
|
|
||||||
12
deps.ts
12
deps.ts
|
|
@ -6,7 +6,9 @@ import {
|
||||||
} from "https://deno.land/std@0.203.0/assert/mod.ts";
|
} from "https://deno.land/std@0.203.0/assert/mod.ts";
|
||||||
import {
|
import {
|
||||||
assertSpyCalls,
|
assertSpyCalls,
|
||||||
|
returnsNext,
|
||||||
spy,
|
spy,
|
||||||
|
stub,
|
||||||
} from "https://deno.land/std@0.203.0/testing/mock.ts";
|
} from "https://deno.land/std@0.203.0/testing/mock.ts";
|
||||||
|
|
||||||
import { CONSTANTS } from "./src/utils.ts";
|
import { CONSTANTS } from "./src/utils.ts";
|
||||||
|
|
@ -18,4 +20,12 @@ const soxa = new ServiceProvider({
|
||||||
baseURL,
|
baseURL,
|
||||||
});
|
});
|
||||||
|
|
||||||
export { assertEquals, assertRejects, assertSpyCalls, soxa, spy };
|
export {
|
||||||
|
assertEquals,
|
||||||
|
assertRejects,
|
||||||
|
assertSpyCalls,
|
||||||
|
returnsNext,
|
||||||
|
soxa,
|
||||||
|
spy,
|
||||||
|
stub,
|
||||||
|
};
|
||||||
|
|
|
||||||
BIN
src/.DS_Store
vendored
BIN
src/.DS_Store
vendored
Binary file not shown.
|
|
@ -1,3 +1,4 @@
|
||||||
|
import { ServiceError } from "../Types/index.ts";
|
||||||
import { Logger } from "./Logger.ts";
|
import { Logger } from "./Logger.ts";
|
||||||
|
|
||||||
export type RetryCallbackProps = {
|
export type RetryCallbackProps = {
|
||||||
|
|
@ -12,11 +13,17 @@ async function* createAsyncIterable<T>(
|
||||||
delay: number,
|
delay: number,
|
||||||
) {
|
) {
|
||||||
for (let i = 0; i < retries; i++) {
|
for (let i = 0; i < retries; i++) {
|
||||||
|
const isLastAttempt = i === retries - 1;
|
||||||
try {
|
try {
|
||||||
const data = await callback({ attempt: i });
|
const data = await callback({ attempt: i });
|
||||||
yield data;
|
yield data;
|
||||||
return;
|
return;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
if (e instanceof ServiceError && isLastAttempt) {
|
||||||
|
yield e;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
yield null;
|
yield null;
|
||||||
Logger.error(e);
|
Logger.error(e);
|
||||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||||
|
|
@ -29,6 +36,7 @@ export class Retry {
|
||||||
async fetch<T = unknown>(
|
async fetch<T = unknown>(
|
||||||
callback: callbackType<T>,
|
callback: callbackType<T>,
|
||||||
) {
|
) {
|
||||||
|
let lastError = null;
|
||||||
for await (
|
for await (
|
||||||
const callbackResult of createAsyncIterable<T>(
|
const callbackResult of createAsyncIterable<T>(
|
||||||
callback,
|
callback,
|
||||||
|
|
@ -36,11 +44,19 @@ export class Retry {
|
||||||
this.retryDelay,
|
this.retryDelay,
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
if (callbackResult) {
|
const isError = callbackResult instanceof Error;
|
||||||
|
|
||||||
|
if (callbackResult && !isError) {
|
||||||
return callbackResult as T;
|
return callbackResult as T;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (isError) {
|
||||||
|
lastError = callbackResult;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
throw new Error(`Max retries (${this.maxRetries}) exceeded.`);
|
throw new Error(`Max retries (${this.maxRetries}) exceeded.`, {
|
||||||
|
cause: lastError,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -12,13 +12,12 @@ import {
|
||||||
queryUserPullRequest,
|
queryUserPullRequest,
|
||||||
queryUserRepository,
|
queryUserRepository,
|
||||||
} from "../Schemas/index.ts";
|
} from "../Schemas/index.ts";
|
||||||
import { soxa } from "../../deps.ts";
|
|
||||||
import { Retry } from "../Helpers/Retry.ts";
|
import { Retry } from "../Helpers/Retry.ts";
|
||||||
import { GithubError, QueryDefaultResponse } from "../Types/index.ts";
|
|
||||||
import { CONSTANTS } from "../utils.ts";
|
import { CONSTANTS } from "../utils.ts";
|
||||||
import { EServiceKindError } from "../Types/EServiceKindError.ts";
|
import { EServiceKindError } from "../Types/EServiceKindError.ts";
|
||||||
import { ServiceError } from "../Types/ServiceError.ts";
|
import { ServiceError } from "../Types/ServiceError.ts";
|
||||||
import { Logger } from "../Helpers/Logger.ts";
|
import { Logger } from "../Helpers/Logger.ts";
|
||||||
|
import { requestGithubData } from "./request.ts";
|
||||||
|
|
||||||
// Need to be here - Exporting from another file makes array of null
|
// Need to be here - Exporting from another file makes array of null
|
||||||
export const TOKENS = [
|
export const TOKENS = [
|
||||||
|
|
@ -59,6 +58,7 @@ export class GithubApiService extends GithubRepository {
|
||||||
async requestUserInfo(username: string): Promise<UserInfo | ServiceError> {
|
async requestUserInfo(username: string): Promise<UserInfo | ServiceError> {
|
||||||
// Avoid to call others if one of them is null
|
// Avoid to call others if one of them is null
|
||||||
const repository = await this.requestUserRepository(username);
|
const repository = await this.requestUserRepository(username);
|
||||||
|
|
||||||
if (repository instanceof ServiceError) {
|
if (repository instanceof ServiceError) {
|
||||||
Logger.error(repository);
|
Logger.error(repository);
|
||||||
return repository;
|
return repository;
|
||||||
|
|
@ -78,7 +78,7 @@ export class GithubApiService extends GithubRepository {
|
||||||
|
|
||||||
if (status.includes("rejected")) {
|
if (status.includes("rejected")) {
|
||||||
Logger.error(`Can not find a user with username:' ${username}'`);
|
Logger.error(`Can not find a user with username:' ${username}'`);
|
||||||
return new ServiceError("not found", EServiceKindError.NOT_FOUND);
|
return new ServiceError("Not found", EServiceKindError.NOT_FOUND);
|
||||||
}
|
}
|
||||||
|
|
||||||
return new UserInfo(
|
return new UserInfo(
|
||||||
|
|
@ -89,27 +89,6 @@ export class GithubApiService extends GithubRepository {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private handleError(responseErrors: GithubError[]): ServiceError {
|
|
||||||
const errors = responseErrors ?? [];
|
|
||||||
|
|
||||||
const isRateLimitExceeded = errors.some((error) =>
|
|
||||||
error.type.includes(EServiceKindError.RATE_LIMIT) ||
|
|
||||||
error.message.includes("rate limit")
|
|
||||||
);
|
|
||||||
|
|
||||||
if (isRateLimitExceeded) {
|
|
||||||
throw new ServiceError(
|
|
||||||
"Rate limit exceeded",
|
|
||||||
EServiceKindError.RATE_LIMIT,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new ServiceError(
|
|
||||||
"unknown error",
|
|
||||||
EServiceKindError.NOT_FOUND,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async executeQuery<T = unknown>(
|
async executeQuery<T = unknown>(
|
||||||
query: string,
|
query: string,
|
||||||
variables: { [key: string]: string },
|
variables: { [key: string]: string },
|
||||||
|
|
@ -120,36 +99,25 @@ export class GithubApiService extends GithubRepository {
|
||||||
CONSTANTS.DEFAULT_GITHUB_RETRY_DELAY,
|
CONSTANTS.DEFAULT_GITHUB_RETRY_DELAY,
|
||||||
);
|
);
|
||||||
const response = await retry.fetch<Promise<T>>(async ({ attempt }) => {
|
const response = await retry.fetch<Promise<T>>(async ({ attempt }) => {
|
||||||
const res = await soxa.post("", {}, {
|
return await requestGithubData(
|
||||||
data: { query: query, variables },
|
query,
|
||||||
headers: {
|
variables,
|
||||||
Authorization: `bearer ${TOKENS[attempt]}`,
|
TOKENS[attempt],
|
||||||
},
|
);
|
||||||
});
|
});
|
||||||
if (res?.data?.errors) {
|
|
||||||
return this.handleError(res?.data?.errors);
|
|
||||||
}
|
|
||||||
return res;
|
|
||||||
}) as QueryDefaultResponse<{ user: T }>;
|
|
||||||
|
|
||||||
return response?.data?.data?.user ??
|
return response;
|
||||||
new ServiceError("not found", EServiceKindError.NOT_FOUND);
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof ServiceError) {
|
if (error.cause instanceof ServiceError) {
|
||||||
Logger.error(error);
|
Logger.error(error.cause.message);
|
||||||
return error;
|
return error.cause;
|
||||||
}
|
}
|
||||||
// TODO: Move this to a logger instance later
|
|
||||||
if (error instanceof Error && error.cause) {
|
if (error instanceof Error && error.cause) {
|
||||||
Logger.error(JSON.stringify(error.cause, null, 2));
|
Logger.error(JSON.stringify(error.cause, null, 2));
|
||||||
} else {
|
} else {
|
||||||
Logger.error(error);
|
Logger.error(error);
|
||||||
}
|
}
|
||||||
|
return new ServiceError("not found", EServiceKindError.NOT_FOUND);
|
||||||
return new ServiceError(
|
|
||||||
"Rate limit exceeded",
|
|
||||||
EServiceKindError.RATE_LIMIT,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
20
src/Services/__mocks__/notFoundUserMock.json
Normal file
20
src/Services/__mocks__/notFoundUserMock.json
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"user": null
|
||||||
|
},
|
||||||
|
"errors": [
|
||||||
|
{
|
||||||
|
"type": "NOT_FOUND",
|
||||||
|
"path": [
|
||||||
|
"user"
|
||||||
|
],
|
||||||
|
"locations": [
|
||||||
|
{
|
||||||
|
"line": 2,
|
||||||
|
"column": 5
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"message": "Could not resolve to a User with the login of 'alekinho'."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
18
src/Services/__mocks__/rateLimitMock.json
Normal file
18
src/Services/__mocks__/rateLimitMock.json
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
{
|
||||||
|
"exceeded": {
|
||||||
|
"data": {
|
||||||
|
"documentation_url": "https://docs.github.com/en/free-pro-team@latest/rest/overview/resources-in-the-rest-api#secondary-rate-limits",
|
||||||
|
"message": "You have exceeded a secondary rate limit. Please wait a few minutes before you try again. If you reach out to GitHub Support for help, please include the request ID DBD8:FB98:31801A8:3222432:65195FDB."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"rate_limit": {
|
||||||
|
"data": {
|
||||||
|
"errors": [
|
||||||
|
{
|
||||||
|
"type": "RATE_LIMITED",
|
||||||
|
"message": "API rate limit exceeded for user ID 10711649."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
1541
src/Services/__mocks__/successGithubResponse.json
Normal file
1541
src/Services/__mocks__/successGithubResponse.json
Normal file
File diff suppressed because it is too large
Load diff
145
src/Services/__tests__/githubApiService.test.ts
Normal file
145
src/Services/__tests__/githubApiService.test.ts
Normal file
|
|
@ -0,0 +1,145 @@
|
||||||
|
import { GithubApiService } from "../GithubApiService.ts";
|
||||||
|
import { assertEquals, returnsNext, soxa, stub } from "../../../deps.ts";
|
||||||
|
import { GitHubUserRepository } from "../../user_info.ts";
|
||||||
|
|
||||||
|
const rateLimitMock = await import("../__mocks__/rateLimitMock.json", {
|
||||||
|
assert: { type: "json" },
|
||||||
|
});
|
||||||
|
|
||||||
|
const successGithubResponseMock = await import(
|
||||||
|
"../__mocks__/successGithubResponse.json",
|
||||||
|
{ assert: { type: "json" } }
|
||||||
|
);
|
||||||
|
|
||||||
|
const notFoundGithubResponseMock = await import(
|
||||||
|
"../__mocks__/notFoundUserMock.json",
|
||||||
|
{ assert: { type: "json" } }
|
||||||
|
);
|
||||||
|
|
||||||
|
import { ServiceError } from "../../Types/index.ts";
|
||||||
|
|
||||||
|
// Unfortunatelly, The spy is a global instance
|
||||||
|
// We can't reset mock as Jest does.
|
||||||
|
stub(
|
||||||
|
soxa,
|
||||||
|
"post",
|
||||||
|
returnsNext([
|
||||||
|
// Should get data in first try
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolve(successGithubResponseMock.default);
|
||||||
|
}),
|
||||||
|
// // Should get data in second Retry
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolve(rateLimitMock.default.rate_limit);
|
||||||
|
}),
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolve(successGithubResponseMock.default);
|
||||||
|
}),
|
||||||
|
// Should throw NOT FOUND
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolve(notFoundGithubResponseMock.default);
|
||||||
|
}),
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolve(notFoundGithubResponseMock.default);
|
||||||
|
}),
|
||||||
|
// Should throw NOT FOUND even if request the user only
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolve(notFoundGithubResponseMock.default);
|
||||||
|
}),
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolve(notFoundGithubResponseMock.default);
|
||||||
|
}),
|
||||||
|
// Should throw RATE LIMIT
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolve(rateLimitMock.default.rate_limit);
|
||||||
|
}),
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolve(rateLimitMock.default.rate_limit);
|
||||||
|
}),
|
||||||
|
// Should throw RATE LIMIT Exceed
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolve(rateLimitMock.default.rate_limit);
|
||||||
|
}),
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolve(rateLimitMock.default.exceeded);
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
|
||||||
|
Deno.test("Should get data in first try", async () => {
|
||||||
|
const provider = new GithubApiService();
|
||||||
|
|
||||||
|
const data = await provider.requestUserRepository(
|
||||||
|
"test",
|
||||||
|
) as GitHubUserRepository;
|
||||||
|
|
||||||
|
assertEquals(data.repositories.totalCount, 128);
|
||||||
|
});
|
||||||
|
|
||||||
|
Deno.test("Should get data in second Retry", async () => {
|
||||||
|
const provider = new GithubApiService();
|
||||||
|
|
||||||
|
const data = await provider.requestUserRepository(
|
||||||
|
"test",
|
||||||
|
) as GitHubUserRepository;
|
||||||
|
|
||||||
|
assertEquals(data.repositories.totalCount, 128);
|
||||||
|
});
|
||||||
|
|
||||||
|
Deno.test("Should throw NOT FOUND", async () => {
|
||||||
|
const provider = new GithubApiService();
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
error = await provider.requestUserInfo("test");
|
||||||
|
} catch (e) {
|
||||||
|
error = e;
|
||||||
|
}
|
||||||
|
|
||||||
|
assertEquals(error.code, 404);
|
||||||
|
assertEquals(error instanceof ServiceError, true);
|
||||||
|
});
|
||||||
|
Deno.test("Should throw NOT FOUND even if request the user only", async () => {
|
||||||
|
const provider = new GithubApiService();
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
error = await provider.requestUserRepository("test");
|
||||||
|
} catch (e) {
|
||||||
|
error = e;
|
||||||
|
}
|
||||||
|
|
||||||
|
assertEquals(error.code, 404);
|
||||||
|
assertEquals(error instanceof ServiceError, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
// The assertRejects() assertion is a little more complicated
|
||||||
|
// mainly because it deals with Promises.
|
||||||
|
// https://docs.deno.com/runtime/manual/basics/testing/assertions#throws
|
||||||
|
Deno.test("Should throw RATE LIMIT", async () => {
|
||||||
|
const provider = new GithubApiService();
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
error = await provider.requestUserRepository("test");
|
||||||
|
} catch (e) {
|
||||||
|
error = e;
|
||||||
|
}
|
||||||
|
|
||||||
|
assertEquals(error.code, 419);
|
||||||
|
assertEquals(error instanceof ServiceError, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
Deno.test("Should throw RATE LIMIT Exceed", async () => {
|
||||||
|
const provider = new GithubApiService();
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
error = await provider.requestUserRepository("test");
|
||||||
|
} catch (e) {
|
||||||
|
error = e;
|
||||||
|
}
|
||||||
|
|
||||||
|
assertEquals(error.code, 419);
|
||||||
|
assertEquals(error instanceof ServiceError, true);
|
||||||
|
});
|
||||||
62
src/Services/request.ts
Normal file
62
src/Services/request.ts
Normal file
|
|
@ -0,0 +1,62 @@
|
||||||
|
import { soxa } from "../../deps.ts";
|
||||||
|
import {
|
||||||
|
EServiceKindError,
|
||||||
|
GithubErrorResponse,
|
||||||
|
GithubExceedError,
|
||||||
|
QueryDefaultResponse,
|
||||||
|
ServiceError,
|
||||||
|
} from "../Types/index.ts";
|
||||||
|
|
||||||
|
export async function requestGithubData<T = unknown>(
|
||||||
|
query: string,
|
||||||
|
variables: { [key: string]: string },
|
||||||
|
token = "",
|
||||||
|
) {
|
||||||
|
const response = await soxa.post("", {}, {
|
||||||
|
data: { query, variables },
|
||||||
|
headers: {
|
||||||
|
Authorization: `bearer ${token}`,
|
||||||
|
},
|
||||||
|
}) as QueryDefaultResponse<{ user: T }>;
|
||||||
|
const responseData = response.data;
|
||||||
|
|
||||||
|
if (responseData?.data?.user) {
|
||||||
|
return responseData.data.user;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw handleError(
|
||||||
|
responseData as unknown as GithubErrorResponse | GithubExceedError,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function handleError(
|
||||||
|
reponseErrors: GithubErrorResponse | GithubExceedError,
|
||||||
|
): ServiceError {
|
||||||
|
let isRateLimitExceeded = false;
|
||||||
|
const arrayErrors = (reponseErrors as GithubErrorResponse)?.errors || [];
|
||||||
|
const objectError = (reponseErrors as GithubExceedError) || {};
|
||||||
|
|
||||||
|
if (Array.isArray(arrayErrors)) {
|
||||||
|
isRateLimitExceeded = arrayErrors.some((error) =>
|
||||||
|
error.type.includes(EServiceKindError.RATE_LIMIT)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (objectError?.message) {
|
||||||
|
isRateLimitExceeded = objectError?.message.includes(
|
||||||
|
"rate limit",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isRateLimitExceeded) {
|
||||||
|
throw new ServiceError(
|
||||||
|
"Rate limit exceeded",
|
||||||
|
EServiceKindError.RATE_LIMIT,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new ServiceError(
|
||||||
|
"unknown error",
|
||||||
|
EServiceKindError.NOT_FOUND,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
@ -3,9 +3,19 @@ export type GithubError = {
|
||||||
type: string;
|
type: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type GithubErrorResponse = {
|
||||||
|
errors: GithubError[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type GithubExceedError = {
|
||||||
|
documentation_url: string;
|
||||||
|
message: string;
|
||||||
|
};
|
||||||
|
|
||||||
export type QueryDefaultResponse<T = unknown> = {
|
export type QueryDefaultResponse<T = unknown> = {
|
||||||
data: {
|
data: {
|
||||||
data: T;
|
data: T;
|
||||||
errors: GithubError[];
|
errors?: GithubErrorResponse;
|
||||||
|
message?: string;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ import { EServiceKindError } from "./EServiceKindError.ts";
|
||||||
export class ServiceError extends Error {
|
export class ServiceError extends Error {
|
||||||
constructor(message: string, kind: EServiceKindError) {
|
constructor(message: string, kind: EServiceKindError) {
|
||||||
super(message);
|
super(message);
|
||||||
|
this.message = message;
|
||||||
this.name = "ServiceError";
|
this.name = "ServiceError";
|
||||||
this.cause = kind;
|
this.cause = kind;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue