Skip to content

Commit 9049e97

Browse files
olivier-landoolivier-lando
authored
Merge pull request #21 from apivideo/improve-retrier
Improve retrier
2 parents 51db6bd + fda0643 commit 9049e97

File tree

11 files changed

+104
-40
lines changed

11 files changed

+104
-40
lines changed

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
# Changelog
22
All changes to this project will be documented in this file.
33

4+
## [1.0.10] - 2022-06-29
5+
- Retry even if the server is not responding.
6+
- Add possibility to define a custom retry policy.
7+
48
## [1.0.9] - 2022-05-24
59
- Progressive upload: prevent last part to be empty
610

‎README.md‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -178,12 +178,13 @@ Using delegated upload tokens for authentication is best options when uploading
178178
#### Common options
179179

180180

181-
| Option name | Mandatory | Type | Description |
182-
| ----------: | --------- | ------ | -------------------------------------------------------------------------- |
183-
| file | **yes** | File | the file you want to upload |
184-
| chunkSize | no | number | number of bytes of each upload chunk (default: 50MB, min: 5MB, max: 128MB) |
185-
| apiHost | no | string | api.video host (default: ws.api.video) |
186-
| retries | no | number | number of retries when an API call fails (default: 5) |
181+
| Option name | Mandatory | Type | Description |
182+
| ------------: | --------- | --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- |
183+
| file | **yes** | File | the file you want to upload |
184+
| chunkSize | no | number | number of bytes of each upload chunk (default: 50MB, min: 5MB, max: 128MB) |
185+
| apiHost | no | string | api.video host (default: ws.api.video) |
186+
| retries | no | number | number of retries when an API call fails (default: 5) |
187+
| retryStrategy | no | (retryCount: number, error: VideoUploadError) => number \| null | function that returns the number of ms to wait before retrying a failed upload. Returns null to stop retrying |
187188

188189

189190
### Example
@@ -276,11 +277,12 @@ Using delegated upload tokens for authentication is best options when uploading
276277
#### Common options
277278

278279

279-
| Option name | Mandatory | Type | Description |
280-
| ----------------: | --------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
281-
| apiHost | no | string | api.video host (default: ws.api.video) |
282-
| retries | no | number | number of retries when an API call fails (default: 5) |
283-
| preventEmptyParts | no | boolean | if true, the upload will succeed even if an empty Blob is passed to uploadLastPart(). This may alter performances a bit in some cases (default: false) |
280+
| Option name | Mandatory | Type | Description |
281+
| ----------------: | --------- | --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ |
282+
| apiHost | no | string | api.video host (default: ws.api.video) |
283+
| retries | no | number | number of retries when an API call fails (default: 5) |
284+
| retryStrategy | no | (retryCount: number, error: VideoUploadError) => number \| null | function that returns the number of ms to wait before retrying a failed upload. Returns null to stop retrying |
285+
| preventEmptyParts | no | boolean | if true, the upload will succeed even if an empty Blob is passed to uploadLastPart(). This may alter performances a bit in some cases (default: false) |
284286

285287

286288
### Example

‎dist/index.js‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎dist/src/abstract-uploader.d.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
export declare const MIN_CHUNK_SIZE: number;
22
export declare const DEFAULT_CHUNK_SIZE: number;
33
export declare const MAX_CHUNK_SIZE: number;
4-
export declare const DEFAULT_RETRIES = 5;
4+
export declare const DEFAULT_RETRIES = 6;
55
export declare const DEFAULT_API_HOST = "ws.api.video";
66
export declare type VideoUploadResponse = {
77
readonly videoId: string;
@@ -29,9 +29,11 @@ export declare type VideoUploadResponse = {
2929
readonly thumbnail?: string;
3030
};
3131
};
32+
declare type RetryStrategy = (retryCount: number, error: VideoUploadError) => number | null;
3233
export interface CommonOptions {
3334
apiHost?: string;
3435
retries?: number;
36+
retryStrategy?: RetryStrategy;
3537
}
3638
export interface WithUploadToken {
3739
uploadToken: string;
@@ -47,7 +49,7 @@ export interface WithApiKey {
4749
videoId: string;
4850
}
4951
export declare type VideoUploadError = {
50-
status: number;
52+
status?: number;
5153
type?: string;
5254
title?: string;
5355
reason?: string;
@@ -61,6 +63,7 @@ declare type HXRRequestParams = {
6163
onProgress?: (e: ProgressEvent) => void;
6264
body: Document | XMLHttpRequestBodyInit | null;
6365
};
66+
export declare const DEFAULT_RETRY_STRATEGY: (maxRetries: number) => (retryCount: number, error: VideoUploadError) => number | null;
6467
export declare abstract class AbstractUploader<T> {
6568
protected uploadEndpoint: string;
6669
protected videoId?: string;
@@ -71,6 +74,7 @@ export declare abstract class AbstractUploader<T> {
7174
protected onProgressCallbacks: ((e: T) => void)[];
7275
protected refreshToken?: string;
7376
protected apiHost: string;
77+
protected retryStrategy: RetryStrategy;
7478
constructor(options: CommonOptions & (WithAccessToken | WithUploadToken | WithApiKey));
7579
onProgress(cb: (e: T) => void): void;
7680
protected parseErrorResponse(xhr: XMLHttpRequest): VideoUploadError;

‎dist/test/abstract-uploader.d.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export {};

‎package-lock.json‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@api.video/video-uploader",
3-
"version": "1.0.9",
3+
"version": "1.0.10",
44
"description": "api.video video uploader",
55
"repository": {
66
"type": "git",

‎src/abstract-uploader.ts‎

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
export const MIN_CHUNK_SIZE = 1024 * 1024 * 5; // 5mb
22
export const DEFAULT_CHUNK_SIZE = 1024 * 1024 * 50; // 50mb
33
export const MAX_CHUNK_SIZE = 1024 * 1024 * 128; // 128mb
4-
export const DEFAULT_RETRIES = 5;
4+
export const DEFAULT_RETRIES = 6;
55
export const DEFAULT_API_HOST = "ws.api.video";
66

77
export declare type VideoUploadResponse = {
@@ -31,9 +31,12 @@ export declare type VideoUploadResponse = {
3131
};
3232
};
3333

34+
type RetryStrategy = (retryCount: number, error: VideoUploadError) => number | null;
35+
3436
export interface CommonOptions {
3537
apiHost?: string;
3638
retries?: number;
39+
retryStrategy?: RetryStrategy;
3740
}
3841

3942
export interface WithUploadToken {
@@ -53,7 +56,7 @@ export interface WithApiKey {
5356
}
5457

5558
export type VideoUploadError = {
56-
status: number;
59+
status?: number;
5760
type?: string;
5861
title?: string;
5962
reason?: string;
@@ -77,6 +80,15 @@ try {
7780
// ignore
7881
}
7982

83+
export const DEFAULT_RETRY_STRATEGY = (maxRetries: number) => {
84+
return (retryCount: number, error: VideoUploadError) => {
85+
if ((error.status && error.status >= 400 && error.status < 500) || retryCount >= maxRetries) {
86+
return null;
87+
}
88+
return Math.floor(200 + 2000 * retryCount * (retryCount + 1));
89+
}
90+
};
91+
8092
export abstract class AbstractUploader<T> {
8193
protected uploadEndpoint: string;
8294
protected videoId?: string;
@@ -85,6 +97,7 @@ export abstract class AbstractUploader<T> {
8597
protected onProgressCallbacks: ((e: T) => void)[] = [];
8698
protected refreshToken?: string;
8799
protected apiHost: string;
100+
protected retryStrategy: RetryStrategy;
88101

89102
constructor(options: CommonOptions & (WithAccessToken | WithUploadToken | WithApiKey)) {
90103
this.apiHost = options.apiHost || DEFAULT_API_HOST;
@@ -116,6 +129,7 @@ export abstract class AbstractUploader<T> {
116129
}
117130
this.headers["AV-Origin-Client"] = "typescript-uploader:" + PACKAGE_VERSION;
118131
this.retries = options.retries || DEFAULT_RETRIES;
132+
this.retryStrategy = options.retryStrategy || DEFAULT_RETRY_STRATEGY(this.retries);
119133
}
120134

121135
public onProgress(cb: (e: T) => void) {
@@ -228,8 +242,22 @@ export abstract class AbstractUploader<T> {
228242
}
229243
}
230244
};
245+
xhr.onerror = (e) => {
246+
reject({
247+
status: undefined,
248+
raw: undefined,
249+
reason: "NETWORK_ERROR",
250+
});
251+
}
252+
xhr.ontimeout = (e) => {
253+
reject({
254+
status: undefined,
255+
raw: undefined,
256+
reason: "NETWORK_TIMEOUT",
257+
});
258+
}
231259
xhr.onload = (_) => {
232-
if(xhr.status < 400) {
260+
if (xhr.status < 400) {
233261
resolve(this.apiResponseToVideoUploadResponse(JSON.parse(xhr.response)));
234262
}
235263
};
@@ -246,11 +274,13 @@ export abstract class AbstractUploader<T> {
246274
resolve(res);
247275
return;
248276
} catch (e: any) {
249-
if (e.status === 401 || retriesCount >= this.retries) {
277+
const retryDelay = this.retryStrategy(retriesCount, e);
278+
if (retryDelay === null) {
250279
reject(e);
251280
return;
252281
}
253-
await this.sleep(200 + retriesCount * 500);
282+
console.log(`video upload: ${e.reason || "ERROR"}, will be retried in ${retryDelay} ms`);
283+
await this.sleep(retryDelay);
254284
retriesCount++;
255285
}
256286
}

‎test/abstract-uploader.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import { expect } from 'chai';
2+
import { ProgressiveUploader, ProgressiveUploadProgressEvent } from '../src/index';
3+
import mock from 'xhr-mock';
4+
import { DEFAULT_RETRY_STRATEGY } from '../src/abstract-uploader';
5+
6+
describe('Default retrier', () => {
7+
const retrier = DEFAULT_RETRY_STRATEGY(10);
8+
9+
it('don\'t retry if it should not', () => {
10+
expect(retrier(11, { status: 500, raw: "" })).to.be.equal(null)
11+
expect(retrier(1, { status: 401, raw: "" })).to.be.equal(null)
12+
});
13+
14+
it('retry if it should', () => {
15+
expect(retrier(1, { status: 500, raw: "" })).to.be.equal(4200);
16+
expect(retrier(8, { status: 502, raw: "" })).to.be.equal(144200);
17+
expect(retrier(8, { raw: "" })).to.be.equal(144200);
18+
});
19+
});

‎test/progressive-video-uploader.test.ts‎

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ describe('Requests synchronization', () => {
2222

2323
it('requests are made sequentially', (done) => {
2424
const uploadToken = "the-upload-token";
25-
const uploader = new ProgressiveUploader({uploadToken});
25+
const uploader = new ProgressiveUploader({ uploadToken });
2626

2727
let isRequesting = false;
2828

@@ -35,9 +35,9 @@ describe('Requests synchronization', () => {
3535
}, 500));
3636
});
3737

38-
uploader.uploadPart(new File([new ArrayBuffer(5*1024*1024)], "filename"));
39-
uploader.uploadPart(new File([new ArrayBuffer(5*1024*1024)], "filename"));
40-
uploader.uploadLastPart(new File([new ArrayBuffer(3*1024*1024)], "filename")).then((r) => done());
38+
uploader.uploadPart(new File([new ArrayBuffer(5 * 1024 * 1024)], "filename"));
39+
uploader.uploadPart(new File([new ArrayBuffer(5 * 1024 * 1024)], "filename"));
40+
uploader.uploadLastPart(new File([new ArrayBuffer(3 * 1024 * 1024)], "filename")).then((r) => done());
4141
});
4242
});
4343

@@ -48,7 +48,7 @@ describe('Content-range', () => {
4848
it('content-range headers are properly set', async () => {
4949
const uploadToken = "the-upload-token";
5050

51-
const uploader = new ProgressiveUploader({uploadToken});
51+
const uploader = new ProgressiveUploader({ uploadToken });
5252

5353
const expectedRanges = [
5454
'part 1/*',
@@ -61,9 +61,9 @@ describe('Content-range', () => {
6161
return res.status(201).body("{}");
6262
});
6363

64-
await uploader.uploadPart(new File([new ArrayBuffer(5*1024*1024)], "filename"));
65-
await uploader.uploadPart(new File([new ArrayBuffer(5*1024*1024)], "filename"));
66-
await uploader.uploadLastPart(new File([new ArrayBuffer(3*1024*1024)], "filename"));
64+
await uploader.uploadPart(new File([new ArrayBuffer(5 * 1024 * 1024)], "filename"));
65+
await uploader.uploadPart(new File([new ArrayBuffer(5 * 1024 * 1024)], "filename"));
66+
await uploader.uploadLastPart(new File([new ArrayBuffer(3 * 1024 * 1024)], "filename"));
6767

6868
expect(expectedRanges).has.lengthOf(0);
6969
});
@@ -76,7 +76,7 @@ describe('Prevent empty part', () => {
7676
it('content-range headers are properly set', async () => {
7777
const uploadToken = "the-upload-token";
7878

79-
const uploader = new ProgressiveUploader({uploadToken, preventEmptyParts: true});
79+
const uploader = new ProgressiveUploader({ uploadToken, preventEmptyParts: true });
8080

8181
const expectedRanges = [
8282
'part 1/*',
@@ -89,9 +89,9 @@ describe('Prevent empty part', () => {
8989
return res.status(201).body("{}");
9090
});
9191

92-
await uploader.uploadPart(new File([new ArrayBuffer(5*1024*1024)], "filename"));
93-
await uploader.uploadPart(new File([new ArrayBuffer(5*1024*1024)], "filename"));
94-
await uploader.uploadPart(new File([new ArrayBuffer(3*1024*1024)], "filename"));
92+
await uploader.uploadPart(new File([new ArrayBuffer(5 * 1024 * 1024)], "filename"));
93+
await uploader.uploadPart(new File([new ArrayBuffer(5 * 1024 * 1024)], "filename"));
94+
await uploader.uploadPart(new File([new ArrayBuffer(3 * 1024 * 1024)], "filename"));
9595
await uploader.uploadLastPart(new Blob());
9696

9797
expect(expectedRanges).has.lengthOf(0);
@@ -144,7 +144,7 @@ describe('Progress listener', () => {
144144

145145
uploader.onProgress((e: ProgressiveUploadProgressEvent) => lastUploadProgressEvent = e);
146146

147-
uploader.uploadPart(new File([new ArrayBuffer(5*1024*1024)], "filename")).then(() => {
147+
uploader.uploadPart(new File([new ArrayBuffer(5 * 1024 * 1024)], "filename")).then(() => {
148148
expect(lastUploadProgressEvent).to.deep.equal({
149149
...lastUploadProgressEvent,
150150
totalBytes: 5242880
@@ -162,7 +162,10 @@ describe('Errors & retries', () => {
162162

163163
const uploadToken = "the-upload-token";
164164

165-
const uploader = new ProgressiveUploader({ uploadToken });
165+
const uploader = new ProgressiveUploader({
166+
uploadToken,
167+
retryStrategy: (retryCount, error) => retryCount > 3 ? null : 10,
168+
});
166169

167170
let postCounts = 0;
168171
mock.post(`https://ws.api.video/upload?token=${uploadToken}`, (req, res) => {
@@ -173,7 +176,7 @@ describe('Errors & retries', () => {
173176
return res.status(500).body('{"error": "oups"}');
174177
});
175178

176-
uploader.uploadPart(new File([new ArrayBuffer(5*1024*1024)], "filename")).then(() => {
179+
uploader.uploadPart(new File([new ArrayBuffer(5 * 1024 * 1024)], "filename")).then(() => {
177180
expect(postCounts).to.be.eq(3);
178181
done();
179182
});
@@ -185,14 +188,14 @@ describe('Errors & retries', () => {
185188

186189
const uploader = new ProgressiveUploader({
187190
uploadToken,
188-
retries: 3,
191+
retryStrategy: (retryCount, error) => retryCount > 3 ? null : 10,
189192
});
190193

191194
mock.post(`https://ws.api.video/upload?token=${uploadToken}`, (req, res) => {
192195
return res.status(500).body('{"error": "oups"}');
193196
});
194197

195-
uploader.uploadPart(new File([new ArrayBuffer(5*1024*1024)], "filename")).then(() => {
198+
uploader.uploadPart(new File([new ArrayBuffer(5 * 1024 * 1024)], "filename")).then(() => {
196199
throw new Error('should not succeed');
197200
}).catch((e) => {
198201
expect(e).to.be.eqls({ status: 500, raw: '{"error": "oups"}', error: 'oups' });

0 commit comments

Comments
 (0)