Bazsalanszky/destreamer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Introduce singleton API client with retry policy (#130)

Browse source 
* Add singleton http client
* Removed refresh token logic
* Further cleanup after refresh token
* Make tests faster maybe
This commit is contained in:
Adrian Calinescu 2020-05-18 20:34:57 +03:00 committed by GitHub
parent 3cf49c5d1c
commit 2c48d103f2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 753 additions and 344 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

829
package-lock.json generated
View file

File diff suppressed because it is too large Load diff

5
package.json
View file

@ -31,14 +31,15 @@
"@types/cli-progress": "^3.4.2",
"@types/jwt-decode": "^2.2.1",
"axios": "^0.19.2",
"axios-retry": "^3.1.8",
"cli-progress": "^3.7.0",
"colors": "^1.4.0",
"is-elevated": "^3.0.0",
"iso8601-duration": "^1.2.0",
"jwt-decode": "^2.2.0",
"puppeteer": "^2.1.1",
"puppeteer": "^3.0.4",
"sanitize-filename": "^1.6.3",
"terminal-image": "^0.2.0",
"terminal-image": "^1.0.1",
"typescript": "^3.8.3",
"yargs": "^15.0.3"
}

91
src/ApiClient.ts Normal file
View file

@ -0,0 +1,91 @@
import axios, { AxiosRequestConfig, AxiosResponse, AxiosInstance, AxiosError } from 'axios';
import axiosRetry, { isNetworkOrIdempotentRequestError } from 'axios-retry';
import { Session } from './Types';
export class ApiClient {
private static instance: ApiClient;
private axiosInstance?: AxiosInstance;
private session?: Session;
private constructor(session?: Session) {
this.session = session;
this.axiosInstance = axios.create({
baseURL: session?.ApiGatewayUri,
timeout: 7000,
headers: { 'User-Agent': 'destreamer/2.0 (Hammer of Dawn)' }
});
axiosRetry(this.axiosInstance, {
shouldResetTimeout: true,
retries: 6,
retryDelay: (retryCount) => {
return retryCount * 2000;
},
retryCondition: (err: AxiosError) => {
const retryCodes = [429, 500, 502, 503];
if (isNetworkOrIdempotentRequestError(err)) {
console.warn(`${err}. Retrying request...`);
return true;
}
console.warn(`Got HTTP ${err?.response?.status}. Retrying request...`);
const condition = retryCodes.includes(err?.response?.status ?? 0);
return condition;
}
});
}
public static getInstance(session?: Session): ApiClient {
if (!ApiClient.instance) {
ApiClient.instance = new ApiClient(session);
}
return ApiClient.instance;
}
/**
* Call Microsoft Stream API. Base URL is sourced from
* the session object and prepended automatically.
*/
public async callApi(
path: string,
method: AxiosRequestConfig['method'] = 'get',
payload?: any): Promise<AxiosResponse | undefined> {
const delimiter = path.split('?').length === 1 ? '?' : '&';
const headers: object = {
'Authorization': 'Bearer ' + this.session?.AccessToken
};
return this.axiosInstance?.request({
method: method,
headers: headers,
data: payload ?? undefined,
url: path + delimiter + 'api-version=' + this.session?.ApiGatewayVersion
});
}
/**
* Call an absolute URL
*/
public async callUrl(
url: string,
method: AxiosRequestConfig['method'] = 'get',
payload?: any,
responseType: AxiosRequestConfig['responseType'] = 'json'): Promise<AxiosResponse | undefined> {
const headers: object = {
'Authorization': 'Bearer ' + this.session?.AccessToken
};
return this.axiosInstance?.request({
method: method,
headers: headers,
data: payload ?? undefined,
url: url,
responseType: responseType
});
}
}

4
src/Errors.ts
View file

@ -13,7 +13,7 @@ export const enum ERROR_CODE {
INVALID_VIDEO_ID,
INVALID_VIDEO_GUID,
UNK_FFMPEG_ERROR,
NO_SESSION_INFO,
NO_SESSION_INFO
}
// TODO: create better errors descriptions
@ -43,7 +43,7 @@ export const Error: IError = {
[ERROR_CODE.INVALID_VIDEO_GUID]: 'Unable to get video GUID from URL',
[ERROR_CODE.NO_SESSION_INFO]: 'Could not evaluate sessionInfo on the page'
}
};
export const enum CLI_ERROR {
GRACEFULLY_STOP = ' ', // gracefully stop execution, yargs way

30
src/Metadata.ts
View file

@ -1,8 +1,9 @@
import { Metadata, Session } from './Types';
import { forEachAsync } from './Utils';
import { ApiClient } from './ApiClient';
import { parse } from 'iso8601-duration';
import axios from 'axios';
function publishedDateToString(date: string) {
const dateJs = new Date(date);
@ -21,8 +22,7 @@ function durationToTotalChunks(duration: string) {
return (hrs * 60) + mins + (secs / 60);
}
export async function getVideoMetadata(videoGuids: string[], session: Session, verbose: boolean): Promise<Metadata[]> {
export async function getVideoMetadata(videoGuids: string[], session: Session): Promise<Metadata[]> {
let metadata: Metadata[] = [];
let title: string;
let date: string;
@ -30,28 +30,20 @@ export async function getVideoMetadata(videoGuids: string[], session: Session, v
let playbackUrl: string;
let posterImage: string;
const apiClient = ApiClient.getInstance(session);
await forEachAsync(videoGuids, async (guid: string) => {
let apiUrl = `${session.ApiGatewayUri}videos/${guid}?api-version=${session.ApiGatewayVersion}`;
let response = await apiClient.callApi('videos/' + guid, 'get');
if (verbose)
console.info(`Calling ${apiUrl}`);
let response = await axios.get(apiUrl,
{
headers: {
Authorization: `Bearer ${session.AccessToken}`
}
});
title = response.data['name'];
playbackUrl = response.data['playbackUrls']
title = response?.data['name'];
playbackUrl = response?.data['playbackUrls']
.filter((item: { [x: string]: string; }) =>
item['mimeType'] == 'application/vnd.apple.mpegurl')
.map((item: { [x: string]: string }) => { return item['playbackUrl']; })[0];
posterImage = response.data['posterImage']['medium']['url'];
date = publishedDateToString(response.data['publishedDate']);
totalChunks = durationToTotalChunks(response.data.media['duration']);
posterImage = response?.data['posterImage']['medium']['url'];
date = publishedDateToString(response?.data['publishedDate']);
totalChunks = durationToTotalChunks(response?.data.media['duration']);
metadata.push({
date: date,

16
src/Thumbnail.ts
View file

@ -1,14 +1,10 @@
import { ApiClient } from './ApiClient';
import { Session } from './Types';
import terminalImage from 'terminal-image';
import axios from 'axios';
export async function drawThumbnail(posterImage: string, accessToken: string) {
let thumbnail = await axios.get(posterImage,
{
headers: {
Authorization: `Bearer ${accessToken}`
},
responseType: 'arraybuffer'
});
console.log(await terminalImage.buffer(thumbnail.data));
export async function drawThumbnail(posterImage: string, session: Session): Promise<void> {
const apiClient = ApiClient.getInstance(session);
let thumbnail = await apiClient.callUrl(posterImage, 'get', null, 'arraybuffer');
console.log(await terminalImage.buffer(thumbnail?.data, { width: 70 } ));
}

34
src/TokenCache.ts
View file

@ -2,15 +2,13 @@ import * as fs from 'fs';
import { Session } from './Types';
import { bgGreen, bgYellow, green } from 'colors';
import jwtDecode from 'jwt-decode';
import axios from 'axios';
import colors from 'colors';
export class TokenCache {
private tokenCacheFile: string = '.token_cache';
public Read(): Session | null {
let j = null;
if(!fs.existsSync(this.tokenCacheFile)) {
if (!fs.existsSync(this.tokenCacheFile)) {
console.warn(bgYellow.black(`${this.tokenCacheFile} not found.\n`));
return null;
@ -55,34 +53,4 @@ export class TokenCache {
console.info(green('Fresh access token dropped into .token_cache'));
});
}
public async RefreshToken(session: Session, cookie?: string | null): Promise<string | null> {
let endpoint = `${session.ApiGatewayUri}refreshtoken?api-version=${session.ApiGatewayVersion}`;
let headers: Function = (): object => {
if (cookie) {
return {
Cookie: cookie
};
}
else {
return {
Authorization: 'Bearer ' + session.AccessToken
};
}
}
let response = await axios.get(endpoint, { headers: headers() });
let freshCookie: string | null = null;
try {
let cookie: string = response.headers["set-cookie"].toString();
freshCookie = cookie.split(',Authorization_Api=')[0];
}
catch (e) {
console.error(colors.yellow("Error when calling /refreshtoken: Missing or unexpected set-cookie header."));
}
return freshCookie;
}
}

84
src/destreamer.ts
View file

@ -11,8 +11,8 @@ import { Metadata, Session } from './Types';
import { drawThumbnail } from './Thumbnail';
import { argv } from './CommandLineParser';
import isElevated from 'is-elevated';
import puppeteer from 'puppeteer';
import isElevated from 'is-elevated';
import colors from 'colors';
import path from 'path';
import fs from 'fs';
@ -23,10 +23,6 @@ import cliProgress from 'cli-progress';
const { FFmpegCommand, FFmpegInput, FFmpegOutput } = require('@tedconf/fessonia')();
const tokenCache = new TokenCache();
// The cookie lifetime is one hour,
// let's refresh every 3000 seconds.
const REFRESH_TOKEN_INTERVAL = 3000;
async function init() {
setProcessEvents(); // must be first!
@ -54,7 +50,11 @@ async function DoInteractiveLogin(url: string, username?: string): Promise<Sessi
const browser = await puppeteer.launch({
executablePath: getPuppeteerChromiumPath(),
headless: false,
args: ['--disable-dev-shm-usage']
args: [
'--disable-dev-shm-usage',
'--fast-start',
'--no-sandbox'
]
});
const page = (await browser.pages())[0];
console.log('Navigating to login page...');
@ -103,6 +103,17 @@ async function DoInteractiveLogin(url: string, username?: string): Promise<Sessi
console.log("At this point Chromium's job is done, shutting it down...\n");
await browser.close();
// --- Ignore all this for now ---
// --- hopefully we won't need it ----
// await sleep(1000);
// let banner = await page.evaluate(
// () => {
// let topbar = document.getElementsByTagName('body')[0];
// topbar.innerHTML =
// '<h1 style="color: red">DESTREAMER NEEDS THIS WINDOW ' +
// 'TO DO SOME ACCESS TOKEN MAGIC. DO NOT CLOSE IT.</h1>';
// });
// --------------------------------
return session;
}
@ -134,11 +145,10 @@ function extractVideoGuid(videoUrls: string[]): string[] {
async function downloadVideo(videoUrls: string[], outputDirectories: string[], session: Session) {
const videoGuids = extractVideoGuid(videoUrls);
let lastTokenRefresh: number;
console.log('Fetching metadata...');
const metadata: Metadata[] = await getVideoMetadata(videoGuids, session, argv.verbose);
const metadata: Metadata[] = await getVideoMetadata(videoGuids, session);
if (argv.simulate) {
metadata.forEach(video => {
@ -152,11 +162,10 @@ async function downloadVideo(videoUrls: string[], outputDirectories: string[], s
return;
}
if (argv.verbose)
if (argv.verbose) {
console.log(outputDirectories);
}
let freshCookie: string | null = null;
const outDirsIdxInc = outputDirectories.length > 1 ? 1:0;
for (let i=0, j=0, l=metadata.length; i<l; ++i, j+=outDirsIdxInc) {
@ -175,46 +184,22 @@ async function downloadVideo(videoUrls: string[], outputDirectories: string[], s
video.title = makeUniqueTitle(sanitize(video.title) + ' - ' + video.date, outputDirectories[j], argv.skip, argv.format);
// Very experimental inline thumbnail rendering
if (!argv.noExperiments)
await drawThumbnail(video.posterImage, session.AccessToken);
console.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...');
if (!process.stdout.columns) {
console.info(colors.red('Unable to get number of columns from terminal.\n' +
'This happens sometimes in Cygwin/MSYS.\n' +
'No progress bar can be rendered, however the download process should not be affected.\n\n' +
'Please use PowerShell or cmd.exe to run destreamer on Windows.'));
'This happens sometimes in Cygwin/MSYS.\n' +
'No progress bar can be rendered, however the download process should not be affected.\n\n' +
'Please use PowerShell or cmd.exe to run destreamer on Windows.'));
}
// Try to get a fresh cookie, else gracefully fall back
// to our session access token (Bearer)
freshCookie = await tokenCache.RefreshToken(session, freshCookie);
const headers = 'Authorization: Bearer ' + session.AccessToken;
// Don't remove the "useless" escapes otherwise ffmpeg will
// not pick up the header
// eslint-disable-next-line no-useless-escape
let headers = 'Authorization:\ Bearer\ ' + session.AccessToken;
if (freshCookie) {
lastTokenRefresh = Date.now();
if (argv.verbose) {
console.info(colors.green('Using a fresh cookie.'));
}
// eslint-disable-next-line no-useless-escape
headers = 'Cookie:\ ' + freshCookie;
// Very experimental inline thumbnail rendering
if (!argv.noExperiments) {
await drawThumbnail(video.posterImage, session);
}
const RefreshTokenMaybe = async (): Promise<void> => {
let elapsed = Date.now() - lastTokenRefresh;
if (elapsed > REFRESH_TOKEN_INTERVAL * 1000) {
if (argv.verbose) {
console.info(colors.green('\nRefreshing access token...'));
}
lastTokenRefresh = Date.now();
freshCookie = await tokenCache.RefreshToken(session, freshCookie);
}
};
const outputPath = outputDirectories[j] + path.sep + video.title + '.' + argv.format;
const ffmpegInpt = new FFmpegInput(video.playbackUrl, new Map([
['headers', headers]
@ -234,8 +219,11 @@ async function downloadVideo(videoUrls: string[], outputDirectories: string[], s
try {
fs.unlinkSync(outputPath);
} catch(e) {}
}
}
catch (e) {
// Future handling of an error maybe
}
};
pbar.start(video.totalChunks, 0, {
speed: '0'
@ -247,7 +235,6 @@ async function downloadVideo(videoUrls: string[], outputDirectories: string[], s
ffmpegCmd.on('update', (data: any) => {
const currentChunks = ffmpegTimemarkToChunk(data.out_time);
RefreshTokenMaybe();
pbar.update(currentChunks, {
speed: data.bitrate
@ -262,7 +249,7 @@ async function downloadVideo(videoUrls: string[], outputDirectories: string[], s
process.on('SIGINT', cleanupFn);
// let the magic begin...
await new Promise((resolve: any, reject: any) => {
await new Promise((resolve: any) => {
ffmpegCmd.on('error', (error: any) => {
if (argv.skip && error.message.includes('exists') && error.message.includes(outputPath)) {
pbar.update(video.totalChunks); // set progress bar to 100%
@ -276,7 +263,7 @@ async function downloadVideo(videoUrls: string[], outputDirectories: string[], s
}
});
ffmpegCmd.on('success', (data:any) => {
ffmpegCmd.on('success', () => {
pbar.update(video.totalChunks); // set progress bar to 100%
console.log(colors.green(`\nDownload finished: ${outputPath}`));
resolve();
@ -304,5 +291,4 @@ async function main() {
downloadVideo(videoUrls, outDirs, session);
}
main();

4
test/test.ts
View file

@ -11,14 +11,14 @@ describe('Puppeteer', () => {
it('should grab GitHub page title', async () => {
browser = await puppeteer.launch({
headless: true,
args: ['--disable-dev-shm-usage']
args: ['--disable-dev-shm-usage', '--fast-start', '--no-sandbox']
});
page = await browser.newPage();
await page.goto("https://github.com/", { waitUntil: 'load' });
let pageTitle = await page.title();
assert.equal(true, pageTitle.includes('GitHub'));
await browser.close();
}).timeout(15000); // yeah, this may take a while...
}).timeout(25000); // yeah, this may take a while...
});
describe('Destreamer', () => {