Bazsalanszky/destreamer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Revert "Fixed parsing for group with more than 100 videos (#288)" (#295)

Browse source 
This reverts commit cd1ac82fea.
This commit is contained in:
Adrian Calinescu 2020-12-15 13:02:23 +02:00 committed by GitHub
parent cd1ac82fea
commit f26204c38a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 1335 additions and 1275 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2519
package-lock.json generated
View file

File diff suppressed because it is too large Load diff

42
package.json
View file

@ -17,34 +17,34 @@
"author": "snobu",
"license": "MIT",
"devDependencies": {
"@types/mocha": "^8.0.4",
"@types/puppeteer": "^5.4.0",
"@types/mocha": "^7.0.2",
"@types/puppeteer": "^1.20.4",
"@types/readline-sync": "^1.4.3",
"@types/tmp": "^0.2.0",
"@types/yargs": "^15.0.11",
"@typescript-eslint/eslint-plugin": "^4.9.0",
"@typescript-eslint/parser": "^4.9.0",
"eslint": "^7.14.0",
"mocha": "^8.2.1",
"tmp": "^0.2.1"
"@types/tmp": "^0.1.0",
"@types/yargs": "^15.0.3",
"@typescript-eslint/eslint-plugin": "^2.25.0",
"@typescript-eslint/parser": "^2.25.0",
"eslint": "^6.8.0",
"mocha": "^7.1.1",
"tmp": "^0.1.0"
},
"dependencies": {
"@tedconf/fessonia": "^2.1.2",
"@types/cli-progress": "^3.8.0",
"@tedconf/fessonia": "^2.1.0",
"@types/cli-progress": "^3.4.2",
"@types/jwt-decode": "^2.2.1",
"axios": "^0.21.0",
"axios-retry": "^3.1.9",
"cli-progress": "^3.8.2",
"axios": "^0.19.2",
"axios-retry": "^3.1.8",
"cli-progress": "^3.7.0",
"colors": "^1.4.0",
"is-elevated": "^3.0.0",
"iso8601-duration": "^1.3.0",
"jwt-decode": "^3.1.2",
"puppeteer": "5.5.0",
"iso8601-duration": "^1.2.0",
"jwt-decode": "^2.2.0",
"puppeteer": "2.1.1",
"readline-sync": "^1.4.10",
"sanitize-filename": "^1.6.3",
"terminal-image": "^1.2.1",
"typescript": "^4.1.2",
"winston": "^3.3.3",
"yargs": "^16.1.1"
"terminal-image": "^1.0.1",
"typescript": "^3.8.3",
"winston": "^3.3.2",
"yargs": "^15.0.3"
}
}

31
src/Utils.ts
View file

@ -20,23 +20,12 @@ async function extractGuids(url: string, client: ApiClient): Promise<Array<strin
return [videoMatch[1]];
}
else if (groupMatch) {
const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get')
.then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
const result: Array<string> = [];
// const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get')
// .then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
// Anything above $top=100 results in 400 Bad Request
// Use $skip to skip the first 100 and get another 100 and so on
for (let index = 0; index <= Math.floor(videoNumber / 100); index++) {
const partial: Array<string> = await client.callApi(
`groups/${groupMatch[1]}/videos?$skip=${100 * index}&` +
'$top=100&$orderby=publishedDate asc', 'get')
.then(
(response: AxiosResponse<any> | undefined) =>
response?.data.value.map((item: any) => item.id)
);
result.push(...partial);
}
// Anything over $top=100 will return a 400 Bad Request
let result: Array<string> = await client.callApi(`groups/${groupMatch[1]}/videos?$top=100&$orderby=publishedDate asc`, 'get')
.then((response: AxiosResponse<any> | undefined) => response?.data.value.map((item: any) => item.id));
return result;
}
@ -60,7 +49,7 @@ export async function parseCLIinput(urlList: Array<string>, defaultOutDir: strin
session: Session): Promise<Array<Array<string>>> {
const apiClient: ApiClient = ApiClient.getInstance(session);
const guidList: Array<string> = [];
let guidList: Array<string> = [];
for (const url of urlList) {
const guids: Array<string> | null = await extractGuids(url, apiClient);
@ -97,8 +86,8 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
.split(/\r?\n/);
const apiClient: ApiClient = ApiClient.getInstance(session);
const guidList: Array<string> = [];
const outDirList: Array<string> = [];
let guidList: Array<string> = [];
let outDirList: Array<string> = [];
// if the last line was an url set this
let foundUrl = false;
@ -113,7 +102,7 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
// parse if line is option
else if (line.includes('-dir')) {
if (foundUrl) {
const outDir: string | null = parseOption('-dir', line);
let outDir: string | null = parseOption('-dir', line);
if (outDir && checkOutDir(outDir)) {
outDirList.push(...Array(guidList.length - outDirList.length)
@ -180,7 +169,7 @@ export function checkOutDir(directory: string): boolean {
logger.info('\nCreated directory: '.yellow + directory);
}
catch (e) {
logger.warn('Cannot create directory: ' + directory +
logger.warn('Cannot create directory: '+ directory +
'\nFalling back to default directory..');
return false;