Bazsalanszky/destreamer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fixes and refactoring (#59)

Browse source 
* Input url list: Fix bad Windows behavior

* Minor output fix

* Fix all download issues
  - downloads are synchronous again
  - fix progress bar (fix #39)
  - nuke fluent and switch to a bug-free ffmpeg module (fessonia)

* Move destreamer process events to a new file, we may add more in the future, lets give them their own space

* Destreamer: Release packages and builder script

ETA when? :P

* Clean up

* Implement yargs checks and add --videoUrlsFile option

* Refactor error handling
  - Human readable
  - No magic numbers

* Handle mkdir error
  - remove reduntant message

* gitignore: don't add hidden files

* Implement --outputDirectories

This gives us more flexibility on where to save videos

..especially if your videos have all the same name <.<

* Rename utils -> Utils

* Fix tests

don't import yargs on files other than main

* Create scripts directory

* Update make_release path

* Fix typo

* Create CONTRIBUTING.md

Co-authored-by: kylon <kylonux@gmail.com>
This commit is contained in:
kylon 2020-04-14 14:59:14 +02:00 committed by GitHub
parent 05c36fe718
commit 176fa6e214
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 709 additions and 208 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.gitignore vendored
View file

@ -1,11 +1,10 @@
.token_cache
.*
*.txt
*.log
*.js
*.zip
node_modules
videos
release
swagger.json
cc.json
.thumbnail.png
cc.json

44
CONTRIBUTING.md Normal file
View file

@ -0,0 +1,44 @@
# Destreamer
<a href="https://github.com/snobu/destreamer/actions">
<img src="https://github.com/snobu/destreamer/workflows/Node%20CI/badge.svg" alt="CI build status" />
</a>
![](logo.png)
## Saves Microsoft Stream videos for offline enjoyment
## HOW TO BUILD FOR RELEASE
Destreamer builder supports the following environments:
* Linux
* WLS (Windows Linux Subsystem)
* MacOS
Requirements
* [pkg](https://www.npmjs.com/package/pkg)
* wget
`Install pkg to your system with the command:`
```
npm i -g pkg
```
You will find your release package in destreamer root directory.
To build a release package, run the following commands:
* `$ npm install`
* `$ cd scripts`
* `$ chmod +x make_release.sh`
* `$ ./make_release.sh`
```
Usage: ./make_realse.sh [option]
help - Show usage
linux - Build for Linux x64
win - Build for Windows x64
macos - Build for MacOS x64
all - Build all
default: all
```

28
README.md
View file

@ -23,7 +23,7 @@ Roadmap -
- [X] Token cache (so you don't have to log in every time you run destreamer)
- [ ] Download closed captions if available
- [ ] Performance improvements (via aria2c maybe?) // _This is under consideration, we're not sure if this borders on abusing the streaming endpoints or not._
- [ ] Single static binary (for each major OS)
- [x] Single static binary (for each major OS)
Send a quality PR first and i'll add you as a contributor to the repository.
@ -38,11 +38,33 @@ Hopefully this doesn't break the end user agreement for Microsoft Stream. Since
Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) approach towards the OS it's running on, tested on Windows, macOS and Linux, results may vary, feel free to open an issue if trouble arise.
## HOW TO BUILD
You can build destreamer on any OS.
You will find destreamer.js in `build/src` folder.
To build destreamer.js run the following commands:
* `npm install`
* `npm run -s build`
## USAGE
* `npm install` to restore packages
* `npm run -s build` to transpile TypeScript to JavaScript
* Unpack destreamer and chromium into the same folder
* Open a new terminal and navigate to that folder
* Run destreamer executable
Linux / MacOS
```
$ ./destreamer
```
Windows
```
destreamer.exe
```
## Options
```
$ node ./destreamer.js

27
package-lock.json generated
View file

@ -70,6 +70,11 @@
}
}
},
"@tedconf/fessonia": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/@tedconf/fessonia/-/fessonia-2.1.0.tgz",
"integrity": "sha512-p1iWLFMRyACW+tZV2ZTcGXt6uj7mvjl3uC1R2O1HHSnUsaS4c37llvtpHQJyiSy3p+SU+1oKWjsJYbDyZpziyw=="
},
"@types/cli-progress": {
"version": "3.4.2",
"resolved": "https://registry.npmjs.org/@types/cli-progress/-/cli-progress-3.4.2.tgz",
@ -89,14 +94,6 @@
"integrity": "sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==",
"dev": true
},
"@types/fluent-ffmpeg": {
"version": "2.1.14",
"resolved": "https://registry.npmjs.org/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.14.tgz",
"integrity": "sha512-nJrAX9ODNI7mUB0b7Y0Stx1a6dOpV3zXsOnWoBuEd9/woQhepBNCMeCyOL6SLJD3jn5sLw5ciDGH0RwJenCoag==",
"requires": {
"@types/node": "*"
}
},
"@types/json-schema": {
"version": "7.0.4",
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.4.tgz",
@ -325,11 +322,6 @@
"integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==",
"dev": true
},
"async": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz",
"integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw=="
},
"async-limiter": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz",
@ -1017,15 +1009,6 @@
"integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==",
"dev": true
},
"fluent-ffmpeg": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz",
"integrity": "sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=",
"requires": {
"async": ">=0.2.9",
"which": "^1.1.1"
}
},
"follow-redirects": {
"version": "1.5.10",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz",

8
package.json
View file

@ -6,10 +6,11 @@
},
"version": "1.0.0",
"description": "Save Microsoft Stream videos for offline enjoyment.",
"main": "destreamer.js",
"main": "build/src/destreamer.js",
"bin": "build/src/destreamer.js",
"scripts": {
"build": "echo Transpiling TypeScript to JavaScript... & node node_modules/typescript/bin/tsc --listEmittedFiles",
"run": "node ./destreamer.js",
"run": "node ./build/src/destreamer.js",
"start": "npm run -s build & npm run -s run",
"test": "mocha build/test"
},
@ -28,13 +29,12 @@
"tmp": "^0.1.0"
},
"dependencies": {
"@tedconf/fessonia": "^2.1.0",
"@types/cli-progress": "^3.4.2",
"@types/fluent-ffmpeg": "^2.1.14",
"@types/jwt-decode": "^2.2.1",
"axios": "^0.19.2",
"cli-progress": "^3.7.0",
"colors": "^1.4.0",
"fluent-ffmpeg": "^2.1.2",
"is-elevated": "^3.0.0",
"iso8601-duration": "^1.2.0",
"jwt-decode": "^2.2.0",

169
scripts/make_release.sh Executable file
View file

@ -0,0 +1,169 @@
#!/bin/bash
set -euo pipefail
cd ..
# vars
chromeRev=`cat node_modules/puppeteer/package.json | grep chromium_revision | grep -oP '"([0-9]+)"' | cut -d"\"" -f2`
baseUrl="https://storage.googleapis.com/chromium-browser-snapshots"
osAr=("Linux_x64" "Mac" "Win_x64")
zipAr=("linux" "mac" "win")
arg="all"
function checkWget() {
command -v wget >/dev/null 2>&1 || { echo -e >&2 "I need wget to work :(\n"; exit 1; }
}
function setupBuildForOS() {
case "$arg" in
"linux")
osAr=("Linux_x64")
zipAr=("linux")
;;
"win")
osAr=("Win_x64")
zipAr=("win")
;;
"macos")
osAr=("Mac")
zipAr=("mac")
;;
*)
echo -e "\nInvalid OS selected!\n"
exit 1
;;
esac;
}
function makeDirectories() {
if [ -d release ]; then
rm -R release
fi
mkdir -p release/temp
}
function buildDestreamer() {
npm run -s build
}
function downloadChromiumPackages() {
local idx=0
for os in "${osAr[@]}"
do
local zipName="chrome-${zipAr[$idx]}.zip"
local finalUrl="$baseUrl/$os/$chromeRev/$zipName"
wget "$finalUrl" -P "release/temp/$os"
unzip "release/temp/$os/$zipName" -d "release/temp/$os"
((++idx))
done;
}
function buildPkg() {
pkg . --out-path release
cd release
cp destreamer-linux temp/Linux_x64
cp destreamer-macos temp/Mac
cp destreamer-win.exe temp/Win_x64
}
function buildPkgForOS() {
pkg -t "$arg" . --out-path release
cd release
cp destreamer* "temp/${osAr[0]}"
}
function buildDestreamerReleasePackages() {
local idx=0
for os in "${osAr[@]}"
do
local chromeFold="chrome-${zipAr[$idx]}"
local osFolder="${zipAr[$idx]}-$chromeRev"
if [[ "$os" == "Win_x64" ]]; then # windows fix
osFolder="win64-$chromeRev"
fi;
cd "temp/$os"
mkdir -p "chromium/$osFolder"
mv "$chromeFold" "chromium/$osFolder"
zip -r "destreamer-$os.zip" chromium destreamer*
mv "destreamer-$os.zip" ../../..
cd ../..
((++idx))
done;
cd ..
}
function usage() {
echo -e "Usage: $0 [option]\n"
echo " help - Show usage"
echo " linux - Build for Linux x64"
echo " win - Build for Windows x64"
echo " macos - Build for MacOS x64"
echo " all - Build all"
echo -e "\n default: all\n"
}
function parseArgument() {
case "$arg" in
"all")
;;
"linux"|"win"|"macos")
setupBuildForOS
;;
*)
usage
exit 0
;;
esac;
}
function main() {
clear
echo "##############################"
echo "# Destreamer release builder #"
echo -e "##############################\n"
parseArgument
checkWget
echo -e "\n> \e[32mMaking directories...\e[39m"
makeDirectories
echo -e "\n> \e[32mBuilding destreamer...\e[39m"
buildDestreamer
echo -e "\n> \e[32mDownloading chromium packages...\e[39m"
downloadChromiumPackages
echo -e "\n> \e[32mBuilding pkg...\e[39m"
if [[ "$arg" == "all" ]]; then
buildPkg
else
buildPkgForOS
fi;
echo -e "\n> \e[32mBuilding destreamer release package\e[39m"
buildDestreamerReleasePackages
rm -R release
exit 0
}
# run
if [[ $# -gt 0 ]]; then
arg="$1"
fi;
main

185
src/CommandLineParser.ts Normal file
View file

@ -0,0 +1,185 @@
import { CLI_ERROR } from './Errors';
import yargs from 'yargs';
import colors from 'colors';
import fs from 'fs';
export const argv = yargs.options({
videoUrls: {
alias: 'V',
describe: 'List of video urls',
type: 'array',
demandOption: false
},
videoUrlsFile: {
alias: 'F',
describe: 'Path to txt file containing the urls',
type: 'string',
demandOption: false
},
username: {
alias: 'u',
type: 'string',
demandOption: false
},
outputDirectory: {
alias: 'o',
describe: 'The directory where destreamer will save your downloads [default: videos]',
type: 'string',
demandOption: false
},
outputDirectories: {
alias: 'O',
describe: 'Path to a txt file containing one output directory per video',
type: 'string',
demandOption: false
},
noThumbnails: {
alias: 'nthumb',
describe: `Do not display video thumbnails`,
type: 'boolean',
default: false,
demandOption: false
},
simulate: {
alias: 's',
describe: `Disable video download and print metadata information to the console`,
type: 'boolean',
default: false,
demandOption: false
},
verbose: {
alias: 'v',
describe: `Print additional information to the console (use this before opening an issue on GitHub)`,
type: 'boolean',
default: false,
demandOption: false
}
})
/**
* Do our own argv magic before destreamer starts.
* ORDER IS IMPORTANT!
* Do not mess with this.
*/
.check(() => isShowHelpRequest())
.check(argv => checkRequiredArgument(argv))
.check(argv => checkVideoUrlsArgConflict(argv))
.check(argv => checkOutputDirArgConflict(argv))
.check(argv => checkVideoUrlsInput(argv))
.check(argv => windowsFileExtensionBadBehaviorFix(argv))
.check(argv => mergeVideoUrlsArguments(argv))
.check(argv => mergeOutputDirArguments(argv))
.argv;
function hasNoArgs() {
return process.argv.length === 2;
}
function isShowHelpRequest() {
if (hasNoArgs())
throw new Error(CLI_ERROR.GRACEFULLY_STOP);
return true;
}
function checkRequiredArgument(argv: any) {
if (hasNoArgs())
return true;
if (!argv.videoUrls && !argv.videoUrlsFile)
throw new Error(colors.red(CLI_ERROR.MISSING_REQUIRED_ARG));
return true;
}
function checkVideoUrlsArgConflict(argv: any) {
if (hasNoArgs())
return true;
if (argv.videoUrls && argv.videoUrlsFile)
throw new Error(colors.red(CLI_ERROR.VIDEOURLS_ARG_CONFLICT));
return true;
}
function checkOutputDirArgConflict(argv: any) {
if (hasNoArgs())
return true;
if (argv.outputDirectory && argv.outputDirectories)
throw new Error(colors.red(CLI_ERROR.OUTPUTDIR_ARG_CONFLICT));
return true;
}
function checkVideoUrlsInput(argv: any) {
if (hasNoArgs() || !argv.videoUrls)
return true;
if (!argv.videoUrls.length)
throw new Error(colors.red(CLI_ERROR.MISSING_REQUIRED_ARG));
const t = argv.videoUrls[0] as string;
if (t.substring(t.length-4) === '.txt')
throw new Error(colors.red(CLI_ERROR.FILE_INPUT_VIDEOURLS_ARG));
return true;
}
/**
* Users see 2 separate options, but we don't really care
* cause both options have no difference in code.
*
* Optimize and make this transparent to destreamer
*/
function mergeVideoUrlsArguments(argv: any) {
if (!argv.videoUrlsFile)
return true;
argv.videoUrls = [argv.videoUrlsFile]; // noone will notice ;)
// these are not valid anymore
delete argv.videoUrlsFile;
delete argv.F;
return true;
}
/**
* Users see 2 separate options, but we don't really care
* cause both options have no difference in code.
*
* Optimize and make this transparent to destreamer
*/
function mergeOutputDirArguments(argv: any) {
if (!argv.outputDirectories && argv.outputDirectory)
return true;
if (!argv.outputDirectory && !argv.outputDirectories)
argv.outputDirectory = 'videos'; // default out dir
else if (argv.outputDirectories)
argv.outputDirectory = argv.outputDirectories;
if (argv.outputDirectories) {
// these are not valid anymore
delete argv.outputDirectories;
delete argv.O;
}
return true;
}
// yeah this is for windows, but lets check everyone, who knows...
function windowsFileExtensionBadBehaviorFix(argv: any) {
if (hasNoArgs() || !argv.videoUrlsFile || !argv.outputDirectories)
return true;
if (!fs.existsSync(argv.videoUrlsFile)) {
if (fs.existsSync(argv.videoUrlsFile + '.txt'))
argv.videoUrlsFile += '.txt';
else
throw new Error(colors.red(CLI_ERROR.INPUT_URLS_FILE_NOT_FOUND));
}
return true;
}

63
src/Errors.ts Normal file
View file

@ -0,0 +1,63 @@
interface IError {
[key: number]: string
}
export const enum ERROR_CODE {
NO_ERROR,
UNHANDLED_ERROR,
MISSING_FFMPEG,
ELEVATED_SHELL,
INVALID_OUTPUT_DIR,
INVALID_INPUT_URLS,
OUTDIRS_URLS_MISMATCH,
INVALID_VIDEO_ID,
INVALID_VIDEO_GUID,
UNK_FFMPEG_ERROR,
NO_SESSION_INFO,
}
// TODO: create better errors descriptions
export const Error: IError = {
[ERROR_CODE.NO_ERROR]: 'Clean exit with code 0',
[ERROR_CODE.UNHANDLED_ERROR]: 'Unhandled error!\n' +
'Timeout or fatal error, please check your downloads directory and try again',
[ERROR_CODE.ELEVATED_SHELL]: 'Running in an elevated shell',
[ERROR_CODE.INVALID_OUTPUT_DIR]: 'Unable to create output directory',
[ERROR_CODE.MISSING_FFMPEG]: 'FFmpeg is missing!\n' +
'Destreamer requires a fairly recent release of FFmpeg to download videos',
[ERROR_CODE.UNK_FFMPEG_ERROR]: 'Unknown FFmpeg error',
[ERROR_CODE.INVALID_INPUT_URLS]: 'No valid URL in the input',
[ERROR_CODE.OUTDIRS_URLS_MISMATCH]: 'Output directories and URLs mismatch!\n' +
'You must input the same number of URLs and output directories',
[ERROR_CODE.INVALID_VIDEO_ID]: 'Unable to get video ID from URL',
[ERROR_CODE.INVALID_VIDEO_GUID]: 'Unable to get video GUID from URL',
[ERROR_CODE.NO_SESSION_INFO]: 'Could not evaluate sessionInfo on the page'
}
export const enum CLI_ERROR {
GRACEFULLY_STOP = ' ', // gracefully stop execution, yargs way
MISSING_REQUIRED_ARG = 'You must specify a URLs source.\n' +
'Valid options are --videoUrls or --videoUrlsFile.',
VIDEOURLS_ARG_CONFLICT = 'Too many URLs sources specified!\n' +
'Please specify a single URLs source with either --videoUrls or --videoUrlsFile.',
OUTPUTDIR_ARG_CONFLICT = 'Too many output arguments specified!\n' +
'Please specify a single output argument with either --outputDirectory or --outputDirectories.',
FILE_INPUT_VIDEOURLS_ARG = 'Wrong input for option --videoUrls.\n' +
'To read URLs from file, use --videoUrlsFile option.',
INPUT_URLS_FILE_NOT_FOUND = 'Input URL list file not found.'
}

27
src/Events.ts Normal file
View file

@ -0,0 +1,27 @@
import { Error, ERROR_CODE } from './Errors';
import colors from 'colors';
/**
* This file contains global destreamer process events
*
* @note SIGINT event is overridden in downloadVideo function
*
* @note function is required for non-packaged destreamer, so we can't do better
*/
export function setProcessEvents() {
// set exit event first so that we can always print cute errors
process.on('exit', (code) => {
if (code == 0)
return;
const msg = code in Error ? `\n\n${Error[code]} \n` : `\n\nUnknown error: exit code ${code} \n`;
console.error(colors.bgRed(msg));
});
process.on('unhandledRejection', (reason) => {
console.error(colors.red(reason as string));
process.exit(ERROR_CODE.UNHANDLED_ERROR);
});
}

6
src/Metadata.ts
View file

@ -25,7 +25,7 @@ export async function getVideoMetadata(videoGuids: string[], session: Session, v
let metadata: Metadata[] = [];
let title: string;
let date: string;
let duration: number;
let totalChunks: number;
let playbackUrl: string;
let posterImage: string;
@ -50,11 +50,11 @@ export async function getVideoMetadata(videoGuids: string[], session: Session, v
posterImage = response.data['posterImage']['medium']['url'];
date = publishedDateToString(response.data['publishedDate']);
duration = durationToTotalChunks(response.data.media['duration']);
totalChunks = durationToTotalChunks(response.data.media['duration']);
metadata.push({
date: date,
duration: duration,
totalChunks: totalChunks,
title: title,
playbackUrl: playbackUrl,
posterImage: posterImage

16
src/PuppeteerHelper.ts Normal file
View file

@ -0,0 +1,16 @@
import path from 'path';
import puppeteer from 'puppeteer';
// Thanks pkg-puppeteer [ cleaned up version :) ]
export function getPuppeteerChromiumPath() {
const isPkg = __filename.includes('snapshot');
const macOS_Linux_rex = /^.*?\/node_modules\/puppeteer\/\.local-chromium/;
const win32_rex = /^.*?\\node_modules\\puppeteer\\\.local-chromium/;
const replaceRegex = process.platform === 'win32' ? win32_rex : macOS_Linux_rex;
if (!isPkg)
return puppeteer.executablePath();
return puppeteer.executablePath()
.replace(replaceRegex, path.join(path.dirname(process.execPath), 'chromium'))
}

28
src/Types.ts
View file

@ -7,32 +7,8 @@ export type Session = {
export type Metadata = {
date: string;
duration: number;
totalChunks: number; // Abstraction of FFmpeg timemark
title: string;
playbackUrl: string;
posterImage: string;
}
interface Errors {
[key: number]: string
}
// I didn't use an enum because there is no real advantage that i can find and
// we can't use multiline string for long errors
// TODO: create better errors descriptions
export const Errors: Errors = {
22: 'FFmpeg is missing.\n' +
'Destreamer requires a fairly recent release of FFmpeg to download videos.\n' +
'Please install it in $PATH or copy the ffmpeg binary to the root directory (next to package.json). \n',
33: "Can't split videoId from videoUrl\n",
44: "Couldn't evaluate sessionInfo on the page\n",
55: 'Running in an elevated shell\n',
66: 'No valid URL in the input\n',
0: "Clean exit with code 0."
}
}

72
src/utils.ts → src/Utils.ts
View file

@ -1,9 +1,10 @@
import { ERROR_CODE } from './Errors';
import { execSync } from 'child_process';
import colors from 'colors';
import fs from 'fs';
import path from 'path';
function sanitizeUrls(urls: string[]) {
const rex = new RegExp(/(?:https:\/\/)?.*\/video\/[a-z0-9]{8}-(?:[a-z0-9]{4}\-){3}[a-z0-9]{12}$/, 'i');
const sanitized: string[] = [];
@ -26,17 +27,35 @@ function sanitizeUrls(urls: string[]) {
sanitized.push(url+query);
}
return sanitized.length ? sanitized : null;
if (!sanitized.length)
process.exit(ERROR_CODE.INVALID_INPUT_URLS);
return sanitized;
}
function sanitizeOutDirsList(dirsList: string[]) {
const sanitized: string[] = [];
dirsList.forEach(dir => {
if (dir !== '')
sanitized.push(dir);
});
return sanitized;
}
function readFileToArray(path: string) {
return fs.readFileSync(path).toString('utf-8').split(/[\r\n]/);
}
export function parseVideoUrls(videoUrls: any) {
const t = videoUrls[0] as string;
let t = videoUrls[0] as string;
const isPath = t.substring(t.length-4) === '.txt';
let urls: string[];
if (isPath)
urls = fs.readFileSync(t).toString('utf-8').split(/[\r\n]/);
urls = readFileToArray(t);
else
urls = videoUrls as string[];
@ -44,6 +63,47 @@ export function parseVideoUrls(videoUrls: any) {
}
export function getOutputDirectoriesList(outDirArg: string) {
const isList = outDirArg.substring(outDirArg.length-4) === '.txt';
let dirsList: string[];
if (isList)
dirsList = sanitizeOutDirsList(readFileToArray(outDirArg));
else
dirsList = [outDirArg];
return dirsList;
}
export function makeOutputDirectories(dirsList: string[]) {
dirsList.forEach(dir => {
if (!fs.existsSync(dir)) {
console.info(colors.yellow('Creating output directory:'));
console.info(colors.green(dir)+'\n');
try {
fs.mkdirSync(dir, { recursive: true });
} catch(e) {
process.exit(ERROR_CODE.INVALID_OUTPUT_DIR);
}
}
});
}
export function checkOutDirsUrlsMismatch(dirsList: string[], urlsList: string[]) {
const dirsListL = dirsList.length;
const urlsListL = urlsList.length;
if (dirsListL == 1) // one out dir, treat this as the chosen one for all
return;
else if (dirsListL != urlsListL)
process.exit(ERROR_CODE.OUTDIRS_URLS_MISMATCH);
}
export function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms));
}
@ -55,10 +115,8 @@ export function checkRequirements() {
console.info(colors.green(`Using ${ffmpegVer}\n`));
} catch (e) {
return null;
process.exit(ERROR_CODE.MISSING_FFMPEG);
}
return true;
}

233
src/destreamer.ts
View file

@ -1,90 +1,33 @@
import { sleep, parseVideoUrls, checkRequirements, makeUniqueTitle, ffmpegTimemarkToChunk } from './utils';
import {
sleep, parseVideoUrls, checkRequirements, makeUniqueTitle, ffmpegTimemarkToChunk,
makeOutputDirectories, getOutputDirectoriesList, checkOutDirsUrlsMismatch
} from './Utils';
import { getPuppeteerChromiumPath } from './PuppeteerHelper';
import { setProcessEvents } from './Events';
import { ERROR_CODE } from './Errors';
import { TokenCache } from './TokenCache';
import { getVideoMetadata } from './Metadata';
import { Metadata, Session, Errors } from './Types';
import { Metadata, Session } from './Types';
import { drawThumbnail } from './Thumbnail';
import { argv } from './CommandLineParser';
import isElevated from 'is-elevated';
import puppeteer from 'puppeteer';
import colors from 'colors';
import fs from 'fs';
import path from 'path';
import yargs from 'yargs';
import sanitize from 'sanitize-filename';
import ffmpeg from 'fluent-ffmpeg';
import cliProgress from 'cli-progress';
let tokenCache = new TokenCache();
const argv = yargs.options({
username: {
alias: 'u',
type: 'string',
demandOption: false
},
outputDirectory: {
alias: 'o',
type: 'string',
default: 'videos',
demandOption: false
},
videoUrls: {
alias: 'V',
describe: 'List of video urls or path to txt file containing the urls',
type: 'array',
demandOption: true
},
simulate: {
alias: 's',
describe: `Disable video download and print metadata information to the console`,
type: 'boolean',
default: false,
demandOption: false
},
noThumbnails: {
alias: 'nthumb',
describe: `Do not display video thumbnails`,
type: 'boolean',
default: false,
demandOption: false
},
verbose: {
alias: 'v',
describe: `Print additional information to the console (use this before opening an issue on GitHub)`,
type: 'boolean',
default: false,
demandOption: false
}
}).argv;
const { FFmpegCommand, FFmpegInput, FFmpegOutput } = require('@tedconf/fessonia')();
const tokenCache = new TokenCache();
async function init() {
process.on('unhandledRejection', (reason) => {
console.error(colors.red('Unhandled error!\nTimeout or fatal error, please check your downloads and try again if necessary.\n'));
console.error(colors.red(reason as string));
});
process.on('exit', (code) => {
if (code == 0) {
return
};
if (code in Errors)
console.error(colors.bgRed(`\n\nError: ${Errors[code]} \n`))
else
console.error(colors.bgRed(`\n\nUnknown exit code ${code} \n`))
});
setProcessEvents(); // must be first!
if (await isElevated())
process.exit(55);
process.exit(ERROR_CODE.ELEVATED_SHELL);
// create output directory
if (!fs.existsSync(argv.outputDirectory)) {
console.log('Creating output directory: ' +
process.cwd() + path.sep + argv.outputDirectory);
fs.mkdirSync(argv.outputDirectory);
}
console.info('Output Directory: %s', argv.outputDirectory);
checkRequirements();
if (argv.username)
console.info('Username: %s', argv.username);
@ -99,11 +42,11 @@ async function init() {
}
async function DoInteractiveLogin(url: string, username?: string): Promise<Session> {
let videoId = url.split("/").pop() ?? process.exit(33)
const videoId = url.split("/").pop() ?? process.exit(ERROR_CODE.INVALID_VIDEO_ID)
console.log('Launching headless Chrome to perform the OpenID Connect dance...');
const browser = await puppeteer.launch({
executablePath: getPuppeteerChromiumPath(),
headless: false,
args: ['--disable-dev-shm-usage']
});
@ -122,7 +65,7 @@ async function DoInteractiveLogin(url: string, username?: string): Promise<Sessi
console.info('We are logged in.');
let session = null;
let tries: number = 0;
let tries: number = 1;
while (!session) {
try {
@ -137,13 +80,12 @@ async function DoInteractiveLogin(url: string, username?: string): Promise<Sessi
}
);
} catch (error) {
if (tries < 5){
session = null;
tries++;
await sleep(3000);
} else {
process.exit(44)
}
if (tries > 5)
process.exit(ERROR_CODE.NO_SESSION_INFO);
session = null;
tries++;
await sleep(3000);
}
}
@ -157,7 +99,7 @@ async function DoInteractiveLogin(url: string, username?: string): Promise<Sessi
}
function extractVideoGuid(videoUrls: string[]): string[] {
let videoGuids: string[] = [];
const videoGuids: string[] = [];
let guid: string | undefined = '';
for (const url of videoUrls) {
@ -165,8 +107,8 @@ function extractVideoGuid(videoUrls: string[]): string[] {
guid = url.split('/').pop();
} catch (e) {
console.error(`Could not split the video GUID from URL: ${e.message}`);
process.exit(33);
console.error(`${e.message}`);
process.exit(ERROR_CODE.INVALID_VIDEO_GUID);
}
if (guid)
@ -181,16 +123,8 @@ function extractVideoGuid(videoUrls: string[]): string[] {
return videoGuids;
}
async function downloadVideo(videoUrls: string[], outputDirectory: string, session: Session) {
async function downloadVideo(videoUrls: string[], outputDirectories: string[], session: Session) {
const videoGuids = extractVideoGuid(videoUrls);
const pbar = new cliProgress.SingleBar({
barCompleteChar: '\u2588',
barIncompleteChar: '\u2591',
format: 'progress [{bar}] {percentage}% {speed}Kbps {eta_formatted}',
barsize: Math.floor(process.stdout.columns / 3),
stopOnComplete: true,
etaBuffer: 20
});
console.log('Fetching metadata...');
@ -208,12 +142,25 @@ async function downloadVideo(videoUrls: string[], outputDirectory: string, sessi
return;
}
await Promise.all(metadata.map(async video => {
console.log(colors.blue(`\nDownloading Video: ${video.title}\n`));
if (argv.verbose)
console.log(outputDirectories);
video.title = makeUniqueTitle(sanitize(video.title) + ' - ' + video.date, argv.outputDirectory);
const outDirsIdxInc = outputDirectories.length > 1 ? 1:0;
for (let i=0, j=0, l=metadata.length; i<l; ++i, j+=outDirsIdxInc) {
const video = metadata[i];
let previousChunks = 0;
const pbar = new cliProgress.SingleBar({
barCompleteChar: '\u2588',
barIncompleteChar: '\u2591',
format: 'progress [{bar}] {percentage}% {speed} {eta_formatted}',
barsize: Math.floor(process.stdout.columns / 3),
stopOnComplete: true,
hideCursor: true,
});
const outputPath = outputDirectory + path.sep + video.title + '.mp4';
console.log(colors.yellow(`\nDownloading Video: ${video.title}\n`));
video.title = makeUniqueTitle(sanitize(video.title) + ' - ' + video.date, outputDirectories[j]);
// Very experimental inline thumbnail rendering
if (!argv.noThumbnails)
@ -221,57 +168,69 @@ async function downloadVideo(videoUrls: string[], outputDirectory: string, sessi
console.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n');
ffmpeg()
.input(video.playbackUrl)
.inputOption([
// Never remove those "useless" escapes or ffmpeg will not
// pick up the header correctly
// eslint-disable-next-line no-useless-escape
'-headers', `Authorization:\ Bearer\ ${session.AccessToken}`,
])
.format('mp4')
.saveToFile(outputPath)
.on('codecData', data => {
console.log(`Input is ${data.video} with ${data.audio} audio.\n`);
const outputPath = outputDirectories[j] + path.sep + video.title + '.mp4';
const ffmpegInpt = new FFmpegInput(video.playbackUrl, new Map([
['headers', `Authorization:\ Bearer\ ${session.AccessToken}`]
]));
const ffmpegOutput = new FFmpegOutput(outputPath);
const ffmpegCmd = new FFmpegCommand();
pbar.start(video.duration, 0, {
speed: '0'
});
pbar.start(video.totalChunks, 0, {
speed: '0'
});
process.on('SIGINT', () => {
pbar.stop();
});
})
.on('progress', progress => {
const currentChunks = ffmpegTimemarkToChunk(progress.timemark);
// prepare ffmpeg command line
ffmpegCmd.addInput(ffmpegInpt);
ffmpegCmd.addOutput(ffmpegOutput);
pbar.update(currentChunks, {
speed: progress.currentKbps
});
})
.on('error', err => {
pbar.stop();
console.log(`ffmpeg returned an error: ${err.message}`);
})
.on('end', () => {
console.log(colors.green(`\nDownload finished: ${outputPath}`));
// set events
ffmpegCmd.on('update', (data: any) => {
const currentChunks = ffmpegTimemarkToChunk(data.out_time);
const incChunk = currentChunks - previousChunks;
pbar.increment(incChunk, {
speed: data.bitrate
});
}));
previousChunks = currentChunks;
});
ffmpegCmd.on('error', (error: any) => {
pbar.stop();
console.log(`\nffmpeg returned an error: ${error.message}`);
process.exit(ERROR_CODE.UNK_FFMPEG_ERROR);
});
process.on('SIGINT', () => {
pbar.stop();
});
// let the magic begin...
await new Promise((resolve: any, reject: any) => {
ffmpegCmd.on('success', (data:any) => {
pbar.update(video.totalChunks); // set progress bar to 100%
console.log(colors.green(`\nDownload finished: ${outputPath}`));
resolve();
});
ffmpegCmd.spawn();
});
}
}
async function main() {
checkRequirements() ?? process.exit(22);
await init();
await init(); // must be first
const videoUrls: string[] = parseVideoUrls(argv.videoUrls) ?? process.exit(66);
const outDirs: string[] = getOutputDirectoriesList(argv.outputDirectory as string);
const videoUrls: string[] = parseVideoUrls(argv.videoUrls);
let session: Session;
let session = tokenCache.Read();
checkOutDirsUrlsMismatch(outDirs, videoUrls);
makeOutputDirectories(outDirs); // create all dirs now to prevent ffmpeg panic
if (session == null) {
session = await DoInteractiveLogin(videoUrls[0], argv.username);
}
session = tokenCache.Read() ?? await DoInteractiveLogin(videoUrls[0], argv.username);
downloadVideo(videoUrls, argv.outputDirectory, session);
downloadVideo(videoUrls, outDirs, session);
}

2
test/test.ts
View file

@ -1,4 +1,4 @@
import { parseVideoUrls } from '../src/utils';
import { parseVideoUrls } from '../src/Utils';
import puppeteer from 'puppeteer';
import assert from 'assert';
import tmp from 'tmp';