diff --git a/.gitignore b/.gitignore index 4cfe012..29ecadd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,10 @@ -.token_cache +.* *.txt *.log *.js +*.zip node_modules videos - +release swagger.json -cc.json - -.thumbnail.png +cc.json \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..2271298 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,44 @@ +# Destreamer + + + CI build status + + +![](logo.png) + +## Saves Microsoft Stream videos for offline enjoyment + +## HOW TO BUILD FOR RELEASE +Destreamer builder supports the following environments: +* Linux +* WLS (Windows Linux Subsystem) +* MacOS + +Requirements +* [pkg](https://www.npmjs.com/package/pkg) +* wget + +`Install pkg to your system with the command:` +``` +npm i -g pkg +``` + +You will find your release package in destreamer root directory. + +To build a release package, run the following commands: +* `$ npm install` +* `$ cd scripts` +* `$ chmod +x make_release.sh` +* `$ ./make_release.sh` + +``` +Usage: ./make_realse.sh [option] + + help - Show usage + linux - Build for Linux x64 + win - Build for Windows x64 + macos - Build for MacOS x64 + all - Build all + + default: all +``` \ No newline at end of file diff --git a/README.md b/README.md index 3db6c10..f873403 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Roadmap - - [X] Token cache (so you don't have to log in every time you run destreamer) - [ ] Download closed captions if available - [ ] Performance improvements (via aria2c maybe?) // _This is under consideration, we're not sure if this borders on abusing the streaming endpoints or not._ -- [ ] Single static binary (for each major OS) +- [x] Single static binary (for each major OS) Send a quality PR first and i'll add you as a contributor to the repository. @@ -38,11 +38,33 @@ Hopefully this doesn't break the end user agreement for Microsoft Stream. Since Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) approach towards the OS it's running on, tested on Windows, macOS and Linux, results may vary, feel free to open an issue if trouble arise. + +## HOW TO BUILD +You can build destreamer on any OS. + +You will find destreamer.js in `build/src` folder. + +To build destreamer.js run the following commands: +* `npm install` +* `npm run -s build` + ## USAGE -* `npm install` to restore packages -* `npm run -s build` to transpile TypeScript to JavaScript +* Unpack destreamer and chromium into the same folder +* Open a new terminal and navigate to that folder +* Run destreamer executable +Linux / MacOS +``` +$ ./destreamer +``` + +Windows +``` +destreamer.exe +``` + +## Options ``` $ node ./destreamer.js diff --git a/package-lock.json b/package-lock.json index 1e0406f..95e5bec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -70,6 +70,11 @@ } } }, + "@tedconf/fessonia": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@tedconf/fessonia/-/fessonia-2.1.0.tgz", + "integrity": "sha512-p1iWLFMRyACW+tZV2ZTcGXt6uj7mvjl3uC1R2O1HHSnUsaS4c37llvtpHQJyiSy3p+SU+1oKWjsJYbDyZpziyw==" + }, "@types/cli-progress": { "version": "3.4.2", "resolved": "https://registry.npmjs.org/@types/cli-progress/-/cli-progress-3.4.2.tgz", @@ -89,14 +94,6 @@ "integrity": "sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==", "dev": true }, - "@types/fluent-ffmpeg": { - "version": "2.1.14", - "resolved": "https://registry.npmjs.org/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.14.tgz", - "integrity": "sha512-nJrAX9ODNI7mUB0b7Y0Stx1a6dOpV3zXsOnWoBuEd9/woQhepBNCMeCyOL6SLJD3jn5sLw5ciDGH0RwJenCoag==", - "requires": { - "@types/node": "*" - } - }, "@types/json-schema": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.4.tgz", @@ -325,11 +322,6 @@ "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, - "async": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz", - "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==" - }, "async-limiter": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", @@ -1017,15 +1009,6 @@ "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", "dev": true }, - "fluent-ffmpeg": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz", - "integrity": "sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=", - "requires": { - "async": ">=0.2.9", - "which": "^1.1.1" - } - }, "follow-redirects": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", diff --git a/package.json b/package.json index cb17a23..fcd3a51 100644 --- a/package.json +++ b/package.json @@ -6,10 +6,11 @@ }, "version": "1.0.0", "description": "Save Microsoft Stream videos for offline enjoyment.", - "main": "destreamer.js", + "main": "build/src/destreamer.js", + "bin": "build/src/destreamer.js", "scripts": { "build": "echo Transpiling TypeScript to JavaScript... & node node_modules/typescript/bin/tsc --listEmittedFiles", - "run": "node ./destreamer.js", + "run": "node ./build/src/destreamer.js", "start": "npm run -s build & npm run -s run", "test": "mocha build/test" }, @@ -28,13 +29,12 @@ "tmp": "^0.1.0" }, "dependencies": { + "@tedconf/fessonia": "^2.1.0", "@types/cli-progress": "^3.4.2", - "@types/fluent-ffmpeg": "^2.1.14", "@types/jwt-decode": "^2.2.1", "axios": "^0.19.2", "cli-progress": "^3.7.0", "colors": "^1.4.0", - "fluent-ffmpeg": "^2.1.2", "is-elevated": "^3.0.0", "iso8601-duration": "^1.2.0", "jwt-decode": "^2.2.0", diff --git a/scripts/make_release.sh b/scripts/make_release.sh new file mode 100755 index 0000000..d14cc42 --- /dev/null +++ b/scripts/make_release.sh @@ -0,0 +1,169 @@ +#!/bin/bash +set -euo pipefail +cd .. + +# vars +chromeRev=`cat node_modules/puppeteer/package.json | grep chromium_revision | grep -oP '"([0-9]+)"' | cut -d"\"" -f2` +baseUrl="https://storage.googleapis.com/chromium-browser-snapshots" +osAr=("Linux_x64" "Mac" "Win_x64") +zipAr=("linux" "mac" "win") +arg="all" + +function checkWget() { + command -v wget >/dev/null 2>&1 || { echo -e >&2 "I need wget to work :(\n"; exit 1; } +} + +function setupBuildForOS() { + case "$arg" in + "linux") + osAr=("Linux_x64") + zipAr=("linux") + ;; + "win") + osAr=("Win_x64") + zipAr=("win") + ;; + "macos") + osAr=("Mac") + zipAr=("mac") + ;; + *) + echo -e "\nInvalid OS selected!\n" + exit 1 + ;; + esac; +} + +function makeDirectories() { + if [ -d release ]; then + rm -R release + fi + + mkdir -p release/temp +} + +function buildDestreamer() { + npm run -s build +} + +function downloadChromiumPackages() { + local idx=0 + + for os in "${osAr[@]}" + do + local zipName="chrome-${zipAr[$idx]}.zip" + local finalUrl="$baseUrl/$os/$chromeRev/$zipName" + + wget "$finalUrl" -P "release/temp/$os" + unzip "release/temp/$os/$zipName" -d "release/temp/$os" + + ((++idx)) + done; +} + +function buildPkg() { + pkg . --out-path release + + cd release + + cp destreamer-linux temp/Linux_x64 + cp destreamer-macos temp/Mac + cp destreamer-win.exe temp/Win_x64 +} + +function buildPkgForOS() { + pkg -t "$arg" . --out-path release + + cd release + + cp destreamer* "temp/${osAr[0]}" +} + +function buildDestreamerReleasePackages() { + local idx=0 + + for os in "${osAr[@]}" + do + local chromeFold="chrome-${zipAr[$idx]}" + local osFolder="${zipAr[$idx]}-$chromeRev" + + if [[ "$os" == "Win_x64" ]]; then # windows fix + osFolder="win64-$chromeRev" + fi; + + cd "temp/$os" + mkdir -p "chromium/$osFolder" + mv "$chromeFold" "chromium/$osFolder" + zip -r "destreamer-$os.zip" chromium destreamer* + mv "destreamer-$os.zip" ../../.. + cd ../.. + + ((++idx)) + done; + + cd .. +} + +function usage() { + echo -e "Usage: $0 [option]\n" + echo " help - Show usage" + echo " linux - Build for Linux x64" + echo " win - Build for Windows x64" + echo " macos - Build for MacOS x64" + echo " all - Build all" + echo -e "\n default: all\n" +} + +function parseArgument() { + case "$arg" in + "all") + ;; + "linux"|"win"|"macos") + setupBuildForOS + ;; + *) + usage + exit 0 + ;; + esac; +} + +function main() { + clear + + echo "##############################" + echo "# Destreamer release builder #" + echo -e "##############################\n" + + parseArgument + checkWget + + echo -e "\n> \e[32mMaking directories...\e[39m" + makeDirectories + + echo -e "\n> \e[32mBuilding destreamer...\e[39m" + buildDestreamer + + echo -e "\n> \e[32mDownloading chromium packages...\e[39m" + downloadChromiumPackages + + echo -e "\n> \e[32mBuilding pkg...\e[39m" + if [[ "$arg" == "all" ]]; then + buildPkg + else + buildPkgForOS + fi; + + echo -e "\n> \e[32mBuilding destreamer release package\e[39m" + buildDestreamerReleasePackages + + rm -R release + exit 0 +} + +# run +if [[ $# -gt 0 ]]; then + arg="$1" +fi; + +main diff --git a/src/CommandLineParser.ts b/src/CommandLineParser.ts new file mode 100644 index 0000000..b887e23 --- /dev/null +++ b/src/CommandLineParser.ts @@ -0,0 +1,185 @@ +import { CLI_ERROR } from './Errors'; + +import yargs from 'yargs'; +import colors from 'colors'; +import fs from 'fs'; + +export const argv = yargs.options({ + videoUrls: { + alias: 'V', + describe: 'List of video urls', + type: 'array', + demandOption: false + }, + videoUrlsFile: { + alias: 'F', + describe: 'Path to txt file containing the urls', + type: 'string', + demandOption: false + }, + username: { + alias: 'u', + type: 'string', + demandOption: false + }, + outputDirectory: { + alias: 'o', + describe: 'The directory where destreamer will save your downloads [default: videos]', + type: 'string', + demandOption: false + }, + outputDirectories: { + alias: 'O', + describe: 'Path to a txt file containing one output directory per video', + type: 'string', + demandOption: false + }, + noThumbnails: { + alias: 'nthumb', + describe: `Do not display video thumbnails`, + type: 'boolean', + default: false, + demandOption: false + }, + simulate: { + alias: 's', + describe: `Disable video download and print metadata information to the console`, + type: 'boolean', + default: false, + demandOption: false + }, + verbose: { + alias: 'v', + describe: `Print additional information to the console (use this before opening an issue on GitHub)`, + type: 'boolean', + default: false, + demandOption: false + } +}) +/** + * Do our own argv magic before destreamer starts. + * ORDER IS IMPORTANT! + * Do not mess with this. + */ +.check(() => isShowHelpRequest()) +.check(argv => checkRequiredArgument(argv)) +.check(argv => checkVideoUrlsArgConflict(argv)) +.check(argv => checkOutputDirArgConflict(argv)) +.check(argv => checkVideoUrlsInput(argv)) +.check(argv => windowsFileExtensionBadBehaviorFix(argv)) +.check(argv => mergeVideoUrlsArguments(argv)) +.check(argv => mergeOutputDirArguments(argv)) +.argv; + +function hasNoArgs() { + return process.argv.length === 2; +} + +function isShowHelpRequest() { + if (hasNoArgs()) + throw new Error(CLI_ERROR.GRACEFULLY_STOP); + + return true; +} + +function checkRequiredArgument(argv: any) { + if (hasNoArgs()) + return true; + + if (!argv.videoUrls && !argv.videoUrlsFile) + throw new Error(colors.red(CLI_ERROR.MISSING_REQUIRED_ARG)); + + return true; +} + +function checkVideoUrlsArgConflict(argv: any) { + if (hasNoArgs()) + return true; + + if (argv.videoUrls && argv.videoUrlsFile) + throw new Error(colors.red(CLI_ERROR.VIDEOURLS_ARG_CONFLICT)); + + return true; +} + +function checkOutputDirArgConflict(argv: any) { + if (hasNoArgs()) + return true; + + if (argv.outputDirectory && argv.outputDirectories) + throw new Error(colors.red(CLI_ERROR.OUTPUTDIR_ARG_CONFLICT)); + + return true; +} + +function checkVideoUrlsInput(argv: any) { + if (hasNoArgs() || !argv.videoUrls) + return true; + + if (!argv.videoUrls.length) + throw new Error(colors.red(CLI_ERROR.MISSING_REQUIRED_ARG)); + + const t = argv.videoUrls[0] as string; + if (t.substring(t.length-4) === '.txt') + throw new Error(colors.red(CLI_ERROR.FILE_INPUT_VIDEOURLS_ARG)); + + return true; +} + +/** + * Users see 2 separate options, but we don't really care + * cause both options have no difference in code. + * + * Optimize and make this transparent to destreamer + */ +function mergeVideoUrlsArguments(argv: any) { + if (!argv.videoUrlsFile) + return true; + + argv.videoUrls = [argv.videoUrlsFile]; // noone will notice ;) + + // these are not valid anymore + delete argv.videoUrlsFile; + delete argv.F; + + return true; +} + +/** + * Users see 2 separate options, but we don't really care + * cause both options have no difference in code. + * + * Optimize and make this transparent to destreamer + */ +function mergeOutputDirArguments(argv: any) { + if (!argv.outputDirectories && argv.outputDirectory) + return true; + + if (!argv.outputDirectory && !argv.outputDirectories) + argv.outputDirectory = 'videos'; // default out dir + else if (argv.outputDirectories) + argv.outputDirectory = argv.outputDirectories; + + if (argv.outputDirectories) { + // these are not valid anymore + delete argv.outputDirectories; + delete argv.O; + } + + return true; +} + +// yeah this is for windows, but lets check everyone, who knows... +function windowsFileExtensionBadBehaviorFix(argv: any) { + if (hasNoArgs() || !argv.videoUrlsFile || !argv.outputDirectories) + return true; + + if (!fs.existsSync(argv.videoUrlsFile)) { + if (fs.existsSync(argv.videoUrlsFile + '.txt')) + argv.videoUrlsFile += '.txt'; + else + throw new Error(colors.red(CLI_ERROR.INPUT_URLS_FILE_NOT_FOUND)); + } + + return true; +} \ No newline at end of file diff --git a/src/Errors.ts b/src/Errors.ts new file mode 100644 index 0000000..11ae750 --- /dev/null +++ b/src/Errors.ts @@ -0,0 +1,63 @@ +interface IError { + [key: number]: string +} + +export const enum ERROR_CODE { + NO_ERROR, + UNHANDLED_ERROR, + MISSING_FFMPEG, + ELEVATED_SHELL, + INVALID_OUTPUT_DIR, + INVALID_INPUT_URLS, + OUTDIRS_URLS_MISMATCH, + INVALID_VIDEO_ID, + INVALID_VIDEO_GUID, + UNK_FFMPEG_ERROR, + NO_SESSION_INFO, +} + +// TODO: create better errors descriptions +export const Error: IError = { + [ERROR_CODE.NO_ERROR]: 'Clean exit with code 0', + + [ERROR_CODE.UNHANDLED_ERROR]: 'Unhandled error!\n' + + 'Timeout or fatal error, please check your downloads directory and try again', + + [ERROR_CODE.ELEVATED_SHELL]: 'Running in an elevated shell', + + [ERROR_CODE.INVALID_OUTPUT_DIR]: 'Unable to create output directory', + + [ERROR_CODE.MISSING_FFMPEG]: 'FFmpeg is missing!\n' + + 'Destreamer requires a fairly recent release of FFmpeg to download videos', + + [ERROR_CODE.UNK_FFMPEG_ERROR]: 'Unknown FFmpeg error', + + [ERROR_CODE.INVALID_INPUT_URLS]: 'No valid URL in the input', + + [ERROR_CODE.OUTDIRS_URLS_MISMATCH]: 'Output directories and URLs mismatch!\n' + + 'You must input the same number of URLs and output directories', + + [ERROR_CODE.INVALID_VIDEO_ID]: 'Unable to get video ID from URL', + + [ERROR_CODE.INVALID_VIDEO_GUID]: 'Unable to get video GUID from URL', + + [ERROR_CODE.NO_SESSION_INFO]: 'Could not evaluate sessionInfo on the page' +} + +export const enum CLI_ERROR { + GRACEFULLY_STOP = ' ', // gracefully stop execution, yargs way + + MISSING_REQUIRED_ARG = 'You must specify a URLs source.\n' + + 'Valid options are --videoUrls or --videoUrlsFile.', + + VIDEOURLS_ARG_CONFLICT = 'Too many URLs sources specified!\n' + + 'Please specify a single URLs source with either --videoUrls or --videoUrlsFile.', + + OUTPUTDIR_ARG_CONFLICT = 'Too many output arguments specified!\n' + + 'Please specify a single output argument with either --outputDirectory or --outputDirectories.', + + FILE_INPUT_VIDEOURLS_ARG = 'Wrong input for option --videoUrls.\n' + + 'To read URLs from file, use --videoUrlsFile option.', + + INPUT_URLS_FILE_NOT_FOUND = 'Input URL list file not found.' +} \ No newline at end of file diff --git a/src/Events.ts b/src/Events.ts new file mode 100644 index 0000000..aa8cd27 --- /dev/null +++ b/src/Events.ts @@ -0,0 +1,27 @@ +import { Error, ERROR_CODE } from './Errors'; + +import colors from 'colors'; + +/** + * This file contains global destreamer process events + * + * @note SIGINT event is overridden in downloadVideo function + * + * @note function is required for non-packaged destreamer, so we can't do better + */ +export function setProcessEvents() { + // set exit event first so that we can always print cute errors + process.on('exit', (code) => { + if (code == 0) + return; + + const msg = code in Error ? `\n\n${Error[code]} \n` : `\n\nUnknown error: exit code ${code} \n`; + + console.error(colors.bgRed(msg)); + }); + + process.on('unhandledRejection', (reason) => { + console.error(colors.red(reason as string)); + process.exit(ERROR_CODE.UNHANDLED_ERROR); + }); +} \ No newline at end of file diff --git a/src/Metadata.ts b/src/Metadata.ts index 6c642e5..1288177 100644 --- a/src/Metadata.ts +++ b/src/Metadata.ts @@ -25,7 +25,7 @@ export async function getVideoMetadata(videoGuids: string[], session: Session, v let metadata: Metadata[] = []; let title: string; let date: string; - let duration: number; + let totalChunks: number; let playbackUrl: string; let posterImage: string; @@ -50,11 +50,11 @@ export async function getVideoMetadata(videoGuids: string[], session: Session, v posterImage = response.data['posterImage']['medium']['url']; date = publishedDateToString(response.data['publishedDate']); - duration = durationToTotalChunks(response.data.media['duration']); + totalChunks = durationToTotalChunks(response.data.media['duration']); metadata.push({ date: date, - duration: duration, + totalChunks: totalChunks, title: title, playbackUrl: playbackUrl, posterImage: posterImage diff --git a/src/PuppeteerHelper.ts b/src/PuppeteerHelper.ts new file mode 100644 index 0000000..6820fb8 --- /dev/null +++ b/src/PuppeteerHelper.ts @@ -0,0 +1,16 @@ +import path from 'path'; +import puppeteer from 'puppeteer'; + +// Thanks pkg-puppeteer [ cleaned up version :) ] +export function getPuppeteerChromiumPath() { + const isPkg = __filename.includes('snapshot'); + const macOS_Linux_rex = /^.*?\/node_modules\/puppeteer\/\.local-chromium/; + const win32_rex = /^.*?\\node_modules\\puppeteer\\\.local-chromium/; + const replaceRegex = process.platform === 'win32' ? win32_rex : macOS_Linux_rex; + + if (!isPkg) + return puppeteer.executablePath(); + + return puppeteer.executablePath() + .replace(replaceRegex, path.join(path.dirname(process.execPath), 'chromium')) +} \ No newline at end of file diff --git a/src/Types.ts b/src/Types.ts index e686d1c..246bd78 100644 --- a/src/Types.ts +++ b/src/Types.ts @@ -7,32 +7,8 @@ export type Session = { export type Metadata = { date: string; - duration: number; + totalChunks: number; // Abstraction of FFmpeg timemark title: string; playbackUrl: string; posterImage: string; -} - - -interface Errors { - [key: number]: string -} - -// I didn't use an enum because there is no real advantage that i can find and -// we can't use multiline string for long errors -// TODO: create better errors descriptions -export const Errors: Errors = { - 22: 'FFmpeg is missing.\n' + - 'Destreamer requires a fairly recent release of FFmpeg to download videos.\n' + - 'Please install it in $PATH or copy the ffmpeg binary to the root directory (next to package.json). \n', - - 33: "Can't split videoId from videoUrl\n", - - 44: "Couldn't evaluate sessionInfo on the page\n", - - 55: 'Running in an elevated shell\n', - - 66: 'No valid URL in the input\n', - - 0: "Clean exit with code 0." -} +} \ No newline at end of file diff --git a/src/utils.ts b/src/Utils.ts similarity index 52% rename from src/utils.ts rename to src/Utils.ts index 19bd49c..2e915cc 100644 --- a/src/utils.ts +++ b/src/Utils.ts @@ -1,9 +1,10 @@ +import { ERROR_CODE } from './Errors'; + import { execSync } from 'child_process'; import colors from 'colors'; import fs from 'fs'; import path from 'path'; - function sanitizeUrls(urls: string[]) { const rex = new RegExp(/(?:https:\/\/)?.*\/video\/[a-z0-9]{8}-(?:[a-z0-9]{4}\-){3}[a-z0-9]{12}$/, 'i'); const sanitized: string[] = []; @@ -26,17 +27,35 @@ function sanitizeUrls(urls: string[]) { sanitized.push(url+query); } - return sanitized.length ? sanitized : null; + if (!sanitized.length) + process.exit(ERROR_CODE.INVALID_INPUT_URLS); + + return sanitized; +} + +function sanitizeOutDirsList(dirsList: string[]) { + const sanitized: string[] = []; + + dirsList.forEach(dir => { + if (dir !== '') + sanitized.push(dir); + }); + + return sanitized; +} + +function readFileToArray(path: string) { + return fs.readFileSync(path).toString('utf-8').split(/[\r\n]/); } export function parseVideoUrls(videoUrls: any) { - const t = videoUrls[0] as string; + let t = videoUrls[0] as string; const isPath = t.substring(t.length-4) === '.txt'; let urls: string[]; if (isPath) - urls = fs.readFileSync(t).toString('utf-8').split(/[\r\n]/); + urls = readFileToArray(t); else urls = videoUrls as string[]; @@ -44,6 +63,47 @@ export function parseVideoUrls(videoUrls: any) { } +export function getOutputDirectoriesList(outDirArg: string) { + const isList = outDirArg.substring(outDirArg.length-4) === '.txt'; + let dirsList: string[]; + + if (isList) + dirsList = sanitizeOutDirsList(readFileToArray(outDirArg)); + else + dirsList = [outDirArg]; + + return dirsList; +} + + +export function makeOutputDirectories(dirsList: string[]) { + dirsList.forEach(dir => { + if (!fs.existsSync(dir)) { + console.info(colors.yellow('Creating output directory:')); + console.info(colors.green(dir)+'\n'); + + try { + fs.mkdirSync(dir, { recursive: true }); + + } catch(e) { + process.exit(ERROR_CODE.INVALID_OUTPUT_DIR); + } + } + }); +} + + +export function checkOutDirsUrlsMismatch(dirsList: string[], urlsList: string[]) { + const dirsListL = dirsList.length; + const urlsListL = urlsList.length; + + if (dirsListL == 1) // one out dir, treat this as the chosen one for all + return; + else if (dirsListL != urlsListL) + process.exit(ERROR_CODE.OUTDIRS_URLS_MISMATCH); +} + + export function sleep(ms: number) { return new Promise(resolve => setTimeout(resolve, ms)); } @@ -55,10 +115,8 @@ export function checkRequirements() { console.info(colors.green(`Using ${ffmpegVer}\n`)); } catch (e) { - return null; + process.exit(ERROR_CODE.MISSING_FFMPEG); } - - return true; } diff --git a/src/destreamer.ts b/src/destreamer.ts index 87eee4f..9b7d9c7 100644 --- a/src/destreamer.ts +++ b/src/destreamer.ts @@ -1,90 +1,33 @@ -import { sleep, parseVideoUrls, checkRequirements, makeUniqueTitle, ffmpegTimemarkToChunk } from './utils'; +import { + sleep, parseVideoUrls, checkRequirements, makeUniqueTitle, ffmpegTimemarkToChunk, + makeOutputDirectories, getOutputDirectoriesList, checkOutDirsUrlsMismatch +} from './Utils'; +import { getPuppeteerChromiumPath } from './PuppeteerHelper'; +import { setProcessEvents } from './Events'; +import { ERROR_CODE } from './Errors'; import { TokenCache } from './TokenCache'; import { getVideoMetadata } from './Metadata'; -import { Metadata, Session, Errors } from './Types'; +import { Metadata, Session } from './Types'; import { drawThumbnail } from './Thumbnail'; +import { argv } from './CommandLineParser'; import isElevated from 'is-elevated'; import puppeteer from 'puppeteer'; import colors from 'colors'; -import fs from 'fs'; import path from 'path'; -import yargs from 'yargs'; import sanitize from 'sanitize-filename'; -import ffmpeg from 'fluent-ffmpeg'; import cliProgress from 'cli-progress'; -let tokenCache = new TokenCache(); - -const argv = yargs.options({ - username: { - alias: 'u', - type: 'string', - demandOption: false - }, - outputDirectory: { - alias: 'o', - type: 'string', - default: 'videos', - demandOption: false - }, - videoUrls: { - alias: 'V', - describe: 'List of video urls or path to txt file containing the urls', - type: 'array', - demandOption: true - }, - simulate: { - alias: 's', - describe: `Disable video download and print metadata information to the console`, - type: 'boolean', - default: false, - demandOption: false - }, - noThumbnails: { - alias: 'nthumb', - describe: `Do not display video thumbnails`, - type: 'boolean', - default: false, - demandOption: false - }, - verbose: { - alias: 'v', - describe: `Print additional information to the console (use this before opening an issue on GitHub)`, - type: 'boolean', - default: false, - demandOption: false - } -}).argv; +const { FFmpegCommand, FFmpegInput, FFmpegOutput } = require('@tedconf/fessonia')(); +const tokenCache = new TokenCache(); async function init() { - - process.on('unhandledRejection', (reason) => { - console.error(colors.red('Unhandled error!\nTimeout or fatal error, please check your downloads and try again if necessary.\n')); - console.error(colors.red(reason as string)); - }); - - process.on('exit', (code) => { - if (code == 0) { - return - }; - if (code in Errors) - console.error(colors.bgRed(`\n\nError: ${Errors[code]} \n`)) - else - console.error(colors.bgRed(`\n\nUnknown exit code ${code} \n`)) - }); + setProcessEvents(); // must be first! if (await isElevated()) - process.exit(55); + process.exit(ERROR_CODE.ELEVATED_SHELL); - // create output directory - if (!fs.existsSync(argv.outputDirectory)) { - console.log('Creating output directory: ' + - process.cwd() + path.sep + argv.outputDirectory); - fs.mkdirSync(argv.outputDirectory); - } - - console.info('Output Directory: %s', argv.outputDirectory); + checkRequirements(); if (argv.username) console.info('Username: %s', argv.username); @@ -99,11 +42,11 @@ async function init() { } async function DoInteractiveLogin(url: string, username?: string): Promise { - - let videoId = url.split("/").pop() ?? process.exit(33) + const videoId = url.split("/").pop() ?? process.exit(ERROR_CODE.INVALID_VIDEO_ID) console.log('Launching headless Chrome to perform the OpenID Connect dance...'); const browser = await puppeteer.launch({ + executablePath: getPuppeteerChromiumPath(), headless: false, args: ['--disable-dev-shm-usage'] }); @@ -122,7 +65,7 @@ async function DoInteractiveLogin(url: string, username?: string): Promise 5) + process.exit(ERROR_CODE.NO_SESSION_INFO); + + session = null; + tries++; + await sleep(3000); } } @@ -157,7 +99,7 @@ async function DoInteractiveLogin(url: string, username?: string): Promise { - console.log(colors.blue(`\nDownloading Video: ${video.title}\n`)); + if (argv.verbose) + console.log(outputDirectories); - video.title = makeUniqueTitle(sanitize(video.title) + ' - ' + video.date, argv.outputDirectory); + const outDirsIdxInc = outputDirectories.length > 1 ? 1:0; + for (let i=0, j=0, l=metadata.length; i { - console.log(`Input is ${data.video} with ${data.audio} audio.\n`); + const outputPath = outputDirectories[j] + path.sep + video.title + '.mp4'; + const ffmpegInpt = new FFmpegInput(video.playbackUrl, new Map([ + ['headers', `Authorization:\ Bearer\ ${session.AccessToken}`] + ])); + const ffmpegOutput = new FFmpegOutput(outputPath); + const ffmpegCmd = new FFmpegCommand(); - pbar.start(video.duration, 0, { - speed: '0' - }); + pbar.start(video.totalChunks, 0, { + speed: '0' + }); - process.on('SIGINT', () => { - pbar.stop(); - }); - }) - .on('progress', progress => { - const currentChunks = ffmpegTimemarkToChunk(progress.timemark); + // prepare ffmpeg command line + ffmpegCmd.addInput(ffmpegInpt); + ffmpegCmd.addOutput(ffmpegOutput); - pbar.update(currentChunks, { - speed: progress.currentKbps - }); - }) - .on('error', err => { - pbar.stop(); - console.log(`ffmpeg returned an error: ${err.message}`); - }) - .on('end', () => { - console.log(colors.green(`\nDownload finished: ${outputPath}`)); + // set events + ffmpegCmd.on('update', (data: any) => { + const currentChunks = ffmpegTimemarkToChunk(data.out_time); + const incChunk = currentChunks - previousChunks; + + pbar.increment(incChunk, { + speed: data.bitrate }); - })); + + previousChunks = currentChunks; + }); + + ffmpegCmd.on('error', (error: any) => { + pbar.stop(); + console.log(`\nffmpeg returned an error: ${error.message}`); + process.exit(ERROR_CODE.UNK_FFMPEG_ERROR); + }); + + process.on('SIGINT', () => { + pbar.stop(); + }); + + // let the magic begin... + await new Promise((resolve: any, reject: any) => { + ffmpegCmd.on('success', (data:any) => { + pbar.update(video.totalChunks); // set progress bar to 100% + console.log(colors.green(`\nDownload finished: ${outputPath}`)); + resolve(); + }); + + ffmpegCmd.spawn(); + }); + } } async function main() { - checkRequirements() ?? process.exit(22); - await init(); + await init(); // must be first - const videoUrls: string[] = parseVideoUrls(argv.videoUrls) ?? process.exit(66); + const outDirs: string[] = getOutputDirectoriesList(argv.outputDirectory as string); + const videoUrls: string[] = parseVideoUrls(argv.videoUrls); + let session: Session; - let session = tokenCache.Read(); + checkOutDirsUrlsMismatch(outDirs, videoUrls); + makeOutputDirectories(outDirs); // create all dirs now to prevent ffmpeg panic - if (session == null) { - session = await DoInteractiveLogin(videoUrls[0], argv.username); - } + session = tokenCache.Read() ?? await DoInteractiveLogin(videoUrls[0], argv.username); - downloadVideo(videoUrls, argv.outputDirectory, session); + downloadVideo(videoUrls, outDirs, session); } diff --git a/test/test.ts b/test/test.ts index c4dda9c..dab4395 100644 --- a/test/test.ts +++ b/test/test.ts @@ -1,4 +1,4 @@ -import { parseVideoUrls } from '../src/utils'; +import { parseVideoUrls } from '../src/Utils'; import puppeteer from 'puppeteer'; import assert from 'assert'; import tmp from 'tmp';