feat: get queue length and concurrency through arguments

README.md

@@ -14,15 +14,23 @@ Copy `.env.example` to `.env` and fill the environment variables based on the Mu
 
 All of the videos from `videos.json` will be uploaded directly to Mux without saving them to the disk.
 
-This process is kinda slow but, if you have a good internet connection and hardware, you can increment the concurrency inside `src/lib/queue.ts` to migrate more videos in parallel.
+This process is kinda slow but, if you have a good internet connection and hardware, you can increment the concurrency passing as argv argument to migrate more videos in parallel, following the example below.
+
+```sh
+$ yarn run-migration 3
+```
 
 ## Getting the Playback ID list
 
 If you're not using signed URLs, you'll need the playback ID to play the videos.
 
-1. Open up the file `src/GetPlaybackIds.ts`;
-2. At line 18 you'll be able to change the page every time you run this script (for now you have to manually set the page number so if you have 850 assets you'll have to run this script 9 times);
-3. Run `yarn get-playback-ids` or `npm run get-playback-ids`;
+The mux assets list are paginated so you'll need to provide the number of the assets you want to reach.
+
+```sh
+# if you want to get 500 videos playbacks id
+
+$ yarn get-playback-ids 500
+```
 
 The script will fill the `data/export.csv` file with the data including the original video ID from `data/videos.json`, the playback ID e the duration.
 

src/GetPlaybackIds.ts

@@ -6,16 +6,26 @@ import path from 'path';
 
 import { mux } from './services/mux';
 
+const QUEUE_LENGTH = Number(process.argv[2])
+
+const isQueueLengthInvalid = isNaN(QUEUE_LENGTH)
+
+if (!QUEUE_LENGTH) throw new Error('Videos quantity must be provided')
+
+if (isQueueLengthInvalid) throw new Error('Queue length must be a number')
+
+const TIMES_TO_RUN = Math.ceil(QUEUE_LENGTH / 100)
+
+let times_ran = 0
+
 const outStream = fs.createWriteStream(path.join(__dirname, '..', 'data', 'export.csv'), {
   flags: 'a',
 });
 
-async function getPlaybackIds() {
+async function getPlaybackIds(page: number) {
   const assets = await mux.Video.Assets.list({
     limit: 100,
-
-    // Change this from 1 to N based on how many assets you have and rerun the script
-    page: 1, 
+    page, 
   })
 
   console.log(chalk.green(`Importing ${assets.length} videos.`));
@@ -40,6 +50,12 @@ async function getPlaybackIds() {
       Math.floor(Number(asset.duration)),
     ].join(',')}\n`);
   })
+
+  times_ran++
+
+  if (times_ran < TIMES_TO_RUN) {
+    getPlaybackIds(times_ran + 1)
+  }
 }
 
-getPlaybackIds()
+getPlaybackIds(1)

src/lib/queue.ts

@@ -1,14 +1,17 @@
 import fastq, { queueAsPromised } from "fastq"
 import { worker } from './worker'
 
+const DEFAULT_CONCURRENCY = 1
+
+const CONCURRENCY = Number(process.argv[2]) || DEFAULT_CONCURRENCY
+
+const isConcurrencyInvalid = isNaN(CONCURRENCY)
+
+if (isConcurrencyInvalid && CONCURRENCY) throw new Error('Queue concurrency must be a number')
+
 export type QueueTask = {
   videoId: string;
   url: string;
 }
 
-/**
- * How many videos will be processed in parallel.
- */
-const CONCURRENCY = 1;
-
 export const queue: queueAsPromised<QueueTask> = fastq.promise(worker, CONCURRENCY)