properly update working progress

dist/twitter.js

@@ -318,11 +318,15 @@ class default_1 {
             });
             const queuedFeeds = lock.feed.slice(0, lock.workon ? lock.workon + 1 : undefined).reverse();
             utils_1.chainPromises(utils_1.Arr.chunk(queuedFeeds, 5).map((arr, i) => () => Promise.all(arr.map((currentFeed, j) => {
-                lock.workon = (queuedFeeds.length - 1) - (i * 5 + j);
+                const workon = (queuedFeeds.length - 1) - (i * 5 + j);
                 fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
                 const promiseDelay = this.workInterval * (Math.random() + j) * 500 / lock.feed.length;
-                logger.debug(`timeout for this pull job: ${promiseDelay * 2}`);
+                const startTime = new Date().getTime();
+                const getTimerTime = () => new Date().getTime() - startTime;
                 const promise = util_1.promisify(setTimeout)(promiseDelay).then(() => {
+                    logger.info(`about to pull from feed #${workon}: ${currentFeed}`);
+                    if (j === arr.length - 1)
+                        logger.info(`timeout for this batch job: ${Math.trunc(promiseDelay * 2)} ms`);
                     const match = /https:\/\/www\.instagram\.com\/([^\/]+)/.exec(currentFeed);
                     if (!match) {
                         logger.error(`current feed "${currentFeed}" is invalid, please remove this feed manually`);
@@ -333,8 +337,7 @@ class default_1 {
                         logger.error(`error scraping media off profile page of ${match[1]}, error: ${error}`);
                         return [];
                     });
-                });
-                promise.then((mediaItems) => {
+                }).then((mediaItems) => {
                     const currentThread = lock.threads[currentFeed];
                     const updateDate = () => currentThread.updatedAt = new Date().toString();
                     if (!mediaItems || mediaItems.length === 0) {
@@ -351,8 +354,14 @@ class default_1 {
                         mediaItems.splice(1);
                     return this.workOnMedia(mediaItems, this.sendMedia(`thread ${currentFeed}`, ...currentThread.subscribers))
                         .then(updateDate).then(updateOffset);
-                }).then(() => fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock)));
-                return util_1.promisify(setTimeout)(promiseDelay * 3);
+                }).then(() => {
+                    lock.workon = workon - 1;
+                    if (j === arr.length - 1) {
+                        logger.info(`batch job #${workon}-${workon + j} completed after ${getTimerTime()} ms`);
+                    }
+                    fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
+                });
+                return Promise.race([promise, util_1.promisify(setTimeout)(promiseDelay * 3)]);
             }))))
                 .then(() => {
                 let timeout = this.workInterval * 500;

src/twitter.ts

@@ -496,13 +496,15 @@ export default class {
     const queuedFeeds = lock.feed.slice(0, lock.workon ? lock.workon + 1 : undefined).reverse();
     chainPromises(Arr.chunk(queuedFeeds, 5).map((arr, i) =>
       () => Promise.all(arr.map((currentFeed, j) => {
-        lock.workon = (queuedFeeds.length - 1) - (i * 5 + j);
+        const workon = (queuedFeeds.length - 1) - (i * 5 + j);
         fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
-
         const promiseDelay = this.workInterval * (Math.random() + j) * 500 / lock.feed.length;
-        logger.debug(`timeout for this pull job: ${promiseDelay * 2}`);
+        const startTime = new Date().getTime();
+        const getTimerTime = () => new Date().getTime() - startTime;
 
         const promise = promisify(setTimeout)(promiseDelay).then(() => {
+          logger.info(`about to pull from feed #${workon}: ${currentFeed}`);
+          if (j === arr.length - 1) logger.info(`timeout for this batch job: ${Math.trunc(promiseDelay * 2)} ms`);
           const match = /https:\/\/www\.instagram\.com\/([^\/]+)/.exec(currentFeed);
           if (!match) {
             logger.error(`current feed "${currentFeed}" is invalid, please remove this feed manually`);
@@ -513,9 +515,7 @@ export default class {
               logger.error(`error scraping media off profile page of ${match[1]}, error: ${error}`);
               return [] as LazyMediaItem[];
             });
-        });
-
-        promise.then((mediaItems: LazyMediaItem[]) => {
+        }).then((mediaItems: LazyMediaItem[]) => {
           const currentThread = lock.threads[currentFeed];
 
           const updateDate = () => currentThread.updatedAt = new Date().toString();
@@ -529,9 +529,15 @@ export default class {
 
           return this.workOnMedia(mediaItems, this.sendMedia(`thread ${currentFeed}`, ...currentThread.subscribers))
             .then(updateDate).then(updateOffset);
-        }).then(() => fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock)));
+        }).then(() => {
+          lock.workon = workon - 1;
+          if (j === arr.length - 1) {
+            logger.info(`batch job #${workon}-${workon + j} completed after ${getTimerTime()} ms`);
+          }
+          fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
+        });
 
-        return promisify(setTimeout)(promiseDelay * 3);
+        return Promise.race([promise, promisify(setTimeout)(promiseDelay * 3)]);
       }))
     ))
       .then(() => {