Merge branch 'ig-optim' into instagram

dist/twitter.js

@@ -224,10 +224,7 @@ class default_1 {
                                 .then(responseHandler);
                             const responseHandler = (res) => {
                                 if (res.status() !== 200) {
-                                    const err = new Error(`error navigating to user page, error was: ${res.status()} ${res.statusText()}`);
-                                    throw Object.defineProperty(err, 'name', {
-                                        value: 'ResponseError',
-                                    });
+                                    throw utils_1.customError('ResponseError')(`error navigating to user page, error was: ${res.status()} ${res.statusText()}`);
                                 }
                                 return res.json()
                                     .catch(redirectionHandler)
@@ -282,6 +279,14 @@ class default_1 {
             });
         };
         this.queryUser = (username) => this.client.user.searchExact(username)
+            .catch((error) => {
+            if (error instanceof instagram_private_api_1.IgLoginRequiredError) {
+                logger.warn('login required, logging in again...');
+                return this.session.login().then(() => this.client.user.searchExact(username));
+            }
+            else
+                throw error;
+        })
             .then(user => `${user.username}:${user.pk}`);
         this.workOnMedia = (lazyMediaItems, sendMedia) => this.webshot(lazyMediaItems, sendMedia, this.webshotDelay);
         this.urlSegmentToId = urlSegmentToId;
@@ -360,13 +365,13 @@ class default_1 {
                     return;
                 const workon = (queuedFeeds.length - 1) - (i * 5 + j);
                 fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
-                const promiseDelay = this.workInterval * (Math.random() + j) * 500 / lock.feed.length;
+                const promiseDelay = this.workInterval * (Math.random() + j) * 250 / lock.feed.length;
                 const startTime = new Date().getTime();
                 const getTimerTime = () => new Date().getTime() - startTime;
-                const promise = util_1.promisify(setTimeout)(promiseDelay).then(() => {
+                const promise = util_1.promisify(setTimeout)(promiseDelay * 3).then(() => {
                     logger.info(`about to pull from feed #${workon}: ${currentFeed}`);
                     if (j === arr.length - 1)
-                        logger.info(`timeout for this batch job: ${Math.trunc(promiseDelay * 2)} ms`);
+                        logger.info(`timeout for this batch job: ${Math.trunc(promiseDelay)} ms`);
                     return this.workOnFeed(currentFeed);
                 }).then(() => {
                     lock.workon = workon - 1;
@@ -375,17 +380,8 @@ class default_1 {
                     }
                     fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
                 });
-                return Promise.race([promise, isWaitingForLogin ? utils_1.neverResolves() : util_1.promisify(setTimeout)(promiseDelay * 3)]);
-            }))))
-                .then(() => {
-                let timeout = this.workInterval * 500;
-                if (timeout < 1000)
-                    timeout = 1000;
-                fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
-                setTimeout(() => {
-                    this.work();
-                }, timeout);
-            });
+                return Promise.race([promise, isWaitingForLogin ? utils_1.neverResolves() : util_1.promisify(setTimeout)(promiseDelay * 4)]);
+            })))).then(this.work);
         };
         this.client = new instagram_private_api_1.IgApiClient();
         if (opt.proxyUrl) {

dist/utils.js

@@ -1,10 +1,28 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Arr = exports.BigNumOps = exports.neverResolves = exports.chainPromises = void 0;
+exports.Arr = exports.BigNumOps = exports.rawRegExp = exports.customError = exports.CustomError = exports.neverResolves = exports.chainPromises = void 0;
+const CallableInstance = require("callable-instance");
 const chainPromises = (lazyPromises, reducer = (lp1, lp2) => (p) => lp1(p).then(lp2), initialValue) => lazyPromises.reduce(reducer, p => Promise.resolve(p))(initialValue);
 exports.chainPromises = chainPromises;
 const neverResolves = () => new Promise(() => undefined);
 exports.neverResolves = neverResolves;
+class CustomErrorConstructor extends CallableInstance {
+    constructor(name) {
+        super('getError');
+        Object.defineProperty(this.constructor, 'name', { value: name });
+    }
+    getError(message) { return new CustomError(this.constructor.name, message); }
+}
+class CustomError extends Error {
+    constructor(name, message) {
+        super(message);
+        this.name = name;
+        Object.defineProperty(this.constructor, 'name', { value: name });
+    }
+}
+exports.CustomError = CustomError;
+const customError = (name) => new CustomErrorConstructor(name);
+exports.customError = customError;
 const chunkArray = (arr, size) => {
     const noOfChunks = Math.ceil(size && arr.length / size);
     const res = Array(noOfChunks);
@@ -13,7 +31,9 @@ const chunkArray = (arr, size) => {
     }
     return res;
 };
-const splitBigNumAt = (num, at) => num.replace(RegExp(String.raw `^([+-]?)(\d+)(\d{${at}})$`), '$1$2,$1$3')
+const rawRegExp = (...args) => RegExp(String.raw(...args));
+exports.rawRegExp = rawRegExp;
+const splitBigNumAt = (num, at) => num.replace(exports.rawRegExp `^([+-]?)(\d+)(\d{${at}})$`, '$1$2,$1$3')
     .replace(/^([^,]*)$/, '0,$1').split(',')
     .map(Number);
 const bigNumPlus = (num1, num2) => {

src/twitter.ts

@@ -11,14 +11,14 @@ import {
 } from 'instagram-id-to-url-segment';
 import {
   IgApiClient,
-  IgClientError, IgExactUserNotFoundError, IgLoginTwoFactorRequiredError, IgResponseError,
+  IgClientError, IgExactUserNotFoundError, IgLoginRequiredError, IgLoginTwoFactorRequiredError, IgResponseError,
   MediaInfoResponseItemsItem, UserFeedResponseItemsItem
 } from 'instagram-private-api';
 import { SocksProxyAgent } from 'socks-proxy-agent';
 
 import { getLogger } from './loggers';
 import QQBot from './koishi';
-import { Arr, BigNumOps, chainPromises, neverResolves } from './utils';
+import { Arr, BigNumOps, chainPromises, customError, neverResolves } from './utils';
 import Webshot, { Cookies, Page } from './webshot';
 
 const parseLink = (link: string): { userName?: string, postUrlSegment?: string } => {
@@ -425,12 +425,9 @@ export default class {
                     .then(responseHandler);
                 const responseHandler = (res: typeof response): ReturnType<typeof response.json> => {
                   if (res.status() !== 200) {
-                    const err = new Error(
+                    throw customError('ResponseError')(
                       `error navigating to user page, error was: ${res.status()} ${res.statusText()}`
                     );
-                    throw Object.defineProperty(err, 'name', {
-                      value: 'ResponseError',
-                    });
                   }
                   return res.json()
                     .catch(redirectionHandler)
@@ -488,6 +485,12 @@ export default class {
   public queryUserMedia: (username: string, targetId?: string) => Promise<LazyMediaItem[]>;
 
   public queryUser = (username: string) => this.client.user.searchExact(username)
+    .catch((error: IgClientError) => {
+      if (error instanceof IgLoginRequiredError) {
+        logger.warn('login required, logging in again...');
+        return this.session.login().then(() => this.client.user.searchExact(username));
+      } else throw error;
+    })
     .then(user => `${user.username}:${user.pk}`);
 
   private workOnMedia = (
@@ -583,13 +586,13 @@ export default class {
 
         const workon = (queuedFeeds.length - 1) - (i * 5 + j);
         fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
-        const promiseDelay = this.workInterval * (Math.random() + j) * 500 / lock.feed.length;
+        const promiseDelay = this.workInterval * (Math.random() + j) * 250 / lock.feed.length;
         const startTime = new Date().getTime();
         const getTimerTime = () => new Date().getTime() - startTime;
 
-        const promise = promisify(setTimeout)(promiseDelay).then(() => {
+        const promise = promisify(setTimeout)(promiseDelay * 3).then(() => {
           logger.info(`about to pull from feed #${workon}: ${currentFeed}`);
-          if (j === arr.length - 1) logger.info(`timeout for this batch job: ${Math.trunc(promiseDelay * 2)} ms`);
+          if (j === arr.length - 1) logger.info(`timeout for this batch job: ${Math.trunc(promiseDelay)} ms`);
           return this.workOnFeed(currentFeed);
         }).then(() => {
           lock.workon = workon - 1;
@@ -599,16 +602,8 @@ export default class {
           fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
         });
 
-        return Promise.race([promise, isWaitingForLogin ? neverResolves() : promisify(setTimeout)(promiseDelay * 3)]);
+        return Promise.race([promise, isWaitingForLogin ? neverResolves() : promisify(setTimeout)(promiseDelay * 4)]);
       }))
-    ))
-      .then(() => {
-        let timeout = this.workInterval * 500;
-        if (timeout < 1000) timeout = 1000;
-        fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
-        setTimeout(() => {
-          this.work();
-        }, timeout);
-      });
+    )).then(this.work);
   };
 }

src/utils.ts

@@ -1,3 +1,5 @@
+import * as CallableInstance from 'callable-instance';
+
 export const chainPromises = <T>(
   lazyPromises: ((p: T) => Promise<T>)[],
   reducer = (lp1: (p: T) => Promise<T>, lp2: (p: T) => Promise<T>) => (p: T) => lp1(p).then(lp2),
@@ -6,6 +8,24 @@ export const chainPromises = <T>(
 
 export const neverResolves = () => new Promise<never>(() => undefined);
 
+class CustomErrorConstructor<Name extends string> extends CallableInstance<[string], CustomError<Name>> {
+  constructor(name: Name) {
+    super('getError');
+    Object.defineProperty(this.constructor, 'name', {value: name});
+  }
+  getError(message?: string) { return new CustomError(this.constructor.name, message); }
+}
+
+export class CustomError<Name extends string> extends Error {
+  constructor(name: Name, message?: string) {
+    super(message);
+    this.name = name;
+    Object.defineProperty(this.constructor, 'name', {value: name});
+  }
+}
+
+export const customError = <Name extends string>(name: Name) => new CustomErrorConstructor(name);
+
 const chunkArray = <T>(arr: T[], size: number) => {
   const noOfChunks = Math.ceil(size && arr.length / size);
   const res = Array<T[]>(noOfChunks);
@@ -15,7 +35,9 @@ const chunkArray = <T>(arr: T[], size: number) => {
   return res;
 };
 
-const splitBigNumAt = (num: string, at: number) => num.replace(RegExp(String.raw`^([+-]?)(\d+)(\d{${at}})$`), '$1$2,$1$3')
+export const rawRegExp = (...args: Parameters<typeof String.raw>) => RegExp(String.raw(...args));
+
+const splitBigNumAt = (num: string, at: number) => num.replace(rawRegExp`^([+-]?)(\d+)(\d{${at}})$`, '$1$2,$1$3')
   .replace(/^([^,]*)$/, '0,$1').split(',')
   .map(Number);