fix pull capped at 30 accounts max

dist/twitter.js

@@ -224,15 +224,15 @@ class default_1 {
                     return idToUserMap[id] = user;
                 });
             }))
-                .then(() => {
-                logger.debug(`pulling stories for users: ${Object.values(idToUserMap).map(user => user.username)}`);
-                return this.client.feed.reelsMedia({ userIds: Object.keys(idToUserMap) }).items()
+                .then(() => utils_1.chainPromises(utils_1.Arr.chunk(utils_1.Arr.shuffle(Object.keys(idToUserMap)), 20).map(userIds => () => {
+                logger.info(`pulling stories for users: ${userIds.map(id => idToUserMap[id].username)}`);
+                return this.client.feed.reelsMedia({ userIds }).items()
                     .then(storyItems => storyItems.forEach(item => {
                     if (!(item.pk in this.cache[idToUserMap[item.user.pk].username].stories)) {
                         this.cache[idToUserMap[item.user.pk].username].stories[item.pk] = item;
                     }
                 }));
-            })
+            }), (lp1, lp2) => () => lp1().then(() => util_1.promisify(setTimeout)(this.workInterval * 1000).then(lp2))))
                 .catch((error) => {
                 if (error instanceof instagram_private_api_1.IgNetworkError) {
                     logger.warn(`error on fetching stories for all: ${JSON.stringify(error.cause)}`);

dist/utils.js

@@ -1,8 +1,24 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.BigNumOps = exports.chainPromises = void 0;
+exports.Arr = exports.BigNumOps = exports.chainPromises = void 0;
 const chainPromises = (lazyPromises, reducer = (lp1, lp2) => (p) => lp1(p).then(lp2), initialValue) => lazyPromises.reduce(reducer, p => Promise.resolve(p))(initialValue);
 exports.chainPromises = chainPromises;
+const shuffleArray = (arr) => {
+    const res = arr.slice(0);
+    for (let i = arr.length - 1; i > 0; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        [res[i], res[j]] = [res[j], res[i]];
+    }
+    return res;
+};
+const chunkArray = (arr, size) => {
+    const noOfChunks = Math.ceil(size && arr.length / size);
+    const res = Array(noOfChunks);
+    for (let [i, j] = [0, 0]; i < noOfChunks; i++) {
+        res[i] = arr.slice(j, j += size);
+    }
+    return res;
+};
 const splitBigNumAt = (num, at) => num.replace(RegExp(String.raw `^([+-]?)(\d+)(\d{${at}})$`), '$1$2,$1$3')
     .replace(/^([^,]*)$/, '0,$1').split(',')
     .map(Number);
@@ -49,3 +65,4 @@ exports.BigNumOps = {
     lShift: bigNumLShift,
     parse: parseBigNum,
 };
+exports.Arr = { chunk: chunkArray, shuffle: shuffleArray };

src/twitter.ts

@@ -15,7 +15,7 @@ import { SocksProxyAgent } from 'socks-proxy-agent';
 
 import { getLogger } from './loggers';
 import QQBot from './koishi';
-import { BigNumOps } from './utils';
+import { Arr, BigNumOps, chainPromises } from './utils';
 import Webshot from './webshot';
 
 const parseLink = (link: string): {userName?: string, storyId?: string} => {
@@ -350,15 +350,18 @@ export default class {
         return idToUserMap[id] = user as UserFeedResponseUser;
       });
     }))
-      .then(() => {
-        logger.debug(`pulling stories for users: ${Object.values(idToUserMap).map(user => user.username)}`);
-        return this.client.feed.reelsMedia({userIds: Object.keys(idToUserMap)}).items()
-          .then(storyItems => storyItems.forEach(item => {
-            if (!(item.pk in this.cache[idToUserMap[item.user.pk].username].stories)) {
-              this.cache[idToUserMap[item.user.pk].username].stories[item.pk] = item;
-            }
-          }));
-      })
+      .then(() => chainPromises(
+        Arr.chunk(Arr.shuffle(Object.keys(idToUserMap)), 20).map(userIds => () => {
+          logger.info(`pulling stories for users: ${userIds.map(id => idToUserMap[id as unknown as number].username)}`);
+          return this.client.feed.reelsMedia({userIds}).items()
+            .then(storyItems => storyItems.forEach(item => {
+              if (!(item.pk in this.cache[idToUserMap[item.user.pk].username].stories)) {
+                this.cache[idToUserMap[item.user.pk].username].stories[item.pk] = item;
+              }
+            }));
+        }),
+        (lp1, lp2) => () => lp1().then(() => promisify(setTimeout)(this.workInterval * 1000).then(lp2))
+      ))
       .catch((error: IgClientError & Partial<RequestError>) => {
         if (error instanceof IgNetworkError) {
           logger.warn(`error on fetching stories for all: ${JSON.stringify(error.cause)}`);

src/utils.ts

@@ -4,6 +4,24 @@ export const chainPromises = <T>(
   initialValue?: T
 ) => lazyPromises.reduce(reducer, p => Promise.resolve(p))(initialValue);
 
+const shuffleArray = <T>(arr: T[]) => {
+  const res = arr.slice(0);
+  for (let i = arr.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1));
+    [res[i], res[j]] = [res[j], res[i]];
+  }
+  return res;
+};
+
+const chunkArray = <T>(arr: T[], size: number) => {
+  const noOfChunks = Math.ceil(size && arr.length / size);
+  const res = Array<T[]>(noOfChunks);
+  for (let [i, j] = [0, 0]; i < noOfChunks; i++) {
+    res[i] = arr.slice(j, j += size);
+  }
+  return res;
+};
+
 const splitBigNumAt = (num: string, at: number) => num.replace(RegExp(String.raw`^([+-]?)(\d+)(\d{${at}})$`), '$1$2,$1$3')
   .replace(/^([^,]*)$/, '0,$1').split(',')
   .map(Number);
@@ -53,3 +71,5 @@ export const BigNumOps = {
   lShift: bigNumLShift,
   parse: parseBigNum,
 };
+
+export const Arr = {chunk: chunkArray, shuffle: shuffleArray};