redownload missing files; add sending delay

dist/koishi.js

@@ -47,11 +47,11 @@ class default_1 {
                 });
             }
         };
-        this.enqueue = (type, id, resolver) => {
+        this.enqueue = (type, id, resolver, delay) => {
             var _a, _b;
             let wasEmpty = false;
             const queue = (_a = this.messageQueues)[_b = `${type}:${id}`] || (_a[_b] = (() => { wasEmpty = true; return []; })());
-            queue.push(() => (0, koishi_1.sleep)(200).then(resolver));
+            queue.push(() => (0, koishi_1.sleep)(wasEmpty ? Math.random() * 3000 + 600 : delay).then(resolver));
             logger.debug(`no. of message currently queued for ${type}:${id}: ${queue.length}`);
             if (wasEmpty)
                 this.next(type, id);
@@ -84,20 +84,23 @@ class default_1 {
                     };
             }
         });
-        this.sendToGroup = (groupID, message) => new Promise((resolve, reject) => {
-            this.enqueue('group', groupID, () => this.bot.sendMessage(groupID, message).then(resolve).catch(reject));
+        this.sendToGroup = (groupID, message, delay = 0) => new Promise((resolve, reject) => {
+            this.enqueue('group', groupID, () => this.bot.sendMessage(groupID, message).then(resolve).catch(reject), delay);
         });
-        this.sendToUser = (userID, message) => new Promise((resolve, reject) => {
-            this.enqueue('private', userID, () => this.bot.sendPrivateMessage(userID, message).then(resolve).catch(reject));
+        this.sendToUser = (userID, message, delay = 0) => new Promise((resolve, reject) => {
+            this.enqueue('private', userID, () => this.bot.sendPrivateMessage(userID, message).then(resolve).catch(reject), delay);
         });
-        this.sendTo = (subscriber, messageChain, noErrors = false) => Promise.all((splitted => [splitted.message, ...splitted.attachments])(exports.Message.separateAttachment(messageChain)).map(msg => {
+        this.sendTo = (subscriber, messageChain, noErrors = false) => Promise.all((splitted => [splitted.message, ...splitted.attachments])(exports.Message.separateAttachment(messageChain)).map((msg, index) => {
+            const delay = (/\[CQ:/.exec(msg) ? Math.random() * 25600 : 0) +
+                (index === 0 ? Math.random() * 16000 : 0) +
+                Math.random() * 3000 + 600;
             switch (subscriber.chatType) {
                 case 'group':
-                    return this.sendToGroup(subscriber.chatID.toString(), msg);
+                    return this.sendToGroup(subscriber.chatID.toString(), msg, delay);
                 case 'private':
-                    return this.sendToUser(subscriber.chatID.toString(), msg);
+                    return this.sendToUser(subscriber.chatID.toString(), msg, delay);
                 case 'temp':
-                    return this.sendToUser(subscriber.chatID.qq.toString(), msg);
+                    return this.sendToUser(subscriber.chatID.qq.toString(), msg, delay);
             }
         }))
             .then(response => {

dist/twitter.js

@@ -198,7 +198,7 @@ class default_1 {
                 const timeout = Math.max(1000, this.workInterval * 1000 / this.lock.feed.length);
                 setInterval(() => { this.pullOrders = utils_1.Arr.shuffle(this.pullOrders); }, 21600000);
                 setTimeout(this.workForAll, timeout);
-                setTimeout(this.work, timeout * 1.2);
+                this.work();
             });
         };
         this.queryUserObject = (userName) => this.client.user.searchExact(userName)
@@ -224,7 +224,16 @@ class default_1 {
                 return `${username}:${pk}`;
             });
         };
-        this.workOnMedia = (mediaItems, sendMedia) => Promise.resolve(mediaItems.forEach(({ msgs, text, author }) => sendMedia(msgs, text, author)));
+        this.workOnMedia = (mediaItems, sendMedia) => (0, utils_1.chainPromises)(mediaItems.map(({ msgs, text, author, original }) => {
+            const findFilePath = (mediaMsg) => /=file:\/\/(.*?)]/.exec(mediaMsg)[1];
+            const filePath = findFilePath(msgs);
+            return () => (fs.existsSync(filePath) ?
+                Promise.resolve() :
+                this.webshot.fetchBestCandidate(original).then(mediaMsg => {
+                    logger.warn(`media file missing, refetched media for ${author}/${original.code}`);
+                    return (0, util_1.promisify)(fs.rename)(findFilePath(mediaMsg), filePath);
+                })).then(() => sendMedia(msgs, text, author));
+        }));
         this.sendStories = (source, ...to) => (msg, text, author) => {
             to.forEach(subscriber => {
                 logger.info(`pushing data${source ? ` of ${source}` : ''} to ${JSON.stringify(subscriber)}`);
@@ -315,7 +324,7 @@ class default_1 {
             const lock = this.lock;
             const timeout = Math.max(1000, this.workInterval * 1000 / lock.feed.length);
             const nextTurn = () => { setTimeout(this.work, timeout); return; };
-            if (this.isInactiveTime || lock.feed.length === 0)
+            if (lock.feed.length === 0)
                 return nextTurn();
             if (lock.workon >= lock.feed.length)
                 lock.workon = 0;

dist/webshot.js

@@ -57,6 +57,18 @@ class Webshot extends CallableInstance {
             logger.warn('unable to find MIME type of fetched media, failing this fetch');
             throw Error();
         })(/\/.+\.(?:.*?(?<=[?&])stp=dst-(jpg)|(.+?)\?)/.exec(url).filter(g => g)[1]));
+        this.fetchBestCandidate = ({ image_versions2, video_versions }) => {
+            const candidates = video_versions || image_versions2.candidates;
+            const url = candidates
+                .sort((var1, var2) => var2.width + ((var2 === null || var2 === void 0 ? void 0 : var2.type) || 0) - var1.width - ((var1 === null || var1 === void 0 ? void 0 : var1.type) || 0))
+                .map(variant => variant.url)[0];
+            const altMessage = `\n[失败的${typeInZH[video_versions ? 'video' : 'photo'].type}：${url}]`;
+            return this.fetchMedia(url)
+                .catch(error => {
+                logger.warn('unable to fetch media, sending plain text instead...');
+                return altMessage;
+            });
+        };
         onready();
     }
     webshot(mediaItems, callback, webshotDelay) {
@@ -67,27 +79,8 @@ class Webshot extends CallableInstance {
             const author = `${item.user.full_name} (@${item.user.username}):\n`;
             const date = `${new Date(item.taken_at * 1000)}\n`;
             messageChain += author + date;
-            const type = (mediaItem) => mediaItem.video_versions ? 'video' : 'photo';
-            const fetchBestCandidate = (candidates, mediaType) => {
-                const url = candidates
-                    .sort((var1, var2) => var2.width + ((var2 === null || var2 === void 0 ? void 0 : var2.type) || 0) - var1.width - ((var1 === null || var1 === void 0 ? void 0 : var1.type) || 0))
-                    .map(variant => variant.url)[0];
-                const altMessage = `\n[失败的${typeInZH[mediaType].type}：${url}]`;
-                return this.fetchMedia(url)
-                    .catch(error => {
-                    logger.warn('unable to fetch media, sending plain text instead...');
-                    return altMessage;
-                })
-                    .then(msg => { messageChain += msg; });
-            };
-            promise = promise.then(() => {
-                if (item.video_versions) {
-                    return fetchBestCandidate(item.video_versions, type(item));
-                }
-                else if (item.image_versions2) {
-                    return fetchBestCandidate(item.image_versions2.candidates, type(item));
-                }
-            });
+            promise = promise.then(() => this.fetchBestCandidate(item))
+                .then(msg => { messageChain += msg; });
             return promise.then(() => {
                 logger.info(`done working on ${item.user.username}/${item.code}, message chain:`);
                 logger.info(JSON.stringify(koishi_1.Message.ellipseBase64(messageChain)));

package.json

@@ -30,6 +30,7 @@
     "lint": "npx eslint --fix --ext .ts ./"
   },
   "dependencies": {
+    "axios": "^0.21.4",
     "callable-instance": "^2.0.0",
     "command-line-usage": "^5.0.5",
     "html-entities": "^1.3.1",

src/koishi.ts

@@ -60,10 +60,10 @@ export default class {
     }
   };
 
-  private enqueue = (type: 'private' | 'group', id: string, resolver: () => Promise<void>) => {
+  private enqueue = (type: 'private' | 'group', id: string, resolver: () => Promise<void>, delay: number) => {
     let wasEmpty = false;
     const queue = this.messageQueues[`${type}:${id}`] ||= (() => { wasEmpty = true; return []; })();
-    queue.push(() => sleep(200).then(resolver));
+    queue.push(() => sleep(wasEmpty ? Math.random() * 3000 + 600 : delay).then(resolver));
     logger.debug(`no. of message currently queued for ${type}:${id}: ${queue.length}`);
     if (wasEmpty) this.next(type, id);
   };
@@ -97,25 +97,29 @@ export default class {
     }
   };
 
-  private sendToGroup = (groupID: string, message: string) => new Promise<string>((resolve, reject) => {
-    this.enqueue('group', groupID, () => this.bot.sendMessage(groupID, message).then(resolve).catch(reject));
+  private sendToGroup = (groupID: string, message: string, delay = 0) => new Promise<string>((resolve, reject) => {
+    this.enqueue('group', groupID, () => this.bot.sendMessage(groupID, message).then(resolve).catch(reject), delay);
   });
 
-  private sendToUser = (userID: string, message: string) => new Promise<string>((resolve, reject) => {
-    this.enqueue('private', userID, () => this.bot.sendPrivateMessage(userID, message).then(resolve).catch(reject));
+  private sendToUser = (userID: string, message: string, delay = 0) => new Promise<string>((resolve, reject) => {
+    this.enqueue('private', userID, () => this.bot.sendPrivateMessage(userID, message).then(resolve).catch(reject), delay);
   });
 
   public sendTo = (subscriber: IChat, messageChain: string, noErrors = false) => Promise.all(
     (splitted => [splitted.message, ...splitted.attachments])(
       Message.separateAttachment(messageChain)
-    ).map(msg => {
+    ).map((msg, index) => {
+      const delay =
+        (/\[CQ:/.exec(msg) ? Math.random() * 25600 : 0) +
+        (index === 0 ? Math.random() * 16000 : 0) +
+        Math.random() * 3000 + 600;
       switch (subscriber.chatType) {
         case 'group':
-          return this.sendToGroup(subscriber.chatID.toString(), msg);
+          return this.sendToGroup(subscriber.chatID.toString(), msg, delay);
         case 'private':
-          return this.sendToUser(subscriber.chatID.toString(), msg);
+          return this.sendToUser(subscriber.chatID.toString(), msg, delay);
         case 'temp': // currently unable to open session, awaiting OneBot v12
-          return this.sendToUser(subscriber.chatID.qq.toString(), msg);
+          return this.sendToUser(subscriber.chatID.qq.toString(), msg, delay);
       }
     }))
     .then(response => {

src/twitter.ts

@@ -391,7 +391,7 @@ export default class {
         const timeout = Math.max(1000, this.workInterval * 1000 / this.lock.feed.length);
         setInterval(() => { this.pullOrders = Arr.shuffle(this.pullOrders); }, 21600000);
         setTimeout(this.workForAll, timeout);
-        setTimeout(this.work, timeout * 1.2);
+        this.work();
       }
     );
   };
@@ -422,7 +422,18 @@ export default class {
   private workOnMedia = (
     mediaItems: CachedMediaItem[],
     sendMedia: (msg: string, text: string, author: string) => void
-  ) => Promise.resolve(mediaItems.forEach(({msgs, text, author}) => sendMedia(msgs, text, author)));
+  ) => chainPromises(mediaItems.map(({msgs, text, author, original}) => {
+    const findFilePath = (mediaMsg) => /=file:\/\/(.*?)]/.exec(mediaMsg)[1];
+    const filePath = findFilePath(msgs);
+    return () =>
+      (fs.existsSync(filePath) ?
+        Promise.resolve() :
+        this.webshot.fetchBestCandidate(original).then(mediaMsg => {
+          logger.warn(`media file missing, refetched media for ${author}/${original.code}`);
+          return promisify(fs.rename)(findFilePath(mediaMsg), filePath);
+        })
+      ).then(() => sendMedia(msgs, text, author));
+  }));
 
   private sendStories = (source?: string, ...to: IChat[]) => (msg: string, text: string, author: string) => {
     to.forEach(subscriber => {
@@ -557,7 +568,7 @@ export default class {
     const lock = this.lock;
     const timeout = Math.max(1000, this.workInterval * 1000 / lock.feed.length);
     const nextTurn = () => { setTimeout(this.work, timeout); return; };
-    if (this.isInactiveTime || lock.feed.length === 0) return nextTurn();
+    if (lock.feed.length === 0) return nextTurn();
     if (lock.workon >= lock.feed.length) lock.workon = 0;
 
     const currentFeed = lock.feed[lock.workon];    

src/webshot.ts

@@ -65,6 +65,21 @@ class Webshot extends CallableInstance<[MediaItem[], (...args) => void, number],
     })(/\/.+\.(?:.*?(?<=[?&])stp=dst-(jpg)|(.+?)\?)/.exec(url).filter(g => g)[1])
   );
 
+  public fetchBestCandidate = ({image_versions2, video_versions}: MediaItem) => {
+    const candidates: (
+      Partial<typeof video_versions[0]> & typeof image_versions2.candidates[0]
+    )[] = video_versions || image_versions2.candidates;
+    const url = candidates
+      .sort((var1, var2) => var2.width + (var2?.type || 0) - var1.width - (var1?.type || 0))
+      .map(variant => variant.url)[0]; // largest media
+    const altMessage = `\n[失败的${typeInZH[video_versions ? 'video' : 'photo'].type}：${url}]`;
+    return this.fetchMedia(url)
+      .catch(error => {
+        logger.warn('unable to fetch media, sending plain text instead...');
+        return altMessage;
+      });
+  };
+
   public webshot(
     mediaItems: MediaItem[],
     callback: (msgs: string, text: string, author: string) => void,
@@ -81,31 +96,8 @@ class Webshot extends CallableInstance<[MediaItem[], (...args) => void, number],
       messageChain += author + date;
 
       // fetch extra entities
-      const type = (mediaItem): keyof typeof typeInZH =>
-        (mediaItem as MediaItem).video_versions ? 'video' : 'photo';
-      const fetchBestCandidate =(
-        candidates: (Partial<typeof item.video_versions[0]> & typeof item.image_versions2.candidates[0])[],
-        mediaType: keyof typeof typeInZH
-      ) => {
-        const url = candidates
-          .sort((var1, var2) => var2.width + (var2?.type || 0) - var1.width - (var1?.type || 0))
-          .map(variant => variant.url)[0]; // largest media
-        const altMessage = `\n[失败的${typeInZH[mediaType].type}：${url}]`;
-        return this.fetchMedia(url)
-          .catch(error => {
-            logger.warn('unable to fetch media, sending plain text instead...');
-            return altMessage;
-          })
-          .then(msg => { messageChain += msg; });
-      };
-
-      promise = promise.then(() => {
-        if (item.video_versions) {
-          return fetchBestCandidate(item.video_versions, type(item));
-        } else if (item.image_versions2) {
-          return fetchBestCandidate(item.image_versions2.candidates, type(item));
-        }
-      });
+      promise = promise.then(() => this.fetchBestCandidate(item))
+        .then(msg => { messageChain += msg; });
       return promise.then(() => {
         logger.info(`done working on ${item.user.username}/${item.code}, message chain:`);
         logger.info(JSON.stringify(Message.ellipseBase64(messageChain)));