Merge branch 'koishi-redis-waiting' into mediaonly-koishi-redis-waiting

dist/webshot.js

@@ -163,9 +163,7 @@ class Webshot extends CallableInstance {
                             const path = temp.path({ suffix: '.html' });
                             fs.writeFileSync(path, html);
                             logger.warn(`saved debug html to ${path}`);
-                        }).then(() => page.route('**/*', route => route.abort().catch(err => {
-                            logger.error(`error aborting all requests for debug screenshot, error: ${err}`);
-                        }))).then(() => page.screenshot({ fullPage: true })).then(screenshot => {
+                        }).then(() => page.route('**/*', route => route.abort().catch(() => { }))).then(() => page.screenshot({ fullPage: true })).then(screenshot => {
                             sharpToFile(sharp(screenshot).jpeg({ quality: 90 })).then(fileUri => {
                                 logger.warn(`saved debug screenshot to ${fileUri.substring(7)}`);
                             });
@@ -285,19 +283,6 @@ class Webshot extends CallableInstance {
             });
             if (this.mode === 0) {
                 const url = `https://mobile.twitter.com/${twi.user.screen_name}/status/${twi.id_str}`;
-                const extendEntity = (cardMedia) => {
-                    var _a;
-                    return (media => {
-                        if (!media.some(entity => entity.id_str === cardMedia.id_str))
-                            media.push(cardMedia);
-                    })((_a = (originTwi.extended_entities || (originTwi.extended_entities = {}))).media || (_a.media = []));
-                };
-                const truncateLongThread = (atId) => {
-                    if (!atId)
-                        return;
-                    logger.info(`thread too long, truncating at tweet ${atId}...`);
-                    truncatedAt = atId;
-                };
                 promise = promise.then(() => this.renderWebshot(url, 1920, webshotDelay, (_, tweetHandle) => tweetHandle.evaluate(div => {
                     try {
                         const selector = '[data-testid="tweet"] :nth-child(2)>:first-child a';
@@ -314,35 +299,48 @@ class Webshot extends CallableInstance {
                     }
                     catch (_a) { }
                     document.documentElement.scrollTop = 0;
-                }).then(truncateLongThread), (page, tweetHandle) => tweetHandle.evaluate(div => {
-                    const cardMedia = div.querySelector('div[data-testid^="card.layout"][data-testid$=".media"] img, video');
+                }).then((id) => {
+                    if (!id)
+                        return;
+                    logger.info(`thread too long, truncating at tweet ${id}...`);
+                    truncatedAt = id;
+                }), (page, tweetHandle) => tweetHandle.evaluate(div => {
+                    const cardMediaDiv = div.querySelector('div[data-testid^="card.layout"][data-testid$=".media"]');
+                    const cardMedia = cardMediaDiv === null || cardMediaDiv === void 0 ? void 0 : cardMediaDiv.querySelector('img, video');
+                    if (!cardMedia)
+                        return {};
                     let match;
-                    if ((cardMedia === null || cardMedia === void 0 ? void 0 : cardMedia.tagName) === 'IMG' && typeof (cardMedia === null || cardMedia === void 0 ? void 0 : cardMedia.getAttribute('src')) === 'string') {
-                        match = /^(.*\/card_img\/(\d+)\/.+\?format=.*)&name=/.exec(cardMedia === null || cardMedia === void 0 ? void 0 : cardMedia.getAttribute('src'));
+                    if (cardMedia.tagName === 'IMG' && typeof cardMedia.getAttribute('src') === 'string') {
+                        match = /^(.*\/card_img\/(\d+)\/.+\?format=.*)&name=/.exec(cardMedia.getAttribute('src'));
                     }
-                    if ((cardMedia === null || cardMedia === void 0 ? void 0 : cardMedia.tagName) === 'VIDEO' && typeof (cardMedia === null || cardMedia === void 0 ? void 0 : cardMedia.getAttribute('poster')) === 'string') {
-                        match = /^(.*\/amplify_video_thumb\/(\d+)\/img\/.*$)/.exec(cardMedia === null || cardMedia === void 0 ? void 0 : cardMedia.getAttribute('poster'));
-                    }
-                    if (match) {
-                        const [media_url_https, id_str] = match.slice(1);
-                        return {
-                            type: cardMedia.tagName,
-                            entityBase: {
-                                media_url: media_url_https.replace(/^https/, 'http'),
-                                media_url_https,
-                                url: '',
-                                display_url: '',
-                                expanded_url: '',
-                                id: Number(id_str),
-                                id_str,
-                                sizes: undefined,
-                            }
-                        };
+                    if (cardMedia.tagName === 'VIDEO' && typeof cardMedia.getAttribute('poster') === 'string') {
+                        match = /^(.*\/amplify_video_thumb\/(\d+)\/img\/.*$)/.exec(cardMedia.getAttribute('poster'));
                     }
-                    return {};
+                    if (!match)
+                        return {};
+                    const [media_url_https, id_str] = match.slice(1);
+                    return {
+                        type: cardMedia.tagName,
+                        entityBase: {
+                            media_url: media_url_https.replace(/^https/, 'http'),
+                            media_url_https,
+                            url: '',
+                            display_url: '',
+                            expanded_url: '',
+                            id: Number(id_str),
+                            id_str,
+                            sizes: undefined,
+                        }
+                    };
                 }).then(({ type, entityBase }) => {
+                    var _a;
+                    if (!type)
+                        return;
+                    const media = (_a = (originTwi.extended_entities || (originTwi.extended_entities = {}))).media || (_a.media = []);
+                    if (media.some(entity => entity.id_str === entityBase.id_str))
+                        return;
                     if (type === 'IMG')
-                        extendEntity(Object.assign(Object.assign({}, entityBase), { type: 'photo' }));
+                        media.push(Object.assign(Object.assign({}, entityBase), { type: 'photo' }));
                     if (type === 'VIDEO')
                         page.evaluate(id_str => {
                             var _a;
@@ -370,7 +368,7 @@ class Webshot extends CallableInstance {
                                         }]
                                 }));
                             });
-                        })).then(videoInfo => extendEntity(Object.assign(Object.assign({}, entityBase), { type: 'video', video_info: videoInfo }))).catch(error => {
+                        })).then(videoInfo => media.push(Object.assign(Object.assign({}, entityBase), { type: 'video', video_info: videoInfo }))).catch(error => {
                             logger.error(`error while fetching scraped video, error: ${error}`);
                             logger.warn('unable to fetch scraped video, ignoring...');
                         });

src/webshot.ts

@@ -200,9 +200,7 @@ class Webshot extends CallableInstance<[Tweet[], (...args) => void, number], Pro
                 const path = temp.path({suffix: '.html'});
                 fs.writeFileSync(path, html);
                 logger.warn(`saved debug html to ${path}`);
-              }).then(() => page.route('**/*', route => route.abort().catch(err => {
-                logger.error(`error aborting all requests for debug screenshot, error: ${err}`);
-              }))
+              }).then(() => page.route('**/*', route => route.abort().catch(() => {/* ignore */}))
               ).then(() => page.screenshot({fullPage: true})).then(screenshot => {
                 sharpToFile(sharp(screenshot).jpeg({ quality: 90 })).then(fileUri => {
                   logger.warn(`saved debug screenshot to ${fileUri.substring(7)}`);
@@ -325,15 +323,6 @@ class Webshot extends CallableInstance<[Tweet[], (...args) => void, number], Pro
       // invoke webshot
       if (this.mode === 0) {
         const url = `https://mobile.twitter.com/${twi.user.screen_name}/status/${twi.id_str}`;
-        const extendEntity = (cardMedia: MediaEntity) =>
-          (media => {
-            if (!media.some(entity => entity.id_str === cardMedia.id_str)) media.push(cardMedia);
-          })((originTwi.extended_entities ||= {}).media ||= []);
-        const truncateLongThread = (atId: string) => {
-          if (!atId) return;
-          logger.info(`thread too long, truncating at tweet ${atId}...`);
-          truncatedAt = atId;
-        };
 
         promise = promise.then(() => this.renderWebshot(url, 1920, webshotDelay,
 
@@ -353,37 +342,44 @@ class Webshot extends CallableInstance<[Tweet[], (...args) => void, number], Pro
               }
             } catch {/* handle errors like none-found cases */}
             document.documentElement.scrollTop = 0;
-          }).then(truncateLongThread),
+          }).then((id: string) => {
+            if (!id) return;
+            logger.info(`thread too long, truncating at tweet ${id}...`);
+            truncatedAt = id;
+          }),
 
           // scrape card media from main tweet
           (page, tweetHandle: puppeteer.ElementHandle<HTMLDivElement>) => tweetHandle.evaluate(div => {
-            const cardMedia = div.querySelector('div[data-testid^="card.layout"][data-testid$=".media"] img, video');
+            const cardMediaDiv = div.querySelector('div[data-testid^="card.layout"][data-testid$=".media"]');
+            const cardMedia = cardMediaDiv?.querySelector('img, video');
+            if (!cardMedia) return {};
             let match: RegExpExecArray;
-            if (cardMedia?.tagName === 'IMG' && typeof cardMedia?.getAttribute('src') === 'string') {
-              match = /^(.*\/card_img\/(\d+)\/.+\?format=.*)&name=/.exec(cardMedia?.getAttribute('src'));
+            if (cardMedia.tagName === 'IMG' && typeof cardMedia.getAttribute('src') === 'string') {
+              match = /^(.*\/card_img\/(\d+)\/.+\?format=.*)&name=/.exec(cardMedia.getAttribute('src'));
             }
-            if (cardMedia?.tagName === 'VIDEO' && typeof cardMedia?.getAttribute('poster') === 'string') {
-              match = /^(.*\/amplify_video_thumb\/(\d+)\/img\/.*$)/.exec(cardMedia?.getAttribute('poster'));
+            if (cardMedia.tagName === 'VIDEO' && typeof cardMedia.getAttribute('poster') === 'string') {
+              match = /^(.*\/amplify_video_thumb\/(\d+)\/img\/.*$)/.exec(cardMedia.getAttribute('poster'));
             }
-            if (match) {
-              const [media_url_https, id_str] = match.slice(1);
-              return {
-                type: cardMedia.tagName,
-                entityBase: {
-                  media_url: media_url_https.replace(/^https/, 'http'),
-                  media_url_https,
-                  url: '',
-                  display_url: '',
-                  expanded_url: '',
-                  id: Number(id_str),
-                  id_str,
-                  sizes: undefined,
-                }
-              };
-            }
-            return {};
+            if (!match) return {};
+            const [media_url_https, id_str] = match.slice(1);
+            return {
+              type: cardMedia.tagName,
+              entityBase: {
+                media_url: media_url_https.replace(/^https/, 'http'),
+                media_url_https,
+                url: '',
+                display_url: '',
+                expanded_url: '',
+                id: Number(id_str),
+                id_str,
+                sizes: undefined,
+              }
+            };
           }).then(({type, entityBase}) => {
-            if (type === 'IMG') extendEntity({
+            if (!type) return;
+            const media = (originTwi.extended_entities ||= {}).media ||= [];
+            if (media.some(entity => entity.id_str === entityBase.id_str)) return;
+            if (type === 'IMG') media.push({
               ...entityBase,
               type: 'photo',
             });
@@ -419,7 +415,7 @@ class Webshot extends CallableInstance<[Tweet[], (...args) => void, number], Pro
                       }) as MediaEntity['video_info'])
                     });
                 })
-              ).then(videoInfo => extendEntity({
+              ).then(videoInfo => media.push({
                 ...entityBase,
                 type: 'video',
                 video_info: videoInfo,