Merge branch 'koishi-redis-waiting' into mediaonly-koishi-redis-waiting

dist/webshot.js

@@ -88,7 +88,8 @@ class Webshot extends CallableInstance {
                             '[data-testid="caret"],[role="group"],[data-testid="tweet"] [class*=" "]+:last-child>*+[class*=" "]~div{display:none}',
                     }))
                         .then(() => page.addStyleTag({
-                        content: '*{font-family:-apple-system,".Helvetica Neue DeskInterface",Hiragino Sans,Hiragino Sans GB,sans-serif!important}',
+                        content: '*{font-family:-apple-system,".Helvetica Neue DeskInterface",Hiragino Sans,Hiragino Sans GB,sans-serif!important}' +
+                            '*{-webkit-font-smoothing:antialiased!important}',
                     }))
                         .then(() => page.evaluate(() => {
                         const poll = setInterval(() => {
@@ -101,12 +102,15 @@ class Webshot extends CallableInstance {
                         }, 250);
                     }))
                         .then(() => page.waitForSelector('xpath=//section/*/*/div[.//article[not(.//time[not(ancestor::div[@aria-labelledby])])]]', { state: 'attached', timeout: getTimeout() }))
-                        .then(handle => handle.$$('xpath=..//a[contains(@href,"content_you_see")]/../../..//*[@role="button"]')
+                        .then(handle => handle.evaluate(div => div.classList.add('mainTweet'))
+                        .then(() => page.addStyleTag({ content: 'div.mainTweet~div{display:none;}' }))
+                        .then(() => handle))
+                        .then(handle => handle.$$('xpath=(.|preceding-sibling::*)//a[contains(@href,"content_you_see")]/../../..//*[@role="button"]')
                         .then(sensitiveToggles => {
                         const count = sensitiveToggles.length;
                         if (count)
                             logger.info(`found ${count} sensitive ${count === 1 ? 'tweet' : 'tweets'} on page, uncollapsing...`);
-                        return (0, utils_1.chainPromises)(sensitiveToggles.filter(toggle => toggle.isVisible()).map(toggle => () => toggle.click()));
+                        return (0, utils_1.chainPromises)(sensitiveToggles.map(toggle => () => toggle.click()));
                     })
                         .then(() => handle))
                         .then(handle => handle.$('[data-testid="tweet"]').then(owner => owner ? handle : null))
@@ -118,7 +122,7 @@ class Webshot extends CallableInstance {
                             const path = temp.path({ suffix: '.html' });
                             (0, fs_1.writeFileSync)(path, html);
                             logger.warn(`saved debug html to ${path}`);
-                        }).then(() => page.screenshot()).then(screenshot => {
+                        }).then(() => page.route('**/*', route => route.abort())).then(() => page.screenshot({ fullPage: true })).then(screenshot => {
                             sharpToFile(sharp(screenshot).jpeg({ quality: 90 })).then(fileUri => {
                                 logger.warn(`saved debug screenshot to ${fileUri.substring(7)}`);
                             });
@@ -127,45 +131,30 @@ class Webshot extends CallableInstance {
                         .then(handle => {
                         if (handle === null)
                             throw new puppeteer.errors.TimeoutError();
-                        return (0, utils_1.chainPromises)(morePostProcessings.map(func => () => func(page, handle)));
+                        return (0, utils_1.chainPromises)(morePostProcessings.map(func => () => func(page, handle)))
+                            .then(() => (0, util_1.promisify)(setTimeout)(getTimeout()))
+                            .then(() => page.evaluate(() => document.activeElement.blur()))
+                            .then(() => handle.evaluateHandle(div => {
+                            const minHeight = Number(div.style.transform.match(/translateY\((.+)px\)/)[1]) + div.offsetHeight;
+                            const parentDiv = div.parentElement;
+                            parentDiv.setAttribute('style', `min-height: ${minHeight}px; margin: 0 -1px; padding: 0 1px`);
+                            return parentDiv;
+                        }))
+                            .catch(err => {
+                            logger.error(`error while parsing content height, failing this webshot`);
+                            throw err;
+                        })
+                            .then(parentDivHandle => parentDivHandle.screenshot());
                     })
-                        .then(() => (0, util_1.promisify)(setTimeout)(getTimeout()))
-                        .then(() => page.evaluate(() => document.activeElement.blur()))
-                        .then(() => page.screenshot())
                         .then(screenshot => {
                         new pngjs_1.PNG({
                             filterType: 4,
                             deflateLevel: 0,
                         }).on('parsed', function () {
-                            const idx = (x, y) => (this.width * y + x) << 2;
-                            let boundary = null;
-                            const x = zoomFactor * 2;
-                            for (let y = x; y < this.height; y += zoomFactor) {
-                                if (this.data[idx(x, y)] !== this.data[idx(x, y - zoomFactor)] &&
-                                    this.data[idx(x, y)] === this.data[idx(x + zoomFactor * 10, y)]) {
-                                    boundary = y;
-                                    break;
-                                }
-                            }
-                            if (boundary !== null) {
-                                logger.info(`found boundary at ${boundary}, cropping image`);
-                                this.data = this.data.slice(0, idx(this.width, boundary));
-                                this.height = boundary;
-                                sharpToFile(jpeg(this.pack())).then(path => {
-                                    logger.info(`finished webshot for ${url}`);
-                                    resolve({ path, boundary });
-                                });
-                            }
-                            else if (height >= 8 * 1920) {
-                                logger.warn('too large, consider as a bug, returning');
-                                sharpToFile(jpeg(this.pack())).then(path => {
-                                    resolve({ path, boundary: 0 });
-                                });
-                            }
-                            else {
-                                logger.info('unable to find boundary, try shooting a larger image');
-                                resolve({ path: '', boundary });
-                            }
+                            sharpToFile(jpeg(this.pack())).then(path => {
+                                logger.info(`finished webshot for ${url}`);
+                                resolve({ path, boundary: this.height });
+                            });
                         }).parse(screenshot);
                     })
                         .catch(err => {

src/webshot.ts

@@ -113,7 +113,8 @@ class Webshot extends CallableInstance<[Tweet[], (...args) => void, number], Pro
                 '[data-testid="caret"],[role="group"],[data-testid="tweet"] [class*=" "]+:last-child>*+[class*=" "]~div{display:none}',
             }))
             .then(() => page.addStyleTag({
-              content: '*{font-family:-apple-system,".Helvetica Neue DeskInterface",Hiragino Sans,Hiragino Sans GB,sans-serif!important}',
+              content: '*{font-family:-apple-system,".Helvetica Neue DeskInterface",Hiragino Sans,Hiragino Sans GB,sans-serif!important}' +
+                '*{-webkit-font-smoothing:antialiased!important}',
             }))
             // remove listeners
             .then(() => page.evaluate(() => {
@@ -130,13 +131,18 @@ class Webshot extends CallableInstance<[Tweet[], (...args) => void, number], Pro
             .then(() => page.waitForSelector(
               'xpath=//section/*/*/div[.//article[not(.//time[not(ancestor::div[@aria-labelledby])])]]',
               {state: 'attached', timeout: getTimeout()}
-            ))
+            ) as Promise<puppeteer.ElementHandle<HTMLDivElement>>)
+            // hide comments
+            .then(handle => handle.evaluate(div => div.classList.add('mainTweet'))
+              .then(() => page.addStyleTag({content: 'div.mainTweet~div{display:none;}'}))
+              .then(() => handle)
+            )
             // toggle visibility of sensitive tweets
-            .then(handle => handle.$$('xpath=..//a[contains(@href,"content_you_see")]/../../..//*[@role="button"]')
+            .then(handle => handle.$$('xpath=(.|preceding-sibling::*)//a[contains(@href,"content_you_see")]/../../..//*[@role="button"]')
               .then(sensitiveToggles => {
                 const count = sensitiveToggles.length;
                 if (count) logger.info(`found ${count} sensitive ${count === 1 ? 'tweet' : 'tweets'} on page, uncollapsing...`);
-                return chainPromises(sensitiveToggles.filter(toggle => toggle.isVisible()).map(toggle => () => toggle.click()));
+                return chainPromises(sensitiveToggles.map(toggle => () => toggle.click()));
               })
               .then(() => handle)
             )
@@ -150,7 +156,8 @@ class Webshot extends CallableInstance<[Tweet[], (...args) => void, number], Pro
                 const path = temp.path({suffix: '.html'});
                 writeFileSync(path, html);
                 logger.warn(`saved debug html to ${path}`);
-              }).then(() => page.screenshot()).then(screenshot => {
+              }).then(() => page.route('**/*', route => route.abort())
+              ).then(() => page.screenshot({fullPage: true})).then(screenshot => {
                 sharpToFile(sharp(screenshot).jpeg({ quality: 90 })).then(fileUri => {
                   logger.warn(`saved debug screenshot to ${fileUri.substring(7)}`);
                 });
@@ -158,50 +165,32 @@ class Webshot extends CallableInstance<[Tweet[], (...args) => void, number], Pro
             })
             .then(handle => {
               if (handle === null) throw new puppeteer.errors.TimeoutError();
-              return chainPromises(morePostProcessings.map(func => () => func(page, handle)));
+              return chainPromises(morePostProcessings.map(func => () => func(page, handle)))
+                .then(() => promisify(setTimeout)(getTimeout()))
+                // hide highlight of retweet header
+                .then(() => page.evaluate(() => (document.activeElement as unknown as HTMLOrSVGElement).blur()))
+                // determine screenshot height
+                .then(() => handle.evaluateHandle(div => {
+                  const minHeight = Number(div.style.transform.match(/translateY\((.+)px\)/)[1]) + div.offsetHeight;
+                  const parentDiv = div.parentElement;
+                  parentDiv.setAttribute('style', `min-height: ${minHeight}px; margin: 0 -1px; padding: 0 1px`);
+                  return parentDiv as HTMLDivElement;
+                }))
+                .catch(err => {
+                  logger.error(`error while parsing content height, failing this webshot`);
+                  throw err;
+                })
+                .then(parentDivHandle => parentDivHandle.screenshot());
             })
-            .then(() => promisify(setTimeout)(getTimeout()))
-            // hide highlight of retweet header
-            .then(() => page.evaluate(() => (document.activeElement as unknown as HTMLOrSVGElement).blur()))
-            .then(() => page.screenshot())
             .then(screenshot => {
               new PNG({
                 filterType: 4,
                 deflateLevel: 0,
               }).on('parsed', function () {
-                // remove comment area
-                // tslint:disable-next-line: no-shadowed-variable
-                // eslint-disable-next-line @typescript-eslint/no-shadow
-                const idx = (x: number, y: number) => (this.width * y + x) << 2;
-                let boundary: number = null;
-                const x = zoomFactor * 2;
-                for (let y = x; y < this.height; y += zoomFactor) {
-                  if (
-                    this.data[idx(x, y)] !== this.data[idx(x, y - zoomFactor)] &&
-                    this.data[idx(x, y)] === this.data[idx(x + zoomFactor * 10, y)]
-                  ) {
-                    boundary = y;
-                    break;
-                  }
-                }
-                if (boundary !== null) {
-                  logger.info(`found boundary at ${boundary}, cropping image`);
-                  this.data = this.data.slice(0, idx(this.width, boundary));
-                  this.height = boundary;
-
-                  sharpToFile(jpeg(this.pack())).then(path => {
-                    logger.info(`finished webshot for ${url}`);
-                    resolve({path, boundary});
-                  });
-                } else if (height >= 8 * 1920) {
-                  logger.warn('too large, consider as a bug, returning');
-                  sharpToFile(jpeg(this.pack())).then(path => {
-                    resolve({path, boundary: 0});
-                  });
-                } else {
-                  logger.info('unable to find boundary, try shooting a larger image');
-                  resolve({path: '', boundary});
-                }
+                sharpToFile(jpeg(this.pack())).then(path => {
+                  logger.info(`finished webshot for ${url}`);
+                  resolve({path, boundary: this.height});
+                });
               }).parse(screenshot);
             })
             .catch(err => {