Merge branch 'koishi-production' into mediaonly-koishi

dist/webshot.js

@@ -80,7 +80,7 @@ class Webshot extends CallableInstance {
                     })
                         .then(() => page.goto(url, { waitUntil: 'load', timeout: getTimeout() }))
                         .then(() => page.addStyleTag({
-                        content: 'header,#layers{display:none!important}' +
+                        content: 'header,#layers{display:none!important}article{background-color:transparent!important}' +
                             '[data-testid="caret"],[role="group"],[data-testid="tweet"]+*>[class*=" "]+div:nth-last-child(2){display:none}',
                     }))
                         .then(() => page.addStyleTag({
@@ -96,16 +96,35 @@ class Webshot extends CallableInstance {
                             });
                         }, 250);
                     }))
-                        .then(() => page.waitForSelector('article', { timeout: getTimeout() }))
+                        .then(() => page.waitForSelector('xpath=//section/*/*/div[.//article/*/*/*/*/*[@role="group"]]', { timeout: getTimeout() }))
+                        .then(handle => handle.$$('xpath=..//a[contains(@href,"content_you_see")]/../../..//*[@role="button"]')
+                        .then(sensitiveToggles => {
+                        const count = sensitiveToggles.length;
+                        if (count)
+                            logger.info(`found ${count} sensitive ${count === 1 ? 'tweet' : 'tweets'} on page, uncollapsing...`);
+                        return Promise.all(sensitiveToggles.map(toggle => toggle.click()));
+                    })
+                        .then(() => handle))
                         .catch((err) => {
                         if (err.name !== 'TimeoutError')
                             throw err;
                         logger.warn(`navigation timed out at ${getTimerTime()} seconds`);
                         return null;
                     })
-                        .then(handle => {
+                        .then((handle) => {
                         if (handle === null)
                             throw new puppeteer.errors.TimeoutError();
+                        return handle.evaluate(div => {
+                            const selector = '[data-testid="tweet"]>:nth-child(2)>:first-child a';
+                            const getProfileUrl = () => div.querySelector(selector).href;
+                            const ownerProfileUrl = getProfileUrl();
+                            while (div = div.previousElementSibling) {
+                                if (getProfileUrl() !== ownerProfileUrl)
+                                    continue;
+                                return document.documentElement.scrollTop = window.scrollY + div.getBoundingClientRect().top;
+                            }
+                            document.documentElement.scrollTop = 0;
+                        });
                     })
                         .then(() => page.evaluate(() => {
                         const cardImg = document.querySelector('div[data-testid^="card.layout"][data-testid$=".media"] img');
@@ -131,9 +150,6 @@ class Webshot extends CallableInstance {
                         if (cardImg)
                             this.extendEntity(cardImg);
                     })
-                        .then(() => page.addScriptTag({
-                        content: 'document.documentElement.scrollTop=0;',
-                    }))
                         .then(() => utils_1.chainPromises(morePostProcessings.map(func => () => func(page))))
                         .then(() => util_1.promisify(setTimeout)(getTimeout()))
                         .then(() => page.screenshot())

src/webshot.ts

@@ -106,7 +106,7 @@ class Webshot extends CallableInstance<[Tweets, (...args) => void, number], Prom
             .then(() => page.goto(url, {waitUntil: 'load', timeout: getTimeout()}))
             // hide header, "more options" button, like and retweet count
             .then(() => page.addStyleTag({
-              content: 'header,#layers{display:none!important}' +
+              content: 'header,#layers{display:none!important}article{background-color:transparent!important}' +
                 '[data-testid="caret"],[role="group"],[data-testid="tweet"]+*>[class*=" "]+div:nth-last-child(2){display:none}',
             }))
             .then(() => page.addStyleTag({
@@ -123,14 +123,35 @@ class Webshot extends CallableInstance<[Tweets, (...args) => void, number], Prom
                 });
               }, 250);
             }))
-            .then(() => page.waitForSelector('article', {timeout: getTimeout()}))
-            .catch((err: Error): Promise<puppeteer.ElementHandle<Element> | null> => {
+            .then(() => page.waitForSelector('xpath=//section/*/*/div[.//article/*/*/*/*/*[@role="group"]]', {timeout: getTimeout()}))
+            // toggle visibility of sensitive tweets
+            .then(handle => handle.$$('xpath=..//a[contains(@href,"content_you_see")]/../../..//*[@role="button"]')
+              .then(sensitiveToggles => {
+                const count = sensitiveToggles.length;
+                if (count) logger.info(`found ${count} sensitive ${count === 1 ? 'tweet' : 'tweets'} on page, uncollapsing...`);
+                return Promise.all(sensitiveToggles.map(toggle => toggle.click()));
+              })
+              .then(() => handle)
+            )
+            .catch((err: Error): Promise<puppeteer.ElementHandle<HTMLDivElement> | null> => {
               if (err.name !== 'TimeoutError') throw err;
               logger.warn(`navigation timed out at ${getTimerTime()} seconds`);
               return null;
             })
-            .then(handle => {
+            // scroll to last tweet by owner in thread, if any, or top of thread
+            .then((handle: puppeteer.ElementHandle<HTMLDivElement>) => {
               if (handle === null) throw new puppeteer.errors.TimeoutError();
+              return handle.evaluate(div => {
+                const selector = '[data-testid="tweet"]>:nth-child(2)>:first-child a';
+                const getProfileUrl = () => div.querySelector<HTMLAnchorElement>(selector).href;
+                const ownerProfileUrl = getProfileUrl();
+                // eslint-disable-next-line no-cond-assign
+                while (div = div.previousElementSibling as HTMLDivElement) {
+                  if (getProfileUrl() !== ownerProfileUrl) continue;
+                  return document.documentElement.scrollTop = window.scrollY + div.getBoundingClientRect().top;
+                }
+                document.documentElement.scrollTop = 0;
+              });
             })
             .then(() => page.evaluate(() => {
               const cardImg = document.querySelector('div[data-testid^="card.layout"][data-testid$=".media"] img');
@@ -156,9 +177,6 @@ class Webshot extends CallableInstance<[Tweets, (...args) => void, number], Prom
             .then(cardImg => {
               if (cardImg) this.extendEntity(cardImg); 
             })
-            .then(() => page.addScriptTag({
-              content: 'document.documentElement.scrollTop=0;',
-            }))
             .then(() => chainPromises(morePostProcessings.map(func => () => func(page))))
             .then(() => promisify(setTimeout)(getTimeout()))
             .then(() => page.screenshot())