gocqhttp-twitter-bot/src/webshot.ts

import { writeFileSync } from 'fs';
import { Readable } from 'stream';
import { promisify } from 'util';

import axios from 'axios';
import * as CallableInstance from 'callable-instance';
import { XmlEntities } from 'html-entities';
import { PNG } from 'pngjs';
import * as puppeteer from 'playwright';
import * as sharp from 'sharp';
import * as temp from 'temp';

import { getLogger } from './loggers';
import { Message } from './koishi';
import { MediaEntity, Tweet } from './twitter';
import { chainPromises } from './utils';

const xmlEntities = new XmlEntities();

const ZHType = (type: string) => new class extends String {
  public type = super.toString();
  public toString = () => `[${super.toString()}]`;
}(type);

const typeInZH = {
  photo: ZHType('图片'),
  video: ZHType('视频'),
  animated_gif: ZHType('GIF'),
};

const logger = getLogger('webshot');

class Webshot extends CallableInstance<[Tweet[], (...args) => void, number], Promise<void>> {

  private browser: puppeteer.Browser;
  private mode: number;
  private wsUrl: string;

  constructor(wsUrl: string, mode: number, onready?: (...args) => void) {
    super('webshot');
    // tslint:disable-next-line: no-conditional-assignment
    // eslint-disable-next-line no-cond-assign
    if (this.mode = mode) {
      onready();
    } else {
      this.wsUrl = wsUrl;
      this.connect(onready);
    }
  }

  private connect = (onready?: (...args) => void): Promise<void> =>
    axios.get<{[key in 'chromium' | 'firefox' | 'webkit']?: string}>(this.wsUrl)
      .then(res => {
        logger.info(`received websocket endpoint: ${JSON.stringify(res.data)}`);
        const browserType = Object.keys(res.data)[0] as keyof typeof res.data;
        return (puppeteer[browserType] as puppeteer.BrowserType<puppeteer.Browser>)
          .connect({wsEndpoint: res.data[browserType]});
      })
      .then(browser => this.browser = browser)
      .then(() => {
        logger.info('launched puppeteer browser');
        if (onready) return onready();
      })
      .catch(error => this.reconnect(error, onready));

  private reconnect = (error, onready?: (...args) => void) => {
    logger.error(`connection error, reason: ${error}`);
    logger.warn('trying to reconnect in 2.5s...');
    return promisify(setTimeout)(2500)
      .then(() => this.connect(onready));
  };

  private renderWebshot = (
    url: string, height: number, webshotDelay: number,
    ...morePostProcessings: ((page?: puppeteer.Page, handle?: puppeteer.ElementHandle) => Promise<any>)[]
  ): Promise<string> => {
    temp.track();
    const jpeg = (data: Readable) => data.pipe(sharp()).jpeg({quality: 90, trellisQuantisation: true});
    const sharpToFile = (pic: sharp.Sharp) => new Promise<string>(resolve => {
      const webshotTempFilePath = temp.path({suffix: '.jpg'});
      pic.toFile(webshotTempFilePath).then(() => resolve(`file://${webshotTempFilePath}`));
    });
    const promise = new Promise<{ path: string, boundary: null | number }>((resolve, reject) => {
      const width = 720;
      const zoomFactor = 2;
      logger.info(`shooting ${width}*${height} webshot for ${url}`);
      this.browser.newPage({
        bypassCSP: true,
        deviceScaleFactor: zoomFactor,
        locale: 'ja-JP',
        timezoneId: 'Asia/Tokyo',
        userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
      })
        .then(page => {
          const startTime = new Date().getTime();
          const getTimerTime = () => new Date().getTime() - startTime;
          const getTimeout = () => Math.max(500, webshotDelay - getTimerTime());
          page.setViewportSize({
            width: width / zoomFactor,
            height: height / zoomFactor,
          })
            .then(() => page.route('*:\/\/video.twimg.com\/**', route => { route.abort(); }))
            .then(() => page.goto(url, {waitUntil: 'load', timeout: getTimeout()}))
            .then(() => Promise.race([
              page.waitForSelector('article', {state: 'attached', timeout: getTimeout()}),
              page.click('#placeholder+#ScriptLoadFailure input[value="Try again"]', {timeout: getTimeout()}),
              page.waitForSelector('div[role="button"]>div>span>:text-matches("^やりなおす|更新$")', {state: 'attached', timeout: getTimeout()})
                .then(() => page.reload({timeout: getTimeout()})),
            ]))
            // hide header, "more options" button, like and retweet count
            .then(() => page.addStyleTag({
              content: 'header,#layers{display:none!important}article{background-color:transparent!important}' +
                '[data-testid="caret"],[role="group"],[data-testid="tweet"] [class*=" "]+:last-child>*+[class*=" "]~div{display:none}',
            }))
            .then(() => page.addStyleTag({
              content: '*{font-family:-apple-system,".Helvetica Neue DeskInterface",Hiragino Sans,Hiragino Sans GB,sans-serif!important}',
            }))
            // remove listeners
            .then(() => page.evaluate(() => {
              const poll = setInterval(() => {
                document.querySelectorAll('div[data-testid="placementTracking"]').forEach(container => {
                  if (container.querySelector('div[role="button"] svg')) {
                    container.innerHTML = container.innerHTML;
                    clearInterval(poll);
                  }
                });
              }, 250);
            }))
            // find main tweet
            .then(() => page.waitForSelector(
              'xpath=//section/*/*/div[.//article[not(.//time[not(ancestor::div[@aria-labelledby])])]]',
              {state: 'attached', timeout: getTimeout()}
            ))
            // toggle visibility of sensitive tweets
            .then(handle => handle.$$('xpath=..//a[contains(@href,"content_you_see")]/../../..//*[@role="button"]')
              .then(sensitiveToggles => {
                const count = sensitiveToggles.length;
                if (count) logger.info(`found ${count} sensitive ${count === 1 ? 'tweet' : 'tweets'} on page, uncollapsing...`);
                return chainPromises(sensitiveToggles.filter(toggle => toggle.isVisible()).map(toggle => () => toggle.click()));
              })
              .then(() => handle)
            )
            // throw early if tweet is unavailable
            .then(handle => handle.$('[data-testid="tweet"]').then(owner => owner ? handle : null))
            .catch((err: Error): Promise<puppeteer.ElementHandle<HTMLDivElement> | null> => {
              if (err.name !== 'TimeoutError')
              throw err;
              logger.warn(`${err} (${getTimerTime()} ms)`);
              return page.evaluate(() => document.documentElement.outerHTML).then(html => {
                const path = temp.path({suffix: '.html'});
                writeFileSync(path, html);
                logger.warn(`saved debug html to ${path}`);
              }).then(() => page.screenshot()).then(screenshot => {
                sharpToFile(sharp(screenshot).jpeg({ quality: 90 })).then(fileUri => {
                  logger.warn(`saved debug screenshot to ${fileUri.substring(7)}`);
                });
              }).then(() => null);
            })
            .then(handle => {
              if (handle === null) throw new puppeteer.errors.TimeoutError();
              return chainPromises(morePostProcessings.map(func => () => func(page, handle)));
            })
            .then(() => promisify(setTimeout)(getTimeout()))
            // hide highlight of retweet header
            .then(() => page.evaluate(() => (document.activeElement as unknown as HTMLOrSVGElement).blur()))
            .then(() => page.screenshot())
            .then(screenshot => {
              new PNG({
                filterType: 4,
                deflateLevel: 0,
              }).on('parsed', function () {
                // remove comment area
                // tslint:disable-next-line: no-shadowed-variable
                // eslint-disable-next-line @typescript-eslint/no-shadow
                const idx = (x: number, y: number) => (this.width * y + x) << 2;
                let boundary: number = null;
                const x = zoomFactor * 2;
                for (let y = x; y < this.height; y += zoomFactor) {
                  if (
                    this.data[idx(x, y)] !== this.data[idx(x, y - zoomFactor)] &&
                    this.data[idx(x, y)] === this.data[idx(x + zoomFactor * 10, y)]
                  ) {
                    boundary = y;
                    break;
                  }
                }
                if (boundary !== null) {
                  logger.info(`found boundary at ${boundary}, cropping image`);
                  this.data = this.data.slice(0, idx(this.width, boundary));
                  this.height = boundary;

                  sharpToFile(jpeg(this.pack())).then(path => {
                    logger.info(`finished webshot for ${url}`);
                    resolve({path, boundary});
                  });
                } else if (height >= 8 * 1920) {
                  logger.warn('too large, consider as a bug, returning');
                  sharpToFile(jpeg(this.pack())).then(path => {
                    resolve({path, boundary: 0});
                  });
                } else {
                  logger.info('unable to find boundary, try shooting a larger image');
                  resolve({path: '', boundary});
                }
              }).parse(screenshot);
            })
            .catch(err => {
              if (err instanceof Error && err.name !== 'TimeoutError') throw err;
              logger.error(`error shooting webshot for ${url}, could not load web page of tweet`);
              resolve({path: '', boundary: 0});
            })
            .finally(() => { page.close(); });
        })
        .catch(reject);
    });
    return promise.then(data => {
      if (data.boundary === null) {
        return this.renderWebshot(url, height + 1920, webshotDelay, ...morePostProcessings);
      } else return data.path;
    }).catch(error => this.reconnect(error)
      .then(() => this.renderWebshot(url, height, webshotDelay, ...morePostProcessings))
    );
  };

  private fetchMedia = (url: string): Promise<string> => new Promise<ArrayBuffer>((resolve, reject) => {
    logger.info(`fetching ${url}`);
    axios({
      method: 'get',
      url,
      responseType: 'arraybuffer',
      timeout: 150000,
    }).then(res => {
      if (res.status === 200) {
        logger.info(`successfully fetched ${url}`);
        resolve(res.data);
      } else {
        logger.error(`failed to fetch ${url}: ${res.status}`);
        reject();
      }
    }).catch (err => {
      logger.error(`failed to fetch ${url}: ${err instanceof Error ? err.message : err}`);
      reject();
    });
  }).then(data =>
    (ext => {
      const mediaTempFilePath = temp.path({suffix: `.${ext}`});
      writeFileSync(mediaTempFilePath, Buffer.from(data));
      const path = `file://${mediaTempFilePath}`;
      switch (ext) {
        case 'jpg':
        case 'png':
          return Message.Image(path);
        case 'mp4':
          return Message.Video(path);
      }
      logger.warn('unable to find MIME type of fetched media, failing this fetch');
      throw Error();
    })(((/\?format=([a-z]+)&/.exec(url)) ?? (/.*\/.*\.([^?]+)/.exec(url)))[1])
  );

  public webshot(
    tweets: Tweet[],
    callback: (cacheId: string, msgs: string, text: string, author: string) => void,
    webshotDelay: number
  ): Promise<void> {
    const promises = tweets.map((twi, index) => {
      let promise = promisify(setTimeout)(webshotDelay / 4 * index).then(() => {
        logger.info(`working on ${twi.user.screen_name}/${twi.id_str}`);
      });
      const originTwi = twi.retweeted_status || twi;
      let messageChain = '';
      let truncatedAt: string;

      // text processing
      let author = `${twi.user.name} (@${twi.user.screen_name}):\n`;
      author += `${new Date(twi.created_at)}\n`;
      if (twi.retweeted_status) author += `RT @${twi.retweeted_status.user.screen_name}: `;

      let text = originTwi.full_text;

      promise = promise.then(() => {
        if (originTwi.entities && originTwi.entities.urls && originTwi.entities.urls.length) {
          originTwi.entities.urls.forEach(url => {
            text = text.replace(new RegExp(url.url, 'gm'), url.expanded_url);
          });
        }
        if (originTwi.extended_entities) {
          originTwi.extended_entities.media.forEach(media => {
            text = text.replace(new RegExp(media.url, 'gm'), this.mode === 1 ? typeInZH[media.type] : '');
          });
        }
        if (this.mode > 0) messageChain += (author + xmlEntities.decode(text));
      });

      // invoke webshot
      if (this.mode === 0) {
        const url = `https://mobile.twitter.com/${twi.user.screen_name}/status/${twi.id_str}`;
        const extendEntity = (cardImg: MediaEntity) => {
          originTwi.extended_entities = {
            ...originTwi.extended_entities,
            media: [
              ...originTwi.extended_entities?.media ?? [],
              cardImg,
            ],
          };
        };
        const truncateLongThread = (atId: string) => {
          if (!atId) return;
          logger.info(`thread too long, truncating at tweet ${atId}...`);
          truncatedAt = atId;
        };

        promise = promise.then(() => this.renderWebshot(url, 1920, webshotDelay,

          // scroll back at least 2 tweets revealing 2nd last tweet by owner in thread, or top of thread
          (_, tweetHandle: puppeteer.ElementHandle<HTMLDivElement>) => tweetHandle.evaluate(div => {
            try {
              const selector = '[data-testid="tweet"] :nth-child(2)>:first-child a';
              const getProfileUrl = () => (div.querySelector<HTMLAnchorElement>(selector) || {href: ''}).href;
              const ownerProfileUrl = getProfileUrl();
              const bottom = div;
              // eslint-disable-next-line no-cond-assign
              while (div = div.previousElementSibling as HTMLDivElement) {
                if (getProfileUrl() !== ownerProfileUrl || div === bottom.previousElementSibling) continue;
                const top = document.documentElement.scrollTop = window.scrollY + div.getBoundingClientRect().top;
                if (top > 10)
                  return div.querySelector<HTMLAnchorElement>('article a[aria-label]').href.replace(/.*\/status\//, '');
              }
            } catch {/* handle errors like none-found cases */}
            document.documentElement.scrollTop = 0;
          }).then(truncateLongThread),

          // scrape card image from main tweet
          (_, tweetHandle: puppeteer.ElementHandle<HTMLDivElement>) => tweetHandle.evaluate(div => {
            const cardImg = div.querySelector('div[data-testid^="card.layout"][data-testid$=".media"] img');
            if (typeof cardImg?.getAttribute('src') === 'string') {
              const match = /^(.*\/card_img\/(\d+)\/.+\?format=.*)&name=/.exec(cardImg?.getAttribute('src'));
              if (match) {
                // tslint:disable-next-line: variable-name
                const [media_url_https, id_str] = match.slice(1);
                return {
                  media_url: media_url_https.replace(/^https/, 'http'),
                  media_url_https,
                  url: '',
                  display_url: '',
                  expanded_url: '',
                  type: 'photo',
                  id: Number(id_str),
                  id_str,
                  sizes: undefined,
                };
              }
            }
          }).then(cardImg => { if (cardImg) extendEntity(cardImg); })
        ))
          .then(fileurl => {
            if (fileurl) return Message.Image(fileurl);
            return '[截图不可用] ' + author + text;
          })
          .then(msg => {
            if (msg) messageChain += msg;
          });
      }
      // fetch extra entities
      // tslint:disable-next-line: curly
      // eslint-disable-next-line curly
      if (1 - this.mode % 2) promise = promise.then(() => {
        if (originTwi.extended_entities) {
          return chainPromises(originTwi.extended_entities.media.map(media => () => {
            let url: string;
            if (media.type === 'photo') {
              url = media.media_url_https.replace(/\.([a-z]+)$/, '?format=$1') + '&name=orig';
            } else {
              url = media.video_info.variants
                .filter(variant => variant.bitrate !== undefined)
                .sort((var1, var2) => var2.bitrate - var1.bitrate)
                .map(variant => variant.url)[0]; // largest video
            }
            const altMessage = `\n[失败的${typeInZH[media.type as keyof typeof typeInZH].type}：${url}]`;
            return this.fetchMedia(url)
              .catch(error => {
                logger.warn('unable to fetch media, sending plain text instead...');
                return altMessage;
              })
              .then(msg => { messageChain += msg; });
          }));
        }
      });
      // append URLs, if any
      if (this.mode === 0) {
        if (originTwi.entities && originTwi.entities.urls && originTwi.entities.urls.length) {
          promise = promise.then(() => {
            const urls = originTwi.entities.urls
              .filter(urlObj => urlObj.indices[0] < originTwi.display_text_range[1])
              .map(urlObj => `\n\ud83d\udd17 ${urlObj.expanded_url}`);
            if (urls.length) {
              messageChain += urls.join('');
            }
          });
        }
      }
      // refer to earlier tweets if thread is truncated
      promise = promise.then(() => {
        if (truncatedAt) {
          messageChain += `\n回复此命令查看对话串中更早的推文：\n/twitter_view ${truncatedAt}`;
        }
      });
      // refer to quoted tweet, if any
      if (originTwi.is_quote_status) {
        promise = promise.then(() => {
          const match = /\/status\/(\d+)/.exec(originTwi.quoted_status_permalink?.expanded);
          const blockQuoteIdStr = match ? match[1] : originTwi.quoted_status?.id_str;
          if (blockQuoteIdStr) messageChain += `\n回复此命令查看引用的推文：\n/twitter_view ${blockQuoteIdStr}`;
        });
      }
      return promise.then(() => {
        logger.info(`done working on ${twi.user.screen_name}/${twi.id_str}, message chain:`);
        logger.info(JSON.stringify(Message.ellipseBase64(messageChain)));
        let cacheId = twi.id_str;
        if (twi.retweeted_status) cacheId += `,rt:${twi.retweeted_status.id_str}`;
        callback(cacheId, messageChain, xmlEntities.decode(text), author);
      });
    });
    return Promise.all(promises).then();
  }
}

export default Webshot;