webshot.ts 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178
  1. import * as CallableInstance from 'callable-instance';
  2. import * as https from 'https';
  3. import * as log4js from 'log4js';
  4. import { PNG } from 'pngjs';
  5. import * as puppeteer from 'puppeteer';
  6. import { Browser } from 'puppeteer';
  7. import * as read from 'read-all-stream';
  8. const logger = log4js.getLogger('webshot');
  9. logger.level = (global as any).loglevel;
  10. class Webshot extends CallableInstance {
  11. private browser: Browser;
  12. constructor(onready?: () => any) {
  13. super('webshot');
  14. puppeteer.launch({args: ['--no-sandbox', '--disable-setuid-sandbox', '--lang=zh-CN,zh']})
  15. .then(browser => this.browser = browser)
  16. .then(() => {
  17. logger.info('launched puppeteer browser');
  18. if (onready) onready();
  19. });
  20. }
  21. private renderWebshot = (url: string, height: number, webshotDelay: number): Promise<string> => {
  22. const promise = new Promise<{ data: string, boundary: null | number }>(resolve => {
  23. const width = 1080;
  24. logger.info(`shooting ${width}*${height} webshot for ${url}`);
  25. this.browser.newPage()
  26. .then(page => {
  27. page.setUserAgent('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36')
  28. .then(() => page.setViewport({
  29. width,
  30. height,
  31. isMobile: true,
  32. }))
  33. .then(() => page.setBypassCSP(true))
  34. .then(() => page.goto(url))
  35. // hide header, "more options" button, like and retweet count
  36. .then(() => page.addStyleTag({
  37. content: 'html{zoom:2}header{display:none!important}path[d=\'M20.207 7.043a1 1 0 0 0-1.414 0L12 13.836 5.207 7.043a1 1 0 0 0-1.414 1.414l7.5 7.5a.996.996 0 0 0 1.414 0l7.5-7.5a1 1 0 0 0 0-1.414z\'],div[role=\'button\']{display: none;}',
  38. }))
  39. .then(() => page.waitFor(webshotDelay))
  40. .then(() => page.addScriptTag({
  41. content: 'document.documentElement.scrollTop=0;',
  42. }))
  43. .then(() => page.screenshot())
  44. .then(screenshot => {
  45. new PNG({
  46. filterType: 4,
  47. }).on('parsed', function () {
  48. // remove comment area
  49. let boundary = null;
  50. let x = 0;
  51. for (let y = 0; y < this.height; y++) {
  52. const idx = (this.width * y + x) << 2;
  53. if (this.data[idx] !== 255) {
  54. boundary = y;
  55. break;
  56. }
  57. }
  58. if (boundary !== null) {
  59. logger.info(`found boundary at ${boundary}, cropping image`);
  60. this.data = this.data.slice(0, (this.width * boundary) << 2);
  61. this.height = boundary;
  62. boundary = null;
  63. x = Math.floor(this.width / 2);
  64. let flag = false;
  65. let cnt = 0;
  66. for (let y = this.height - 1; y >= 0; y--) {
  67. const idx = (this.width * y + x) << 2;
  68. if ((this.data[idx] === 255) === flag) {
  69. cnt++;
  70. flag = !flag;
  71. } else continue;
  72. // line above the "comment", "retweet", "like", "share" button row
  73. if (cnt === 2) {
  74. boundary = y + 1;
  75. }
  76. // if there are a "retweet" count and "like" count row, this will be the line above it
  77. if (cnt === 4) {
  78. const b = y + 1;
  79. if (this.height - b <= 200) boundary = b;
  80. break;
  81. }
  82. }
  83. if (boundary != null) {
  84. logger.info(`found boundary at ${boundary}, trimming image`);
  85. this.data = this.data.slice(0, (this.width * boundary) << 2);
  86. this.height = boundary;
  87. }
  88. read(this.pack(), 'base64').then(data => {
  89. logger.info(`finished webshot for ${url}`);
  90. resolve({data, boundary});
  91. });
  92. } else if (height >= 8 * 1920) {
  93. logger.warn('too large, consider as a bug, returning');
  94. read(this.pack(), 'base64').then(data => {
  95. logger.info(`finished webshot for ${url}`);
  96. resolve({data, boundary: 0});
  97. });
  98. } else {
  99. logger.info('unable to found boundary, try shooting a larger image');
  100. resolve({data: '', boundary});
  101. }
  102. }).parse(screenshot);
  103. })
  104. .then(() => page.close());
  105. });
  106. });
  107. return promise.then(data => {
  108. if (data.boundary === null) return this.renderWebshot(url, height + 1920, webshotDelay);
  109. else return data.data;
  110. });
  111. }
  112. private fetchImage = (url: string): Promise<string> =>
  113. new Promise<string>(resolve => {
  114. logger.info(`fetching ${url}`);
  115. https.get(url, res => {
  116. if (res.statusCode === 200) {
  117. read(res, 'base64').then(data => {
  118. logger.info(`successfully fetched ${url}`);
  119. resolve(data);
  120. });
  121. } else {
  122. logger.error(`failed to fetch ${url}: ${res.statusCode}`);
  123. resolve();
  124. }
  125. }).on('error', (err) => {
  126. logger.error(`failed to fetch ${url}: ${err.message}`);
  127. resolve();
  128. });
  129. })
  130. public webshot(tweets, callback, webshotDelay: number): Promise<void> {
  131. let promise = new Promise<void>(resolve => {
  132. resolve();
  133. });
  134. tweets.forEach(twi => {
  135. let cqstr = '';
  136. const url = `https://mobile.twitter.com/${twi.user.screen_name}/status/${twi.id_str}`;
  137. promise = promise.then(() => this.renderWebshot(url, 1920, webshotDelay))
  138. .then(base64Webshot => {
  139. if (base64Webshot) cqstr += `[CQ:image,file=base64://${base64Webshot}]`;
  140. });
  141. if (twi.extended_entities) {
  142. twi.extended_entities.media.forEach(media => {
  143. promise = promise.then(() => this.fetchImage(media.media_url_https))
  144. .then(base64Image => {
  145. if (base64Image) cqstr += `[CQ:image,file=base64://${base64Image}]`;
  146. });
  147. });
  148. }
  149. if (twi.entities && twi.entities.urls && twi.entities.urls.length) {
  150. promise = promise.then(() => {
  151. const urls = twi.entities.urls
  152. .filter(urlObj => urlObj.indices[0] < twi.display_text_range[1])
  153. .map(urlObj => urlObj.expanded_url);
  154. if (urls.length) {
  155. cqstr += '\n';
  156. cqstr += urls.join('\n');
  157. }
  158. });
  159. }
  160. promise.then(() => callback(cqstr));
  161. });
  162. return promise;
  163. }
  164. }
  165. export default Webshot;