webshot.js 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240
  1. "use strict";
  2. Object.defineProperty(exports, "__esModule", { value: true });
  3. const axios_1 = require("axios");
  4. const CallableInstance = require("callable-instance");
  5. const fs_1 = require("fs");
  6. const html_entities_1 = require("html-entities");
  7. const pngjs_1 = require("pngjs");
  8. const puppeteer = require("puppeteer");
  9. const loggers_1 = require("./loggers");
  10. const mirai_1 = require("./mirai");
  11. const writeOutTo = (path, data) => new Promise(resolve => {
  12. data.pipe(fs_1.createWriteStream(path)).on('close', () => resolve(path));
  13. });
  14. const xmlEntities = new html_entities_1.XmlEntities();
  15. const typeInZH = {
  16. photo: '[图片]',
  17. video: '[视频]',
  18. animated_gif: '[GIF]',
  19. };
  20. const logger = loggers_1.getLogger('webshot');
  21. const mkdirP = dir => { if (!fs_1.existsSync(dir))
  22. fs_1.mkdirSync(dir, { recursive: true }); };
  23. const baseName = path => path.split(/[/\\]/).slice(-1)[0];
  24. class Webshot extends CallableInstance {
  25. constructor(outDir, mode, onready) {
  26. super('webshot');
  27. this.renderWebshot = (url, height, webshotDelay) => {
  28. const writeOutPic = (pic) => writeOutTo(`${this.outDir}/${url.replace(/[:\/]/g, '_')}.png`, pic);
  29. const promise = new Promise(resolve => {
  30. const width = 1080;
  31. logger.info(`shooting ${width}*${height} webshot for ${url}`);
  32. this.browser.newPage()
  33. .then(page => {
  34. page.setUserAgent('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36')
  35. .then(() => page.setViewport({
  36. width: width / 2,
  37. height: height / 2,
  38. isMobile: true,
  39. deviceScaleFactor: 2,
  40. }))
  41. .then(() => page.setBypassCSP(true))
  42. .then(() => page.goto(url, { waitUntil: 'load', timeout: 150000 }))
  43. // hide header, "more options" button, like and retweet count
  44. .then(() => page.addStyleTag({
  45. content: 'header{display:none!important}path[d=\'M20.207 7.043a1 1 0 0 0-1.414 0L12 13.836 5.207 7.043a1 1 0 0 0-1.414 1.414l7.5 7.5a.996.996 0 0 0 1.414 0l7.5-7.5a1 1 0 0 0 0-1.414z\'],div[role=\'button\']{display: none;}',
  46. }))
  47. .then(() => page.waitFor(webshotDelay))
  48. .then(() => page.addScriptTag({
  49. content: 'document.documentElement.scrollTop=0;',
  50. }))
  51. .then(() => page.screenshot())
  52. .then(screenshot => {
  53. new pngjs_1.PNG({
  54. filterType: 4,
  55. }).on('parsed', function () {
  56. // remove comment area
  57. let boundary = null;
  58. let x = Math.floor(this.width / 180);
  59. for (let y = 0; y < this.height; y++) {
  60. const idx = (this.width * y + x) << 2;
  61. if (this.data[idx] !== 255) {
  62. if (this.data[idx + this.width * 10] !== 255) {
  63. // footer kicks in
  64. boundary = null;
  65. }
  66. else {
  67. boundary = y;
  68. }
  69. break;
  70. }
  71. }
  72. if (boundary !== null) {
  73. logger.info(`found boundary at ${boundary}, cropping image`);
  74. this.data = this.data.slice(0, (this.width * boundary) << 2);
  75. this.height = boundary;
  76. boundary = null;
  77. x = Math.floor(this.width / 30);
  78. let flag = false;
  79. let cnt = 0;
  80. for (let y = this.height - 1; y >= 0; y--) {
  81. const idx = (this.width * y + x) << 2;
  82. if ((this.data[idx] === 255) === flag) {
  83. cnt++;
  84. flag = !flag;
  85. }
  86. else
  87. continue;
  88. // line above the "comment", "retweet", "like", "share" button row
  89. if (cnt === 2) {
  90. boundary = y + 1;
  91. }
  92. // if there are a "retweet" count and "like" count row, this will be the line above it
  93. if (cnt === 4) {
  94. const b = y + 1;
  95. if (this.height - b <= 200)
  96. boundary = b;
  97. break;
  98. }
  99. }
  100. if (boundary != null) {
  101. logger.info(`found boundary at ${boundary}, trimming image`);
  102. this.data = this.data.slice(0, (this.width * boundary) << 2);
  103. this.height = boundary;
  104. }
  105. writeOutPic(this.pack()).then(path => {
  106. logger.info(`finished webshot for ${url}`);
  107. resolve({ path, boundary });
  108. });
  109. }
  110. else if (height >= 8 * 1920) {
  111. logger.warn('too large, consider as a bug, returning');
  112. writeOutPic(this.pack()).then(path => {
  113. resolve({ path, boundary: 0 });
  114. });
  115. }
  116. else {
  117. logger.info('unable to find boundary, try shooting a larger image');
  118. resolve({ path: '', boundary });
  119. }
  120. }).parse(screenshot);
  121. })
  122. .then(() => page.close());
  123. });
  124. });
  125. return promise.then(data => {
  126. if (data.boundary === null)
  127. return this.renderWebshot(url, height + 1920, webshotDelay);
  128. else
  129. return data.path;
  130. });
  131. };
  132. this.fetchImage = (url, tag) => new Promise(resolve => {
  133. logger.info(`fetching ${url}`);
  134. axios_1.default({
  135. method: 'get',
  136. url,
  137. responseType: 'stream',
  138. }).then(res => {
  139. if (res.status === 200) {
  140. logger.info(`successfully fetched ${url}`);
  141. resolve(res.data);
  142. }
  143. else {
  144. logger.error(`failed to fetch ${url}: ${res.status}`);
  145. resolve();
  146. }
  147. }).catch(err => {
  148. logger.error(`failed to fetch ${url}: ${err.message}`);
  149. resolve();
  150. });
  151. }).then(data => {
  152. const imgName = `${tag}${baseName(url.replace(/(\.[a-z]+)(:.*)/, '$1$2$1'))}`;
  153. return writeOutTo(`${this.outDir}/${imgName}`, data);
  154. });
  155. mkdirP(this.outDir = outDir);
  156. // tslint:disable-next-line: no-conditional-assignment
  157. if (this.mode = mode) {
  158. onready();
  159. }
  160. else {
  161. // use local Chromium
  162. puppeteer.connect({ browserURL: 'http://127.0.0.1:9222' })
  163. .then(browser => this.browser = browser)
  164. .then(() => {
  165. logger.info('launched puppeteer browser');
  166. if (onready)
  167. onready();
  168. });
  169. }
  170. }
  171. webshot(tweets, callback, webshotDelay) {
  172. let promise = new Promise(resolve => {
  173. resolve();
  174. });
  175. tweets.forEach(twi => {
  176. promise = promise.then(() => {
  177. logger.info(`working on ${twi.user.screen_name}/${twi.id_str}`);
  178. });
  179. const originTwi = twi.retweeted_status || twi;
  180. const messageChain = [];
  181. // text processing
  182. let author = `${twi.user.name} (@${twi.user.screen_name}):\n`;
  183. if (twi.retweeted_status)
  184. author += `RT @${twi.retweeted_status.user.screen_name}: `;
  185. let text = originTwi.full_text;
  186. promise = promise.then(() => {
  187. if (originTwi.entities && originTwi.entities.urls && originTwi.entities.urls.length) {
  188. originTwi.entities.urls.forEach(url => {
  189. text = text.replace(new RegExp(url.url, 'gm'), url.expanded_url);
  190. });
  191. }
  192. if (originTwi.extended_entities) {
  193. originTwi.extended_entities.media.forEach(media => {
  194. text = text.replace(new RegExp(media.url, 'gm'), this.mode === 1 ? typeInZH[media.type] : '');
  195. });
  196. }
  197. if (this.mode > 0)
  198. messageChain.push(mirai_1.MiraiMessage.Plain(author + xmlEntities.decode(text)));
  199. });
  200. // invoke webshot
  201. if (this.mode === 0) {
  202. const url = `https://mobile.twitter.com/${twi.user.screen_name}/status/${twi.id_str}`;
  203. promise = promise.then(() => this.renderWebshot(url, 1920, webshotDelay))
  204. .then(webshotFilePath => {
  205. if (webshotFilePath)
  206. messageChain.push(mirai_1.MiraiMessage.Image('', '', baseName(webshotFilePath)));
  207. });
  208. }
  209. // fetch extra images
  210. if (1 - this.mode % 2) {
  211. if (originTwi.extended_entities) {
  212. originTwi.extended_entities.media.forEach(media => promise = promise.then(() => this.fetchImage(media.media_url_https + ':orig', `${twi.user.screen_name}-${twi.id_str}--`)
  213. .then(path => {
  214. messageChain.push(mirai_1.MiraiMessage.Image('', '', baseName(path)));
  215. })));
  216. }
  217. }
  218. // append URLs, if any
  219. if (this.mode === 0) {
  220. if (originTwi.entities && originTwi.entities.urls && originTwi.entities.urls.length) {
  221. promise = promise.then(() => {
  222. const urls = originTwi.entities.urls
  223. .filter(urlObj => urlObj.indices[0] < originTwi.display_text_range[1])
  224. .map(urlObj => urlObj.expanded_url);
  225. if (urls.length) {
  226. messageChain.push(mirai_1.MiraiMessage.Plain(urls.join('\n')));
  227. }
  228. });
  229. }
  230. }
  231. promise.then(() => {
  232. logger.info(`done working on ${twi.user.screen_name}/${twi.id_str}, message chain:`);
  233. logger.info(JSON.stringify(messageChain));
  234. callback(messageChain, text, author);
  235. });
  236. });
  237. return promise;
  238. }
  239. }
  240. exports.default = Webshot;