twitter.js 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349
  1. "use strict";
  2. var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
  3. function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
  4. return new (P || (P = Promise))(function (resolve, reject) {
  5. function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
  6. function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
  7. function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
  8. step((generator = generator.apply(thisArg, _arguments || [])).next());
  9. });
  10. };
  11. Object.defineProperty(exports, "__esModule", { value: true });
  12. exports.sendTimeline = exports.sendTweet = exports.ScreenNameNormalizer = void 0;
  13. const fs = require("fs");
  14. const path = require("path");
  15. const Twitter = require("twitter");
  16. const loggers_1 = require("./loggers");
  17. const utils_1 = require("./utils");
  18. const webshot_1 = require("./webshot");
  19. class ScreenNameNormalizer {
  20. static normalizeLive(username) {
  21. return __awaiter(this, void 0, void 0, function* () {
  22. if (this._queryUser) {
  23. return yield this._queryUser(username)
  24. .catch((err) => {
  25. if (err[0].code !== 50) {
  26. logger.warn(`error looking up user: ${err[0].message}`);
  27. return username;
  28. }
  29. return null;
  30. });
  31. }
  32. return this.normalize(username);
  33. });
  34. }
  35. }
  36. exports.ScreenNameNormalizer = ScreenNameNormalizer;
  37. ScreenNameNormalizer.normalize = (username) => username.toLowerCase().replace(/^@/, '');
  38. exports.sendTweet = (id, receiver) => {
  39. throw Error();
  40. };
  41. exports.sendTimeline = (conf, receiver) => {
  42. throw Error();
  43. };
  44. const TWITTER_EPOCH = 1288834974657;
  45. const snowflake = (epoch) => Number.isNaN(epoch) ? undefined :
  46. utils_1.BigNumOps.lShift(String(epoch - 1 - TWITTER_EPOCH), 22);
  47. const logger = loggers_1.getLogger('twitter');
  48. const maxTrials = 3;
  49. const uploadTimeout = 10000;
  50. const retryInterval = 1500;
  51. const ordinal = (n) => {
  52. switch ((Math.trunc(n / 10) % 10 === 1) ? 0 : n % 10) {
  53. case 1:
  54. return `${n}st`;
  55. case 2:
  56. return `${n}nd`;
  57. case 3:
  58. return `${n}rd`;
  59. default:
  60. return `${n}th`;
  61. }
  62. };
  63. const retryOnError = (doWork, onRetry) => new Promise(resolve => {
  64. const retry = (reason, count) => {
  65. setTimeout(() => {
  66. let terminate = false;
  67. onRetry(reason, count, defaultValue => { terminate = true; resolve(defaultValue); });
  68. if (!terminate)
  69. doWork().then(resolve).catch(error => retry(error, count + 1));
  70. }, retryInterval);
  71. };
  72. doWork().then(resolve).catch(error => retry(error, 1));
  73. });
  74. class default_1 {
  75. constructor(opt) {
  76. this.launch = () => {
  77. this.webshot = new webshot_1.default(this.mode, () => setTimeout(this.work, this.workInterval * 1000));
  78. };
  79. this.queryUser = (username) => this.client.get('users/show', { screen_name: username })
  80. .then((user) => user.screen_name);
  81. this.queryTimelineReverse = (conf) => {
  82. if (!conf.since)
  83. return this.queryTimeline(conf);
  84. const count = conf.count;
  85. const maxID = conf.until;
  86. conf.count = undefined;
  87. const until = () => utils_1.BigNumOps.min(maxID, utils_1.BigNumOps.plus(conf.since, String(7 * 24 * 3600 * 1000 * Math.pow(2, 22))));
  88. conf.until = until();
  89. const promise = (tweets) => this.queryTimeline(conf).then(newTweets => {
  90. tweets = newTweets.concat(tweets);
  91. conf.since = conf.until;
  92. conf.until = until();
  93. if (tweets.length >= count ||
  94. utils_1.BigNumOps.compare(conf.since, conf.until) >= 0) {
  95. return tweets.slice(-count);
  96. }
  97. return promise(tweets);
  98. });
  99. return promise([]);
  100. };
  101. this.queryTimeline = ({ username, count, since, until, noreps, norts }) => {
  102. username = username.replace(/^@?(.*)$/, '@$1');
  103. logger.info(`querying timeline of ${username} with config: ${JSON.stringify(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, (count && { count })), (since && { since })), (until && { until })), (noreps && { noreps })), (norts && { norts })))}`);
  104. const fetchTimeline = (config = {
  105. screen_name: username.slice(1),
  106. trim_user: true,
  107. exclude_replies: noreps !== null && noreps !== void 0 ? noreps : true,
  108. include_rts: !(norts !== null && norts !== void 0 ? norts : false),
  109. since_id: since,
  110. max_id: until,
  111. tweet_mode: 'extended',
  112. }, tweets = []) => this.client.get('statuses/user_timeline', config)
  113. .then((newTweets) => {
  114. if (newTweets.length) {
  115. logger.debug(`fetched tweets: ${JSON.stringify(newTweets)}`);
  116. config.max_id = utils_1.BigNumOps.plus('-1', newTweets[newTweets.length - 1].id_str);
  117. logger.info(`timeline query of ${username} yielded ${newTweets.length} new tweets, next query will start at offset ${config.max_id}`);
  118. tweets.push(...newTweets.filter(tweet => tweet.extended_entities));
  119. }
  120. if (!newTweets.length || tweets.length >= count) {
  121. logger.info(`timeline query of ${username} finished successfully, ${tweets.length} tweets with extended entities have been fetched`);
  122. return tweets.slice(0, count);
  123. }
  124. return fetchTimeline(config, tweets);
  125. });
  126. return fetchTimeline();
  127. };
  128. this.workOnTweets = (tweets, sendTweets) => {
  129. const uploader = (message, lastResort) => {
  130. let timeout = uploadTimeout;
  131. return retryOnError(() => this.bot.uploadPic(message, timeout).then(() => message), (_, count, terminate) => {
  132. if (count <= maxTrials) {
  133. timeout *= (count + 2) / (count + 1);
  134. logger.warn(`retry uploading for the ${ordinal(count)} time...`);
  135. }
  136. else {
  137. logger.warn(`${count - 1} consecutive failures while uploading, trying plain text instead...`);
  138. terminate(lastResort());
  139. }
  140. });
  141. };
  142. return this.webshot(tweets, uploader, sendTweets, this.webshotDelay);
  143. };
  144. this.getTweet = (id, sender) => {
  145. const endpoint = 'statuses/show';
  146. const config = {
  147. id,
  148. tweet_mode: 'extended',
  149. };
  150. return this.client.get(endpoint, config)
  151. .then((tweet) => {
  152. logger.debug(`api returned tweet ${JSON.stringify(tweet)} for query id=${id}`);
  153. return this.workOnTweets([tweet], sender);
  154. });
  155. };
  156. this.sendTweets = (source, ...to) => (msg, text, author) => {
  157. to.forEach(subscriber => {
  158. logger.info(`pushing data${source ? ` of ${source}` : ''} to ${JSON.stringify(subscriber)}`);
  159. retryOnError(() => this.bot.sendTo(subscriber, msg), (_, count, terminate) => {
  160. if (count <= maxTrials) {
  161. logger.warn(`retry sending to ${subscriber.chatID} for the ${ordinal(count)} time...`);
  162. }
  163. else {
  164. logger.warn(`${count - 1} consecutive failures while sending` +
  165. 'message chain, trying plain text instead...');
  166. terminate(this.bot.sendTo(subscriber, author + text));
  167. }
  168. });
  169. });
  170. };
  171. this.work = () => {
  172. const lock = this.lock;
  173. if (this.workInterval < 1)
  174. this.workInterval = 1;
  175. if (lock.feed.length === 0) {
  176. setTimeout(() => {
  177. this.work();
  178. }, this.workInterval * 1000);
  179. return;
  180. }
  181. if (lock.workon >= lock.feed.length)
  182. lock.workon = 0;
  183. if (!lock.threads[lock.feed[lock.workon]] ||
  184. !lock.threads[lock.feed[lock.workon]].subscribers ||
  185. lock.threads[lock.feed[lock.workon]].subscribers.length === 0) {
  186. logger.warn(`nobody subscribes thread ${lock.feed[lock.workon]}, removing from feed`);
  187. delete lock.threads[lock.feed[lock.workon]];
  188. lock.feed.splice(lock.workon, 1);
  189. fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
  190. this.work();
  191. return;
  192. }
  193. const currentFeed = lock.feed[lock.workon];
  194. logger.debug(`pulling feed ${currentFeed}`);
  195. const promise = new Promise(resolve => {
  196. let match = currentFeed.match(/https:\/\/twitter.com\/([^\/]+)\/lists\/([^\/]+)/);
  197. let config;
  198. let endpoint;
  199. if (match) {
  200. if (match[1] === 'i') {
  201. config = {
  202. list_id: match[2],
  203. tweet_mode: 'extended',
  204. };
  205. }
  206. else {
  207. config = {
  208. owner_screen_name: match[1],
  209. slug: match[2],
  210. tweet_mode: 'extended',
  211. };
  212. }
  213. endpoint = 'lists/statuses';
  214. }
  215. else {
  216. match = currentFeed.match(/https:\/\/twitter.com\/([^\/]+)/);
  217. if (match) {
  218. config = {
  219. screen_name: match[1],
  220. exclude_replies: false,
  221. tweet_mode: 'extended',
  222. };
  223. endpoint = 'statuses/user_timeline';
  224. }
  225. }
  226. if (endpoint) {
  227. const offset = lock.threads[currentFeed].offset;
  228. if (offset > 0)
  229. config.since_id = offset;
  230. if (offset < -1)
  231. config.max_id = offset.slice(1);
  232. this.client.get(endpoint, config, (error, tweets, response) => {
  233. if (error) {
  234. if (error instanceof Array && error.length > 0 && error[0].code === 34) {
  235. logger.warn(`error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
  236. lock.threads[currentFeed].subscribers.forEach(subscriber => {
  237. logger.info(`sending notfound message of ${currentFeed} to ${JSON.stringify(subscriber)}`);
  238. this.bot.sendTo(subscriber, `链接 ${currentFeed} 指向的用户或列表不存在,请退订。`).catch();
  239. });
  240. }
  241. else {
  242. logger.error(`unhandled error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
  243. }
  244. resolve();
  245. }
  246. else
  247. resolve(tweets);
  248. });
  249. }
  250. });
  251. promise.then((tweets) => {
  252. logger.debug(`api returned ${JSON.stringify(tweets)} for feed ${currentFeed}`);
  253. const currentThread = lock.threads[currentFeed];
  254. const updateDate = () => currentThread.updatedAt = new Date().toString();
  255. if (!tweets || tweets.length === 0) {
  256. updateDate();
  257. return;
  258. }
  259. const topOfFeed = tweets[0].id_str;
  260. logger.info(`current offset: ${currentThread.offset}, current top of feed: ${topOfFeed}`);
  261. const bottomOfFeed = tweets[tweets.length - 1].id_str;
  262. const setOffset = (offset) => currentThread.offset = offset;
  263. const updateOffset = () => setOffset(topOfFeed);
  264. tweets = tweets.filter(twi => !twi.retweeted_status && twi.extended_entities);
  265. logger.info(`found ${tweets.length} tweets with extended entities`);
  266. if (currentThread.offset === '-1') {
  267. updateOffset();
  268. return;
  269. }
  270. if (currentThread.offset <= 0) {
  271. if (tweets.length === 0) {
  272. setOffset(utils_1.BigNumOps.plus('1', '-' + bottomOfFeed));
  273. lock.workon--;
  274. return;
  275. }
  276. tweets.splice(1);
  277. }
  278. if (tweets.length === 0) {
  279. updateDate();
  280. updateOffset();
  281. return;
  282. }
  283. return this.workOnTweets(tweets, this.sendTweets(`thread ${currentFeed}`, ...currentThread.subscribers))
  284. .then(updateDate).then(updateOffset);
  285. })
  286. .then(() => {
  287. lock.workon++;
  288. let timeout = this.workInterval * 1000 / lock.feed.length;
  289. if (timeout < 1000)
  290. timeout = 1000;
  291. fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
  292. setTimeout(() => {
  293. this.work();
  294. }, timeout);
  295. });
  296. };
  297. this.client = new Twitter({
  298. consumer_key: opt.consumer_key,
  299. consumer_secret: opt.consumer_secret,
  300. access_token_key: opt.access_token_key,
  301. access_token_secret: opt.access_token_secret,
  302. });
  303. this.lockfile = opt.lockfile;
  304. this.lock = opt.lock;
  305. this.workInterval = opt.workInterval;
  306. this.bot = opt.bot;
  307. this.webshotDelay = opt.webshotDelay;
  308. this.mode = opt.mode;
  309. ScreenNameNormalizer._queryUser = this.queryUser;
  310. exports.sendTweet = (id, receiver) => {
  311. this.getTweet(id, this.sendTweets(`tweet ${id}`, receiver))
  312. .catch((err) => {
  313. if (err[0].code !== 144) {
  314. logger.warn(`error retrieving tweet: ${err[0].message}`);
  315. this.bot.sendTo(receiver, `获取推文时出现错误:${err[0].message}`);
  316. }
  317. this.bot.sendTo(receiver, '找不到请求的推文,它可能已被删除。');
  318. });
  319. };
  320. exports.sendTimeline = ({ username, count, since, until, noreps, norts }, receiver) => {
  321. const countNum = Number(count) || 10;
  322. (countNum > 0 ? this.queryTimeline : this.queryTimelineReverse)({
  323. username,
  324. count: Math.abs(countNum),
  325. since: utils_1.BigNumOps.parse(since) || snowflake(new Date(since).getTime()),
  326. until: utils_1.BigNumOps.parse(until) || snowflake(new Date(until).getTime()),
  327. noreps: { on: true, off: false }[noreps],
  328. norts: { on: true, off: false }[norts],
  329. })
  330. .then(tweets => utils_1.chainPromises(tweets.map(tweet => this.bot.sendTo(receiver, `\
  331. 编号:${tweet.id_str}
  332. 时间:${tweet.created_at}
  333. 媒体:${tweet.extended_entities ? '有' : '无'}
  334. 正文:\n${tweet.full_text.replace(/^([\s\S\n]{50})[\s\S\n]+( https:\/\/t.co\/.*)$/, '$1…$2')}`))
  335. .concat(this.bot.sendTo(receiver, tweets.length ?
  336. '时间线查询完毕,使用 /twitterpic_view <编号> 查看媒体推文详细内容。' :
  337. '时间线查询完毕,没有找到符合条件的媒体推文。'))))
  338. .catch((err) => {
  339. var _a, _b, _c;
  340. if (((_a = err[0]) === null || _a === void 0 ? void 0 : _a.code) !== 34) {
  341. logger.warn(`error retrieving timeline: ${((_b = err[0]) === null || _b === void 0 ? void 0 : _b.message) || err}`);
  342. return this.bot.sendTo(receiver, `获取时间线时出现错误:${((_c = err[0]) === null || _c === void 0 ? void 0 : _c.message) || err}`);
  343. }
  344. this.bot.sendTo(receiver, `找不到用户 ${username.replace(/^@?(.*)$/, '@$1')}。`);
  345. });
  346. };
  347. }
  348. }
  349. exports.default = default_1;