"use strict"; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; Object.defineProperty(exports, "__esModule", { value: true }); exports.sendTimeline = exports.sendTweet = exports.ScreenNameNormalizer = exports.linkBuilder = exports.parseLink = void 0; const fs = require("fs"); const path = require("path"); const Twitter = require("twitter-api-v2"); const loggers_1 = require("./loggers"); const redis_1 = require("./redis"); const utils_1 = require("./utils"); const webshot_1 = require("./webshot"); const parseLink = (link) => { let match = /twitter.com\/([^\/?#]+)\/lists\/([^\/?#]+)/.exec(link) || /^([^\/?#]+)\/([^\/?#]+)$/.exec(link); if (match) return [match[1], `/lists/${match[2]}`]; match = /twitter.com\/([^\/?#]+)\/status\/(\d+)/.exec(link); if (match) return [match[1], `/status/${match[2]}`]; match = /twitter.com\/([^\/?#]+)/.exec(link) || /^([^\/?#]+)$/.exec(link); if (match) return [match[1]]; return; }; exports.parseLink = parseLink; const linkBuilder = (userName, more = '') => { if (!userName) return; return `https://twitter.com/${userName}${more}`; }; exports.linkBuilder = linkBuilder; class ScreenNameNormalizer { static normalizeLive(username) { return __awaiter(this, void 0, void 0, function* () { if (this._queryUser) { return yield this._queryUser(username) .then(userNameId => userNameId.split(':')[0]) .catch((err) => { if (err.title === 'Not Found Error') { logger.warn(`error looking up user: ${showApiError(err)}`); return username; } return null; }); } return this.normalize(username); }); } } exports.ScreenNameNormalizer = ScreenNameNormalizer; ScreenNameNormalizer.normalize = (username) => username.toLowerCase().replace(/^@/, ''); let sendTweet = (id, receiver, forceRefresh) => { throw Error(); }; exports.sendTweet = sendTweet; let sendTimeline = (conf, receiver) => { throw Error(); }; exports.sendTimeline = sendTimeline; const TWITTER_EPOCH = 1288834974657; const snowflake = (epoch) => Number.isNaN(epoch) ? undefined : utils_1.BigNumOps.lShift(String(epoch - 1 - TWITTER_EPOCH), 22); const logger = (0, loggers_1.getLogger)('twitter'); const maxTrials = 3; const retryInterval = 1500; const ordinal = (n) => { switch ((Math.trunc(n / 10) % 10 === 1) ? 0 : n % 10) { case 1: return `${n}st`; case 2: return `${n}nd`; case 3: return `${n}rd`; default: return `${n}th`; } }; const retryOnError = (doWork, onRetry) => new Promise(resolve => { const retry = (reason, count) => { setTimeout(() => { let terminate = false; onRetry(reason, count, defaultValue => { terminate = true; resolve(defaultValue); }); if (!terminate) doWork().then(resolve).catch(error => retry(error, count + 1)); }, retryInterval); }; doWork().then(resolve).catch(error => retry(error, 1)); }); const showApiError = (err) => err.errors && err.errors[0].message || err.detail || err.stack || JSON.stringify(err); const toMutableConst = (o) => { return o; }; const v2SingleParams = toMutableConst({ expansions: ['attachments.media_keys', 'author_id', 'referenced_tweets.id'], 'tweet.fields': ['created_at', 'entities'], 'media.fields': ['url', 'variants', 'alt_text'], 'user.fields': ['id', 'name', 'username'] }); ; class default_1 { constructor(opt) { this.launch = () => { this.client.appLogin().then(client => { this.client = client.readOnly; this.webshot = new webshot_1.default(this.wsUrl, this.mode, () => setTimeout(this.work, this.workInterval * 1000)); }); }; this.queryUser = (username) => { const thread = this.lock.threads[(0, exports.linkBuilder)(username)]; if (thread && thread.id) return Promise.resolve(`${username}:${thread.id}`); return this.client.v2.userByUsername(username).then(({ data: { username, id }, errors }) => { if (errors && errors.length > 0) throw errors[0]; if (thread) thread.id = id; return `${username}:${id}`; }); }; this.queryTimelineReverse = (conf) => { if (!conf.since) return this.queryTimeline(conf); const count = conf.count; const maxID = conf.until; conf.count = undefined; const until = () => utils_1.BigNumOps.min(maxID, utils_1.BigNumOps.plus(conf.since, String(7 * 24 * 3600 * 1000 * Math.pow(2, 22)))); conf.until = until(); const promise = (tweets) => this.queryTimeline(conf).then(newTweets => { tweets = newTweets.concat(tweets); conf.since = conf.until; conf.until = until(); if (tweets.length >= count || utils_1.BigNumOps.compare(conf.since, conf.until) >= 0) { return tweets.slice(-count); } return promise(tweets); }); return promise([]); }; this.queryTimeline = ({ username, count, since, until, noreps, norts }) => { username = username.replace(/^@?(.*)$/, '@$1'); logger.info(`querying timeline of ${username} with config: ${JSON.stringify(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, (count && { count })), (since && { since })), (until && { until })), (noreps && { noreps })), (norts && { norts })))}`); return this.queryUser(username.slice(1)).then(userNameId => this.get('userTimeline', userNameId.split(':')[1], Object.assign(Object.assign(Object.assign({ expansions: ['attachments.media_keys', 'author_id'], 'tweet.fields': ['created_at'], exclude: [ ...(noreps !== null && noreps !== void 0 ? noreps : true) ? ['replies'] : [], ...(norts !== null && norts !== void 0 ? norts : false) ? ['retweets'] : [], ] }, (count && { max_results: Math.min(Math.max(count, 5), 100) })), (since && { since_id: since })), (until && { until_id: until }))).then(tweets => tweets.slice(0, count))); }; this.workOnTweets = (tweets, sendTweets, refresh = false) => Promise.all(tweets.map(({ data, includes }) => ((this.redis && !refresh) ? this.redis.waitForProcess(`webshot/${data.id}`, this.webshotDelay * 4) .then(() => this.redis.getContent(`webshot/${data.id}`)) : Promise.reject()) .then(content => { if (content === null) throw Error(); logger.info(`retrieved cached webshot of tweet ${data.id} from redis database`); const { msg, text, author } = JSON.parse(content); let cacheId = data.id; const retweetRef = (data.referenced_tweets || []).find(ref => ref.type === 'retweeted'); if (retweetRef) cacheId += `,rt:${retweetRef.id}`; sendTweets(cacheId, msg, text, author); return null; }) .catch(() => { this.redis.startProcess(`webshot/${data.id}`); return { data, includes }; }))).then(tweets => this.webshot(tweets.filter(t => t), (cacheId, msg, text, author) => { Promise.resolve() .then(() => { if (!this.redis) return; const [twid, rtid] = cacheId.split(',rt:'); logger.info(`caching webshot of tweet ${twid} to redis database`); this.redis.cacheContent(`webshot/${twid}`, JSON.stringify({ msg, text, author, rtid })) .then(() => this.redis.finishProcess(`webshot/${twid}`)); }) .then(() => sendTweets(cacheId, msg, text, author)); }, this.webshotDelay)); this.getTweet = (id, sender, refresh = false) => ((this.redis && !refresh) ? this.redis.waitForProcess(`webshot/${id}`, this.webshotDelay * 4) .then(() => this.redis.getContent(`webshot/${id}`)) .then(content => { if (content === null) throw Error(); const { rtid } = JSON.parse(content); return { data: Object.assign({ id }, rtid && { referenced_tweets: [{ type: 'retweeted', id: rtid }] }) }; }) : Promise.reject()) .catch(() => this.client.v2.singleTweet(id, v2SingleParams)) .then((tweet) => { if (tweet.data.text) { logger.debug(`api returned tweet ${JSON.stringify(tweet)} for query id=${id}`); } else { logger.debug(`skipped querying api as this tweet has been cached`); } return this.workOnTweets([tweet], sender, refresh); }); this.sendTweets = (config = { reportOnSkip: false, force: false }, ...to) => (id, msg, text, author) => { to.forEach(subscriber => { const [twid, rtid] = id.split(',rt:'); const { sourceInfo: source, reportOnSkip, force } = config; const targetStr = JSON.stringify(subscriber); const send = () => retryOnError(() => this.bot.sendTo(subscriber, msg), (_, count, terminate) => { if (count <= maxTrials) { logger.warn(`retry sending to ${subscriber.chatID} for the ${ordinal(count)} time...`); } else { logger.warn(`${count - 1} consecutive failures while sending message chain, trying plain text instead...`); terminate(this.bot.sendTo(subscriber, author + text, true)); } }).then(() => { if (this.redis) { logger.info(`caching push status of tweet ${rtid ? `${rtid} (RTed as ${twid})` : twid} for ${targetStr}...`); return this.redis.cacheForChat(rtid || twid, subscriber); } }); ((this.redis && !force) ? this.redis.isCachedForChat(rtid || twid, subscriber) : Promise.resolve(false)) .then(isCached => { if (isCached) { logger.info(`skipped subscriber ${targetStr} as tweet ${rtid ? `${rtid} (or its RT)` : twid} has been sent already`); if (!reportOnSkip) return; text = `[最近发送过的推文:${rtid || twid}]`; msg = author + text; } logger.info(`pushing data${source ? ` of ${source}` : ''} to ${targetStr}`); return send(); }); }); }; this.get = (type, targetId, params) => { const { since_id, max_results } = params; const getMore = (res) => { if (res.errors && res.errors.length > 0) throw res.errors[0]; if (!res.meta.next_token || utils_1.BigNumOps.compare(res.tweets.slice(-1)[0].id, since_id || '0') !== 1 || !since_id && res.meta.result_count >= max_results) return res; return res.fetchNext().then(getMore); }; if (type === 'listTweets') delete params.since_id; return this.client.v2[type](targetId, params).then(getMore) .then(({ includes, tweets }) => tweets.map((tweet) => ({ data: tweet, includes: { media: includes.medias(tweet), users: [includes.author(tweet)] } }))); }; this.work = () => { const lock = this.lock; if (this.workInterval < 1) this.workInterval = 1; if (lock.feed.length === 0) { setTimeout(() => { this.work(); }, this.workInterval * 1000); return; } if (lock.workon >= lock.feed.length) lock.workon = 0; if (!lock.threads[lock.feed[lock.workon]] || !lock.threads[lock.feed[lock.workon]].subscribers || lock.threads[lock.feed[lock.workon]].subscribers.length === 0) { logger.warn(`nobody subscribes thread ${lock.feed[lock.workon]}, removing from feed`); delete lock.threads[lock.feed[lock.workon]]; lock.feed.splice(lock.workon, 1); fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock)); this.work(); return; } const currentFeed = lock.feed[lock.workon]; logger.debug(`pulling feed ${currentFeed}`); const promise = new Promise(resolve => { let job = Promise.resolve(); let id = lock.threads[currentFeed].id; let endpoint; let match = /https:\/\/twitter.com\/([^\/]+)\/lists\/([^\/]+)/.exec(currentFeed); if (match) { endpoint = 'listTweets'; if (match[1] === 'i') { id = match[2]; } else if (id === undefined) { job = job.then(() => this.client.v1.list({ owner_screen_name: match[1], slug: match[2], })).then(({ id_str }) => { lock.threads[currentFeed].id = id = id_str; }); } } else { match = /https:\/\/twitter.com\/([^\/]+)/.exec(currentFeed); if (match) { endpoint = 'userTimeline'; if (id === undefined) { job = job.then(() => this.queryUser(match[1].replace(/^@?(.*)$/, '$1'))).then(userNameId => { lock.threads[currentFeed].id = id = userNameId.split(':')[1]; }); } } } const offset = lock.threads[currentFeed].offset; job.then(() => this.get(endpoint, id, Object.assign(Object.assign(Object.assign({}, v2SingleParams), { max_results: 20 }), (offset > 0) && { since_id: offset }))).catch((err) => { if (err.title === 'Not Found Error') { logger.warn(`error on fetching tweets for ${currentFeed}: ${showApiError(err)}`); lock.threads[currentFeed].subscribers.forEach(subscriber => { logger.info(`sending notfound message of ${currentFeed} to ${JSON.stringify(subscriber)}`); this.bot.sendTo(subscriber, `链接 ${currentFeed} 指向的用户或列表不存在,请退订。`).catch(); }); } else { logger.error(`unhandled error on fetching tweets for ${currentFeed}: ${showApiError(err)}`); } return []; }).then(resolve); }); promise.then((tweets) => { logger.debug(`api returned ${JSON.stringify(tweets)} for feed ${currentFeed}`); const currentThread = lock.threads[currentFeed]; const updateDate = () => currentThread.updatedAt = new Date().toString(); if (tweets.length === 0) { updateDate(); return; } const topOfFeed = tweets[0].data.id; const updateOffset = () => currentThread.offset = topOfFeed; if (currentThread.offset === '-1') { updateOffset(); return; } if (currentThread.offset === '0') tweets.splice(1); return this.workOnTweets(tweets, this.sendTweets({ sourceInfo: `thread ${currentFeed}` }, ...currentThread.subscribers)) .then(updateDate).then(updateOffset); }) .then(() => { lock.workon++; let timeout = this.workInterval * 1000 / lock.feed.length; if (timeout < 1000) timeout = 1000; fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock)); setTimeout(() => { this.work(); }, timeout); }); }; this.client = new Twitter.TwitterApi({ appKey: opt.consumerKey, appSecret: opt.consumerSecret, }).readOnly; this.lockfile = opt.lockfile; this.lock = opt.lock; this.workInterval = opt.workInterval; this.bot = opt.bot; this.webshotDelay = opt.webshotDelay; this.mode = opt.mode; this.wsUrl = opt.wsUrl; if (opt.redis) this.redis = new redis_1.default(opt.redis); ScreenNameNormalizer._queryUser = this.queryUser; exports.sendTweet = (idOrQuery, receiver, forceRefresh) => { const match = /^last(|-\d+)@([^\/?#,]+)((?:,no.*?=[^,]*)*)$/.exec(idOrQuery); const query = () => this.queryTimeline({ username: match[2], count: 1 - Number(match[1]), noreps: { on: true, off: false }[match[3].replace(/.*,noreps=([^,]*).*/, '$1')], norts: { on: true, off: false }[match[3].replace(/.*,norts=([^,]*).*/, '$1')], }).then(tweets => tweets.slice(-1)[0].data.id); (match ? query() : Promise.resolve(idOrQuery)) .then((id) => this.getTweet(id, this.sendTweets({ sourceInfo: `tweet ${id}`, reportOnSkip: true, force: forceRefresh }, receiver), forceRefresh)) .catch((err) => { if (err.title !== 'Not Found Error') { logger.warn(`error retrieving tweet: ${showApiError(err)}`); this.bot.sendTo(receiver, `获取推文时出现错误:${showApiError(err)}`); } if (err.resource_type === 'user') { return this.bot.sendTo(receiver, `找不到用户 ${match[2].replace(/^@?(.*)$/, '@$1')}。`); } this.bot.sendTo(receiver, '找不到请求的推文,它可能已被删除。'); }); }; exports.sendTimeline = ({ username, count, since, until, noreps, norts }, receiver) => { const countNum = Number(count) || 10; (countNum > 0 ? this.queryTimeline : this.queryTimelineReverse)({ username, count: Math.abs(countNum), since: utils_1.BigNumOps.parse(since) || snowflake(new Date(since).getTime()), until: utils_1.BigNumOps.parse(until) || snowflake(new Date(until).getTime()), noreps: { on: true, off: false }[noreps], norts: { on: true, off: false }[norts], }) .then(tweets => (0, utils_1.chainPromises)(tweets.map(({ data }) => () => this.bot.sendTo(receiver, `\ 编号:${data.id} 时间:${data.created_at} 媒体:${(data.attachments || {}).media_keys ? '有' : '无'} 正文:\n${data.text.replace(/^([\s\S\n]{50})[\s\S\n]+?( https:\/\/t.co\/.*)?$/, '$1…$2')}`)) .concat(() => this.bot.sendTo(receiver, tweets.length ? '时间线查询完毕,使用 /twitter_view <编号> 查看推文详细内容。' : '时间线查询完毕,没有找到符合条件的推文。')))) .catch((err) => { if (err.title !== 'Not Found Error') { logger.warn(`error retrieving timeline: ${showApiError(err)}`); return this.bot.sendTo(receiver, `获取时间线时出现错误:${showApiError(err)}`); } this.bot.sendTo(receiver, `找不到用户 ${username.replace(/^@?(.*)$/, '@$1')}。`); }); }; } } exports.default = default_1;