import * as Twitter from 'twitter'; import TwitterTypes from 'twitter-d'; import { getLogger } from './loggers'; import { BigNumOps } from './utils'; interface IWorkerOption { consumerKey: string; consumerSecret: string; accessTokenKey: string; accessTokenSecret: string; } export interface ITimelineQueryConfig { username: string; count?: number; since?: string; until?: string; noreps?: boolean; norts?: boolean; } const TWITTER_EPOCH = 1288834974657; export const snowflake = (epoch: number) => Number.isNaN(epoch) ? undefined : BigNumOps.lShift(String(epoch - 1 - TWITTER_EPOCH), 22); const logger = getLogger('twitter'); export type FullUser = TwitterTypes.FullUser; export type Entities = TwitterTypes.Entities; export type ExtendedEntities = TwitterTypes.ExtendedEntities; export type MediaEntity = TwitterTypes.MediaEntity; interface ITweet extends TwitterTypes.Status { user: FullUser; retweeted_status?: Tweet; } export type Tweet = ITweet; export type Tweets = ITweet[]; export let queryByRegExp = (username: string, regexp: RegExp, cacheSeconds?: number, until?: string) => Promise.resolve(null); export default class { private client: Twitter; private lastQueries: {[key: string]: {match: RegExpExecArray, id: string}} = {}; constructor(opt: IWorkerOption) { this.client = new Twitter({ consumer_key: opt.consumerKey, consumer_secret: opt.consumerSecret, access_token_key: opt.accessTokenKey, access_token_secret: opt.accessTokenSecret, }); queryByRegExp = (username, regexp, cacheSeconds?: number, until?: string) => { logger.info(`searching timeline of @${username} for matches of ${regexp}...`); const normalizedUsername = username.toLowerCase().replace(/^@/, ''); const queryKey = `${normalizedUsername}:${regexp.toString()}`; const isOld = (then: string) => { if (!then) return true; return BigNumOps.compare(snowflake(Date.now() - cacheSeconds * 1000), then) >= 0; }; if (queryKey in this.lastQueries && !isOld(this.lastQueries[queryKey].id)) { const {match, id} = this.lastQueries[queryKey]; logger.info(`found match ${JSON.stringify(match)} from cached tweet of id ${id}`); return Promise.resolve(match); } return this.queryTimeline({username, norts: true, until}) .then(tweets => { const found = tweets.find(tweet => regexp.test(tweet.full_text)); if (found) { const match = regexp.exec(found.full_text); this.lastQueries[queryKey] = {match, id: found.id_str}; logger.info(`found match ${JSON.stringify(match)} in tweet of id ${found.id_str} from timeline`); return match; } const last = tweets.slice(-1)[0].id_str; if (isOld(last)) return null; queryByRegExp(username, regexp, cacheSeconds, last); }); }; } public queryUser = (username: string) => this.client.get('users/show', {screen_name: username}) .then((user: FullUser) => user.screen_name); public queryTimelineReverse = (conf: ITimelineQueryConfig) => { if (!conf.since) return this.queryTimeline(conf); const count = conf.count; const maxID = conf.until; conf.count = undefined; const until = () => BigNumOps.min(maxID, BigNumOps.plus(conf.since, String(7 * 24 * 3600 * 1000 * 2 ** 22))); conf.until = until(); const promise = (tweets: ITweet[]): Promise =>this.queryTimeline(conf).then(newTweets => { tweets = newTweets.concat(tweets); conf.since = conf.until; conf.until = until(); if ( tweets.length >= count || BigNumOps.compare(conf.since, conf.until) >= 0 ) { return tweets.slice(-count); } return promise(tweets); }); return promise([]); }; public queryTimeline = ( { username, count, since, until, noreps, norts }: ITimelineQueryConfig ) => { username = username.replace(/^@?(.*)$/, '@$1'); logger.info(`querying timeline of ${username} with config: ${ JSON.stringify({ ...(count && {count}), ...(since && {since}), ...(until && {until}), ...(noreps && {noreps}), ...(norts && {norts}), })}`); const fetchTimeline = ( config = { screen_name: username.slice(1), trim_user: true, exclude_replies: noreps ?? true, include_rts: !(norts ?? false), since_id: since, max_id: until, tweet_mode: 'extended', }, tweets: ITweet[] = [] ): Promise => this.client.get('statuses/user_timeline', config) .then((newTweets: ITweet[]) => { if (newTweets.length) { logger.debug(`fetched tweets: ${JSON.stringify(newTweets)}`); config.max_id = BigNumOps.plus('-1', newTweets[newTweets.length - 1].id_str); logger.info(`timeline query of ${username} yielded ${ newTweets.length } new tweets, next query will start at offset ${config.max_id}`); tweets.push(...newTweets); } if (!newTweets.length || count === undefined || tweets.length >= count) { logger.info(`timeline query of ${username} finished successfully, ${ tweets.length } tweets have been fetched`); return tweets.slice(0, count); } return fetchTimeline(config, tweets); }); return fetchTimeline(); }; }