|
@@ -0,0 +1,103 @@
|
|
|
|
+"use strict";
|
|
|
|
+Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
|
+exports.queryByRegExp = exports.snowflake = void 0;
|
|
|
|
+const Twitter = require("twitter");
|
|
|
|
+const loggers_1 = require("./loggers");
|
|
|
|
+const utils_1 = require("./utils");
|
|
|
|
+const TWITTER_EPOCH = 1288834974657;
|
|
|
|
+const snowflake = (epoch) => Number.isNaN(epoch) ? undefined :
|
|
|
|
+ utils_1.BigNumOps.lShift(String(epoch - 1 - TWITTER_EPOCH), 22);
|
|
|
|
+exports.snowflake = snowflake;
|
|
|
|
+const logger = loggers_1.getLogger('twitter');
|
|
|
|
+let queryByRegExp = (username, regexp, cacheSeconds, until) => Promise.resolve(null);
|
|
|
|
+exports.queryByRegExp = queryByRegExp;
|
|
|
|
+class default_1 {
|
|
|
|
+ constructor(opt) {
|
|
|
|
+ this.lastQueries = {};
|
|
|
|
+ this.queryUser = (username) => this.client.get('users/show', { screen_name: username })
|
|
|
|
+ .then((user) => user.screen_name);
|
|
|
|
+ this.queryTimelineReverse = (conf) => {
|
|
|
|
+ if (!conf.since)
|
|
|
|
+ return this.queryTimeline(conf);
|
|
|
|
+ const count = conf.count;
|
|
|
|
+ const maxID = conf.until;
|
|
|
|
+ conf.count = undefined;
|
|
|
|
+ const until = () => utils_1.BigNumOps.min(maxID, utils_1.BigNumOps.plus(conf.since, String(7 * 24 * 3600 * 1000 * Math.pow(2, 22))));
|
|
|
|
+ conf.until = until();
|
|
|
|
+ const promise = (tweets) => this.queryTimeline(conf).then(newTweets => {
|
|
|
|
+ tweets = newTweets.concat(tweets);
|
|
|
|
+ conf.since = conf.until;
|
|
|
|
+ conf.until = until();
|
|
|
|
+ if (tweets.length >= count ||
|
|
|
|
+ utils_1.BigNumOps.compare(conf.since, conf.until) >= 0) {
|
|
|
|
+ return tweets.slice(-count);
|
|
|
|
+ }
|
|
|
|
+ return promise(tweets);
|
|
|
|
+ });
|
|
|
|
+ return promise([]);
|
|
|
|
+ };
|
|
|
|
+ this.queryTimeline = ({ username, count, since, until, noreps, norts }) => {
|
|
|
|
+ username = username.replace(/^@?(.*)$/, '@$1');
|
|
|
|
+ logger.info(`querying timeline of ${username} with config: ${JSON.stringify(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, (count && { count })), (since && { since })), (until && { until })), (noreps && { noreps })), (norts && { norts })))}`);
|
|
|
|
+ const fetchTimeline = (config = {
|
|
|
|
+ screen_name: username.slice(1),
|
|
|
|
+ trim_user: true,
|
|
|
|
+ exclude_replies: noreps !== null && noreps !== void 0 ? noreps : true,
|
|
|
|
+ include_rts: !(norts !== null && norts !== void 0 ? norts : false),
|
|
|
|
+ since_id: since,
|
|
|
|
+ max_id: until,
|
|
|
|
+ tweet_mode: 'extended',
|
|
|
|
+ }, tweets = []) => this.client.get('statuses/user_timeline', config)
|
|
|
|
+ .then((newTweets) => {
|
|
|
|
+ if (newTweets.length) {
|
|
|
|
+ logger.debug(`fetched tweets: ${JSON.stringify(newTweets)}`);
|
|
|
|
+ config.max_id = utils_1.BigNumOps.plus('-1', newTweets[newTweets.length - 1].id_str);
|
|
|
|
+ logger.info(`timeline query of ${username} yielded ${newTweets.length} new tweets, next query will start at offset ${config.max_id}`);
|
|
|
|
+ tweets.push(...newTweets);
|
|
|
|
+ }
|
|
|
|
+ if (!newTweets.length || count === undefined || tweets.length >= count) {
|
|
|
|
+ logger.info(`timeline query of ${username} finished successfully, ${tweets.length} tweets have been fetched`);
|
|
|
|
+ return tweets.slice(0, count);
|
|
|
|
+ }
|
|
|
|
+ return fetchTimeline(config, tweets);
|
|
|
|
+ });
|
|
|
|
+ return fetchTimeline();
|
|
|
|
+ };
|
|
|
|
+ this.client = new Twitter({
|
|
|
|
+ consumer_key: opt.consumerKey,
|
|
|
|
+ consumer_secret: opt.consumerSecret,
|
|
|
|
+ access_token_key: opt.accessTokenKey,
|
|
|
|
+ access_token_secret: opt.accessTokenSecret,
|
|
|
|
+ });
|
|
|
|
+ exports.queryByRegExp = (username, regexp, cacheSeconds, until) => {
|
|
|
|
+ logger.info(`searching timeline of @${username} for matches of ${regexp}...`);
|
|
|
|
+ const normalizedUsername = username.toLowerCase().replace(/^@/, '');
|
|
|
|
+ const queryKey = `${normalizedUsername}:${regexp.toString()}`;
|
|
|
|
+ const isOld = (then) => {
|
|
|
|
+ if (!then)
|
|
|
|
+ return true;
|
|
|
|
+ return utils_1.BigNumOps.compare(exports.snowflake(Date.now() - cacheSeconds * 1000), then) >= 0;
|
|
|
|
+ };
|
|
|
|
+ if (queryKey in this.lastQueries && !isOld(this.lastQueries[queryKey].id)) {
|
|
|
|
+ const { match, id } = this.lastQueries[queryKey];
|
|
|
|
+ logger.info(`found match ${JSON.stringify(match)} from cached tweet of id ${id}`);
|
|
|
|
+ return Promise.resolve(match);
|
|
|
|
+ }
|
|
|
|
+ return this.queryTimeline({ username, norts: true, until })
|
|
|
|
+ .then(tweets => {
|
|
|
|
+ const found = tweets.find(tweet => regexp.test(tweet.full_text));
|
|
|
|
+ if (found) {
|
|
|
|
+ const match = regexp.exec(found.full_text);
|
|
|
|
+ this.lastQueries[queryKey] = { match, id: found.id_str };
|
|
|
|
+ logger.info(`found match ${JSON.stringify(match)} in tweet of id ${found.id_str} from timeline`);
|
|
|
|
+ return match;
|
|
|
|
+ }
|
|
|
|
+ const last = tweets.slice(-1)[0].id_str;
|
|
|
|
+ if (isOld(last))
|
|
|
|
+ return null;
|
|
|
|
+ exports.queryByRegExp(username, regexp, cacheSeconds, last);
|
|
|
|
+ });
|
|
|
|
+ };
|
|
|
|
+ }
|
|
|
|
+}
|
|
|
|
+exports.default = default_1;
|