|
|
@@ -1,10 +1,9 @@
|
|
|
import * as fs from 'fs';
|
|
|
import * as path from 'path';
|
|
|
-import * as Twitter from 'twitter';
|
|
|
-import TwitterTypes from 'twitter-d';
|
|
|
+import * as Twitter from 'twitter-api-v2';
|
|
|
|
|
|
import { getLogger } from './loggers';
|
|
|
-import QQBot from './koishi';
|
|
|
+import QQBot, { Message } from './koishi';
|
|
|
import RedisSvc from './redis';
|
|
|
import { chainPromises, BigNumOps } from './utils';
|
|
|
import Webshot from './webshot';
|
|
|
@@ -17,13 +16,33 @@ interface IWorkerOption {
|
|
|
webshotDelay: number;
|
|
|
consumerKey: string;
|
|
|
consumerSecret: string;
|
|
|
- accessTokenKey: string;
|
|
|
- accessTokenSecret: string;
|
|
|
+ accessTokenKey?: string;
|
|
|
+ accessTokenSecret?: string;
|
|
|
mode: number;
|
|
|
wsUrl: string;
|
|
|
redis?: IRedisConfig;
|
|
|
}
|
|
|
|
|
|
+export const parseLink = (link: string): string[] => {
|
|
|
+ let match =
|
|
|
+ /twitter.com\/([^\/?#]+)\/lists\/([^\/?#]+)/.exec(link) ||
|
|
|
+ /^([^\/?#]+)\/([^\/?#]+)$/.exec(link);
|
|
|
+ if (match) return [match[1], `/lists/${match[2]}`];
|
|
|
+ match =
|
|
|
+ /twitter.com\/([^\/?#]+)\/status\/(\d+)/.exec(link);
|
|
|
+ if (match) return [match[1], `/status/${match[2]}`];
|
|
|
+ match =
|
|
|
+ /twitter.com\/([^\/?#]+)/.exec(link) ||
|
|
|
+ /^([^\/?#]+)$/.exec(link);
|
|
|
+ if (match) return [match[1]];
|
|
|
+ return;
|
|
|
+}
|
|
|
+
|
|
|
+export const linkBuilder = (userName: string, more = ''): string => {
|
|
|
+ if (!userName) return;
|
|
|
+ return `https://twitter.com/${userName}${more}`;
|
|
|
+}
|
|
|
+
|
|
|
export class ScreenNameNormalizer {
|
|
|
|
|
|
// tslint:disable-next-line: variable-name
|
|
|
@@ -32,17 +51,19 @@ export class ScreenNameNormalizer {
|
|
|
public static normalize = (username: string) => username.toLowerCase().replace(/^@/, '');
|
|
|
|
|
|
public static async normalizeLive(username: string) {
|
|
|
+ username = this.normalize(username);
|
|
|
if (this._queryUser) {
|
|
|
return await this._queryUser(username)
|
|
|
- .catch((err: {code: number, message: string}[]) => {
|
|
|
- if (err[0].code !== 50) {
|
|
|
- logger.warn(`error looking up user: ${err[0].message}`);
|
|
|
+ .then(userNameId => userNameId.split(':')[0])
|
|
|
+ .catch((err: Twitter.InlineErrorV2) => {
|
|
|
+ if (err.title === 'Not Found Error') {
|
|
|
+ logger.warn(`error looking up user: ${showApiError(err)}`);
|
|
|
return username;
|
|
|
}
|
|
|
return null;
|
|
|
});
|
|
|
}
|
|
|
- return this.normalize(username);
|
|
|
+ return username;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
@@ -99,19 +120,53 @@ const retryOnError = <T, U>(
|
|
|
doWork().then(resolve).catch(error => retry(error, 1));
|
|
|
});
|
|
|
|
|
|
-export type FullUser = TwitterTypes.FullUser;
|
|
|
-export type Entities = TwitterTypes.Entities;
|
|
|
-export type ExtendedEntities = TwitterTypes.ExtendedEntities;
|
|
|
-export type MediaEntity = TwitterTypes.MediaEntity;
|
|
|
+const showApiError = (err: Partial<Twitter.InlineErrorV2 & Twitter.ErrorV2 & Error>) =>
|
|
|
+ err.errors && err.errors[0].message || err.detail || err.stack || JSON.stringify(err);
|
|
|
|
|
|
-export interface Tweet extends TwitterTypes.Status {
|
|
|
- user: FullUser;
|
|
|
- retweeted_status?: Tweet;
|
|
|
-}
|
|
|
+const toMutableConst = <T>(o: T) => {
|
|
|
+ // credits: https://stackoverflow.com/a/60493166
|
|
|
+ type DeepMutableArrays<T> =
|
|
|
+ (T extends object ? { [K in keyof T]: DeepMutableArrays<T[K]> } : T) extends infer O ?
|
|
|
+ O extends ReadonlyArray<any> ? { -readonly [K in keyof O]: O[K] } : O : never;
|
|
|
+ return o as DeepMutableArrays<T>
|
|
|
+};
|
|
|
+
|
|
|
+const v2SingleParams = toMutableConst({
|
|
|
+ expansions: ['attachments.media_keys', 'author_id', 'referenced_tweets.id'],
|
|
|
+ 'tweet.fields': ['created_at', 'entities'],
|
|
|
+ 'media.fields': ['url', 'variants', 'alt_text'],
|
|
|
+ 'user.fields': ['id', 'name', 'username']
|
|
|
+} as const);
|
|
|
+
|
|
|
+type PickRequired<T, K extends keyof T> = Pick<Required<T>, K> & T;
|
|
|
+
|
|
|
+export type TweetObject = PickRequired<
|
|
|
+ Twitter.TweetV2SingleResult['data'],
|
|
|
+ typeof v2SingleParams['tweet.fields'][number]
|
|
|
+>;
|
|
|
+
|
|
|
+export type UserObject = PickRequired<
|
|
|
+ Twitter.TweetV2SingleResult['includes']['users'][number],
|
|
|
+ typeof v2SingleParams['user.fields'][number]
|
|
|
+>;
|
|
|
+
|
|
|
+export type MediaObject =
|
|
|
+ Twitter.TweetV2SingleResult['includes']['media'][number] & (
|
|
|
+ {type: 'video' | 'animated_gif', variants: Twitter.MediaVariantsV2[]} |
|
|
|
+ {type: 'photo', url: string}
|
|
|
+ );
|
|
|
+
|
|
|
+export interface Tweet extends Twitter.TweetV2SingleResult {
|
|
|
+ data: TweetObject,
|
|
|
+ includes: {
|
|
|
+ media: MediaObject[],
|
|
|
+ users: UserObject[],
|
|
|
+ }
|
|
|
+};
|
|
|
|
|
|
export default class {
|
|
|
|
|
|
- private client: Twitter;
|
|
|
+ private client: Twitter.TwitterApiReadOnly;
|
|
|
private lock: ILock;
|
|
|
private lockfile: string;
|
|
|
private workInterval: number;
|
|
|
@@ -123,12 +178,10 @@ export default class {
|
|
|
private redis: RedisSvc;
|
|
|
|
|
|
constructor(opt: IWorkerOption) {
|
|
|
- this.client = new Twitter({
|
|
|
- consumer_key: opt.consumerKey,
|
|
|
- consumer_secret: opt.consumerSecret,
|
|
|
- access_token_key: opt.accessTokenKey,
|
|
|
- access_token_secret: opt.accessTokenSecret,
|
|
|
- });
|
|
|
+ this.client = new Twitter.TwitterApi({
|
|
|
+ appKey: opt.consumerKey,
|
|
|
+ appSecret: opt.consumerSecret,
|
|
|
+ }).readOnly;
|
|
|
this.lockfile = opt.lockfile;
|
|
|
this.lock = opt.lock;
|
|
|
this.workInterval = opt.workInterval;
|
|
|
@@ -145,19 +198,20 @@ export default class {
|
|
|
count: 1 - Number(match[1]),
|
|
|
noreps: {on: true, off: false}[match[3].replace(/.*,noreps=([^,]*).*/, '$1')],
|
|
|
norts: {on: true, off: false}[match[3].replace(/.*,norts=([^,]*).*/, '$1')],
|
|
|
- }).then(tweets => tweets.slice(-1)[0].id_str);
|
|
|
+ }).then(tweets => tweets.slice(-1)[0].data.id);
|
|
|
(match ? query() : Promise.resolve(idOrQuery))
|
|
|
.then((id: string) => this.getTweet(
|
|
|
id,
|
|
|
this.sendTweets({sourceInfo: `tweet ${id}`, reportOnSkip: true, force: forceRefresh}, receiver),
|
|
|
forceRefresh
|
|
|
))
|
|
|
- .catch((err: {code: number, message: string}[]) => {
|
|
|
- if (err[0]?.code === 34)
|
|
|
+ .catch((err: Twitter.InlineErrorV2) => {
|
|
|
+ if (err.title !== 'Not Found Error') {
|
|
|
+ logger.warn(`error retrieving tweet: ${showApiError(err)}`);
|
|
|
+ this.bot.sendTo(receiver, `获取推文时出现错误：${showApiError(err)}`);
|
|
|
+ }
|
|
|
+ if (err.resource_type === 'user') {
|
|
|
return this.bot.sendTo(receiver, `找不到用户 ${match[2].replace(/^@?(.*)$/, '@$1')}。`);
|
|
|
- if (err[0].code !== 144) {
|
|
|
- logger.warn(`error retrieving tweet: ${err[0].message}`);
|
|
|
- this.bot.sendTo(receiver, `获取推文时出现错误：${err[0].message}`);
|
|
|
}
|
|
|
this.bot.sendTo(receiver, '找不到请求的推文，它可能已被删除。');
|
|
|
});
|
|
|
@@ -173,21 +227,21 @@ export default class {
|
|
|
norts: {on: true, off: false}[norts],
|
|
|
})
|
|
|
.then(tweets => chainPromises(
|
|
|
- tweets.map(tweet => () => this.bot.sendTo(receiver, `\
|
|
|
-编号：${tweet.id_str}
|
|
|
-时间：${tweet.created_at}
|
|
|
-媒体：${tweet.extended_entities ? '有' : '无'}
|
|
|
-正文：\n${tweet.full_text.replace(/^([\s\S\n]{50})[\s\S\n]+?( https:\/\/t.co\/.*)?$/, '$1…$2')}`
|
|
|
+ tweets.map(({data}) => () => this.bot.sendTo(receiver, `\
|
|
|
+编号：${data.id}
|
|
|
+时间：${data.created_at}
|
|
|
+媒体：${(data.attachments || {}).media_keys ? '有' : '无'}
|
|
|
+正文：\n${data.text.replace(/^([\s\S\n]{50})[\s\S\n]+?( https:\/\/t.co\/.*)?$/, '$1…$2')}`
|
|
|
))
|
|
|
.concat(() => this.bot.sendTo(receiver, tweets.length ?
|
|
|
- '时间线查询完毕，使用 /twitterpic_view <编号> 查看媒体推文详细内容。' :
|
|
|
+ '时间线查询完毕，使用 /twipic_view <编号> 查看媒体推文详细内容。' :
|
|
|
'时间线查询完毕，没有找到符合条件的媒体推文。'
|
|
|
))
|
|
|
))
|
|
|
- .catch((err: {code: number, message: string}[]) => {
|
|
|
- if (err[0]?.code !== 34) {
|
|
|
- logger.warn(`error retrieving timeline: ${err[0]?.message || err}`);
|
|
|
- return this.bot.sendTo(receiver, `获取时间线时出现错误：${err[0]?.message || err}`);
|
|
|
+ .catch((err: Twitter.InlineErrorV2) => {
|
|
|
+ if (err.title !== 'Not Found Error') {
|
|
|
+ logger.warn(`error retrieving timeline: ${showApiError(err)}`);
|
|
|
+ return this.bot.sendTo(receiver, `获取时间线时出现错误：${showApiError(err)}`);
|
|
|
}
|
|
|
this.bot.sendTo(receiver, `找不到用户 ${username.replace(/^@?(.*)$/, '@$1')}。`);
|
|
|
});
|
|
|
@@ -195,15 +249,25 @@ export default class {
|
|
|
}
|
|
|
|
|
|
public launch = () => {
|
|
|
- this.webshot = new Webshot(
|
|
|
- this.wsUrl,
|
|
|
- this.mode,
|
|
|
- () => setTimeout(this.work, this.workInterval * 1000)
|
|
|
- );
|
|
|
+ this.client.appLogin().then(client => {
|
|
|
+ this.client = client.readOnly;
|
|
|
+ this.webshot = new Webshot(
|
|
|
+ this.wsUrl,
|
|
|
+ this.mode,
|
|
|
+ () => setTimeout(this.work, this.workInterval * 1000)
|
|
|
+ );
|
|
|
+ });
|
|
|
};
|
|
|
|
|
|
- public queryUser = (username: string) => this.client.get('users/show', {screen_name: username})
|
|
|
- .then((user: FullUser) => user.screen_name);
|
|
|
+ public queryUser = (username: string) => {
|
|
|
+ const thread = this.lock.threads[linkBuilder(username)];
|
|
|
+ if (thread && thread.id) return Promise.resolve(`${username}:${thread.id}`);
|
|
|
+ return this.client.v2.userByUsername(username).then(({data: {username, id}, errors}) => {
|
|
|
+ if (errors && errors.length > 0) throw errors[0];
|
|
|
+ if (thread) thread.id = id;
|
|
|
+ return `${username}:${id}`;
|
|
|
+ })
|
|
|
+ }
|
|
|
|
|
|
public queryTimelineReverse = (conf: ITimelineQueryConfig) => {
|
|
|
if (!conf.since) return this.queryTimeline(conf);
|
|
|
@@ -230,70 +294,70 @@ export default class {
|
|
|
};
|
|
|
|
|
|
public queryTimeline = (
|
|
|
- { username, count, since, until, noreps, norts }: ITimelineQueryConfig
|
|
|
+ {username, count, since, until, noreps, norts}: ITimelineQueryConfig
|
|
|
) => {
|
|
|
username = username.replace(/^@?(.*)$/, '@$1');
|
|
|
- logger.info(`querying timeline of ${username} with config: ${
|
|
|
- JSON.stringify({
|
|
|
- ...(count && {count}),
|
|
|
- ...(since && {since}),
|
|
|
- ...(until && {until}),
|
|
|
- ...(noreps && {noreps}),
|
|
|
- ...(norts && {norts}),
|
|
|
- })}`);
|
|
|
- const fetchTimeline = (
|
|
|
- config = {
|
|
|
- screen_name: username.slice(1),
|
|
|
- trim_user: true,
|
|
|
- exclude_replies: noreps ?? true,
|
|
|
- include_rts: !(norts ?? false),
|
|
|
- since_id: since,
|
|
|
- max_id: until,
|
|
|
- tweet_mode: 'extended',
|
|
|
- },
|
|
|
- tweets: Tweet[] = []
|
|
|
- ): Promise<Tweet[]> => this.client.get('statuses/user_timeline', config)
|
|
|
- .then((newTweets: Tweet[]) => {
|
|
|
- if (newTweets.length) {
|
|
|
- logger.debug(`fetched tweets: ${JSON.stringify(newTweets)}`);
|
|
|
- config.max_id = BigNumOps.plus('-1', newTweets[newTweets.length - 1].id_str);
|
|
|
- logger.info(`timeline query of ${username} yielded ${
|
|
|
- newTweets.length
|
|
|
- } new tweets, next query will start at offset ${config.max_id}`);
|
|
|
- tweets.push(...newTweets.filter(tweet => tweet.extended_entities));
|
|
|
- }
|
|
|
- if (!newTweets.length || tweets.length >= count) {
|
|
|
+ return this.queryUser(username.slice(1)).then(userNameId => {
|
|
|
+ const getMore = (lastTweets: Tweet[] = []) => {
|
|
|
+ logger.info(`querying timeline of ${username} with config: ${
|
|
|
+ JSON.stringify({
|
|
|
+ ...(count && {count}),
|
|
|
+ ...(since && {since}),
|
|
|
+ ...(until && {until}),
|
|
|
+ ...(noreps && {noreps}),
|
|
|
+ ...(norts && {norts}),
|
|
|
+ })}`);
|
|
|
+ return this.get('userTimeline', userNameId.split(':')[1], {
|
|
|
+ expansions: ['attachments.media_keys', 'author_id'],
|
|
|
+ 'tweet.fields': ['created_at'],
|
|
|
+ exclude: [
|
|
|
+ ...(noreps ?? true) ? ['replies' as const] : [],
|
|
|
+ ...(norts ?? false) ? ['retweets' as const] : [],
|
|
|
+ ],
|
|
|
+ max_results: Math.min(Math.max(count || 0, 20), 100),
|
|
|
+ ...(since && {since_id: since}),
|
|
|
+ ...(until && {until_id: until}),
|
|
|
+ }).then(newTweets => {
|
|
|
+ logger.info(`timeline query of ${username} yielded ${newTweets.length} new tweets`);
|
|
|
+ const tweets = lastTweets.concat(newTweets.filter(({data}) => (data.attachments || {}).media_keys));
|
|
|
+ if (tweets.length < count) {
|
|
|
+ until = BigNumOps.plus('-1', newTweets.slice(-1)[0].data.id);
|
|
|
+ logger.info(`starting next query at offset ${until}...`);
|
|
|
+ return getMore(tweets);
|
|
|
+ }
|
|
|
logger.info(`timeline query of ${username} finished successfully, ${
|
|
|
tweets.length
|
|
|
- } tweets with extended entities have been fetched`);
|
|
|
+ } media tweets have been fetched`);
|
|
|
return tweets.slice(0, count);
|
|
|
- }
|
|
|
- return fetchTimeline(config, tweets);
|
|
|
- });
|
|
|
- return fetchTimeline();
|
|
|
+ });
|
|
|
+ }
|
|
|
+ return getMore();
|
|
|
+ });
|
|
|
};
|
|
|
|
|
|
private workOnTweets = (
|
|
|
tweets: Tweet[],
|
|
|
sendTweets: (cacheId: string, msg: string, text: string, author: string) => void,
|
|
|
refresh = false
|
|
|
- ) => Promise.all(tweets.map(tweet =>
|
|
|
+ ) => Promise.all(tweets.map(({data, includes}) =>
|
|
|
((this.redis && !refresh) ?
|
|
|
- this.redis.waitForProcess(`webshot/${tweet.id_str}`, this.webshotDelay * 4)
|
|
|
- .then(() => this.redis.getContent(`webshot/${tweet.id_str}`)) :
|
|
|
+ this.redis.waitForProcess(`webshot/${data.id}`, this.webshotDelay * 4)
|
|
|
+ .then(() => this.redis.getContent(`webshot/${data.id}`)) :
|
|
|
Promise.reject())
|
|
|
.then(content => {
|
|
|
if (content === null) throw Error();
|
|
|
- logger.info(`retrieved cached webshot of tweet ${tweet.id_str} from redis database`);
|
|
|
+ logger.info(`retrieved cached webshot of tweet ${data.id} from redis database, message chain:`);
|
|
|
const {msg, text, author} = JSON.parse(content) as {[key: string]: string};
|
|
|
- let cacheId = tweet.id_str;
|
|
|
- if (tweet.retweeted_status) cacheId += `,rt:${tweet.retweeted_status.id_str}`;
|
|
|
- sendTweets(cacheId, msg, text, author);
|
|
|
+ let cacheId = data.id;
|
|
|
+ const retweetRef = (data.referenced_tweets || []).find(ref => ref.type === 'retweeted');
|
|
|
+ if (retweetRef) cacheId += `,rt:${retweetRef.id}`;
|
|
|
+ logger.info(JSON.stringify(Message.parseCQCode(msg)));
|
|
|
+ sendTweets(cacheId, Message.parseCQCode(msg), text, author);
|
|
|
return null as Tweet;
|
|
|
})
|
|
|
.catch(() => {
|
|
|
- this.redis.startProcess(`webshot/${tweet.id_str}`);
|
|
|
- return tweet;
|
|
|
+ this.redis.startProcess(`webshot/${data.id}`);
|
|
|
+ return {data, includes} as Tweet;
|
|
|
})
|
|
|
)).then(tweets =>
|
|
|
this.webshot(
|
|
|
@@ -304,8 +368,9 @@ export default class {
|
|
|
if (!this.redis) return;
|
|
|
const [twid, rtid] = cacheId.split(',rt:');
|
|
|
logger.info(`caching webshot of tweet ${twid} to redis database`);
|
|
|
- this.redis.cacheContent(`webshot/${twid}`, JSON.stringify({msg, text, author, rtid}))
|
|
|
- .then(() => this.redis.finishProcess(`webshot/${twid}`));
|
|
|
+ this.redis.cacheContent(`webshot/${twid}`,
|
|
|
+ JSON.stringify({msg: Message.toCQCode(msg), text, author, rtid})
|
|
|
+ ).then(() => this.redis.finishProcess(`webshot/${twid}`));
|
|
|
})
|
|
|
.then(() => sendTweets(cacheId, msg, text, author));
|
|
|
},
|
|
|
@@ -317,31 +382,30 @@ export default class {
|
|
|
id: string,
|
|
|
sender: (cacheId: string, msg: string, text: string, author: string) => void,
|
|
|
refresh = false
|
|
|
- ) => {
|
|
|
- const endpoint = 'statuses/show';
|
|
|
- const config = {
|
|
|
- id,
|
|
|
- tweet_mode: 'extended',
|
|
|
- };
|
|
|
- return ((this.redis && !refresh) ?
|
|
|
+ ) =>
|
|
|
+ ((this.redis && !refresh) ?
|
|
|
this.redis.waitForProcess(`webshot/${id}`, this.webshotDelay * 4)
|
|
|
.then(() => this.redis.getContent(`webshot/${id}`))
|
|
|
.then(content => {
|
|
|
if (content === null) throw Error();
|
|
|
const {rtid} = JSON.parse(content);
|
|
|
- return {id_str: id, retweeted_status: rtid ? {id_str: rtid} : undefined} as Tweet;
|
|
|
+ return {data: {id, ...rtid && {referenced_tweets: [{type: 'retweeted', id: rtid}]}}} as Tweet;
|
|
|
}) :
|
|
|
- Promise.reject())
|
|
|
- .catch(() => this.client.get(endpoint, config))
|
|
|
+ Promise.reject()
|
|
|
+ )
|
|
|
+ .catch(() => this.client.v2.singleTweet(id, v2SingleParams))
|
|
|
.then((tweet: Tweet) => {
|
|
|
- if (tweet.id) {
|
|
|
+ if (tweet.data.text) {
|
|
|
logger.debug(`api returned tweet ${JSON.stringify(tweet)} for query id=${id}`);
|
|
|
+ const retweetRef = (tweet.data.referenced_tweets || []).find(ref => ref.type === 'retweeted');
|
|
|
+ if (retweetRef) return this.client.v2.singleTweet(retweetRef.id, v2SingleParams)
|
|
|
+ .then(({includes: {media}}) => ({...tweet, includes: {...tweet.includes, media}}) as Tweet);
|
|
|
} else {
|
|
|
- logger.debug(`skipped querying api as this tweet has been cached`)
|
|
|
+ logger.debug(`skipped querying api as this tweet has been cached`);
|
|
|
}
|
|
|
- return this.workOnTweets([tweet], sender, refresh);
|
|
|
- });
|
|
|
- };
|
|
|
+ return tweet;
|
|
|
+ })
|
|
|
+ .then((tweet: Tweet) => this.workOnTweets([tweet], sender, refresh));
|
|
|
|
|
|
private sendTweets = (
|
|
|
config: {sourceInfo?: string, reportOnSkip?: boolean, force?: boolean}
|
|
|
@@ -382,6 +446,43 @@ export default class {
|
|
|
});
|
|
|
};
|
|
|
|
|
|
+ private get = <T extends 'userTimeline' | 'listTweets'>(
|
|
|
+ type: T, targetId: string, params: Parameters<typeof this.client.v2[T]>[1]
|
|
|
+ ) => {
|
|
|
+ const {since_id, max_results} = (params as Twitter.TweetV2UserTimelineParams);
|
|
|
+ const getMore = (res: Twitter.TweetUserTimelineV2Paginator | Twitter.TweetV2ListTweetsPaginator) => {
|
|
|
+ if (res.errors && res.errors.length > 0) {
|
|
|
+ const [err] = res.errors;
|
|
|
+ if (!res.data) throw err;
|
|
|
+ if (err.title === 'Authorization Error') {
|
|
|
+ logger.warn(`non-fatal error while querying ${type} with id ${targetId}, error: ${err.detail}`);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (!res.meta.next_token || // at last page
|
|
|
+ BigNumOps.compare(res.tweets.slice(-1)[0].id, since_id || '0') !== 1 || // at specified boundary
|
|
|
+ !since_id && res.meta.result_count >= max_results // at specified max count
|
|
|
+ ) return res;
|
|
|
+ return res.fetchNext().then<typeof res>(getMore);
|
|
|
+ };
|
|
|
+ if (type === 'listTweets') delete (params as any).since_id;
|
|
|
+ return this.client.v2[type](targetId, params).then(getMore)
|
|
|
+ .then(({includes, tweets}) => tweets.map((tweet): Tweet =>
|
|
|
+ ({
|
|
|
+ data: tweet as TweetObject,
|
|
|
+ includes: {
|
|
|
+ media: includes.medias(tweet) as MediaObject[],
|
|
|
+ users: [includes.author(tweet)]
|
|
|
+ }
|
|
|
+ })
|
|
|
+ ))
|
|
|
+ .then(tweets => Promise.all(tweets.map(tweet => {
|
|
|
+ const retweetRef = (tweet.data.referenced_tweets || []).find(ref => ref.type === 'retweeted');
|
|
|
+ if (retweetRef) return this.client.v2.singleTweet(retweetRef.id, v2SingleParams)
|
|
|
+ .then(({includes: {media}}) => ({...tweet, includes: {...tweet.includes, media}}) as Tweet);
|
|
|
+ return tweet;
|
|
|
+ })));
|
|
|
+ };
|
|
|
+
|
|
|
public work = () => {
|
|
|
const lock = this.lock;
|
|
|
if (this.workInterval < 1) this.workInterval = 1;
|
|
|
@@ -406,62 +507,57 @@ export default class {
|
|
|
const currentFeed = lock.feed[lock.workon];
|
|
|
logger.debug(`pulling feed ${currentFeed}`);
|
|
|
|
|
|
- const promise = new Promise(resolve => {
|
|
|
+ const promise = new Promise<Tweet[]>(resolve => {
|
|
|
+ let job = Promise.resolve();
|
|
|
+ let id = lock.threads[currentFeed].id;
|
|
|
+ let endpoint: Parameters<typeof this.get>[0];
|
|
|
+
|
|
|
let match = /https:\/\/twitter.com\/([^\/]+)\/lists\/([^\/]+)/.exec(currentFeed);
|
|
|
- let config: {[key: string]: any};
|
|
|
- let endpoint: string;
|
|
|
if (match) {
|
|
|
+ endpoint = 'listTweets';
|
|
|
if (match[1] === 'i') {
|
|
|
- config = {
|
|
|
- list_id: match[2],
|
|
|
- tweet_mode: 'extended',
|
|
|
- };
|
|
|
- } else {
|
|
|
- config = {
|
|
|
+ id = match[2];
|
|
|
+ } else if (id === undefined) {
|
|
|
+ job = job.then(() => this.client.v1.list({
|
|
|
owner_screen_name: match[1],
|
|
|
slug: match[2],
|
|
|
- tweet_mode: 'extended',
|
|
|
- };
|
|
|
+ })).then(({id_str}) => {
|
|
|
+ lock.threads[currentFeed].id = id = id_str;
|
|
|
+ });
|
|
|
}
|
|
|
- endpoint = 'lists/statuses';
|
|
|
} else {
|
|
|
match = /https:\/\/twitter.com\/([^\/]+)/.exec(currentFeed);
|
|
|
if (match) {
|
|
|
- config = {
|
|
|
- screen_name: match[1],
|
|
|
- exclude_replies: false,
|
|
|
- tweet_mode: 'extended',
|
|
|
- };
|
|
|
- endpoint = 'statuses/user_timeline';
|
|
|
+ endpoint = 'userTimeline';
|
|
|
+ if (id === undefined) {
|
|
|
+ job = job.then(() => this.queryUser(
|
|
|
+ match[1].replace(/^@?(.*)$/, '$1')
|
|
|
+ )).then(userNameId => {
|
|
|
+ lock.threads[currentFeed].id = id = userNameId.split(':')[1];
|
|
|
+ });
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- if (endpoint) {
|
|
|
- const offset = lock.threads[currentFeed].offset;
|
|
|
- config.include_rts = false;
|
|
|
- if (offset as unknown as number > 0) config.since_id = offset;
|
|
|
- if (offset as unknown as number < -1) config.max_id = offset.slice(1);
|
|
|
- const getMore = (lastTweets: Tweet[] = []) => this.client.get(
|
|
|
- endpoint, config, (error: {[key: string]: any}[], tweets: Tweet[]
|
|
|
- ) => {
|
|
|
- if (error) {
|
|
|
- if (error instanceof Array && error.length > 0 && error[0].code === 34) {
|
|
|
- logger.warn(`error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
|
|
|
- lock.threads[currentFeed].subscribers.forEach(subscriber => {
|
|
|
- logger.info(`sending notfound message of ${currentFeed} to ${JSON.stringify(subscriber)}`);
|
|
|
- this.bot.sendTo(subscriber, `链接 ${currentFeed} 指向的用户或列表不存在，请退订。`).catch();
|
|
|
- });
|
|
|
- } else {
|
|
|
- logger.error(`unhandled error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
|
|
|
- }
|
|
|
- }
|
|
|
- if (!(tweets instanceof Array) || tweets.length === 0) return resolve(lastTweets);
|
|
|
- if (offset as unknown as number <= 0) return resolve(lastTweets.concat(tweets));
|
|
|
- config.max_id = BigNumOps.plus(tweets.slice(-1)[0].id_str, '-1');
|
|
|
- getMore(lastTweets.concat(tweets));
|
|
|
- });
|
|
|
- getMore();
|
|
|
- }
|
|
|
+ const offset = lock.threads[currentFeed].offset;
|
|
|
+ job.then(() => this.get(endpoint, id, {
|
|
|
+ ...v2SingleParams,
|
|
|
+ max_results: 20,
|
|
|
+ exclude: ['retweets'],
|
|
|
+ ...(offset as unknown as number > 0) && {since_id: offset},
|
|
|
+ ...(offset as unknown as number < -1) && {until_id: offset.slice(1)},
|
|
|
+ })).catch((err: Twitter.InlineErrorV2) => {
|
|
|
+ if (err.title === 'Not Found Error') {
|
|
|
+ logger.warn(`error on fetching tweets for ${currentFeed}: ${showApiError(err)}`);
|
|
|
+ lock.threads[currentFeed].subscribers.forEach(subscriber => {
|
|
|
+ logger.info(`sending notfound message of ${currentFeed} to ${JSON.stringify(subscriber)}`);
|
|
|
+ this.bot.sendTo(subscriber, `链接 ${currentFeed} 指向的用户或列表不存在，请退订。`).catch();
|
|
|
+ });
|
|
|
+ } else {
|
|
|
+ logger.error(`unhandled error on fetching tweets for ${currentFeed}: ${showApiError(err)}`);
|
|
|
+ }
|
|
|
+ return [] as Tweet[];
|
|
|
+ }).then(resolve);
|
|
|
});
|
|
|
|
|
|
promise.then((tweets: Tweet[]) => {
|
|
|
@@ -478,12 +574,18 @@ export default class {
|
|
|
return;
|
|
|
}
|
|
|
|
|
|
- const topOfFeed = tweets[0].id_str;
|
|
|
+ const currentUser = tweets[0].includes.users.find(user => user.id === currentThread.id);
|
|
|
+ if (currentUser.username !== parseLink(currentFeed)[1]) {
|
|
|
+ lock.feed[lock.workon] = linkBuilder(currentUser.username);
|
|
|
+ }
|
|
|
+
|
|
|
+ const topOfFeed = tweets[0].data.id;
|
|
|
logger.info(`current offset: ${currentThread.offset}, current top of feed: ${topOfFeed}`);
|
|
|
- const bottomOfFeed = tweets[tweets.length - 1].id_str;
|
|
|
+ const bottomOfFeed = tweets.slice(-1)[0].data.id;
|
|
|
const updateOffset = () => setOffset(topOfFeed);
|
|
|
- tweets = tweets.filter(twi => twi.extended_entities);
|
|
|
+ tweets = tweets.filter(({data}) => (data.attachments || {}).media_keys);
|
|
|
logger.info(`found ${tweets.length} tweets with extended entities`);
|
|
|
+
|
|
|
if (currentThread.offset === '-1') { updateOffset(); return; }
|
|
|
if (currentThread.offset as unknown as number <= 0) {
|
|
|
if (tweets.length === 0) {
|
